1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.Iterator;
25 import java.util.List;
26 import static org.junit.Assert.assertEquals;
27 import static org.junit.Assert.assertFalse;
28 import static org.junit.Assert.assertTrue;
29
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FSDataOutputStream;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.Cell;
35 import org.apache.hadoop.hbase.CellUtil;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.KeyValue;
39 import org.apache.hadoop.hbase.Tag;
40 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
41 import org.apache.hadoop.hbase.testclassification.SmallTests;
42 import org.apache.hadoop.hbase.util.Bytes;
43 import org.junit.Before;
44 import org.junit.Test;
45 import org.junit.experimental.categories.Category;
46 import org.junit.runner.RunWith;
47 import org.junit.runners.Parameterized;
48 import org.junit.runners.Parameterized.Parameters;
49
50
51
52
53 @Category(SmallTests.class)
54 @RunWith(Parameterized.class)
55 public class TestSeekTo {
56
57 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
58 private final DataBlockEncoding encoding;
59
60 @Parameters
61 public static Collection<Object[]> parameters() {
62 List<Object[]> paramList = new ArrayList<Object[]>();
63 for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
64 paramList.add(new Object[] { encoding });
65 }
66 return paramList;
67 }
68
69 static boolean switchKVs = false;
70
71 public TestSeekTo(DataBlockEncoding encoding) {
72 this.encoding = encoding;
73 }
74
75 @Before
76 public void setUp() {
77
78 switchKVs = false;
79 }
80
81
82 static KeyValue toKV(String row, TagUsage tagUsage) {
83 if (tagUsage == TagUsage.NO_TAG) {
84 return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes.toBytes("qualifier"),
85 Bytes.toBytes("value"));
86 } else if (tagUsage == TagUsage.ONLY_TAG) {
87 Tag t = new Tag((byte) 1, "myTag1");
88 Tag[] tags = new Tag[1];
89 tags[0] = t;
90 return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes.toBytes("qualifier"),
91 HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"), tags);
92 } else {
93 if (!switchKVs) {
94 switchKVs = true;
95 return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"),
96 Bytes.toBytes("qualifier"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
97 } else {
98 switchKVs = false;
99 Tag t = new Tag((byte) 1, "myTag1");
100 Tag[] tags = new Tag[1];
101 tags[0] = t;
102 return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"),
103 Bytes.toBytes("qualifier"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"), tags);
104 }
105 }
106 }
107 static String toRowStr(KeyValue kv) {
108 return Bytes.toString(kv.getRow());
109 }
110
111 Path makeNewFile(TagUsage tagUsage) throws IOException {
112 Path ncTFile = new Path(TEST_UTIL.getDataTestDir(), "basic.hfile");
113 FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile);
114 Configuration conf = TEST_UTIL.getConfiguration();
115 if (tagUsage != TagUsage.NO_TAG) {
116 conf.setInt("hfile.format.version", 3);
117 } else {
118 conf.setInt("hfile.format.version", 2);
119 }
120 int blocksize = toKV("a", tagUsage).getLength() * 3;
121 HFileContext context = new HFileContextBuilder().withBlockSize(blocksize)
122 .withDataBlockEncoding(encoding)
123 .withIncludesTags(true).build();
124 HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout)
125 .withFileContext(context)
126
127
128 .withComparator(KeyValue.COMPARATOR).create();
129
130
131 writer.append(toKV("c", tagUsage));
132 writer.append(toKV("e", tagUsage));
133 writer.append(toKV("g", tagUsage));
134
135 writer.append(toKV("i", tagUsage));
136 writer.append(toKV("k", tagUsage));
137 writer.close();
138 fout.close();
139 return ncTFile;
140 }
141
142 @Test
143 public void testSeekBefore() throws Exception {
144 testSeekBeforeInternals(TagUsage.NO_TAG);
145 testSeekBeforeInternals(TagUsage.ONLY_TAG);
146 testSeekBeforeInternals(TagUsage.PARTIAL_TAG);
147 }
148
149 protected void testSeekBeforeInternals(TagUsage tagUsage) throws IOException {
150 Path p = makeNewFile(tagUsage);
151 FileSystem fs = TEST_UTIL.getTestFileSystem();
152 Configuration conf = TEST_UTIL.getConfiguration();
153 HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
154 reader.loadFileInfo();
155 HFileScanner scanner = reader.getScanner(false, true);
156 assertFalse(scanner.seekBefore(toKV("a", tagUsage).getKey()));
157
158 assertFalse(scanner.seekBefore(toKV("c", tagUsage).getKey()));
159
160 assertTrue(scanner.seekBefore(toKV("d", tagUsage).getKey()));
161 assertEquals("c", toRowStr(scanner.getKeyValue()));
162
163 assertTrue(scanner.seekBefore(toKV("e", tagUsage).getKey()));
164 assertEquals("c", toRowStr(scanner.getKeyValue()));
165
166 assertTrue(scanner.seekBefore(toKV("f", tagUsage).getKey()));
167 assertEquals("e", toRowStr(scanner.getKeyValue()));
168
169 assertTrue(scanner.seekBefore(toKV("g", tagUsage).getKey()));
170 assertEquals("e", toRowStr(scanner.getKeyValue()));
171 assertTrue(scanner.seekBefore(toKV("h", tagUsage).getKey()));
172 assertEquals("g", toRowStr(scanner.getKeyValue()));
173 assertTrue(scanner.seekBefore(toKV("i", tagUsage).getKey()));
174 assertEquals("g", toRowStr(scanner.getKeyValue()));
175 assertTrue(scanner.seekBefore(toKV("j", tagUsage).getKey()));
176 assertEquals("i", toRowStr(scanner.getKeyValue()));
177 Cell cell = scanner.getKeyValue();
178 if (tagUsage != TagUsage.NO_TAG && cell.getTagsLength() > 0) {
179 Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
180 cell.getTagsLength());
181 while (tagsIterator.hasNext()) {
182 Tag next = tagsIterator.next();
183 assertEquals("myTag1", Bytes.toString(next.getValue()));
184 }
185 }
186 assertTrue(scanner.seekBefore(toKV("k", tagUsage).getKey()));
187 assertEquals("i", toRowStr(scanner.getKeyValue()));
188 assertTrue(scanner.seekBefore(toKV("l", tagUsage).getKey()));
189 assertEquals("k", toRowStr(scanner.getKeyValue()));
190
191 reader.close();
192
193 reader.close();
194 }
195
196 @Test
197 public void testSeekBeforeWithReSeekTo() throws Exception {
198 testSeekBeforeWithReSeekToInternals(TagUsage.NO_TAG);
199 testSeekBeforeWithReSeekToInternals(TagUsage.ONLY_TAG);
200 testSeekBeforeWithReSeekToInternals(TagUsage.PARTIAL_TAG);
201 }
202
203 protected void testSeekBeforeWithReSeekToInternals(TagUsage tagUsage) throws IOException {
204 Path p = makeNewFile(tagUsage);
205 FileSystem fs = TEST_UTIL.getTestFileSystem();
206 Configuration conf = TEST_UTIL.getConfiguration();
207 HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
208 reader.loadFileInfo();
209 HFileScanner scanner = reader.getScanner(false, true);
210 assertFalse(scanner.seekBefore(toKV("a", tagUsage).getKey()));
211 assertFalse(scanner.seekBefore(toKV("b", tagUsage).getKey()));
212 assertFalse(scanner.seekBefore(toKV("c", tagUsage).getKey()));
213
214
215 assertTrue(scanner.seekBefore(toKV("d", tagUsage).getKey()));
216 assertEquals("c", toRowStr(scanner.getKeyValue()));
217
218 assertEquals(0, scanner.reseekTo(toKV("c", tagUsage).getKey()));
219 assertEquals("c", toRowStr(scanner.getKeyValue()));
220 assertEquals(0, scanner.reseekTo(toKV("g", tagUsage).getKey()));
221 assertEquals("g", toRowStr(scanner.getKeyValue()));
222
223
224 assertTrue(scanner.seekBefore(toKV("e", tagUsage).getKey()));
225 assertEquals("c", toRowStr(scanner.getKeyValue()));
226
227 assertEquals(0, scanner.reseekTo(toKV("e", tagUsage).getKey()));
228 assertEquals("e", toRowStr(scanner.getKeyValue()));
229 assertEquals(0, scanner.reseekTo(toKV("g", tagUsage).getKey()));
230 assertEquals("g", toRowStr(scanner.getKeyValue()));
231
232
233 assertTrue(scanner.seekBefore(toKV("f", tagUsage).getKey()));
234 assertEquals("e", toRowStr(scanner.getKeyValue()));
235
236 assertEquals(0, scanner.reseekTo(toKV("e", tagUsage).getKey()));
237 assertEquals("e", toRowStr(scanner.getKeyValue()));
238 assertEquals(0, scanner.reseekTo(toKV("g", tagUsage).getKey()));
239 assertEquals("g", toRowStr(scanner.getKeyValue()));
240
241
242 assertTrue(scanner.seekBefore(toKV("g", tagUsage).getKey()));
243 assertEquals("e", toRowStr(scanner.getKeyValue()));
244
245 assertEquals(0, scanner.reseekTo(toKV("e", tagUsage).getKey()));
246 assertEquals("e", toRowStr(scanner.getKeyValue()));
247 assertEquals(0, scanner.reseekTo(toKV("g", tagUsage).getKey()));
248 assertEquals("g", toRowStr(scanner.getKeyValue()));
249
250
251 assertTrue(scanner.seekBefore(toKV("h", tagUsage).getKey()));
252 assertEquals("g", toRowStr(scanner.getKeyValue()));
253
254 assertEquals(0, scanner.reseekTo(toKV("g", tagUsage).getKey()));
255 assertEquals("g", toRowStr(scanner.getKeyValue()));
256
257
258 assertTrue(scanner.seekBefore(toKV("i", tagUsage).getKey()));
259 assertEquals("g", toRowStr(scanner.getKeyValue()));
260
261 assertEquals(0, scanner.reseekTo(toKV("g", tagUsage).getKey()));
262 assertEquals("g", toRowStr(scanner.getKeyValue()));
263
264
265 assertTrue(scanner.seekBefore(toKV("j", tagUsage).getKey()));
266 assertEquals("i", toRowStr(scanner.getKeyValue()));
267
268 assertEquals(0, scanner.reseekTo(toKV("i", tagUsage).getKey()));
269 assertEquals("i", toRowStr(scanner.getKeyValue()));
270
271
272 assertTrue(scanner.seekBefore(toKV("k", tagUsage).getKey()));
273 assertEquals("i", toRowStr(scanner.getKeyValue()));
274
275 assertEquals(0, scanner.reseekTo(toKV("i", tagUsage).getKey()));
276 assertEquals("i", toRowStr(scanner.getKeyValue()));
277 assertEquals(0, scanner.reseekTo(toKV("k", tagUsage).getKey()));
278 assertEquals("k", toRowStr(scanner.getKeyValue()));
279
280
281 assertTrue(scanner.seekBefore(toKV("l", tagUsage).getKey()));
282 assertEquals("k", toRowStr(scanner.getKeyValue()));
283
284 assertEquals(0, scanner.reseekTo(toKV("k", tagUsage).getKey()));
285 assertEquals("k", toRowStr(scanner.getKeyValue()));
286 deleteTestDir(fs);
287 }
288
289 protected void deleteTestDir(FileSystem fs) throws IOException {
290 Path dataTestDir = TEST_UTIL.getDataTestDir();
291 if (fs.exists(dataTestDir)) {
292 fs.delete(dataTestDir, true);
293 }
294 }
295 @Test
296 public void testSeekTo() throws Exception {
297 testSeekToInternals(TagUsage.NO_TAG);
298 testSeekToInternals(TagUsage.ONLY_TAG);
299 testSeekToInternals(TagUsage.PARTIAL_TAG);
300 }
301
302 protected void testSeekToInternals(TagUsage tagUsage) throws IOException {
303 Path p = makeNewFile(tagUsage);
304 FileSystem fs = TEST_UTIL.getTestFileSystem();
305 Configuration conf = TEST_UTIL.getConfiguration();
306 HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
307 reader.loadFileInfo();
308 assertEquals(2, reader.getDataBlockIndexReader().getRootBlockCount());
309 HFileScanner scanner = reader.getScanner(false, true);
310
311 assertEquals(-1, scanner.seekTo(toKV("a", tagUsage).getKey()));
312
313 assertEquals(1, scanner.seekTo(toKV("d", tagUsage).getKey()));
314 assertEquals("c", toRowStr(scanner.getKeyValue()));
315
316
317 assertEquals(0, scanner.seekTo(toKV("i", tagUsage).getKey()));
318 assertEquals("i", toRowStr(scanner.getKeyValue()));
319
320 assertEquals(1, scanner.seekTo(toKV("l", tagUsage).getKey()));
321 if (encoding == DataBlockEncoding.PREFIX_TREE) {
322
323 assertEquals(null, scanner.getKeyValue());
324 } else {
325 assertEquals("k", toRowStr(scanner.getKeyValue()));
326 }
327
328 reader.close();
329 }
330
331 @Test
332 public void testBlockContainingKey() throws Exception {
333 testBlockContainingKeyInternals(TagUsage.NO_TAG);
334 testBlockContainingKeyInternals(TagUsage.ONLY_TAG);
335 testBlockContainingKeyInternals(TagUsage.PARTIAL_TAG);
336 }
337
338 protected void testBlockContainingKeyInternals(TagUsage tagUsage) throws IOException {
339 Path p = makeNewFile(tagUsage);
340 FileSystem fs = TEST_UTIL.getTestFileSystem();
341 Configuration conf = TEST_UTIL.getConfiguration();
342 HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
343 reader.loadFileInfo();
344 HFileBlockIndex.BlockIndexReader blockIndexReader =
345 reader.getDataBlockIndexReader();
346 System.out.println(blockIndexReader.toString());
347 int klen = toKV("a", tagUsage).getKey().length;
348
349 assertEquals(-1, blockIndexReader.rootBlockContainingKey(
350 toKV("a", tagUsage).getKey(), 0, klen));
351 assertEquals(0, blockIndexReader.rootBlockContainingKey(
352 toKV("c", tagUsage).getKey(), 0, klen));
353 assertEquals(0, blockIndexReader.rootBlockContainingKey(
354 toKV("d", tagUsage).getKey(), 0, klen));
355 assertEquals(0, blockIndexReader.rootBlockContainingKey(
356 toKV("e", tagUsage).getKey(), 0, klen));
357 assertEquals(0, blockIndexReader.rootBlockContainingKey(
358 toKV("g", tagUsage).getKey(), 0, klen));
359 assertEquals(1, blockIndexReader.rootBlockContainingKey(
360 toKV("h", tagUsage).getKey(), 0, klen));
361 assertEquals(1, blockIndexReader.rootBlockContainingKey(
362 toKV("i", tagUsage).getKey(), 0, klen));
363 assertEquals(1, blockIndexReader.rootBlockContainingKey(
364 toKV("j", tagUsage).getKey(), 0, klen));
365 assertEquals(1, blockIndexReader.rootBlockContainingKey(
366 toKV("k", tagUsage).getKey(), 0, klen));
367 assertEquals(1, blockIndexReader.rootBlockContainingKey(
368 toKV("l", tagUsage).getKey(), 0, klen));
369
370 reader.close();
371 deleteTestDir(fs);
372 }
373
374
375 }
376