View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.Collection;
24  import java.util.Iterator;
25  import java.util.List;
26  import static org.junit.Assert.assertEquals;
27  import static org.junit.Assert.assertFalse;
28  import static org.junit.Assert.assertTrue;
29  
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.FSDataOutputStream;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.Cell;
35  import org.apache.hadoop.hbase.CellUtil;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.KeyValue;
39  import org.apache.hadoop.hbase.Tag;
40  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
41  import org.apache.hadoop.hbase.testclassification.SmallTests;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.junit.Before;
44  import org.junit.Test;
45  import org.junit.experimental.categories.Category;
46  import org.junit.runner.RunWith;
47  import org.junit.runners.Parameterized;
48  import org.junit.runners.Parameterized.Parameters;
49  
50  /**
51   * Test {@link HFileScanner#seekTo(byte[])} and its variants.
52   */
53  @Category(SmallTests.class)
54  @RunWith(Parameterized.class)
55  public class TestSeekTo {
56  
57    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
58    private final DataBlockEncoding encoding;
59  
60    @Parameters
61    public static Collection<Object[]> parameters() {
62      List<Object[]> paramList = new ArrayList<Object[]>();
63      for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
64        paramList.add(new Object[] { encoding });
65      }
66      return paramList;
67    }
68  
69    static boolean switchKVs = false;
70  
71    public TestSeekTo(DataBlockEncoding encoding) {
72      this.encoding = encoding;
73    }
74  
75    @Before
76    public void setUp() {
77      // reset
78      switchKVs = false;
79       }
80  
81  
82    static KeyValue toKV(String row, TagUsage tagUsage) {
83      if (tagUsage == TagUsage.NO_TAG) {
84        return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes.toBytes("qualifier"),
85            Bytes.toBytes("value"));
86      } else if (tagUsage == TagUsage.ONLY_TAG) {
87        Tag t = new Tag((byte) 1, "myTag1");
88        Tag[] tags = new Tag[1];
89        tags[0] = t;
90        return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes.toBytes("qualifier"),
91            HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"), tags);
92      } else {
93        if (!switchKVs) {
94          switchKVs = true;
95          return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"),
96              Bytes.toBytes("qualifier"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
97        } else {
98          switchKVs = false;
99          Tag t = new Tag((byte) 1, "myTag1");
100         Tag[] tags = new Tag[1];
101         tags[0] = t;
102         return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"),
103             Bytes.toBytes("qualifier"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"), tags);
104       }
105     }
106   }
107   static String toRowStr(KeyValue kv) {
108     return Bytes.toString(kv.getRow());
109   }
110 
111   Path makeNewFile(TagUsage tagUsage) throws IOException {
112     Path ncTFile = new Path(TEST_UTIL.getDataTestDir(), "basic.hfile");
113     FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile);
114     Configuration conf = TEST_UTIL.getConfiguration();
115     if (tagUsage != TagUsage.NO_TAG) {
116       conf.setInt("hfile.format.version", 3);
117     } else {
118       conf.setInt("hfile.format.version", 2);
119     }
120     int blocksize = toKV("a", tagUsage).getLength() * 3;
121     HFileContext context = new HFileContextBuilder().withBlockSize(blocksize)
122         .withDataBlockEncoding(encoding)
123         .withIncludesTags(true).build();
124     HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout)
125           .withFileContext(context)
126           // NOTE: This test is dependent on this deprecated nonstandard
127           // comparator
128           .withComparator(KeyValue.COMPARATOR).create();
129     // 4 bytes * 3 * 2 for each key/value +
130     // 3 for keys, 15 for values = 42 (woot)
131     writer.append(toKV("c", tagUsage));
132     writer.append(toKV("e", tagUsage));
133     writer.append(toKV("g", tagUsage));
134     // block transition
135     writer.append(toKV("i", tagUsage));
136     writer.append(toKV("k", tagUsage));
137     writer.close();
138     fout.close();
139     return ncTFile;
140   }
141 
142   @Test
143   public void testSeekBefore() throws Exception {
144     testSeekBeforeInternals(TagUsage.NO_TAG);
145     testSeekBeforeInternals(TagUsage.ONLY_TAG);
146     testSeekBeforeInternals(TagUsage.PARTIAL_TAG);
147   }
148 
149   protected void testSeekBeforeInternals(TagUsage tagUsage) throws IOException {
150     Path p = makeNewFile(tagUsage);
151     FileSystem fs = TEST_UTIL.getTestFileSystem();
152     Configuration conf = TEST_UTIL.getConfiguration();
153     HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
154     reader.loadFileInfo();
155     HFileScanner scanner = reader.getScanner(false, true);
156     assertFalse(scanner.seekBefore(toKV("a", tagUsage).getKey()));
157 
158     assertFalse(scanner.seekBefore(toKV("c", tagUsage).getKey()));
159 
160     assertTrue(scanner.seekBefore(toKV("d", tagUsage).getKey()));
161     assertEquals("c", toRowStr(scanner.getKeyValue()));
162 
163     assertTrue(scanner.seekBefore(toKV("e", tagUsage).getKey()));
164     assertEquals("c", toRowStr(scanner.getKeyValue()));
165 
166     assertTrue(scanner.seekBefore(toKV("f", tagUsage).getKey()));
167     assertEquals("e", toRowStr(scanner.getKeyValue()));
168 
169     assertTrue(scanner.seekBefore(toKV("g", tagUsage).getKey()));
170     assertEquals("e", toRowStr(scanner.getKeyValue()));
171     assertTrue(scanner.seekBefore(toKV("h", tagUsage).getKey()));
172     assertEquals("g", toRowStr(scanner.getKeyValue()));
173     assertTrue(scanner.seekBefore(toKV("i", tagUsage).getKey()));
174     assertEquals("g", toRowStr(scanner.getKeyValue()));
175     assertTrue(scanner.seekBefore(toKV("j", tagUsage).getKey()));
176     assertEquals("i", toRowStr(scanner.getKeyValue()));
177     Cell cell = scanner.getKeyValue();
178     if (tagUsage != TagUsage.NO_TAG && cell.getTagsLength() > 0) {
179       Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
180           cell.getTagsLength());
181       while (tagsIterator.hasNext()) {
182         Tag next = tagsIterator.next();
183         assertEquals("myTag1", Bytes.toString(next.getValue()));
184       }
185     }
186     assertTrue(scanner.seekBefore(toKV("k", tagUsage).getKey()));
187     assertEquals("i", toRowStr(scanner.getKeyValue()));
188     assertTrue(scanner.seekBefore(toKV("l", tagUsage).getKey()));
189     assertEquals("k", toRowStr(scanner.getKeyValue()));
190 
191     reader.close();
192 
193     reader.close();
194   }
195 
196   @Test
197   public void testSeekBeforeWithReSeekTo() throws Exception {
198     testSeekBeforeWithReSeekToInternals(TagUsage.NO_TAG);
199     testSeekBeforeWithReSeekToInternals(TagUsage.ONLY_TAG);
200     testSeekBeforeWithReSeekToInternals(TagUsage.PARTIAL_TAG);
201   }
202 
203   protected void testSeekBeforeWithReSeekToInternals(TagUsage tagUsage) throws IOException {
204     Path p = makeNewFile(tagUsage);
205     FileSystem fs = TEST_UTIL.getTestFileSystem();
206     Configuration conf = TEST_UTIL.getConfiguration();
207     HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
208     reader.loadFileInfo();
209     HFileScanner scanner = reader.getScanner(false, true);
210     assertFalse(scanner.seekBefore(toKV("a", tagUsage).getKey()));
211     assertFalse(scanner.seekBefore(toKV("b", tagUsage).getKey()));
212     assertFalse(scanner.seekBefore(toKV("c", tagUsage).getKey()));
213 
214     // seekBefore d, so the scanner points to c
215     assertTrue(scanner.seekBefore(toKV("d", tagUsage).getKey()));
216     assertEquals("c", toRowStr(scanner.getKeyValue()));
217     // reseekTo e and g
218     assertEquals(0, scanner.reseekTo(toKV("c", tagUsage).getKey()));
219     assertEquals("c", toRowStr(scanner.getKeyValue()));
220     assertEquals(0, scanner.reseekTo(toKV("g", tagUsage).getKey()));
221     assertEquals("g", toRowStr(scanner.getKeyValue()));
222 
223     // seekBefore e, so the scanner points to c
224     assertTrue(scanner.seekBefore(toKV("e", tagUsage).getKey()));
225     assertEquals("c", toRowStr(scanner.getKeyValue()));
226     // reseekTo e and g
227     assertEquals(0, scanner.reseekTo(toKV("e", tagUsage).getKey()));
228     assertEquals("e", toRowStr(scanner.getKeyValue()));
229     assertEquals(0, scanner.reseekTo(toKV("g", tagUsage).getKey()));
230     assertEquals("g", toRowStr(scanner.getKeyValue()));
231 
232     // seekBefore f, so the scanner points to e
233     assertTrue(scanner.seekBefore(toKV("f", tagUsage).getKey()));
234     assertEquals("e", toRowStr(scanner.getKeyValue()));
235     // reseekTo e and g
236     assertEquals(0, scanner.reseekTo(toKV("e", tagUsage).getKey()));
237     assertEquals("e", toRowStr(scanner.getKeyValue()));
238     assertEquals(0, scanner.reseekTo(toKV("g", tagUsage).getKey()));
239     assertEquals("g", toRowStr(scanner.getKeyValue()));
240 
241     // seekBefore g, so the scanner points to e
242     assertTrue(scanner.seekBefore(toKV("g", tagUsage).getKey()));
243     assertEquals("e", toRowStr(scanner.getKeyValue()));
244     // reseekTo e and g again
245     assertEquals(0, scanner.reseekTo(toKV("e", tagUsage).getKey()));
246     assertEquals("e", toRowStr(scanner.getKeyValue()));
247     assertEquals(0, scanner.reseekTo(toKV("g", tagUsage).getKey()));
248     assertEquals("g", toRowStr(scanner.getKeyValue()));
249 
250     // seekBefore h, so the scanner points to g
251     assertTrue(scanner.seekBefore(toKV("h", tagUsage).getKey()));
252     assertEquals("g", toRowStr(scanner.getKeyValue()));
253     // reseekTo g
254     assertEquals(0, scanner.reseekTo(toKV("g", tagUsage).getKey()));
255     assertEquals("g", toRowStr(scanner.getKeyValue()));
256 
257     // seekBefore i, so the scanner points to g
258     assertTrue(scanner.seekBefore(toKV("i", tagUsage).getKey()));
259     assertEquals("g", toRowStr(scanner.getKeyValue()));
260     // reseekTo g
261     assertEquals(0, scanner.reseekTo(toKV("g", tagUsage).getKey()));
262     assertEquals("g", toRowStr(scanner.getKeyValue()));
263 
264     // seekBefore j, so the scanner points to i
265     assertTrue(scanner.seekBefore(toKV("j", tagUsage).getKey()));
266     assertEquals("i", toRowStr(scanner.getKeyValue()));
267     // reseekTo i
268     assertEquals(0, scanner.reseekTo(toKV("i", tagUsage).getKey()));
269     assertEquals("i", toRowStr(scanner.getKeyValue()));
270 
271     // seekBefore k, so the scanner points to i
272     assertTrue(scanner.seekBefore(toKV("k", tagUsage).getKey()));
273     assertEquals("i", toRowStr(scanner.getKeyValue()));
274     // reseekTo i and k
275     assertEquals(0, scanner.reseekTo(toKV("i", tagUsage).getKey()));
276     assertEquals("i", toRowStr(scanner.getKeyValue()));
277     assertEquals(0, scanner.reseekTo(toKV("k", tagUsage).getKey()));
278     assertEquals("k", toRowStr(scanner.getKeyValue()));
279 
280     // seekBefore l, so the scanner points to k
281     assertTrue(scanner.seekBefore(toKV("l", tagUsage).getKey()));
282     assertEquals("k", toRowStr(scanner.getKeyValue()));
283     // reseekTo k
284     assertEquals(0, scanner.reseekTo(toKV("k", tagUsage).getKey()));
285     assertEquals("k", toRowStr(scanner.getKeyValue()));
286     deleteTestDir(fs);
287   }
288 
289   protected void deleteTestDir(FileSystem fs) throws IOException {
290     Path dataTestDir = TEST_UTIL.getDataTestDir();
291     if (fs.exists(dataTestDir)) {
292       fs.delete(dataTestDir, true);
293     }
294        }
295   @Test
296   public void testSeekTo() throws Exception {
297     testSeekToInternals(TagUsage.NO_TAG);
298     testSeekToInternals(TagUsage.ONLY_TAG);
299     testSeekToInternals(TagUsage.PARTIAL_TAG);
300   }
301 
302   protected void testSeekToInternals(TagUsage tagUsage) throws IOException {
303     Path p = makeNewFile(tagUsage);
304     FileSystem fs = TEST_UTIL.getTestFileSystem();
305     Configuration conf = TEST_UTIL.getConfiguration();
306     HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
307     reader.loadFileInfo();
308     assertEquals(2, reader.getDataBlockIndexReader().getRootBlockCount());
309     HFileScanner scanner = reader.getScanner(false, true);
310     // lies before the start of the file.
311     assertEquals(-1, scanner.seekTo(toKV("a", tagUsage).getKey()));
312 
313     assertEquals(1, scanner.seekTo(toKV("d", tagUsage).getKey()));
314     assertEquals("c", toRowStr(scanner.getKeyValue()));
315 
316     // Across a block boundary now.
317     assertEquals(0, scanner.seekTo(toKV("i", tagUsage).getKey()));
318     assertEquals("i", toRowStr(scanner.getKeyValue()));
319 
320     assertEquals(1, scanner.seekTo(toKV("l", tagUsage).getKey()));
321     if (encoding == DataBlockEncoding.PREFIX_TREE) {
322       // TODO : Fix this
323       assertEquals(null, scanner.getKeyValue());
324     } else {
325       assertEquals("k", toRowStr(scanner.getKeyValue()));
326     }
327 
328     reader.close();
329   }
330 
331   @Test
332   public void testBlockContainingKey() throws Exception {
333     testBlockContainingKeyInternals(TagUsage.NO_TAG);
334     testBlockContainingKeyInternals(TagUsage.ONLY_TAG);
335     testBlockContainingKeyInternals(TagUsage.PARTIAL_TAG);
336   }
337 
338   protected void testBlockContainingKeyInternals(TagUsage tagUsage) throws IOException {
339     Path p = makeNewFile(tagUsage);
340     FileSystem fs = TEST_UTIL.getTestFileSystem();
341     Configuration conf = TEST_UTIL.getConfiguration();
342     HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
343     reader.loadFileInfo();
344     HFileBlockIndex.BlockIndexReader blockIndexReader = 
345       reader.getDataBlockIndexReader();
346     System.out.println(blockIndexReader.toString());
347     int klen = toKV("a", tagUsage).getKey().length;
348     // falls before the start of the file.
349     assertEquals(-1, blockIndexReader.rootBlockContainingKey(
350         toKV("a", tagUsage).getKey(), 0, klen));
351     assertEquals(0, blockIndexReader.rootBlockContainingKey(
352         toKV("c", tagUsage).getKey(), 0, klen));
353     assertEquals(0, blockIndexReader.rootBlockContainingKey(
354         toKV("d", tagUsage).getKey(), 0, klen));
355     assertEquals(0, blockIndexReader.rootBlockContainingKey(
356         toKV("e", tagUsage).getKey(), 0, klen));
357     assertEquals(0, blockIndexReader.rootBlockContainingKey(
358         toKV("g", tagUsage).getKey(), 0, klen));
359     assertEquals(1, blockIndexReader.rootBlockContainingKey(
360         toKV("h", tagUsage).getKey(), 0, klen));
361     assertEquals(1, blockIndexReader.rootBlockContainingKey(
362         toKV("i", tagUsage).getKey(), 0, klen));
363     assertEquals(1, blockIndexReader.rootBlockContainingKey(
364         toKV("j", tagUsage).getKey(), 0, klen));
365     assertEquals(1, blockIndexReader.rootBlockContainingKey(
366         toKV("k", tagUsage).getKey(), 0, klen));
367     assertEquals(1, blockIndexReader.rootBlockContainingKey(
368         toKV("l", tagUsage).getKey(), 0, klen));
369 
370     reader.close();
371     deleteTestDir(fs);
372   }
373 
374 
375 }
376