View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.io.IOException;
22  
23  import org.apache.hadoop.fs.FSDataOutputStream;
24  import org.apache.hadoop.fs.Path;
25  import org.apache.hadoop.hbase.Cell;
26  import org.apache.hadoop.hbase.HBaseTestCase;
27  import org.apache.hadoop.hbase.HConstants;
28  import org.apache.hadoop.hbase.KeyValue;
29  import org.apache.hadoop.hbase.KeyValueUtil;
30  import org.apache.hadoop.hbase.testclassification.SmallTests;
31  import org.apache.hadoop.hbase.Tag;
32  import org.apache.hadoop.hbase.util.Bytes;
33  import org.junit.experimental.categories.Category;
34  
35  /**
36   * Test {@link HFileScanner#seekTo(byte[])} and its variants.
37   */
38  @Category(SmallTests.class)
39  public class TestSeekTo extends HBaseTestCase {
40  
41    static boolean switchKVs = false;
42  
43    static KeyValue toKV(String row, TagUsage tagUsage) {
44      if (tagUsage == TagUsage.NO_TAG) {
45        return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes.toBytes("qualifier"),
46            Bytes.toBytes("value"));
47      } else if (tagUsage == TagUsage.ONLY_TAG) {
48        Tag t = new Tag((byte) 1, "myTag1");
49        Tag[] tags = new Tag[1];
50        tags[0] = t;
51        return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes.toBytes("qualifier"),
52            HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"), tags);
53      } else {
54        if (!switchKVs) {
55          switchKVs = true;
56          return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"),
57              Bytes.toBytes("qualifier"), Bytes.toBytes("value"));
58        } else {
59          switchKVs = false;
60          Tag t = new Tag((byte) 1, "myTag1");
61          Tag[] tags = new Tag[1];
62          tags[0] = t;
63          return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"),
64              Bytes.toBytes("qualifier"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"), tags);
65        }
66      }
67    }
68    static String toRowStr(Cell kv) {
69      return Bytes.toString(KeyValueUtil.ensureKeyValue(kv).getRow());
70    }
71  
72    Path makeNewFile(TagUsage tagUsage) throws IOException {
73      Path ncTFile = new Path(this.testDir, "basic.hfile");
74      if (tagUsage != TagUsage.NO_TAG) {
75        conf.setInt("hfile.format.version", 3);
76      } else {
77        conf.setInt("hfile.format.version", 2);
78      }
79      FSDataOutputStream fout = this.fs.create(ncTFile);
80      int blocksize = toKV("a", tagUsage).getLength() * 3;
81      HFileContext context = new HFileContextBuilder().withBlockSize(blocksize)
82          .withIncludesTags(true).build();
83      HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout)
84          .withFileContext(context)
85          .withComparator(KeyValue.COMPARATOR).create();
86      // 4 bytes * 3 * 2 for each key/value +
87      // 3 for keys, 15 for values = 42 (woot)
88      writer.append(toKV("c", tagUsage));
89      writer.append(toKV("e", tagUsage));
90      writer.append(toKV("g", tagUsage));
91      // block transition
92      writer.append(toKV("i", tagUsage));
93      writer.append(toKV("k", tagUsage));
94      writer.close();
95      fout.close();
96      return ncTFile;
97    }
98  
99    public void testSeekBefore() throws Exception {
100     testSeekBeforeInternals(TagUsage.NO_TAG);
101     testSeekBeforeInternals(TagUsage.ONLY_TAG);
102     testSeekBeforeInternals(TagUsage.PARTIAL_TAG);
103   }
104 
105   protected void testSeekBeforeInternals(TagUsage tagUsage) throws IOException {
106     Path p = makeNewFile(tagUsage);
107     HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
108     reader.loadFileInfo();
109     HFileScanner scanner = reader.getScanner(false, true);
110     assertEquals(false, scanner.seekBefore(toKV("a", tagUsage)));
111 
112     assertEquals(false, scanner.seekBefore(toKV("c", tagUsage)));
113 
114     assertEquals(true, scanner.seekBefore(toKV("d", tagUsage)));
115     assertEquals("c", toRowStr(scanner.getKeyValue()));
116 
117     assertEquals(true, scanner.seekBefore(toKV("e", tagUsage)));
118     assertEquals("c", toRowStr(scanner.getKeyValue()));
119 
120     assertEquals(true, scanner.seekBefore(toKV("f", tagUsage)));
121     assertEquals("e", toRowStr(scanner.getKeyValue()));
122 
123     assertEquals(true, scanner.seekBefore(toKV("g", tagUsage)));
124     assertEquals("e", toRowStr(scanner.getKeyValue()));
125 
126     assertEquals(true, scanner.seekBefore(toKV("h", tagUsage)));
127     assertEquals("g", toRowStr(scanner.getKeyValue()));
128     assertEquals(true, scanner.seekBefore(toKV("i", tagUsage)));
129     assertEquals("g", toRowStr(scanner.getKeyValue()));
130     assertEquals(true, scanner.seekBefore(toKV("j", tagUsage)));
131     assertEquals("i", toRowStr(scanner.getKeyValue()));
132     assertEquals(true, scanner.seekBefore(toKV("k", tagUsage)));
133     assertEquals("i", toRowStr(scanner.getKeyValue()));
134     assertEquals(true, scanner.seekBefore(toKV("l", tagUsage)));
135     assertEquals("k", toRowStr(scanner.getKeyValue()));
136 
137     reader.close();
138   }
139 
140   public void testSeekBeforeWithReSeekTo() throws Exception {
141     testSeekBeforeWithReSeekToInternals(TagUsage.NO_TAG);
142     testSeekBeforeWithReSeekToInternals(TagUsage.ONLY_TAG);
143     testSeekBeforeWithReSeekToInternals(TagUsage.PARTIAL_TAG);
144   }
145 
146   protected void testSeekBeforeWithReSeekToInternals(TagUsage tagUsage) throws IOException {
147     Path p = makeNewFile(tagUsage);
148     HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
149     reader.loadFileInfo();
150     HFileScanner scanner = reader.getScanner(false, true);
151     assertEquals(false, scanner.seekBefore(toKV("a", tagUsage)));
152     assertEquals(false, scanner.seekBefore(toKV("b", tagUsage)));
153     assertEquals(false, scanner.seekBefore(toKV("c", tagUsage)));
154 
155     // seekBefore d, so the scanner points to c
156     assertEquals(true, scanner.seekBefore(toKV("d", tagUsage)));
157     assertEquals("c", toRowStr(scanner.getKeyValue()));
158     // reseekTo e and g
159     assertEquals(0, scanner.reseekTo(toKV("c", tagUsage)));
160     assertEquals("c", toRowStr(scanner.getKeyValue()));
161     assertEquals(0, scanner.reseekTo(toKV("g", tagUsage)));
162     assertEquals("g", toRowStr(scanner.getKeyValue()));
163 
164     // seekBefore e, so the scanner points to c
165     assertEquals(true, scanner.seekBefore(toKV("e", tagUsage)));
166     assertEquals("c", toRowStr(scanner.getKeyValue()));
167     // reseekTo e and g
168     assertEquals(0, scanner.reseekTo(toKV("e", tagUsage)));
169     assertEquals("e", toRowStr(scanner.getKeyValue()));
170     assertEquals(0, scanner.reseekTo(toKV("g", tagUsage)));
171     assertEquals("g", toRowStr(scanner.getKeyValue()));
172 
173     // seekBefore f, so the scanner points to e
174     assertEquals(true, scanner.seekBefore(toKV("f", tagUsage)));
175     assertEquals("e", toRowStr(scanner.getKeyValue()));
176     // reseekTo e and g
177     assertEquals(0, scanner.reseekTo(toKV("e", tagUsage)));
178     assertEquals("e", toRowStr(scanner.getKeyValue()));
179     assertEquals(0, scanner.reseekTo(toKV("g", tagUsage)));
180     assertEquals("g", toRowStr(scanner.getKeyValue()));
181 
182     // seekBefore g, so the scanner points to e
183     assertEquals(true, scanner.seekBefore(toKV("g", tagUsage)));
184     assertEquals("e", toRowStr(scanner.getKeyValue()));
185     // reseekTo e and g again
186     assertEquals(0, scanner.reseekTo(toKV("e", tagUsage)));
187     assertEquals("e", toRowStr(scanner.getKeyValue()));
188     assertEquals(0, scanner.reseekTo(toKV("g", tagUsage)));
189     assertEquals("g", toRowStr(scanner.getKeyValue()));
190 
191     // seekBefore h, so the scanner points to g
192     assertEquals(true, scanner.seekBefore(toKV("h", tagUsage)));
193     assertEquals("g", toRowStr(scanner.getKeyValue()));
194     // reseekTo g
195     assertEquals(0, scanner.reseekTo(toKV("g", tagUsage)));
196     assertEquals("g", toRowStr(scanner.getKeyValue()));
197 
198     // seekBefore i, so the scanner points to g
199     assertEquals(true, scanner.seekBefore(toKV("i", tagUsage)));
200     assertEquals("g", toRowStr(scanner.getKeyValue()));
201     // reseekTo g
202     assertEquals(0, scanner.reseekTo(toKV("g", tagUsage)));
203     assertEquals("g", toRowStr(scanner.getKeyValue()));
204 
205     // seekBefore j, so the scanner points to i
206     assertEquals(true, scanner.seekBefore(toKV("j", tagUsage)));
207     assertEquals("i", toRowStr(scanner.getKeyValue()));
208     // reseekTo i
209     assertEquals(0, scanner.reseekTo(toKV("i", tagUsage)));
210     assertEquals("i", toRowStr(scanner.getKeyValue()));
211 
212     // seekBefore k, so the scanner points to i
213     assertEquals(true, scanner.seekBefore(toKV("k", tagUsage)));
214     assertEquals("i", toRowStr(scanner.getKeyValue()));
215     // reseekTo i and k
216     assertEquals(0, scanner.reseekTo(toKV("i", tagUsage)));
217     assertEquals("i", toRowStr(scanner.getKeyValue()));
218     assertEquals(0, scanner.reseekTo(toKV("k", tagUsage)));
219     assertEquals("k", toRowStr(scanner.getKeyValue()));
220 
221     // seekBefore l, so the scanner points to k
222     assertEquals(true, scanner.seekBefore(toKV("l", tagUsage)));
223     assertEquals("k", toRowStr(scanner.getKeyValue()));
224     // reseekTo k
225     assertEquals(0, scanner.reseekTo(toKV("k", tagUsage)));
226     assertEquals("k", toRowStr(scanner.getKeyValue()));
227   }
228 
229   public void testSeekTo() throws Exception {
230     testSeekToInternals(TagUsage.NO_TAG);
231     testSeekToInternals(TagUsage.ONLY_TAG);
232     testSeekToInternals(TagUsage.PARTIAL_TAG);
233   }
234 
235   protected void testSeekToInternals(TagUsage tagUsage) throws IOException {
236     Path p = makeNewFile(tagUsage);
237     HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
238     reader.loadFileInfo();
239     assertEquals(2, reader.getDataBlockIndexReader().getRootBlockCount());
240     HFileScanner scanner = reader.getScanner(false, true);
241     // lies before the start of the file.
242     assertEquals(-1, scanner.seekTo(toKV("a", tagUsage)));
243 
244     assertEquals(1, scanner.seekTo(toKV("d", tagUsage)));
245     assertEquals("c", toRowStr(scanner.getKeyValue()));
246 
247     // Across a block boundary now.
248     // 'h' does not exist so we will get a '1' back for not found.
249     assertEquals(0, scanner.seekTo(toKV("i", tagUsage)));
250     assertEquals("i", toRowStr(scanner.getKeyValue()));
251 
252     assertEquals(1, scanner.seekTo(toKV("l", tagUsage)));
253     assertEquals("k", toRowStr(scanner.getKeyValue()));
254 
255     reader.close();
256   }
257   public void testBlockContainingKey() throws Exception {
258     testBlockContainingKeyInternals(TagUsage.NO_TAG);
259     testBlockContainingKeyInternals(TagUsage.ONLY_TAG);
260     testBlockContainingKeyInternals(TagUsage.PARTIAL_TAG);
261   }
262 
263   protected void testBlockContainingKeyInternals(TagUsage tagUsage) throws IOException {
264     Path p = makeNewFile(tagUsage);
265     HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
266     reader.loadFileInfo();
267     HFileBlockIndex.BlockIndexReader blockIndexReader = 
268       reader.getDataBlockIndexReader();
269     System.out.println(blockIndexReader.toString());
270     // falls before the start of the file.
271     assertEquals(-1, blockIndexReader.rootBlockContainingKey(
272         toKV("a", tagUsage)));
273     assertEquals(0, blockIndexReader.rootBlockContainingKey(
274         toKV("c", tagUsage)));
275     assertEquals(0, blockIndexReader.rootBlockContainingKey(
276         toKV("d", tagUsage)));
277     assertEquals(0, blockIndexReader.rootBlockContainingKey(
278         toKV("e", tagUsage)));
279     assertEquals(0, blockIndexReader.rootBlockContainingKey(
280         toKV("g", tagUsage)));
281     assertEquals(1, blockIndexReader.rootBlockContainingKey(toKV("h", tagUsage)));
282     assertEquals(1, blockIndexReader.rootBlockContainingKey(
283         toKV("i", tagUsage)));
284     assertEquals(1, blockIndexReader.rootBlockContainingKey(
285         toKV("j", tagUsage)));
286     assertEquals(1, blockIndexReader.rootBlockContainingKey(
287         toKV("k", tagUsage)));
288     assertEquals(1, blockIndexReader.rootBlockContainingKey(
289         toKV("l", tagUsage)));
290     reader.close();
291   }
292 }