View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.io;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertNull;
24  import static org.junit.Assert.assertTrue;
25  
26  import java.io.IOException;
27  import java.util.ArrayList;
28  import java.util.List;
29  
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.FileSystem;
32  import org.apache.hadoop.fs.Path;
33  import org.apache.hadoop.hbase.*;
34  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
35  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
36  import org.apache.hadoop.hbase.io.hfile.HFile;
37  import org.apache.hadoop.hbase.io.hfile.HFileScanner;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.junit.AfterClass;
40  import org.junit.BeforeClass;
41  import org.junit.Test;
42  import org.junit.experimental.categories.Category;
43  
44  @Category(SmallTests.class)
45  public class TestHalfStoreFileReader {
46    private static HBaseTestingUtility TEST_UTIL;
47  
48    @BeforeClass
49    public static void setupBeforeClass() throws Exception {
50      TEST_UTIL = new HBaseTestingUtility();
51    }
52  
53    @AfterClass
54    public static void tearDownAfterClass() throws Exception {
55      TEST_UTIL.cleanupTestDir();
56    }
57  
58    /**
59     * Test the scanner and reseek of a half hfile scanner. The scanner API
60     * demands that seekTo and reseekTo() only return < 0 if the key lies
61     * before the start of the file (with no position on the scanner). Returning
62     * 0 if perfect match (rare), and return > 1 if we got an imperfect match.
63     *
64     * The latter case being the most common, we should generally be returning 1,
65     * and if we do, there may or may not be a 'next' in the scanner/file.
66     *
67     * A bug in the half file scanner was returning -1 at the end of the bottom
68     * half, and that was causing the infrastructure above to go null causing NPEs
69     * and other problems.  This test reproduces that failure, and also tests
70     * both the bottom and top of the file while we are at it.
71     *
72     * @throws IOException
73     */
74    @Test
75    public void testHalfScanAndReseek() throws IOException {
76      String root_dir = TEST_UTIL.getDataTestDir().toString();
77      Path p = new Path(root_dir, "test");
78  
79      Configuration conf = TEST_UTIL.getConfiguration();
80      FileSystem fs = FileSystem.get(conf);
81      CacheConfig cacheConf = new CacheConfig(conf);
82  
83      HFile.Writer w = HFile.getWriterFactory(conf, cacheConf)
84          .withPath(fs, p)
85          .withBlockSize(1024)
86          .withComparator(KeyValue.KEY_COMPARATOR)
87          .create();
88  
89      // write some things.
90      List<KeyValue> items = genSomeKeys();
91      for (KeyValue kv : items) {
92        w.append(kv);
93      }
94      w.close();
95  
96      HFile.Reader r = HFile.createReader(fs, p, cacheConf);
97      r.loadFileInfo();
98      byte [] midkey = r.midkey();
99      KeyValue midKV = KeyValue.createKeyValueFromKey(midkey);
100     midkey = midKV.getRow();
101 
102     //System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey));
103 
104     Reference bottom = new Reference(midkey, Reference.Range.bottom);
105     doTestOfScanAndReseek(p, fs, bottom, cacheConf);
106 
107     Reference top = new Reference(midkey, Reference.Range.top);
108     doTestOfScanAndReseek(p, fs, top, cacheConf);
109 
110     r.close();
111   }
112 
113   private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom,
114       CacheConfig cacheConf)
115       throws IOException {
116     final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p,
117         cacheConf, bottom, DataBlockEncoding.NONE);
118     halfreader.loadFileInfo();
119     final HFileScanner scanner = halfreader.getScanner(false, false);
120 
121     scanner.seekTo();
122     KeyValue curr;
123     do {
124       curr = scanner.getKeyValue();
125       KeyValue reseekKv =
126           getLastOnCol(curr);
127       int ret = scanner.reseekTo(reseekKv.getKey());
128       assertTrue("reseek to returned: " + ret, ret > 0);
129       //System.out.println(curr + ": " + ret);
130     } while (scanner.next());
131 
132     int ret = scanner.reseekTo(getLastOnCol(curr).getKey());
133     //System.out.println("Last reseek: " + ret);
134     assertTrue( ret > 0 );
135 
136     halfreader.close(true);
137   }
138 
139 
140   // Tests the scanner on an HFile that is backed by HalfStoreFiles
141   @Test
142   public void testHalfScanner() throws IOException {
143       String root_dir = TEST_UTIL.getDataTestDir().toString();
144       Path p = new Path(root_dir, "test");
145       Configuration conf = TEST_UTIL.getConfiguration();
146       FileSystem fs = FileSystem.get(conf);
147       CacheConfig cacheConf = new CacheConfig(conf);
148 
149       HFile.Writer w = HFile.getWriterFactory(conf, cacheConf)
150               .withPath(fs, p)
151               .withBlockSize(1024)
152               .withComparator(KeyValue.KEY_COMPARATOR)
153               .create();
154 
155       // write some things.
156       List<KeyValue> items = genSomeKeys();
157       for (KeyValue kv : items) {
158           w.append(kv);
159       }
160       w.close();
161 
162 
163       HFile.Reader r = HFile.createReader(fs, p, cacheConf);
164       r.loadFileInfo();
165       byte[] midkey = r.midkey();
166       KeyValue midKV = KeyValue.createKeyValueFromKey(midkey);
167       midkey = midKV.getRow();
168 
169       Reference bottom = new Reference(midkey, Reference.Range.bottom);
170       Reference top = new Reference(midkey, Reference.Range.top);
171 
172       // Ugly code to get the item before the midkey
173       KeyValue beforeMidKey = null;
174       for (KeyValue item : items) {
175           if (item.equals(midKV)) {
176               break;
177           }
178           beforeMidKey = item;
179       }
180 
181 
182       // Seek on the splitKey, should be in top, not in bottom
183       KeyValue foundKeyValue = doTestOfSeekBefore(p, fs, bottom, midKV, cacheConf);
184       assertEquals(beforeMidKey, foundKeyValue);
185 
186       // Seek tot the last thing should be the penultimate on the top, the one before the midkey on the bottom.
187       foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(items.size() - 1), cacheConf);
188       assertEquals(items.get(items.size() - 2), foundKeyValue);
189 
190       foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(items.size() - 1), cacheConf);
191       assertEquals(beforeMidKey, foundKeyValue);
192 
193       // Try and seek before something that is in the bottom.
194       foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(0), cacheConf);
195       assertNull(foundKeyValue);
196 
197       // Try and seek before the first thing.
198       foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(0), cacheConf);
199       assertNull(foundKeyValue);
200 
201       // Try and seek before the second thing in the top and bottom.
202       foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(1), cacheConf);
203       assertNull(foundKeyValue);
204 
205       foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(1), cacheConf);
206       assertEquals(items.get(0), foundKeyValue);
207 
208       // Try to seek before the splitKey in the top file
209       foundKeyValue = doTestOfSeekBefore(p, fs, top, midKV, cacheConf);
210       assertNull(foundKeyValue);
211     }
212 
213   private KeyValue doTestOfSeekBefore(Path p, FileSystem fs, Reference bottom, KeyValue seekBefore,
214                                         CacheConfig cacheConfig)
215             throws IOException {
216       final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p,
217               cacheConfig, bottom, DataBlockEncoding.NONE);
218       halfreader.loadFileInfo();
219       final HFileScanner scanner = halfreader.getScanner(false, false);
220       scanner.seekBefore(seekBefore.getKey());
221       return scanner.getKeyValue();
222   }
223 
224   private KeyValue getLastOnCol(KeyValue curr) {
225     return KeyValue.createLastOnRow(
226         curr.getBuffer(), curr.getRowOffset(), curr.getRowLength(),
227         curr.getBuffer(), curr.getFamilyOffset(), curr.getFamilyLength(),
228         curr.getBuffer(), curr.getQualifierOffset(), curr.getQualifierLength());
229   }
230 
231   static final int SIZE = 1000;
232 
233   static byte[] _b(String s) {
234     return Bytes.toBytes(s);
235   }
236 
237   List<KeyValue> genSomeKeys() {
238     List<KeyValue> ret = new ArrayList<KeyValue>(SIZE);
239     for (int i = 0; i < SIZE; i++) {
240       KeyValue kv =
241           new KeyValue(
242               _b(String.format("row_%04d", i)),
243               _b("family"),
244               _b("qualifier"),
245               1000, // timestamp
246               _b("value"));
247       ret.add(kv);
248     }
249     return ret;
250   }
251 
252 
253 
254 }
255