1   /*
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.io;
22  
23  import static org.junit.Assert.assertEquals;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  
27  import java.io.IOException;
28  import java.util.ArrayList;
29  import java.util.List;
30  
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.*;
35  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
36  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
37  import org.apache.hadoop.hbase.io.hfile.HFile;
38  import org.apache.hadoop.hbase.io.hfile.HFileScanner;
39  import org.apache.hadoop.hbase.util.Bytes;
40  import org.junit.Test;
41  import org.junit.experimental.categories.Category;
42  
43  @Category(SmallTests.class)
44  public class TestHalfStoreFileReader {
45  
46    /**
47     * Test the scanner and reseek of a half hfile scanner. The scanner API
48     * demands that seekTo and reseekTo() only return < 0 if the key lies
49     * before the start of the file (with no position on the scanner). Returning
50     * 0 if perfect match (rare), and return > 1 if we got an imperfect match.
51     *
52     * The latter case being the most common, we should generally be returning 1,
53     * and if we do, there may or may not be a 'next' in the scanner/file.
54     *
55     * A bug in the half file scanner was returning -1 at the end of the bottom
56     * half, and that was causing the infrastructure above to go null causing NPEs
57     * and other problems.  This test reproduces that failure, and also tests
58     * both the bottom and top of the file while we are at it.
59     *
60     * @throws IOException
61     */
62    @Test
63    public void testHalfScanAndReseek() throws IOException {
64      HBaseTestingUtility test_util = new HBaseTestingUtility();
65      String root_dir = test_util.getDataTestDir("TestHalfStoreFile").toString();
66      Path p = new Path(root_dir, "test");
67  
68      Configuration conf = test_util.getConfiguration();
69      FileSystem fs = FileSystem.get(conf);
70      CacheConfig cacheConf = new CacheConfig(conf);
71  
72      HFile.Writer w = HFile.getWriterFactory(conf, cacheConf)
73          .withPath(fs, p)
74          .withBlockSize(1024)
75          .withComparator(KeyValue.KEY_COMPARATOR)
76          .create();
77  
78      // write some things.
79      List<KeyValue> items = genSomeKeys();
80      for (KeyValue kv : items) {
81        w.append(kv);
82      }
83      w.close();
84  
85      HFile.Reader r = HFile.createReader(fs, p, cacheConf);
86      r.loadFileInfo();
87      byte [] midkey = r.midkey();
88      KeyValue midKV = KeyValue.createKeyValueFromKey(midkey);
89      midkey = midKV.getRow();
90  
91      //System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey));
92  
93      Reference bottom = new Reference(midkey, Reference.Range.bottom);
94      doTestOfScanAndReseek(p, fs, bottom, cacheConf);
95  
96      Reference top = new Reference(midkey, Reference.Range.top);
97      doTestOfScanAndReseek(p, fs, top, cacheConf);
98  
99      r.close();
100   }
101 
102   private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom,
103       CacheConfig cacheConf)
104       throws IOException {
105     final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p,
106         cacheConf, bottom, DataBlockEncoding.NONE);
107     halfreader.loadFileInfo();
108     final HFileScanner scanner = halfreader.getScanner(false, false);
109 
110     scanner.seekTo();
111     KeyValue curr;
112     do {
113       curr = scanner.getKeyValue();
114       KeyValue reseekKv =
115           getLastOnCol(curr);
116       int ret = scanner.reseekTo(reseekKv.getKey());
117       assertTrue("reseek to returned: " + ret, ret > 0);
118       //System.out.println(curr + ": " + ret);
119     } while (scanner.next());
120 
121     int ret = scanner.reseekTo(getLastOnCol(curr).getKey());
122     //System.out.println("Last reseek: " + ret);
123     assertTrue( ret > 0 );
124 
125     halfreader.close(true);
126   }
127 
128 
129   // Tests the scanner on an HFile that is backed by HalfStoreFiles
130   @Test
131   public void testHalfScanner() throws IOException {
132       HBaseTestingUtility test_util = new HBaseTestingUtility();
133       String root_dir = test_util.getDataTestDir("TestHalfStoreFileScanBefore").toString();
134       Path p = new Path(root_dir, "test");
135       Configuration conf = test_util.getConfiguration();
136       FileSystem fs = FileSystem.get(conf);
137       CacheConfig cacheConf = new CacheConfig(conf);
138 
139       HFile.Writer w = HFile.getWriterFactory(conf, cacheConf)
140               .withPath(fs, p)
141               .withBlockSize(1024)
142               .withComparator(KeyValue.KEY_COMPARATOR)
143               .create();
144 
145       // write some things.
146       List<KeyValue> items = genSomeKeys();
147       for (KeyValue kv : items) {
148           w.append(kv);
149       }
150       w.close();
151 
152 
153       HFile.Reader r = HFile.createReader(fs, p, cacheConf);
154       r.loadFileInfo();
155       byte[] midkey = r.midkey();
156       KeyValue midKV = KeyValue.createKeyValueFromKey(midkey);
157       midkey = midKV.getRow();
158 
159       Reference bottom = new Reference(midkey, Reference.Range.bottom);
160       Reference top = new Reference(midkey, Reference.Range.top);
161 
162       // Ugly code to get the item before the midkey
163       KeyValue beforeMidKey = null;
164       for (KeyValue item : items) {
165           if (item.equals(midKV)) {
166               break;
167           }
168           beforeMidKey = item;
169       }
170 
171 
172       // Seek on the splitKey, should be in top, not in bottom
173       KeyValue foundKeyValue = doTestOfSeekBefore(p, fs, bottom, midKV, cacheConf);
174       assertEquals(beforeMidKey, foundKeyValue);
175 
176       // Seek tot the last thing should be the penultimate on the top, the one before the midkey on the bottom.
177       foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(items.size() - 1), cacheConf);
178       assertEquals(items.get(items.size() - 2), foundKeyValue);
179 
180       foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(items.size() - 1), cacheConf);
181       assertEquals(beforeMidKey, foundKeyValue);
182 
183       // Try and seek before something that is in the bottom.
184       foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(0), cacheConf);
185       assertNull(foundKeyValue);
186 
187       // Try and seek before the first thing.
188       foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(0), cacheConf);
189       assertNull(foundKeyValue);
190 
191       // Try and seek before the second thing in the top and bottom.
192       foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(1), cacheConf);
193       assertNull(foundKeyValue);
194 
195       foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(1), cacheConf);
196       assertEquals(items.get(0), foundKeyValue);
197 
198       // Try to seek before the splitKey in the top file
199       foundKeyValue = doTestOfSeekBefore(p, fs, top, midKV, cacheConf);
200       assertNull(foundKeyValue);
201     }
202 
203   private KeyValue doTestOfSeekBefore(Path p, FileSystem fs, Reference bottom, KeyValue seekBefore,
204                                         CacheConfig cacheConfig)
205             throws IOException {
206       final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p,
207               cacheConfig, bottom, DataBlockEncoding.NONE);
208       halfreader.loadFileInfo();
209       final HFileScanner scanner = halfreader.getScanner(false, false);
210       scanner.seekBefore(seekBefore.getKey());
211       return scanner.getKeyValue();
212   }
213 
214   private KeyValue getLastOnCol(KeyValue curr) {
215     return KeyValue.createLastOnRow(
216         curr.getBuffer(), curr.getRowOffset(), curr.getRowLength(),
217         curr.getBuffer(), curr.getFamilyOffset(), curr.getFamilyLength(),
218         curr.getBuffer(), curr.getQualifierOffset(), curr.getQualifierLength());
219   }
220 
221   static final int SIZE = 1000;
222 
223   static byte[] _b(String s) {
224     return Bytes.toBytes(s);
225   }
226 
227   List<KeyValue> genSomeKeys() {
228     List<KeyValue> ret = new ArrayList<KeyValue>(SIZE);
229     for (int i = 0; i < SIZE; i++) {
230       KeyValue kv =
231           new KeyValue(
232               _b(String.format("row_%04d", i)),
233               _b("family"),
234               _b("qualifier"),
235               1000, // timestamp
236               _b("value"));
237       ret.add(kv);
238     }
239     return ret;
240   }
241 
242 
243 
244   @org.junit.Rule
245   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
246     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
247 }
248