View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.io.DataInput;
22  import java.io.DataOutput;
23  import java.io.IOException;
24  import java.nio.ByteBuffer;
25  import java.util.Arrays;
26  import java.util.Map;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.fs.FSDataInputStream;
31  import org.apache.hadoop.fs.FSDataOutputStream;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.hbase.HBaseTestCase;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.KeyValue.KeyComparator;
38  import org.apache.hadoop.hbase.SmallTests;
39  import org.apache.hadoop.hbase.io.compress.Compression;
40  import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
41  import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.io.Writable;
44  import org.junit.experimental.categories.Category;
45  
46  /**
47   * test hfile features.
48   * <p>
49   * Copied from
50   * <a href="https://issues.apache.org/jira/browse/HADOOP-3315">hadoop-3315 tfile</a>.
51   * Remove after tfile is committed and use the tfile version of this class
52   * instead.</p>
53   */
54  @Category(SmallTests.class)
55  public class TestHFile extends HBaseTestCase {
56    static final Log LOG = LogFactory.getLog(TestHFile.class);
57  
58    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
59    private static String ROOT_DIR =
60      TEST_UTIL.getDataTestDir("TestHFile").toString();
61    private final int minBlockSize = 512;
62    private static String localFormatter = "%010d";
63    private static CacheConfig cacheConf = null;
64    private Map<String, Long> startingMetrics;
65  
66    @Override
67    public void setUp() throws Exception {
68      super.setUp();
69    }
70  
71    @Override
72    public void tearDown() throws Exception {
73      super.tearDown();
74    }
75  
76  
77    /**
78     * Test empty HFile.
79     * Test all features work reasonably when hfile is empty of entries.
80     * @throws IOException
81     */
82    public void testEmptyHFile() throws IOException {
83      if (cacheConf == null) cacheConf = new CacheConfig(conf);
84      Path f = new Path(ROOT_DIR, getName());
85      Writer w =
86          HFile.getWriterFactory(conf, cacheConf).withPath(fs, f).create();
87      w.close();
88      Reader r = HFile.createReader(fs, f, cacheConf);
89      r.loadFileInfo();
90      assertNull(r.getFirstKey());
91      assertNull(r.getLastKey());
92    }
93  
94    /**
95     * Create 0-length hfile and show that it fails
96     */
97    public void testCorrupt0LengthHFile() throws IOException {
98      if (cacheConf == null) cacheConf = new CacheConfig(conf);
99      Path f = new Path(ROOT_DIR, getName());
100     FSDataOutputStream fsos = fs.create(f);
101     fsos.close();
102 
103     try {
104       Reader r = HFile.createReader(fs, f, cacheConf);
105     } catch (CorruptHFileException che) {
106       // Expected failure
107       return;
108     }
109     fail("Should have thrown exception");
110   }
111 
112   public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException {
113     FileStatus fst = fs.getFileStatus(src);
114     long len = fst.getLen();
115     len = len / 2 ;
116 
117     // create a truncated hfile
118     FSDataOutputStream fdos = fs.create(dst);
119     byte[] buf = new byte[(int)len];
120     FSDataInputStream fdis = fs.open(src);
121     fdis.read(buf);
122     fdos.write(buf);
123     fdis.close();
124     fdos.close();
125   }
126 
127   /**
128    * Create a truncated hfile and verify that exception thrown.
129    */
130   public void testCorruptTruncatedHFile() throws IOException {
131     if (cacheConf == null) cacheConf = new CacheConfig(conf);
132     Path f = new Path(ROOT_DIR, getName());
133     Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f).create();
134     writeSomeRecords(w, 0, 100);
135     w.close();
136 
137     Path trunc = new Path(f.getParent(), "trucated");
138     truncateFile(fs, w.getPath(), trunc);
139 
140     try {
141       Reader r = HFile.createReader(fs, trunc, cacheConf);
142     } catch (CorruptHFileException che) {
143       // Expected failure
144       return;
145     }
146     fail("Should have thrown exception");
147   }
148 
149   // write some records into the tfile
150   // write them twice
151   private int writeSomeRecords(Writer writer, int start, int n)
152       throws IOException {
153     String value = "value";
154     for (int i = start; i < (start + n); i++) {
155       String key = String.format(localFormatter, Integer.valueOf(i));
156       writer.append(Bytes.toBytes(key), Bytes.toBytes(value + key));
157     }
158     return (start + n);
159   }
160 
161   private void readAllRecords(HFileScanner scanner) throws IOException {
162     readAndCheckbytes(scanner, 0, 100);
163   }
164 
165   // read the records and check
166   private int readAndCheckbytes(HFileScanner scanner, int start, int n)
167       throws IOException {
168     String value = "value";
169     int i = start;
170     for (; i < (start + n); i++) {
171       ByteBuffer key = scanner.getKey();
172       ByteBuffer val = scanner.getValue();
173       String keyStr = String.format(localFormatter, Integer.valueOf(i));
174       String valStr = value + keyStr;
175       byte [] keyBytes = Bytes.toBytes(key);
176       assertTrue("bytes for keys do not match " + keyStr + " " +
177         Bytes.toString(Bytes.toBytes(key)),
178           Arrays.equals(Bytes.toBytes(keyStr), keyBytes));
179       byte [] valBytes = Bytes.toBytes(val);
180       assertTrue("bytes for vals do not match " + valStr + " " +
181         Bytes.toString(valBytes),
182         Arrays.equals(Bytes.toBytes(valStr), valBytes));
183       if (!scanner.next()) {
184         break;
185       }
186     }
187     assertEquals(i, start + n - 1);
188     return (start + n);
189   }
190 
191   private byte[] getSomeKey(int rowId) {
192     return String.format(localFormatter, Integer.valueOf(rowId)).getBytes();
193   }
194 
195   private void writeRecords(Writer writer) throws IOException {
196     writeSomeRecords(writer, 0, 100);
197     writer.close();
198   }
199 
200   private FSDataOutputStream createFSOutput(Path name) throws IOException {
201     //if (fs.exists(name)) fs.delete(name, true);
202     FSDataOutputStream fout = fs.create(name);
203     return fout;
204   }
205 
206   /**
207    * test none codecs
208    */
209   void basicWithSomeCodec(String codec) throws IOException {
210     if (cacheConf == null) cacheConf = new CacheConfig(conf);
211     Path ncTFile = new Path(ROOT_DIR, "basic.hfile." + codec.toString());
212     FSDataOutputStream fout = createFSOutput(ncTFile);
213     Writer writer = HFile.getWriterFactory(conf, cacheConf)
214         .withOutputStream(fout)
215         .withBlockSize(minBlockSize)
216         .withCompression(codec)
217         .create();
218     LOG.info(writer);
219     writeRecords(writer);
220     fout.close();
221     FSDataInputStream fin = fs.open(ncTFile);
222     Reader reader = HFile.createReaderFromStream(ncTFile, fs.open(ncTFile),
223       fs.getFileStatus(ncTFile).getLen(), cacheConf);
224     System.out.println(cacheConf.toString());
225     // Load up the index.
226     reader.loadFileInfo();
227     // Get a scanner that caches and that does not use pread.
228     HFileScanner scanner = reader.getScanner(true, false);
229     // Align scanner at start of the file.
230     scanner.seekTo();
231     readAllRecords(scanner);
232     scanner.seekTo(getSomeKey(50));
233     assertTrue("location lookup failed", scanner.seekTo(getSomeKey(50)) == 0);
234     // read the key and see if it matches
235     ByteBuffer readKey = scanner.getKey();
236     assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50),
237       Bytes.toBytes(readKey)));
238 
239     scanner.seekTo(new byte[0]);
240     ByteBuffer val1 = scanner.getValue();
241     scanner.seekTo(new byte[0]);
242     ByteBuffer val2 = scanner.getValue();
243     assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2)));
244 
245     reader.close();
246     fin.close();
247     fs.delete(ncTFile, true);
248   }
249 
250   public void testTFileFeatures() throws IOException {
251     basicWithSomeCodec("none");
252     basicWithSomeCodec("gz");
253   }
254 
255   private void writeNumMetablocks(Writer writer, int n) {
256     for (int i = 0; i < n; i++) {
257       writer.appendMetaBlock("HFileMeta" + i, new Writable() {
258         private int val;
259         public Writable setVal(int val) { this.val = val; return this; }
260         
261         @Override
262         public void write(DataOutput out) throws IOException {
263           out.write(("something to test" + val).getBytes());
264         }
265         
266         @Override
267         public void readFields(DataInput in) throws IOException { }
268       }.setVal(i));
269     }
270   }
271 
272   private void someTestingWithMetaBlock(Writer writer) {
273     writeNumMetablocks(writer, 10);
274   }
275 
276   private void readNumMetablocks(Reader reader, int n) throws IOException {
277     for (int i = 0; i < n; i++) {
278       ByteBuffer actual = reader.getMetaBlock("HFileMeta" + i, false);
279       ByteBuffer expected = 
280         ByteBuffer.wrap(("something to test" + i).getBytes());
281       assertTrue("failed to match metadata", actual.compareTo(expected) == 0);
282     }
283   }
284 
285   private void someReadingWithMetaBlock(Reader reader) throws IOException {
286     readNumMetablocks(reader, 10);
287   }
288 
289   private void metablocks(final String compress) throws Exception {
290     if (cacheConf == null) cacheConf = new CacheConfig(conf);
291     Path mFile = new Path(ROOT_DIR, "meta.hfile");
292     FSDataOutputStream fout = createFSOutput(mFile);
293     Writer writer = HFile.getWriterFactory(conf, cacheConf)
294         .withOutputStream(fout)
295         .withBlockSize(minBlockSize)
296         .withCompression(compress)
297         .create();
298     someTestingWithMetaBlock(writer);
299     writer.close();
300     fout.close();
301     FSDataInputStream fin = fs.open(mFile);
302     Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile),
303         this.fs.getFileStatus(mFile).getLen(), cacheConf);
304     reader.loadFileInfo();
305     // No data -- this should return false.
306     assertFalse(reader.getScanner(false, false).seekTo());
307     someReadingWithMetaBlock(reader);
308     fs.delete(mFile, true);
309     reader.close();
310     fin.close();
311   }
312 
313   // test meta blocks for tfiles
314   public void testMetaBlocks() throws Exception {
315     metablocks("none");
316     metablocks("gz");
317   }
318 
319   public void testNullMetaBlocks() throws Exception {
320     if (cacheConf == null) cacheConf = new CacheConfig(conf);
321     for (Compression.Algorithm compressAlgo : 
322         HBaseTestingUtility.COMPRESSION_ALGORITHMS) {
323       Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile");
324       FSDataOutputStream fout = createFSOutput(mFile);
325       Writer writer = HFile.getWriterFactory(conf, cacheConf)
326           .withOutputStream(fout)
327           .withBlockSize(minBlockSize)
328           .withCompression(compressAlgo)
329           .create();
330       writer.append("foo".getBytes(), "value".getBytes());
331       writer.close();
332       fout.close();
333       Reader reader = HFile.createReader(fs, mFile, cacheConf);
334       reader.loadFileInfo();
335       assertNull(reader.getMetaBlock("non-existant", false));
336     }
337   }
338 
339   /**
340    * Make sure the ordinals for our compression algorithms do not change on us.
341    */
342   public void testCompressionOrdinance() {
343     assertTrue(Compression.Algorithm.LZO.ordinal() == 0);
344     assertTrue(Compression.Algorithm.GZ.ordinal() == 1);
345     assertTrue(Compression.Algorithm.NONE.ordinal() == 2);
346     assertTrue(Compression.Algorithm.SNAPPY.ordinal() == 3);
347     assertTrue(Compression.Algorithm.LZ4.ordinal() == 4);
348   }
349 
350   // This can't be an anonymous class because the compiler will not generate
351   // a nullary constructor for it.
352   static class CustomKeyComparator extends KeyComparator {
353     @Override
354     public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2,
355         int l2) {
356       return -Bytes.compareTo(b1, s1, l1, b2, s2, l2);
357     }
358     @Override
359     public int compare(byte[] o1, byte[] o2) {
360       return compare(o1, 0, o1.length, o2, 0, o2.length);
361     }
362   }
363 
364   public void testComparator() throws IOException {
365     if (cacheConf == null) cacheConf = new CacheConfig(conf);
366     Path mFile = new Path(ROOT_DIR, "meta.tfile");
367     FSDataOutputStream fout = createFSOutput(mFile);
368     KeyComparator comparator = new CustomKeyComparator();
369     Writer writer = HFile.getWriterFactory(conf, cacheConf)
370         .withOutputStream(fout)
371         .withBlockSize(minBlockSize)
372         .withComparator(comparator)
373         .create();
374     writer.append("3".getBytes(), "0".getBytes());
375     writer.append("2".getBytes(), "0".getBytes());
376     writer.append("1".getBytes(), "0".getBytes());
377     writer.close();
378   }
379 
380 
381 }
382