View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.io.DataInput;
22  import java.io.DataOutput;
23  import java.io.IOException;
24  import java.nio.ByteBuffer;
25  import java.util.Arrays;
26  import java.util.Map;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.fs.FSDataInputStream;
31  import org.apache.hadoop.fs.FSDataOutputStream;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.hbase.HBaseTestCase;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.KeyValue;
38  import org.apache.hadoop.hbase.SmallTests;
39  import org.apache.hadoop.hbase.io.compress.Compression;
40  import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
41  import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.io.Writable;
44  import org.junit.experimental.categories.Category;
45  
46  /**
47   * test hfile features.
48   * <p>
49   * Copied from
50   * <a href="https://issues.apache.org/jira/browse/HADOOP-3315">hadoop-3315 tfile</a>.
51   * Remove after tfile is committed and use the tfile version of this class
52   * instead.</p>
53   */
54  @Category(SmallTests.class)
55  public class TestHFile extends HBaseTestCase {
56    static final Log LOG = LogFactory.getLog(TestHFile.class);
57  
58    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
59    private static String ROOT_DIR =
60      TEST_UTIL.getDataTestDir("TestHFile").toString();
61    private final int minBlockSize = 512;
62    private static String localFormatter = "%010d";
63    private static CacheConfig cacheConf = null;
64    private Map<String, Long> startingMetrics;
65  
66    @Override
67    public void setUp() throws Exception {
68      super.setUp();
69    }
70  
71    @Override
72    public void tearDown() throws Exception {
73      super.tearDown();
74    }
75  
76  
77    /**
78     * Test empty HFile.
79     * Test all features work reasonably when hfile is empty of entries.
80     * @throws IOException
81     */
82    public void testEmptyHFile() throws IOException {
83      if (cacheConf == null) cacheConf = new CacheConfig(conf);
84      Path f = new Path(ROOT_DIR, getName());
85      Writer w =
86          HFile.getWriterFactory(conf, cacheConf).withPath(fs, f).create();
87      w.close();
88      Reader r = HFile.createReader(fs, f, cacheConf);
89      r.loadFileInfo();
90      assertNull(r.getFirstKey());
91      assertNull(r.getLastKey());
92    }
93  
94    /**
95     * Create 0-length hfile and show that it fails
96     */
97    public void testCorrupt0LengthHFile() throws IOException {
98      if (cacheConf == null) cacheConf = new CacheConfig(conf);
99      Path f = new Path(ROOT_DIR, getName());
100     FSDataOutputStream fsos = fs.create(f);
101     fsos.close();
102 
103     try {
104       Reader r = HFile.createReader(fs, f, cacheConf);
105     } catch (CorruptHFileException che) {
106       // Expected failure
107       return;
108     }
109     fail("Should have thrown exception");
110   }
111 
112   public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException {
113     FileStatus fst = fs.getFileStatus(src);
114     long len = fst.getLen();
115     len = len / 2 ;
116 
117     // create a truncated hfile
118     FSDataOutputStream fdos = fs.create(dst);
119     byte[] buf = new byte[(int)len];
120     FSDataInputStream fdis = fs.open(src);
121     fdis.read(buf);
122     fdos.write(buf);
123     fdis.close();
124     fdos.close();
125   }
126 
127   /**
128    * Create a truncated hfile and verify that exception thrown.
129    */
130   public void testCorruptTruncatedHFile() throws IOException {
131     if (cacheConf == null) cacheConf = new CacheConfig(conf);
132     Path f = new Path(ROOT_DIR, getName());
133     Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f).create();
134     writeSomeRecords(w, 0, 100);
135     w.close();
136 
137     Path trunc = new Path(f.getParent(), "trucated");
138     truncateFile(fs, w.getPath(), trunc);
139 
140     try {
141       Reader r = HFile.createReader(fs, trunc, cacheConf);
142     } catch (CorruptHFileException che) {
143       // Expected failure
144       return;
145     }
146     fail("Should have thrown exception");
147   }
148 
149   // write some records into the tfile
150   // write them twice
151   private int writeSomeRecords(Writer writer, int start, int n)
152       throws IOException {
153     String value = "value";
154     for (int i = start; i < (start + n); i++) {
155       String key = String.format(localFormatter, Integer.valueOf(i));
156       writer.append(Bytes.toBytes(key), Bytes.toBytes(value + key));
157     }
158     return (start + n);
159   }
160 
161   private void readAllRecords(HFileScanner scanner) throws IOException {
162     readAndCheckbytes(scanner, 0, 100);
163   }
164 
165   // read the records and check
166   private int readAndCheckbytes(HFileScanner scanner, int start, int n)
167       throws IOException {
168     String value = "value";
169     int i = start;
170     for (; i < (start + n); i++) {
171       ByteBuffer key = scanner.getKey();
172       ByteBuffer val = scanner.getValue();
173       String keyStr = String.format(localFormatter, Integer.valueOf(i));
174       String valStr = value + keyStr;
175       byte [] keyBytes = Bytes.toBytes(key);
176       assertTrue("bytes for keys do not match " + keyStr + " " +
177         Bytes.toString(Bytes.toBytes(key)),
178           Arrays.equals(Bytes.toBytes(keyStr), keyBytes));
179       byte [] valBytes = Bytes.toBytes(val);
180       assertTrue("bytes for vals do not match " + valStr + " " +
181         Bytes.toString(valBytes),
182         Arrays.equals(Bytes.toBytes(valStr), valBytes));
183       if (!scanner.next()) {
184         break;
185       }
186     }
187     assertEquals(i, start + n - 1);
188     return (start + n);
189   }
190 
191   private byte[] getSomeKey(int rowId) {
192     return String.format(localFormatter, Integer.valueOf(rowId)).getBytes();
193   }
194 
195   private void writeRecords(Writer writer) throws IOException {
196     writeSomeRecords(writer, 0, 100);
197     writer.close();
198   }
199 
200   private FSDataOutputStream createFSOutput(Path name) throws IOException {
201     //if (fs.exists(name)) fs.delete(name, true);
202     FSDataOutputStream fout = fs.create(name);
203     return fout;
204   }
205 
206   /**
207    * test none codecs
208    */
209   void basicWithSomeCodec(String codec) throws IOException {
210     if (cacheConf == null) cacheConf = new CacheConfig(conf);
211     Path ncTFile = new Path(ROOT_DIR, "basic.hfile." + codec.toString());
212     FSDataOutputStream fout = createFSOutput(ncTFile);
213     Writer writer = HFile.getWriterFactory(conf, cacheConf)
214         .withOutputStream(fout)
215         .withBlockSize(minBlockSize)
216         .withCompression(codec)
217         // NOTE: This test is dependent on this deprecated nonstandard comparator
218         .withComparator(new KeyValue.RawBytesComparator())
219         .create();
220     LOG.info(writer);
221     writeRecords(writer);
222     fout.close();
223     FSDataInputStream fin = fs.open(ncTFile);
224     Reader reader = HFile.createReaderFromStream(ncTFile, fs.open(ncTFile),
225       fs.getFileStatus(ncTFile).getLen(), cacheConf);
226     System.out.println(cacheConf.toString());
227     // Load up the index.
228     reader.loadFileInfo();
229     // Get a scanner that caches and that does not use pread.
230     HFileScanner scanner = reader.getScanner(true, false);
231     // Align scanner at start of the file.
232     scanner.seekTo();
233     readAllRecords(scanner);
234     scanner.seekTo(getSomeKey(50));
235     assertTrue("location lookup failed", scanner.seekTo(getSomeKey(50)) == 0);
236     // read the key and see if it matches
237     ByteBuffer readKey = scanner.getKey();
238     assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50),
239       Bytes.toBytes(readKey)));
240 
241     scanner.seekTo(new byte[0]);
242     ByteBuffer val1 = scanner.getValue();
243     scanner.seekTo(new byte[0]);
244     ByteBuffer val2 = scanner.getValue();
245     assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2)));
246 
247     reader.close();
248     fin.close();
249     fs.delete(ncTFile, true);
250   }
251 
252   public void testTFileFeatures() throws IOException {
253     basicWithSomeCodec("none");
254     basicWithSomeCodec("gz");
255   }
256 
257   private void writeNumMetablocks(Writer writer, int n) {
258     for (int i = 0; i < n; i++) {
259       writer.appendMetaBlock("HFileMeta" + i, new Writable() {
260         private int val;
261         public Writable setVal(int val) { this.val = val; return this; }
262         
263         @Override
264         public void write(DataOutput out) throws IOException {
265           out.write(("something to test" + val).getBytes());
266         }
267         
268         @Override
269         public void readFields(DataInput in) throws IOException { }
270       }.setVal(i));
271     }
272   }
273 
274   private void someTestingWithMetaBlock(Writer writer) {
275     writeNumMetablocks(writer, 10);
276   }
277 
278   private void readNumMetablocks(Reader reader, int n) throws IOException {
279     for (int i = 0; i < n; i++) {
280       ByteBuffer actual = reader.getMetaBlock("HFileMeta" + i, false);
281       ByteBuffer expected = 
282         ByteBuffer.wrap(("something to test" + i).getBytes());
283       assertTrue("failed to match metadata", actual.compareTo(expected) == 0);
284     }
285   }
286 
287   private void someReadingWithMetaBlock(Reader reader) throws IOException {
288     readNumMetablocks(reader, 10);
289   }
290 
291   private void metablocks(final String compress) throws Exception {
292     if (cacheConf == null) cacheConf = new CacheConfig(conf);
293     Path mFile = new Path(ROOT_DIR, "meta.hfile");
294     FSDataOutputStream fout = createFSOutput(mFile);
295     Writer writer = HFile.getWriterFactory(conf, cacheConf)
296         .withOutputStream(fout)
297         .withBlockSize(minBlockSize)
298         .withCompression(compress)
299         .create();
300     someTestingWithMetaBlock(writer);
301     writer.close();
302     fout.close();
303     FSDataInputStream fin = fs.open(mFile);
304     Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile),
305         this.fs.getFileStatus(mFile).getLen(), cacheConf);
306     reader.loadFileInfo();
307     // No data -- this should return false.
308     assertFalse(reader.getScanner(false, false).seekTo());
309     someReadingWithMetaBlock(reader);
310     fs.delete(mFile, true);
311     reader.close();
312     fin.close();
313   }
314 
315   // test meta blocks for tfiles
316   public void testMetaBlocks() throws Exception {
317     metablocks("none");
318     metablocks("gz");
319   }
320 
321   public void testNullMetaBlocks() throws Exception {
322     if (cacheConf == null) cacheConf = new CacheConfig(conf);
323     for (Compression.Algorithm compressAlgo : 
324         HBaseTestingUtility.COMPRESSION_ALGORITHMS) {
325       Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile");
326       FSDataOutputStream fout = createFSOutput(mFile);
327       Writer writer = HFile.getWriterFactory(conf, cacheConf)
328           .withOutputStream(fout)
329           .withBlockSize(minBlockSize)
330           .withCompression(compressAlgo)
331           .create();
332       writer.append("foo".getBytes(), "value".getBytes());
333       writer.close();
334       fout.close();
335       Reader reader = HFile.createReader(fs, mFile, cacheConf);
336       reader.loadFileInfo();
337       assertNull(reader.getMetaBlock("non-existant", false));
338     }
339   }
340 
341   /**
342    * Make sure the ordinals for our compression algorithms do not change on us.
343    */
344   public void testCompressionOrdinance() {
345     assertTrue(Compression.Algorithm.LZO.ordinal() == 0);
346     assertTrue(Compression.Algorithm.GZ.ordinal() == 1);
347     assertTrue(Compression.Algorithm.NONE.ordinal() == 2);
348     assertTrue(Compression.Algorithm.SNAPPY.ordinal() == 3);
349     assertTrue(Compression.Algorithm.LZ4.ordinal() == 4);
350   }
351 
352 }
353