1   /**
2    * Copyright 2009 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.io.hfile;
21  
22  import java.io.DataInput;
23  import java.io.DataOutput;
24  import java.io.IOException;
25  import java.nio.ByteBuffer;
26  import java.util.Arrays;
27  import java.util.Map;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.fs.FSDataInputStream;
32  import org.apache.hadoop.fs.FSDataOutputStream;
33  import org.apache.hadoop.fs.FileStatus;
34  import org.apache.hadoop.fs.FileSystem;
35  import org.apache.hadoop.fs.Path;
36  import org.apache.hadoop.hbase.HBaseTestCase;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.KeyValue.KeyComparator;
39  import org.apache.hadoop.hbase.SmallTests;
40  import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
41  import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.io.Writable;
44  import org.junit.experimental.categories.Category;
45  
46  /**
47   * test hfile features.
48   * <p>
49   * Copied from
50   * <a href="https://issues.apache.org/jira/browse/HADOOP-3315">hadoop-3315 tfile</a>.
51   * Remove after tfile is committed and use the tfile version of this class
52   * instead.</p>
53   */
54  @Category(SmallTests.class)
55  public class TestHFile extends HBaseTestCase {
56    static final Log LOG = LogFactory.getLog(TestHFile.class);
57  
58    private String ROOT_DIR;
59    private final int minBlockSize = 512;
60    private static String localFormatter = "%010d";
61    private static CacheConfig cacheConf = null;
62  
63    @Override
64    public void setUp() throws Exception {
65      ROOT_DIR = this.getUnitTestdir("TestHFile").toString();
66      super.setUp();
67    }
68  
69    @Override
70    public void tearDown() throws Exception {
71      super.tearDown();
72    }
73  
74  
75    /**
76     * Test empty HFile.
77     * Test all features work reasonably when hfile is empty of entries.
78     * @throws IOException
79     */
80    public void testEmptyHFile() throws IOException {
81      if (cacheConf == null) cacheConf = new CacheConfig(conf);
82      Path f = new Path(ROOT_DIR, getName());
83      Writer w =
84          HFile.getWriterFactory(conf, cacheConf).withPath(fs, f).create();
85      w.close();
86      Reader r = HFile.createReader(fs, f, cacheConf);
87      r.loadFileInfo();
88      assertNull(r.getFirstKey());
89      assertNull(r.getLastKey());
90    }
91  
92    /**
93     * Create 0-length hfile and show that it fails
94     */
95    public void testCorrupt0LengthHFile() throws IOException {
96      if (cacheConf == null) cacheConf = new CacheConfig(conf);
97      Path f = new Path(ROOT_DIR, getName());
98      FSDataOutputStream fsos = fs.create(f);
99      fsos.close();
100 
101     try {
102       Reader r = HFile.createReader(fs, f, cacheConf);
103     } catch (CorruptHFileException che) {
104       // Expected failure
105       return;
106     }
107     fail("Should have thrown exception");
108   }
109 
110   public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException {
111     FileStatus fst = fs.getFileStatus(src);
112     long len = fst.getLen();
113     len = len / 2 ;
114 
115     // create a truncated hfile
116     FSDataOutputStream fdos = fs.create(dst);
117     byte[] buf = new byte[(int)len];
118     FSDataInputStream fdis = fs.open(src);
119     fdis.read(buf);
120     fdos.write(buf);
121     fdis.close();
122     fdos.close();
123   }
124 
125   /**
126    * Create a truncated hfile and verify that exception thrown.
127    */
128   public void testCorruptTruncatedHFile() throws IOException {
129     if (cacheConf == null) cacheConf = new CacheConfig(conf);
130     Path f = new Path(ROOT_DIR, getName());
131     Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f).create();
132     writeSomeRecords(w, 0, 100);
133     w.close();
134 
135     Path trunc = new Path(f.getParent(), "trucated");
136     truncateFile(fs, w.getPath(), trunc);
137 
138     try {
139       Reader r = HFile.createReader(fs, trunc, cacheConf);
140     } catch (CorruptHFileException che) {
141       // Expected failure
142       return;
143     }
144     fail("Should have thrown exception");
145   }
146 
147   // write some records into the tfile
148   // write them twice
149   private int writeSomeRecords(Writer writer, int start, int n)
150       throws IOException {
151     String value = "value";
152     for (int i = start; i < (start + n); i++) {
153       String key = String.format(localFormatter, Integer.valueOf(i));
154       writer.append(Bytes.toBytes(key), Bytes.toBytes(value + key));
155     }
156     return (start + n);
157   }
158 
159   private void readAllRecords(HFileScanner scanner) throws IOException {
160     readAndCheckbytes(scanner, 0, 100);
161   }
162 
163   // read the records and check
164   private int readAndCheckbytes(HFileScanner scanner, int start, int n)
165       throws IOException {
166     String value = "value";
167     int i = start;
168     for (; i < (start + n); i++) {
169       ByteBuffer key = scanner.getKey();
170       ByteBuffer val = scanner.getValue();
171       String keyStr = String.format(localFormatter, Integer.valueOf(i));
172       String valStr = value + keyStr;
173       byte [] keyBytes = Bytes.toBytes(key);
174       assertTrue("bytes for keys do not match " + keyStr + " " +
175         Bytes.toString(Bytes.toBytes(key)),
176           Arrays.equals(Bytes.toBytes(keyStr), keyBytes));
177       byte [] valBytes = Bytes.toBytes(val);
178       assertTrue("bytes for vals do not match " + valStr + " " +
179         Bytes.toString(valBytes),
180         Arrays.equals(Bytes.toBytes(valStr), valBytes));
181       if (!scanner.next()) {
182         break;
183       }
184     }
185     assertEquals(i, start + n - 1);
186     return (start + n);
187   }
188 
189   private byte[] getSomeKey(int rowId) {
190     return String.format(localFormatter, Integer.valueOf(rowId)).getBytes();
191   }
192 
193   private void writeRecords(Writer writer) throws IOException {
194     writeSomeRecords(writer, 0, 100);
195     writer.close();
196   }
197 
198   private FSDataOutputStream createFSOutput(Path name) throws IOException {
199     //if (fs.exists(name)) fs.delete(name, true);
200     FSDataOutputStream fout = fs.create(name);
201     return fout;
202   }
203 
204   /**
205    * test none codecs
206    */
207   void basicWithSomeCodec(String codec) throws IOException {
208     if (cacheConf == null) cacheConf = new CacheConfig(conf);
209     Path ncTFile = new Path(ROOT_DIR, "basic.hfile." + codec.toString());
210     FSDataOutputStream fout = createFSOutput(ncTFile);
211     Writer writer = HFile.getWriterFactory(conf, cacheConf)
212         .withOutputStream(fout)
213         .withBlockSize(minBlockSize)
214         .withCompression(codec)
215         .create();
216     LOG.info(writer);
217     writeRecords(writer);
218     fout.close();
219     FSDataInputStream fin = fs.open(ncTFile);
220     Reader reader = HFile.createReaderFromStream(ncTFile, fs.open(ncTFile),
221       fs.getFileStatus(ncTFile).getLen(), cacheConf);
222     System.out.println(cacheConf.toString());
223     // Load up the index.
224     reader.loadFileInfo();
225     // Get a scanner that caches and that does not use pread.
226     HFileScanner scanner = reader.getScanner(true, false);
227     // Align scanner at start of the file.
228     scanner.seekTo();
229     readAllRecords(scanner);
230     scanner.seekTo(getSomeKey(50));
231     assertTrue("location lookup failed", scanner.seekTo(getSomeKey(50)) == 0);
232     // read the key and see if it matches
233     ByteBuffer readKey = scanner.getKey();
234     assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50),
235       Bytes.toBytes(readKey)));
236 
237     scanner.seekTo(new byte[0]);
238     ByteBuffer val1 = scanner.getValue();
239     scanner.seekTo(new byte[0]);
240     ByteBuffer val2 = scanner.getValue();
241     assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2)));
242 
243     reader.close();
244     fin.close();
245     fs.delete(ncTFile, true);
246   }
247 
248   public void testTFileFeatures() throws IOException {
249     basicWithSomeCodec("none");
250     basicWithSomeCodec("gz");
251   }
252 
253   private void writeNumMetablocks(Writer writer, int n) {
254     for (int i = 0; i < n; i++) {
255       writer.appendMetaBlock("HFileMeta" + i, new Writable() {
256         private int val;
257         public Writable setVal(int val) { this.val = val; return this; }
258         
259         @Override
260         public void write(DataOutput out) throws IOException {
261           out.write(("something to test" + val).getBytes());
262         }
263         
264         @Override
265         public void readFields(DataInput in) throws IOException { }
266       }.setVal(i));
267     }
268   }
269 
270   private void someTestingWithMetaBlock(Writer writer) {
271     writeNumMetablocks(writer, 10);
272   }
273 
274   private void readNumMetablocks(Reader reader, int n) throws IOException {
275     for (int i = 0; i < n; i++) {
276       ByteBuffer actual = reader.getMetaBlock("HFileMeta" + i, false);
277       ByteBuffer expected = 
278         ByteBuffer.wrap(("something to test" + i).getBytes());
279       assertTrue("failed to match metadata", actual.compareTo(expected) == 0);
280     }
281   }
282 
283   private void someReadingWithMetaBlock(Reader reader) throws IOException {
284     readNumMetablocks(reader, 10);
285   }
286 
287   private void metablocks(final String compress) throws Exception {
288     if (cacheConf == null) cacheConf = new CacheConfig(conf);
289     Path mFile = new Path(ROOT_DIR, "meta.hfile");
290     FSDataOutputStream fout = createFSOutput(mFile);
291     Writer writer = HFile.getWriterFactory(conf, cacheConf)
292         .withOutputStream(fout)
293         .withBlockSize(minBlockSize)
294         .withCompression(compress)
295         .create();
296     someTestingWithMetaBlock(writer);
297     writer.close();
298     fout.close();
299     FSDataInputStream fin = fs.open(mFile);
300     Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile),
301         this.fs.getFileStatus(mFile).getLen(), cacheConf);
302     reader.loadFileInfo();
303     // No data -- this should return false.
304     assertFalse(reader.getScanner(false, false).seekTo());
305     someReadingWithMetaBlock(reader);
306     fs.delete(mFile, true);
307     reader.close();
308     fin.close();
309   }
310 
311   // test meta blocks for tfiles
312   public void testMetaBlocks() throws Exception {
313     metablocks("none");
314     metablocks("gz");
315   }
316 
317   public void testNullMetaBlocks() throws Exception {
318     if (cacheConf == null) cacheConf = new CacheConfig(conf);
319     for (Compression.Algorithm compressAlgo : 
320         HBaseTestingUtility.COMPRESSION_ALGORITHMS) {
321       Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile");
322       FSDataOutputStream fout = createFSOutput(mFile);
323       Writer writer = HFile.getWriterFactory(conf, cacheConf)
324           .withOutputStream(fout)
325           .withBlockSize(minBlockSize)
326           .withCompression(compressAlgo)
327           .create();
328       writer.append("foo".getBytes(), "value".getBytes());
329       writer.close();
330       fout.close();
331       Reader reader = HFile.createReader(fs, mFile, cacheConf);
332       reader.loadFileInfo();
333       assertNull(reader.getMetaBlock("non-existant", false));
334     }
335   }
336 
337   /**
338    * Make sure the ordinals for our compression algorithms do not change on us.
339    */
340   public void testCompressionOrdinance() {
341     assertTrue(Compression.Algorithm.LZO.ordinal() == 0);
342     assertTrue(Compression.Algorithm.GZ.ordinal() == 1);
343     assertTrue(Compression.Algorithm.NONE.ordinal() == 2);
344     assertTrue(Compression.Algorithm.SNAPPY.ordinal() == 3);
345     assertTrue(Compression.Algorithm.LZ4.ordinal() == 4);
346   }
347 
348   public void testComparator() throws IOException {
349     if (cacheConf == null) cacheConf = new CacheConfig(conf);
350     Path mFile = new Path(ROOT_DIR, "meta.tfile");
351     FSDataOutputStream fout = createFSOutput(mFile);
352     KeyComparator comparator = new KeyComparator() {
353       @Override
354       public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2,
355           int l2) {
356         return -Bytes.compareTo(b1, s1, l1, b2, s2, l2);
357       }
358       @Override
359       public int compare(byte[] o1, byte[] o2) {
360         return compare(o1, 0, o1.length, o2, 0, o2.length);
361       }
362     };
363     Writer writer = HFile.getWriterFactory(conf, cacheConf)
364         .withOutputStream(fout)
365         .withBlockSize(minBlockSize)
366         .withComparator(comparator)
367         .create();
368     writer.append("3".getBytes(), "0".getBytes());
369     writer.append("2".getBytes(), "0".getBytes());
370     writer.append("1".getBytes(), "0".getBytes());
371     writer.close();
372   }
373 
374 
375   @org.junit.Rule
376   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
377     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
378 }
379