1   /**
2    * Copyright 2009 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.io.hfile;
21  
22  import java.io.DataInput;
23  import java.io.DataOutput;
24  import java.io.IOException;
25  import java.nio.ByteBuffer;
26  import java.util.Arrays;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.fs.FSDataInputStream;
31  import org.apache.hadoop.fs.FSDataOutputStream;
32  import org.apache.hadoop.fs.Path;
33  import org.apache.hadoop.hbase.HBaseTestCase;
34  import org.apache.hadoop.hbase.HBaseTestingUtility;
35  import org.apache.hadoop.hbase.KeyValue.KeyComparator;
36  import org.apache.hadoop.hbase.io.hfile.HFile.BlockIndex;
37  import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
38  import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
39  import org.apache.hadoop.hbase.util.Bytes;
40  import org.apache.hadoop.hbase.util.ClassSize;
41  import org.apache.hadoop.io.Writable;
42  
43  /**
44   * test hfile features.
45   * <p>
46   * Copied from
47   * <a href="https://issues.apache.org/jira/browse/HADOOP-3315">hadoop-3315 tfile</a>.
48   * Remove after tfile is committed and use the tfile version of this class
49   * instead.</p>
50   */
51  public class TestHFile extends HBaseTestCase {
52    static final Log LOG = LogFactory.getLog(TestHFile.class);
53  
54    private static String ROOT_DIR =
55      HBaseTestingUtility.getTestDir("TestHFile").toString();
56    private final int minBlockSize = 512;
57    private static String localFormatter = "%010d";
58  
59    /**
60     * Test empty HFile.
61     * Test all features work reasonably when hfile is empty of entries.
62     * @throws IOException
63     */
64    public void testEmptyHFile() throws IOException {
65      Path f = new Path(ROOT_DIR, getName());
66      Writer w = new Writer(this.fs, f);
67      w.close();
68      Reader r = new Reader(fs, f, null, false);
69      r.loadFileInfo();
70      assertNull(r.getFirstKey());
71      assertNull(r.getLastKey());
72    }
73  
74    // write some records into the tfile
75    // write them twice
76    private int writeSomeRecords(Writer writer, int start, int n)
77        throws IOException {
78      String value = "value";
79      for (int i = start; i < (start + n); i++) {
80        String key = String.format(localFormatter, Integer.valueOf(i));
81        writer.append(Bytes.toBytes(key), Bytes.toBytes(value + key));
82      }
83      return (start + n);
84    }
85  
86    private void readAllRecords(HFileScanner scanner) throws IOException {
87      readAndCheckbytes(scanner, 0, 100);
88    }
89  
90    // read the records and check
91    private int readAndCheckbytes(HFileScanner scanner, int start, int n)
92        throws IOException {
93      String value = "value";
94      int i = start;
95      for (; i < (start + n); i++) {
96        ByteBuffer key = scanner.getKey();
97        ByteBuffer val = scanner.getValue();
98        String keyStr = String.format(localFormatter, Integer.valueOf(i));
99        String valStr = value + keyStr;
100       byte [] keyBytes = Bytes.toBytes(key);
101       assertTrue("bytes for keys do not match " + keyStr + " " +
102         Bytes.toString(Bytes.toBytes(key)),
103           Arrays.equals(Bytes.toBytes(keyStr), keyBytes));
104       byte [] valBytes = Bytes.toBytes(val);
105       assertTrue("bytes for vals do not match " + valStr + " " +
106         Bytes.toString(valBytes),
107         Arrays.equals(Bytes.toBytes(valStr), valBytes));
108       if (!scanner.next()) {
109         break;
110       }
111     }
112     assertEquals(i, start + n - 1);
113     return (start + n);
114   }
115 
116   private byte[] getSomeKey(int rowId) {
117     return String.format(localFormatter, Integer.valueOf(rowId)).getBytes();
118   }
119 
120   private void writeRecords(Writer writer) throws IOException {
121     writeSomeRecords(writer, 0, 100);
122     writer.close();
123   }
124 
125   private FSDataOutputStream createFSOutput(Path name) throws IOException {
126     if (fs.exists(name)) fs.delete(name, true);
127     FSDataOutputStream fout = fs.create(name);
128     return fout;
129   }
130 
131   /**
132    * test none codecs
133    */
134   void basicWithSomeCodec(String codec) throws IOException {
135     Path ncTFile = new Path(ROOT_DIR, "basic.hfile");
136     FSDataOutputStream fout = createFSOutput(ncTFile);
137     Writer writer = new Writer(fout, minBlockSize,
138       Compression.getCompressionAlgorithmByName(codec), null);
139     LOG.info(writer);
140     writeRecords(writer);
141     fout.close();
142     FSDataInputStream fin = fs.open(ncTFile);
143     Reader reader = new Reader(fs.open(ncTFile),
144       fs.getFileStatus(ncTFile).getLen(), null, false);
145     // Load up the index.
146     reader.loadFileInfo();
147     // Get a scanner that caches and that does not use pread.
148     HFileScanner scanner = reader.getScanner(true, false);
149     // Align scanner at start of the file.
150     scanner.seekTo();
151     readAllRecords(scanner);
152     scanner.seekTo(getSomeKey(50));
153     assertTrue("location lookup failed", scanner.seekTo(getSomeKey(50)) == 0);
154     // read the key and see if it matches
155     ByteBuffer readKey = scanner.getKey();
156     assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50),
157       Bytes.toBytes(readKey)));
158 
159     scanner.seekTo(new byte[0]);
160     ByteBuffer val1 = scanner.getValue();
161     scanner.seekTo(new byte[0]);
162     ByteBuffer val2 = scanner.getValue();
163     assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2)));
164 
165     reader.close();
166     fin.close();
167     fs.delete(ncTFile, true);
168   }
169 
170   public void testTFileFeatures() throws IOException {
171     basicWithSomeCodec("none");
172     basicWithSomeCodec("gz");
173   }
174 
175   private void writeNumMetablocks(Writer writer, int n) {
176     for (int i = 0; i < n; i++) {
177       writer.appendMetaBlock("HFileMeta" + i, new Writable() {
178         private int val;
179         public Writable setVal(int val) { this.val = val; return this; }
180         
181         @Override
182         public void write(DataOutput out) throws IOException {
183           out.write(("something to test" + val).getBytes());
184         }
185         
186         @Override
187         public void readFields(DataInput in) throws IOException { }
188       }.setVal(i));
189     }
190   }
191 
192   private void someTestingWithMetaBlock(Writer writer) {
193     writeNumMetablocks(writer, 10);
194   }
195 
196   private void readNumMetablocks(Reader reader, int n) throws IOException {
197     for (int i = 0; i < n; i++) {
198       ByteBuffer actual = reader.getMetaBlock("HFileMeta" + i, false);
199       ByteBuffer expected = 
200         ByteBuffer.wrap(("something to test" + i).getBytes());
201       assertTrue("failed to match metadata", actual.compareTo(expected) == 0);
202     }
203   }
204 
205   private void someReadingWithMetaBlock(Reader reader) throws IOException {
206     readNumMetablocks(reader, 10);
207   }
208 
209   private void metablocks(final String compress) throws Exception {
210     Path mFile = new Path(ROOT_DIR, "meta.hfile");
211     FSDataOutputStream fout = createFSOutput(mFile);
212     Writer writer = new Writer(fout, minBlockSize,
213       Compression.getCompressionAlgorithmByName(compress), null);
214     someTestingWithMetaBlock(writer);
215     writer.close();
216     fout.close();
217     FSDataInputStream fin = fs.open(mFile);
218     Reader reader = new Reader(fs.open(mFile), this.fs.getFileStatus(mFile)
219         .getLen(), null, false);
220     reader.loadFileInfo();
221     // No data -- this should return false.
222     assertFalse(reader.getScanner(false, false).seekTo());
223     someReadingWithMetaBlock(reader);
224     fs.delete(mFile, true);
225     reader.close();
226     fin.close();
227   }
228 
229   // test meta blocks for tfiles
230   public void testMetaBlocks() throws Exception {
231     metablocks("none");
232     metablocks("gz");
233   }
234 
235   public void testNullMetaBlocks() throws Exception {
236     Path mFile = new Path(ROOT_DIR, "nometa.hfile");
237     FSDataOutputStream fout = createFSOutput(mFile);
238     Writer writer = new Writer(fout, minBlockSize,
239         Compression.Algorithm.NONE, null);
240     writer.append("foo".getBytes(), "value".getBytes());
241     writer.close();
242     fout.close();
243     Reader reader = new Reader(fs, mFile, null, false);
244     reader.loadFileInfo();
245     assertNull(reader.getMetaBlock("non-existant", false));
246   }
247 
248   /**
249    * Make sure the orginals for our compression libs doesn't change on us.
250    */
251   public void testCompressionOrdinance() {
252     //assertTrue(Compression.Algorithm.LZO.ordinal() == 0);
253     assertTrue(Compression.Algorithm.GZ.ordinal() == 1);
254     assertTrue(Compression.Algorithm.NONE.ordinal() == 2);
255   }
256 
257 
258   public void testComparator() throws IOException {
259     Path mFile = new Path(ROOT_DIR, "meta.tfile");
260     FSDataOutputStream fout = createFSOutput(mFile);
261     Writer writer = new Writer(fout, minBlockSize, (Compression.Algorithm) null,
262       new KeyComparator() {
263         @Override
264         public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2,
265             int l2) {
266           return -Bytes.compareTo(b1, s1, l1, b2, s2, l2);
267 
268         }
269         @Override
270         public int compare(byte[] o1, byte[] o2) {
271           return compare(o1, 0, o1.length, o2, 0, o2.length);
272         }
273       });
274     writer.append("3".getBytes(), "0".getBytes());
275     writer.append("2".getBytes(), "0".getBytes());
276     writer.append("1".getBytes(), "0".getBytes());
277     writer.close();
278   }
279 
280   /**
281    * Checks if the HeapSize calculator is within reason
282    */
283   @SuppressWarnings("unchecked")
284   public void testHeapSizeForBlockIndex() throws IOException{
285     Class cl = null;
286     long expected = 0L;
287     long actual = 0L;
288 
289     cl = BlockIndex.class;
290     expected = ClassSize.estimateBase(cl, false);
291     BlockIndex bi = new BlockIndex(Bytes.BYTES_RAWCOMPARATOR);
292     actual = bi.heapSize();
293     //Since the arrays in BlockIndex(byte [][] blockKeys, long [] blockOffsets,
294     //int [] blockDataSizes) are all null they are not going to show up in the
295     //HeapSize calculation, so need to remove those array costs from ecpected.
296     expected -= ClassSize.align(3 * ClassSize.ARRAY);
297     if(expected != actual) {
298       ClassSize.estimateBase(cl, true);
299       assertEquals(expected, actual);
300     }
301   }
302 
303 }