View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.io.ByteArrayInputStream;
22  import java.io.ByteArrayOutputStream;
23  import java.io.DataInput;
24  import java.io.DataInputStream;
25  import java.io.DataOutputStream;
26  import java.io.IOException;
27  import java.nio.ByteBuffer;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.classification.InterfaceAudience;
32  import org.apache.hadoop.fs.FSDataInputStream;
33  import org.apache.hadoop.hbase.KeyValue;
34  import org.apache.hadoop.hbase.io.compress.Compression;
35  import org.apache.hadoop.hbase.protobuf.generated.HFileProtos;
36  import org.apache.hadoop.hbase.util.Bytes;
37  import org.apache.hadoop.io.RawComparator;
38  
39  import com.google.common.io.NullOutputStream;
40  
41  /**
42   * The {@link HFile} has a fixed trailer which contains offsets to other
43   * variable parts of the file. Also includes basic metadata on this file. The
44   * trailer size is fixed within a given {@link HFile} format version only, but
45   * we always store the version number as the last four-byte integer of the file.
46   * The version number itself is split into two portions, a major 
47   * version and a minor version. 
48   * The last three bytes of a file is the major
49   * version and a single preceding byte is the minor number. The major version
50   * determines which readers/writers to use to read/write a hfile while a minor
51   * version determines smaller changes in hfile format that do not need a new
52   * reader/writer type.
53   */
54  @InterfaceAudience.Private
55  public class FixedFileTrailer {
56  
57    private static final Log LOG = LogFactory.getLog(FixedFileTrailer.class);
58  
59    /** HFile minor version that introduced pbuf filetrailer */
60    private static final int PBUF_TRAILER_MINOR_VERSION = 2;
61  
62    /**
63     * We store the comparator class name as a fixed-length field in the trailer.
64     */
65    private static final int MAX_COMPARATOR_NAME_LENGTH = 128;
66  
67    /**
68     * Offset to the fileinfo data, a small block of vitals. Necessary in v1 but
69     * only potentially useful for pretty-printing in v2.
70     */
71    private long fileInfoOffset;
72  
73    /**
74     * In version 1, the offset to the data block index. Starting from version 2,
75     * the meaning of this field is the offset to the section of the file that
76     * should be loaded at the time the file is being opened, and as of the time
77     * of writing, this happens to be the offset of the file info section.
78     */
79    private long loadOnOpenDataOffset;
80  
81    /** The number of entries in the root data index. */
82    private int dataIndexCount;
83  
84    /** Total uncompressed size of all blocks of the data index */
85    private long uncompressedDataIndexSize;
86  
87    /** The number of entries in the meta index */
88    private int metaIndexCount;
89  
90    /** The total uncompressed size of keys/values stored in the file. */
91    private long totalUncompressedBytes;
92  
93    /**
94     * The number of key/value pairs in the file. This field was int in version 1,
95     * but is now long.
96     */
97    private long entryCount;
98  
99    /** The compression codec used for all blocks. */
100   private Compression.Algorithm compressionCodec = Compression.Algorithm.NONE;
101 
102   /**
103    * The number of levels in the potentially multi-level data index. Used from
104    * version 2 onwards.
105    */
106   private int numDataIndexLevels;
107 
108   /** The offset of the first data block. */
109   private long firstDataBlockOffset;
110 
111   /**
112    * It is guaranteed that no key/value data blocks start after this offset in
113    * the file.
114    */
115   private long lastDataBlockOffset;
116 
117   /** Raw key comparator class name in version 2 */
118   private String comparatorClassName = KeyValue.KEY_COMPARATOR.getClass().getName();
119 
120   /** The {@link HFile} format major version. */
121   private final int majorVersion;
122 
123   /** The {@link HFile} format minor version. */
124   private final int minorVersion;
125 
126   FixedFileTrailer(int majorVersion, int minorVersion) {
127     this.majorVersion = majorVersion;
128     this.minorVersion = minorVersion;
129     HFile.checkFormatVersion(majorVersion);
130   }
131 
132   private static int[] computeTrailerSizeByVersion() {
133     int versionToSize[] = new int[HFile.MAX_FORMAT_VERSION + 1];
134     for (int version = HFile.MIN_FORMAT_VERSION;
135          version <= HFile.MAX_FORMAT_VERSION;
136          ++version) {
137       FixedFileTrailer fft = new FixedFileTrailer(version, HFileBlock.MINOR_VERSION_NO_CHECKSUM);
138       DataOutputStream dos = new DataOutputStream(new NullOutputStream());
139       try {
140         fft.serialize(dos);
141       } catch (IOException ex) {
142         // The above has no reason to fail.
143         throw new RuntimeException(ex);
144       }
145       versionToSize[version] = dos.size();
146     }
147     return versionToSize;
148   }
149 
150   private static int getMaxTrailerSize() {
151     int maxSize = 0;
152     for (int version = HFile.MIN_FORMAT_VERSION;
153          version <= HFile.MAX_FORMAT_VERSION;
154          ++version)
155       maxSize = Math.max(getTrailerSize(version), maxSize);
156     return maxSize;
157   }
158 
159   private static final int TRAILER_SIZE[] = computeTrailerSizeByVersion();
160   private static final int MAX_TRAILER_SIZE = getMaxTrailerSize();
161 
162   private static final int NOT_PB_SIZE = BlockType.MAGIC_LENGTH + Bytes.SIZEOF_INT;
163 
164   static int getTrailerSize(int version) {
165     return TRAILER_SIZE[version];
166   }
167 
168   public int getTrailerSize() {
169     return getTrailerSize(majorVersion);
170   }
171 
172   /**
173    * Write the trailer to a data stream. We support writing version 1 for
174    * testing and for determining version 1 trailer size. It is also easy to see
175    * what fields changed in version 2.
176    *
177    * @param outputStream
178    * @throws IOException
179    */
180   void serialize(DataOutputStream outputStream) throws IOException {
181     HFile.checkFormatVersion(majorVersion);
182 
183     ByteArrayOutputStream baos = new ByteArrayOutputStream();
184     DataOutputStream baosDos = new DataOutputStream(baos);
185 
186     BlockType.TRAILER.write(baosDos);
187     if (majorVersion > 2 || (majorVersion == 2 && minorVersion >= PBUF_TRAILER_MINOR_VERSION)) {
188       serializeAsPB(baosDos);
189     } else {
190       serializeAsWritable(baosDos);
191     }
192 
193     // The last 4 bytes of the file encode the major and minor version universally
194     baosDos.writeInt(materializeVersion(majorVersion, minorVersion));
195 
196     outputStream.write(baos.toByteArray());
197   }
198 
199   /**
200    * Write trailer data as protobuf
201    * @param outputStream
202    * @throws IOException
203    */
204   void serializeAsPB(DataOutputStream output) throws IOException {
205     ByteArrayOutputStream baos = new ByteArrayOutputStream();
206     HFileProtos.FileTrailerProto.newBuilder()
207       .setFileInfoOffset(fileInfoOffset)
208       .setLoadOnOpenDataOffset(loadOnOpenDataOffset)
209       .setUncompressedDataIndexSize(uncompressedDataIndexSize)
210       .setTotalUncompressedBytes(totalUncompressedBytes)
211       .setDataIndexCount(dataIndexCount)
212       .setMetaIndexCount(metaIndexCount)
213       .setEntryCount(entryCount)
214       .setNumDataIndexLevels(numDataIndexLevels)
215       .setFirstDataBlockOffset(firstDataBlockOffset)
216       .setLastDataBlockOffset(lastDataBlockOffset)
217       .setComparatorClassName(comparatorClassName)
218       .setCompressionCodec(compressionCodec.ordinal())
219       .build().writeDelimitedTo(baos);
220     output.write(baos.toByteArray());
221     // Pad to make up the difference between variable PB encoding length and the
222     // length when encoded as writable under earlier V2 formats. Failure to pad
223     // properly or if the PB encoding is too big would mean the trailer wont be read
224     // in properly by HFile.
225     int padding = getTrailerSize() - NOT_PB_SIZE - baos.size();
226     if (padding < 0) {
227       throw new IOException("Pbuf encoding size exceeded fixed trailer size limit");
228     }
229     for (int i = 0; i < padding; i++) {
230       output.write(0);
231     }
232   }
233 
234   /**
235    * Write trailer data as writable
236    * @param outputStream
237    * @throws IOException
238    */
239   void serializeAsWritable(DataOutputStream output) throws IOException {
240     output.writeLong(fileInfoOffset);
241     output.writeLong(loadOnOpenDataOffset);
242     output.writeInt(dataIndexCount);
243 
244     if (majorVersion == 1) {
245       // This used to be metaIndexOffset, but it was not used in version 1.
246       output.writeLong(0);
247     } else {
248       output.writeLong(uncompressedDataIndexSize);
249     }
250 
251     output.writeInt(metaIndexCount);
252     output.writeLong(totalUncompressedBytes);
253     if (majorVersion == 1) {
254       output.writeInt((int) Math.min(Integer.MAX_VALUE, entryCount));
255     } else {
256       // This field is long from version 2 onwards.
257       output.writeLong(entryCount);
258     }
259     output.writeInt(compressionCodec.ordinal());
260 
261     if (majorVersion > 1) {
262       output.writeInt(numDataIndexLevels);
263       output.writeLong(firstDataBlockOffset);
264       output.writeLong(lastDataBlockOffset);
265       Bytes.writeStringFixedSize(output, comparatorClassName, MAX_COMPARATOR_NAME_LENGTH);
266     }
267   }
268 
269   /**
270    * Deserialize the fixed file trailer from the given stream. The version needs
271    * to already be specified. Make sure this is consistent with
272    * {@link #serialize(DataOutputStream)}.
273    *
274    * @param inputStream
275    * @throws IOException
276    */
277   void deserialize(DataInputStream inputStream) throws IOException {
278     HFile.checkFormatVersion(majorVersion);
279 
280     BlockType.TRAILER.readAndCheck(inputStream);
281 
282     if (majorVersion > 2 || (majorVersion == 2 && minorVersion >= PBUF_TRAILER_MINOR_VERSION)) {
283       deserializeFromPB(inputStream);
284     } else {
285       deserializeFromWritable(inputStream);
286     }
287 
288     // The last 4 bytes of the file encode the major and minor version universally
289     int version = inputStream.readInt();
290     expectMajorVersion(extractMajorVersion(version));
291     expectMinorVersion(extractMinorVersion(version));
292   }
293 
294   /**
295    * Deserialize the file trailer as protobuf
296    * @param inputStream
297    * @throws IOException
298    */
299   void deserializeFromPB(DataInputStream inputStream) throws IOException {
300     // read PB and skip padding
301     int start = inputStream.available();
302     HFileProtos.FileTrailerProto.Builder builder = HFileProtos.FileTrailerProto.newBuilder();
303     builder.mergeDelimitedFrom(inputStream);
304     int size = start - inputStream.available();
305     inputStream.skip(getTrailerSize() - NOT_PB_SIZE - size);
306 
307     // process the PB
308     if (builder.hasFileInfoOffset()) {
309       fileInfoOffset = builder.getFileInfoOffset();
310     }
311     if (builder.hasLoadOnOpenDataOffset()) {
312       loadOnOpenDataOffset = builder.getLoadOnOpenDataOffset();
313     }
314     if (builder.hasUncompressedDataIndexSize()) {
315       uncompressedDataIndexSize = builder.getUncompressedDataIndexSize();
316     }
317     if (builder.hasTotalUncompressedBytes()) {
318       totalUncompressedBytes = builder.getTotalUncompressedBytes();
319     }
320     if (builder.hasDataIndexCount()) {
321       dataIndexCount = builder.getDataIndexCount();
322     }
323     if (builder.hasMetaIndexCount()) {
324       metaIndexCount = builder.getMetaIndexCount();
325     }
326     if (builder.hasEntryCount()) {
327       entryCount = builder.getEntryCount();
328     }
329     if (builder.hasNumDataIndexLevels()) {
330       numDataIndexLevels = builder.getNumDataIndexLevels();
331     }
332     if (builder.hasFirstDataBlockOffset()) {
333       firstDataBlockOffset = builder.getFirstDataBlockOffset();
334     }
335     if (builder.hasLastDataBlockOffset()) {
336       lastDataBlockOffset = builder.getLastDataBlockOffset();
337     }
338     if (builder.hasComparatorClassName()) {
339       setComparatorClass(getComparatorClass(builder.getComparatorClassName()));
340     }
341     if (builder.hasCompressionCodec()) {
342       compressionCodec = Compression.Algorithm.values()[builder.getCompressionCodec()];
343     } else {
344       compressionCodec = Compression.Algorithm.NONE;
345     }
346   }
347 
348   /**
349    * Deserialize the file trailer as writable data
350    * @param input
351    * @throws IOException
352    */
353   void deserializeFromWritable(DataInput input) throws IOException {
354     fileInfoOffset = input.readLong();
355     loadOnOpenDataOffset = input.readLong();
356     dataIndexCount = input.readInt();
357     if (majorVersion == 1) {
358       input.readLong(); // Read and skip metaIndexOffset.
359     } else {
360       uncompressedDataIndexSize = input.readLong();
361     }
362     metaIndexCount = input.readInt();
363 
364     totalUncompressedBytes = input.readLong();
365     entryCount = majorVersion == 1 ? input.readInt() : input.readLong();
366     compressionCodec = Compression.Algorithm.values()[input.readInt()];
367     if (majorVersion > 1) {
368       numDataIndexLevels = input.readInt();
369       firstDataBlockOffset = input.readLong();
370       lastDataBlockOffset = input.readLong();
371       setComparatorClass(getComparatorClass(Bytes.readStringFixedSize(input,
372         MAX_COMPARATOR_NAME_LENGTH)));
373     }
374   }
375   
376   private void append(StringBuilder sb, String s) {
377     if (sb.length() > 0)
378       sb.append(", ");
379     sb.append(s);
380   }
381 
382   @Override
383   public String toString() {
384     StringBuilder sb = new StringBuilder();
385     append(sb, "fileinfoOffset=" + fileInfoOffset);
386     append(sb, "loadOnOpenDataOffset=" + loadOnOpenDataOffset);
387     append(sb, "dataIndexCount=" + dataIndexCount);
388     append(sb, "metaIndexCount=" + metaIndexCount);
389     append(sb, "totalUncomressedBytes=" + totalUncompressedBytes);
390     append(sb, "entryCount=" + entryCount);
391     append(sb, "compressionCodec=" + compressionCodec);
392     if (majorVersion == 2) {
393       append(sb, "uncompressedDataIndexSize=" + uncompressedDataIndexSize);
394       append(sb, "numDataIndexLevels=" + numDataIndexLevels);
395       append(sb, "firstDataBlockOffset=" + firstDataBlockOffset);
396       append(sb, "lastDataBlockOffset=" + lastDataBlockOffset);
397       append(sb, "comparatorClassName=" + comparatorClassName);
398     }
399     append(sb, "majorVersion=" + majorVersion);
400     append(sb, "minorVersion=" + minorVersion);
401 
402     return sb.toString();
403   }
404 
405   /**
406    * Reads a file trailer from the given file.
407    *
408    * @param istream the input stream with the ability to seek. Does not have to
409    *          be buffered, as only one read operation is made.
410    * @param fileSize the file size. Can be obtained using
411    *          {@link org.apache.hadoop.fs.FileSystem#getFileStatus(
412    *          org.apache.hadoop.fs.Path)}.
413    * @return the fixed file trailer read
414    * @throws IOException if failed to read from the underlying stream, or the
415    *           trailer is corrupted, or the version of the trailer is
416    *           unsupported
417    */
418   public static FixedFileTrailer readFromStream(FSDataInputStream istream,
419       long fileSize) throws IOException {
420     int bufferSize = MAX_TRAILER_SIZE;
421     long seekPoint = fileSize - bufferSize;
422     if (seekPoint < 0) {
423       // It is hard to imagine such a small HFile.
424       seekPoint = 0;
425       bufferSize = (int) fileSize;
426     }
427 
428     istream.seek(seekPoint);
429     ByteBuffer buf = ByteBuffer.allocate(bufferSize);
430     istream.readFully(buf.array(), buf.arrayOffset(),
431         buf.arrayOffset() + buf.limit());
432 
433     // Read the version from the last int of the file.
434     buf.position(buf.limit() - Bytes.SIZEOF_INT);
435     int version = buf.getInt();
436 
437     // Extract the major and minor versions.
438     int majorVersion = extractMajorVersion(version);
439     int minorVersion = extractMinorVersion(version);
440 
441     HFile.checkFormatVersion(majorVersion); // throws IAE if invalid
442 
443     int trailerSize = getTrailerSize(majorVersion);
444 
445     FixedFileTrailer fft = new FixedFileTrailer(majorVersion, minorVersion);
446     fft.deserialize(new DataInputStream(new ByteArrayInputStream(buf.array(),
447         buf.arrayOffset() + bufferSize - trailerSize, trailerSize)));
448     return fft;
449   }
450 
451   public void expectMajorVersion(int expected) {
452     if (majorVersion != expected) {
453       throw new IllegalArgumentException("Invalid HFile major version: "
454           + majorVersion 
455           + " (expected: " + expected + ")");
456     }
457   }
458 
459   public void expectMinorVersion(int expected) {
460     if (minorVersion != expected) {
461       throw new IllegalArgumentException("Invalid HFile minor version: "
462           + minorVersion + " (expected: " + expected + ")");
463     }
464   }
465 
466   public void expectAtLeastMajorVersion(int lowerBound) {
467     if (majorVersion < lowerBound) {
468       throw new IllegalArgumentException("Invalid HFile major version: "
469           + majorVersion
470           + " (expected: " + lowerBound + " or higher).");
471     }
472   }
473 
474   public long getFileInfoOffset() {
475     return fileInfoOffset;
476   }
477 
478   public void setFileInfoOffset(long fileInfoOffset) {
479     this.fileInfoOffset = fileInfoOffset;
480   }
481 
482   public long getLoadOnOpenDataOffset() {
483     return loadOnOpenDataOffset;
484   }
485 
486   public void setLoadOnOpenOffset(long loadOnOpenDataOffset) {
487     this.loadOnOpenDataOffset = loadOnOpenDataOffset;
488   }
489 
490   public int getDataIndexCount() {
491     return dataIndexCount;
492   }
493 
494   public void setDataIndexCount(int dataIndexCount) {
495     this.dataIndexCount = dataIndexCount;
496   }
497 
498   public int getMetaIndexCount() {
499     return metaIndexCount;
500   }
501 
502   public void setMetaIndexCount(int metaIndexCount) {
503     this.metaIndexCount = metaIndexCount;
504   }
505 
506   public long getTotalUncompressedBytes() {
507     return totalUncompressedBytes;
508   }
509 
510   public void setTotalUncompressedBytes(long totalUncompressedBytes) {
511     this.totalUncompressedBytes = totalUncompressedBytes;
512   }
513 
514   public long getEntryCount() {
515     return entryCount;
516   }
517 
518   public void setEntryCount(long newEntryCount) {
519     if (majorVersion == 1) {
520       int intEntryCount = (int) Math.min(Integer.MAX_VALUE, newEntryCount);
521       if (intEntryCount != newEntryCount) {
522         LOG.info("Warning: entry count is " + newEntryCount + " but writing "
523             + intEntryCount + " into the version " + majorVersion + " trailer");
524       }
525       entryCount = intEntryCount;
526       return;
527     }
528     entryCount = newEntryCount;
529   }
530 
531   public Compression.Algorithm getCompressionCodec() {
532     return compressionCodec;
533   }
534 
535   public void setCompressionCodec(Compression.Algorithm compressionCodec) {
536     this.compressionCodec = compressionCodec;
537   }
538 
539   public int getNumDataIndexLevels() {
540     expectAtLeastMajorVersion(2);
541     return numDataIndexLevels;
542   }
543 
544   public void setNumDataIndexLevels(int numDataIndexLevels) {
545     expectAtLeastMajorVersion(2);
546     this.numDataIndexLevels = numDataIndexLevels;
547   }
548 
549   public long getLastDataBlockOffset() {
550     expectAtLeastMajorVersion(2);
551     return lastDataBlockOffset;
552   }
553 
554   public void setLastDataBlockOffset(long lastDataBlockOffset) {
555     expectAtLeastMajorVersion(2);
556     this.lastDataBlockOffset = lastDataBlockOffset;
557   }
558 
559   public long getFirstDataBlockOffset() {
560     expectAtLeastMajorVersion(2);
561     return firstDataBlockOffset;
562   }
563 
564   public void setFirstDataBlockOffset(long firstDataBlockOffset) {
565     expectAtLeastMajorVersion(2);
566     this.firstDataBlockOffset = firstDataBlockOffset;
567   }
568 
569   public String getComparatorClassName() {
570     return comparatorClassName;
571   }
572 
573   /**
574    * Returns the major version of this HFile format
575    */
576   public int getMajorVersion() {
577     return majorVersion;
578   }
579 
580   /**
581    * Returns the minor version of this HFile format
582    */
583   int getMinorVersion() {
584     return minorVersion;
585   }
586 
587   @SuppressWarnings("rawtypes")
588   public void setComparatorClass(Class<? extends RawComparator> klass) {
589     // Is the comparator instantiable
590     try {
591       klass.newInstance();
592     } catch (Exception e) {
593       throw new RuntimeException("Comparator class " + klass.getName() +
594         " is not instantiable", e);
595     }
596     comparatorClassName = klass.getName();
597   }
598 
599   @SuppressWarnings("unchecked")
600   private static Class<? extends RawComparator<byte[]>> getComparatorClass(
601       String comparatorClassName) throws IOException {
602     try {
603       return (Class<? extends RawComparator<byte[]>>)
604           Class.forName(comparatorClassName);
605     } catch (ClassNotFoundException ex) {
606       throw new IOException(ex);
607     }
608   }
609 
610   public static RawComparator<byte[]> createComparator(
611       String comparatorClassName) throws IOException {
612     try {
613       return getComparatorClass(comparatorClassName).newInstance();
614     } catch (InstantiationException e) {
615       throw new IOException("Comparator class " + comparatorClassName +
616         " is not instantiable", e);
617     } catch (IllegalAccessException e) {
618       throw new IOException("Comparator class " + comparatorClassName +
619         " is not instantiable", e);
620     }
621   }
622 
623   RawComparator<byte[]> createComparator() throws IOException {
624     expectAtLeastMajorVersion(2);
625     return createComparator(comparatorClassName);
626   }
627 
628   public long getUncompressedDataIndexSize() {
629     if (majorVersion == 1)
630       return 0;
631     return uncompressedDataIndexSize;
632   }
633 
634   public void setUncompressedDataIndexSize(
635       long uncompressedDataIndexSize) {
636     expectAtLeastMajorVersion(2);
637     this.uncompressedDataIndexSize = uncompressedDataIndexSize;
638   }
639 
640   /**
641    * Extracts the major version for a 4-byte serialized version data.
642    * The major version is the 3 least significant bytes
643    */
644   private static int extractMajorVersion(int serializedVersion) {
645     return (serializedVersion & 0x00ffffff);
646   }
647 
648   /**
649    * Extracts the minor version for a 4-byte serialized version data.
650    * The major version are the 3 the most significant bytes
651    */
652   private static int extractMinorVersion(int serializedVersion) {
653     return (serializedVersion >>> 24);
654   }
655 
656   /**
657    * Create a 4 byte serialized version number by combining the
658    * minor and major version numbers.
659    */
660   private static int materializeVersion(int majorVersion, int minorVersion) {
661     return ((majorVersion & 0x00ffffff) | (minorVersion << 24));
662   }
663 }