1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile;
20
21 import java.io.ByteArrayInputStream;
22 import java.io.ByteArrayOutputStream;
23 import java.io.DataInput;
24 import java.io.DataInputStream;
25 import java.io.DataOutputStream;
26 import java.io.IOException;
27 import java.nio.ByteBuffer;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.classification.InterfaceAudience;
32 import org.apache.hadoop.fs.FSDataInputStream;
33 import org.apache.hadoop.hbase.KeyValue;
34 import org.apache.hadoop.hbase.io.compress.Compression;
35 import org.apache.hadoop.hbase.protobuf.generated.HFileProtos;
36 import org.apache.hadoop.hbase.util.Bytes;
37 import org.apache.hadoop.io.RawComparator;
38
39 import com.google.common.io.NullOutputStream;
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54 @InterfaceAudience.Private
55 public class FixedFileTrailer {
56
57 private static final Log LOG = LogFactory.getLog(FixedFileTrailer.class);
58
59
60 private static final int PBUF_TRAILER_MINOR_VERSION = 2;
61
62
63
64
65 private static final int MAX_COMPARATOR_NAME_LENGTH = 128;
66
67
68
69
70
71 private long fileInfoOffset;
72
73
74
75
76
77
78
79 private long loadOnOpenDataOffset;
80
81
82 private int dataIndexCount;
83
84
85 private long uncompressedDataIndexSize;
86
87
88 private int metaIndexCount;
89
90
91 private long totalUncompressedBytes;
92
93
94
95
96
97 private long entryCount;
98
99
100 private Compression.Algorithm compressionCodec = Compression.Algorithm.NONE;
101
102
103
104
105
106 private int numDataIndexLevels;
107
108
109 private long firstDataBlockOffset;
110
111
112
113
114
115 private long lastDataBlockOffset;
116
117
118 private String comparatorClassName = KeyValue.KEY_COMPARATOR.getClass().getName();
119
120
121 private final int majorVersion;
122
123
124 private final int minorVersion;
125
126 FixedFileTrailer(int majorVersion, int minorVersion) {
127 this.majorVersion = majorVersion;
128 this.minorVersion = minorVersion;
129 HFile.checkFormatVersion(majorVersion);
130 }
131
132 private static int[] computeTrailerSizeByVersion() {
133 int versionToSize[] = new int[HFile.MAX_FORMAT_VERSION + 1];
134 for (int version = HFile.MIN_FORMAT_VERSION;
135 version <= HFile.MAX_FORMAT_VERSION;
136 ++version) {
137 FixedFileTrailer fft = new FixedFileTrailer(version, HFileBlock.MINOR_VERSION_NO_CHECKSUM);
138 DataOutputStream dos = new DataOutputStream(new NullOutputStream());
139 try {
140 fft.serialize(dos);
141 } catch (IOException ex) {
142
143 throw new RuntimeException(ex);
144 }
145 versionToSize[version] = dos.size();
146 }
147 return versionToSize;
148 }
149
150 private static int getMaxTrailerSize() {
151 int maxSize = 0;
152 for (int version = HFile.MIN_FORMAT_VERSION;
153 version <= HFile.MAX_FORMAT_VERSION;
154 ++version)
155 maxSize = Math.max(getTrailerSize(version), maxSize);
156 return maxSize;
157 }
158
159 private static final int TRAILER_SIZE[] = computeTrailerSizeByVersion();
160 private static final int MAX_TRAILER_SIZE = getMaxTrailerSize();
161
162 private static final int NOT_PB_SIZE = BlockType.MAGIC_LENGTH + Bytes.SIZEOF_INT;
163
164 static int getTrailerSize(int version) {
165 return TRAILER_SIZE[version];
166 }
167
168 public int getTrailerSize() {
169 return getTrailerSize(majorVersion);
170 }
171
172
173
174
175
176
177
178
179
180 void serialize(DataOutputStream outputStream) throws IOException {
181 HFile.checkFormatVersion(majorVersion);
182
183 ByteArrayOutputStream baos = new ByteArrayOutputStream();
184 DataOutputStream baosDos = new DataOutputStream(baos);
185
186 BlockType.TRAILER.write(baosDos);
187 if (majorVersion > 2 || (majorVersion == 2 && minorVersion >= PBUF_TRAILER_MINOR_VERSION)) {
188 serializeAsPB(baosDos);
189 } else {
190 serializeAsWritable(baosDos);
191 }
192
193
194 baosDos.writeInt(materializeVersion(majorVersion, minorVersion));
195
196 outputStream.write(baos.toByteArray());
197 }
198
199
200
201
202
203
204 void serializeAsPB(DataOutputStream output) throws IOException {
205 ByteArrayOutputStream baos = new ByteArrayOutputStream();
206 HFileProtos.FileTrailerProto.newBuilder()
207 .setFileInfoOffset(fileInfoOffset)
208 .setLoadOnOpenDataOffset(loadOnOpenDataOffset)
209 .setUncompressedDataIndexSize(uncompressedDataIndexSize)
210 .setTotalUncompressedBytes(totalUncompressedBytes)
211 .setDataIndexCount(dataIndexCount)
212 .setMetaIndexCount(metaIndexCount)
213 .setEntryCount(entryCount)
214 .setNumDataIndexLevels(numDataIndexLevels)
215 .setFirstDataBlockOffset(firstDataBlockOffset)
216 .setLastDataBlockOffset(lastDataBlockOffset)
217 .setComparatorClassName(comparatorClassName)
218 .setCompressionCodec(compressionCodec.ordinal())
219 .build().writeDelimitedTo(baos);
220 output.write(baos.toByteArray());
221
222
223
224
225 int padding = getTrailerSize() - NOT_PB_SIZE - baos.size();
226 if (padding < 0) {
227 throw new IOException("Pbuf encoding size exceeded fixed trailer size limit");
228 }
229 for (int i = 0; i < padding; i++) {
230 output.write(0);
231 }
232 }
233
234
235
236
237
238
239 void serializeAsWritable(DataOutputStream output) throws IOException {
240 output.writeLong(fileInfoOffset);
241 output.writeLong(loadOnOpenDataOffset);
242 output.writeInt(dataIndexCount);
243
244 if (majorVersion == 1) {
245
246 output.writeLong(0);
247 } else {
248 output.writeLong(uncompressedDataIndexSize);
249 }
250
251 output.writeInt(metaIndexCount);
252 output.writeLong(totalUncompressedBytes);
253 if (majorVersion == 1) {
254 output.writeInt((int) Math.min(Integer.MAX_VALUE, entryCount));
255 } else {
256
257 output.writeLong(entryCount);
258 }
259 output.writeInt(compressionCodec.ordinal());
260
261 if (majorVersion > 1) {
262 output.writeInt(numDataIndexLevels);
263 output.writeLong(firstDataBlockOffset);
264 output.writeLong(lastDataBlockOffset);
265 Bytes.writeStringFixedSize(output, comparatorClassName, MAX_COMPARATOR_NAME_LENGTH);
266 }
267 }
268
269
270
271
272
273
274
275
276
277 void deserialize(DataInputStream inputStream) throws IOException {
278 HFile.checkFormatVersion(majorVersion);
279
280 BlockType.TRAILER.readAndCheck(inputStream);
281
282 if (majorVersion > 2 || (majorVersion == 2 && minorVersion >= PBUF_TRAILER_MINOR_VERSION)) {
283 deserializeFromPB(inputStream);
284 } else {
285 deserializeFromWritable(inputStream);
286 }
287
288
289 int version = inputStream.readInt();
290 expectMajorVersion(extractMajorVersion(version));
291 expectMinorVersion(extractMinorVersion(version));
292 }
293
294
295
296
297
298
299 void deserializeFromPB(DataInputStream inputStream) throws IOException {
300
301 int start = inputStream.available();
302 HFileProtos.FileTrailerProto.Builder builder = HFileProtos.FileTrailerProto.newBuilder();
303 builder.mergeDelimitedFrom(inputStream);
304 int size = start - inputStream.available();
305 inputStream.skip(getTrailerSize() - NOT_PB_SIZE - size);
306
307
308 if (builder.hasFileInfoOffset()) {
309 fileInfoOffset = builder.getFileInfoOffset();
310 }
311 if (builder.hasLoadOnOpenDataOffset()) {
312 loadOnOpenDataOffset = builder.getLoadOnOpenDataOffset();
313 }
314 if (builder.hasUncompressedDataIndexSize()) {
315 uncompressedDataIndexSize = builder.getUncompressedDataIndexSize();
316 }
317 if (builder.hasTotalUncompressedBytes()) {
318 totalUncompressedBytes = builder.getTotalUncompressedBytes();
319 }
320 if (builder.hasDataIndexCount()) {
321 dataIndexCount = builder.getDataIndexCount();
322 }
323 if (builder.hasMetaIndexCount()) {
324 metaIndexCount = builder.getMetaIndexCount();
325 }
326 if (builder.hasEntryCount()) {
327 entryCount = builder.getEntryCount();
328 }
329 if (builder.hasNumDataIndexLevels()) {
330 numDataIndexLevels = builder.getNumDataIndexLevels();
331 }
332 if (builder.hasFirstDataBlockOffset()) {
333 firstDataBlockOffset = builder.getFirstDataBlockOffset();
334 }
335 if (builder.hasLastDataBlockOffset()) {
336 lastDataBlockOffset = builder.getLastDataBlockOffset();
337 }
338 if (builder.hasComparatorClassName()) {
339 setComparatorClass(getComparatorClass(builder.getComparatorClassName()));
340 }
341 if (builder.hasCompressionCodec()) {
342 compressionCodec = Compression.Algorithm.values()[builder.getCompressionCodec()];
343 } else {
344 compressionCodec = Compression.Algorithm.NONE;
345 }
346 }
347
348
349
350
351
352
353 void deserializeFromWritable(DataInput input) throws IOException {
354 fileInfoOffset = input.readLong();
355 loadOnOpenDataOffset = input.readLong();
356 dataIndexCount = input.readInt();
357 if (majorVersion == 1) {
358 input.readLong();
359 } else {
360 uncompressedDataIndexSize = input.readLong();
361 }
362 metaIndexCount = input.readInt();
363
364 totalUncompressedBytes = input.readLong();
365 entryCount = majorVersion == 1 ? input.readInt() : input.readLong();
366 compressionCodec = Compression.Algorithm.values()[input.readInt()];
367 if (majorVersion > 1) {
368 numDataIndexLevels = input.readInt();
369 firstDataBlockOffset = input.readLong();
370 lastDataBlockOffset = input.readLong();
371 setComparatorClass(getComparatorClass(Bytes.readStringFixedSize(input,
372 MAX_COMPARATOR_NAME_LENGTH)));
373 }
374 }
375
376 private void append(StringBuilder sb, String s) {
377 if (sb.length() > 0)
378 sb.append(", ");
379 sb.append(s);
380 }
381
382 @Override
383 public String toString() {
384 StringBuilder sb = new StringBuilder();
385 append(sb, "fileinfoOffset=" + fileInfoOffset);
386 append(sb, "loadOnOpenDataOffset=" + loadOnOpenDataOffset);
387 append(sb, "dataIndexCount=" + dataIndexCount);
388 append(sb, "metaIndexCount=" + metaIndexCount);
389 append(sb, "totalUncomressedBytes=" + totalUncompressedBytes);
390 append(sb, "entryCount=" + entryCount);
391 append(sb, "compressionCodec=" + compressionCodec);
392 if (majorVersion == 2) {
393 append(sb, "uncompressedDataIndexSize=" + uncompressedDataIndexSize);
394 append(sb, "numDataIndexLevels=" + numDataIndexLevels);
395 append(sb, "firstDataBlockOffset=" + firstDataBlockOffset);
396 append(sb, "lastDataBlockOffset=" + lastDataBlockOffset);
397 append(sb, "comparatorClassName=" + comparatorClassName);
398 }
399 append(sb, "majorVersion=" + majorVersion);
400 append(sb, "minorVersion=" + minorVersion);
401
402 return sb.toString();
403 }
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418 public static FixedFileTrailer readFromStream(FSDataInputStream istream,
419 long fileSize) throws IOException {
420 int bufferSize = MAX_TRAILER_SIZE;
421 long seekPoint = fileSize - bufferSize;
422 if (seekPoint < 0) {
423
424 seekPoint = 0;
425 bufferSize = (int) fileSize;
426 }
427
428 istream.seek(seekPoint);
429 ByteBuffer buf = ByteBuffer.allocate(bufferSize);
430 istream.readFully(buf.array(), buf.arrayOffset(),
431 buf.arrayOffset() + buf.limit());
432
433
434 buf.position(buf.limit() - Bytes.SIZEOF_INT);
435 int version = buf.getInt();
436
437
438 int majorVersion = extractMajorVersion(version);
439 int minorVersion = extractMinorVersion(version);
440
441 HFile.checkFormatVersion(majorVersion);
442
443 int trailerSize = getTrailerSize(majorVersion);
444
445 FixedFileTrailer fft = new FixedFileTrailer(majorVersion, minorVersion);
446 fft.deserialize(new DataInputStream(new ByteArrayInputStream(buf.array(),
447 buf.arrayOffset() + bufferSize - trailerSize, trailerSize)));
448 return fft;
449 }
450
451 public void expectMajorVersion(int expected) {
452 if (majorVersion != expected) {
453 throw new IllegalArgumentException("Invalid HFile major version: "
454 + majorVersion
455 + " (expected: " + expected + ")");
456 }
457 }
458
459 public void expectMinorVersion(int expected) {
460 if (minorVersion != expected) {
461 throw new IllegalArgumentException("Invalid HFile minor version: "
462 + minorVersion + " (expected: " + expected + ")");
463 }
464 }
465
466 public void expectAtLeastMajorVersion(int lowerBound) {
467 if (majorVersion < lowerBound) {
468 throw new IllegalArgumentException("Invalid HFile major version: "
469 + majorVersion
470 + " (expected: " + lowerBound + " or higher).");
471 }
472 }
473
474 public long getFileInfoOffset() {
475 return fileInfoOffset;
476 }
477
478 public void setFileInfoOffset(long fileInfoOffset) {
479 this.fileInfoOffset = fileInfoOffset;
480 }
481
482 public long getLoadOnOpenDataOffset() {
483 return loadOnOpenDataOffset;
484 }
485
486 public void setLoadOnOpenOffset(long loadOnOpenDataOffset) {
487 this.loadOnOpenDataOffset = loadOnOpenDataOffset;
488 }
489
490 public int getDataIndexCount() {
491 return dataIndexCount;
492 }
493
494 public void setDataIndexCount(int dataIndexCount) {
495 this.dataIndexCount = dataIndexCount;
496 }
497
498 public int getMetaIndexCount() {
499 return metaIndexCount;
500 }
501
502 public void setMetaIndexCount(int metaIndexCount) {
503 this.metaIndexCount = metaIndexCount;
504 }
505
506 public long getTotalUncompressedBytes() {
507 return totalUncompressedBytes;
508 }
509
510 public void setTotalUncompressedBytes(long totalUncompressedBytes) {
511 this.totalUncompressedBytes = totalUncompressedBytes;
512 }
513
514 public long getEntryCount() {
515 return entryCount;
516 }
517
518 public void setEntryCount(long newEntryCount) {
519 if (majorVersion == 1) {
520 int intEntryCount = (int) Math.min(Integer.MAX_VALUE, newEntryCount);
521 if (intEntryCount != newEntryCount) {
522 LOG.info("Warning: entry count is " + newEntryCount + " but writing "
523 + intEntryCount + " into the version " + majorVersion + " trailer");
524 }
525 entryCount = intEntryCount;
526 return;
527 }
528 entryCount = newEntryCount;
529 }
530
531 public Compression.Algorithm getCompressionCodec() {
532 return compressionCodec;
533 }
534
535 public void setCompressionCodec(Compression.Algorithm compressionCodec) {
536 this.compressionCodec = compressionCodec;
537 }
538
539 public int getNumDataIndexLevels() {
540 expectAtLeastMajorVersion(2);
541 return numDataIndexLevels;
542 }
543
544 public void setNumDataIndexLevels(int numDataIndexLevels) {
545 expectAtLeastMajorVersion(2);
546 this.numDataIndexLevels = numDataIndexLevels;
547 }
548
549 public long getLastDataBlockOffset() {
550 expectAtLeastMajorVersion(2);
551 return lastDataBlockOffset;
552 }
553
554 public void setLastDataBlockOffset(long lastDataBlockOffset) {
555 expectAtLeastMajorVersion(2);
556 this.lastDataBlockOffset = lastDataBlockOffset;
557 }
558
559 public long getFirstDataBlockOffset() {
560 expectAtLeastMajorVersion(2);
561 return firstDataBlockOffset;
562 }
563
564 public void setFirstDataBlockOffset(long firstDataBlockOffset) {
565 expectAtLeastMajorVersion(2);
566 this.firstDataBlockOffset = firstDataBlockOffset;
567 }
568
569 public String getComparatorClassName() {
570 return comparatorClassName;
571 }
572
573
574
575
576 public int getMajorVersion() {
577 return majorVersion;
578 }
579
580
581
582
583 int getMinorVersion() {
584 return minorVersion;
585 }
586
587 @SuppressWarnings("rawtypes")
588 public void setComparatorClass(Class<? extends RawComparator> klass) {
589
590 try {
591 klass.newInstance();
592 } catch (Exception e) {
593 throw new RuntimeException("Comparator class " + klass.getName() +
594 " is not instantiable", e);
595 }
596 comparatorClassName = klass.getName();
597 }
598
599 @SuppressWarnings("unchecked")
600 private static Class<? extends RawComparator<byte[]>> getComparatorClass(
601 String comparatorClassName) throws IOException {
602 try {
603 return (Class<? extends RawComparator<byte[]>>)
604 Class.forName(comparatorClassName);
605 } catch (ClassNotFoundException ex) {
606 throw new IOException(ex);
607 }
608 }
609
610 public static RawComparator<byte[]> createComparator(
611 String comparatorClassName) throws IOException {
612 try {
613 return getComparatorClass(comparatorClassName).newInstance();
614 } catch (InstantiationException e) {
615 throw new IOException("Comparator class " + comparatorClassName +
616 " is not instantiable", e);
617 } catch (IllegalAccessException e) {
618 throw new IOException("Comparator class " + comparatorClassName +
619 " is not instantiable", e);
620 }
621 }
622
623 RawComparator<byte[]> createComparator() throws IOException {
624 expectAtLeastMajorVersion(2);
625 return createComparator(comparatorClassName);
626 }
627
628 public long getUncompressedDataIndexSize() {
629 if (majorVersion == 1)
630 return 0;
631 return uncompressedDataIndexSize;
632 }
633
634 public void setUncompressedDataIndexSize(
635 long uncompressedDataIndexSize) {
636 expectAtLeastMajorVersion(2);
637 this.uncompressedDataIndexSize = uncompressedDataIndexSize;
638 }
639
640
641
642
643
644 private static int extractMajorVersion(int serializedVersion) {
645 return (serializedVersion & 0x00ffffff);
646 }
647
648
649
650
651
652 private static int extractMinorVersion(int serializedVersion) {
653 return (serializedVersion >>> 24);
654 }
655
656
657
658
659
660 private static int materializeVersion(int majorVersion, int minorVersion) {
661 return ((majorVersion & 0x00ffffff) | (minorVersion << 24));
662 }
663 }