1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile;
20
21 import java.io.ByteArrayInputStream;
22 import java.io.ByteArrayOutputStream;
23 import java.io.DataInput;
24 import java.io.DataInputStream;
25 import java.io.DataOutputStream;
26 import java.io.IOException;
27 import java.nio.ByteBuffer;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.commons.io.output.NullOutputStream;
32 import org.apache.hadoop.classification.InterfaceAudience;
33 import org.apache.hadoop.fs.FSDataInputStream;
34 import org.apache.hadoop.hbase.KeyValue;
35 import org.apache.hadoop.hbase.KeyValue.KVComparator;
36 import org.apache.hadoop.hbase.io.compress.Compression;
37 import org.apache.hadoop.hbase.protobuf.generated.HFileProtos;
38 import org.apache.hadoop.hbase.util.Bytes;
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53 @InterfaceAudience.Private
54 public class FixedFileTrailer {
55
56 private static final Log LOG = LogFactory.getLog(FixedFileTrailer.class);
57
58
59 private static final int PBUF_TRAILER_MINOR_VERSION = 2;
60
61
62
63
64 private static final int MAX_COMPARATOR_NAME_LENGTH = 128;
65
66
67
68
69
70 private long fileInfoOffset;
71
72
73
74
75
76
77
78 private long loadOnOpenDataOffset;
79
80
81 private int dataIndexCount;
82
83
84 private long uncompressedDataIndexSize;
85
86
87 private int metaIndexCount;
88
89
90 private long totalUncompressedBytes;
91
92
93
94
95
96 private long entryCount;
97
98
99 private Compression.Algorithm compressionCodec = Compression.Algorithm.NONE;
100
101
102
103
104
105 private int numDataIndexLevels;
106
107
108 private long firstDataBlockOffset;
109
110
111
112
113
114 private long lastDataBlockOffset;
115
116
117 private String comparatorClassName = KeyValue.COMPARATOR.getLegacyKeyComparatorName();
118
119
120 private final int majorVersion;
121
122
123 private final int minorVersion;
124
125 FixedFileTrailer(int majorVersion, int minorVersion) {
126 this.majorVersion = majorVersion;
127 this.minorVersion = minorVersion;
128 HFile.checkFormatVersion(majorVersion);
129 }
130
131 private static int[] computeTrailerSizeByVersion() {
132 int versionToSize[] = new int[HFile.MAX_FORMAT_VERSION + 1];
133 for (int version = HFile.MIN_FORMAT_VERSION;
134 version <= HFile.MAX_FORMAT_VERSION;
135 ++version) {
136 FixedFileTrailer fft = new FixedFileTrailer(version, HFileBlock.MINOR_VERSION_NO_CHECKSUM);
137 DataOutputStream dos = new DataOutputStream(NullOutputStream.NULL_OUTPUT_STREAM);
138 try {
139 fft.serialize(dos);
140 } catch (IOException ex) {
141
142 throw new RuntimeException(ex);
143 }
144 versionToSize[version] = dos.size();
145 }
146 return versionToSize;
147 }
148
149 private static int getMaxTrailerSize() {
150 int maxSize = 0;
151 for (int version = HFile.MIN_FORMAT_VERSION;
152 version <= HFile.MAX_FORMAT_VERSION;
153 ++version)
154 maxSize = Math.max(getTrailerSize(version), maxSize);
155 return maxSize;
156 }
157
158 private static final int TRAILER_SIZE[] = computeTrailerSizeByVersion();
159 private static final int MAX_TRAILER_SIZE = getMaxTrailerSize();
160
161 private static final int NOT_PB_SIZE = BlockType.MAGIC_LENGTH + Bytes.SIZEOF_INT;
162
163 static int getTrailerSize(int version) {
164 return TRAILER_SIZE[version];
165 }
166
167 public int getTrailerSize() {
168 return getTrailerSize(majorVersion);
169 }
170
171
172
173
174
175
176
177
178
179 void serialize(DataOutputStream outputStream) throws IOException {
180 HFile.checkFormatVersion(majorVersion);
181
182 ByteArrayOutputStream baos = new ByteArrayOutputStream();
183 DataOutputStream baosDos = new DataOutputStream(baos);
184
185 BlockType.TRAILER.write(baosDos);
186 if (majorVersion > 2 || (majorVersion == 2 && minorVersion >= PBUF_TRAILER_MINOR_VERSION)) {
187 serializeAsPB(baosDos);
188 } else {
189 serializeAsWritable(baosDos);
190 }
191
192
193 baosDos.writeInt(materializeVersion(majorVersion, minorVersion));
194
195 baos.writeTo(outputStream);
196 }
197
198
199
200
201
202
203 void serializeAsPB(DataOutputStream output) throws IOException {
204 ByteArrayOutputStream baos = new ByteArrayOutputStream();
205 HFileProtos.FileTrailerProto.newBuilder()
206 .setFileInfoOffset(fileInfoOffset)
207 .setLoadOnOpenDataOffset(loadOnOpenDataOffset)
208 .setUncompressedDataIndexSize(uncompressedDataIndexSize)
209 .setTotalUncompressedBytes(totalUncompressedBytes)
210 .setDataIndexCount(dataIndexCount)
211 .setMetaIndexCount(metaIndexCount)
212 .setEntryCount(entryCount)
213 .setNumDataIndexLevels(numDataIndexLevels)
214 .setFirstDataBlockOffset(firstDataBlockOffset)
215 .setLastDataBlockOffset(lastDataBlockOffset)
216
217
218 .setComparatorClassName(comparatorClassName)
219 .setCompressionCodec(compressionCodec.ordinal())
220 .build()
221
222
223 .writeDelimitedTo(baos);
224 baos.writeTo(output);
225
226
227
228
229 int padding = getTrailerSize() - NOT_PB_SIZE - baos.size();
230 if (padding < 0) {
231 throw new IOException("Pbuf encoding size exceeded fixed trailer size limit");
232 }
233 for (int i = 0; i < padding; i++) {
234 output.write(0);
235 }
236 }
237
238
239
240
241
242
243 void serializeAsWritable(DataOutputStream output) throws IOException {
244 output.writeLong(fileInfoOffset);
245 output.writeLong(loadOnOpenDataOffset);
246 output.writeInt(dataIndexCount);
247
248 output.writeLong(uncompressedDataIndexSize);
249
250 output.writeInt(metaIndexCount);
251 output.writeLong(totalUncompressedBytes);
252 output.writeLong(entryCount);
253 output.writeInt(compressionCodec.ordinal());
254
255 output.writeInt(numDataIndexLevels);
256 output.writeLong(firstDataBlockOffset);
257 output.writeLong(lastDataBlockOffset);
258 Bytes.writeStringFixedSize(output, comparatorClassName, MAX_COMPARATOR_NAME_LENGTH);
259 }
260
261
262
263
264
265
266
267
268
269 void deserialize(DataInputStream inputStream) throws IOException {
270 HFile.checkFormatVersion(majorVersion);
271
272 BlockType.TRAILER.readAndCheck(inputStream);
273
274 if (majorVersion > 2 || (majorVersion == 2 && minorVersion >= PBUF_TRAILER_MINOR_VERSION)) {
275 deserializeFromPB(inputStream);
276 } else {
277 deserializeFromWritable(inputStream);
278 }
279
280
281 int version = inputStream.readInt();
282 expectMajorVersion(extractMajorVersion(version));
283 expectMinorVersion(extractMinorVersion(version));
284 }
285
286
287
288
289
290
291 void deserializeFromPB(DataInputStream inputStream) throws IOException {
292
293 int start = inputStream.available();
294 HFileProtos.FileTrailerProto.Builder builder = HFileProtos.FileTrailerProto.newBuilder();
295 builder.mergeDelimitedFrom(inputStream);
296 int size = start - inputStream.available();
297 inputStream.skip(getTrailerSize() - NOT_PB_SIZE - size);
298
299
300 if (builder.hasFileInfoOffset()) {
301 fileInfoOffset = builder.getFileInfoOffset();
302 }
303 if (builder.hasLoadOnOpenDataOffset()) {
304 loadOnOpenDataOffset = builder.getLoadOnOpenDataOffset();
305 }
306 if (builder.hasUncompressedDataIndexSize()) {
307 uncompressedDataIndexSize = builder.getUncompressedDataIndexSize();
308 }
309 if (builder.hasTotalUncompressedBytes()) {
310 totalUncompressedBytes = builder.getTotalUncompressedBytes();
311 }
312 if (builder.hasDataIndexCount()) {
313 dataIndexCount = builder.getDataIndexCount();
314 }
315 if (builder.hasMetaIndexCount()) {
316 metaIndexCount = builder.getMetaIndexCount();
317 }
318 if (builder.hasEntryCount()) {
319 entryCount = builder.getEntryCount();
320 }
321 if (builder.hasNumDataIndexLevels()) {
322 numDataIndexLevels = builder.getNumDataIndexLevels();
323 }
324 if (builder.hasFirstDataBlockOffset()) {
325 firstDataBlockOffset = builder.getFirstDataBlockOffset();
326 }
327 if (builder.hasLastDataBlockOffset()) {
328 lastDataBlockOffset = builder.getLastDataBlockOffset();
329 }
330 if (builder.hasComparatorClassName()) {
331
332
333 setComparatorClass(getComparatorClass(builder.getComparatorClassName()));
334 }
335 if (builder.hasCompressionCodec()) {
336 compressionCodec = Compression.Algorithm.values()[builder.getCompressionCodec()];
337 } else {
338 compressionCodec = Compression.Algorithm.NONE;
339 }
340 }
341
342
343
344
345
346
347 void deserializeFromWritable(DataInput input) throws IOException {
348 fileInfoOffset = input.readLong();
349 loadOnOpenDataOffset = input.readLong();
350 dataIndexCount = input.readInt();
351 uncompressedDataIndexSize = input.readLong();
352 metaIndexCount = input.readInt();
353
354 totalUncompressedBytes = input.readLong();
355 entryCount = input.readLong();
356 compressionCodec = Compression.Algorithm.values()[input.readInt()];
357 numDataIndexLevels = input.readInt();
358 firstDataBlockOffset = input.readLong();
359 lastDataBlockOffset = input.readLong();
360
361
362 setComparatorClass(getComparatorClass(Bytes.readStringFixedSize(input,
363 MAX_COMPARATOR_NAME_LENGTH)));
364 }
365
366 private void append(StringBuilder sb, String s) {
367 if (sb.length() > 0)
368 sb.append(", ");
369 sb.append(s);
370 }
371
372 @Override
373 public String toString() {
374 StringBuilder sb = new StringBuilder();
375 append(sb, "fileinfoOffset=" + fileInfoOffset);
376 append(sb, "loadOnOpenDataOffset=" + loadOnOpenDataOffset);
377 append(sb, "dataIndexCount=" + dataIndexCount);
378 append(sb, "metaIndexCount=" + metaIndexCount);
379 append(sb, "totalUncomressedBytes=" + totalUncompressedBytes);
380 append(sb, "entryCount=" + entryCount);
381 append(sb, "compressionCodec=" + compressionCodec);
382 append(sb, "uncompressedDataIndexSize=" + uncompressedDataIndexSize);
383 append(sb, "numDataIndexLevels=" + numDataIndexLevels);
384 append(sb, "firstDataBlockOffset=" + firstDataBlockOffset);
385 append(sb, "lastDataBlockOffset=" + lastDataBlockOffset);
386 append(sb, "comparatorClassName=" + comparatorClassName);
387 append(sb, "majorVersion=" + majorVersion);
388 append(sb, "minorVersion=" + minorVersion);
389
390 return sb.toString();
391 }
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406 public static FixedFileTrailer readFromStream(FSDataInputStream istream,
407 long fileSize) throws IOException {
408 int bufferSize = MAX_TRAILER_SIZE;
409 long seekPoint = fileSize - bufferSize;
410 if (seekPoint < 0) {
411
412 seekPoint = 0;
413 bufferSize = (int) fileSize;
414 }
415
416 istream.seek(seekPoint);
417 ByteBuffer buf = ByteBuffer.allocate(bufferSize);
418 istream.readFully(buf.array(), buf.arrayOffset(),
419 buf.arrayOffset() + buf.limit());
420
421
422 buf.position(buf.limit() - Bytes.SIZEOF_INT);
423 int version = buf.getInt();
424
425
426 int majorVersion = extractMajorVersion(version);
427 int minorVersion = extractMinorVersion(version);
428
429 HFile.checkFormatVersion(majorVersion);
430
431 int trailerSize = getTrailerSize(majorVersion);
432
433 FixedFileTrailer fft = new FixedFileTrailer(majorVersion, minorVersion);
434 fft.deserialize(new DataInputStream(new ByteArrayInputStream(buf.array(),
435 buf.arrayOffset() + bufferSize - trailerSize, trailerSize)));
436 return fft;
437 }
438
439 public void expectMajorVersion(int expected) {
440 if (majorVersion != expected) {
441 throw new IllegalArgumentException("Invalid HFile major version: "
442 + majorVersion
443 + " (expected: " + expected + ")");
444 }
445 }
446
447 public void expectMinorVersion(int expected) {
448 if (minorVersion != expected) {
449 throw new IllegalArgumentException("Invalid HFile minor version: "
450 + minorVersion + " (expected: " + expected + ")");
451 }
452 }
453
454 public void expectAtLeastMajorVersion(int lowerBound) {
455 if (majorVersion < lowerBound) {
456 throw new IllegalArgumentException("Invalid HFile major version: "
457 + majorVersion
458 + " (expected: " + lowerBound + " or higher).");
459 }
460 }
461
462 public long getFileInfoOffset() {
463 return fileInfoOffset;
464 }
465
466 public void setFileInfoOffset(long fileInfoOffset) {
467 this.fileInfoOffset = fileInfoOffset;
468 }
469
470 public long getLoadOnOpenDataOffset() {
471 return loadOnOpenDataOffset;
472 }
473
474 public void setLoadOnOpenOffset(long loadOnOpenDataOffset) {
475 this.loadOnOpenDataOffset = loadOnOpenDataOffset;
476 }
477
478 public int getDataIndexCount() {
479 return dataIndexCount;
480 }
481
482 public void setDataIndexCount(int dataIndexCount) {
483 this.dataIndexCount = dataIndexCount;
484 }
485
486 public int getMetaIndexCount() {
487 return metaIndexCount;
488 }
489
490 public void setMetaIndexCount(int metaIndexCount) {
491 this.metaIndexCount = metaIndexCount;
492 }
493
494 public long getTotalUncompressedBytes() {
495 return totalUncompressedBytes;
496 }
497
498 public void setTotalUncompressedBytes(long totalUncompressedBytes) {
499 this.totalUncompressedBytes = totalUncompressedBytes;
500 }
501
502 public long getEntryCount() {
503 return entryCount;
504 }
505
506 public void setEntryCount(long newEntryCount) {
507 entryCount = newEntryCount;
508 }
509
510 public Compression.Algorithm getCompressionCodec() {
511 return compressionCodec;
512 }
513
514 public void setCompressionCodec(Compression.Algorithm compressionCodec) {
515 this.compressionCodec = compressionCodec;
516 }
517
518 public int getNumDataIndexLevels() {
519 expectAtLeastMajorVersion(2);
520 return numDataIndexLevels;
521 }
522
523 public void setNumDataIndexLevels(int numDataIndexLevels) {
524 expectAtLeastMajorVersion(2);
525 this.numDataIndexLevels = numDataIndexLevels;
526 }
527
528 public long getLastDataBlockOffset() {
529 expectAtLeastMajorVersion(2);
530 return lastDataBlockOffset;
531 }
532
533 public void setLastDataBlockOffset(long lastDataBlockOffset) {
534 expectAtLeastMajorVersion(2);
535 this.lastDataBlockOffset = lastDataBlockOffset;
536 }
537
538 public long getFirstDataBlockOffset() {
539 expectAtLeastMajorVersion(2);
540 return firstDataBlockOffset;
541 }
542
543 public void setFirstDataBlockOffset(long firstDataBlockOffset) {
544 expectAtLeastMajorVersion(2);
545 this.firstDataBlockOffset = firstDataBlockOffset;
546 }
547
548 public String getComparatorClassName() {
549 return comparatorClassName;
550 }
551
552
553
554
555 public int getMajorVersion() {
556 return majorVersion;
557 }
558
559
560
561
562 public int getMinorVersion() {
563 return minorVersion;
564 }
565
566 public void setComparatorClass(Class<? extends KVComparator> klass) {
567
568 try {
569 KVComparator comp = klass.newInstance();
570
571
572 if (KeyValue.COMPARATOR.getClass().equals(klass)) {
573 comparatorClassName = KeyValue.COMPARATOR.getLegacyKeyComparatorName();
574 } else if (KeyValue.META_COMPARATOR.getClass().equals(klass)) {
575 comparatorClassName = KeyValue.META_COMPARATOR.getLegacyKeyComparatorName();
576 } else if (KeyValue.RAW_COMPARATOR.getClass().equals(klass)) {
577 comparatorClassName = KeyValue.RAW_COMPARATOR.getLegacyKeyComparatorName();
578 } else {
579
580 comparatorClassName = klass.getName();
581 }
582
583 } catch (Exception e) {
584 throw new RuntimeException("Comparator class " + klass.getName() +
585 " is not instantiable", e);
586 }
587
588 }
589
590 @SuppressWarnings("unchecked")
591 private static Class<? extends KVComparator> getComparatorClass(
592 String comparatorClassName) throws IOException {
593 try {
594
595 if (comparatorClassName.equals(KeyValue.COMPARATOR.getLegacyKeyComparatorName())) {
596 comparatorClassName = KeyValue.COMPARATOR.getClass().getName();
597 } else if (comparatorClassName.equals(KeyValue.META_COMPARATOR.getLegacyKeyComparatorName())) {
598 comparatorClassName = KeyValue.META_COMPARATOR.getClass().getName();
599 } else if (comparatorClassName.equals(KeyValue.RAW_COMPARATOR.getLegacyKeyComparatorName())) {
600 comparatorClassName = KeyValue.RAW_COMPARATOR.getClass().getName();
601 }
602
603
604
605 return (Class<? extends KVComparator>)
606 Class.forName(comparatorClassName);
607 } catch (ClassNotFoundException ex) {
608 throw new IOException(ex);
609 }
610 }
611
612 public static KVComparator createComparator(
613 String comparatorClassName) throws IOException {
614 try {
615 return getComparatorClass(comparatorClassName).newInstance();
616 } catch (InstantiationException e) {
617 throw new IOException("Comparator class " + comparatorClassName +
618 " is not instantiable", e);
619 } catch (IllegalAccessException e) {
620 throw new IOException("Comparator class " + comparatorClassName +
621 " is not instantiable", e);
622 }
623 }
624
625 KVComparator createComparator() throws IOException {
626 expectAtLeastMajorVersion(2);
627 return createComparator(comparatorClassName);
628 }
629
630 public long getUncompressedDataIndexSize() {
631 return uncompressedDataIndexSize;
632 }
633
634 public void setUncompressedDataIndexSize(
635 long uncompressedDataIndexSize) {
636 expectAtLeastMajorVersion(2);
637 this.uncompressedDataIndexSize = uncompressedDataIndexSize;
638 }
639
640
641
642
643
644 private static int extractMajorVersion(int serializedVersion) {
645 return (serializedVersion & 0x00ffffff);
646 }
647
648
649
650
651
652 private static int extractMinorVersion(int serializedVersion) {
653 return (serializedVersion >>> 24);
654 }
655
656
657
658
659
660 private static int materializeVersion(int majorVersion, int minorVersion) {
661 return ((majorVersion & 0x00ffffff) | (minorVersion << 24));
662 }
663 }