1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.io.hfile;
21
22 import static org.apache.hadoop.hbase.io.hfile.HFile.MAX_FORMAT_VERSION;
23 import static org.apache.hadoop.hbase.io.hfile.HFile.MIN_FORMAT_VERSION;
24
25 import java.io.ByteArrayInputStream;
26 import java.io.ByteArrayOutputStream;
27 import java.io.DataInputStream;
28 import java.io.DataOutput;
29 import java.io.DataOutputStream;
30 import java.io.IOException;
31 import java.nio.ByteBuffer;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.fs.FSDataInputStream;
36 import org.apache.hadoop.hbase.util.Bytes;
37 import org.apache.hadoop.io.RawComparator;
38
39 import com.google.common.io.NullOutputStream;
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54 public class FixedFileTrailer {
55
56 private static final Log LOG = LogFactory.getLog(FixedFileTrailer.class);
57
58
59
60
61 private static final int MAX_COMPARATOR_NAME_LENGTH = 128;
62
63
64
65
66
67 private long fileInfoOffset;
68
69
70
71
72
73
74
75 private long loadOnOpenDataOffset;
76
77
78 private int dataIndexCount;
79
80
81 private long uncompressedDataIndexSize;
82
83
84 private int metaIndexCount;
85
86
87 private long totalUncompressedBytes;
88
89
90
91
92
93 private long entryCount;
94
95
96 private Compression.Algorithm compressionCodec = Compression.Algorithm.NONE;
97
98
99
100
101
102 private int numDataIndexLevels;
103
104
105 private long firstDataBlockOffset;
106
107
108
109
110
111 private long lastDataBlockOffset;
112
113
114 private String comparatorClassName = RawComparator.class.getName();
115
116
117 private final int majorVersion;
118
119
120 private final int minorVersion;
121
122 FixedFileTrailer(int majorVersion, int minorVersion) {
123 this.majorVersion = majorVersion;
124 this.minorVersion = minorVersion;
125 HFile.checkFormatVersion(majorVersion);
126 }
127
128 private static int[] computeTrailerSizeByVersion() {
129 int versionToSize[] = new int[HFile.MAX_FORMAT_VERSION + 1];
130 for (int version = MIN_FORMAT_VERSION;
131 version <= MAX_FORMAT_VERSION;
132 ++version) {
133 FixedFileTrailer fft = new FixedFileTrailer(version,
134 HFileBlock.MINOR_VERSION_NO_CHECKSUM);
135 DataOutputStream dos = new DataOutputStream(new NullOutputStream());
136 try {
137 fft.serialize(dos);
138 } catch (IOException ex) {
139
140 throw new RuntimeException(ex);
141 }
142 versionToSize[version] = dos.size();
143 }
144 return versionToSize;
145 }
146
147 private static int getMaxTrailerSize() {
148 int maxSize = 0;
149 for (int version = MIN_FORMAT_VERSION;
150 version <= MAX_FORMAT_VERSION;
151 ++version)
152 maxSize = Math.max(getTrailerSize(version), maxSize);
153 return maxSize;
154 }
155
156 private static final int TRAILER_SIZE[] = computeTrailerSizeByVersion();
157 private static final int MAX_TRAILER_SIZE = getMaxTrailerSize();
158
159 static int getTrailerSize(int version) {
160 return TRAILER_SIZE[version];
161 }
162
163 public int getTrailerSize() {
164 return getTrailerSize(majorVersion);
165 }
166
167
168
169
170
171
172
173
174
175 void serialize(DataOutputStream outputStream) throws IOException {
176 HFile.checkFormatVersion(majorVersion);
177
178 ByteArrayOutputStream baos = new ByteArrayOutputStream();
179 DataOutput baosDos = new DataOutputStream(baos);
180
181 BlockType.TRAILER.write(baosDos);
182 baosDos.writeLong(fileInfoOffset);
183 baosDos.writeLong(loadOnOpenDataOffset);
184 baosDos.writeInt(dataIndexCount);
185
186 if (majorVersion == 1) {
187
188 baosDos.writeLong(0);
189 } else {
190 baosDos.writeLong(uncompressedDataIndexSize);
191 }
192
193 baosDos.writeInt(metaIndexCount);
194 baosDos.writeLong(totalUncompressedBytes);
195 if (majorVersion == 1) {
196 baosDos.writeInt((int) Math.min(Integer.MAX_VALUE, entryCount));
197 } else {
198
199 baosDos.writeLong(entryCount);
200 }
201 baosDos.writeInt(compressionCodec.ordinal());
202
203 if (majorVersion > 1) {
204 baosDos.writeInt(numDataIndexLevels);
205 baosDos.writeLong(firstDataBlockOffset);
206 baosDos.writeLong(lastDataBlockOffset);
207 Bytes.writeStringFixedSize(baosDos, comparatorClassName,
208 MAX_COMPARATOR_NAME_LENGTH);
209 }
210
211
212 baosDos.writeInt(materializeVersion(majorVersion, minorVersion));
213
214 outputStream.write(baos.toByteArray());
215 }
216
217
218
219
220
221
222
223
224
225
226 void deserialize(DataInputStream inputStream) throws IOException {
227 HFile.checkFormatVersion(majorVersion);
228
229 BlockType.TRAILER.readAndCheck(inputStream);
230
231 fileInfoOffset = inputStream.readLong();
232 loadOnOpenDataOffset = inputStream.readLong();
233 dataIndexCount = inputStream.readInt();
234
235 if (majorVersion == 1) {
236 inputStream.readLong();
237 } else {
238 uncompressedDataIndexSize = inputStream.readLong();
239 }
240 metaIndexCount = inputStream.readInt();
241
242 totalUncompressedBytes = inputStream.readLong();
243 entryCount = majorVersion == 1 ? inputStream.readInt() : inputStream.readLong();
244 compressionCodec = Compression.Algorithm.values()[inputStream.readInt()];
245 if (majorVersion > 1) {
246 numDataIndexLevels = inputStream.readInt();
247 firstDataBlockOffset = inputStream.readLong();
248 lastDataBlockOffset = inputStream.readLong();
249 comparatorClassName =
250 Bytes.readStringFixedSize(inputStream, MAX_COMPARATOR_NAME_LENGTH);
251 }
252
253 int version = inputStream.readInt();
254 expectMajorVersion(extractMajorVersion(version));
255 expectMinorVersion(extractMinorVersion(version));
256 }
257
258 private void append(StringBuilder sb, String s) {
259 if (sb.length() > 0)
260 sb.append(", ");
261 sb.append(s);
262 }
263
264 @Override
265 public String toString() {
266 StringBuilder sb = new StringBuilder();
267 append(sb, "fileinfoOffset=" + fileInfoOffset);
268 append(sb, "loadOnOpenDataOffset=" + loadOnOpenDataOffset);
269 append(sb, "dataIndexCount=" + dataIndexCount);
270 append(sb, "metaIndexCount=" + metaIndexCount);
271 append(sb, "totalUncomressedBytes=" + totalUncompressedBytes);
272 append(sb, "entryCount=" + entryCount);
273 append(sb, "compressionCodec=" + compressionCodec);
274 if (majorVersion == 2) {
275 append(sb, "uncompressedDataIndexSize=" + uncompressedDataIndexSize);
276 append(sb, "numDataIndexLevels=" + numDataIndexLevels);
277 append(sb, "firstDataBlockOffset=" + firstDataBlockOffset);
278 append(sb, "lastDataBlockOffset=" + lastDataBlockOffset);
279 append(sb, "comparatorClassName=" + comparatorClassName);
280 }
281 append(sb, "majorVersion=" + majorVersion);
282 append(sb, "minorVersion=" + minorVersion);
283
284 return sb.toString();
285 }
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300 public static FixedFileTrailer readFromStream(FSDataInputStream istream,
301 long fileSize) throws IOException {
302 int bufferSize = MAX_TRAILER_SIZE;
303 long seekPoint = fileSize - bufferSize;
304 if (seekPoint < 0) {
305
306 seekPoint = 0;
307 bufferSize = (int) fileSize;
308 }
309
310 istream.seek(seekPoint);
311 ByteBuffer buf = ByteBuffer.allocate(bufferSize);
312 istream.readFully(buf.array(), buf.arrayOffset(),
313 buf.arrayOffset() + buf.limit());
314
315
316 buf.position(buf.limit() - Bytes.SIZEOF_INT);
317 int version = buf.getInt();
318
319
320 int majorVersion = extractMajorVersion(version);
321 int minorVersion = extractMinorVersion(version);
322
323 HFile.checkFormatVersion(majorVersion);
324
325 int trailerSize = getTrailerSize(majorVersion);
326
327 FixedFileTrailer fft = new FixedFileTrailer(majorVersion, minorVersion);
328 fft.deserialize(new DataInputStream(new ByteArrayInputStream(buf.array(),
329 buf.arrayOffset() + bufferSize - trailerSize, trailerSize)));
330 return fft;
331 }
332
333 public void expectMajorVersion(int expected) {
334 if (majorVersion != expected) {
335 throw new IllegalArgumentException("Invalid HFile major version: "
336 + majorVersion
337 + " (expected: " + expected + ")");
338 }
339 }
340
341 public void expectMinorVersion(int expected) {
342 if (minorVersion != expected) {
343 throw new IllegalArgumentException("Invalid HFile minor version: "
344 + minorVersion + " (expected: " + expected + ")");
345 }
346 }
347
348 public void expectAtLeastMajorVersion(int lowerBound) {
349 if (majorVersion < lowerBound) {
350 throw new IllegalArgumentException("Invalid HFile major version: "
351 + majorVersion
352 + " (expected: " + lowerBound + " or higher).");
353 }
354 }
355
356 public long getFileInfoOffset() {
357 return fileInfoOffset;
358 }
359
360 public void setFileInfoOffset(long fileInfoOffset) {
361 this.fileInfoOffset = fileInfoOffset;
362 }
363
364 public long getLoadOnOpenDataOffset() {
365 return loadOnOpenDataOffset;
366 }
367
368 public void setLoadOnOpenOffset(long loadOnOpenDataOffset) {
369 this.loadOnOpenDataOffset = loadOnOpenDataOffset;
370 }
371
372 public int getDataIndexCount() {
373 return dataIndexCount;
374 }
375
376 public void setDataIndexCount(int dataIndexCount) {
377 this.dataIndexCount = dataIndexCount;
378 }
379
380 public int getMetaIndexCount() {
381 return metaIndexCount;
382 }
383
384 public void setMetaIndexCount(int metaIndexCount) {
385 this.metaIndexCount = metaIndexCount;
386 }
387
388 public long getTotalUncompressedBytes() {
389 return totalUncompressedBytes;
390 }
391
392 public void setTotalUncompressedBytes(long totalUncompressedBytes) {
393 this.totalUncompressedBytes = totalUncompressedBytes;
394 }
395
396 public long getEntryCount() {
397 return entryCount;
398 }
399
400 public void setEntryCount(long newEntryCount) {
401 if (majorVersion == 1) {
402 int intEntryCount = (int) Math.min(Integer.MAX_VALUE, newEntryCount);
403 if (intEntryCount != newEntryCount) {
404 LOG.info("Warning: entry count is " + newEntryCount + " but writing "
405 + intEntryCount + " into the version " + majorVersion + " trailer");
406 }
407 entryCount = intEntryCount;
408 return;
409 }
410 entryCount = newEntryCount;
411 }
412
413 public Compression.Algorithm getCompressionCodec() {
414 return compressionCodec;
415 }
416
417 public void setCompressionCodec(Compression.Algorithm compressionCodec) {
418 this.compressionCodec = compressionCodec;
419 }
420
421 public int getNumDataIndexLevels() {
422 expectAtLeastMajorVersion(2);
423 return numDataIndexLevels;
424 }
425
426 public void setNumDataIndexLevels(int numDataIndexLevels) {
427 expectAtLeastMajorVersion(2);
428 this.numDataIndexLevels = numDataIndexLevels;
429 }
430
431 public long getLastDataBlockOffset() {
432 expectAtLeastMajorVersion(2);
433 return lastDataBlockOffset;
434 }
435
436 public void setLastDataBlockOffset(long lastDataBlockOffset) {
437 expectAtLeastMajorVersion(2);
438 this.lastDataBlockOffset = lastDataBlockOffset;
439 }
440
441 public long getFirstDataBlockOffset() {
442 expectAtLeastMajorVersion(2);
443 return firstDataBlockOffset;
444 }
445
446 public void setFirstDataBlockOffset(long firstDataBlockOffset) {
447 expectAtLeastMajorVersion(2);
448 this.firstDataBlockOffset = firstDataBlockOffset;
449 }
450
451
452
453
454 public int getMajorVersion() {
455 return majorVersion;
456 }
457
458
459
460
461 int getMinorVersion() {
462 return minorVersion;
463 }
464
465 @SuppressWarnings("rawtypes")
466 public void setComparatorClass(Class<? extends RawComparator> klass) {
467 expectAtLeastMajorVersion(2);
468 comparatorClassName = klass.getName();
469 }
470
471 @SuppressWarnings("unchecked")
472 private static Class<? extends RawComparator<byte[]>> getComparatorClass(
473 String comparatorClassName) throws IOException {
474 try {
475 return (Class<? extends RawComparator<byte[]>>)
476 Class.forName(comparatorClassName);
477 } catch (ClassNotFoundException ex) {
478 throw new IOException(ex);
479 }
480 }
481
482 public static RawComparator<byte[]> createComparator(
483 String comparatorClassName) throws IOException {
484 try {
485 return getComparatorClass(comparatorClassName).newInstance();
486 } catch (InstantiationException e) {
487 throw new IOException(e);
488 } catch (IllegalAccessException e) {
489 throw new IOException(e);
490 }
491 }
492
493 RawComparator<byte[]> createComparator() throws IOException {
494 expectAtLeastMajorVersion(2);
495 return createComparator(comparatorClassName);
496 }
497
498 public long getUncompressedDataIndexSize() {
499 if (majorVersion == 1)
500 return 0;
501 return uncompressedDataIndexSize;
502 }
503
504 public void setUncompressedDataIndexSize(
505 long uncompressedDataIndexSize) {
506 expectAtLeastMajorVersion(2);
507 this.uncompressedDataIndexSize = uncompressedDataIndexSize;
508 }
509
510
511
512
513
514 private static int extractMajorVersion(int serializedVersion) {
515 return (serializedVersion & 0x00ffffff);
516 }
517
518
519
520
521
522 private static int extractMinorVersion(int serializedVersion) {
523 return (serializedVersion >>> 24);
524 }
525
526
527
528
529
530 private static int materializeVersion(int majorVersion, int minorVersion) {
531 return ((majorVersion & 0x00ffffff) | (minorVersion << 24));
532 }
533 }