1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.io.hfile;
21
22 import java.io.Closeable;
23 import java.io.DataInputStream;
24 import java.io.DataOutputStream;
25 import java.io.IOException;
26 import java.io.InputStream;
27 import java.io.OutputStream;
28 import java.nio.ByteBuffer;
29 import java.util.ArrayList;
30 import java.util.Arrays;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.SortedSet;
34
35 import org.apache.commons.cli.CommandLine;
36 import org.apache.commons.cli.CommandLineParser;
37 import org.apache.commons.cli.HelpFormatter;
38 import org.apache.commons.cli.Options;
39 import org.apache.commons.cli.PosixParser;
40 import org.apache.commons.logging.Log;
41 import org.apache.commons.logging.LogFactory;
42 import org.apache.hadoop.conf.Configuration;
43 import org.apache.hadoop.fs.FSDataInputStream;
44 import org.apache.hadoop.fs.FSDataOutputStream;
45 import org.apache.hadoop.fs.FileStatus;
46 import org.apache.hadoop.fs.FileSystem;
47 import org.apache.hadoop.fs.Path;
48 import org.apache.hadoop.fs.PathFilter;
49 import org.apache.hadoop.hbase.KeyValue.KeyComparator;
50 import org.apache.hadoop.hbase.HBaseConfiguration;
51 import org.apache.hadoop.hbase.HRegionInfo;
52 import org.apache.hadoop.hbase.KeyValue;
53 import org.apache.hadoop.hbase.io.HbaseMapWritable;
54 import org.apache.hadoop.hbase.io.HeapSize;
55 import org.apache.hadoop.hbase.util.Bytes;
56 import org.apache.hadoop.hbase.util.ClassSize;
57 import org.apache.hadoop.hbase.util.FSUtils;
58 import org.apache.hadoop.io.IOUtils;
59 import org.apache.hadoop.io.RawComparator;
60 import org.apache.hadoop.io.Writable;
61 import org.apache.hadoop.io.compress.Compressor;
62 import org.apache.hadoop.io.compress.Decompressor;
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127 public class HFile {
128 static final Log LOG = LogFactory.getLog(HFile.class);
129
130
131
132
133 final static byte [] DATABLOCKMAGIC =
134 {'D', 'A', 'T', 'A', 'B', 'L', 'K', 42 };
135 final static byte [] INDEXBLOCKMAGIC =
136 { 'I', 'D', 'X', 'B', 'L', 'K', 41, 43 };
137 final static byte [] METABLOCKMAGIC =
138 { 'M', 'E', 'T', 'A', 'B', 'L', 'K', 99 };
139 final static byte [] TRAILERBLOCKMAGIC =
140 { 'T', 'R', 'A', 'B', 'L', 'K', 34, 36 };
141
142
143
144
145 public final static int MAXIMUM_KEY_LENGTH = Integer.MAX_VALUE;
146
147
148
149
150 public final static int DEFAULT_BLOCKSIZE = 64 * 1024;
151
152
153
154
155 public final static Compression.Algorithm DEFAULT_COMPRESSION_ALGORITHM =
156 Compression.Algorithm.NONE;
157
158 public final static String DEFAULT_COMPRESSION =
159 DEFAULT_COMPRESSION_ALGORITHM.getName();
160
161
162 private static volatile long readOps;
163 private static volatile long readTime;
164 private static volatile long writeOps;
165 private static volatile long writeTime;
166
167 public static final long getReadOps() {
168 long ret = readOps;
169 readOps = 0;
170 return ret;
171 }
172
173 public static final long getReadTime() {
174 long ret = readTime;
175 readTime = 0;
176 return ret;
177 }
178
179 public static final long getWriteOps() {
180 long ret = writeOps;
181 writeOps = 0;
182 return ret;
183 }
184
185 public static final long getWriteTime() {
186 long ret = writeTime;
187 writeTime = 0;
188 return ret;
189 }
190
191
192
193
194 public static class Writer implements Closeable {
195
196 private FSDataOutputStream outputStream;
197
198 private boolean closeOutputStream;
199
200
201
202 protected String name;
203
204
205 private long totalBytes = 0;
206
207
208 private int entryCount = 0;
209
210
211 private long keylength = 0;
212 private long valuelength = 0;
213
214
215 private final RawComparator<byte []> rawComparator;
216
217
218 private DataOutputStream out;
219
220
221
222 private int blocksize;
223
224
225 private long blockBegin;
226
227
228 private byte [] firstKey = null;
229
230
231 private byte [] lastKeyBuffer = null;
232 private int lastKeyOffset = -1;
233 private int lastKeyLength = -1;
234
235
236
237 ArrayList<byte[]> blockKeys = new ArrayList<byte[]>();
238
239 ArrayList<Long> blockOffsets = new ArrayList<Long>();
240
241 ArrayList<Integer> blockDataSizes = new ArrayList<Integer>();
242
243
244 private ArrayList<byte []> metaNames = new ArrayList<byte []>();
245 private ArrayList<Writable> metaData = new ArrayList<Writable>();
246
247
248 private final Compression.Algorithm compressAlgo;
249 private Compressor compressor;
250
251
252 private FileInfo fileinfo = new FileInfo();
253
254
255 private Path path = null;
256
257
258
259
260
261
262
263 public Writer(FileSystem fs, Path path)
264 throws IOException {
265 this(fs, path, DEFAULT_BLOCKSIZE, (Compression.Algorithm) null, null);
266 }
267
268
269
270
271
272
273
274
275
276
277
278 public Writer(FileSystem fs, Path path, int blocksize,
279 String compress, final KeyComparator comparator)
280 throws IOException {
281 this(fs, path, blocksize,
282 compress == null? DEFAULT_COMPRESSION_ALGORITHM:
283 Compression.getCompressionAlgorithmByName(compress),
284 comparator);
285 }
286
287
288
289
290
291
292
293
294
295
296 public Writer(FileSystem fs, Path path, int blocksize,
297 Compression.Algorithm compress,
298 final KeyComparator comparator)
299 throws IOException {
300 this(fs.create(path), blocksize, compress, comparator);
301 this.closeOutputStream = true;
302 this.name = path.toString();
303 this.path = path;
304 }
305
306
307
308
309
310
311
312
313
314 public Writer(final FSDataOutputStream ostream, final int blocksize,
315 final String compress, final KeyComparator c)
316 throws IOException {
317 this(ostream, blocksize,
318 Compression.getCompressionAlgorithmByName(compress), c);
319 }
320
321
322
323
324
325
326
327
328
329 public Writer(final FSDataOutputStream ostream, final int blocksize,
330 final Compression.Algorithm compress, final KeyComparator c)
331 throws IOException {
332 this.outputStream = ostream;
333 this.closeOutputStream = false;
334 this.blocksize = blocksize;
335 this.rawComparator = c == null? Bytes.BYTES_RAWCOMPARATOR: c;
336 this.name = this.outputStream.toString();
337 this.compressAlgo = compress == null?
338 DEFAULT_COMPRESSION_ALGORITHM: compress;
339 }
340
341
342
343
344
345 private void checkBlockBoundary() throws IOException {
346 if (this.out != null && this.out.size() < blocksize) return;
347 finishBlock();
348 newBlock();
349 }
350
351
352
353
354
355 private void finishBlock() throws IOException {
356 if (this.out == null) return;
357 long now = System.currentTimeMillis();
358
359 int size = releaseCompressingStream(this.out);
360 this.out = null;
361 blockKeys.add(firstKey);
362 blockOffsets.add(Long.valueOf(blockBegin));
363 blockDataSizes.add(Integer.valueOf(size));
364 this.totalBytes += size;
365
366 writeTime += System.currentTimeMillis() - now;
367 writeOps++;
368 }
369
370
371
372
373
374 private void newBlock() throws IOException {
375
376 blockBegin = outputStream.getPos();
377 this.out = getCompressingStream();
378 this.out.write(DATABLOCKMAGIC);
379 firstKey = null;
380 }
381
382
383
384
385
386
387
388
389
390 private DataOutputStream getCompressingStream() throws IOException {
391 this.compressor = compressAlgo.getCompressor();
392
393
394
395
396
397
398
399
400 OutputStream os =
401 this.compressAlgo.createCompressionStream(this.outputStream,
402 this.compressor, 0);
403 return new DataOutputStream(os);
404 }
405
406
407
408
409
410
411
412
413
414 private int releaseCompressingStream(final DataOutputStream dos)
415 throws IOException {
416 dos.flush();
417 this.compressAlgo.returnCompressor(this.compressor);
418 this.compressor = null;
419 return dos.size();
420 }
421
422
423
424
425
426
427
428
429
430
431 public void appendMetaBlock(String metaBlockName, Writable content) {
432 byte[] key = Bytes.toBytes(metaBlockName);
433 int i;
434 for (i = 0; i < metaNames.size(); ++i) {
435
436 byte[] cur = metaNames.get(i);
437 if (this.rawComparator.compare(cur, 0, cur.length, key, 0, key.length)
438 > 0) {
439 break;
440 }
441 }
442 metaNames.add(i, key);
443 metaData.add(i, content);
444 }
445
446
447
448
449
450
451
452
453 public void appendFileInfo(final byte [] k, final byte [] v)
454 throws IOException {
455 appendFileInfo(this.fileinfo, k, v, true);
456 }
457
458 static FileInfo appendFileInfo(FileInfo fi, final byte [] k, final byte [] v,
459 final boolean checkPrefix)
460 throws IOException {
461 if (k == null || v == null) {
462 throw new NullPointerException("Key nor value may be null");
463 }
464 if (checkPrefix &&
465 Bytes.startsWith(k, FileInfo.RESERVED_PREFIX_BYTES)) {
466 throw new IOException("Keys with a " + FileInfo.RESERVED_PREFIX +
467 " are reserved");
468 }
469 fi.put(k, v);
470 return fi;
471 }
472
473
474
475
476 public Path getPath() {
477 return this.path;
478 }
479
480 @Override
481 public String toString() {
482 return "writer=" + this.name + ", compression=" +
483 this.compressAlgo.getName();
484 }
485
486
487
488
489
490
491
492
493 public void append(final KeyValue kv)
494 throws IOException {
495 append(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength(),
496 kv.getBuffer(), kv.getValueOffset(), kv.getValueLength());
497 }
498
499
500
501
502
503
504
505
506
507 public void append(final byte [] key, final byte [] value)
508 throws IOException {
509 append(key, 0, key.length, value, 0, value.length);
510 }
511
512
513
514
515
516
517
518
519
520
521
522
523
524 private void append(final byte [] key, final int koffset, final int klength,
525 final byte [] value, final int voffset, final int vlength)
526 throws IOException {
527 boolean dupKey = checkKey(key, koffset, klength);
528 checkValue(value, voffset, vlength);
529 if (!dupKey) {
530 checkBlockBoundary();
531 }
532
533 this.out.writeInt(klength);
534 this.keylength += klength;
535 this.out.writeInt(vlength);
536 this.valuelength += vlength;
537 this.out.write(key, koffset, klength);
538 this.out.write(value, voffset, vlength);
539
540 if (this.firstKey == null) {
541
542 this.firstKey = new byte [klength];
543 System.arraycopy(key, koffset, this.firstKey, 0, klength);
544 }
545 this.lastKeyBuffer = key;
546 this.lastKeyOffset = koffset;
547 this.lastKeyLength = klength;
548 this.entryCount ++;
549 }
550
551
552
553
554
555
556 private boolean checkKey(final byte [] key, final int offset, final int length)
557 throws IOException {
558 boolean dupKey = false;
559
560 if (key == null || length <= 0) {
561 throw new IOException("Key cannot be null or empty");
562 }
563 if (length > MAXIMUM_KEY_LENGTH) {
564 throw new IOException("Key length " + length + " > " +
565 MAXIMUM_KEY_LENGTH);
566 }
567 if (this.lastKeyBuffer != null) {
568 int keyComp = this.rawComparator.compare(this.lastKeyBuffer, this.lastKeyOffset,
569 this.lastKeyLength, key, offset, length);
570 if (keyComp > 0) {
571 throw new IOException("Added a key not lexically larger than" +
572 " previous key=" + Bytes.toStringBinary(key, offset, length) +
573 ", lastkey=" + Bytes.toStringBinary(this.lastKeyBuffer, this.lastKeyOffset,
574 this.lastKeyLength));
575 } else if (keyComp == 0) {
576 dupKey = true;
577 }
578 }
579 return dupKey;
580 }
581
582 private void checkValue(final byte [] value, final int offset,
583 final int length) throws IOException {
584 if (value == null) {
585 throw new IOException("Value cannot be null");
586 }
587 }
588
589 public long getTotalBytes() {
590 return this.totalBytes;
591 }
592
593 public void close() throws IOException {
594 if (this.outputStream == null) {
595 return;
596 }
597
598
599
600 finishBlock();
601
602 FixedFileTrailer trailer = new FixedFileTrailer();
603
604
605 ArrayList<Long> metaOffsets = null;
606 ArrayList<Integer> metaDataSizes = null;
607 if (metaNames.size() > 0) {
608 metaOffsets = new ArrayList<Long>(metaNames.size());
609 metaDataSizes = new ArrayList<Integer>(metaNames.size());
610 for (int i = 0 ; i < metaNames.size() ; ++ i ) {
611
612 long curPos = outputStream.getPos();
613 metaOffsets.add(curPos);
614
615 DataOutputStream dos = getCompressingStream();
616 dos.write(METABLOCKMAGIC);
617 metaData.get(i).write(dos);
618 int size = releaseCompressingStream(dos);
619
620 metaDataSizes.add(size);
621 }
622 }
623
624
625 trailer.fileinfoOffset = writeFileInfo(this.outputStream);
626
627
628 trailer.dataIndexOffset = BlockIndex.writeIndex(this.outputStream,
629 this.blockKeys, this.blockOffsets, this.blockDataSizes);
630
631
632 if (metaNames.size() > 0) {
633 trailer.metaIndexOffset = BlockIndex.writeIndex(this.outputStream,
634 this.metaNames, metaOffsets, metaDataSizes);
635 }
636
637
638 trailer.dataIndexCount = blockKeys.size();
639 trailer.metaIndexCount = metaNames.size();
640
641 trailer.totalUncompressedBytes = totalBytes;
642 trailer.entryCount = entryCount;
643
644 trailer.compressionCodec = this.compressAlgo.ordinal();
645
646 trailer.serialize(outputStream);
647
648 if (this.closeOutputStream) {
649 this.outputStream.close();
650 this.outputStream = null;
651 }
652 }
653
654
655
656
657
658
659
660
661 private long writeFileInfo(FSDataOutputStream o) throws IOException {
662 if (this.lastKeyBuffer != null) {
663
664
665 byte [] b = new byte[this.lastKeyLength];
666 System.arraycopy(this.lastKeyBuffer, this.lastKeyOffset, b, 0,
667 this.lastKeyLength);
668 appendFileInfo(this.fileinfo, FileInfo.LASTKEY, b, false);
669 }
670 int avgKeyLen = this.entryCount == 0? 0:
671 (int)(this.keylength/this.entryCount);
672 appendFileInfo(this.fileinfo, FileInfo.AVG_KEY_LEN,
673 Bytes.toBytes(avgKeyLen), false);
674 int avgValueLen = this.entryCount == 0? 0:
675 (int)(this.valuelength/this.entryCount);
676 appendFileInfo(this.fileinfo, FileInfo.AVG_VALUE_LEN,
677 Bytes.toBytes(avgValueLen), false);
678 appendFileInfo(this.fileinfo, FileInfo.COMPARATOR,
679 Bytes.toBytes(this.rawComparator.getClass().getName()), false);
680 long pos = o.getPos();
681 this.fileinfo.write(o);
682 return pos;
683 }
684 }
685
686
687
688
689 public static class Reader implements Closeable {
690
691 private FSDataInputStream istream;
692
693
694 private boolean closeIStream;
695
696
697 HFile.BlockIndex blockIndex;
698 private BlockIndex metaIndex;
699 FixedFileTrailer trailer;
700 private volatile boolean fileInfoLoaded = false;
701
702
703 private Compression.Algorithm compressAlgo;
704
705
706 private byte [] lastkey = null;
707
708 private int avgKeyLen = -1;
709 private int avgValueLen = -1;
710
711
712 RawComparator<byte []> comparator;
713
714
715 private final long fileSize;
716
717
718 private final BlockCache cache;
719 public int cacheHits = 0;
720 public int blockLoads = 0;
721 public int metaLoads = 0;
722
723
724 private boolean inMemory = false;
725
726
727
728
729 protected String name;
730
731
732
733
734
735
736
737
738
739
740 public Reader(FileSystem fs, Path path, BlockCache cache, boolean inMemory)
741 throws IOException {
742 this(fs.open(path), fs.getFileStatus(path).getLen(), cache, inMemory);
743 this.closeIStream = true;
744 this.name = path.toString();
745 }
746
747
748
749
750
751
752
753
754
755
756
757 public Reader(final FSDataInputStream fsdis, final long size,
758 final BlockCache cache, final boolean inMemory) {
759 this.cache = cache;
760 this.fileSize = size;
761 this.istream = fsdis;
762 this.closeIStream = false;
763 this.name = this.istream == null? "": this.istream.toString();
764 this.inMemory = inMemory;
765 }
766
767 @Override
768 public String toString() {
769 return "reader=" + this.name +
770 (!isFileInfoLoaded()? "":
771 ", compression=" + this.compressAlgo.getName() +
772 ", inMemory=" + this.inMemory +
773 ", firstKey=" + toStringFirstKey() +
774 ", lastKey=" + toStringLastKey()) +
775 ", avgKeyLen=" + this.avgKeyLen +
776 ", avgValueLen=" + this.avgValueLen +
777 ", entries=" + this.trailer.entryCount +
778 ", length=" + this.fileSize;
779 }
780
781 protected String toStringFirstKey() {
782 return KeyValue.keyToString(getFirstKey());
783 }
784
785 protected String toStringLastKey() {
786 return KeyValue.keyToString(getLastKey());
787 }
788
789 public long length() {
790 return this.fileSize;
791 }
792
793 public boolean inMemory() {
794 return this.inMemory;
795 }
796
797
798
799
800
801
802
803 public Map<byte [], byte []> loadFileInfo()
804 throws IOException {
805 this.trailer = readTrailer();
806
807
808 this.istream.seek(this.trailer.fileinfoOffset);
809 FileInfo fi = new FileInfo();
810 fi.readFields(this.istream);
811 this.lastkey = fi.get(FileInfo.LASTKEY);
812 this.avgKeyLen = Bytes.toInt(fi.get(FileInfo.AVG_KEY_LEN));
813 this.avgValueLen = Bytes.toInt(fi.get(FileInfo.AVG_VALUE_LEN));
814 String clazzName = Bytes.toString(fi.get(FileInfo.COMPARATOR));
815 this.comparator = getComparator(clazzName);
816
817
818 this.blockIndex = BlockIndex.readIndex(this.comparator, this.istream,
819 this.trailer.dataIndexOffset, this.trailer.dataIndexCount);
820
821
822 if (trailer.metaIndexCount > 0) {
823 this.metaIndex = BlockIndex.readIndex(Bytes.BYTES_RAWCOMPARATOR,
824 this.istream, this.trailer.metaIndexOffset, trailer.metaIndexCount);
825 }
826 this.fileInfoLoaded = true;
827 return fi;
828 }
829
830 boolean isFileInfoLoaded() {
831 return this.fileInfoLoaded;
832 }
833
834 @SuppressWarnings("unchecked")
835 private RawComparator<byte []> getComparator(final String clazzName)
836 throws IOException {
837 if (clazzName == null || clazzName.length() == 0) {
838 return null;
839 }
840 try {
841 return (RawComparator<byte []>)Class.forName(clazzName).newInstance();
842 } catch (InstantiationException e) {
843 throw new IOException(e);
844 } catch (IllegalAccessException e) {
845 throw new IOException(e);
846 } catch (ClassNotFoundException e) {
847 throw new IOException(e);
848 }
849 }
850
851
852
853
854
855
856 private FixedFileTrailer readTrailer() throws IOException {
857 FixedFileTrailer fft = new FixedFileTrailer();
858 long seekPoint = this.fileSize - FixedFileTrailer.trailerSize();
859 this.istream.seek(seekPoint);
860 fft.deserialize(this.istream);
861
862 this.compressAlgo =
863 Compression.Algorithm.values()[fft.compressionCodec];
864 return fft;
865 }
866
867
868
869
870
871
872
873
874
875
876
877 public HFileScanner getScanner(boolean cacheBlocks, final boolean pread) {
878 return new Scanner(this, cacheBlocks, pread);
879 }
880
881
882
883
884
885
886 protected int blockContainingKey(final byte [] key, int offset, int length) {
887 if (blockIndex == null) {
888 throw new RuntimeException("Block index not loaded");
889 }
890 return blockIndex.blockContainingKey(key, offset, length);
891 }
892
893
894
895
896
897
898 public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock)
899 throws IOException {
900 if (trailer.metaIndexCount == 0) {
901 return null;
902 }
903 if (metaIndex == null) {
904 throw new IOException("Meta index not loaded");
905 }
906
907 byte [] mbname = Bytes.toBytes(metaBlockName);
908 int block = metaIndex.blockContainingKey(mbname, 0, mbname.length);
909 if (block == -1)
910 return null;
911 long blockSize;
912 if (block == metaIndex.count - 1) {
913 blockSize = trailer.fileinfoOffset - metaIndex.blockOffsets[block];
914 } else {
915 blockSize = metaIndex.blockOffsets[block+1] - metaIndex.blockOffsets[block];
916 }
917
918 long now = System.currentTimeMillis();
919
920
921 synchronized (metaIndex.blockKeys[block]) {
922 metaLoads++;
923
924 if (cache != null) {
925 ByteBuffer cachedBuf = cache.getBlock(name + "meta" + block);
926 if (cachedBuf != null) {
927
928
929 cacheHits++;
930 return cachedBuf.duplicate();
931 }
932
933 }
934
935 ByteBuffer buf = decompress(metaIndex.blockOffsets[block],
936 longToInt(blockSize), metaIndex.blockDataSizes[block], true);
937 byte [] magic = new byte[METABLOCKMAGIC.length];
938 buf.get(magic, 0, magic.length);
939
940 if (! Arrays.equals(magic, METABLOCKMAGIC)) {
941 throw new IOException("Meta magic is bad in block " + block);
942 }
943
944
945 buf = buf.slice();
946
947 readTime += System.currentTimeMillis() - now;
948 readOps++;
949
950
951 if(cacheBlock && cache != null) {
952 cache.cacheBlock(name + "meta" + block, buf.duplicate(), inMemory);
953 }
954
955 return buf;
956 }
957 }
958
959
960
961
962
963
964
965
966
967 ByteBuffer readBlock(int block, boolean cacheBlock, final boolean pread)
968 throws IOException {
969 if (blockIndex == null) {
970 throw new IOException("Block index not loaded");
971 }
972 if (block < 0 || block >= blockIndex.count) {
973 throw new IOException("Requested block is out of range: " + block +
974 ", max: " + blockIndex.count);
975 }
976
977
978
979
980 synchronized (blockIndex.blockKeys[block]) {
981 blockLoads++;
982
983 if (cache != null) {
984 ByteBuffer cachedBuf = cache.getBlock(name + block);
985 if (cachedBuf != null) {
986
987
988 cacheHits++;
989 return cachedBuf.duplicate();
990 }
991
992 }
993
994
995 long now = System.currentTimeMillis();
996 long onDiskBlockSize;
997 if (block == blockIndex.count - 1) {
998
999
1000 long offset = this.metaIndex != null?
1001 this.metaIndex.blockOffsets[0]: this.trailer.fileinfoOffset;
1002 onDiskBlockSize = offset - blockIndex.blockOffsets[block];
1003 } else {
1004 onDiskBlockSize = blockIndex.blockOffsets[block+1] -
1005 blockIndex.blockOffsets[block];
1006 }
1007 ByteBuffer buf = decompress(blockIndex.blockOffsets[block],
1008 longToInt(onDiskBlockSize), this.blockIndex.blockDataSizes[block],
1009 pread);
1010
1011 byte [] magic = new byte[DATABLOCKMAGIC.length];
1012 buf.get(magic, 0, magic.length);
1013 if (!Arrays.equals(magic, DATABLOCKMAGIC)) {
1014 throw new IOException("Data magic is bad in block " + block);
1015 }
1016
1017
1018
1019
1020 buf = buf.slice();
1021
1022 readTime += System.currentTimeMillis() - now;
1023 readOps++;
1024
1025
1026 if(cacheBlock && cache != null) {
1027 cache.cacheBlock(name + block, buf.duplicate(), inMemory);
1028 }
1029
1030 return buf;
1031 }
1032 }
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044 private ByteBuffer decompress(final long offset, final int compressedSize,
1045 final int decompressedSize, final boolean pread)
1046 throws IOException {
1047 Decompressor decompressor = null;
1048 ByteBuffer buf = null;
1049 try {
1050 decompressor = this.compressAlgo.getDecompressor();
1051
1052
1053
1054
1055 InputStream is = this.compressAlgo.createDecompressionStream(
1056 new BoundedRangeFileInputStream(this.istream, offset, compressedSize,
1057 pread),
1058 decompressor, 0);
1059 buf = ByteBuffer.allocate(decompressedSize);
1060 IOUtils.readFully(is, buf.array(), 0, buf.capacity());
1061 is.close();
1062 } finally {
1063 if (null != decompressor) {
1064 this.compressAlgo.returnDecompressor(decompressor);
1065 }
1066 }
1067 return buf;
1068 }
1069
1070
1071
1072
1073
1074
1075 public byte [] getFirstKey() {
1076 if (blockIndex == null) {
1077 throw new RuntimeException("Block index not loaded");
1078 }
1079 return this.blockIndex.isEmpty()? null: this.blockIndex.blockKeys[0];
1080 }
1081
1082
1083
1084
1085
1086
1087 public byte[] getFirstRowKey() {
1088 byte[] firstKey = getFirstKey();
1089 if (firstKey == null) return null;
1090 return KeyValue.createKeyValueFromKey(firstKey).getRow();
1091 }
1092
1093
1094
1095
1096 public int getEntries() {
1097 if (!this.isFileInfoLoaded()) {
1098 throw new RuntimeException("File info not loaded");
1099 }
1100 return this.trailer.entryCount;
1101 }
1102
1103
1104
1105
1106
1107
1108 public byte [] getLastKey() {
1109 if (!isFileInfoLoaded()) {
1110 throw new RuntimeException("Load file info first");
1111 }
1112 return this.blockIndex.isEmpty()? null: this.lastkey;
1113 }
1114
1115
1116
1117
1118
1119
1120 public byte[] getLastRowKey() {
1121 byte[] lastKey = getLastKey();
1122 if (lastKey == null) return null;
1123 return KeyValue.createKeyValueFromKey(lastKey).getRow();
1124 }
1125
1126
1127
1128
1129 public int getFilterEntries() {
1130 return getEntries();
1131 }
1132
1133
1134
1135
1136 public RawComparator<byte []> getComparator() {
1137 return this.comparator;
1138 }
1139
1140
1141
1142
1143 public long indexSize() {
1144 return (this.blockIndex != null? this.blockIndex.heapSize(): 0) +
1145 ((this.metaIndex != null)? this.metaIndex.heapSize(): 0);
1146 }
1147
1148
1149
1150
1151
1152
1153 public byte [] midkey() throws IOException {
1154 if (!isFileInfoLoaded() || this.blockIndex.isEmpty()) {
1155 return null;
1156 }
1157 return this.blockIndex.midkey();
1158 }
1159
1160 public void close() throws IOException {
1161 if (this.closeIStream && this.istream != null) {
1162 this.istream.close();
1163 this.istream = null;
1164 }
1165 }
1166
1167 public String getName() {
1168 return name;
1169 }
1170
1171
1172
1173
1174 protected static class Scanner implements HFileScanner {
1175 private final Reader reader;
1176 private ByteBuffer block;
1177 private int currBlock;
1178
1179 private final boolean cacheBlocks;
1180 private final boolean pread;
1181
1182 private int currKeyLen = 0;
1183 private int currValueLen = 0;
1184
1185 public int blockFetches = 0;
1186
1187 public Scanner(Reader r, boolean cacheBlocks, final boolean pread) {
1188 this.reader = r;
1189 this.cacheBlocks = cacheBlocks;
1190 this.pread = pread;
1191 }
1192
1193 public KeyValue getKeyValue() {
1194 if(this.block == null) {
1195 return null;
1196 }
1197 return new KeyValue(this.block.array(),
1198 this.block.arrayOffset() + this.block.position() - 8);
1199 }
1200
1201 public ByteBuffer getKey() {
1202 if (this.block == null || this.currKeyLen == 0) {
1203 throw new RuntimeException("you need to seekTo() before calling getKey()");
1204 }
1205 ByteBuffer keyBuff = this.block.slice();
1206 keyBuff.limit(this.currKeyLen);
1207 keyBuff.rewind();
1208
1209 return keyBuff;
1210 }
1211
1212 public ByteBuffer getValue() {
1213 if (block == null || currKeyLen == 0) {
1214 throw new RuntimeException("you need to seekTo() before calling getValue()");
1215 }
1216
1217 ByteBuffer valueBuff = this.block.slice();
1218 valueBuff.position(this.currKeyLen);
1219 valueBuff = valueBuff.slice();
1220 valueBuff.limit(currValueLen);
1221 valueBuff.rewind();
1222 return valueBuff;
1223 }
1224
1225 public boolean next() throws IOException {
1226
1227
1228 if (block == null) {
1229 throw new IOException("Next called on non-seeked scanner");
1230 }
1231 block.position(block.position() + currKeyLen + currValueLen);
1232 if (block.remaining() <= 0) {
1233
1234 currBlock++;
1235 if (currBlock >= reader.blockIndex.count) {
1236
1237 currBlock = 0;
1238 block = null;
1239 return false;
1240 }
1241 block = reader.readBlock(this.currBlock, this.cacheBlocks, this.pread);
1242 currKeyLen = block.getInt();
1243 currValueLen = block.getInt();
1244 blockFetches++;
1245 return true;
1246 }
1247
1248
1249
1250 currKeyLen = block.getInt();
1251 currValueLen = block.getInt();
1252 return true;
1253 }
1254
1255 public int seekTo(byte [] key) throws IOException {
1256 return seekTo(key, 0, key.length);
1257 }
1258
1259
1260 public int seekTo(byte[] key, int offset, int length) throws IOException {
1261 int b = reader.blockContainingKey(key, offset, length);
1262 if (b < 0) return -1;
1263
1264 loadBlock(b);
1265
1266 return blockSeek(key, offset, length, false);
1267 }
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279 private int blockSeek(byte[] key, int offset, int length, boolean seekBefore) {
1280 int klen, vlen;
1281 int lastLen = 0;
1282 do {
1283 klen = block.getInt();
1284 vlen = block.getInt();
1285 int comp = this.reader.comparator.compare(key, offset, length,
1286 block.array(), block.arrayOffset() + block.position(), klen);
1287 if (comp == 0) {
1288 if (seekBefore) {
1289 block.position(block.position() - lastLen - 16);
1290 currKeyLen = block.getInt();
1291 currValueLen = block.getInt();
1292 return 1;
1293 }
1294 currKeyLen = klen;
1295 currValueLen = vlen;
1296 return 0;
1297 }
1298 if (comp < 0) {
1299
1300 block.position(block.position() - lastLen - 16);
1301 currKeyLen = block.getInt();
1302 currValueLen = block.getInt();
1303 return 1;
1304 }
1305 block.position(block.position() + klen + vlen);
1306 lastLen = klen + vlen ;
1307 } while(block.remaining() > 0);
1308
1309
1310
1311 block.position(block.position() - lastLen - 8);
1312 currKeyLen = block.getInt();
1313 currValueLen = block.getInt();
1314 return 1;
1315 }
1316
1317 public boolean seekBefore(byte [] key) throws IOException {
1318 return seekBefore(key, 0, key.length);
1319 }
1320
1321 public boolean seekBefore(byte[] key, int offset, int length)
1322 throws IOException {
1323 int b = reader.blockContainingKey(key, offset, length);
1324 if (b < 0)
1325 return false;
1326
1327
1328 if (this.reader.comparator.compare(reader.blockIndex.blockKeys[b],
1329 0, reader.blockIndex.blockKeys[b].length,
1330 key, offset, length) == 0) {
1331
1332 if (b == 0) {
1333
1334 return false;
1335 }
1336 b--;
1337
1338 }
1339 loadBlock(b);
1340 blockSeek(key, offset, length, true);
1341 return true;
1342 }
1343
1344 public String getKeyString() {
1345 return Bytes.toStringBinary(block.array(), block.arrayOffset() +
1346 block.position(), currKeyLen);
1347 }
1348
1349 public String getValueString() {
1350 return Bytes.toString(block.array(), block.arrayOffset() +
1351 block.position() + currKeyLen, currValueLen);
1352 }
1353
1354 public Reader getReader() {
1355 return this.reader;
1356 }
1357
1358 public boolean isSeeked(){
1359 return this.block != null;
1360 }
1361
1362 public boolean seekTo() throws IOException {
1363 if (this.reader.blockIndex.isEmpty()) {
1364 return false;
1365 }
1366 if (block != null && currBlock == 0) {
1367 block.rewind();
1368 currKeyLen = block.getInt();
1369 currValueLen = block.getInt();
1370 return true;
1371 }
1372 currBlock = 0;
1373 block = reader.readBlock(this.currBlock, this.cacheBlocks, this.pread);
1374 currKeyLen = block.getInt();
1375 currValueLen = block.getInt();
1376 blockFetches++;
1377 return true;
1378 }
1379
1380 private void loadBlock(int bloc) throws IOException {
1381 if (block == null) {
1382 block = reader.readBlock(bloc, this.cacheBlocks, this.pread);
1383 currBlock = bloc;
1384 blockFetches++;
1385 } else {
1386 if (bloc != currBlock) {
1387 block = reader.readBlock(bloc, this.cacheBlocks, this.pread);
1388 currBlock = bloc;
1389 blockFetches++;
1390 } else {
1391
1392 block.rewind();
1393 }
1394 }
1395 }
1396
1397 @Override
1398 public String toString() {
1399 return "HFileScanner for reader " + String.valueOf(reader);
1400 }
1401 }
1402
1403 public String getTrailerInfo() {
1404 return trailer.toString();
1405 }
1406 }
1407
1408
1409
1410
1411
1412 private static class FixedFileTrailer {
1413
1414 long fileinfoOffset;
1415
1416 long dataIndexOffset;
1417
1418 int dataIndexCount;
1419
1420 long metaIndexOffset;
1421
1422 int metaIndexCount;
1423 long totalUncompressedBytes;
1424 int entryCount;
1425 int compressionCodec;
1426 int version = 1;
1427
1428 FixedFileTrailer() {
1429 super();
1430 }
1431
1432 static int trailerSize() {
1433
1434 return
1435 ( Bytes.SIZEOF_INT * 5 ) +
1436 ( Bytes.SIZEOF_LONG * 4 ) +
1437 TRAILERBLOCKMAGIC.length;
1438 }
1439
1440 void serialize(DataOutputStream outputStream) throws IOException {
1441 outputStream.write(TRAILERBLOCKMAGIC);
1442 outputStream.writeLong(fileinfoOffset);
1443 outputStream.writeLong(dataIndexOffset);
1444 outputStream.writeInt(dataIndexCount);
1445 outputStream.writeLong(metaIndexOffset);
1446 outputStream.writeInt(metaIndexCount);
1447 outputStream.writeLong(totalUncompressedBytes);
1448 outputStream.writeInt(entryCount);
1449 outputStream.writeInt(compressionCodec);
1450 outputStream.writeInt(version);
1451 }
1452
1453 void deserialize(DataInputStream inputStream) throws IOException {
1454 byte [] header = new byte[TRAILERBLOCKMAGIC.length];
1455 inputStream.readFully(header);
1456 if ( !Arrays.equals(header, TRAILERBLOCKMAGIC)) {
1457 throw new IOException("Trailer 'header' is wrong; does the trailer " +
1458 "size match content?");
1459 }
1460 fileinfoOffset = inputStream.readLong();
1461 dataIndexOffset = inputStream.readLong();
1462 dataIndexCount = inputStream.readInt();
1463
1464 metaIndexOffset = inputStream.readLong();
1465 metaIndexCount = inputStream.readInt();
1466
1467 totalUncompressedBytes = inputStream.readLong();
1468 entryCount = inputStream.readInt();
1469 compressionCodec = inputStream.readInt();
1470 version = inputStream.readInt();
1471
1472 if (version != 1) {
1473 throw new IOException("Wrong version: " + version);
1474 }
1475 }
1476
1477 @Override
1478 public String toString() {
1479 return "fileinfoOffset=" + fileinfoOffset +
1480 ", dataIndexOffset=" + dataIndexOffset +
1481 ", dataIndexCount=" + dataIndexCount +
1482 ", metaIndexOffset=" + metaIndexOffset +
1483 ", metaIndexCount=" + metaIndexCount +
1484 ", totalBytes=" + totalUncompressedBytes +
1485 ", entryCount=" + entryCount +
1486 ", version=" + version;
1487 }
1488 }
1489
1490
1491
1492
1493
1494 static class BlockIndex implements HeapSize {
1495
1496 int count = 0;
1497 byte [][] blockKeys;
1498 long [] blockOffsets;
1499 int [] blockDataSizes;
1500 int size = 0;
1501
1502
1503
1504 final RawComparator<byte []> comparator;
1505
1506
1507
1508
1509 @SuppressWarnings("unused")
1510 private BlockIndex() {
1511 this(null);
1512 }
1513
1514
1515
1516
1517
1518 BlockIndex(final RawComparator<byte []>c) {
1519 this.comparator = c;
1520
1521 this.size += (4 * 8);
1522 }
1523
1524
1525
1526
1527 boolean isEmpty() {
1528 return this.blockKeys.length <= 0;
1529 }
1530
1531
1532
1533
1534
1535
1536
1537
1538 void add(final byte[] key, final long offset, final int dataSize) {
1539 blockOffsets[count] = offset;
1540 blockKeys[count] = key;
1541 blockDataSizes[count] = dataSize;
1542 count++;
1543 this.size += (Bytes.SIZEOF_INT * 2 + key.length);
1544 }
1545
1546
1547
1548
1549
1550
1551 int blockContainingKey(final byte[] key, int offset, int length) {
1552 int pos = Bytes.binarySearch(blockKeys, key, offset, length, this.comparator);
1553 if (pos < 0) {
1554 pos ++;
1555 pos *= -1;
1556 if (pos == 0) {
1557
1558 return -1;
1559 }
1560
1561
1562
1563 pos --;
1564
1565 return pos;
1566 }
1567
1568 return pos;
1569 }
1570
1571
1572
1573
1574
1575 byte [] midkey() throws IOException {
1576 int pos = ((this.count - 1)/2);
1577 if (pos < 0) {
1578 throw new IOException("HFile empty");
1579 }
1580 return this.blockKeys[pos];
1581 }
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595 static long writeIndex(final FSDataOutputStream o,
1596 final List<byte []> keys, final List<Long> offsets,
1597 final List<Integer> sizes)
1598 throws IOException {
1599 long pos = o.getPos();
1600
1601 if (keys.size() > 0) {
1602 o.write(INDEXBLOCKMAGIC);
1603
1604 for (int i = 0; i < keys.size(); ++i) {
1605 o.writeLong(offsets.get(i).longValue());
1606 o.writeInt(sizes.get(i).intValue());
1607 byte [] key = keys.get(i);
1608 Bytes.writeByteArray(o, key);
1609 }
1610 }
1611 return pos;
1612 }
1613
1614
1615
1616
1617
1618
1619
1620
1621 static BlockIndex readIndex(final RawComparator<byte []> c,
1622 final FSDataInputStream in, final long indexOffset, final int indexSize)
1623 throws IOException {
1624 BlockIndex bi = new BlockIndex(c);
1625 bi.blockOffsets = new long[indexSize];
1626 bi.blockKeys = new byte[indexSize][];
1627 bi.blockDataSizes = new int[indexSize];
1628
1629 if (indexSize > 0) {
1630 in.seek(indexOffset);
1631 byte [] magic = new byte[INDEXBLOCKMAGIC.length];
1632 IOUtils.readFully(in, magic, 0, magic.length);
1633 if (!Arrays.equals(magic, INDEXBLOCKMAGIC)) {
1634 throw new IOException("Index block magic is wrong: " +
1635 Arrays.toString(magic));
1636 }
1637 for (int i = 0; i < indexSize; ++i ) {
1638 long offset = in.readLong();
1639 int dataSize = in.readInt();
1640 byte [] key = Bytes.readByteArray(in);
1641 bi.add(key, offset, dataSize);
1642 }
1643 }
1644 return bi;
1645 }
1646
1647 @Override
1648 public String toString() {
1649 StringBuilder sb = new StringBuilder();
1650 sb.append("size=" + count);
1651 for (int i = 0; i < count ; i++) {
1652 sb.append(", ");
1653 sb.append("key=").append(Bytes.toStringBinary(blockKeys[i])).
1654 append(", offset=").append(blockOffsets[i]).
1655 append(", dataSize=" + blockDataSizes[i]);
1656 }
1657 return sb.toString();
1658 }
1659
1660 public long heapSize() {
1661 long heapsize = ClassSize.align(ClassSize.OBJECT +
1662 2 * Bytes.SIZEOF_INT + (3 + 1) * ClassSize.REFERENCE);
1663
1664 if(blockKeys != null) {
1665
1666 heapsize += ClassSize.align(ClassSize.ARRAY +
1667 blockKeys.length * ClassSize.REFERENCE);
1668
1669 for(byte [] bs : blockKeys) {
1670 heapsize += ClassSize.align(ClassSize.ARRAY + bs.length);
1671 }
1672 }
1673 if(blockOffsets != null) {
1674 heapsize += ClassSize.align(ClassSize.ARRAY +
1675 blockOffsets.length * Bytes.SIZEOF_LONG);
1676 }
1677 if(blockDataSizes != null) {
1678 heapsize += ClassSize.align(ClassSize.ARRAY +
1679 blockDataSizes.length * Bytes.SIZEOF_INT);
1680 }
1681
1682 return ClassSize.align(heapsize);
1683 }
1684
1685 }
1686
1687
1688
1689
1690 static class FileInfo extends HbaseMapWritable<byte [], byte []> {
1691 static final String RESERVED_PREFIX = "hfile.";
1692 static final byte[] RESERVED_PREFIX_BYTES = Bytes.toBytes(RESERVED_PREFIX);
1693 static final byte [] LASTKEY = Bytes.toBytes(RESERVED_PREFIX + "LASTKEY");
1694 static final byte [] AVG_KEY_LEN =
1695 Bytes.toBytes(RESERVED_PREFIX + "AVG_KEY_LEN");
1696 static final byte [] AVG_VALUE_LEN =
1697 Bytes.toBytes(RESERVED_PREFIX + "AVG_VALUE_LEN");
1698 static final byte [] COMPARATOR =
1699 Bytes.toBytes(RESERVED_PREFIX + "COMPARATOR");
1700
1701
1702
1703
1704 FileInfo() {
1705 super();
1706 }
1707 }
1708
1709
1710
1711
1712
1713 public static boolean isReservedFileInfoKey(byte[] key) {
1714 return Bytes.startsWith(key, FileInfo.RESERVED_PREFIX_BYTES);
1715 }
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730 public static String[] getSupportedCompressionAlgorithms() {
1731 return Compression.getSupportedAlgorithms();
1732 }
1733
1734
1735
1736
1737
1738
1739 static int longToInt(final long l) {
1740
1741
1742 return (int)(l & 0x00000000ffffffffL);
1743 }
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754 static List<Path> getStoreFiles(FileSystem fs, Path regionDir)
1755 throws IOException {
1756 List<Path> res = new ArrayList<Path>();
1757 PathFilter dirFilter = new FSUtils.DirFilter(fs);
1758 FileStatus[] familyDirs = fs.listStatus(regionDir, dirFilter);
1759 for(FileStatus dir : familyDirs) {
1760 FileStatus[] files = fs.listStatus(dir.getPath());
1761 for (FileStatus file : files) {
1762 if (!file.isDir()) {
1763 res.add(file.getPath());
1764 }
1765 }
1766 }
1767 return res;
1768 }
1769
1770 public static void main(String []args) throws IOException {
1771 try {
1772
1773 Options options = new Options();
1774 options.addOption("v", "verbose", false, "Verbose output; emits file and meta data delimiters");
1775 options.addOption("p", "printkv", false, "Print key/value pairs");
1776 options.addOption("m", "printmeta", false, "Print meta data of file");
1777 options.addOption("k", "checkrow", false,
1778 "Enable row order check; looks for out-of-order keys");
1779 options.addOption("a", "checkfamily", false, "Enable family check");
1780 options.addOption("f", "file", true,
1781 "File to scan. Pass full-path; e.g. hdfs://a:9000/hbase/.META./12/34");
1782 options.addOption("r", "region", true,
1783 "Region to scan. Pass region name; e.g. '.META.,,1'");
1784 if (args.length == 0) {
1785 HelpFormatter formatter = new HelpFormatter();
1786 formatter.printHelp("HFile ", options, true);
1787 System.exit(-1);
1788 }
1789 CommandLineParser parser = new PosixParser();
1790 CommandLine cmd = parser.parse(options, args);
1791 boolean verbose = cmd.hasOption("v");
1792 boolean printKeyValue = cmd.hasOption("p");
1793 boolean printMeta = cmd.hasOption("m");
1794 boolean checkRow = cmd.hasOption("k");
1795 boolean checkFamily = cmd.hasOption("a");
1796
1797 Configuration conf = HBaseConfiguration.create();
1798 conf.set("fs.defaultFS",
1799 conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR));
1800 FileSystem fs = FileSystem.get(conf);
1801 ArrayList<Path> files = new ArrayList<Path>();
1802 if (cmd.hasOption("f")) {
1803 files.add(new Path(cmd.getOptionValue("f")));
1804 }
1805 if (cmd.hasOption("r")) {
1806 String regionName = cmd.getOptionValue("r");
1807 byte[] rn = Bytes.toBytes(regionName);
1808 byte[][] hri = HRegionInfo.parseRegionName(rn);
1809 Path rootDir = FSUtils.getRootDir(conf);
1810 Path tableDir = new Path(rootDir, Bytes.toString(hri[0]));
1811 String enc = HRegionInfo.encodeRegionName(rn);
1812 Path regionDir = new Path(tableDir, enc);
1813 if (verbose) System.out.println("region dir -> " + regionDir);
1814 List<Path> regionFiles = getStoreFiles(fs, regionDir);
1815 if (verbose) System.out.println("Number of region files found -> " +
1816 regionFiles.size());
1817 if (verbose) {
1818 int i = 1;
1819 for (Path p : regionFiles) {
1820 if (verbose) System.out.println("Found file[" + i++ + "] -> " + p);
1821 }
1822 }
1823 files.addAll(regionFiles);
1824 }
1825
1826 for (Path file : files) {
1827 if (verbose) System.out.println("Scanning -> " + file);
1828 if (!fs.exists(file)) {
1829 System.err.println("ERROR, file doesnt exist: " + file);
1830 continue;
1831 }
1832
1833 HFile.Reader reader = new HFile.Reader(fs, file, null, false);
1834 Map<byte[],byte[]> fileInfo = reader.loadFileInfo();
1835
1836 HFileScanner scanner = reader.getScanner(false, false);
1837 scanner.seekTo();
1838 KeyValue pkv = null;
1839 int count = 0;
1840 do {
1841 KeyValue kv = scanner.getKeyValue();
1842
1843 if (printKeyValue) {
1844 System.out.println("K: " + kv +
1845 " V: " + Bytes.toStringBinary(kv.getValue()));
1846 }
1847
1848 if (checkRow && pkv != null) {
1849 if (Bytes.compareTo(pkv.getRow(), kv.getRow()) > 0) {
1850 System.err.println("WARNING, previous row is greater then" +
1851 " current row\n\tfilename -> " + file +
1852 "\n\tprevious -> " + Bytes.toStringBinary(pkv.getKey()) +
1853 "\n\tcurrent -> " + Bytes.toStringBinary(kv.getKey()));
1854 }
1855 }
1856
1857 if (checkFamily) {
1858 String fam = Bytes.toString(kv.getFamily());
1859 if (!file.toString().contains(fam)) {
1860 System.err.println("WARNING, filename does not match kv family," +
1861 "\n\tfilename -> " + file +
1862 "\n\tkeyvalue -> " + Bytes.toStringBinary(kv.getKey()));
1863 }
1864 if (pkv != null && Bytes.compareTo(pkv.getFamily(), kv.getFamily()) != 0) {
1865 System.err.println("WARNING, previous kv has different family" +
1866 " compared to current key\n\tfilename -> " + file +
1867 "\n\tprevious -> " + Bytes.toStringBinary(pkv.getKey()) +
1868 "\n\tcurrent -> " + Bytes.toStringBinary(kv.getKey()));
1869 }
1870 }
1871 pkv = kv;
1872 count++;
1873 } while (scanner.next());
1874 if (verbose || printKeyValue) {
1875 System.out.println("Scanned kv count -> " + count);
1876 }
1877
1878 if (printMeta) {
1879 System.out.println("Block index size as per heapsize: " + reader.indexSize());
1880 System.out.println(reader.toString());
1881 System.out.println(reader.getTrailerInfo());
1882 System.out.println("Fileinfo:");
1883 for (Map.Entry<byte[], byte[]> e : fileInfo.entrySet()) {
1884 System.out.print(Bytes.toString(e.getKey()) + " = " );
1885 if (Bytes.compareTo(e.getKey(), Bytes.toBytes("MAX_SEQ_ID_KEY"))==0) {
1886 long seqid = Bytes.toLong(e.getValue());
1887 System.out.println(seqid);
1888 } else {
1889 System.out.println(Bytes.toStringBinary(e.getValue()));
1890 }
1891 }
1892 }
1893 reader.close();
1894 }
1895 } catch (Exception e) {
1896 e.printStackTrace();
1897 }
1898 }
1899 }