1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.DataInput;
21 import java.io.IOException;
22 import java.nio.ByteBuffer;
23 import java.util.ArrayList;
24 import java.util.List;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.classification.InterfaceAudience;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.KeyValue;
33 import org.apache.hadoop.hbase.KeyValue.KVComparator;
34 import org.apache.hadoop.hbase.fs.HFileSystem;
35 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
36 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
37 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
38 import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
39 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
40 import org.apache.hadoop.hbase.util.ByteBufferUtils;
41 import org.apache.hadoop.hbase.util.Bytes;
42 import org.apache.hadoop.hbase.util.IdLock;
43 import org.apache.hadoop.io.WritableUtils;
44 import org.cloudera.htrace.Trace;
45 import org.cloudera.htrace.TraceScope;
46
47
48
49
50 @InterfaceAudience.Private
51 public class HFileReaderV2 extends AbstractHFileReader {
52
53 private static final Log LOG = LogFactory.getLog(HFileReaderV2.class);
54
55
56 public static final int MINOR_VERSION_WITH_CHECKSUM = 1;
57
58 public static final int MINOR_VERSION_NO_CHECKSUM = 0;
59
60
61 public static final int PBUF_TRAILER_MINOR_VERSION = 2;
62
63
64
65
66
67 public final static int KEY_VALUE_LEN_SIZE = 2 * Bytes.SIZEOF_INT;
68
69 protected boolean includesMemstoreTS = false;
70 protected boolean decodeMemstoreTS = false;
71 protected boolean shouldIncludeMemstoreTS() {
72 return includesMemstoreTS;
73 }
74
75
76 protected HFileBlock.FSReader fsBlockReader;
77
78
79
80
81
82
83
84 private IdLock offsetLock = new IdLock();
85
86
87
88
89
90 private List<HFileBlock> loadOnOpenBlocks = new ArrayList<HFileBlock>();
91
92
93 static final int MIN_MINOR_VERSION = 0;
94
95
96
97
98 static final int MAX_MINOR_VERSION = 3;
99
100
101 static final int MINOR_VERSION_WITH_FAKED_KEY = 3;
102
103 protected HFileContext hfileContext;
104
105
106
107
108
109
110
111
112
113
114
115
116
117 public HFileReaderV2(Path path, FixedFileTrailer trailer,
118 final FSDataInputStreamWrapper fsdis, final long size, final CacheConfig cacheConf,
119 final HFileSystem hfs, final Configuration conf) throws IOException {
120 super(path, trailer, size, cacheConf, hfs, conf);
121 this.conf = conf;
122 trailer.expectMajorVersion(getMajorVersion());
123 validateMinorVersion(path, trailer.getMinorVersion());
124 this.hfileContext = createHFileContext(fsdis, fileSize, hfs, path, trailer);
125 HFileBlock.FSReaderV2 fsBlockReaderV2 = new HFileBlock.FSReaderV2(fsdis, fileSize, hfs, path,
126 hfileContext);
127 this.fsBlockReader = fsBlockReaderV2;
128
129
130 comparator = trailer.createComparator();
131 dataBlockIndexReader = new HFileBlockIndex.BlockIndexReader(comparator,
132 trailer.getNumDataIndexLevels(), this);
133 metaBlockIndexReader = new HFileBlockIndex.BlockIndexReader(
134 KeyValue.RAW_COMPARATOR, 1);
135
136
137
138 HFileBlock.BlockIterator blockIter = fsBlockReaderV2.blockRange(
139 trailer.getLoadOnOpenDataOffset(),
140 fileSize - trailer.getTrailerSize());
141
142
143
144 dataBlockIndexReader.readMultiLevelIndexRoot(
145 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
146 trailer.getDataIndexCount());
147
148
149 metaBlockIndexReader.readRootIndex(
150 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
151 trailer.getMetaIndexCount());
152
153
154 fileInfo = new FileInfo();
155 fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
156 lastKey = fileInfo.get(FileInfo.LASTKEY);
157 avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN));
158 avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN));
159 byte [] keyValueFormatVersion =
160 fileInfo.get(HFileWriterV2.KEY_VALUE_VERSION);
161 includesMemstoreTS = keyValueFormatVersion != null &&
162 Bytes.toInt(keyValueFormatVersion) ==
163 HFileWriterV2.KEY_VALUE_VER_WITH_MEMSTORE;
164 fsBlockReaderV2.setIncludesMemstoreTS(includesMemstoreTS);
165 if (includesMemstoreTS) {
166 decodeMemstoreTS = Bytes.toLong(fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY)) > 0;
167 }
168
169
170 dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo);
171 fsBlockReaderV2.setDataBlockEncoder(dataBlockEncoder);
172
173
174 HFileBlock b;
175 while ((b = blockIter.nextBlock()) != null) {
176 loadOnOpenBlocks.add(b);
177 }
178 }
179
180 protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
181 HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
182 return new HFileContextBuilder()
183 .withIncludesMvcc(this.includesMemstoreTS)
184 .withCompression(this.compressAlgo)
185 .withHBaseCheckSum(trailer.getMinorVersion() >= MINOR_VERSION_WITH_CHECKSUM)
186 .build();
187 }
188
189
190
191
192
193
194
195
196
197
198
199
200
201 @Override
202 public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
203 final boolean isCompaction) {
204 if (dataBlockEncoder.useEncodedScanner()) {
205 return new EncodedScannerV2(this, cacheBlocks, pread, isCompaction,
206 hfileContext);
207 }
208
209 return new ScannerV2(this, cacheBlocks, pread, isCompaction);
210 }
211
212
213
214
215
216
217
218 @Override
219 public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock)
220 throws IOException {
221 if (trailer.getMetaIndexCount() == 0) {
222 return null;
223 }
224 if (metaBlockIndexReader == null) {
225 throw new IOException("Meta index not loaded");
226 }
227
228 byte[] mbname = Bytes.toBytes(metaBlockName);
229 int block = metaBlockIndexReader.rootBlockContainingKey(mbname, 0,
230 mbname.length);
231 if (block == -1)
232 return null;
233 long blockSize = metaBlockIndexReader.getRootBlockDataSize(block);
234 long startTimeNs = System.nanoTime();
235
236
237
238
239 synchronized (metaBlockIndexReader.getRootBlockKey(block)) {
240
241 long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block);
242 BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset,
243 DataBlockEncoding.NONE, BlockType.META);
244
245 cacheBlock &= cacheConf.shouldCacheDataOnRead();
246 if (cacheConf.isBlockCacheEnabled()) {
247 HFileBlock cachedBlock =
248 (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey, cacheBlock, false);
249 if (cachedBlock != null) {
250
251
252 return cachedBlock.getBufferWithoutHeader();
253 }
254
255 }
256
257 HFileBlock metaBlock = fsBlockReader.readBlockData(metaBlockOffset,
258 blockSize, -1, true);
259
260 final long delta = System.nanoTime() - startTimeNs;
261 HFile.offerReadLatency(delta, true);
262
263
264 if (cacheBlock) {
265 cacheConf.getBlockCache().cacheBlock(cacheKey, metaBlock,
266 cacheConf.isInMemory());
267 }
268
269 return metaBlock.getBufferWithoutHeader();
270 }
271 }
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288 @Override
289 public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize,
290 final boolean cacheBlock, boolean pread, final boolean isCompaction,
291 BlockType expectedBlockType)
292 throws IOException {
293 if (dataBlockIndexReader == null) {
294 throw new IOException("Block index not loaded");
295 }
296 if (dataBlockOffset < 0
297 || dataBlockOffset >= trailer.getLoadOnOpenDataOffset()) {
298 throw new IOException("Requested block is out of range: "
299 + dataBlockOffset + ", lastDataBlockOffset: "
300 + trailer.getLastDataBlockOffset());
301 }
302
303
304
305
306
307
308 BlockCacheKey cacheKey =
309 new BlockCacheKey(name, dataBlockOffset,
310 dataBlockEncoder.getDataBlockEncoding(),
311 expectedBlockType);
312
313 boolean useLock = false;
314 IdLock.Entry lockEntry = null;
315 TraceScope traceScope = Trace.startSpan("HFileReaderV2.readBlock");
316 try {
317 while (true) {
318 if (useLock) {
319 lockEntry = offsetLock.getLockEntry(dataBlockOffset);
320 }
321
322
323 if (cacheConf.isBlockCacheEnabled()) {
324
325
326 HFileBlock cachedBlock = (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey,
327 cacheBlock, useLock);
328 if (cachedBlock != null) {
329 validateBlockType(cachedBlock, expectedBlockType);
330 if (cachedBlock.getBlockType().isData()) {
331 HFile.dataBlockReadCnt.incrementAndGet();
332
333
334
335 if (cachedBlock.getDataBlockEncoding() != dataBlockEncoder.getDataBlockEncoding()) {
336 throw new IOException("Cached block under key " + cacheKey + " "
337 + "has wrong encoding: " + cachedBlock.getDataBlockEncoding() + " (expected: "
338 + dataBlockEncoder.getDataBlockEncoding() + ")");
339 }
340 }
341 return cachedBlock;
342 }
343
344 }
345 if (!useLock) {
346
347 useLock = true;
348 continue;
349 }
350 if (Trace.isTracing()) {
351 traceScope.getSpan().addTimelineAnnotation("blockCacheMiss");
352 }
353
354 long startTimeNs = System.nanoTime();
355 HFileBlock hfileBlock = fsBlockReader.readBlockData(dataBlockOffset, onDiskBlockSize, -1,
356 pread);
357 validateBlockType(hfileBlock, expectedBlockType);
358
359 final long delta = System.nanoTime() - startTimeNs;
360 HFile.offerReadLatency(delta, pread);
361
362
363 if (cacheBlock && cacheConf.shouldCacheBlockOnRead(hfileBlock.getBlockType().getCategory())) {
364 cacheConf.getBlockCache().cacheBlock(cacheKey, hfileBlock, cacheConf.isInMemory());
365 }
366
367 if (hfileBlock.getBlockType().isData()) {
368 HFile.dataBlockReadCnt.incrementAndGet();
369 }
370
371 return hfileBlock;
372 }
373 } finally {
374 traceScope.close();
375 if (lockEntry != null) {
376 offsetLock.releaseLockEntry(lockEntry);
377 }
378 }
379 }
380
381 @Override
382 public boolean hasMVCCInfo() {
383 return includesMemstoreTS && decodeMemstoreTS;
384 }
385
386
387
388
389
390
391
392
393
394
395 private void validateBlockType(HFileBlock block,
396 BlockType expectedBlockType) throws IOException {
397 if (expectedBlockType == null) {
398 return;
399 }
400 BlockType actualBlockType = block.getBlockType();
401 if (actualBlockType == BlockType.ENCODED_DATA &&
402 expectedBlockType == BlockType.DATA) {
403
404
405 return;
406 }
407 if (actualBlockType != expectedBlockType) {
408 throw new IOException("Expected block type " + expectedBlockType + ", " +
409 "but got " + actualBlockType + ": " + block);
410 }
411 }
412
413
414
415
416
417
418 @Override
419 public byte[] getLastKey() {
420 return dataBlockIndexReader.isEmpty() ? null : lastKey;
421 }
422
423
424
425
426
427
428 @Override
429 public byte[] midkey() throws IOException {
430 return dataBlockIndexReader.midkey();
431 }
432
433 @Override
434 public void close() throws IOException {
435 close(cacheConf.shouldEvictOnClose());
436 }
437
438 public void close(boolean evictOnClose) throws IOException {
439 if (evictOnClose && cacheConf.isBlockCacheEnabled()) {
440 int numEvicted = cacheConf.getBlockCache().evictBlocksByHfileName(name);
441 if (LOG.isTraceEnabled()) {
442 LOG.trace("On close, file=" + name + " evicted=" + numEvicted
443 + " block(s)");
444 }
445 }
446 fsBlockReader.closeStreams();
447 }
448
449
450 @Override
451 HFileBlock.FSReader getUncachedBlockReader() {
452 return fsBlockReader;
453 }
454
455
456 protected abstract static class AbstractScannerV2
457 extends AbstractHFileReader.Scanner {
458 protected HFileBlock block;
459
460
461
462
463
464
465
466
467 protected byte[] nextIndexedKey;
468
469 public AbstractScannerV2(HFileReaderV2 r, boolean cacheBlocks,
470 final boolean pread, final boolean isCompaction) {
471 super(r, cacheBlocks, pread, isCompaction);
472 }
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490 protected int seekTo(byte[] key, int offset, int length, boolean rewind)
491 throws IOException {
492 HFileBlockIndex.BlockIndexReader indexReader =
493 reader.getDataBlockIndexReader();
494 BlockWithScanInfo blockWithScanInfo =
495 indexReader.loadDataBlockWithScanInfo(key, offset, length, block,
496 cacheBlocks, pread, isCompaction);
497 if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) {
498
499 return -1;
500 }
501 return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(),
502 blockWithScanInfo.getNextIndexedKey(), rewind, key, offset, length, false);
503 }
504
505 protected abstract ByteBuffer getFirstKeyInBlock(HFileBlock curBlock);
506
507 protected abstract int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
508 boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
509 throws IOException;
510
511 @Override
512 public int seekTo(byte[] key, int offset, int length) throws IOException {
513
514
515 return seekTo(key, offset, length, true);
516 }
517
518 @Override
519 public int reseekTo(byte[] key, int offset, int length) throws IOException {
520 int compared;
521 if (isSeeked()) {
522 compared = compareKey(reader.getComparator(), key, offset, length);
523 if (compared < 1) {
524
525
526 return compared;
527 } else {
528 if (this.nextIndexedKey != null &&
529 (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY ||
530 reader.getComparator().compareFlatKey(key, offset, length,
531 nextIndexedKey, 0, nextIndexedKey.length) < 0)) {
532
533
534
535 return loadBlockAndSeekToKey(this.block, this.nextIndexedKey,
536 false, key, offset, length, false);
537 }
538 }
539 }
540
541
542 return seekTo(key, offset, length, false);
543 }
544
545 @Override
546 public boolean seekBefore(byte[] key, int offset, int length)
547 throws IOException {
548 HFileBlock seekToBlock =
549 reader.getDataBlockIndexReader().seekToDataBlock(key, offset, length,
550 block, cacheBlocks, pread, isCompaction);
551 if (seekToBlock == null) {
552 return false;
553 }
554 ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock);
555
556 if (reader.getComparator().compareFlatKey(firstKey.array(),
557 firstKey.arrayOffset(), firstKey.limit(), key, offset, length) >= 0)
558 {
559 long previousBlockOffset = seekToBlock.getPrevBlockOffset();
560
561 if (previousBlockOffset == -1) {
562
563 return false;
564 }
565
566
567
568
569 seekToBlock = reader.readBlock(previousBlockOffset,
570 seekToBlock.getOffset() - previousBlockOffset, cacheBlocks,
571 pread, isCompaction, BlockType.DATA);
572
573
574 }
575 byte[] firstKeyInCurrentBlock = Bytes.getBytes(firstKey);
576 loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, offset, length, true);
577 return true;
578 }
579
580
581
582
583
584
585
586
587
588 protected HFileBlock readNextDataBlock() throws IOException {
589 long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset();
590 if (block == null)
591 return null;
592
593 HFileBlock curBlock = block;
594
595 do {
596 if (curBlock.getOffset() >= lastDataBlockOffset)
597 return null;
598
599 if (curBlock.getOffset() < 0) {
600 throw new IOException("Invalid block file offset: " + block);
601 }
602
603
604
605 curBlock = reader.readBlock(curBlock.getOffset()
606 + curBlock.getOnDiskSizeWithHeader(),
607 curBlock.getNextBlockOnDiskSizeWithHeader(), cacheBlocks, pread,
608 isCompaction, null);
609 } while (!curBlock.getBlockType().isData());
610
611 return curBlock;
612 }
613
614
615
616
617
618
619
620
621 public abstract int compareKey(KVComparator comparator, byte[] key, int offset,
622 int length);
623 }
624
625
626
627
628 protected static class ScannerV2 extends AbstractScannerV2 {
629 private HFileReaderV2 reader;
630
631 public ScannerV2(HFileReaderV2 r, boolean cacheBlocks,
632 final boolean pread, final boolean isCompaction) {
633 super(r, cacheBlocks, pread, isCompaction);
634 this.reader = r;
635 }
636
637 @Override
638 public KeyValue getKeyValue() {
639 if (!isSeeked())
640 return null;
641
642 KeyValue ret = new KeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
643 + blockBuffer.position(), getCellBufSize());
644 if (this.reader.shouldIncludeMemstoreTS()) {
645 ret.setMvccVersion(currMemstoreTS);
646 }
647 return ret;
648 }
649
650 protected int getCellBufSize() {
651 return KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen;
652 }
653
654 @Override
655 public ByteBuffer getKey() {
656 assertSeeked();
657 return ByteBuffer.wrap(
658 blockBuffer.array(),
659 blockBuffer.arrayOffset() + blockBuffer.position()
660 + KEY_VALUE_LEN_SIZE, currKeyLen).slice();
661 }
662
663 @Override
664 public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
665 return comparator.compareFlatKey(key, offset, length, blockBuffer.array(),
666 blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen);
667 }
668
669 @Override
670 public ByteBuffer getValue() {
671 assertSeeked();
672 return ByteBuffer.wrap(
673 blockBuffer.array(),
674 blockBuffer.arrayOffset() + blockBuffer.position()
675 + KEY_VALUE_LEN_SIZE + currKeyLen, currValueLen).slice();
676 }
677
678 protected void setNonSeekedState() {
679 block = null;
680 blockBuffer = null;
681 currKeyLen = 0;
682 currValueLen = 0;
683 currMemstoreTS = 0;
684 currMemstoreTSLen = 0;
685 }
686
687
688
689
690
691
692
693
694 @Override
695 public boolean next() throws IOException {
696 assertSeeked();
697
698 try {
699 blockBuffer.position(getNextCellStartPosition());
700 } catch (IllegalArgumentException e) {
701 LOG.error("Current pos = " + blockBuffer.position()
702 + "; currKeyLen = " + currKeyLen + "; currValLen = "
703 + currValueLen + "; block limit = " + blockBuffer.limit()
704 + "; HFile name = " + reader.getName()
705 + "; currBlock currBlockOffset = " + block.getOffset());
706 throw e;
707 }
708
709 if (blockBuffer.remaining() <= 0) {
710 long lastDataBlockOffset =
711 reader.getTrailer().getLastDataBlockOffset();
712
713 if (block.getOffset() >= lastDataBlockOffset) {
714 setNonSeekedState();
715 return false;
716 }
717
718
719 HFileBlock nextBlock = readNextDataBlock();
720 if (nextBlock == null) {
721 setNonSeekedState();
722 return false;
723 }
724
725 updateCurrBlock(nextBlock);
726 return true;
727 }
728
729
730 readKeyValueLen();
731 return true;
732 }
733
734 protected int getNextCellStartPosition() {
735 return blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen
736 + currMemstoreTSLen;
737 }
738
739
740
741
742
743
744
745
746 @Override
747 public boolean seekTo() throws IOException {
748 if (reader == null) {
749 return false;
750 }
751
752 if (reader.getTrailer().getEntryCount() == 0) {
753
754 return false;
755 }
756
757 long firstDataBlockOffset =
758 reader.getTrailer().getFirstDataBlockOffset();
759 if (block != null && block.getOffset() == firstDataBlockOffset) {
760 blockBuffer.rewind();
761 readKeyValueLen();
762 return true;
763 }
764
765 block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
766 isCompaction, BlockType.DATA);
767 if (block.getOffset() < 0) {
768 throw new IOException("Invalid block offset: " + block.getOffset());
769 }
770 updateCurrBlock(block);
771 return true;
772 }
773
774 @Override
775 protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
776 boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
777 throws IOException {
778 if (block == null || block.getOffset() != seekToBlock.getOffset()) {
779 updateCurrBlock(seekToBlock);
780 } else if (rewind) {
781 blockBuffer.rewind();
782 }
783
784
785 this.nextIndexedKey = nextIndexedKey;
786 return blockSeek(key, offset, length, seekBefore);
787 }
788
789
790
791
792
793
794
795 protected void updateCurrBlock(HFileBlock newBlock) {
796 block = newBlock;
797
798
799 if (block.getBlockType() != BlockType.DATA) {
800 throw new IllegalStateException("ScannerV2 works only on data " +
801 "blocks, got " + block.getBlockType() + "; " +
802 "fileName=" + reader.name + ", " +
803 "dataBlockEncoder=" + reader.dataBlockEncoder + ", " +
804 "isCompaction=" + isCompaction);
805 }
806
807 blockBuffer = block.getBufferWithoutHeader();
808 readKeyValueLen();
809 blockFetches++;
810
811
812 this.nextIndexedKey = null;
813 }
814
815 protected void readKeyValueLen() {
816 blockBuffer.mark();
817 currKeyLen = blockBuffer.getInt();
818 currValueLen = blockBuffer.getInt();
819 ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
820 readMvccVersion();
821 if (currKeyLen < 0 || currValueLen < 0
822 || currKeyLen > blockBuffer.limit()
823 || currValueLen > blockBuffer.limit()) {
824 throw new IllegalStateException("Invalid currKeyLen " + currKeyLen
825 + " or currValueLen " + currValueLen + ". Block offset: "
826 + block.getOffset() + ", block length: " + blockBuffer.limit()
827 + ", position: " + blockBuffer.position() + " (without header).");
828 }
829 blockBuffer.reset();
830 }
831
832 protected void readMvccVersion() {
833 if (this.reader.shouldIncludeMemstoreTS()) {
834 if (this.reader.decodeMemstoreTS) {
835 try {
836 currMemstoreTS = Bytes.readVLong(blockBuffer.array(), blockBuffer.arrayOffset()
837 + blockBuffer.position());
838 currMemstoreTSLen = WritableUtils.getVIntSize(currMemstoreTS);
839 } catch (Exception e) {
840 throw new RuntimeException("Error reading memstore timestamp", e);
841 }
842 } else {
843 currMemstoreTS = 0;
844 currMemstoreTSLen = 1;
845 }
846 }
847 }
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864 protected int blockSeek(byte[] key, int offset, int length,
865 boolean seekBefore) {
866 int klen, vlen;
867 long memstoreTS = 0;
868 int memstoreTSLen = 0;
869 int lastKeyValueSize = -1;
870 do {
871 blockBuffer.mark();
872 klen = blockBuffer.getInt();
873 vlen = blockBuffer.getInt();
874 blockBuffer.reset();
875 if (this.reader.shouldIncludeMemstoreTS()) {
876 if (this.reader.decodeMemstoreTS) {
877 try {
878 int memstoreTSOffset = blockBuffer.arrayOffset()
879 + blockBuffer.position() + KEY_VALUE_LEN_SIZE + klen + vlen;
880 memstoreTS = Bytes.readVLong(blockBuffer.array(),
881 memstoreTSOffset);
882 memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
883 } catch (Exception e) {
884 throw new RuntimeException("Error reading memstore timestamp", e);
885 }
886 } else {
887 memstoreTS = 0;
888 memstoreTSLen = 1;
889 }
890 }
891
892 int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position()
893 + KEY_VALUE_LEN_SIZE;
894 int comp = reader.getComparator().compareFlatKey(key, offset, length,
895 blockBuffer.array(), keyOffset, klen);
896
897 if (comp == 0) {
898 if (seekBefore) {
899 if (lastKeyValueSize < 0) {
900 throw new IllegalStateException("blockSeek with seekBefore "
901 + "at the first key of the block: key="
902 + Bytes.toStringBinary(key) + ", blockOffset="
903 + block.getOffset() + ", onDiskSize="
904 + block.getOnDiskSizeWithHeader());
905 }
906 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
907 readKeyValueLen();
908 return 1;
909 }
910 currKeyLen = klen;
911 currValueLen = vlen;
912 if (this.reader.shouldIncludeMemstoreTS()) {
913 currMemstoreTS = memstoreTS;
914 currMemstoreTSLen = memstoreTSLen;
915 }
916 return 0;
917 } else if (comp < 0) {
918 if (lastKeyValueSize > 0)
919 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
920 readKeyValueLen();
921 if (lastKeyValueSize == -1 && blockBuffer.position() == 0
922 && this.reader.trailer.getMinorVersion() >= MINOR_VERSION_WITH_FAKED_KEY) {
923 return HConstants.INDEX_KEY_MAGIC;
924 }
925 return 1;
926 }
927
928
929 lastKeyValueSize = klen + vlen + memstoreTSLen + KEY_VALUE_LEN_SIZE;
930 blockBuffer.position(blockBuffer.position() + lastKeyValueSize);
931 } while (blockBuffer.remaining() > 0);
932
933
934
935
936 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
937 readKeyValueLen();
938 return 1;
939 }
940
941 @Override
942 protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
943 ByteBuffer buffer = curBlock.getBufferWithoutHeader();
944
945 buffer.rewind();
946 int klen = buffer.getInt();
947 buffer.getInt();
948 ByteBuffer keyBuff = buffer.slice();
949 keyBuff.limit(klen);
950 keyBuff.rewind();
951 return keyBuff;
952 }
953
954 @Override
955 public String getKeyString() {
956 return Bytes.toStringBinary(blockBuffer.array(),
957 blockBuffer.arrayOffset() + blockBuffer.position()
958 + KEY_VALUE_LEN_SIZE, currKeyLen);
959 }
960
961 @Override
962 public String getValueString() {
963 return Bytes.toString(blockBuffer.array(), blockBuffer.arrayOffset()
964 + blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen,
965 currValueLen);
966 }
967 }
968
969
970
971
972 protected static class EncodedScannerV2 extends AbstractScannerV2 {
973 private final HFileBlockDecodingContext decodingCtx;
974 private final DataBlockEncoder.EncodedSeeker seeker;
975 private final DataBlockEncoder dataBlockEncoder;
976 protected final HFileContext meta;
977
978 public EncodedScannerV2(HFileReaderV2 reader, boolean cacheBlocks,
979 boolean pread, boolean isCompaction, HFileContext meta) {
980 super(reader, cacheBlocks, pread, isCompaction);
981 DataBlockEncoding encoding = reader.dataBlockEncoder.getDataBlockEncoding();
982 dataBlockEncoder = encoding.getEncoder();
983 decodingCtx = dataBlockEncoder.newDataBlockDecodingContext(meta);
984 seeker = dataBlockEncoder.createSeeker(
985 reader.getComparator(), decodingCtx);
986 this.meta = meta;
987 }
988
989 @Override
990 public boolean isSeeked(){
991 return this.block != null;
992 }
993
994
995
996
997
998
999
1000
1001 private void updateCurrentBlock(HFileBlock newBlock) throws CorruptHFileException {
1002 block = newBlock;
1003
1004
1005 if (block.getBlockType() != BlockType.ENCODED_DATA) {
1006 throw new IllegalStateException(
1007 "EncodedScanner works only on encoded data blocks");
1008 }
1009 short dataBlockEncoderId = block.getDataBlockEncodingId();
1010 if (!DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) {
1011 String encoderCls = dataBlockEncoder.getClass().getName();
1012 throw new CorruptHFileException("Encoder " + encoderCls
1013 + " doesn't support data block encoding "
1014 + DataBlockEncoding.getNameFromId(dataBlockEncoderId));
1015 }
1016
1017 seeker.setCurrentBuffer(getEncodedBuffer(newBlock));
1018 blockFetches++;
1019
1020
1021 this.nextIndexedKey = null;
1022 }
1023
1024 private ByteBuffer getEncodedBuffer(HFileBlock newBlock) {
1025 ByteBuffer origBlock = newBlock.getBufferReadOnly();
1026 ByteBuffer encodedBlock = ByteBuffer.wrap(origBlock.array(),
1027 origBlock.arrayOffset() + newBlock.headerSize() +
1028 DataBlockEncoding.ID_SIZE,
1029 newBlock.getUncompressedSizeWithoutHeader() -
1030 DataBlockEncoding.ID_SIZE).slice();
1031 return encodedBlock;
1032 }
1033
1034 @Override
1035 public boolean seekTo() throws IOException {
1036 if (reader == null) {
1037 return false;
1038 }
1039
1040 if (reader.getTrailer().getEntryCount() == 0) {
1041
1042 return false;
1043 }
1044
1045 long firstDataBlockOffset =
1046 reader.getTrailer().getFirstDataBlockOffset();
1047 if (block != null && block.getOffset() == firstDataBlockOffset) {
1048 seeker.rewind();
1049 return true;
1050 }
1051
1052 block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
1053 isCompaction, BlockType.DATA);
1054 if (block.getOffset() < 0) {
1055 throw new IOException("Invalid block offset: " + block.getOffset());
1056 }
1057 updateCurrentBlock(block);
1058 return true;
1059 }
1060
1061 @Override
1062 public boolean next() throws IOException {
1063 boolean isValid = seeker.next();
1064 if (!isValid) {
1065 block = readNextDataBlock();
1066 isValid = block != null;
1067 if (isValid) {
1068 updateCurrentBlock(block);
1069 }
1070 }
1071 return isValid;
1072 }
1073
1074 @Override
1075 public ByteBuffer getKey() {
1076 assertValidSeek();
1077 return seeker.getKeyDeepCopy();
1078 }
1079
1080 @Override
1081 public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
1082 return seeker.compareKey(comparator, key, offset, length);
1083 }
1084
1085 @Override
1086 public ByteBuffer getValue() {
1087 assertValidSeek();
1088 return seeker.getValueShallowCopy();
1089 }
1090
1091 @Override
1092 public KeyValue getKeyValue() {
1093 if (block == null) {
1094 return null;
1095 }
1096 return seeker.getKeyValue();
1097 }
1098
1099 @Override
1100 public String getKeyString() {
1101 ByteBuffer keyBuffer = getKey();
1102 return Bytes.toStringBinary(keyBuffer.array(),
1103 keyBuffer.arrayOffset(), keyBuffer.limit());
1104 }
1105
1106 @Override
1107 public String getValueString() {
1108 ByteBuffer valueBuffer = getValue();
1109 return Bytes.toStringBinary(valueBuffer.array(),
1110 valueBuffer.arrayOffset(), valueBuffer.limit());
1111 }
1112
1113 private void assertValidSeek() {
1114 if (block == null) {
1115 throw new NotSeekedException();
1116 }
1117 }
1118
1119 @Override
1120 protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
1121 return dataBlockEncoder.getFirstKeyInBlock(getEncodedBuffer(curBlock));
1122 }
1123
1124 @Override
1125 protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
1126 boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
1127 throws IOException {
1128 if (block == null || block.getOffset() != seekToBlock.getOffset()) {
1129 updateCurrentBlock(seekToBlock);
1130 } else if (rewind) {
1131 seeker.rewind();
1132 }
1133 this.nextIndexedKey = nextIndexedKey;
1134 return seeker.seekToKeyInBlock(key, offset, length, seekBefore);
1135 }
1136 }
1137
1138
1139
1140
1141
1142 @Override
1143 public DataInput getGeneralBloomFilterMetadata() throws IOException {
1144 return this.getBloomFilterMetadata(BlockType.GENERAL_BLOOM_META);
1145 }
1146
1147 @Override
1148 public DataInput getDeleteBloomFilterMetadata() throws IOException {
1149 return this.getBloomFilterMetadata(BlockType.DELETE_FAMILY_BLOOM_META);
1150 }
1151
1152 private DataInput getBloomFilterMetadata(BlockType blockType)
1153 throws IOException {
1154 if (blockType != BlockType.GENERAL_BLOOM_META &&
1155 blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
1156 throw new RuntimeException("Block Type: " + blockType.toString() +
1157 " is not supported") ;
1158 }
1159
1160 for (HFileBlock b : loadOnOpenBlocks)
1161 if (b.getBlockType() == blockType)
1162 return b.getByteStream();
1163 return null;
1164 }
1165
1166 @Override
1167 public boolean isFileInfoLoaded() {
1168 return true;
1169 }
1170
1171
1172
1173
1174
1175 private void validateMinorVersion(Path path, int minorVersion) {
1176 if (minorVersion < MIN_MINOR_VERSION ||
1177 minorVersion > MAX_MINOR_VERSION) {
1178 String msg = "Minor version for path " + path +
1179 " is expected to be between " +
1180 MIN_MINOR_VERSION + " and " + MAX_MINOR_VERSION +
1181 " but is found to be " + minorVersion;
1182 LOG.error(msg);
1183 throw new RuntimeException(msg);
1184 }
1185 }
1186
1187 @Override
1188 public int getMajorVersion() {
1189 return 2;
1190 }
1191
1192 @Override
1193 public HFileContext getFileContext() {
1194 return hfileContext;
1195 }
1196 }