1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.DataInput;
21 import java.io.IOException;
22 import java.nio.ByteBuffer;
23 import java.util.ArrayList;
24 import java.util.List;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.classification.InterfaceAudience;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.KeyValue;
33 import org.apache.hadoop.hbase.KeyValue.KVComparator;
34 import org.apache.hadoop.hbase.fs.HFileSystem;
35 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
36 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
37 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
38 import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
39 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
40 import org.apache.hadoop.hbase.util.ByteBufferUtils;
41 import org.apache.hadoop.hbase.util.Bytes;
42 import org.apache.hadoop.hbase.util.IdLock;
43 import org.apache.hadoop.io.WritableUtils;
44 import org.cloudera.htrace.Trace;
45 import org.cloudera.htrace.TraceScope;
46
47 import com.google.common.annotations.VisibleForTesting;
48
49
50
51
52 @InterfaceAudience.Private
53 public class HFileReaderV2 extends AbstractHFileReader {
54
55 private static final Log LOG = LogFactory.getLog(HFileReaderV2.class);
56
57
58 public static final int MINOR_VERSION_WITH_CHECKSUM = 1;
59
60 public static final int MINOR_VERSION_NO_CHECKSUM = 0;
61
62
63 public static final int PBUF_TRAILER_MINOR_VERSION = 2;
64
65
66
67
68
69 public final static int KEY_VALUE_LEN_SIZE = 2 * Bytes.SIZEOF_INT;
70
71 protected boolean includesMemstoreTS = false;
72 protected boolean decodeMemstoreTS = false;
73 protected boolean shouldIncludeMemstoreTS() {
74 return includesMemstoreTS;
75 }
76
77
78 protected HFileBlock.FSReader fsBlockReader;
79
80
81
82
83
84
85
86 private IdLock offsetLock = new IdLock();
87
88
89
90
91
92 private List<HFileBlock> loadOnOpenBlocks = new ArrayList<HFileBlock>();
93
94
95 static final int MIN_MINOR_VERSION = 0;
96
97
98
99
100 static final int MAX_MINOR_VERSION = 3;
101
102
103 static final int MINOR_VERSION_WITH_FAKED_KEY = 3;
104
105 protected HFileContext hfileContext;
106
107
108
109
110
111
112
113
114
115
116
117
118
119 public HFileReaderV2(final Path path, final FixedFileTrailer trailer,
120 final FSDataInputStreamWrapper fsdis, final long size, final CacheConfig cacheConf,
121 final HFileSystem hfs, final Configuration conf) throws IOException {
122 super(path, trailer, size, cacheConf, hfs, conf);
123 this.conf = conf;
124 trailer.expectMajorVersion(getMajorVersion());
125 validateMinorVersion(path, trailer.getMinorVersion());
126 this.hfileContext = createHFileContext(fsdis, fileSize, hfs, path, trailer);
127 HFileBlock.FSReaderV2 fsBlockReaderV2 = new HFileBlock.FSReaderV2(fsdis, fileSize, hfs, path,
128 hfileContext);
129 this.fsBlockReader = fsBlockReaderV2;
130
131
132 comparator = trailer.createComparator();
133 dataBlockIndexReader = new HFileBlockIndex.BlockIndexReader(comparator,
134 trailer.getNumDataIndexLevels(), this);
135 metaBlockIndexReader = new HFileBlockIndex.BlockIndexReader(
136 KeyValue.RAW_COMPARATOR, 1);
137
138
139
140 HFileBlock.BlockIterator blockIter = fsBlockReaderV2.blockRange(
141 trailer.getLoadOnOpenDataOffset(),
142 fileSize - trailer.getTrailerSize());
143
144
145
146 dataBlockIndexReader.readMultiLevelIndexRoot(
147 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
148 trailer.getDataIndexCount());
149
150
151 metaBlockIndexReader.readRootIndex(
152 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
153 trailer.getMetaIndexCount());
154
155
156 fileInfo = new FileInfo();
157 fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
158 lastKey = fileInfo.get(FileInfo.LASTKEY);
159 avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN));
160 avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN));
161 byte [] keyValueFormatVersion =
162 fileInfo.get(HFileWriterV2.KEY_VALUE_VERSION);
163 includesMemstoreTS = keyValueFormatVersion != null &&
164 Bytes.toInt(keyValueFormatVersion) ==
165 HFileWriterV2.KEY_VALUE_VER_WITH_MEMSTORE;
166 fsBlockReaderV2.setIncludesMemstoreTS(includesMemstoreTS);
167 if (includesMemstoreTS) {
168 decodeMemstoreTS = Bytes.toLong(fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY)) > 0;
169 }
170
171
172 dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo);
173 fsBlockReaderV2.setDataBlockEncoder(dataBlockEncoder);
174
175
176 HFileBlock b;
177 while ((b = blockIter.nextBlock()) != null) {
178 loadOnOpenBlocks.add(b);
179 }
180
181
182 if (cacheConf.shouldPrefetchOnOpen()) {
183 PrefetchExecutor.request(path, new Runnable() {
184 public void run() {
185 try {
186 long offset = 0;
187 long end = fileSize - getTrailer().getTrailerSize();
188 HFileBlock prevBlock = null;
189 while (offset < end) {
190 if (Thread.interrupted()) {
191 break;
192 }
193 long onDiskSize = -1;
194 if (prevBlock != null) {
195 onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
196 }
197 HFileBlock block = readBlock(offset, onDiskSize, true, false, false, false, null);
198 prevBlock = block;
199 offset += block.getOnDiskSizeWithHeader();
200 }
201 } catch (IOException e) {
202
203 if (LOG.isTraceEnabled()) {
204 LOG.trace("Exception encountered while prefetching " + path + ":", e);
205 }
206 } catch (Exception e) {
207
208 LOG.warn("Exception encountered while prefetching " + path + ":", e);
209 } finally {
210 PrefetchExecutor.complete(path);
211 }
212 }
213 });
214 }
215 }
216
217 protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
218 HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
219 return new HFileContextBuilder()
220 .withIncludesMvcc(this.includesMemstoreTS)
221 .withCompression(this.compressAlgo)
222 .withHBaseCheckSum(trailer.getMinorVersion() >= MINOR_VERSION_WITH_CHECKSUM)
223 .build();
224 }
225
226
227
228
229
230
231
232
233
234
235
236
237
238 @Override
239 public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
240 final boolean isCompaction) {
241 if (dataBlockEncoder.useEncodedScanner()) {
242 return new EncodedScannerV2(this, cacheBlocks, pread, isCompaction,
243 hfileContext);
244 }
245
246 return new ScannerV2(this, cacheBlocks, pread, isCompaction);
247 }
248
249
250
251
252
253
254
255 @Override
256 public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock)
257 throws IOException {
258 if (trailer.getMetaIndexCount() == 0) {
259 return null;
260 }
261 if (metaBlockIndexReader == null) {
262 throw new IOException("Meta index not loaded");
263 }
264
265 byte[] mbname = Bytes.toBytes(metaBlockName);
266 int block = metaBlockIndexReader.rootBlockContainingKey(mbname, 0,
267 mbname.length);
268 if (block == -1)
269 return null;
270 long blockSize = metaBlockIndexReader.getRootBlockDataSize(block);
271
272
273
274
275 synchronized (metaBlockIndexReader.getRootBlockKey(block)) {
276
277 long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block);
278 BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset,
279 DataBlockEncoding.NONE, BlockType.META);
280
281 cacheBlock &= cacheConf.shouldCacheDataOnRead();
282 if (cacheConf.isBlockCacheEnabled()) {
283 HFileBlock cachedBlock =
284 (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey, cacheBlock, false, true);
285 if (cachedBlock != null) {
286
287
288 return cachedBlock.getBufferWithoutHeader();
289 }
290
291 }
292
293 HFileBlock metaBlock = fsBlockReader.readBlockData(metaBlockOffset,
294 blockSize, -1, true);
295
296
297 if (cacheBlock) {
298 cacheConf.getBlockCache().cacheBlock(cacheKey, metaBlock,
299 cacheConf.isInMemory());
300 }
301
302 return metaBlock.getBufferWithoutHeader();
303 }
304 }
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321 @Override
322 public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize,
323 final boolean cacheBlock, boolean pread, final boolean isCompaction,
324 final boolean updateCacheMetrics, BlockType expectedBlockType)
325 throws IOException {
326 if (dataBlockIndexReader == null) {
327 throw new IOException("Block index not loaded");
328 }
329 if (dataBlockOffset < 0
330 || dataBlockOffset >= trailer.getLoadOnOpenDataOffset()) {
331 throw new IOException("Requested block is out of range: "
332 + dataBlockOffset + ", lastDataBlockOffset: "
333 + trailer.getLastDataBlockOffset());
334 }
335
336
337
338
339
340
341 BlockCacheKey cacheKey =
342 new BlockCacheKey(name, dataBlockOffset,
343 dataBlockEncoder.getDataBlockEncoding(),
344 expectedBlockType);
345
346 boolean useLock = false;
347 IdLock.Entry lockEntry = null;
348 TraceScope traceScope = Trace.startSpan("HFileReaderV2.readBlock");
349 try {
350 while (true) {
351 if (useLock) {
352 lockEntry = offsetLock.getLockEntry(dataBlockOffset);
353 }
354
355
356 if (cacheConf.isBlockCacheEnabled()) {
357
358
359 HFileBlock cachedBlock = (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey,
360 cacheBlock, useLock, updateCacheMetrics);
361 if (cachedBlock != null) {
362 validateBlockType(cachedBlock, expectedBlockType);
363 if (cachedBlock.getBlockType().isData()) {
364 HFile.dataBlockReadCnt.incrementAndGet();
365
366
367
368 if (cachedBlock.getDataBlockEncoding() != dataBlockEncoder.getDataBlockEncoding()) {
369 throw new IOException("Cached block under key " + cacheKey + " "
370 + "has wrong encoding: " + cachedBlock.getDataBlockEncoding() + " (expected: "
371 + dataBlockEncoder.getDataBlockEncoding() + ")");
372 }
373 }
374 return cachedBlock;
375 }
376
377 }
378 if (!useLock) {
379
380 useLock = true;
381 continue;
382 }
383 if (Trace.isTracing()) {
384 traceScope.getSpan().addTimelineAnnotation("blockCacheMiss");
385 }
386
387 HFileBlock hfileBlock = fsBlockReader.readBlockData(dataBlockOffset, onDiskBlockSize, -1,
388 pread);
389 validateBlockType(hfileBlock, expectedBlockType);
390
391
392 if (cacheBlock && cacheConf.shouldCacheBlockOnRead(hfileBlock.getBlockType().getCategory())) {
393 cacheConf.getBlockCache().cacheBlock(cacheKey, hfileBlock, cacheConf.isInMemory());
394 }
395
396 if (updateCacheMetrics && hfileBlock.getBlockType().isData()) {
397 HFile.dataBlockReadCnt.incrementAndGet();
398 }
399
400 return hfileBlock;
401 }
402 } finally {
403 traceScope.close();
404 if (lockEntry != null) {
405 offsetLock.releaseLockEntry(lockEntry);
406 }
407 }
408 }
409
410 @Override
411 public boolean hasMVCCInfo() {
412 return includesMemstoreTS && decodeMemstoreTS;
413 }
414
415
416
417
418
419
420
421
422
423
424 private void validateBlockType(HFileBlock block,
425 BlockType expectedBlockType) throws IOException {
426 if (expectedBlockType == null) {
427 return;
428 }
429 BlockType actualBlockType = block.getBlockType();
430 if (actualBlockType == BlockType.ENCODED_DATA &&
431 expectedBlockType == BlockType.DATA) {
432
433
434 return;
435 }
436 if (actualBlockType != expectedBlockType) {
437 throw new IOException("Expected block type " + expectedBlockType + ", " +
438 "but got " + actualBlockType + ": " + block);
439 }
440 }
441
442
443
444
445
446
447 @Override
448 public byte[] getLastKey() {
449 return dataBlockIndexReader.isEmpty() ? null : lastKey;
450 }
451
452
453
454
455
456
457 @Override
458 public byte[] midkey() throws IOException {
459 return dataBlockIndexReader.midkey();
460 }
461
462 @Override
463 public void close() throws IOException {
464 close(cacheConf.shouldEvictOnClose());
465 }
466
467 public void close(boolean evictOnClose) throws IOException {
468 PrefetchExecutor.cancel(path);
469 if (evictOnClose && cacheConf.isBlockCacheEnabled()) {
470 int numEvicted = cacheConf.getBlockCache().evictBlocksByHfileName(name);
471 if (LOG.isTraceEnabled()) {
472 LOG.trace("On close, file=" + name + " evicted=" + numEvicted
473 + " block(s)");
474 }
475 }
476 fsBlockReader.closeStreams();
477 }
478
479
480 @Override
481 HFileBlock.FSReader getUncachedBlockReader() {
482 return fsBlockReader;
483 }
484
485
486 protected abstract static class AbstractScannerV2
487 extends AbstractHFileReader.Scanner {
488 protected HFileBlock block;
489
490
491
492
493
494
495
496
497 protected byte[] nextIndexedKey;
498
499 public AbstractScannerV2(HFileReaderV2 r, boolean cacheBlocks,
500 final boolean pread, final boolean isCompaction) {
501 super(r, cacheBlocks, pread, isCompaction);
502 }
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520 protected int seekTo(byte[] key, int offset, int length, boolean rewind)
521 throws IOException {
522 HFileBlockIndex.BlockIndexReader indexReader =
523 reader.getDataBlockIndexReader();
524 BlockWithScanInfo blockWithScanInfo =
525 indexReader.loadDataBlockWithScanInfo(key, offset, length, block,
526 cacheBlocks, pread, isCompaction);
527 if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) {
528
529 return -1;
530 }
531 return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(),
532 blockWithScanInfo.getNextIndexedKey(), rewind, key, offset, length, false);
533 }
534
535 protected abstract ByteBuffer getFirstKeyInBlock(HFileBlock curBlock);
536
537 protected abstract int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
538 boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
539 throws IOException;
540
541 @Override
542 public int seekTo(byte[] key, int offset, int length) throws IOException {
543
544
545 return seekTo(key, offset, length, true);
546 }
547
548 @Override
549 public int reseekTo(byte[] key, int offset, int length) throws IOException {
550 int compared;
551 if (isSeeked()) {
552 compared = compareKey(reader.getComparator(), key, offset, length);
553 if (compared < 1) {
554
555
556 return compared;
557 } else {
558 if (this.nextIndexedKey != null &&
559 (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY ||
560 reader.getComparator().compareFlatKey(key, offset, length,
561 nextIndexedKey, 0, nextIndexedKey.length) < 0)) {
562
563
564
565 return loadBlockAndSeekToKey(this.block, this.nextIndexedKey,
566 false, key, offset, length, false);
567 }
568 }
569 }
570
571
572 return seekTo(key, offset, length, false);
573 }
574
575 @Override
576 public boolean seekBefore(byte[] key, int offset, int length)
577 throws IOException {
578 HFileBlock seekToBlock =
579 reader.getDataBlockIndexReader().seekToDataBlock(key, offset, length,
580 block, cacheBlocks, pread, isCompaction);
581 if (seekToBlock == null) {
582 return false;
583 }
584 ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock);
585
586 if (reader.getComparator().compareFlatKey(firstKey.array(),
587 firstKey.arrayOffset(), firstKey.limit(), key, offset, length) >= 0)
588 {
589 long previousBlockOffset = seekToBlock.getPrevBlockOffset();
590
591 if (previousBlockOffset == -1) {
592
593 return false;
594 }
595
596
597
598
599 seekToBlock = reader.readBlock(previousBlockOffset,
600 seekToBlock.getOffset() - previousBlockOffset, cacheBlocks,
601 pread, isCompaction, true, BlockType.DATA);
602
603
604 }
605 byte[] firstKeyInCurrentBlock = Bytes.getBytes(firstKey);
606 loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, offset, length, true);
607 return true;
608 }
609
610
611
612
613
614
615
616
617
618 protected HFileBlock readNextDataBlock() throws IOException {
619 long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset();
620 if (block == null)
621 return null;
622
623 HFileBlock curBlock = block;
624
625 do {
626 if (curBlock.getOffset() >= lastDataBlockOffset)
627 return null;
628
629 if (curBlock.getOffset() < 0) {
630 throw new IOException("Invalid block file offset: " + block);
631 }
632
633
634
635 curBlock = reader.readBlock(curBlock.getOffset()
636 + curBlock.getOnDiskSizeWithHeader(),
637 curBlock.getNextBlockOnDiskSizeWithHeader(), cacheBlocks, pread,
638 isCompaction, true, null);
639 } while (!curBlock.getBlockType().isData());
640
641 return curBlock;
642 }
643
644
645
646
647
648
649
650
651 public abstract int compareKey(KVComparator comparator, byte[] key, int offset,
652 int length);
653 }
654
655
656
657
658 protected static class ScannerV2 extends AbstractScannerV2 {
659 private HFileReaderV2 reader;
660
661 public ScannerV2(HFileReaderV2 r, boolean cacheBlocks,
662 final boolean pread, final boolean isCompaction) {
663 super(r, cacheBlocks, pread, isCompaction);
664 this.reader = r;
665 }
666
667 @Override
668 public KeyValue getKeyValue() {
669 if (!isSeeked())
670 return null;
671
672 KeyValue ret = new KeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
673 + blockBuffer.position(), getCellBufSize());
674 if (this.reader.shouldIncludeMemstoreTS()) {
675 ret.setMvccVersion(currMemstoreTS);
676 }
677 return ret;
678 }
679
680 protected int getCellBufSize() {
681 return KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen;
682 }
683
684 @Override
685 public ByteBuffer getKey() {
686 assertSeeked();
687 return ByteBuffer.wrap(
688 blockBuffer.array(),
689 blockBuffer.arrayOffset() + blockBuffer.position()
690 + KEY_VALUE_LEN_SIZE, currKeyLen).slice();
691 }
692
693 @Override
694 public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
695 return comparator.compareFlatKey(key, offset, length, blockBuffer.array(),
696 blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen);
697 }
698
699 @Override
700 public ByteBuffer getValue() {
701 assertSeeked();
702 return ByteBuffer.wrap(
703 blockBuffer.array(),
704 blockBuffer.arrayOffset() + blockBuffer.position()
705 + KEY_VALUE_LEN_SIZE + currKeyLen, currValueLen).slice();
706 }
707
708 protected void setNonSeekedState() {
709 block = null;
710 blockBuffer = null;
711 currKeyLen = 0;
712 currValueLen = 0;
713 currMemstoreTS = 0;
714 currMemstoreTSLen = 0;
715 }
716
717
718
719
720
721
722
723
724 @Override
725 public boolean next() throws IOException {
726 assertSeeked();
727
728 try {
729 blockBuffer.position(getNextCellStartPosition());
730 } catch (IllegalArgumentException e) {
731 LOG.error("Current pos = " + blockBuffer.position()
732 + "; currKeyLen = " + currKeyLen + "; currValLen = "
733 + currValueLen + "; block limit = " + blockBuffer.limit()
734 + "; HFile name = " + reader.getName()
735 + "; currBlock currBlockOffset = " + block.getOffset());
736 throw e;
737 }
738
739 if (blockBuffer.remaining() <= 0) {
740 long lastDataBlockOffset =
741 reader.getTrailer().getLastDataBlockOffset();
742
743 if (block.getOffset() >= lastDataBlockOffset) {
744 setNonSeekedState();
745 return false;
746 }
747
748
749 HFileBlock nextBlock = readNextDataBlock();
750 if (nextBlock == null) {
751 setNonSeekedState();
752 return false;
753 }
754
755 updateCurrBlock(nextBlock);
756 return true;
757 }
758
759
760 readKeyValueLen();
761 return true;
762 }
763
764 protected int getNextCellStartPosition() {
765 return blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen
766 + currMemstoreTSLen;
767 }
768
769
770
771
772
773
774
775
776 @Override
777 public boolean seekTo() throws IOException {
778 if (reader == null) {
779 return false;
780 }
781
782 if (reader.getTrailer().getEntryCount() == 0) {
783
784 return false;
785 }
786
787 long firstDataBlockOffset =
788 reader.getTrailer().getFirstDataBlockOffset();
789 if (block != null && block.getOffset() == firstDataBlockOffset) {
790 blockBuffer.rewind();
791 readKeyValueLen();
792 return true;
793 }
794
795 block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
796 isCompaction, true, BlockType.DATA);
797 if (block.getOffset() < 0) {
798 throw new IOException("Invalid block offset: " + block.getOffset());
799 }
800 updateCurrBlock(block);
801 return true;
802 }
803
804 @Override
805 protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
806 boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
807 throws IOException {
808 if (block == null || block.getOffset() != seekToBlock.getOffset()) {
809 updateCurrBlock(seekToBlock);
810 } else if (rewind) {
811 blockBuffer.rewind();
812 }
813
814
815 this.nextIndexedKey = nextIndexedKey;
816 return blockSeek(key, offset, length, seekBefore);
817 }
818
819
820
821
822
823
824
825 protected void updateCurrBlock(HFileBlock newBlock) {
826 block = newBlock;
827
828
829 if (block.getBlockType() != BlockType.DATA) {
830 throw new IllegalStateException("ScannerV2 works only on data " +
831 "blocks, got " + block.getBlockType() + "; " +
832 "fileName=" + reader.name + ", " +
833 "dataBlockEncoder=" + reader.dataBlockEncoder + ", " +
834 "isCompaction=" + isCompaction);
835 }
836
837 blockBuffer = block.getBufferWithoutHeader();
838 readKeyValueLen();
839 blockFetches++;
840
841
842 this.nextIndexedKey = null;
843 }
844
845 protected void readKeyValueLen() {
846 blockBuffer.mark();
847 currKeyLen = blockBuffer.getInt();
848 currValueLen = blockBuffer.getInt();
849 ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
850 readMvccVersion();
851 if (currKeyLen < 0 || currValueLen < 0
852 || currKeyLen > blockBuffer.limit()
853 || currValueLen > blockBuffer.limit()) {
854 throw new IllegalStateException("Invalid currKeyLen " + currKeyLen
855 + " or currValueLen " + currValueLen + ". Block offset: "
856 + block.getOffset() + ", block length: " + blockBuffer.limit()
857 + ", position: " + blockBuffer.position() + " (without header).");
858 }
859 blockBuffer.reset();
860 }
861
862 protected void readMvccVersion() {
863 if (this.reader.shouldIncludeMemstoreTS()) {
864 if (this.reader.decodeMemstoreTS) {
865 try {
866 currMemstoreTS = Bytes.readVLong(blockBuffer.array(), blockBuffer.arrayOffset()
867 + blockBuffer.position());
868 currMemstoreTSLen = WritableUtils.getVIntSize(currMemstoreTS);
869 } catch (Exception e) {
870 throw new RuntimeException("Error reading memstore timestamp", e);
871 }
872 } else {
873 currMemstoreTS = 0;
874 currMemstoreTSLen = 1;
875 }
876 }
877 }
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894 protected int blockSeek(byte[] key, int offset, int length,
895 boolean seekBefore) {
896 int klen, vlen;
897 long memstoreTS = 0;
898 int memstoreTSLen = 0;
899 int lastKeyValueSize = -1;
900 do {
901 blockBuffer.mark();
902 klen = blockBuffer.getInt();
903 vlen = blockBuffer.getInt();
904 blockBuffer.reset();
905 if (this.reader.shouldIncludeMemstoreTS()) {
906 if (this.reader.decodeMemstoreTS) {
907 try {
908 int memstoreTSOffset = blockBuffer.arrayOffset()
909 + blockBuffer.position() + KEY_VALUE_LEN_SIZE + klen + vlen;
910 memstoreTS = Bytes.readVLong(blockBuffer.array(),
911 memstoreTSOffset);
912 memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
913 } catch (Exception e) {
914 throw new RuntimeException("Error reading memstore timestamp", e);
915 }
916 } else {
917 memstoreTS = 0;
918 memstoreTSLen = 1;
919 }
920 }
921
922 int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position()
923 + KEY_VALUE_LEN_SIZE;
924 int comp = reader.getComparator().compareFlatKey(key, offset, length,
925 blockBuffer.array(), keyOffset, klen);
926
927 if (comp == 0) {
928 if (seekBefore) {
929 if (lastKeyValueSize < 0) {
930 throw new IllegalStateException("blockSeek with seekBefore "
931 + "at the first key of the block: key="
932 + Bytes.toStringBinary(key) + ", blockOffset="
933 + block.getOffset() + ", onDiskSize="
934 + block.getOnDiskSizeWithHeader());
935 }
936 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
937 readKeyValueLen();
938 return 1;
939 }
940 currKeyLen = klen;
941 currValueLen = vlen;
942 if (this.reader.shouldIncludeMemstoreTS()) {
943 currMemstoreTS = memstoreTS;
944 currMemstoreTSLen = memstoreTSLen;
945 }
946 return 0;
947 } else if (comp < 0) {
948 if (lastKeyValueSize > 0)
949 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
950 readKeyValueLen();
951 if (lastKeyValueSize == -1 && blockBuffer.position() == 0
952 && this.reader.trailer.getMinorVersion() >= MINOR_VERSION_WITH_FAKED_KEY) {
953 return HConstants.INDEX_KEY_MAGIC;
954 }
955 return 1;
956 }
957
958
959 lastKeyValueSize = klen + vlen + memstoreTSLen + KEY_VALUE_LEN_SIZE;
960 blockBuffer.position(blockBuffer.position() + lastKeyValueSize);
961 } while (blockBuffer.remaining() > 0);
962
963
964
965
966 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
967 readKeyValueLen();
968 return 1;
969 }
970
971 @Override
972 protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
973 ByteBuffer buffer = curBlock.getBufferWithoutHeader();
974
975 buffer.rewind();
976 int klen = buffer.getInt();
977 buffer.getInt();
978 ByteBuffer keyBuff = buffer.slice();
979 keyBuff.limit(klen);
980 keyBuff.rewind();
981 return keyBuff;
982 }
983
984 @Override
985 public String getKeyString() {
986 return Bytes.toStringBinary(blockBuffer.array(),
987 blockBuffer.arrayOffset() + blockBuffer.position()
988 + KEY_VALUE_LEN_SIZE, currKeyLen);
989 }
990
991 @Override
992 public String getValueString() {
993 return Bytes.toString(blockBuffer.array(), blockBuffer.arrayOffset()
994 + blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen,
995 currValueLen);
996 }
997 }
998
999
1000
1001
1002 protected static class EncodedScannerV2 extends AbstractScannerV2 {
1003 private final HFileBlockDecodingContext decodingCtx;
1004 private final DataBlockEncoder.EncodedSeeker seeker;
1005 private final DataBlockEncoder dataBlockEncoder;
1006 protected final HFileContext meta;
1007
1008 public EncodedScannerV2(HFileReaderV2 reader, boolean cacheBlocks,
1009 boolean pread, boolean isCompaction, HFileContext meta) {
1010 super(reader, cacheBlocks, pread, isCompaction);
1011 DataBlockEncoding encoding = reader.dataBlockEncoder.getDataBlockEncoding();
1012 dataBlockEncoder = encoding.getEncoder();
1013 decodingCtx = dataBlockEncoder.newDataBlockDecodingContext(meta);
1014 seeker = dataBlockEncoder.createSeeker(
1015 reader.getComparator(), decodingCtx);
1016 this.meta = meta;
1017 }
1018
1019 @Override
1020 public boolean isSeeked(){
1021 return this.block != null;
1022 }
1023
1024
1025
1026
1027
1028
1029
1030
1031 private void updateCurrentBlock(HFileBlock newBlock) throws CorruptHFileException {
1032 block = newBlock;
1033
1034
1035 if (block.getBlockType() != BlockType.ENCODED_DATA) {
1036 throw new IllegalStateException(
1037 "EncodedScanner works only on encoded data blocks");
1038 }
1039 short dataBlockEncoderId = block.getDataBlockEncodingId();
1040 if (!DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) {
1041 String encoderCls = dataBlockEncoder.getClass().getName();
1042 throw new CorruptHFileException("Encoder " + encoderCls
1043 + " doesn't support data block encoding "
1044 + DataBlockEncoding.getNameFromId(dataBlockEncoderId));
1045 }
1046
1047 seeker.setCurrentBuffer(getEncodedBuffer(newBlock));
1048 blockFetches++;
1049
1050
1051 this.nextIndexedKey = null;
1052 }
1053
1054 private ByteBuffer getEncodedBuffer(HFileBlock newBlock) {
1055 ByteBuffer origBlock = newBlock.getBufferReadOnly();
1056 ByteBuffer encodedBlock = ByteBuffer.wrap(origBlock.array(),
1057 origBlock.arrayOffset() + newBlock.headerSize() +
1058 DataBlockEncoding.ID_SIZE,
1059 newBlock.getUncompressedSizeWithoutHeader() -
1060 DataBlockEncoding.ID_SIZE).slice();
1061 return encodedBlock;
1062 }
1063
1064 @Override
1065 public boolean seekTo() throws IOException {
1066 if (reader == null) {
1067 return false;
1068 }
1069
1070 if (reader.getTrailer().getEntryCount() == 0) {
1071
1072 return false;
1073 }
1074
1075 long firstDataBlockOffset =
1076 reader.getTrailer().getFirstDataBlockOffset();
1077 if (block != null && block.getOffset() == firstDataBlockOffset) {
1078 seeker.rewind();
1079 return true;
1080 }
1081
1082 block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
1083 isCompaction, true, BlockType.DATA);
1084 if (block.getOffset() < 0) {
1085 throw new IOException("Invalid block offset: " + block.getOffset());
1086 }
1087 updateCurrentBlock(block);
1088 return true;
1089 }
1090
1091 @Override
1092 public boolean next() throws IOException {
1093 boolean isValid = seeker.next();
1094 if (!isValid) {
1095 block = readNextDataBlock();
1096 isValid = block != null;
1097 if (isValid) {
1098 updateCurrentBlock(block);
1099 }
1100 }
1101 return isValid;
1102 }
1103
1104 @Override
1105 public ByteBuffer getKey() {
1106 assertValidSeek();
1107 return seeker.getKeyDeepCopy();
1108 }
1109
1110 @Override
1111 public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
1112 return seeker.compareKey(comparator, key, offset, length);
1113 }
1114
1115 @Override
1116 public ByteBuffer getValue() {
1117 assertValidSeek();
1118 return seeker.getValueShallowCopy();
1119 }
1120
1121 @Override
1122 public KeyValue getKeyValue() {
1123 if (block == null) {
1124 return null;
1125 }
1126 return seeker.getKeyValue();
1127 }
1128
1129 @Override
1130 public String getKeyString() {
1131 ByteBuffer keyBuffer = getKey();
1132 return Bytes.toStringBinary(keyBuffer.array(),
1133 keyBuffer.arrayOffset(), keyBuffer.limit());
1134 }
1135
1136 @Override
1137 public String getValueString() {
1138 ByteBuffer valueBuffer = getValue();
1139 return Bytes.toStringBinary(valueBuffer.array(),
1140 valueBuffer.arrayOffset(), valueBuffer.limit());
1141 }
1142
1143 private void assertValidSeek() {
1144 if (block == null) {
1145 throw new NotSeekedException();
1146 }
1147 }
1148
1149 @Override
1150 protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
1151 return dataBlockEncoder.getFirstKeyInBlock(getEncodedBuffer(curBlock));
1152 }
1153
1154 @Override
1155 protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
1156 boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
1157 throws IOException {
1158 if (block == null || block.getOffset() != seekToBlock.getOffset()) {
1159 updateCurrentBlock(seekToBlock);
1160 } else if (rewind) {
1161 seeker.rewind();
1162 }
1163 this.nextIndexedKey = nextIndexedKey;
1164 return seeker.seekToKeyInBlock(key, offset, length, seekBefore);
1165 }
1166 }
1167
1168
1169
1170
1171
1172 @Override
1173 public DataInput getGeneralBloomFilterMetadata() throws IOException {
1174 return this.getBloomFilterMetadata(BlockType.GENERAL_BLOOM_META);
1175 }
1176
1177 @Override
1178 public DataInput getDeleteBloomFilterMetadata() throws IOException {
1179 return this.getBloomFilterMetadata(BlockType.DELETE_FAMILY_BLOOM_META);
1180 }
1181
1182 private DataInput getBloomFilterMetadata(BlockType blockType)
1183 throws IOException {
1184 if (blockType != BlockType.GENERAL_BLOOM_META &&
1185 blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
1186 throw new RuntimeException("Block Type: " + blockType.toString() +
1187 " is not supported") ;
1188 }
1189
1190 for (HFileBlock b : loadOnOpenBlocks)
1191 if (b.getBlockType() == blockType)
1192 return b.getByteStream();
1193 return null;
1194 }
1195
1196 @Override
1197 public boolean isFileInfoLoaded() {
1198 return true;
1199 }
1200
1201
1202
1203
1204
1205 private void validateMinorVersion(Path path, int minorVersion) {
1206 if (minorVersion < MIN_MINOR_VERSION ||
1207 minorVersion > MAX_MINOR_VERSION) {
1208 String msg = "Minor version for path " + path +
1209 " is expected to be between " +
1210 MIN_MINOR_VERSION + " and " + MAX_MINOR_VERSION +
1211 " but is found to be " + minorVersion;
1212 LOG.error(msg);
1213 throw new RuntimeException(msg);
1214 }
1215 }
1216
1217 @Override
1218 public int getMajorVersion() {
1219 return 2;
1220 }
1221
1222 @Override
1223 public HFileContext getFileContext() {
1224 return hfileContext;
1225 }
1226
1227
1228
1229
1230
1231 @VisibleForTesting
1232 boolean prefetchComplete() {
1233 return PrefetchExecutor.isCompleted(path);
1234 }
1235 }