1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.DataInput;
21 import java.io.IOException;
22 import java.nio.ByteBuffer;
23 import java.util.ArrayList;
24 import java.util.List;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.hbase.classification.InterfaceAudience;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.KeyValue;
33 import org.apache.hadoop.hbase.KeyValue.KVComparator;
34 import org.apache.hadoop.hbase.NoTagsKeyValue;
35 import org.apache.hadoop.hbase.fs.HFileSystem;
36 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
37 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
38 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
39 import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
40 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
41 import org.apache.hadoop.hbase.util.ByteBufferUtils;
42 import org.apache.hadoop.hbase.util.Bytes;
43 import org.apache.hadoop.hbase.util.IdLock;
44 import org.apache.hadoop.io.WritableUtils;
45 import org.cloudera.htrace.Trace;
46 import org.cloudera.htrace.TraceScope;
47
48 import com.google.common.annotations.VisibleForTesting;
49
50
51
52
53 @InterfaceAudience.Private
54 public class HFileReaderV2 extends AbstractHFileReader {
55
56 private static final Log LOG = LogFactory.getLog(HFileReaderV2.class);
57
58
59 public static final int MINOR_VERSION_WITH_CHECKSUM = 1;
60
61 public static final int MINOR_VERSION_NO_CHECKSUM = 0;
62
63
64 public static final int PBUF_TRAILER_MINOR_VERSION = 2;
65
66
67
68
69
70 public final static int KEY_VALUE_LEN_SIZE = 2 * Bytes.SIZEOF_INT;
71
72 protected boolean includesMemstoreTS = false;
73 protected boolean decodeMemstoreTS = false;
74 protected boolean shouldIncludeMemstoreTS() {
75 return includesMemstoreTS;
76 }
77
78
79 protected HFileBlock.FSReader fsBlockReader;
80
81
82
83
84
85
86
87 private IdLock offsetLock = new IdLock();
88
89
90
91
92
93 private List<HFileBlock> loadOnOpenBlocks = new ArrayList<HFileBlock>();
94
95
96 static final int MIN_MINOR_VERSION = 0;
97
98
99
100
101 static final int MAX_MINOR_VERSION = 3;
102
103
104 static final int MINOR_VERSION_WITH_FAKED_KEY = 3;
105
106 protected HFileContext hfileContext;
107
108
109
110
111
112
113
114
115
116
117
118
119
120 public HFileReaderV2(final Path path, final FixedFileTrailer trailer,
121 final FSDataInputStreamWrapper fsdis, final long size, final CacheConfig cacheConf,
122 final HFileSystem hfs, final Configuration conf) throws IOException {
123 super(path, trailer, size, cacheConf, hfs, conf);
124 this.conf = conf;
125 trailer.expectMajorVersion(getMajorVersion());
126 validateMinorVersion(path, trailer.getMinorVersion());
127 this.hfileContext = createHFileContext(fsdis, fileSize, hfs, path, trailer);
128 HFileBlock.FSReaderV2 fsBlockReaderV2 = new HFileBlock.FSReaderV2(fsdis, fileSize, hfs, path,
129 hfileContext);
130 this.fsBlockReader = fsBlockReaderV2;
131
132
133 comparator = trailer.createComparator();
134 dataBlockIndexReader = new HFileBlockIndex.BlockIndexReader(comparator,
135 trailer.getNumDataIndexLevels(), this);
136 metaBlockIndexReader = new HFileBlockIndex.BlockIndexReader(
137 KeyValue.RAW_COMPARATOR, 1);
138
139
140
141 HFileBlock.BlockIterator blockIter = fsBlockReaderV2.blockRange(
142 trailer.getLoadOnOpenDataOffset(),
143 fileSize - trailer.getTrailerSize());
144
145
146
147 dataBlockIndexReader.readMultiLevelIndexRoot(
148 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
149 trailer.getDataIndexCount());
150
151
152 metaBlockIndexReader.readRootIndex(
153 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
154 trailer.getMetaIndexCount());
155
156
157 fileInfo = new FileInfo();
158 fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
159 lastKey = fileInfo.get(FileInfo.LASTKEY);
160 avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN));
161 avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN));
162 byte [] keyValueFormatVersion =
163 fileInfo.get(HFileWriterV2.KEY_VALUE_VERSION);
164 includesMemstoreTS = keyValueFormatVersion != null &&
165 Bytes.toInt(keyValueFormatVersion) ==
166 HFileWriterV2.KEY_VALUE_VER_WITH_MEMSTORE;
167 fsBlockReaderV2.setIncludesMemstoreTS(includesMemstoreTS);
168 if (includesMemstoreTS) {
169 decodeMemstoreTS = Bytes.toLong(fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY)) > 0;
170 }
171
172
173 dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo);
174 fsBlockReaderV2.setDataBlockEncoder(dataBlockEncoder);
175
176
177 HFileBlock b;
178 while ((b = blockIter.nextBlock()) != null) {
179 loadOnOpenBlocks.add(b);
180 }
181
182
183 if (cacheConf.shouldPrefetchOnOpen()) {
184 PrefetchExecutor.request(path, new Runnable() {
185 public void run() {
186 try {
187 long offset = 0;
188 long end = fileSize - getTrailer().getTrailerSize();
189 HFileBlock prevBlock = null;
190 while (offset < end) {
191 if (Thread.interrupted()) {
192 break;
193 }
194 long onDiskSize = -1;
195 if (prevBlock != null) {
196 onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
197 }
198 HFileBlock block = readBlock(offset, onDiskSize, true, false, false, false, null);
199 prevBlock = block;
200 offset += block.getOnDiskSizeWithHeader();
201 }
202 } catch (IOException e) {
203
204 if (LOG.isTraceEnabled()) {
205 LOG.trace("Exception encountered while prefetching " + path + ":", e);
206 }
207 } catch (Exception e) {
208
209 LOG.warn("Exception encountered while prefetching " + path + ":", e);
210 } finally {
211 PrefetchExecutor.complete(path);
212 }
213 }
214 });
215 }
216 }
217
218 protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
219 HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
220 return new HFileContextBuilder()
221 .withIncludesMvcc(this.includesMemstoreTS)
222 .withCompression(this.compressAlgo)
223 .withHBaseCheckSum(trailer.getMinorVersion() >= MINOR_VERSION_WITH_CHECKSUM)
224 .build();
225 }
226
227
228
229
230
231
232
233
234
235
236
237
238
239 @Override
240 public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
241 final boolean isCompaction) {
242 if (dataBlockEncoder.useEncodedScanner()) {
243 return new EncodedScannerV2(this, cacheBlocks, pread, isCompaction,
244 hfileContext);
245 }
246
247 return new ScannerV2(this, cacheBlocks, pread, isCompaction);
248 }
249
250
251
252
253
254
255
256 @Override
257 public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock)
258 throws IOException {
259 if (trailer.getMetaIndexCount() == 0) {
260 return null;
261 }
262 if (metaBlockIndexReader == null) {
263 throw new IOException("Meta index not loaded");
264 }
265
266 byte[] mbname = Bytes.toBytes(metaBlockName);
267 int block = metaBlockIndexReader.rootBlockContainingKey(mbname, 0,
268 mbname.length);
269 if (block == -1)
270 return null;
271 long blockSize = metaBlockIndexReader.getRootBlockDataSize(block);
272
273
274
275
276 synchronized (metaBlockIndexReader.getRootBlockKey(block)) {
277
278 long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block);
279 BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset,
280 DataBlockEncoding.NONE, BlockType.META);
281
282 cacheBlock &= cacheConf.shouldCacheDataOnRead();
283 if (cacheConf.isBlockCacheEnabled()) {
284 HFileBlock cachedBlock =
285 (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey, cacheBlock, false, true);
286 if (cachedBlock != null) {
287 assert cachedBlock.isUnpacked() : "Packed block leak.";
288
289
290 return cachedBlock.getBufferWithoutHeader();
291 }
292
293 }
294
295 HFileBlock metaBlock = fsBlockReader.readBlockData(metaBlockOffset,
296 blockSize, -1, true).unpack(hfileContext, fsBlockReader);
297
298
299 if (cacheBlock) {
300 cacheConf.getBlockCache().cacheBlock(cacheKey, metaBlock,
301 cacheConf.isInMemory());
302 }
303
304 return metaBlock.getBufferWithoutHeader();
305 }
306 }
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323 @Override
324 public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize,
325 final boolean cacheBlock, boolean pread, final boolean isCompaction,
326 final boolean updateCacheMetrics, BlockType expectedBlockType)
327 throws IOException {
328 if (dataBlockIndexReader == null) {
329 throw new IOException("Block index not loaded");
330 }
331 if (dataBlockOffset < 0
332 || dataBlockOffset >= trailer.getLoadOnOpenDataOffset()) {
333 throw new IOException("Requested block is out of range: "
334 + dataBlockOffset + ", lastDataBlockOffset: "
335 + trailer.getLastDataBlockOffset());
336 }
337
338
339
340
341
342
343 BlockCacheKey cacheKey =
344 new BlockCacheKey(name, dataBlockOffset,
345 dataBlockEncoder.getDataBlockEncoding(),
346 expectedBlockType);
347
348 boolean useLock = false;
349 IdLock.Entry lockEntry = null;
350 TraceScope traceScope = Trace.startSpan("HFileReaderV2.readBlock");
351 try {
352 while (true) {
353
354 if (cacheConf.shouldReadBlockFromCache(expectedBlockType)) {
355 if (useLock) {
356 lockEntry = offsetLock.getLockEntry(dataBlockOffset);
357 }
358
359
360 HFileBlock cachedBlock = (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey,
361 cacheBlock, useLock, updateCacheMetrics);
362 if (cachedBlock != null) {
363 if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) {
364 cachedBlock = cachedBlock.unpack(hfileContext, fsBlockReader);
365 }
366 if (Trace.isTracing()) {
367 traceScope.getSpan().addTimelineAnnotation("blockCacheHit");
368 }
369 assert cachedBlock.isUnpacked() : "Packed block leak.";
370 if (cachedBlock.getBlockType().isData()) {
371 HFile.dataBlockReadCnt.incrementAndGet();
372
373
374
375 if (cachedBlock.getDataBlockEncoding() != dataBlockEncoder.getDataBlockEncoding()) {
376 throw new IOException("Cached block under key " + cacheKey + " "
377 + "has wrong encoding: " + cachedBlock.getDataBlockEncoding() + " (expected: "
378 + dataBlockEncoder.getDataBlockEncoding() + ")");
379 }
380 }
381 return cachedBlock;
382 }
383 if (!useLock && cacheBlock && cacheConf.shouldLockOnCacheMiss(expectedBlockType)) {
384
385 useLock = true;
386 continue;
387 }
388
389 }
390
391 if (Trace.isTracing()) {
392 traceScope.getSpan().addTimelineAnnotation("blockCacheMiss");
393 }
394
395 HFileBlock hfileBlock = fsBlockReader.readBlockData(dataBlockOffset, onDiskBlockSize, -1,
396 pread);
397 validateBlockType(hfileBlock, expectedBlockType);
398 HFileBlock unpacked = hfileBlock.unpack(hfileContext, fsBlockReader);
399 BlockType.BlockCategory category = hfileBlock.getBlockType().getCategory();
400
401
402 if (cacheBlock && cacheConf.shouldCacheBlockOnRead(category)) {
403 cacheConf.getBlockCache().cacheBlock(cacheKey,
404 cacheConf.shouldCacheCompressed(category) ? hfileBlock : unpacked,
405 cacheConf.isInMemory());
406 }
407
408 if (updateCacheMetrics && hfileBlock.getBlockType().isData()) {
409 HFile.dataBlockReadCnt.incrementAndGet();
410 }
411
412 return unpacked;
413 }
414 } finally {
415 traceScope.close();
416 if (lockEntry != null) {
417 offsetLock.releaseLockEntry(lockEntry);
418 }
419 }
420 }
421
422 @Override
423 public boolean hasMVCCInfo() {
424 return includesMemstoreTS && decodeMemstoreTS;
425 }
426
427
428
429
430
431
432
433
434
435
436 private void validateBlockType(HFileBlock block,
437 BlockType expectedBlockType) throws IOException {
438 if (expectedBlockType == null) {
439 return;
440 }
441 BlockType actualBlockType = block.getBlockType();
442 if (actualBlockType == BlockType.ENCODED_DATA &&
443 expectedBlockType == BlockType.DATA) {
444
445
446 return;
447 }
448 if (actualBlockType != expectedBlockType) {
449 throw new IOException("Expected block type " + expectedBlockType + ", " +
450 "but got " + actualBlockType + ": " + block);
451 }
452 }
453
454
455
456
457
458
459 @Override
460 public byte[] getLastKey() {
461 return dataBlockIndexReader.isEmpty() ? null : lastKey;
462 }
463
464
465
466
467
468
469 @Override
470 public byte[] midkey() throws IOException {
471 return dataBlockIndexReader.midkey();
472 }
473
474 @Override
475 public void close() throws IOException {
476 close(cacheConf.shouldEvictOnClose());
477 }
478
479 public void close(boolean evictOnClose) throws IOException {
480 PrefetchExecutor.cancel(path);
481 if (evictOnClose && cacheConf.isBlockCacheEnabled()) {
482 int numEvicted = cacheConf.getBlockCache().evictBlocksByHfileName(name);
483 if (LOG.isTraceEnabled()) {
484 LOG.trace("On close, file=" + name + " evicted=" + numEvicted
485 + " block(s)");
486 }
487 }
488 fsBlockReader.closeStreams();
489 }
490
491
492 @Override
493 HFileBlock.FSReader getUncachedBlockReader() {
494 return fsBlockReader;
495 }
496
497
498 protected abstract static class AbstractScannerV2
499 extends AbstractHFileReader.Scanner {
500 protected HFileBlock block;
501
502 @Override
503 public byte[] getNextIndexedKey() {
504 return nextIndexedKey;
505 }
506
507
508
509
510
511
512
513 protected byte[] nextIndexedKey;
514
515 public AbstractScannerV2(HFileReaderV2 r, boolean cacheBlocks,
516 final boolean pread, final boolean isCompaction) {
517 super(r, cacheBlocks, pread, isCompaction);
518 }
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536 protected int seekTo(byte[] key, int offset, int length, boolean rewind)
537 throws IOException {
538 HFileBlockIndex.BlockIndexReader indexReader =
539 reader.getDataBlockIndexReader();
540 BlockWithScanInfo blockWithScanInfo =
541 indexReader.loadDataBlockWithScanInfo(key, offset, length, block,
542 cacheBlocks, pread, isCompaction);
543 if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) {
544
545 return -1;
546 }
547 return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(),
548 blockWithScanInfo.getNextIndexedKey(), rewind, key, offset, length, false);
549 }
550
551 protected abstract ByteBuffer getFirstKeyInBlock(HFileBlock curBlock);
552
553 protected abstract int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
554 boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
555 throws IOException;
556
557 @Override
558 public int seekTo(byte[] key, int offset, int length) throws IOException {
559
560
561 return seekTo(key, offset, length, true);
562 }
563
564 @Override
565 public int reseekTo(byte[] key, int offset, int length) throws IOException {
566 int compared;
567 if (isSeeked()) {
568 compared = compareKey(reader.getComparator(), key, offset, length);
569 if (compared < 1) {
570
571
572 return compared;
573 } else {
574 if (this.nextIndexedKey != null &&
575 (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY ||
576 reader.getComparator().compareFlatKey(key, offset, length,
577 nextIndexedKey, 0, nextIndexedKey.length) < 0)) {
578
579
580
581 return loadBlockAndSeekToKey(this.block, this.nextIndexedKey,
582 false, key, offset, length, false);
583 }
584 }
585 }
586
587
588 return seekTo(key, offset, length, false);
589 }
590
591 @Override
592 public boolean seekBefore(byte[] key, int offset, int length)
593 throws IOException {
594 HFileBlock seekToBlock =
595 reader.getDataBlockIndexReader().seekToDataBlock(key, offset, length,
596 block, cacheBlocks, pread, isCompaction);
597 if (seekToBlock == null) {
598 return false;
599 }
600 ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock);
601
602 if (reader.getComparator().compareFlatKey(firstKey.array(),
603 firstKey.arrayOffset(), firstKey.limit(), key, offset, length) >= 0)
604 {
605 long previousBlockOffset = seekToBlock.getPrevBlockOffset();
606
607 if (previousBlockOffset == -1) {
608
609 return false;
610 }
611
612
613
614
615 seekToBlock = reader.readBlock(previousBlockOffset,
616 seekToBlock.getOffset() - previousBlockOffset, cacheBlocks,
617 pread, isCompaction, true, BlockType.DATA);
618
619
620 }
621 byte[] firstKeyInCurrentBlock = Bytes.getBytes(firstKey);
622 loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, offset, length, true);
623 return true;
624 }
625
626
627
628
629
630
631
632
633
634 protected HFileBlock readNextDataBlock() throws IOException {
635 long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset();
636 if (block == null)
637 return null;
638
639 HFileBlock curBlock = block;
640
641 do {
642 if (curBlock.getOffset() >= lastDataBlockOffset)
643 return null;
644
645 if (curBlock.getOffset() < 0) {
646 throw new IOException("Invalid block file offset: " + block);
647 }
648
649
650
651 curBlock = reader.readBlock(curBlock.getOffset()
652 + curBlock.getOnDiskSizeWithHeader(),
653 curBlock.getNextBlockOnDiskSizeWithHeader(), cacheBlocks, pread,
654 isCompaction, true, null);
655 } while (!curBlock.getBlockType().isData());
656
657 return curBlock;
658 }
659
660
661
662
663
664
665
666
667 public abstract int compareKey(KVComparator comparator, byte[] key, int offset,
668 int length);
669 }
670
671
672
673
674 protected static class ScannerV2 extends AbstractScannerV2 {
675 private HFileReaderV2 reader;
676
677 public ScannerV2(HFileReaderV2 r, boolean cacheBlocks,
678 final boolean pread, final boolean isCompaction) {
679 super(r, cacheBlocks, pread, isCompaction);
680 this.reader = r;
681 }
682
683 @Override
684 public KeyValue getKeyValue() {
685 if (!isSeeked())
686 return null;
687
688
689 return formNoTagsKeyValue();
690 }
691
692 protected KeyValue formNoTagsKeyValue() {
693 KeyValue ret = new NoTagsKeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
694 + blockBuffer.position(), getCellBufSize());
695 if (this.reader.shouldIncludeMemstoreTS()) {
696 ret.setMvccVersion(currMemstoreTS);
697 }
698 return ret;
699 }
700
701 protected int getCellBufSize() {
702 return KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen;
703 }
704
705 @Override
706 public ByteBuffer getKey() {
707 assertSeeked();
708 return ByteBuffer.wrap(
709 blockBuffer.array(),
710 blockBuffer.arrayOffset() + blockBuffer.position()
711 + KEY_VALUE_LEN_SIZE, currKeyLen).slice();
712 }
713
714 @Override
715 public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
716 return comparator.compareFlatKey(key, offset, length, blockBuffer.array(),
717 blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen);
718 }
719
720 @Override
721 public ByteBuffer getValue() {
722 assertSeeked();
723 return ByteBuffer.wrap(
724 blockBuffer.array(),
725 blockBuffer.arrayOffset() + blockBuffer.position()
726 + KEY_VALUE_LEN_SIZE + currKeyLen, currValueLen).slice();
727 }
728
729 protected void setNonSeekedState() {
730 block = null;
731 blockBuffer = null;
732 currKeyLen = 0;
733 currValueLen = 0;
734 currMemstoreTS = 0;
735 currMemstoreTSLen = 0;
736 }
737
738
739
740
741
742
743
744
745 @Override
746 public boolean next() throws IOException {
747 assertSeeked();
748
749 try {
750 blockBuffer.position(getNextCellStartPosition());
751 } catch (IllegalArgumentException e) {
752 LOG.error("Current pos = " + blockBuffer.position()
753 + "; currKeyLen = " + currKeyLen + "; currValLen = "
754 + currValueLen + "; block limit = " + blockBuffer.limit()
755 + "; HFile name = " + reader.getName()
756 + "; currBlock currBlockOffset = " + block.getOffset());
757 throw e;
758 }
759
760 if (blockBuffer.remaining() <= 0) {
761 long lastDataBlockOffset =
762 reader.getTrailer().getLastDataBlockOffset();
763
764 if (block.getOffset() >= lastDataBlockOffset) {
765 setNonSeekedState();
766 return false;
767 }
768
769
770 HFileBlock nextBlock = readNextDataBlock();
771 if (nextBlock == null) {
772 setNonSeekedState();
773 return false;
774 }
775
776 updateCurrBlock(nextBlock);
777 return true;
778 }
779
780
781 readKeyValueLen();
782 return true;
783 }
784
785 protected int getNextCellStartPosition() {
786 return blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen
787 + currMemstoreTSLen;
788 }
789
790
791
792
793
794
795
796
797 @Override
798 public boolean seekTo() throws IOException {
799 if (reader == null) {
800 return false;
801 }
802
803 if (reader.getTrailer().getEntryCount() == 0) {
804
805 return false;
806 }
807
808 long firstDataBlockOffset =
809 reader.getTrailer().getFirstDataBlockOffset();
810 if (block != null && block.getOffset() == firstDataBlockOffset) {
811 blockBuffer.rewind();
812 readKeyValueLen();
813 return true;
814 }
815
816 block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
817 isCompaction, true, BlockType.DATA);
818 if (block.getOffset() < 0) {
819 throw new IOException("Invalid block offset: " + block.getOffset());
820 }
821 updateCurrBlock(block);
822 return true;
823 }
824
825 @Override
826 protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
827 boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
828 throws IOException {
829 if (block == null || block.getOffset() != seekToBlock.getOffset()) {
830 updateCurrBlock(seekToBlock);
831 } else if (rewind) {
832 blockBuffer.rewind();
833 }
834
835
836 this.nextIndexedKey = nextIndexedKey;
837 return blockSeek(key, offset, length, seekBefore);
838 }
839
840
841
842
843
844
845
846 protected void updateCurrBlock(HFileBlock newBlock) {
847 block = newBlock;
848
849
850 if (block.getBlockType() != BlockType.DATA) {
851 throw new IllegalStateException("ScannerV2 works only on data " +
852 "blocks, got " + block.getBlockType() + "; " +
853 "fileName=" + reader.name + ", " +
854 "dataBlockEncoder=" + reader.dataBlockEncoder + ", " +
855 "isCompaction=" + isCompaction);
856 }
857
858 blockBuffer = block.getBufferWithoutHeader();
859 readKeyValueLen();
860 blockFetches++;
861
862
863 this.nextIndexedKey = null;
864 }
865
866 protected void readKeyValueLen() {
867 blockBuffer.mark();
868 currKeyLen = blockBuffer.getInt();
869 currValueLen = blockBuffer.getInt();
870 ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
871 readMvccVersion();
872 if (currKeyLen < 0 || currValueLen < 0
873 || currKeyLen > blockBuffer.limit()
874 || currValueLen > blockBuffer.limit()) {
875 throw new IllegalStateException("Invalid currKeyLen " + currKeyLen
876 + " or currValueLen " + currValueLen + ". Block offset: "
877 + block.getOffset() + ", block length: " + blockBuffer.limit()
878 + ", position: " + blockBuffer.position() + " (without header).");
879 }
880 blockBuffer.reset();
881 }
882
883 protected void readMvccVersion() {
884 if (this.reader.shouldIncludeMemstoreTS()) {
885 if (this.reader.decodeMemstoreTS) {
886 currMemstoreTS = Bytes.readAsVLong(blockBuffer.array(), blockBuffer.arrayOffset()
887 + blockBuffer.position());
888 currMemstoreTSLen = WritableUtils.getVIntSize(currMemstoreTS);
889 } else {
890 currMemstoreTS = 0;
891 currMemstoreTSLen = 1;
892 }
893 }
894 }
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911 protected int blockSeek(byte[] key, int offset, int length,
912 boolean seekBefore) {
913 int klen, vlen;
914 long memstoreTS = 0;
915 int memstoreTSLen = 0;
916 int lastKeyValueSize = -1;
917 do {
918 blockBuffer.mark();
919 klen = blockBuffer.getInt();
920 vlen = blockBuffer.getInt();
921 blockBuffer.reset();
922 if (this.reader.shouldIncludeMemstoreTS()) {
923 if (this.reader.decodeMemstoreTS) {
924 int memstoreTSOffset = blockBuffer.arrayOffset() + blockBuffer.position()
925 + KEY_VALUE_LEN_SIZE + klen + vlen;
926 memstoreTS = Bytes.readAsVLong(blockBuffer.array(), memstoreTSOffset);
927 memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
928 } else {
929 memstoreTS = 0;
930 memstoreTSLen = 1;
931 }
932 }
933
934 int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position()
935 + KEY_VALUE_LEN_SIZE;
936 int comp = reader.getComparator().compareFlatKey(key, offset, length,
937 blockBuffer.array(), keyOffset, klen);
938
939 if (comp == 0) {
940 if (seekBefore) {
941 if (lastKeyValueSize < 0) {
942 throw new IllegalStateException("blockSeek with seekBefore "
943 + "at the first key of the block: key="
944 + Bytes.toStringBinary(key) + ", blockOffset="
945 + block.getOffset() + ", onDiskSize="
946 + block.getOnDiskSizeWithHeader());
947 }
948 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
949 readKeyValueLen();
950 return 1;
951 }
952 currKeyLen = klen;
953 currValueLen = vlen;
954 if (this.reader.shouldIncludeMemstoreTS()) {
955 currMemstoreTS = memstoreTS;
956 currMemstoreTSLen = memstoreTSLen;
957 }
958 return 0;
959 } else if (comp < 0) {
960 if (lastKeyValueSize > 0)
961 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
962 readKeyValueLen();
963 if (lastKeyValueSize == -1 && blockBuffer.position() == 0
964 && this.reader.trailer.getMinorVersion() >= MINOR_VERSION_WITH_FAKED_KEY) {
965 return HConstants.INDEX_KEY_MAGIC;
966 }
967 return 1;
968 }
969
970
971 lastKeyValueSize = klen + vlen + memstoreTSLen + KEY_VALUE_LEN_SIZE;
972 blockBuffer.position(blockBuffer.position() + lastKeyValueSize);
973 } while (blockBuffer.remaining() > 0);
974
975
976
977
978 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
979 readKeyValueLen();
980 return 1;
981 }
982
983 @Override
984 protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
985 ByteBuffer buffer = curBlock.getBufferWithoutHeader();
986
987 buffer.rewind();
988 int klen = buffer.getInt();
989 buffer.getInt();
990 ByteBuffer keyBuff = buffer.slice();
991 keyBuff.limit(klen);
992 keyBuff.rewind();
993 return keyBuff;
994 }
995
996 @Override
997 public String getKeyString() {
998 return Bytes.toStringBinary(blockBuffer.array(),
999 blockBuffer.arrayOffset() + blockBuffer.position()
1000 + KEY_VALUE_LEN_SIZE, currKeyLen);
1001 }
1002
1003 @Override
1004 public String getValueString() {
1005 return Bytes.toString(blockBuffer.array(), blockBuffer.arrayOffset()
1006 + blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen,
1007 currValueLen);
1008 }
1009 }
1010
1011
1012
1013
1014 protected static class EncodedScannerV2 extends AbstractScannerV2 {
1015 private final HFileBlockDecodingContext decodingCtx;
1016 private final DataBlockEncoder.EncodedSeeker seeker;
1017 private final DataBlockEncoder dataBlockEncoder;
1018 protected final HFileContext meta;
1019
1020 public EncodedScannerV2(HFileReaderV2 reader, boolean cacheBlocks,
1021 boolean pread, boolean isCompaction, HFileContext meta) {
1022 super(reader, cacheBlocks, pread, isCompaction);
1023 DataBlockEncoding encoding = reader.dataBlockEncoder.getDataBlockEncoding();
1024 dataBlockEncoder = encoding.getEncoder();
1025 decodingCtx = dataBlockEncoder.newDataBlockDecodingContext(meta);
1026 seeker = dataBlockEncoder.createSeeker(
1027 reader.getComparator(), decodingCtx);
1028 this.meta = meta;
1029 }
1030
1031 @Override
1032 public boolean isSeeked(){
1033 return this.block != null;
1034 }
1035
1036
1037
1038
1039
1040
1041
1042
1043 private void updateCurrentBlock(HFileBlock newBlock) throws CorruptHFileException {
1044 block = newBlock;
1045
1046
1047 if (block.getBlockType() != BlockType.ENCODED_DATA) {
1048 throw new IllegalStateException(
1049 "EncodedScanner works only on encoded data blocks");
1050 }
1051 short dataBlockEncoderId = block.getDataBlockEncodingId();
1052 if (!DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) {
1053 String encoderCls = dataBlockEncoder.getClass().getName();
1054 throw new CorruptHFileException("Encoder " + encoderCls
1055 + " doesn't support data block encoding "
1056 + DataBlockEncoding.getNameFromId(dataBlockEncoderId));
1057 }
1058
1059 seeker.setCurrentBuffer(getEncodedBuffer(newBlock));
1060 blockFetches++;
1061
1062
1063 this.nextIndexedKey = null;
1064 }
1065
1066 private ByteBuffer getEncodedBuffer(HFileBlock newBlock) {
1067 ByteBuffer origBlock = newBlock.getBufferReadOnly();
1068 ByteBuffer encodedBlock = ByteBuffer.wrap(origBlock.array(),
1069 origBlock.arrayOffset() + newBlock.headerSize() +
1070 DataBlockEncoding.ID_SIZE,
1071 newBlock.getUncompressedSizeWithoutHeader() -
1072 DataBlockEncoding.ID_SIZE).slice();
1073 return encodedBlock;
1074 }
1075
1076 @Override
1077 public boolean seekTo() throws IOException {
1078 if (reader == null) {
1079 return false;
1080 }
1081
1082 if (reader.getTrailer().getEntryCount() == 0) {
1083
1084 return false;
1085 }
1086
1087 long firstDataBlockOffset =
1088 reader.getTrailer().getFirstDataBlockOffset();
1089 if (block != null && block.getOffset() == firstDataBlockOffset) {
1090 seeker.rewind();
1091 return true;
1092 }
1093
1094 block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
1095 isCompaction, true, BlockType.DATA);
1096 if (block.getOffset() < 0) {
1097 throw new IOException("Invalid block offset: " + block.getOffset());
1098 }
1099 updateCurrentBlock(block);
1100 return true;
1101 }
1102
1103 @Override
1104 public boolean next() throws IOException {
1105 boolean isValid = seeker.next();
1106 if (!isValid) {
1107 block = readNextDataBlock();
1108 isValid = block != null;
1109 if (isValid) {
1110 updateCurrentBlock(block);
1111 }
1112 }
1113 return isValid;
1114 }
1115
1116 @Override
1117 public ByteBuffer getKey() {
1118 assertValidSeek();
1119 return seeker.getKeyDeepCopy();
1120 }
1121
1122 @Override
1123 public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
1124 return seeker.compareKey(comparator, key, offset, length);
1125 }
1126
1127 @Override
1128 public ByteBuffer getValue() {
1129 assertValidSeek();
1130 return seeker.getValueShallowCopy();
1131 }
1132
1133 @Override
1134 public KeyValue getKeyValue() {
1135 if (block == null) {
1136 return null;
1137 }
1138 return seeker.getKeyValue();
1139 }
1140
1141 @Override
1142 public String getKeyString() {
1143 ByteBuffer keyBuffer = getKey();
1144 return Bytes.toStringBinary(keyBuffer.array(),
1145 keyBuffer.arrayOffset(), keyBuffer.limit());
1146 }
1147
1148 @Override
1149 public String getValueString() {
1150 ByteBuffer valueBuffer = getValue();
1151 return Bytes.toStringBinary(valueBuffer.array(),
1152 valueBuffer.arrayOffset(), valueBuffer.limit());
1153 }
1154
1155 private void assertValidSeek() {
1156 if (block == null) {
1157 throw new NotSeekedException();
1158 }
1159 }
1160
1161 @Override
1162 protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
1163 return dataBlockEncoder.getFirstKeyInBlock(getEncodedBuffer(curBlock));
1164 }
1165
1166 @Override
1167 protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
1168 boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
1169 throws IOException {
1170 if (block == null || block.getOffset() != seekToBlock.getOffset()) {
1171 updateCurrentBlock(seekToBlock);
1172 } else if (rewind) {
1173 seeker.rewind();
1174 }
1175 this.nextIndexedKey = nextIndexedKey;
1176 return seeker.seekToKeyInBlock(key, offset, length, seekBefore);
1177 }
1178 }
1179
1180
1181
1182
1183
1184 @Override
1185 public DataInput getGeneralBloomFilterMetadata() throws IOException {
1186 return this.getBloomFilterMetadata(BlockType.GENERAL_BLOOM_META);
1187 }
1188
1189 @Override
1190 public DataInput getDeleteBloomFilterMetadata() throws IOException {
1191 return this.getBloomFilterMetadata(BlockType.DELETE_FAMILY_BLOOM_META);
1192 }
1193
1194 private DataInput getBloomFilterMetadata(BlockType blockType)
1195 throws IOException {
1196 if (blockType != BlockType.GENERAL_BLOOM_META &&
1197 blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
1198 throw new RuntimeException("Block Type: " + blockType.toString() +
1199 " is not supported") ;
1200 }
1201
1202 for (HFileBlock b : loadOnOpenBlocks)
1203 if (b.getBlockType() == blockType)
1204 return b.getByteStream();
1205 return null;
1206 }
1207
1208 @Override
1209 public boolean isFileInfoLoaded() {
1210 return true;
1211 }
1212
1213
1214
1215
1216
1217 private void validateMinorVersion(Path path, int minorVersion) {
1218 if (minorVersion < MIN_MINOR_VERSION ||
1219 minorVersion > MAX_MINOR_VERSION) {
1220 String msg = "Minor version for path " + path +
1221 " is expected to be between " +
1222 MIN_MINOR_VERSION + " and " + MAX_MINOR_VERSION +
1223 " but is found to be " + minorVersion;
1224 LOG.error(msg);
1225 throw new RuntimeException(msg);
1226 }
1227 }
1228
1229 @Override
1230 public int getMajorVersion() {
1231 return 2;
1232 }
1233
1234 @Override
1235 public HFileContext getFileContext() {
1236 return hfileContext;
1237 }
1238
1239
1240
1241
1242
1243 @VisibleForTesting
1244 boolean prefetchComplete() {
1245 return PrefetchExecutor.isCompleted(path);
1246 }
1247 }