1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.DataInput;
21 import java.io.IOException;
22 import java.nio.ByteBuffer;
23 import java.util.ArrayList;
24 import java.util.List;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.hbase.classification.InterfaceAudience;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.KeyValue;
33 import org.apache.hadoop.hbase.KeyValue.KVComparator;
34 import org.apache.hadoop.hbase.NoTagsKeyValue;
35 import org.apache.hadoop.hbase.fs.HFileSystem;
36 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
37 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
38 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
39 import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
40 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
41 import org.apache.hadoop.hbase.util.ByteBufferUtils;
42 import org.apache.hadoop.hbase.util.Bytes;
43 import org.apache.hadoop.hbase.util.IdLock;
44 import org.apache.hadoop.io.WritableUtils;
45 import org.cloudera.htrace.Trace;
46 import org.cloudera.htrace.TraceScope;
47
48 import com.google.common.annotations.VisibleForTesting;
49
50
51
52
53 @InterfaceAudience.Private
54 public class HFileReaderV2 extends AbstractHFileReader {
55
56 private static final Log LOG = LogFactory.getLog(HFileReaderV2.class);
57
58
59 public static final int MINOR_VERSION_WITH_CHECKSUM = 1;
60
61 public static final int MINOR_VERSION_NO_CHECKSUM = 0;
62
63
64 public static final int PBUF_TRAILER_MINOR_VERSION = 2;
65
66
67
68
69
70 public final static int KEY_VALUE_LEN_SIZE = 2 * Bytes.SIZEOF_INT;
71
72 protected boolean includesMemstoreTS = false;
73 protected boolean decodeMemstoreTS = false;
74 protected boolean shouldIncludeMemstoreTS() {
75 return includesMemstoreTS;
76 }
77
78
79 protected HFileBlock.FSReader fsBlockReader;
80
81
82
83
84
85
86
87 private IdLock offsetLock = new IdLock();
88
89
90
91
92
93 private List<HFileBlock> loadOnOpenBlocks = new ArrayList<HFileBlock>();
94
95
96 static final int MIN_MINOR_VERSION = 0;
97
98
99
100
101 static final int MAX_MINOR_VERSION = 3;
102
103
104 static final int MINOR_VERSION_WITH_FAKED_KEY = 3;
105
106 protected HFileContext hfileContext;
107
108
109
110
111
112
113
114
115
116
117
118
119
120 public HFileReaderV2(final Path path, final FixedFileTrailer trailer,
121 final FSDataInputStreamWrapper fsdis, final long size, final CacheConfig cacheConf,
122 final HFileSystem hfs, final Configuration conf) throws IOException {
123 super(path, trailer, size, cacheConf, hfs, conf);
124 this.conf = conf;
125 trailer.expectMajorVersion(getMajorVersion());
126 validateMinorVersion(path, trailer.getMinorVersion());
127 this.hfileContext = createHFileContext(fsdis, fileSize, hfs, path, trailer);
128 HFileBlock.FSReaderV2 fsBlockReaderV2 = new HFileBlock.FSReaderV2(fsdis, fileSize, hfs, path,
129 hfileContext);
130 this.fsBlockReader = fsBlockReaderV2;
131
132
133 comparator = trailer.createComparator();
134 dataBlockIndexReader = new HFileBlockIndex.BlockIndexReader(comparator,
135 trailer.getNumDataIndexLevels(), this);
136 metaBlockIndexReader = new HFileBlockIndex.BlockIndexReader(
137 KeyValue.RAW_COMPARATOR, 1);
138
139
140
141 HFileBlock.BlockIterator blockIter = fsBlockReaderV2.blockRange(
142 trailer.getLoadOnOpenDataOffset(),
143 fileSize - trailer.getTrailerSize());
144
145
146
147 dataBlockIndexReader.readMultiLevelIndexRoot(
148 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
149 trailer.getDataIndexCount());
150
151
152 metaBlockIndexReader.readRootIndex(
153 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
154 trailer.getMetaIndexCount());
155
156
157 fileInfo = new FileInfo();
158 fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
159 lastKey = fileInfo.get(FileInfo.LASTKEY);
160 avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN));
161 avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN));
162 byte [] keyValueFormatVersion =
163 fileInfo.get(HFileWriterV2.KEY_VALUE_VERSION);
164 includesMemstoreTS = keyValueFormatVersion != null &&
165 Bytes.toInt(keyValueFormatVersion) ==
166 HFileWriterV2.KEY_VALUE_VER_WITH_MEMSTORE;
167 fsBlockReaderV2.setIncludesMemstoreTS(includesMemstoreTS);
168 if (includesMemstoreTS) {
169 decodeMemstoreTS = Bytes.toLong(fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY)) > 0;
170 }
171
172
173 dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo);
174 fsBlockReaderV2.setDataBlockEncoder(dataBlockEncoder);
175
176
177 HFileBlock b;
178 while ((b = blockIter.nextBlock()) != null) {
179 loadOnOpenBlocks.add(b);
180 }
181
182
183 if (cacheConf.shouldPrefetchOnOpen()) {
184 PrefetchExecutor.request(path, new Runnable() {
185 public void run() {
186 try {
187 long offset = 0;
188 long end = fileSize - getTrailer().getTrailerSize();
189 HFileBlock prevBlock = null;
190 while (offset < end) {
191 if (Thread.interrupted()) {
192 break;
193 }
194 long onDiskSize = -1;
195 if (prevBlock != null) {
196 onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
197 }
198 HFileBlock block = readBlock(offset, onDiskSize, true, false, false, false, null);
199 prevBlock = block;
200 offset += block.getOnDiskSizeWithHeader();
201 }
202 } catch (IOException e) {
203
204 if (LOG.isTraceEnabled()) {
205 LOG.trace("Exception encountered while prefetching " + path + ":", e);
206 }
207 } catch (Exception e) {
208
209 LOG.warn("Exception encountered while prefetching " + path + ":", e);
210 } finally {
211 PrefetchExecutor.complete(path);
212 }
213 }
214 });
215 }
216 }
217
218 protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
219 HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
220 return new HFileContextBuilder()
221 .withIncludesMvcc(this.includesMemstoreTS)
222 .withCompression(this.compressAlgo)
223 .withHBaseCheckSum(trailer.getMinorVersion() >= MINOR_VERSION_WITH_CHECKSUM)
224 .build();
225 }
226
227
228
229
230
231
232
233
234
235
236
237
238
239 @Override
240 public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
241 final boolean isCompaction) {
242 if (dataBlockEncoder.useEncodedScanner()) {
243 return new EncodedScannerV2(this, cacheBlocks, pread, isCompaction,
244 hfileContext);
245 }
246
247 return new ScannerV2(this, cacheBlocks, pread, isCompaction);
248 }
249
250
251
252
253
254
255
256 @Override
257 public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock)
258 throws IOException {
259 if (trailer.getMetaIndexCount() == 0) {
260 return null;
261 }
262 if (metaBlockIndexReader == null) {
263 throw new IOException("Meta index not loaded");
264 }
265
266 byte[] mbname = Bytes.toBytes(metaBlockName);
267 int block = metaBlockIndexReader.rootBlockContainingKey(mbname, 0,
268 mbname.length);
269 if (block == -1)
270 return null;
271 long blockSize = metaBlockIndexReader.getRootBlockDataSize(block);
272
273
274
275
276 synchronized (metaBlockIndexReader.getRootBlockKey(block)) {
277
278 long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block);
279 BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset,
280 DataBlockEncoding.NONE, BlockType.META);
281
282 cacheBlock &= cacheConf.shouldCacheDataOnRead();
283 if (cacheConf.isBlockCacheEnabled()) {
284 HFileBlock cachedBlock =
285 (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey, cacheBlock, false, true);
286 if (cachedBlock != null) {
287 assert cachedBlock.isUnpacked() : "Packed block leak.";
288
289
290 return cachedBlock.getBufferWithoutHeader();
291 }
292
293 }
294
295 HFileBlock metaBlock = fsBlockReader.readBlockData(metaBlockOffset,
296 blockSize, -1, true).unpack(hfileContext, fsBlockReader);
297
298
299 if (cacheBlock) {
300 cacheConf.getBlockCache().cacheBlock(cacheKey, metaBlock,
301 cacheConf.isInMemory());
302 }
303
304 return metaBlock.getBufferWithoutHeader();
305 }
306 }
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323 @Override
324 public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize,
325 final boolean cacheBlock, boolean pread, final boolean isCompaction,
326 final boolean updateCacheMetrics, BlockType expectedBlockType)
327 throws IOException {
328 if (dataBlockIndexReader == null) {
329 throw new IOException("Block index not loaded");
330 }
331 if (dataBlockOffset < 0
332 || dataBlockOffset >= trailer.getLoadOnOpenDataOffset()) {
333 throw new IOException("Requested block is out of range: "
334 + dataBlockOffset + ", lastDataBlockOffset: "
335 + trailer.getLastDataBlockOffset());
336 }
337
338
339
340
341
342
343 BlockCacheKey cacheKey =
344 new BlockCacheKey(name, dataBlockOffset,
345 dataBlockEncoder.getDataBlockEncoding(),
346 expectedBlockType);
347
348 boolean useLock = false;
349 IdLock.Entry lockEntry = null;
350 TraceScope traceScope = Trace.startSpan("HFileReaderV2.readBlock");
351 try {
352 while (true) {
353
354 if (cacheConf.shouldReadBlockFromCache(expectedBlockType)) {
355 if (useLock) {
356 lockEntry = offsetLock.getLockEntry(dataBlockOffset);
357 }
358
359
360 HFileBlock cachedBlock = (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey,
361 cacheBlock, useLock, updateCacheMetrics);
362 if (cachedBlock != null) {
363 if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) {
364 cachedBlock = cachedBlock.unpack(hfileContext, fsBlockReader);
365 }
366 if (Trace.isTracing()) {
367 traceScope.getSpan().addTimelineAnnotation("blockCacheHit");
368 }
369 assert cachedBlock.isUnpacked() : "Packed block leak.";
370 if (cachedBlock.getBlockType().isData()) {
371 HFile.dataBlockReadCnt.incrementAndGet();
372
373
374
375 if (cachedBlock.getDataBlockEncoding() != dataBlockEncoder.getDataBlockEncoding()) {
376 throw new IOException("Cached block under key " + cacheKey + " "
377 + "has wrong encoding: " + cachedBlock.getDataBlockEncoding() + " (expected: "
378 + dataBlockEncoder.getDataBlockEncoding() + ")");
379 }
380 }
381 return cachedBlock;
382 }
383 if (!useLock && cacheBlock && cacheConf.shouldLockOnCacheMiss(expectedBlockType)) {
384
385 useLock = true;
386 continue;
387 }
388
389 }
390
391 if (Trace.isTracing()) {
392 traceScope.getSpan().addTimelineAnnotation("blockCacheMiss");
393 }
394
395 HFileBlock hfileBlock = fsBlockReader.readBlockData(dataBlockOffset, onDiskBlockSize, -1,
396 pread);
397 validateBlockType(hfileBlock, expectedBlockType);
398 HFileBlock unpacked = hfileBlock.unpack(hfileContext, fsBlockReader);
399 BlockType.BlockCategory category = hfileBlock.getBlockType().getCategory();
400
401
402 if (cacheBlock && cacheConf.shouldCacheBlockOnRead(category)) {
403 cacheConf.getBlockCache().cacheBlock(cacheKey,
404 cacheConf.shouldCacheCompressed(category) ? hfileBlock : unpacked,
405 cacheConf.isInMemory());
406 }
407
408 if (updateCacheMetrics && hfileBlock.getBlockType().isData()) {
409 HFile.dataBlockReadCnt.incrementAndGet();
410 }
411
412 return unpacked;
413 }
414 } finally {
415 traceScope.close();
416 if (lockEntry != null) {
417 offsetLock.releaseLockEntry(lockEntry);
418 }
419 }
420 }
421
422 @Override
423 public boolean hasMVCCInfo() {
424 return includesMemstoreTS && decodeMemstoreTS;
425 }
426
427
428
429
430
431
432
433
434
435
436 private void validateBlockType(HFileBlock block,
437 BlockType expectedBlockType) throws IOException {
438 if (expectedBlockType == null) {
439 return;
440 }
441 BlockType actualBlockType = block.getBlockType();
442 if (actualBlockType == BlockType.ENCODED_DATA &&
443 expectedBlockType == BlockType.DATA) {
444
445
446 return;
447 }
448 if (actualBlockType != expectedBlockType) {
449 throw new IOException("Expected block type " + expectedBlockType + ", " +
450 "but got " + actualBlockType + ": " + block);
451 }
452 }
453
454
455
456
457
458
459 @Override
460 public byte[] getLastKey() {
461 return dataBlockIndexReader.isEmpty() ? null : lastKey;
462 }
463
464
465
466
467
468
469 @Override
470 public byte[] midkey() throws IOException {
471 return dataBlockIndexReader.midkey();
472 }
473
474 @Override
475 public void close() throws IOException {
476 close(cacheConf.shouldEvictOnClose());
477 }
478
479 public void close(boolean evictOnClose) throws IOException {
480 PrefetchExecutor.cancel(path);
481 if (evictOnClose && cacheConf.isBlockCacheEnabled()) {
482 int numEvicted = cacheConf.getBlockCache().evictBlocksByHfileName(name);
483 if (LOG.isTraceEnabled()) {
484 LOG.trace("On close, file=" + name + " evicted=" + numEvicted
485 + " block(s)");
486 }
487 }
488 fsBlockReader.closeStreams();
489 }
490
491
492 @Override
493 HFileBlock.FSReader getUncachedBlockReader() {
494 return fsBlockReader;
495 }
496
497
498 protected abstract static class AbstractScannerV2
499 extends AbstractHFileReader.Scanner {
500 protected HFileBlock block;
501
502 @Override
503 public byte[] getNextIndexedKey() {
504 return nextIndexedKey;
505 }
506
507
508
509
510
511
512
513 protected byte[] nextIndexedKey;
514
515 public AbstractScannerV2(HFileReaderV2 r, boolean cacheBlocks,
516 final boolean pread, final boolean isCompaction) {
517 super(r, cacheBlocks, pread, isCompaction);
518 }
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536 protected int seekTo(byte[] key, int offset, int length, boolean rewind)
537 throws IOException {
538 HFileBlockIndex.BlockIndexReader indexReader =
539 reader.getDataBlockIndexReader();
540 BlockWithScanInfo blockWithScanInfo =
541 indexReader.loadDataBlockWithScanInfo(key, offset, length, block,
542 cacheBlocks, pread, isCompaction);
543 if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) {
544
545 return -1;
546 }
547 return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(),
548 blockWithScanInfo.getNextIndexedKey(), rewind, key, offset, length, false);
549 }
550
551 protected abstract ByteBuffer getFirstKeyInBlock(HFileBlock curBlock);
552
553 protected abstract int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
554 boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
555 throws IOException;
556
557 @Override
558 public int seekTo(byte[] key, int offset, int length) throws IOException {
559
560
561 return seekTo(key, offset, length, true);
562 }
563
564 @Override
565 public int reseekTo(byte[] key, int offset, int length) throws IOException {
566 int compared;
567 if (isSeeked()) {
568 compared = compareKey(reader.getComparator(), key, offset, length);
569 if (compared < 1) {
570
571
572 return compared;
573 } else {
574 if (this.nextIndexedKey != null &&
575 (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY ||
576 reader.getComparator().compareFlatKey(key, offset, length,
577 nextIndexedKey, 0, nextIndexedKey.length) < 0)) {
578
579
580
581 return loadBlockAndSeekToKey(this.block, this.nextIndexedKey,
582 false, key, offset, length, false);
583 }
584 }
585 }
586
587
588 return seekTo(key, offset, length, false);
589 }
590
591 @Override
592 public boolean seekBefore(byte[] key, int offset, int length)
593 throws IOException {
594 HFileBlock seekToBlock =
595 reader.getDataBlockIndexReader().seekToDataBlock(key, offset, length,
596 block, cacheBlocks, pread, isCompaction);
597 if (seekToBlock == null) {
598 return false;
599 }
600 ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock);
601
602 if (reader.getComparator().compareFlatKey(firstKey.array(),
603 firstKey.arrayOffset(), firstKey.limit(), key, offset, length) >= 0)
604 {
605 long previousBlockOffset = seekToBlock.getPrevBlockOffset();
606
607 if (previousBlockOffset == -1) {
608
609 return false;
610 }
611
612
613
614
615
616
617 int prevBlockSize = -1;
618 seekToBlock = reader.readBlock(previousBlockOffset,
619 prevBlockSize, cacheBlocks,
620 pread, isCompaction, true, BlockType.DATA);
621
622
623 }
624 byte[] firstKeyInCurrentBlock = Bytes.getBytes(firstKey);
625 loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, offset, length, true);
626 return true;
627 }
628
629
630
631
632
633
634
635
636
637 protected HFileBlock readNextDataBlock() throws IOException {
638 long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset();
639 if (block == null)
640 return null;
641
642 HFileBlock curBlock = block;
643
644 do {
645 if (curBlock.getOffset() >= lastDataBlockOffset)
646 return null;
647
648 if (curBlock.getOffset() < 0) {
649 throw new IOException("Invalid block file offset: " + block);
650 }
651
652
653
654 curBlock = reader.readBlock(curBlock.getOffset()
655 + curBlock.getOnDiskSizeWithHeader(),
656 curBlock.getNextBlockOnDiskSizeWithHeader(), cacheBlocks, pread,
657 isCompaction, true, null);
658 } while (!curBlock.getBlockType().isData());
659
660 return curBlock;
661 }
662
663
664
665
666
667
668
669
670 public abstract int compareKey(KVComparator comparator, byte[] key, int offset,
671 int length);
672 }
673
674
675
676
677 protected static class ScannerV2 extends AbstractScannerV2 {
678 private HFileReaderV2 reader;
679
680 public ScannerV2(HFileReaderV2 r, boolean cacheBlocks,
681 final boolean pread, final boolean isCompaction) {
682 super(r, cacheBlocks, pread, isCompaction);
683 this.reader = r;
684 }
685
686 @Override
687 public KeyValue getKeyValue() {
688 if (!isSeeked())
689 return null;
690
691
692 return formNoTagsKeyValue();
693 }
694
695 protected KeyValue formNoTagsKeyValue() {
696 KeyValue ret = new NoTagsKeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
697 + blockBuffer.position(), getCellBufSize());
698 if (this.reader.shouldIncludeMemstoreTS()) {
699 ret.setMvccVersion(currMemstoreTS);
700 }
701 return ret;
702 }
703
704 protected int getCellBufSize() {
705 return KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen;
706 }
707
708 @Override
709 public ByteBuffer getKey() {
710 assertSeeked();
711 return ByteBuffer.wrap(
712 blockBuffer.array(),
713 blockBuffer.arrayOffset() + blockBuffer.position()
714 + KEY_VALUE_LEN_SIZE, currKeyLen).slice();
715 }
716
717 @Override
718 public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
719 return comparator.compareFlatKey(key, offset, length, blockBuffer.array(),
720 blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen);
721 }
722
723 @Override
724 public ByteBuffer getValue() {
725 assertSeeked();
726 return ByteBuffer.wrap(
727 blockBuffer.array(),
728 blockBuffer.arrayOffset() + blockBuffer.position()
729 + KEY_VALUE_LEN_SIZE + currKeyLen, currValueLen).slice();
730 }
731
732 protected void setNonSeekedState() {
733 block = null;
734 blockBuffer = null;
735 currKeyLen = 0;
736 currValueLen = 0;
737 currMemstoreTS = 0;
738 currMemstoreTSLen = 0;
739 }
740
741
742
743
744
745
746
747
748 @Override
749 public boolean next() throws IOException {
750 assertSeeked();
751
752 try {
753 blockBuffer.position(getNextCellStartPosition());
754 } catch (IllegalArgumentException e) {
755 LOG.error("Current pos = " + blockBuffer.position()
756 + "; currKeyLen = " + currKeyLen + "; currValLen = "
757 + currValueLen + "; block limit = " + blockBuffer.limit()
758 + "; HFile name = " + reader.getName()
759 + "; currBlock currBlockOffset = " + block.getOffset());
760 throw e;
761 }
762
763 if (blockBuffer.remaining() <= 0) {
764 long lastDataBlockOffset =
765 reader.getTrailer().getLastDataBlockOffset();
766
767 if (block.getOffset() >= lastDataBlockOffset) {
768 setNonSeekedState();
769 return false;
770 }
771
772
773 HFileBlock nextBlock = readNextDataBlock();
774 if (nextBlock == null) {
775 setNonSeekedState();
776 return false;
777 }
778
779 updateCurrBlock(nextBlock);
780 return true;
781 }
782
783
784 readKeyValueLen();
785 return true;
786 }
787
788 protected int getNextCellStartPosition() {
789 return blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen
790 + currMemstoreTSLen;
791 }
792
793
794
795
796
797
798
799
800 @Override
801 public boolean seekTo() throws IOException {
802 if (reader == null) {
803 return false;
804 }
805
806 if (reader.getTrailer().getEntryCount() == 0) {
807
808 return false;
809 }
810
811 long firstDataBlockOffset =
812 reader.getTrailer().getFirstDataBlockOffset();
813 if (block != null && block.getOffset() == firstDataBlockOffset) {
814 blockBuffer.rewind();
815 readKeyValueLen();
816 return true;
817 }
818
819 block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
820 isCompaction, true, BlockType.DATA);
821 if (block.getOffset() < 0) {
822 throw new IOException("Invalid block offset: " + block.getOffset());
823 }
824 updateCurrBlock(block);
825 return true;
826 }
827
828 @Override
829 protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
830 boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
831 throws IOException {
832 if (block == null || block.getOffset() != seekToBlock.getOffset()) {
833 updateCurrBlock(seekToBlock);
834 } else if (rewind) {
835 blockBuffer.rewind();
836 }
837
838
839 this.nextIndexedKey = nextIndexedKey;
840 return blockSeek(key, offset, length, seekBefore);
841 }
842
843
844
845
846
847
848
849 protected void updateCurrBlock(HFileBlock newBlock) {
850 block = newBlock;
851
852
853 if (block.getBlockType() != BlockType.DATA) {
854 throw new IllegalStateException("ScannerV2 works only on data " +
855 "blocks, got " + block.getBlockType() + "; " +
856 "fileName=" + reader.name + ", " +
857 "dataBlockEncoder=" + reader.dataBlockEncoder + ", " +
858 "isCompaction=" + isCompaction);
859 }
860
861 blockBuffer = block.getBufferWithoutHeader();
862 readKeyValueLen();
863 blockFetches++;
864
865
866 this.nextIndexedKey = null;
867 }
868
869 protected void readKeyValueLen() {
870 blockBuffer.mark();
871 currKeyLen = blockBuffer.getInt();
872 currValueLen = blockBuffer.getInt();
873 ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
874 readMvccVersion();
875 if (currKeyLen < 0 || currValueLen < 0
876 || currKeyLen > blockBuffer.limit()
877 || currValueLen > blockBuffer.limit()) {
878 throw new IllegalStateException("Invalid currKeyLen " + currKeyLen
879 + " or currValueLen " + currValueLen + ". Block offset: "
880 + block.getOffset() + ", block length: " + blockBuffer.limit()
881 + ", position: " + blockBuffer.position() + " (without header).");
882 }
883 blockBuffer.reset();
884 }
885
886 protected void readMvccVersion() {
887 if (this.reader.shouldIncludeMemstoreTS()) {
888 if (this.reader.decodeMemstoreTS) {
889 currMemstoreTS = Bytes.readAsVLong(blockBuffer.array(), blockBuffer.arrayOffset()
890 + blockBuffer.position());
891 currMemstoreTSLen = WritableUtils.getVIntSize(currMemstoreTS);
892 } else {
893 currMemstoreTS = 0;
894 currMemstoreTSLen = 1;
895 }
896 }
897 }
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914 protected int blockSeek(byte[] key, int offset, int length,
915 boolean seekBefore) {
916 int klen, vlen;
917 long memstoreTS = 0;
918 int memstoreTSLen = 0;
919 int lastKeyValueSize = -1;
920 do {
921 blockBuffer.mark();
922 klen = blockBuffer.getInt();
923 vlen = blockBuffer.getInt();
924 blockBuffer.reset();
925 if (this.reader.shouldIncludeMemstoreTS()) {
926 if (this.reader.decodeMemstoreTS) {
927 int memstoreTSOffset = blockBuffer.arrayOffset() + blockBuffer.position()
928 + KEY_VALUE_LEN_SIZE + klen + vlen;
929 memstoreTS = Bytes.readAsVLong(blockBuffer.array(), memstoreTSOffset);
930 memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
931 } else {
932 memstoreTS = 0;
933 memstoreTSLen = 1;
934 }
935 }
936
937 int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position()
938 + KEY_VALUE_LEN_SIZE;
939 int comp = reader.getComparator().compareFlatKey(key, offset, length,
940 blockBuffer.array(), keyOffset, klen);
941
942 if (comp == 0) {
943 if (seekBefore) {
944 if (lastKeyValueSize < 0) {
945 throw new IllegalStateException("blockSeek with seekBefore "
946 + "at the first key of the block: key="
947 + Bytes.toStringBinary(key) + ", blockOffset="
948 + block.getOffset() + ", onDiskSize="
949 + block.getOnDiskSizeWithHeader());
950 }
951 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
952 readKeyValueLen();
953 return 1;
954 }
955 currKeyLen = klen;
956 currValueLen = vlen;
957 if (this.reader.shouldIncludeMemstoreTS()) {
958 currMemstoreTS = memstoreTS;
959 currMemstoreTSLen = memstoreTSLen;
960 }
961 return 0;
962 } else if (comp < 0) {
963 if (lastKeyValueSize > 0)
964 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
965 readKeyValueLen();
966 if (lastKeyValueSize == -1 && blockBuffer.position() == 0
967 && this.reader.trailer.getMinorVersion() >= MINOR_VERSION_WITH_FAKED_KEY) {
968 return HConstants.INDEX_KEY_MAGIC;
969 }
970 return 1;
971 }
972
973
974 lastKeyValueSize = klen + vlen + memstoreTSLen + KEY_VALUE_LEN_SIZE;
975 blockBuffer.position(blockBuffer.position() + lastKeyValueSize);
976 } while (blockBuffer.remaining() > 0);
977
978
979
980
981 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
982 readKeyValueLen();
983 return 1;
984 }
985
986 @Override
987 protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
988 ByteBuffer buffer = curBlock.getBufferWithoutHeader();
989
990 buffer.rewind();
991 int klen = buffer.getInt();
992 buffer.getInt();
993 ByteBuffer keyBuff = buffer.slice();
994 keyBuff.limit(klen);
995 keyBuff.rewind();
996 return keyBuff;
997 }
998
999 @Override
1000 public String getKeyString() {
1001 return Bytes.toStringBinary(blockBuffer.array(),
1002 blockBuffer.arrayOffset() + blockBuffer.position()
1003 + KEY_VALUE_LEN_SIZE, currKeyLen);
1004 }
1005
1006 @Override
1007 public String getValueString() {
1008 return Bytes.toString(blockBuffer.array(), blockBuffer.arrayOffset()
1009 + blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen,
1010 currValueLen);
1011 }
1012 }
1013
1014
1015
1016
1017 protected static class EncodedScannerV2 extends AbstractScannerV2 {
1018 private final HFileBlockDecodingContext decodingCtx;
1019 private final DataBlockEncoder.EncodedSeeker seeker;
1020 private final DataBlockEncoder dataBlockEncoder;
1021 protected final HFileContext meta;
1022
1023 public EncodedScannerV2(HFileReaderV2 reader, boolean cacheBlocks,
1024 boolean pread, boolean isCompaction, HFileContext meta) {
1025 super(reader, cacheBlocks, pread, isCompaction);
1026 DataBlockEncoding encoding = reader.dataBlockEncoder.getDataBlockEncoding();
1027 dataBlockEncoder = encoding.getEncoder();
1028 decodingCtx = dataBlockEncoder.newDataBlockDecodingContext(meta);
1029 seeker = dataBlockEncoder.createSeeker(
1030 reader.getComparator(), decodingCtx);
1031 this.meta = meta;
1032 }
1033
1034 @Override
1035 public boolean isSeeked(){
1036 return this.block != null;
1037 }
1038
1039
1040
1041
1042
1043
1044
1045
1046 private void updateCurrentBlock(HFileBlock newBlock) throws CorruptHFileException {
1047 block = newBlock;
1048
1049
1050 if (block.getBlockType() != BlockType.ENCODED_DATA) {
1051 throw new IllegalStateException(
1052 "EncodedScanner works only on encoded data blocks");
1053 }
1054 short dataBlockEncoderId = block.getDataBlockEncodingId();
1055 if (!DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) {
1056 String encoderCls = dataBlockEncoder.getClass().getName();
1057 throw new CorruptHFileException("Encoder " + encoderCls
1058 + " doesn't support data block encoding "
1059 + DataBlockEncoding.getNameFromId(dataBlockEncoderId));
1060 }
1061
1062 seeker.setCurrentBuffer(getEncodedBuffer(newBlock));
1063 blockFetches++;
1064
1065
1066 this.nextIndexedKey = null;
1067 }
1068
1069 private ByteBuffer getEncodedBuffer(HFileBlock newBlock) {
1070 ByteBuffer origBlock = newBlock.getBufferReadOnly();
1071 ByteBuffer encodedBlock = ByteBuffer.wrap(origBlock.array(),
1072 origBlock.arrayOffset() + newBlock.headerSize() +
1073 DataBlockEncoding.ID_SIZE,
1074 newBlock.getUncompressedSizeWithoutHeader() -
1075 DataBlockEncoding.ID_SIZE).slice();
1076 return encodedBlock;
1077 }
1078
1079 @Override
1080 public boolean seekTo() throws IOException {
1081 if (reader == null) {
1082 return false;
1083 }
1084
1085 if (reader.getTrailer().getEntryCount() == 0) {
1086
1087 return false;
1088 }
1089
1090 long firstDataBlockOffset =
1091 reader.getTrailer().getFirstDataBlockOffset();
1092 if (block != null && block.getOffset() == firstDataBlockOffset) {
1093 seeker.rewind();
1094 return true;
1095 }
1096
1097 block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
1098 isCompaction, true, BlockType.DATA);
1099 if (block.getOffset() < 0) {
1100 throw new IOException("Invalid block offset: " + block.getOffset());
1101 }
1102 updateCurrentBlock(block);
1103 return true;
1104 }
1105
1106 @Override
1107 public boolean next() throws IOException {
1108 boolean isValid = seeker.next();
1109 if (!isValid) {
1110 block = readNextDataBlock();
1111 isValid = block != null;
1112 if (isValid) {
1113 updateCurrentBlock(block);
1114 }
1115 }
1116 return isValid;
1117 }
1118
1119 @Override
1120 public ByteBuffer getKey() {
1121 assertValidSeek();
1122 return seeker.getKeyDeepCopy();
1123 }
1124
1125 @Override
1126 public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
1127 return seeker.compareKey(comparator, key, offset, length);
1128 }
1129
1130 @Override
1131 public ByteBuffer getValue() {
1132 assertValidSeek();
1133 return seeker.getValueShallowCopy();
1134 }
1135
1136 @Override
1137 public KeyValue getKeyValue() {
1138 if (block == null) {
1139 return null;
1140 }
1141 return seeker.getKeyValue();
1142 }
1143
1144 @Override
1145 public String getKeyString() {
1146 ByteBuffer keyBuffer = getKey();
1147 return Bytes.toStringBinary(keyBuffer.array(),
1148 keyBuffer.arrayOffset(), keyBuffer.limit());
1149 }
1150
1151 @Override
1152 public String getValueString() {
1153 ByteBuffer valueBuffer = getValue();
1154 return Bytes.toStringBinary(valueBuffer.array(),
1155 valueBuffer.arrayOffset(), valueBuffer.limit());
1156 }
1157
1158 private void assertValidSeek() {
1159 if (block == null) {
1160 throw new NotSeekedException();
1161 }
1162 }
1163
1164 @Override
1165 protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
1166 return dataBlockEncoder.getFirstKeyInBlock(getEncodedBuffer(curBlock));
1167 }
1168
1169 @Override
1170 protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
1171 boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
1172 throws IOException {
1173 if (block == null || block.getOffset() != seekToBlock.getOffset()) {
1174 updateCurrentBlock(seekToBlock);
1175 } else if (rewind) {
1176 seeker.rewind();
1177 }
1178 this.nextIndexedKey = nextIndexedKey;
1179 return seeker.seekToKeyInBlock(key, offset, length, seekBefore);
1180 }
1181 }
1182
1183
1184
1185
1186
1187 @Override
1188 public DataInput getGeneralBloomFilterMetadata() throws IOException {
1189 return this.getBloomFilterMetadata(BlockType.GENERAL_BLOOM_META);
1190 }
1191
1192 @Override
1193 public DataInput getDeleteBloomFilterMetadata() throws IOException {
1194 return this.getBloomFilterMetadata(BlockType.DELETE_FAMILY_BLOOM_META);
1195 }
1196
1197 private DataInput getBloomFilterMetadata(BlockType blockType)
1198 throws IOException {
1199 if (blockType != BlockType.GENERAL_BLOOM_META &&
1200 blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
1201 throw new RuntimeException("Block Type: " + blockType.toString() +
1202 " is not supported") ;
1203 }
1204
1205 for (HFileBlock b : loadOnOpenBlocks)
1206 if (b.getBlockType() == blockType)
1207 return b.getByteStream();
1208 return null;
1209 }
1210
1211 @Override
1212 public boolean isFileInfoLoaded() {
1213 return true;
1214 }
1215
1216
1217
1218
1219
1220 private void validateMinorVersion(Path path, int minorVersion) {
1221 if (minorVersion < MIN_MINOR_VERSION ||
1222 minorVersion > MAX_MINOR_VERSION) {
1223 String msg = "Minor version for path " + path +
1224 " is expected to be between " +
1225 MIN_MINOR_VERSION + " and " + MAX_MINOR_VERSION +
1226 " but is found to be " + minorVersion;
1227 LOG.error(msg);
1228 throw new RuntimeException(msg);
1229 }
1230 }
1231
1232 @Override
1233 public int getMajorVersion() {
1234 return 2;
1235 }
1236
1237 @Override
1238 public HFileContext getFileContext() {
1239 return hfileContext;
1240 }
1241
1242
1243
1244
1245
1246 @VisibleForTesting
1247 boolean prefetchComplete() {
1248 return PrefetchExecutor.isCompleted(path);
1249 }
1250 }