1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.DataInput;
21 import java.io.IOException;
22 import java.nio.ByteBuffer;
23 import java.util.ArrayList;
24 import java.util.List;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.hbase.classification.InterfaceAudience;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hbase.Cell;
32 import org.apache.hadoop.hbase.CellUtil;
33 import org.apache.hadoop.hbase.HConstants;
34 import org.apache.hadoop.hbase.KeyValue;
35 import org.apache.hadoop.hbase.KeyValue.KVComparator;
36 import org.apache.hadoop.hbase.fs.HFileSystem;
37 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
38 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
39 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
40 import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
41 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
42 import org.apache.hadoop.hbase.util.ByteBufferUtils;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.util.IdLock;
45 import org.apache.hadoop.io.WritableUtils;
46 import org.apache.htrace.Trace;
47 import org.apache.htrace.TraceScope;
48
49 import com.google.common.annotations.VisibleForTesting;
50
51
52
53
54 @InterfaceAudience.Private
55 public class HFileReaderV2 extends AbstractHFileReader {
56
57 private static final Log LOG = LogFactory.getLog(HFileReaderV2.class);
58
59
60 public static final int MINOR_VERSION_WITH_CHECKSUM = 1;
61
62 public static final int MINOR_VERSION_NO_CHECKSUM = 0;
63
64
65 public static final int PBUF_TRAILER_MINOR_VERSION = 2;
66
67
68
69
70
71 public final static int KEY_VALUE_LEN_SIZE = 2 * Bytes.SIZEOF_INT;
72
73 protected boolean includesMemstoreTS = false;
74 protected boolean decodeMemstoreTS = false;
75 protected boolean shouldIncludeMemstoreTS() {
76 return includesMemstoreTS;
77 }
78
79
80 protected HFileBlock.FSReader fsBlockReader;
81
82
83
84
85
86
87
88 private IdLock offsetLock = new IdLock();
89
90
91
92
93
94 private List<HFileBlock> loadOnOpenBlocks = new ArrayList<HFileBlock>();
95
96
97 static final int MIN_MINOR_VERSION = 0;
98
99
100
101
102 static final int MAX_MINOR_VERSION = 3;
103
104
105 static final int MINOR_VERSION_WITH_FAKED_KEY = 3;
106
107 protected HFileContext hfileContext;
108
109
110
111
112
113
114
115
116
117
118
119
120
121 public HFileReaderV2(final Path path, final FixedFileTrailer trailer,
122 final FSDataInputStreamWrapper fsdis, final long size, final CacheConfig cacheConf,
123 final HFileSystem hfs, final Configuration conf) throws IOException {
124 super(path, trailer, size, cacheConf, hfs, conf);
125 this.conf = conf;
126 trailer.expectMajorVersion(getMajorVersion());
127 validateMinorVersion(path, trailer.getMinorVersion());
128 this.hfileContext = createHFileContext(fsdis, fileSize, hfs, path, trailer);
129 HFileBlock.FSReaderImpl fsBlockReaderV2 =
130 new HFileBlock.FSReaderImpl(fsdis, fileSize, hfs, path, hfileContext);
131 this.fsBlockReader = fsBlockReaderV2;
132
133
134 comparator = trailer.createComparator();
135 dataBlockIndexReader = new HFileBlockIndex.BlockIndexReader(comparator,
136 trailer.getNumDataIndexLevels(), this);
137 metaBlockIndexReader = new HFileBlockIndex.BlockIndexReader(
138 KeyValue.RAW_COMPARATOR, 1);
139
140
141
142 HFileBlock.BlockIterator blockIter = fsBlockReaderV2.blockRange(
143 trailer.getLoadOnOpenDataOffset(),
144 fileSize - trailer.getTrailerSize());
145
146
147
148 dataBlockIndexReader.readMultiLevelIndexRoot(
149 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
150 trailer.getDataIndexCount());
151
152
153 metaBlockIndexReader.readRootIndex(
154 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
155 trailer.getMetaIndexCount());
156
157
158 fileInfo = new FileInfo();
159 fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
160 lastKey = fileInfo.get(FileInfo.LASTKEY);
161 avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN));
162 avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN));
163 byte [] keyValueFormatVersion =
164 fileInfo.get(HFileWriterV2.KEY_VALUE_VERSION);
165 includesMemstoreTS = keyValueFormatVersion != null &&
166 Bytes.toInt(keyValueFormatVersion) ==
167 HFileWriterV2.KEY_VALUE_VER_WITH_MEMSTORE;
168 fsBlockReaderV2.setIncludesMemstoreTS(includesMemstoreTS);
169 if (includesMemstoreTS) {
170 decodeMemstoreTS = Bytes.toLong(fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY)) > 0;
171 }
172
173
174 dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo);
175 fsBlockReaderV2.setDataBlockEncoder(dataBlockEncoder);
176
177
178 HFileBlock b;
179 while ((b = blockIter.nextBlock()) != null) {
180 loadOnOpenBlocks.add(b);
181 }
182
183
184 if (cacheConf.shouldPrefetchOnOpen()) {
185 PrefetchExecutor.request(path, new Runnable() {
186 public void run() {
187 try {
188 long offset = 0;
189 long end = fileSize - getTrailer().getTrailerSize();
190 HFileBlock prevBlock = null;
191 while (offset < end) {
192 if (Thread.interrupted()) {
193 break;
194 }
195 long onDiskSize = -1;
196 if (prevBlock != null) {
197 onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
198 }
199 HFileBlock block = readBlock(offset, onDiskSize, true, false, false, false,
200 null, null);
201 prevBlock = block;
202 offset += block.getOnDiskSizeWithHeader();
203 }
204 } catch (IOException e) {
205
206 if (LOG.isTraceEnabled()) {
207 LOG.trace("Exception encountered while prefetching " + path + ":", e);
208 }
209 } catch (Exception e) {
210
211 LOG.warn("Exception encountered while prefetching " + path + ":", e);
212 } finally {
213 PrefetchExecutor.complete(path);
214 }
215 }
216 });
217 }
218 }
219
220 protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
221 HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
222 return new HFileContextBuilder()
223 .withIncludesMvcc(this.includesMemstoreTS)
224 .withCompression(this.compressAlgo)
225 .withHBaseCheckSum(trailer.getMinorVersion() >= MINOR_VERSION_WITH_CHECKSUM)
226 .build();
227 }
228
229
230
231
232
233
234
235
236
237
238
239
240
241 @Override
242 public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
243 final boolean isCompaction) {
244 if (dataBlockEncoder.useEncodedScanner()) {
245 return new EncodedScannerV2(this, cacheBlocks, pread, isCompaction,
246 hfileContext);
247 }
248
249 return new ScannerV2(this, cacheBlocks, pread, isCompaction);
250 }
251
252
253
254
255
256 private HFileBlock getCachedBlock(BlockCacheKey cacheKey, boolean cacheBlock, boolean useLock,
257 boolean isCompaction, boolean updateCacheMetrics, BlockType expectedBlockType,
258 DataBlockEncoding expectedDataBlockEncoding) throws IOException {
259
260 if (cacheConf.isBlockCacheEnabled()) {
261 BlockCache cache = cacheConf.getBlockCache();
262 HFileBlock cachedBlock = (HFileBlock) cache.getBlock(cacheKey, cacheBlock, useLock,
263 updateCacheMetrics);
264 if (cachedBlock != null) {
265 if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) {
266 cachedBlock = cachedBlock.unpack(hfileContext, fsBlockReader);
267 }
268 validateBlockType(cachedBlock, expectedBlockType);
269
270 if (expectedDataBlockEncoding == null) {
271 return cachedBlock;
272 }
273 DataBlockEncoding actualDataBlockEncoding =
274 cachedBlock.getDataBlockEncoding();
275
276
277
278 if (cachedBlock.getBlockType().isData() &&
279 !actualDataBlockEncoding.equals(expectedDataBlockEncoding)) {
280
281
282
283
284
285
286
287
288
289
290 if (!expectedDataBlockEncoding.equals(DataBlockEncoding.NONE) &&
291 !actualDataBlockEncoding.equals(DataBlockEncoding.NONE)) {
292
293
294
295
296
297
298 LOG.info("Evicting cached block with key " + cacheKey +
299 " because of a data block encoding mismatch" +
300 "; expected: " + expectedDataBlockEncoding +
301 ", actual: " + actualDataBlockEncoding);
302 cache.evictBlock(cacheKey);
303 }
304 return null;
305 }
306 return cachedBlock;
307 }
308 }
309 return null;
310 }
311
312
313
314
315
316
317 @Override
318 public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock)
319 throws IOException {
320 if (trailer.getMetaIndexCount() == 0) {
321 return null;
322 }
323 if (metaBlockIndexReader == null) {
324 throw new IOException("Meta index not loaded");
325 }
326
327 byte[] mbname = Bytes.toBytes(metaBlockName);
328 int block = metaBlockIndexReader.rootBlockContainingKey(mbname,
329 0, mbname.length);
330 if (block == -1)
331 return null;
332 long blockSize = metaBlockIndexReader.getRootBlockDataSize(block);
333
334
335
336
337 synchronized (metaBlockIndexReader.getRootBlockKey(block)) {
338
339 long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block);
340 BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset);
341
342 cacheBlock &= cacheConf.shouldCacheDataOnRead();
343 if (cacheConf.isBlockCacheEnabled()) {
344 HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, false, true, true,
345 BlockType.META, null);
346 if (cachedBlock != null) {
347 assert cachedBlock.isUnpacked() : "Packed block leak.";
348
349
350 return cachedBlock.getBufferWithoutHeader();
351 }
352
353 }
354
355 HFileBlock metaBlock = fsBlockReader.readBlockData(metaBlockOffset,
356 blockSize, -1, true).unpack(hfileContext, fsBlockReader);
357
358
359 if (cacheBlock) {
360 cacheConf.getBlockCache().cacheBlock(cacheKey, metaBlock,
361 cacheConf.isInMemory(), this.cacheConf.isCacheDataInL1());
362 }
363
364 return metaBlock.getBufferWithoutHeader();
365 }
366 }
367
368 @Override
369 public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize,
370 final boolean cacheBlock, boolean pread, final boolean isCompaction,
371 boolean updateCacheMetrics, BlockType expectedBlockType,
372 DataBlockEncoding expectedDataBlockEncoding)
373 throws IOException {
374 if (dataBlockIndexReader == null) {
375 throw new IOException("Block index not loaded");
376 }
377 if (dataBlockOffset < 0 || dataBlockOffset >= trailer.getLoadOnOpenDataOffset()) {
378 throw new IOException("Requested block is out of range: " + dataBlockOffset +
379 ", lastDataBlockOffset: " + trailer.getLastDataBlockOffset());
380 }
381
382
383
384
385
386 BlockCacheKey cacheKey = new BlockCacheKey(name, dataBlockOffset);
387 boolean useLock = false;
388 IdLock.Entry lockEntry = null;
389 TraceScope traceScope = Trace.startSpan("HFileReaderV2.readBlock");
390 try {
391 while (true) {
392 if (useLock) {
393 lockEntry = offsetLock.getLockEntry(dataBlockOffset);
394 }
395
396
397 if (cacheConf.isBlockCacheEnabled()) {
398
399
400 HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, useLock, isCompaction,
401 updateCacheMetrics, expectedBlockType, expectedDataBlockEncoding);
402 if (cachedBlock != null) {
403 if (Trace.isTracing()) {
404 traceScope.getSpan().addTimelineAnnotation("blockCacheHit");
405 }
406 assert cachedBlock.isUnpacked() : "Packed block leak.";
407 if (cachedBlock.getBlockType().isData()) {
408 if (updateCacheMetrics) {
409 HFile.dataBlockReadCnt.incrementAndGet();
410 }
411
412
413 if (cachedBlock.getDataBlockEncoding() != dataBlockEncoder.getDataBlockEncoding()) {
414 throw new IOException("Cached block under key " + cacheKey + " "
415 + "has wrong encoding: " + cachedBlock.getDataBlockEncoding() + " (expected: "
416 + dataBlockEncoder.getDataBlockEncoding() + ")");
417 }
418 }
419
420 return cachedBlock;
421 }
422
423 }
424 if (!useLock) {
425
426 useLock = true;
427 continue;
428 }
429 if (Trace.isTracing()) {
430 traceScope.getSpan().addTimelineAnnotation("blockCacheMiss");
431 }
432
433 HFileBlock hfileBlock = fsBlockReader.readBlockData(dataBlockOffset, onDiskBlockSize, -1,
434 pread);
435 validateBlockType(hfileBlock, expectedBlockType);
436 HFileBlock unpacked = hfileBlock.unpack(hfileContext, fsBlockReader);
437 BlockType.BlockCategory category = hfileBlock.getBlockType().getCategory();
438
439
440 if (cacheBlock && cacheConf.shouldCacheBlockOnRead(category)) {
441 cacheConf.getBlockCache().cacheBlock(cacheKey,
442 cacheConf.shouldCacheCompressed(category) ? hfileBlock : unpacked,
443 cacheConf.isInMemory(), this.cacheConf.isCacheDataInL1());
444 }
445
446 if (updateCacheMetrics && hfileBlock.getBlockType().isData()) {
447 HFile.dataBlockReadCnt.incrementAndGet();
448 }
449
450 return unpacked;
451 }
452 } finally {
453 traceScope.close();
454 if (lockEntry != null) {
455 offsetLock.releaseLockEntry(lockEntry);
456 }
457 }
458 }
459
460 @Override
461 public boolean hasMVCCInfo() {
462 return includesMemstoreTS && decodeMemstoreTS;
463 }
464
465
466
467
468
469
470
471
472
473
474 private void validateBlockType(HFileBlock block,
475 BlockType expectedBlockType) throws IOException {
476 if (expectedBlockType == null) {
477 return;
478 }
479 BlockType actualBlockType = block.getBlockType();
480 if (expectedBlockType.isData() && actualBlockType.isData()) {
481
482
483 return;
484 }
485 if (actualBlockType != expectedBlockType) {
486 throw new IOException("Expected block type " + expectedBlockType + ", " +
487 "but got " + actualBlockType + ": " + block);
488 }
489 }
490
491
492
493
494
495
496 @Override
497 public byte[] getLastKey() {
498 return dataBlockIndexReader.isEmpty() ? null : lastKey;
499 }
500
501
502
503
504
505
506 @Override
507 public byte[] midkey() throws IOException {
508 return dataBlockIndexReader.midkey();
509 }
510
511 @Override
512 public void close() throws IOException {
513 close(cacheConf.shouldEvictOnClose());
514 }
515
516 public void close(boolean evictOnClose) throws IOException {
517 PrefetchExecutor.cancel(path);
518 if (evictOnClose && cacheConf.isBlockCacheEnabled()) {
519 int numEvicted = cacheConf.getBlockCache().evictBlocksByHfileName(name);
520 if (LOG.isTraceEnabled()) {
521 LOG.trace("On close, file=" + name + " evicted=" + numEvicted
522 + " block(s)");
523 }
524 }
525 fsBlockReader.closeStreams();
526 }
527
528 public DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction) {
529 return dataBlockEncoder.getEffectiveEncodingInCache(isCompaction);
530 }
531
532
533 @Override
534 HFileBlock.FSReader getUncachedBlockReader() {
535 return fsBlockReader;
536 }
537
538
539 protected abstract static class AbstractScannerV2
540 extends AbstractHFileReader.Scanner {
541 protected HFileBlock block;
542
543 @Override
544 public Cell getNextIndexedKey() {
545 return nextIndexedKey;
546 }
547
548
549
550
551
552
553
554 protected Cell nextIndexedKey;
555
556 public AbstractScannerV2(HFileReaderV2 r, boolean cacheBlocks,
557 final boolean pread, final boolean isCompaction) {
558 super(r, cacheBlocks, pread, isCompaction);
559 }
560
561 protected abstract ByteBuffer getFirstKeyInBlock(HFileBlock curBlock);
562
563 protected abstract int loadBlockAndSeekToKey(HFileBlock seekToBlock, Cell nextIndexedKey,
564 boolean rewind, Cell key, boolean seekBefore) throws IOException;
565
566 @Override
567 public int seekTo(byte[] key, int offset, int length) throws IOException {
568
569
570 return seekTo(new KeyValue.KeyOnlyKeyValue(key, offset, length));
571 }
572
573 @Override
574 public int reseekTo(byte[] key, int offset, int length) throws IOException {
575 return reseekTo(new KeyValue.KeyOnlyKeyValue(key, offset, length));
576 }
577
578 @Override
579 public int seekTo(Cell key) throws IOException {
580 return seekTo(key, true);
581 }
582
583 @Override
584 public int reseekTo(Cell key) throws IOException {
585 int compared;
586 if (isSeeked()) {
587 compared = compareKey(reader.getComparator(), key);
588 if (compared < 1) {
589
590
591 return compared;
592 } else {
593
594 if (this.nextIndexedKey != null &&
595 (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY || reader
596 .getComparator()
597 .compareOnlyKeyPortion(key, nextIndexedKey) < 0)) {
598
599
600
601
602
603
604 return loadBlockAndSeekToKey(this.block, nextIndexedKey, false, key, false);
605 }
606 }
607 }
608
609
610 return seekTo(key, false);
611 }
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628 public int seekTo(Cell key, boolean rewind) throws IOException {
629 HFileBlockIndex.BlockIndexReader indexReader = reader.getDataBlockIndexReader();
630 BlockWithScanInfo blockWithScanInfo = indexReader.loadDataBlockWithScanInfo(key, block,
631 cacheBlocks, pread, isCompaction, getEffectiveDataBlockEncoding());
632 if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) {
633
634 return -1;
635 }
636 return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(),
637 blockWithScanInfo.getNextIndexedKey(), rewind, key, false);
638 }
639
640 @Override
641 public boolean seekBefore(byte[] key, int offset, int length) throws IOException {
642 return seekBefore(new KeyValue.KeyOnlyKeyValue(key, offset, length));
643 }
644
645 @Override
646 public boolean seekBefore(Cell key) throws IOException {
647 HFileBlock seekToBlock = reader.getDataBlockIndexReader().seekToDataBlock(key, block,
648 cacheBlocks, pread, isCompaction,
649 ((HFileReaderV2) reader).getEffectiveEncodingInCache(isCompaction));
650 if (seekToBlock == null) {
651 return false;
652 }
653 ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock);
654
655 if (reader.getComparator()
656 .compareOnlyKeyPortion(
657 new KeyValue.KeyOnlyKeyValue(firstKey.array(), firstKey.arrayOffset(),
658 firstKey.limit()), key) >= 0) {
659 long previousBlockOffset = seekToBlock.getPrevBlockOffset();
660
661 if (previousBlockOffset == -1) {
662
663 return false;
664 }
665
666
667
668
669 seekToBlock = reader.readBlock(previousBlockOffset,
670 seekToBlock.getOffset() - previousBlockOffset, cacheBlocks,
671 pread, isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
672
673
674 }
675 Cell firstKeyInCurrentBlock = new KeyValue.KeyOnlyKeyValue(Bytes.getBytes(firstKey));
676 loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, true);
677 return true;
678 }
679
680
681
682
683
684
685
686
687 protected HFileBlock readNextDataBlock() throws IOException {
688 long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset();
689 if (block == null)
690 return null;
691
692 HFileBlock curBlock = block;
693
694 do {
695 if (curBlock.getOffset() >= lastDataBlockOffset)
696 return null;
697
698 if (curBlock.getOffset() < 0) {
699 throw new IOException("Invalid block file offset: " + block);
700 }
701
702
703
704 curBlock = reader.readBlock(curBlock.getOffset()
705 + curBlock.getOnDiskSizeWithHeader(),
706 curBlock.getNextBlockOnDiskSizeWithHeader(), cacheBlocks, pread,
707 isCompaction, true, null, getEffectiveDataBlockEncoding());
708 } while (!curBlock.getBlockType().isData());
709
710 return curBlock;
711 }
712
713 public DataBlockEncoding getEffectiveDataBlockEncoding() {
714 return ((HFileReaderV2)reader).getEffectiveEncodingInCache(isCompaction);
715 }
716
717
718
719
720
721
722
723
724 public abstract int compareKey(KVComparator comparator, byte[] key, int offset,
725 int length);
726
727 public abstract int compareKey(KVComparator comparator, Cell kv);
728 }
729
730
731
732
733 protected static class ScannerV2 extends AbstractScannerV2 {
734 private HFileReaderV2 reader;
735
736 public ScannerV2(HFileReaderV2 r, boolean cacheBlocks,
737 final boolean pread, final boolean isCompaction) {
738 super(r, cacheBlocks, pread, isCompaction);
739 this.reader = r;
740 }
741
742 @Override
743 public Cell getKeyValue() {
744 if (!isSeeked())
745 return null;
746
747 KeyValue ret = new KeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
748 + blockBuffer.position(), getCellBufSize());
749 if (this.reader.shouldIncludeMemstoreTS()) {
750 ret.setSequenceId(currMemstoreTS);
751 }
752 return ret;
753 }
754
755 protected int getCellBufSize() {
756 return KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen;
757 }
758
759 @Override
760 public ByteBuffer getKey() {
761 assertSeeked();
762 return ByteBuffer.wrap(
763 blockBuffer.array(),
764 blockBuffer.arrayOffset() + blockBuffer.position()
765 + KEY_VALUE_LEN_SIZE, currKeyLen).slice();
766 }
767
768 @Override
769 public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
770 return comparator.compareFlatKey(key, offset, length, blockBuffer.array(),
771 blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen);
772 }
773
774 @Override
775 public ByteBuffer getValue() {
776 assertSeeked();
777 return ByteBuffer.wrap(
778 blockBuffer.array(),
779 blockBuffer.arrayOffset() + blockBuffer.position()
780 + KEY_VALUE_LEN_SIZE + currKeyLen, currValueLen).slice();
781 }
782
783 protected void setNonSeekedState() {
784 block = null;
785 blockBuffer = null;
786 currKeyLen = 0;
787 currValueLen = 0;
788 currMemstoreTS = 0;
789 currMemstoreTSLen = 0;
790 }
791
792
793
794
795
796
797
798
799 @Override
800 public boolean next() throws IOException {
801 assertSeeked();
802
803 try {
804 blockBuffer.position(getNextCellStartPosition());
805 } catch (IllegalArgumentException e) {
806 LOG.error("Current pos = " + blockBuffer.position()
807 + "; currKeyLen = " + currKeyLen + "; currValLen = "
808 + currValueLen + "; block limit = " + blockBuffer.limit()
809 + "; HFile name = " + reader.getName()
810 + "; currBlock currBlockOffset = " + block.getOffset());
811 throw e;
812 }
813
814 if (blockBuffer.remaining() <= 0) {
815 long lastDataBlockOffset =
816 reader.getTrailer().getLastDataBlockOffset();
817
818 if (block.getOffset() >= lastDataBlockOffset) {
819 setNonSeekedState();
820 return false;
821 }
822
823
824 HFileBlock nextBlock = readNextDataBlock();
825 if (nextBlock == null) {
826 setNonSeekedState();
827 return false;
828 }
829
830 updateCurrBlock(nextBlock);
831 return true;
832 }
833
834
835 readKeyValueLen();
836 return true;
837 }
838
839 protected int getNextCellStartPosition() {
840 return blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen
841 + currMemstoreTSLen;
842 }
843
844
845
846
847
848
849
850
851 @Override
852 public boolean seekTo() throws IOException {
853 if (reader == null) {
854 return false;
855 }
856
857 if (reader.getTrailer().getEntryCount() == 0) {
858
859 return false;
860 }
861
862 long firstDataBlockOffset =
863 reader.getTrailer().getFirstDataBlockOffset();
864 if (block != null && block.getOffset() == firstDataBlockOffset) {
865 blockBuffer.rewind();
866 readKeyValueLen();
867 return true;
868 }
869
870 block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
871 isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
872 if (block.getOffset() < 0) {
873 throw new IOException("Invalid block offset: " + block.getOffset());
874 }
875 updateCurrBlock(block);
876 return true;
877 }
878
879 @Override
880 protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, Cell nextIndexedKey,
881 boolean rewind, Cell key, boolean seekBefore) throws IOException {
882 if (block == null || block.getOffset() != seekToBlock.getOffset()) {
883 updateCurrBlock(seekToBlock);
884 } else if (rewind) {
885 blockBuffer.rewind();
886 }
887
888
889 this.nextIndexedKey = nextIndexedKey;
890 return blockSeek(key, seekBefore);
891 }
892
893
894
895
896
897
898
899 protected void updateCurrBlock(HFileBlock newBlock) {
900 block = newBlock;
901
902
903 if (block.getBlockType() != BlockType.DATA) {
904 throw new IllegalStateException("ScannerV2 works only on data " +
905 "blocks, got " + block.getBlockType() + "; " +
906 "fileName=" + reader.name + ", " +
907 "dataBlockEncoder=" + reader.dataBlockEncoder + ", " +
908 "isCompaction=" + isCompaction);
909 }
910
911 blockBuffer = block.getBufferWithoutHeader();
912 readKeyValueLen();
913 blockFetches++;
914
915
916 this.nextIndexedKey = null;
917 }
918
919 protected void readKeyValueLen() {
920 blockBuffer.mark();
921 currKeyLen = blockBuffer.getInt();
922 currValueLen = blockBuffer.getInt();
923 ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
924 readMvccVersion();
925 if (currKeyLen < 0 || currValueLen < 0
926 || currKeyLen > blockBuffer.limit()
927 || currValueLen > blockBuffer.limit()) {
928 throw new IllegalStateException("Invalid currKeyLen " + currKeyLen
929 + " or currValueLen " + currValueLen + ". Block offset: "
930 + block.getOffset() + ", block length: " + blockBuffer.limit()
931 + ", position: " + blockBuffer.position() + " (without header).");
932 }
933 blockBuffer.reset();
934 }
935
936 protected void readMvccVersion() {
937 if (this.reader.shouldIncludeMemstoreTS()) {
938 if (this.reader.decodeMemstoreTS) {
939 try {
940 currMemstoreTS = Bytes.readVLong(blockBuffer.array(), blockBuffer.arrayOffset()
941 + blockBuffer.position());
942 currMemstoreTSLen = WritableUtils.getVIntSize(currMemstoreTS);
943 } catch (Exception e) {
944 throw new RuntimeException("Error reading memstore timestamp", e);
945 }
946 } else {
947 currMemstoreTS = 0;
948 currMemstoreTSLen = 1;
949 }
950 }
951 }
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970 protected int blockSeek(Cell key, boolean seekBefore) {
971 int klen, vlen;
972 long memstoreTS = 0;
973 int memstoreTSLen = 0;
974 int lastKeyValueSize = -1;
975 KeyValue.KeyOnlyKeyValue keyOnlykv = new KeyValue.KeyOnlyKeyValue();
976 do {
977 blockBuffer.mark();
978 klen = blockBuffer.getInt();
979 vlen = blockBuffer.getInt();
980 blockBuffer.reset();
981 if (this.reader.shouldIncludeMemstoreTS()) {
982 if (this.reader.decodeMemstoreTS) {
983 try {
984 int memstoreTSOffset = blockBuffer.arrayOffset() + blockBuffer.position()
985 + KEY_VALUE_LEN_SIZE + klen + vlen;
986 memstoreTS = Bytes.readVLong(blockBuffer.array(), memstoreTSOffset);
987 memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
988 } catch (Exception e) {
989 throw new RuntimeException("Error reading memstore timestamp", e);
990 }
991 } else {
992 memstoreTS = 0;
993 memstoreTSLen = 1;
994 }
995 }
996
997 int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE;
998 keyOnlykv.setKey(blockBuffer.array(), keyOffset, klen);
999 int comp = reader.getComparator().compareOnlyKeyPortion(key, keyOnlykv);
1000
1001 if (comp == 0) {
1002 if (seekBefore) {
1003 if (lastKeyValueSize < 0) {
1004 throw new IllegalStateException("blockSeek with seekBefore "
1005 + "at the first key of the block: key="
1006 + CellUtil.getCellKeyAsString(key)
1007 + ", blockOffset=" + block.getOffset() + ", onDiskSize="
1008 + block.getOnDiskSizeWithHeader());
1009 }
1010 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1011 readKeyValueLen();
1012 return 1;
1013 }
1014 currKeyLen = klen;
1015 currValueLen = vlen;
1016 if (this.reader.shouldIncludeMemstoreTS()) {
1017 currMemstoreTS = memstoreTS;
1018 currMemstoreTSLen = memstoreTSLen;
1019 }
1020 return 0;
1021 } else if (comp < 0) {
1022 if (lastKeyValueSize > 0)
1023 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1024 readKeyValueLen();
1025 if (lastKeyValueSize == -1 && blockBuffer.position() == 0
1026 && this.reader.trailer.getMinorVersion() >= MINOR_VERSION_WITH_FAKED_KEY) {
1027 return HConstants.INDEX_KEY_MAGIC;
1028 }
1029 return 1;
1030 }
1031
1032
1033 lastKeyValueSize = klen + vlen + memstoreTSLen + KEY_VALUE_LEN_SIZE;
1034 blockBuffer.position(blockBuffer.position() + lastKeyValueSize);
1035 } while (blockBuffer.remaining() > 0);
1036
1037
1038
1039
1040 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1041 readKeyValueLen();
1042 return 1;
1043 }
1044
1045 @Override
1046 protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
1047 ByteBuffer buffer = curBlock.getBufferWithoutHeader();
1048
1049 buffer.rewind();
1050 int klen = buffer.getInt();
1051 buffer.getInt();
1052 ByteBuffer keyBuff = buffer.slice();
1053 keyBuff.limit(klen);
1054 keyBuff.rewind();
1055 return keyBuff;
1056 }
1057
1058 @Override
1059 public String getKeyString() {
1060 return Bytes.toStringBinary(blockBuffer.array(),
1061 blockBuffer.arrayOffset() + blockBuffer.position()
1062 + KEY_VALUE_LEN_SIZE, currKeyLen);
1063 }
1064
1065 @Override
1066 public String getValueString() {
1067 return Bytes.toString(blockBuffer.array(), blockBuffer.arrayOffset()
1068 + blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen,
1069 currValueLen);
1070 }
1071
1072 @Override
1073 public int compareKey(KVComparator comparator, Cell key) {
1074 return comparator.compareOnlyKeyPortion(
1075 key,
1076 new KeyValue.KeyOnlyKeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
1077 + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen));
1078 }
1079 }
1080
1081
1082
1083
1084 protected static class EncodedScannerV2 extends AbstractScannerV2 {
1085 private final HFileBlockDecodingContext decodingCtx;
1086 private final DataBlockEncoder.EncodedSeeker seeker;
1087 private final DataBlockEncoder dataBlockEncoder;
1088 protected final HFileContext meta;
1089
1090 public EncodedScannerV2(HFileReaderV2 reader, boolean cacheBlocks,
1091 boolean pread, boolean isCompaction, HFileContext meta) {
1092 super(reader, cacheBlocks, pread, isCompaction);
1093 DataBlockEncoding encoding = reader.dataBlockEncoder.getDataBlockEncoding();
1094 dataBlockEncoder = encoding.getEncoder();
1095 decodingCtx = dataBlockEncoder.newDataBlockDecodingContext(meta);
1096 seeker = dataBlockEncoder.createSeeker(
1097 reader.getComparator(), decodingCtx);
1098 this.meta = meta;
1099 }
1100
1101 @Override
1102 public boolean isSeeked(){
1103 return this.block != null;
1104 }
1105
1106
1107
1108
1109
1110
1111
1112
1113 private void updateCurrentBlock(HFileBlock newBlock) throws CorruptHFileException {
1114 block = newBlock;
1115
1116
1117 if (block.getBlockType() != BlockType.ENCODED_DATA) {
1118 throw new IllegalStateException(
1119 "EncodedScanner works only on encoded data blocks");
1120 }
1121 short dataBlockEncoderId = block.getDataBlockEncodingId();
1122 if (!DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) {
1123 String encoderCls = dataBlockEncoder.getClass().getName();
1124 throw new CorruptHFileException("Encoder " + encoderCls
1125 + " doesn't support data block encoding "
1126 + DataBlockEncoding.getNameFromId(dataBlockEncoderId));
1127 }
1128
1129 seeker.setCurrentBuffer(getEncodedBuffer(newBlock));
1130 blockFetches++;
1131
1132
1133 this.nextIndexedKey = null;
1134 }
1135
1136 private ByteBuffer getEncodedBuffer(HFileBlock newBlock) {
1137 ByteBuffer origBlock = newBlock.getBufferReadOnly();
1138 ByteBuffer encodedBlock = ByteBuffer.wrap(origBlock.array(),
1139 origBlock.arrayOffset() + newBlock.headerSize() +
1140 DataBlockEncoding.ID_SIZE,
1141 newBlock.getUncompressedSizeWithoutHeader() -
1142 DataBlockEncoding.ID_SIZE).slice();
1143 return encodedBlock;
1144 }
1145
1146 @Override
1147 public boolean seekTo() throws IOException {
1148 if (reader == null) {
1149 return false;
1150 }
1151
1152 if (reader.getTrailer().getEntryCount() == 0) {
1153
1154 return false;
1155 }
1156
1157 long firstDataBlockOffset =
1158 reader.getTrailer().getFirstDataBlockOffset();
1159 if (block != null && block.getOffset() == firstDataBlockOffset) {
1160 seeker.rewind();
1161 return true;
1162 }
1163
1164 block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
1165 isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
1166 if (block.getOffset() < 0) {
1167 throw new IOException("Invalid block offset: " + block.getOffset());
1168 }
1169 updateCurrentBlock(block);
1170 return true;
1171 }
1172
1173 @Override
1174 public boolean next() throws IOException {
1175 boolean isValid = seeker.next();
1176 if (!isValid) {
1177 block = readNextDataBlock();
1178 isValid = block != null;
1179 if (isValid) {
1180 updateCurrentBlock(block);
1181 }
1182 }
1183 return isValid;
1184 }
1185
1186 @Override
1187 public ByteBuffer getKey() {
1188 assertValidSeek();
1189 return seeker.getKeyDeepCopy();
1190 }
1191
1192 @Override
1193 public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
1194 return seeker.compareKey(comparator, key, offset, length);
1195 }
1196
1197 @Override
1198 public ByteBuffer getValue() {
1199 assertValidSeek();
1200 return seeker.getValueShallowCopy();
1201 }
1202
1203 @Override
1204 public Cell getKeyValue() {
1205 if (block == null) {
1206 return null;
1207 }
1208 return seeker.getKeyValue();
1209 }
1210
1211 @Override
1212 public String getKeyString() {
1213 ByteBuffer keyBuffer = getKey();
1214 return Bytes.toStringBinary(keyBuffer.array(),
1215 keyBuffer.arrayOffset(), keyBuffer.limit());
1216 }
1217
1218 @Override
1219 public String getValueString() {
1220 ByteBuffer valueBuffer = getValue();
1221 return Bytes.toStringBinary(valueBuffer.array(),
1222 valueBuffer.arrayOffset(), valueBuffer.limit());
1223 }
1224
1225 private void assertValidSeek() {
1226 if (block == null) {
1227 throw new NotSeekedException();
1228 }
1229 }
1230
1231 @Override
1232 protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
1233 return dataBlockEncoder.getFirstKeyInBlock(getEncodedBuffer(curBlock));
1234 }
1235
1236 @Override
1237 protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, Cell nextIndexedKey,
1238 boolean rewind, Cell key, boolean seekBefore) throws IOException {
1239 if (block == null || block.getOffset() != seekToBlock.getOffset()) {
1240 updateCurrentBlock(seekToBlock);
1241 } else if (rewind) {
1242 seeker.rewind();
1243 }
1244 this.nextIndexedKey = nextIndexedKey;
1245 return seeker.seekToKeyInBlock(key, seekBefore);
1246 }
1247
1248 @Override
1249 public int compareKey(KVComparator comparator, Cell key) {
1250 return seeker.compareKey(comparator, key);
1251 }
1252 }
1253
1254
1255
1256
1257
1258 @Override
1259 public DataInput getGeneralBloomFilterMetadata() throws IOException {
1260 return this.getBloomFilterMetadata(BlockType.GENERAL_BLOOM_META);
1261 }
1262
1263 @Override
1264 public DataInput getDeleteBloomFilterMetadata() throws IOException {
1265 return this.getBloomFilterMetadata(BlockType.DELETE_FAMILY_BLOOM_META);
1266 }
1267
1268 private DataInput getBloomFilterMetadata(BlockType blockType)
1269 throws IOException {
1270 if (blockType != BlockType.GENERAL_BLOOM_META &&
1271 blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
1272 throw new RuntimeException("Block Type: " + blockType.toString() +
1273 " is not supported") ;
1274 }
1275
1276 for (HFileBlock b : loadOnOpenBlocks)
1277 if (b.getBlockType() == blockType)
1278 return b.getByteStream();
1279 return null;
1280 }
1281
1282 @Override
1283 public boolean isFileInfoLoaded() {
1284 return true;
1285 }
1286
1287
1288
1289
1290
1291 private void validateMinorVersion(Path path, int minorVersion) {
1292 if (minorVersion < MIN_MINOR_VERSION ||
1293 minorVersion > MAX_MINOR_VERSION) {
1294 String msg = "Minor version for path " + path +
1295 " is expected to be between " +
1296 MIN_MINOR_VERSION + " and " + MAX_MINOR_VERSION +
1297 " but is found to be " + minorVersion;
1298 LOG.error(msg);
1299 throw new RuntimeException(msg);
1300 }
1301 }
1302
1303 @Override
1304 public int getMajorVersion() {
1305 return 2;
1306 }
1307
1308 @Override
1309 public HFileContext getFileContext() {
1310 return hfileContext;
1311 }
1312
1313
1314
1315
1316
1317 @VisibleForTesting
1318 boolean prefetchComplete() {
1319 return PrefetchExecutor.isCompleted(path);
1320 }
1321 }