1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.DataInput;
21 import java.io.IOException;
22 import java.nio.ByteBuffer;
23 import java.util.ArrayList;
24 import java.util.List;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.hbase.classification.InterfaceAudience;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hbase.Cell;
32 import org.apache.hadoop.hbase.CellUtil;
33 import org.apache.hadoop.hbase.HConstants;
34 import org.apache.hadoop.hbase.KeyValue;
35 import org.apache.hadoop.hbase.KeyValue.KVComparator;
36 import org.apache.hadoop.hbase.fs.HFileSystem;
37 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
38 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
39 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
40 import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
41 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
42 import org.apache.hadoop.hbase.util.ByteBufferUtils;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.util.IdLock;
45 import org.apache.hadoop.io.WritableUtils;
46 import org.apache.htrace.Trace;
47 import org.apache.htrace.TraceScope;
48
49 import com.google.common.annotations.VisibleForTesting;
50
51
52
53
54 @InterfaceAudience.Private
55 public class HFileReaderV2 extends AbstractHFileReader {
56
57 private static final Log LOG = LogFactory.getLog(HFileReaderV2.class);
58
59
60 public static final int MINOR_VERSION_WITH_CHECKSUM = 1;
61
62 public static final int MINOR_VERSION_NO_CHECKSUM = 0;
63
64
65 public static final int PBUF_TRAILER_MINOR_VERSION = 2;
66
67
68
69
70
71 public final static int KEY_VALUE_LEN_SIZE = 2 * Bytes.SIZEOF_INT;
72
73 protected boolean includesMemstoreTS = false;
74 protected boolean decodeMemstoreTS = false;
75 protected boolean shouldIncludeMemstoreTS() {
76 return includesMemstoreTS;
77 }
78
79
80 protected HFileBlock.FSReader fsBlockReader;
81
82
83
84
85
86
87
88 private IdLock offsetLock = new IdLock();
89
90
91
92
93
94 private List<HFileBlock> loadOnOpenBlocks = new ArrayList<HFileBlock>();
95
96
97 static final int MIN_MINOR_VERSION = 0;
98
99
100
101
102 static final int MAX_MINOR_VERSION = 3;
103
104
105 static final int MINOR_VERSION_WITH_FAKED_KEY = 3;
106
107 protected HFileContext hfileContext;
108
109
110
111
112
113
114
115
116
117
118
119
120
121 public HFileReaderV2(final Path path, final FixedFileTrailer trailer,
122 final FSDataInputStreamWrapper fsdis, final long size, final CacheConfig cacheConf,
123 final HFileSystem hfs, final Configuration conf) throws IOException {
124 super(path, trailer, size, cacheConf, hfs, conf);
125 this.conf = conf;
126 trailer.expectMajorVersion(getMajorVersion());
127 validateMinorVersion(path, trailer.getMinorVersion());
128 this.hfileContext = createHFileContext(fsdis, fileSize, hfs, path, trailer);
129 HFileBlock.FSReaderImpl fsBlockReaderV2 =
130 new HFileBlock.FSReaderImpl(fsdis, fileSize, hfs, path, hfileContext);
131 this.fsBlockReader = fsBlockReaderV2;
132
133
134 comparator = trailer.createComparator();
135 dataBlockIndexReader = new HFileBlockIndex.BlockIndexReader(comparator,
136 trailer.getNumDataIndexLevels(), this);
137 metaBlockIndexReader = new HFileBlockIndex.BlockIndexReader(
138 KeyValue.RAW_COMPARATOR, 1);
139
140
141
142 HFileBlock.BlockIterator blockIter = fsBlockReaderV2.blockRange(
143 trailer.getLoadOnOpenDataOffset(),
144 fileSize - trailer.getTrailerSize());
145
146
147
148 dataBlockIndexReader.readMultiLevelIndexRoot(
149 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
150 trailer.getDataIndexCount());
151
152
153 metaBlockIndexReader.readRootIndex(
154 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
155 trailer.getMetaIndexCount());
156
157
158 fileInfo = new FileInfo();
159 fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
160 lastKey = fileInfo.get(FileInfo.LASTKEY);
161 avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN));
162 avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN));
163 byte [] keyValueFormatVersion =
164 fileInfo.get(HFileWriterV2.KEY_VALUE_VERSION);
165 includesMemstoreTS = keyValueFormatVersion != null &&
166 Bytes.toInt(keyValueFormatVersion) ==
167 HFileWriterV2.KEY_VALUE_VER_WITH_MEMSTORE;
168 fsBlockReaderV2.setIncludesMemstoreTS(includesMemstoreTS);
169 if (includesMemstoreTS) {
170 decodeMemstoreTS = Bytes.toLong(fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY)) > 0;
171 }
172
173
174 dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo);
175 fsBlockReaderV2.setDataBlockEncoder(dataBlockEncoder);
176
177
178 HFileBlock b;
179 while ((b = blockIter.nextBlock()) != null) {
180 loadOnOpenBlocks.add(b);
181 }
182
183
184 if (cacheConf.shouldPrefetchOnOpen()) {
185 PrefetchExecutor.request(path, new Runnable() {
186 public void run() {
187 try {
188 long offset = 0;
189 long end = fileSize - getTrailer().getTrailerSize();
190 HFileBlock prevBlock = null;
191 while (offset < end) {
192 if (Thread.interrupted()) {
193 break;
194 }
195 long onDiskSize = -1;
196 if (prevBlock != null) {
197 onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
198 }
199 HFileBlock block = readBlock(offset, onDiskSize, true, false, false, false,
200 null, null);
201 prevBlock = block;
202 offset += block.getOnDiskSizeWithHeader();
203 }
204 } catch (IOException e) {
205
206 if (LOG.isTraceEnabled()) {
207 LOG.trace("Exception encountered while prefetching " + path + ":", e);
208 }
209 } catch (Exception e) {
210
211 LOG.warn("Exception encountered while prefetching " + path + ":", e);
212 } finally {
213 PrefetchExecutor.complete(path);
214 }
215 }
216 });
217 }
218 }
219
220 protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
221 HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
222 return new HFileContextBuilder()
223 .withIncludesMvcc(this.includesMemstoreTS)
224 .withCompression(this.compressAlgo)
225 .withHBaseCheckSum(trailer.getMinorVersion() >= MINOR_VERSION_WITH_CHECKSUM)
226 .build();
227 }
228
229
230
231
232
233
234
235
236
237
238
239
240
241 @Override
242 public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
243 final boolean isCompaction) {
244 if (dataBlockEncoder.useEncodedScanner()) {
245 return new EncodedScannerV2(this, cacheBlocks, pread, isCompaction,
246 hfileContext);
247 }
248
249 return new ScannerV2(this, cacheBlocks, pread, isCompaction);
250 }
251
252
253
254
255
256 private HFileBlock getCachedBlock(BlockCacheKey cacheKey, boolean cacheBlock, boolean useLock,
257 boolean isCompaction, boolean updateCacheMetrics, BlockType expectedBlockType,
258 DataBlockEncoding expectedDataBlockEncoding) throws IOException {
259
260 if (cacheConf.isBlockCacheEnabled()) {
261 BlockCache cache = cacheConf.getBlockCache();
262 HFileBlock cachedBlock = (HFileBlock) cache.getBlock(cacheKey, cacheBlock, useLock,
263 updateCacheMetrics);
264 if (cachedBlock != null) {
265 if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) {
266 cachedBlock = cachedBlock.unpack(hfileContext, fsBlockReader);
267 }
268 validateBlockType(cachedBlock, expectedBlockType);
269
270 if (expectedDataBlockEncoding == null) {
271 return cachedBlock;
272 }
273 DataBlockEncoding actualDataBlockEncoding =
274 cachedBlock.getDataBlockEncoding();
275
276
277
278 if (cachedBlock.getBlockType().isData() &&
279 !actualDataBlockEncoding.equals(expectedDataBlockEncoding)) {
280
281
282
283
284
285
286
287
288
289
290 if (!expectedDataBlockEncoding.equals(DataBlockEncoding.NONE) &&
291 !actualDataBlockEncoding.equals(DataBlockEncoding.NONE)) {
292
293
294
295
296
297
298 LOG.info("Evicting cached block with key " + cacheKey +
299 " because of a data block encoding mismatch" +
300 "; expected: " + expectedDataBlockEncoding +
301 ", actual: " + actualDataBlockEncoding);
302 cache.evictBlock(cacheKey);
303 }
304 return null;
305 }
306 return cachedBlock;
307 }
308 }
309 return null;
310 }
311
312
313
314
315
316
317 @Override
318 public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock)
319 throws IOException {
320 if (trailer.getMetaIndexCount() == 0) {
321 return null;
322 }
323 if (metaBlockIndexReader == null) {
324 throw new IOException("Meta index not loaded");
325 }
326
327 byte[] mbname = Bytes.toBytes(metaBlockName);
328 int block = metaBlockIndexReader.rootBlockContainingKey(mbname,
329 0, mbname.length);
330 if (block == -1)
331 return null;
332 long blockSize = metaBlockIndexReader.getRootBlockDataSize(block);
333
334
335
336
337 synchronized (metaBlockIndexReader.getRootBlockKey(block)) {
338
339 long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block);
340 BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset);
341
342 cacheBlock &= cacheConf.shouldCacheDataOnRead();
343 if (cacheConf.isBlockCacheEnabled()) {
344 HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, false, true, true,
345 BlockType.META, null);
346 if (cachedBlock != null) {
347 assert cachedBlock.isUnpacked() : "Packed block leak.";
348
349
350 return cachedBlock.getBufferWithoutHeader();
351 }
352
353 }
354
355 HFileBlock metaBlock = fsBlockReader.readBlockData(metaBlockOffset,
356 blockSize, -1, true).unpack(hfileContext, fsBlockReader);
357
358
359 if (cacheBlock) {
360 cacheConf.getBlockCache().cacheBlock(cacheKey, metaBlock,
361 cacheConf.isInMemory(), this.cacheConf.isCacheDataInL1());
362 }
363
364 return metaBlock.getBufferWithoutHeader();
365 }
366 }
367
368 @Override
369 public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize,
370 final boolean cacheBlock, boolean pread, final boolean isCompaction,
371 boolean updateCacheMetrics, BlockType expectedBlockType,
372 DataBlockEncoding expectedDataBlockEncoding)
373 throws IOException {
374 if (dataBlockIndexReader == null) {
375 throw new IOException("Block index not loaded");
376 }
377 if (dataBlockOffset < 0 || dataBlockOffset >= trailer.getLoadOnOpenDataOffset()) {
378 throw new IOException("Requested block is out of range: " + dataBlockOffset +
379 ", lastDataBlockOffset: " + trailer.getLastDataBlockOffset());
380 }
381
382
383
384
385
386 BlockCacheKey cacheKey = new BlockCacheKey(name, dataBlockOffset);
387 boolean useLock = false;
388 IdLock.Entry lockEntry = null;
389 TraceScope traceScope = Trace.startSpan("HFileReaderV2.readBlock");
390 try {
391 while (true) {
392 if (useLock) {
393 lockEntry = offsetLock.getLockEntry(dataBlockOffset);
394 }
395
396
397 if (cacheConf.isBlockCacheEnabled()) {
398
399
400 HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, useLock, isCompaction,
401 updateCacheMetrics, expectedBlockType, expectedDataBlockEncoding);
402 if (cachedBlock != null) {
403 assert cachedBlock.isUnpacked() : "Packed block leak.";
404 if (cachedBlock.getBlockType().isData()) {
405 if (updateCacheMetrics) {
406 HFile.dataBlockReadCnt.incrementAndGet();
407 }
408
409
410 if (cachedBlock.getDataBlockEncoding() != dataBlockEncoder.getDataBlockEncoding()) {
411 throw new IOException("Cached block under key " + cacheKey + " "
412 + "has wrong encoding: " + cachedBlock.getDataBlockEncoding() + " (expected: "
413 + dataBlockEncoder.getDataBlockEncoding() + ")");
414 }
415 }
416
417 return cachedBlock;
418 }
419
420 }
421 if (!useLock) {
422
423 useLock = true;
424 continue;
425 }
426 if (Trace.isTracing()) {
427 traceScope.getSpan().addTimelineAnnotation("blockCacheMiss");
428 }
429
430 HFileBlock hfileBlock = fsBlockReader.readBlockData(dataBlockOffset, onDiskBlockSize, -1,
431 pread);
432 validateBlockType(hfileBlock, expectedBlockType);
433 HFileBlock unpacked = hfileBlock.unpack(hfileContext, fsBlockReader);
434 BlockType.BlockCategory category = hfileBlock.getBlockType().getCategory();
435
436
437 if (cacheBlock && cacheConf.shouldCacheBlockOnRead(category)) {
438 cacheConf.getBlockCache().cacheBlock(cacheKey,
439 cacheConf.shouldCacheCompressed(category) ? hfileBlock : unpacked,
440 cacheConf.isInMemory(), this.cacheConf.isCacheDataInL1());
441 }
442
443 if (updateCacheMetrics && hfileBlock.getBlockType().isData()) {
444 HFile.dataBlockReadCnt.incrementAndGet();
445 }
446
447 return unpacked;
448 }
449 } finally {
450 traceScope.close();
451 if (lockEntry != null) {
452 offsetLock.releaseLockEntry(lockEntry);
453 }
454 }
455 }
456
457 @Override
458 public boolean hasMVCCInfo() {
459 return includesMemstoreTS && decodeMemstoreTS;
460 }
461
462
463
464
465
466
467
468
469
470
471 private void validateBlockType(HFileBlock block,
472 BlockType expectedBlockType) throws IOException {
473 if (expectedBlockType == null) {
474 return;
475 }
476 BlockType actualBlockType = block.getBlockType();
477 if (expectedBlockType.isData() && actualBlockType.isData()) {
478
479
480 return;
481 }
482 if (actualBlockType != expectedBlockType) {
483 throw new IOException("Expected block type " + expectedBlockType + ", " +
484 "but got " + actualBlockType + ": " + block);
485 }
486 }
487
488
489
490
491
492
493 @Override
494 public byte[] getLastKey() {
495 return dataBlockIndexReader.isEmpty() ? null : lastKey;
496 }
497
498
499
500
501
502
503 @Override
504 public byte[] midkey() throws IOException {
505 return dataBlockIndexReader.midkey();
506 }
507
508 @Override
509 public void close() throws IOException {
510 close(cacheConf.shouldEvictOnClose());
511 }
512
513 public void close(boolean evictOnClose) throws IOException {
514 PrefetchExecutor.cancel(path);
515 if (evictOnClose && cacheConf.isBlockCacheEnabled()) {
516 int numEvicted = cacheConf.getBlockCache().evictBlocksByHfileName(name);
517 if (LOG.isTraceEnabled()) {
518 LOG.trace("On close, file=" + name + " evicted=" + numEvicted
519 + " block(s)");
520 }
521 }
522 fsBlockReader.closeStreams();
523 }
524
525 public DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction) {
526 return dataBlockEncoder.getEffectiveEncodingInCache(isCompaction);
527 }
528
529
530 @Override
531 HFileBlock.FSReader getUncachedBlockReader() {
532 return fsBlockReader;
533 }
534
535
536 protected abstract static class AbstractScannerV2
537 extends AbstractHFileReader.Scanner {
538 protected HFileBlock block;
539
540
541
542
543
544
545
546
547 protected byte[] nextIndexedKey;
548
549 public AbstractScannerV2(HFileReaderV2 r, boolean cacheBlocks,
550 final boolean pread, final boolean isCompaction) {
551 super(r, cacheBlocks, pread, isCompaction);
552 }
553
554 protected abstract ByteBuffer getFirstKeyInBlock(HFileBlock curBlock);
555
556 protected abstract int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
557 boolean rewind, Cell key, boolean seekBefore) throws IOException;
558
559 @Override
560 public int seekTo(byte[] key, int offset, int length) throws IOException {
561
562
563 return seekTo(new KeyValue.KeyOnlyKeyValue(key, offset, length));
564 }
565
566 @Override
567 public int reseekTo(byte[] key, int offset, int length) throws IOException {
568 return reseekTo(new KeyValue.KeyOnlyKeyValue(key, offset, length));
569 }
570
571 @Override
572 public int seekTo(Cell key) throws IOException {
573 return seekTo(key, true);
574 }
575
576 @Override
577 public int reseekTo(Cell key) throws IOException {
578 int compared;
579 if (isSeeked()) {
580 compared = compareKey(reader.getComparator(), key);
581 if (compared < 1) {
582
583
584 return compared;
585 } else {
586
587 if (this.nextIndexedKey != null &&
588 (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY || reader
589 .getComparator()
590 .compareOnlyKeyPortion(key,
591 new KeyValue.KeyOnlyKeyValue(nextIndexedKey, 0,
592 nextIndexedKey.length)) < 0)) {
593
594
595
596
597
598
599 return loadBlockAndSeekToKey(this.block, nextIndexedKey, false, key, false);
600 }
601 }
602 }
603
604
605 return seekTo(key, false);
606 }
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623 public int seekTo(Cell key, boolean rewind) throws IOException {
624 HFileBlockIndex.BlockIndexReader indexReader = reader.getDataBlockIndexReader();
625 BlockWithScanInfo blockWithScanInfo = indexReader.loadDataBlockWithScanInfo(key, block,
626 cacheBlocks, pread, isCompaction, getEffectiveDataBlockEncoding());
627 if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) {
628
629 return -1;
630 }
631 return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(),
632 blockWithScanInfo.getNextIndexedKey(), rewind, key, false);
633 }
634
635 @Override
636 public boolean seekBefore(byte[] key, int offset, int length) throws IOException {
637 return seekBefore(new KeyValue.KeyOnlyKeyValue(key, offset, length));
638 }
639
640 @Override
641 public boolean seekBefore(Cell key) throws IOException {
642 HFileBlock seekToBlock = reader.getDataBlockIndexReader().seekToDataBlock(key, block,
643 cacheBlocks, pread, isCompaction,
644 ((HFileReaderV2) reader).getEffectiveEncodingInCache(isCompaction));
645 if (seekToBlock == null) {
646 return false;
647 }
648 ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock);
649
650 if (reader.getComparator()
651 .compareOnlyKeyPortion(
652 new KeyValue.KeyOnlyKeyValue(firstKey.array(), firstKey.arrayOffset(),
653 firstKey.limit()), key) >= 0) {
654 long previousBlockOffset = seekToBlock.getPrevBlockOffset();
655
656 if (previousBlockOffset == -1) {
657
658 return false;
659 }
660
661
662
663
664 seekToBlock = reader.readBlock(previousBlockOffset,
665 seekToBlock.getOffset() - previousBlockOffset, cacheBlocks,
666 pread, isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
667
668
669 }
670 byte[] firstKeyInCurrentBlock = Bytes.getBytes(firstKey);
671 loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, true);
672 return true;
673 }
674
675
676
677
678
679
680
681
682 protected HFileBlock readNextDataBlock() throws IOException {
683 long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset();
684 if (block == null)
685 return null;
686
687 HFileBlock curBlock = block;
688
689 do {
690 if (curBlock.getOffset() >= lastDataBlockOffset)
691 return null;
692
693 if (curBlock.getOffset() < 0) {
694 throw new IOException("Invalid block file offset: " + block);
695 }
696
697
698
699 curBlock = reader.readBlock(curBlock.getOffset()
700 + curBlock.getOnDiskSizeWithHeader(),
701 curBlock.getNextBlockOnDiskSizeWithHeader(), cacheBlocks, pread,
702 isCompaction, true, null, getEffectiveDataBlockEncoding());
703 } while (!curBlock.getBlockType().isData());
704
705 return curBlock;
706 }
707
708 public DataBlockEncoding getEffectiveDataBlockEncoding() {
709 return ((HFileReaderV2)reader).getEffectiveEncodingInCache(isCompaction);
710 }
711
712
713
714
715
716
717
718
719 public abstract int compareKey(KVComparator comparator, byte[] key, int offset,
720 int length);
721
722 public abstract int compareKey(KVComparator comparator, Cell kv);
723 }
724
725
726
727
728 protected static class ScannerV2 extends AbstractScannerV2 {
729 private HFileReaderV2 reader;
730
731 public ScannerV2(HFileReaderV2 r, boolean cacheBlocks,
732 final boolean pread, final boolean isCompaction) {
733 super(r, cacheBlocks, pread, isCompaction);
734 this.reader = r;
735 }
736
737 @Override
738 public Cell getKeyValue() {
739 if (!isSeeked())
740 return null;
741
742 KeyValue ret = new KeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
743 + blockBuffer.position(), getCellBufSize());
744 if (this.reader.shouldIncludeMemstoreTS()) {
745 ret.setSequenceId(currMemstoreTS);
746 }
747 return ret;
748 }
749
750 protected int getCellBufSize() {
751 return KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen;
752 }
753
754 @Override
755 public ByteBuffer getKey() {
756 assertSeeked();
757 return ByteBuffer.wrap(
758 blockBuffer.array(),
759 blockBuffer.arrayOffset() + blockBuffer.position()
760 + KEY_VALUE_LEN_SIZE, currKeyLen).slice();
761 }
762
763 @Override
764 public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
765 return comparator.compareFlatKey(key, offset, length, blockBuffer.array(),
766 blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen);
767 }
768
769 @Override
770 public ByteBuffer getValue() {
771 assertSeeked();
772 return ByteBuffer.wrap(
773 blockBuffer.array(),
774 blockBuffer.arrayOffset() + blockBuffer.position()
775 + KEY_VALUE_LEN_SIZE + currKeyLen, currValueLen).slice();
776 }
777
778 protected void setNonSeekedState() {
779 block = null;
780 blockBuffer = null;
781 currKeyLen = 0;
782 currValueLen = 0;
783 currMemstoreTS = 0;
784 currMemstoreTSLen = 0;
785 }
786
787
788
789
790
791
792
793
794 @Override
795 public boolean next() throws IOException {
796 assertSeeked();
797
798 try {
799 blockBuffer.position(getNextCellStartPosition());
800 } catch (IllegalArgumentException e) {
801 LOG.error("Current pos = " + blockBuffer.position()
802 + "; currKeyLen = " + currKeyLen + "; currValLen = "
803 + currValueLen + "; block limit = " + blockBuffer.limit()
804 + "; HFile name = " + reader.getName()
805 + "; currBlock currBlockOffset = " + block.getOffset());
806 throw e;
807 }
808
809 if (blockBuffer.remaining() <= 0) {
810 long lastDataBlockOffset =
811 reader.getTrailer().getLastDataBlockOffset();
812
813 if (block.getOffset() >= lastDataBlockOffset) {
814 setNonSeekedState();
815 return false;
816 }
817
818
819 HFileBlock nextBlock = readNextDataBlock();
820 if (nextBlock == null) {
821 setNonSeekedState();
822 return false;
823 }
824
825 updateCurrBlock(nextBlock);
826 return true;
827 }
828
829
830 readKeyValueLen();
831 return true;
832 }
833
834 protected int getNextCellStartPosition() {
835 return blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen
836 + currMemstoreTSLen;
837 }
838
839
840
841
842
843
844
845
846 @Override
847 public boolean seekTo() throws IOException {
848 if (reader == null) {
849 return false;
850 }
851
852 if (reader.getTrailer().getEntryCount() == 0) {
853
854 return false;
855 }
856
857 long firstDataBlockOffset =
858 reader.getTrailer().getFirstDataBlockOffset();
859 if (block != null && block.getOffset() == firstDataBlockOffset) {
860 blockBuffer.rewind();
861 readKeyValueLen();
862 return true;
863 }
864
865 block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
866 isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
867 if (block.getOffset() < 0) {
868 throw new IOException("Invalid block offset: " + block.getOffset());
869 }
870 updateCurrBlock(block);
871 return true;
872 }
873
874 @Override
875 protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
876 boolean rewind, Cell key, boolean seekBefore) throws IOException {
877 if (block == null || block.getOffset() != seekToBlock.getOffset()) {
878 updateCurrBlock(seekToBlock);
879 } else if (rewind) {
880 blockBuffer.rewind();
881 }
882
883
884 this.nextIndexedKey = nextIndexedKey;
885 return blockSeek(key, seekBefore);
886 }
887
888
889
890
891
892
893
894 protected void updateCurrBlock(HFileBlock newBlock) {
895 block = newBlock;
896
897
898 if (block.getBlockType() != BlockType.DATA) {
899 throw new IllegalStateException("ScannerV2 works only on data " +
900 "blocks, got " + block.getBlockType() + "; " +
901 "fileName=" + reader.name + ", " +
902 "dataBlockEncoder=" + reader.dataBlockEncoder + ", " +
903 "isCompaction=" + isCompaction);
904 }
905
906 blockBuffer = block.getBufferWithoutHeader();
907 readKeyValueLen();
908 blockFetches++;
909
910
911 this.nextIndexedKey = null;
912 }
913
914 protected void readKeyValueLen() {
915 blockBuffer.mark();
916 currKeyLen = blockBuffer.getInt();
917 currValueLen = blockBuffer.getInt();
918 ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
919 readMvccVersion();
920 if (currKeyLen < 0 || currValueLen < 0
921 || currKeyLen > blockBuffer.limit()
922 || currValueLen > blockBuffer.limit()) {
923 throw new IllegalStateException("Invalid currKeyLen " + currKeyLen
924 + " or currValueLen " + currValueLen + ". Block offset: "
925 + block.getOffset() + ", block length: " + blockBuffer.limit()
926 + ", position: " + blockBuffer.position() + " (without header).");
927 }
928 blockBuffer.reset();
929 }
930
931 protected void readMvccVersion() {
932 if (this.reader.shouldIncludeMemstoreTS()) {
933 if (this.reader.decodeMemstoreTS) {
934 try {
935 currMemstoreTS = Bytes.readVLong(blockBuffer.array(), blockBuffer.arrayOffset()
936 + blockBuffer.position());
937 currMemstoreTSLen = WritableUtils.getVIntSize(currMemstoreTS);
938 } catch (Exception e) {
939 throw new RuntimeException("Error reading memstore timestamp", e);
940 }
941 } else {
942 currMemstoreTS = 0;
943 currMemstoreTSLen = 1;
944 }
945 }
946 }
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965 protected int blockSeek(Cell key, boolean seekBefore) {
966 int klen, vlen;
967 long memstoreTS = 0;
968 int memstoreTSLen = 0;
969 int lastKeyValueSize = -1;
970 KeyValue.KeyOnlyKeyValue keyOnlykv = new KeyValue.KeyOnlyKeyValue();
971 do {
972 blockBuffer.mark();
973 klen = blockBuffer.getInt();
974 vlen = blockBuffer.getInt();
975 blockBuffer.reset();
976 if (this.reader.shouldIncludeMemstoreTS()) {
977 if (this.reader.decodeMemstoreTS) {
978 try {
979 int memstoreTSOffset = blockBuffer.arrayOffset() + blockBuffer.position()
980 + KEY_VALUE_LEN_SIZE + klen + vlen;
981 memstoreTS = Bytes.readVLong(blockBuffer.array(), memstoreTSOffset);
982 memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
983 } catch (Exception e) {
984 throw new RuntimeException("Error reading memstore timestamp", e);
985 }
986 } else {
987 memstoreTS = 0;
988 memstoreTSLen = 1;
989 }
990 }
991
992 int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE;
993 keyOnlykv.setKey(blockBuffer.array(), keyOffset, klen);
994 int comp = reader.getComparator().compareOnlyKeyPortion(key, keyOnlykv);
995
996 if (comp == 0) {
997 if (seekBefore) {
998 if (lastKeyValueSize < 0) {
999 throw new IllegalStateException("blockSeek with seekBefore "
1000 + "at the first key of the block: key="
1001 + CellUtil.getCellKeyAsString(key)
1002 + ", blockOffset=" + block.getOffset() + ", onDiskSize="
1003 + block.getOnDiskSizeWithHeader());
1004 }
1005 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1006 readKeyValueLen();
1007 return 1;
1008 }
1009 currKeyLen = klen;
1010 currValueLen = vlen;
1011 if (this.reader.shouldIncludeMemstoreTS()) {
1012 currMemstoreTS = memstoreTS;
1013 currMemstoreTSLen = memstoreTSLen;
1014 }
1015 return 0;
1016 } else if (comp < 0) {
1017 if (lastKeyValueSize > 0)
1018 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1019 readKeyValueLen();
1020 if (lastKeyValueSize == -1 && blockBuffer.position() == 0
1021 && this.reader.trailer.getMinorVersion() >= MINOR_VERSION_WITH_FAKED_KEY) {
1022 return HConstants.INDEX_KEY_MAGIC;
1023 }
1024 return 1;
1025 }
1026
1027
1028 lastKeyValueSize = klen + vlen + memstoreTSLen + KEY_VALUE_LEN_SIZE;
1029 blockBuffer.position(blockBuffer.position() + lastKeyValueSize);
1030 } while (blockBuffer.remaining() > 0);
1031
1032
1033
1034
1035 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1036 readKeyValueLen();
1037 return 1;
1038 }
1039
1040 @Override
1041 protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
1042 ByteBuffer buffer = curBlock.getBufferWithoutHeader();
1043
1044 buffer.rewind();
1045 int klen = buffer.getInt();
1046 buffer.getInt();
1047 ByteBuffer keyBuff = buffer.slice();
1048 keyBuff.limit(klen);
1049 keyBuff.rewind();
1050 return keyBuff;
1051 }
1052
1053 @Override
1054 public String getKeyString() {
1055 return Bytes.toStringBinary(blockBuffer.array(),
1056 blockBuffer.arrayOffset() + blockBuffer.position()
1057 + KEY_VALUE_LEN_SIZE, currKeyLen);
1058 }
1059
1060 @Override
1061 public String getValueString() {
1062 return Bytes.toString(blockBuffer.array(), blockBuffer.arrayOffset()
1063 + blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen,
1064 currValueLen);
1065 }
1066
1067 @Override
1068 public int compareKey(KVComparator comparator, Cell key) {
1069 return comparator.compareOnlyKeyPortion(
1070 key,
1071 new KeyValue.KeyOnlyKeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
1072 + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen));
1073 }
1074 }
1075
1076
1077
1078
1079 protected static class EncodedScannerV2 extends AbstractScannerV2 {
1080 private final HFileBlockDecodingContext decodingCtx;
1081 private final DataBlockEncoder.EncodedSeeker seeker;
1082 private final DataBlockEncoder dataBlockEncoder;
1083 protected final HFileContext meta;
1084
1085 public EncodedScannerV2(HFileReaderV2 reader, boolean cacheBlocks,
1086 boolean pread, boolean isCompaction, HFileContext meta) {
1087 super(reader, cacheBlocks, pread, isCompaction);
1088 DataBlockEncoding encoding = reader.dataBlockEncoder.getDataBlockEncoding();
1089 dataBlockEncoder = encoding.getEncoder();
1090 decodingCtx = dataBlockEncoder.newDataBlockDecodingContext(meta);
1091 seeker = dataBlockEncoder.createSeeker(
1092 reader.getComparator(), decodingCtx);
1093 this.meta = meta;
1094 }
1095
1096 @Override
1097 public boolean isSeeked(){
1098 return this.block != null;
1099 }
1100
1101
1102
1103
1104
1105
1106
1107
1108 private void updateCurrentBlock(HFileBlock newBlock) throws CorruptHFileException {
1109 block = newBlock;
1110
1111
1112 if (block.getBlockType() != BlockType.ENCODED_DATA) {
1113 throw new IllegalStateException(
1114 "EncodedScanner works only on encoded data blocks");
1115 }
1116 short dataBlockEncoderId = block.getDataBlockEncodingId();
1117 if (!DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) {
1118 String encoderCls = dataBlockEncoder.getClass().getName();
1119 throw new CorruptHFileException("Encoder " + encoderCls
1120 + " doesn't support data block encoding "
1121 + DataBlockEncoding.getNameFromId(dataBlockEncoderId));
1122 }
1123
1124 seeker.setCurrentBuffer(getEncodedBuffer(newBlock));
1125 blockFetches++;
1126
1127
1128 this.nextIndexedKey = null;
1129 }
1130
1131 private ByteBuffer getEncodedBuffer(HFileBlock newBlock) {
1132 ByteBuffer origBlock = newBlock.getBufferReadOnly();
1133 ByteBuffer encodedBlock = ByteBuffer.wrap(origBlock.array(),
1134 origBlock.arrayOffset() + newBlock.headerSize() +
1135 DataBlockEncoding.ID_SIZE,
1136 newBlock.getUncompressedSizeWithoutHeader() -
1137 DataBlockEncoding.ID_SIZE).slice();
1138 return encodedBlock;
1139 }
1140
1141 @Override
1142 public boolean seekTo() throws IOException {
1143 if (reader == null) {
1144 return false;
1145 }
1146
1147 if (reader.getTrailer().getEntryCount() == 0) {
1148
1149 return false;
1150 }
1151
1152 long firstDataBlockOffset =
1153 reader.getTrailer().getFirstDataBlockOffset();
1154 if (block != null && block.getOffset() == firstDataBlockOffset) {
1155 seeker.rewind();
1156 return true;
1157 }
1158
1159 block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
1160 isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
1161 if (block.getOffset() < 0) {
1162 throw new IOException("Invalid block offset: " + block.getOffset());
1163 }
1164 updateCurrentBlock(block);
1165 return true;
1166 }
1167
1168 @Override
1169 public boolean next() throws IOException {
1170 boolean isValid = seeker.next();
1171 if (!isValid) {
1172 block = readNextDataBlock();
1173 isValid = block != null;
1174 if (isValid) {
1175 updateCurrentBlock(block);
1176 }
1177 }
1178 return isValid;
1179 }
1180
1181 @Override
1182 public ByteBuffer getKey() {
1183 assertValidSeek();
1184 return seeker.getKeyDeepCopy();
1185 }
1186
1187 @Override
1188 public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
1189 return seeker.compareKey(comparator, key, offset, length);
1190 }
1191
1192 @Override
1193 public ByteBuffer getValue() {
1194 assertValidSeek();
1195 return seeker.getValueShallowCopy();
1196 }
1197
1198 @Override
1199 public Cell getKeyValue() {
1200 if (block == null) {
1201 return null;
1202 }
1203 return seeker.getKeyValue();
1204 }
1205
1206 @Override
1207 public String getKeyString() {
1208 ByteBuffer keyBuffer = getKey();
1209 return Bytes.toStringBinary(keyBuffer.array(),
1210 keyBuffer.arrayOffset(), keyBuffer.limit());
1211 }
1212
1213 @Override
1214 public String getValueString() {
1215 ByteBuffer valueBuffer = getValue();
1216 return Bytes.toStringBinary(valueBuffer.array(),
1217 valueBuffer.arrayOffset(), valueBuffer.limit());
1218 }
1219
1220 private void assertValidSeek() {
1221 if (block == null) {
1222 throw new NotSeekedException();
1223 }
1224 }
1225
1226 @Override
1227 protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
1228 return dataBlockEncoder.getFirstKeyInBlock(getEncodedBuffer(curBlock));
1229 }
1230
1231 @Override
1232 protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
1233 boolean rewind, Cell key, boolean seekBefore) throws IOException {
1234 if (block == null || block.getOffset() != seekToBlock.getOffset()) {
1235 updateCurrentBlock(seekToBlock);
1236 } else if (rewind) {
1237 seeker.rewind();
1238 }
1239 this.nextIndexedKey = nextIndexedKey;
1240 return seeker.seekToKeyInBlock(key, seekBefore);
1241 }
1242
1243 @Override
1244 public int compareKey(KVComparator comparator, Cell key) {
1245 return seeker.compareKey(comparator, key);
1246 }
1247 }
1248
1249
1250
1251
1252
1253 @Override
1254 public DataInput getGeneralBloomFilterMetadata() throws IOException {
1255 return this.getBloomFilterMetadata(BlockType.GENERAL_BLOOM_META);
1256 }
1257
1258 @Override
1259 public DataInput getDeleteBloomFilterMetadata() throws IOException {
1260 return this.getBloomFilterMetadata(BlockType.DELETE_FAMILY_BLOOM_META);
1261 }
1262
1263 private DataInput getBloomFilterMetadata(BlockType blockType)
1264 throws IOException {
1265 if (blockType != BlockType.GENERAL_BLOOM_META &&
1266 blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
1267 throw new RuntimeException("Block Type: " + blockType.toString() +
1268 " is not supported") ;
1269 }
1270
1271 for (HFileBlock b : loadOnOpenBlocks)
1272 if (b.getBlockType() == blockType)
1273 return b.getByteStream();
1274 return null;
1275 }
1276
1277 @Override
1278 public boolean isFileInfoLoaded() {
1279 return true;
1280 }
1281
1282
1283
1284
1285
1286 private void validateMinorVersion(Path path, int minorVersion) {
1287 if (minorVersion < MIN_MINOR_VERSION ||
1288 minorVersion > MAX_MINOR_VERSION) {
1289 String msg = "Minor version for path " + path +
1290 " is expected to be between " +
1291 MIN_MINOR_VERSION + " and " + MAX_MINOR_VERSION +
1292 " but is found to be " + minorVersion;
1293 LOG.error(msg);
1294 throw new RuntimeException(msg);
1295 }
1296 }
1297
1298 @Override
1299 public int getMajorVersion() {
1300 return 2;
1301 }
1302
1303 @Override
1304 public HFileContext getFileContext() {
1305 return hfileContext;
1306 }
1307
1308
1309
1310
1311
1312 @VisibleForTesting
1313 boolean prefetchComplete() {
1314 return PrefetchExecutor.isCompleted(path);
1315 }
1316 }