View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import java.io.DataInput;
21  import java.io.IOException;
22  import java.nio.ByteBuffer;
23  import java.util.ArrayList;
24  import java.util.List;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.hbase.classification.InterfaceAudience;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.hbase.Cell;
32  import org.apache.hadoop.hbase.CellUtil;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.KeyValue;
35  import org.apache.hadoop.hbase.KeyValue.KVComparator;
36  import org.apache.hadoop.hbase.fs.HFileSystem;
37  import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
38  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
39  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
40  import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
41  import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
42  import org.apache.hadoop.hbase.util.ByteBufferUtils;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.util.IdLock;
45  import org.apache.hadoop.io.WritableUtils;
46  import org.apache.htrace.Trace;
47  import org.apache.htrace.TraceScope;
48  
49  import com.google.common.annotations.VisibleForTesting;
50  
51  /**
52   * {@link HFile} reader for version 2.
53   */
54  @InterfaceAudience.Private
55  public class HFileReaderV2 extends AbstractHFileReader {
56  
57    private static final Log LOG = LogFactory.getLog(HFileReaderV2.class);
58  
59    /** Minor versions in HFile V2 starting with this number have hbase checksums */
60    public static final int MINOR_VERSION_WITH_CHECKSUM = 1;
61    /** In HFile V2 minor version that does not support checksums */
62    public static final int MINOR_VERSION_NO_CHECKSUM = 0;
63  
64    /** HFile minor version that introduced pbuf filetrailer */
65    public static final int PBUF_TRAILER_MINOR_VERSION = 2;
66  
67    /**
68     * The size of a (key length, value length) tuple that prefixes each entry in
69     * a data block.
70     */
71    public final static int KEY_VALUE_LEN_SIZE = 2 * Bytes.SIZEOF_INT;
72  
73    protected boolean includesMemstoreTS = false;
74    protected boolean decodeMemstoreTS = false;
75    protected boolean shouldIncludeMemstoreTS() {
76      return includesMemstoreTS;
77    }
78  
79    /** Filesystem-level block reader. */
80    protected HFileBlock.FSReader fsBlockReader;
81  
82    /**
83     * A "sparse lock" implementation allowing to lock on a particular block
84     * identified by offset. The purpose of this is to avoid two clients loading
85     * the same block, and have all but one client wait to get the block from the
86     * cache.
87     */
88    private IdLock offsetLock = new IdLock();
89  
90    /**
91     * Blocks read from the load-on-open section, excluding data root index, meta
92     * index, and file info.
93     */
94    private List<HFileBlock> loadOnOpenBlocks = new ArrayList<HFileBlock>();
95  
96    /** Minimum minor version supported by this HFile format */
97    static final int MIN_MINOR_VERSION = 0;
98  
99    /** Maximum minor version supported by this HFile format */
100   // We went to version 2 when we moved to pb'ing fileinfo and the trailer on
101   // the file. This version can read Writables version 1.
102   static final int MAX_MINOR_VERSION = 3;
103 
104   /** Minor versions starting with this number have faked index key */
105   static final int MINOR_VERSION_WITH_FAKED_KEY = 3;
106 
107   protected HFileContext hfileContext;
108 
109   /**
110    * Opens a HFile. You must load the index before you can use it by calling
111    * {@link #loadFileInfo()}.
112    *
113    * @param path Path to HFile.
114    * @param trailer File trailer.
115    * @param fsdis input stream.
116    * @param size Length of the stream.
117    * @param cacheConf Cache configuration.
118    * @param hfs
119    * @param conf
120    */
121   public HFileReaderV2(final Path path, final FixedFileTrailer trailer,
122       final FSDataInputStreamWrapper fsdis, final long size, final CacheConfig cacheConf,
123       final HFileSystem hfs, final Configuration conf) throws IOException {
124     super(path, trailer, size, cacheConf, hfs, conf);
125     this.conf = conf;
126     trailer.expectMajorVersion(getMajorVersion());
127     validateMinorVersion(path, trailer.getMinorVersion());
128     this.hfileContext = createHFileContext(fsdis, fileSize, hfs, path, trailer);
129     HFileBlock.FSReaderImpl fsBlockReaderV2 =
130       new HFileBlock.FSReaderImpl(fsdis, fileSize, hfs, path, hfileContext);
131     this.fsBlockReader = fsBlockReaderV2; // upcast
132 
133     // Comparator class name is stored in the trailer in version 2.
134     comparator = trailer.createComparator();
135     dataBlockIndexReader = new HFileBlockIndex.BlockIndexReader(comparator,
136         trailer.getNumDataIndexLevels(), this);
137     metaBlockIndexReader = new HFileBlockIndex.BlockIndexReader(
138         KeyValue.RAW_COMPARATOR, 1);
139 
140     // Parse load-on-open data.
141 
142     HFileBlock.BlockIterator blockIter = fsBlockReaderV2.blockRange(
143         trailer.getLoadOnOpenDataOffset(),
144         fileSize - trailer.getTrailerSize());
145 
146     // Data index. We also read statistics about the block index written after
147     // the root level.
148     dataBlockIndexReader.readMultiLevelIndexRoot(
149         blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
150         trailer.getDataIndexCount());
151 
152     // Meta index.
153     metaBlockIndexReader.readRootIndex(
154         blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
155         trailer.getMetaIndexCount());
156 
157     // File info
158     fileInfo = new FileInfo();
159     fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
160     lastKey = fileInfo.get(FileInfo.LASTKEY);
161     avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN));
162     avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN));
163     byte [] keyValueFormatVersion =
164         fileInfo.get(HFileWriterV2.KEY_VALUE_VERSION);
165     includesMemstoreTS = keyValueFormatVersion != null &&
166         Bytes.toInt(keyValueFormatVersion) ==
167             HFileWriterV2.KEY_VALUE_VER_WITH_MEMSTORE;
168     fsBlockReaderV2.setIncludesMemstoreTS(includesMemstoreTS);
169     if (includesMemstoreTS) {
170       decodeMemstoreTS = Bytes.toLong(fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY)) > 0;
171     }
172 
173     // Read data block encoding algorithm name from file info.
174     dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo);
175     fsBlockReaderV2.setDataBlockEncoder(dataBlockEncoder);
176 
177     // Store all other load-on-open blocks for further consumption.
178     HFileBlock b;
179     while ((b = blockIter.nextBlock()) != null) {
180       loadOnOpenBlocks.add(b);
181     }
182 
183     // Prefetch file blocks upon open if requested
184     if (cacheConf.shouldPrefetchOnOpen()) {
185       PrefetchExecutor.request(path, new Runnable() {
186         public void run() {
187           try {
188             long offset = 0;
189             long end = fileSize - getTrailer().getTrailerSize();
190             HFileBlock prevBlock = null;
191             while (offset < end) {
192               if (Thread.interrupted()) {
193                 break;
194               }
195               long onDiskSize = -1;
196               if (prevBlock != null) {
197                 onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
198               }
199               HFileBlock block = readBlock(offset, onDiskSize, true, false, false, false,
200                 null, null);
201               prevBlock = block;
202               offset += block.getOnDiskSizeWithHeader();
203             }
204           } catch (IOException e) {
205             // IOExceptions are probably due to region closes (relocation, etc.)
206             if (LOG.isTraceEnabled()) {
207               LOG.trace("Exception encountered while prefetching " + path + ":", e);
208             }
209           } catch (Exception e) {
210             // Other exceptions are interesting
211             LOG.warn("Exception encountered while prefetching " + path + ":", e);
212           } finally {
213             PrefetchExecutor.complete(path);
214           }
215         }
216       });
217     }
218   }
219 
220   protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
221       HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
222     return new HFileContextBuilder()
223       .withIncludesMvcc(this.includesMemstoreTS)
224       .withCompression(this.compressAlgo)
225       .withHBaseCheckSum(trailer.getMinorVersion() >= MINOR_VERSION_WITH_CHECKSUM)
226       .build();
227   }
228 
229   /**
230    * Create a Scanner on this file. No seeks or reads are done on creation. Call
231    * {@link HFileScanner#seekTo(byte[])} to position an start the read. There is
232    * nothing to clean up in a Scanner. Letting go of your references to the
233    * scanner is sufficient.
234    *
235    * @param cacheBlocks True if we should cache blocks read in by this scanner.
236    * @param pread Use positional read rather than seek+read if true (pread is
237    *          better for random reads, seek+read is better scanning).
238    * @param isCompaction is scanner being used for a compaction?
239    * @return Scanner on this file.
240    */
241    @Override
242    public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
243       final boolean isCompaction) {
244     if (dataBlockEncoder.useEncodedScanner()) {
245       return new EncodedScannerV2(this, cacheBlocks, pread, isCompaction,
246           hfileContext);
247     }
248 
249     return new ScannerV2(this, cacheBlocks, pread, isCompaction);
250   }
251 
252   /**
253    * Retrieve block from cache. Validates the retrieved block's type vs {@code expectedBlockType}
254    * and its encoding vs. {@code expectedDataBlockEncoding}. Unpacks the block as necessary.
255    */
256    private HFileBlock getCachedBlock(BlockCacheKey cacheKey, boolean cacheBlock, boolean useLock,
257        boolean isCompaction, boolean updateCacheMetrics, BlockType expectedBlockType,
258        DataBlockEncoding expectedDataBlockEncoding) throws IOException {
259      // Check cache for block. If found return.
260      if (cacheConf.isBlockCacheEnabled()) {
261        BlockCache cache = cacheConf.getBlockCache();
262        HFileBlock cachedBlock = (HFileBlock) cache.getBlock(cacheKey, cacheBlock, useLock,
263          updateCacheMetrics);
264        if (cachedBlock != null) {
265          if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) {
266            cachedBlock = cachedBlock.unpack(hfileContext, fsBlockReader);
267          }
268          validateBlockType(cachedBlock, expectedBlockType);
269 
270          if (expectedDataBlockEncoding == null) {
271            return cachedBlock;
272          }
273          DataBlockEncoding actualDataBlockEncoding =
274                  cachedBlock.getDataBlockEncoding();
275          // Block types other than data blocks always have
276          // DataBlockEncoding.NONE. To avoid false negative cache misses, only
277          // perform this check if cached block is a data block.
278          if (cachedBlock.getBlockType().isData() &&
279                  !actualDataBlockEncoding.equals(expectedDataBlockEncoding)) {
280            // This mismatch may happen if a ScannerV2, which is used for say a
281            // compaction, tries to read an encoded block from the block cache.
282            // The reverse might happen when an EncodedScannerV2 tries to read
283            // un-encoded blocks which were cached earlier.
284            //
285            // Because returning a data block with an implicit BlockType mismatch
286            // will cause the requesting scanner to throw a disk read should be
287            // forced here. This will potentially cause a significant number of
288            // cache misses, so update so we should keep track of this as it might
289            // justify the work on a CompoundScannerV2.
290            if (!expectedDataBlockEncoding.equals(DataBlockEncoding.NONE) &&
291                    !actualDataBlockEncoding.equals(DataBlockEncoding.NONE)) {
292              // If the block is encoded but the encoding does not match the
293              // expected encoding it is likely the encoding was changed but the
294              // block was not yet evicted. Evictions on file close happen async
295              // so blocks with the old encoding still linger in cache for some
296              // period of time. This event should be rare as it only happens on
297              // schema definition change.
298              LOG.info("Evicting cached block with key " + cacheKey +
299                      " because of a data block encoding mismatch" +
300                      "; expected: " + expectedDataBlockEncoding +
301                      ", actual: " + actualDataBlockEncoding);
302              cache.evictBlock(cacheKey);
303            }
304            return null;
305          }
306          return cachedBlock;
307        }
308      }
309      return null;
310    }
311   /**
312    * @param metaBlockName
313    * @param cacheBlock Add block to cache, if found
314    * @return block wrapped in a ByteBuffer, with header skipped
315    * @throws IOException
316    */
317   @Override
318   public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock)
319       throws IOException {
320     if (trailer.getMetaIndexCount() == 0) {
321       return null; // there are no meta blocks
322     }
323     if (metaBlockIndexReader == null) {
324       throw new IOException("Meta index not loaded");
325     }
326 
327     byte[] mbname = Bytes.toBytes(metaBlockName);
328     int block = metaBlockIndexReader.rootBlockContainingKey(mbname,
329         0, mbname.length);
330     if (block == -1)
331       return null;
332     long blockSize = metaBlockIndexReader.getRootBlockDataSize(block);
333 
334     // Per meta key from any given file, synchronize reads for said block. This
335     // is OK to do for meta blocks because the meta block index is always
336     // single-level.
337     synchronized (metaBlockIndexReader.getRootBlockKey(block)) {
338       // Check cache for block. If found return.
339       long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block);
340       BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset);
341 
342       cacheBlock &= cacheConf.shouldCacheDataOnRead();
343       if (cacheConf.isBlockCacheEnabled()) {
344         HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, false, true, true,
345           BlockType.META, null);
346         if (cachedBlock != null) {
347           assert cachedBlock.isUnpacked() : "Packed block leak.";
348           // Return a distinct 'shallow copy' of the block,
349           // so pos does not get messed by the scanner
350           return cachedBlock.getBufferWithoutHeader();
351         }
352         // Cache Miss, please load.
353       }
354 
355       HFileBlock metaBlock = fsBlockReader.readBlockData(metaBlockOffset,
356           blockSize, -1, true).unpack(hfileContext, fsBlockReader);
357 
358       // Cache the block
359       if (cacheBlock) {
360         cacheConf.getBlockCache().cacheBlock(cacheKey, metaBlock,
361             cacheConf.isInMemory(), this.cacheConf.isCacheDataInL1());
362       }
363 
364       return metaBlock.getBufferWithoutHeader();
365     }
366   }
367 
368   @Override
369   public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize,
370       final boolean cacheBlock, boolean pread, final boolean isCompaction,
371       boolean updateCacheMetrics, BlockType expectedBlockType,
372       DataBlockEncoding expectedDataBlockEncoding)
373       throws IOException {
374     if (dataBlockIndexReader == null) {
375       throw new IOException("Block index not loaded");
376     }
377     if (dataBlockOffset < 0 || dataBlockOffset >= trailer.getLoadOnOpenDataOffset()) {
378       throw new IOException("Requested block is out of range: " + dataBlockOffset +
379         ", lastDataBlockOffset: " + trailer.getLastDataBlockOffset());
380     }
381 
382     // For any given block from any given file, synchronize reads for said block.
383     // Without a cache, this synchronizing is needless overhead, but really
384     // the other choice is to duplicate work (which the cache would prevent you
385     // from doing).
386     BlockCacheKey cacheKey = new BlockCacheKey(name, dataBlockOffset);
387     boolean useLock = false;
388     IdLock.Entry lockEntry = null;
389     TraceScope traceScope = Trace.startSpan("HFileReaderV2.readBlock");
390     try {
391       while (true) {
392         if (useLock) {
393           lockEntry = offsetLock.getLockEntry(dataBlockOffset);
394         }
395 
396         // Check cache for block. If found return.
397         if (cacheConf.isBlockCacheEnabled()) {
398           // Try and get the block from the block cache. If the useLock variable is true then this
399           // is the second time through the loop and it should not be counted as a block cache miss.
400           HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, useLock, isCompaction,
401             updateCacheMetrics, expectedBlockType, expectedDataBlockEncoding);
402           if (cachedBlock != null) {
403             if (Trace.isTracing()) {
404               traceScope.getSpan().addTimelineAnnotation("blockCacheHit");
405             }
406             assert cachedBlock.isUnpacked() : "Packed block leak.";
407             if (cachedBlock.getBlockType().isData()) {
408               if (updateCacheMetrics) {
409                 HFile.dataBlockReadCnt.incrementAndGet();
410               }
411               // Validate encoding type for data blocks. We include encoding
412               // type in the cache key, and we expect it to match on a cache hit.
413               if (cachedBlock.getDataBlockEncoding() != dataBlockEncoder.getDataBlockEncoding()) {
414                 throw new IOException("Cached block under key " + cacheKey + " "
415                   + "has wrong encoding: " + cachedBlock.getDataBlockEncoding() + " (expected: "
416                   + dataBlockEncoder.getDataBlockEncoding() + ")");
417               }
418             }
419             // Cache-hit. Return!
420             return cachedBlock;
421           }
422           // Carry on, please load.
423         }
424         if (!useLock) {
425           // check cache again with lock
426           useLock = true;
427           continue;
428         }
429         if (Trace.isTracing()) {
430           traceScope.getSpan().addTimelineAnnotation("blockCacheMiss");
431         }
432         // Load block from filesystem.
433         HFileBlock hfileBlock = fsBlockReader.readBlockData(dataBlockOffset, onDiskBlockSize, -1,
434             pread);
435         validateBlockType(hfileBlock, expectedBlockType);
436         HFileBlock unpacked = hfileBlock.unpack(hfileContext, fsBlockReader);
437         BlockType.BlockCategory category = hfileBlock.getBlockType().getCategory();
438 
439         // Cache the block if necessary
440         if (cacheBlock && cacheConf.shouldCacheBlockOnRead(category)) {
441           cacheConf.getBlockCache().cacheBlock(cacheKey,
442             cacheConf.shouldCacheCompressed(category) ? hfileBlock : unpacked,
443             cacheConf.isInMemory(), this.cacheConf.isCacheDataInL1());
444         }
445 
446         if (updateCacheMetrics && hfileBlock.getBlockType().isData()) {
447           HFile.dataBlockReadCnt.incrementAndGet();
448         }
449 
450         return unpacked;
451       }
452     } finally {
453       traceScope.close();
454       if (lockEntry != null) {
455         offsetLock.releaseLockEntry(lockEntry);
456       }
457     }
458   }
459 
460   @Override
461   public boolean hasMVCCInfo() {
462     return includesMemstoreTS && decodeMemstoreTS;
463   }
464 
465   /**
466    * Compares the actual type of a block retrieved from cache or disk with its
467    * expected type and throws an exception in case of a mismatch. Expected
468    * block type of {@link BlockType#DATA} is considered to match the actual
469    * block type [@link {@link BlockType#ENCODED_DATA} as well.
470    * @param block a block retrieved from cache or disk
471    * @param expectedBlockType the expected block type, or null to skip the
472    *          check
473    */
474   private void validateBlockType(HFileBlock block,
475       BlockType expectedBlockType) throws IOException {
476     if (expectedBlockType == null) {
477       return;
478     }
479     BlockType actualBlockType = block.getBlockType();
480     if (expectedBlockType.isData() && actualBlockType.isData()) {
481       // We consider DATA to match ENCODED_DATA for the purpose of this
482       // verification.
483       return;
484     }
485     if (actualBlockType != expectedBlockType) {
486       throw new IOException("Expected block type " + expectedBlockType + ", " +
487           "but got " + actualBlockType + ": " + block);
488     }
489   }
490 
491   /**
492    * @return Last key in the file. May be null if file has no entries. Note that
493    *         this is not the last row key, but rather the byte form of the last
494    *         KeyValue.
495    */
496   @Override
497   public byte[] getLastKey() {
498     return dataBlockIndexReader.isEmpty() ? null : lastKey;
499   }
500 
501   /**
502    * @return Midkey for this file. We work with block boundaries only so
503    *         returned midkey is an approximation only.
504    * @throws IOException
505    */
506   @Override
507   public byte[] midkey() throws IOException {
508     return dataBlockIndexReader.midkey();
509   }
510 
511   @Override
512   public void close() throws IOException {
513     close(cacheConf.shouldEvictOnClose());
514   }
515 
516   public void close(boolean evictOnClose) throws IOException {
517     PrefetchExecutor.cancel(path);
518     if (evictOnClose && cacheConf.isBlockCacheEnabled()) {
519       int numEvicted = cacheConf.getBlockCache().evictBlocksByHfileName(name);
520       if (LOG.isTraceEnabled()) {
521         LOG.trace("On close, file=" + name + " evicted=" + numEvicted
522           + " block(s)");
523       }
524     }
525     fsBlockReader.closeStreams();
526   }
527 
528   public DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction) {
529     return dataBlockEncoder.getEffectiveEncodingInCache(isCompaction);
530   }
531 
532   /** For testing */
533   @Override
534   HFileBlock.FSReader getUncachedBlockReader() {
535     return fsBlockReader;
536   }
537 
538 
539   protected abstract static class AbstractScannerV2
540       extends AbstractHFileReader.Scanner {
541     protected HFileBlock block;
542 
543     @Override
544     public Cell getNextIndexedKey() {
545       return nextIndexedKey;
546     }
547     /**
548      * The next indexed key is to keep track of the indexed key of the next data block.
549      * If the nextIndexedKey is HConstants.NO_NEXT_INDEXED_KEY, it means that the
550      * current data block is the last data block.
551      *
552      * If the nextIndexedKey is null, it means the nextIndexedKey has not been loaded yet.
553      */
554     protected Cell nextIndexedKey;
555 
556     public AbstractScannerV2(HFileReaderV2 r, boolean cacheBlocks,
557         final boolean pread, final boolean isCompaction) {
558       super(r, cacheBlocks, pread, isCompaction);
559     }
560 
561     protected abstract ByteBuffer getFirstKeyInBlock(HFileBlock curBlock);
562 
563     protected abstract int loadBlockAndSeekToKey(HFileBlock seekToBlock, Cell nextIndexedKey,
564         boolean rewind, Cell key, boolean seekBefore) throws IOException;
565 
566     @Override
567     public int seekTo(byte[] key, int offset, int length) throws IOException {
568       // Always rewind to the first key of the block, because the given key
569       // might be before or after the current key.
570       return seekTo(new KeyValue.KeyOnlyKeyValue(key, offset, length));
571     }
572 
573     @Override
574     public int reseekTo(byte[] key, int offset, int length) throws IOException {
575       return reseekTo(new KeyValue.KeyOnlyKeyValue(key, offset, length));
576     }
577 
578     @Override
579     public int seekTo(Cell key) throws IOException {
580       return seekTo(key, true);
581     }
582 
583     @Override
584     public int reseekTo(Cell key) throws IOException {
585       int compared;
586       if (isSeeked()) {
587         compared = compareKey(reader.getComparator(), key);
588         if (compared < 1) {
589           // If the required key is less than or equal to current key, then
590           // don't do anything.
591           return compared;
592         } else {
593           // The comparison with no_next_index_key has to be checked
594           if (this.nextIndexedKey != null &&
595               (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY || reader
596               .getComparator()
597                   .compareOnlyKeyPortion(key, nextIndexedKey) < 0)) {
598             // The reader shall continue to scan the current data block instead
599             // of querying the
600             // block index as long as it knows the target key is strictly
601             // smaller than
602             // the next indexed key or the current data block is the last data
603             // block.
604             return loadBlockAndSeekToKey(this.block, nextIndexedKey, false, key, false);
605           }
606         }
607       }
608       // Don't rewind on a reseek operation, because reseek implies that we are
609       // always going forward in the file.
610       return seekTo(key, false);
611     }
612 
613 
614     /**
615      * An internal API function. Seek to the given key, optionally rewinding to
616      * the first key of the block before doing the seek.
617      *
618      * @param key - a cell representing the key that we need to fetch
619      * @param rewind whether to rewind to the first key of the block before
620      *        doing the seek. If this is false, we are assuming we never go
621      *        back, otherwise the result is undefined.
622      * @return -1 if the key is earlier than the first key of the file,
623      *         0 if we are at the given key, 1 if we are past the given key
624      *         -2 if the key is earlier than the first key of the file while
625      *         using a faked index key
626      * @throws IOException
627      */
628     public int seekTo(Cell key, boolean rewind) throws IOException {
629       HFileBlockIndex.BlockIndexReader indexReader = reader.getDataBlockIndexReader();
630       BlockWithScanInfo blockWithScanInfo = indexReader.loadDataBlockWithScanInfo(key, block,
631           cacheBlocks, pread, isCompaction, getEffectiveDataBlockEncoding());
632       if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) {
633         // This happens if the key e.g. falls before the beginning of the file.
634         return -1;
635       }
636       return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(),
637           blockWithScanInfo.getNextIndexedKey(), rewind, key, false);
638     }
639 
640     @Override
641     public boolean seekBefore(byte[] key, int offset, int length) throws IOException {
642       return seekBefore(new KeyValue.KeyOnlyKeyValue(key, offset, length));
643     }
644 
645     @Override
646     public boolean seekBefore(Cell key) throws IOException {
647       HFileBlock seekToBlock = reader.getDataBlockIndexReader().seekToDataBlock(key, block,
648           cacheBlocks, pread, isCompaction,
649           ((HFileReaderV2) reader).getEffectiveEncodingInCache(isCompaction));
650       if (seekToBlock == null) {
651         return false;
652       }
653       ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock);
654 
655       if (reader.getComparator()
656           .compareOnlyKeyPortion(
657               new KeyValue.KeyOnlyKeyValue(firstKey.array(), firstKey.arrayOffset(),
658                   firstKey.limit()), key) >= 0) {
659         long previousBlockOffset = seekToBlock.getPrevBlockOffset();
660         // The key we are interested in
661         if (previousBlockOffset == -1) {
662           // we have a 'problem', the key we want is the first of the file.
663           return false;
664         }
665 
666         // It is important that we compute and pass onDiskSize to the block
667         // reader so that it does not have to read the header separately to
668         // figure out the size.
669         seekToBlock = reader.readBlock(previousBlockOffset,
670             seekToBlock.getOffset() - previousBlockOffset, cacheBlocks,
671             pread, isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
672         // TODO shortcut: seek forward in this block to the last key of the
673         // block.
674       }
675       Cell firstKeyInCurrentBlock = new KeyValue.KeyOnlyKeyValue(Bytes.getBytes(firstKey));
676       loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, true);
677       return true;
678     }
679 
680     /**
681      * Scans blocks in the "scanned" section of the {@link HFile} until the next
682      * data block is found.
683      *
684      * @return the next block, or null if there are no more data blocks
685      * @throws IOException
686      */
687     protected HFileBlock readNextDataBlock() throws IOException {
688       long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset();
689       if (block == null)
690         return null;
691 
692       HFileBlock curBlock = block;
693 
694       do {
695         if (curBlock.getOffset() >= lastDataBlockOffset)
696           return null;
697 
698         if (curBlock.getOffset() < 0) {
699           throw new IOException("Invalid block file offset: " + block);
700         }
701 
702         // We are reading the next block without block type validation, because
703         // it might turn out to be a non-data block.
704         curBlock = reader.readBlock(curBlock.getOffset()
705             + curBlock.getOnDiskSizeWithHeader(),
706             curBlock.getNextBlockOnDiskSizeWithHeader(), cacheBlocks, pread,
707             isCompaction, true, null, getEffectiveDataBlockEncoding());
708       } while (!curBlock.getBlockType().isData());
709 
710       return curBlock;
711     }
712 
713     public DataBlockEncoding getEffectiveDataBlockEncoding() {
714       return ((HFileReaderV2)reader).getEffectiveEncodingInCache(isCompaction);
715     }
716     /**
717      * Compare the given key against the current key
718      * @param comparator
719      * @param key
720      * @param offset
721      * @param length
722      * @return -1 is the passed key is smaller than the current key, 0 if equal and 1 if greater
723      */
724     public abstract int compareKey(KVComparator comparator, byte[] key, int offset,
725         int length);
726 
727     public abstract int compareKey(KVComparator comparator, Cell kv);
728   }
729 
730   /**
731    * Implementation of {@link HFileScanner} interface.
732    */
733   protected static class ScannerV2 extends AbstractScannerV2 {
734     private HFileReaderV2 reader;
735 
736     public ScannerV2(HFileReaderV2 r, boolean cacheBlocks,
737         final boolean pread, final boolean isCompaction) {
738       super(r, cacheBlocks, pread, isCompaction);
739       this.reader = r;
740     }
741 
742     @Override
743     public Cell getKeyValue() {
744       if (!isSeeked())
745         return null;
746 
747       KeyValue ret = new KeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
748           + blockBuffer.position(), getCellBufSize());
749       if (this.reader.shouldIncludeMemstoreTS()) {
750         ret.setSequenceId(currMemstoreTS);
751       }
752       return ret;
753     }
754 
755     protected int getCellBufSize() {
756       return KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen;
757     }
758 
759     @Override
760     public ByteBuffer getKey() {
761       assertSeeked();
762       return ByteBuffer.wrap(
763           blockBuffer.array(),
764           blockBuffer.arrayOffset() + blockBuffer.position()
765               + KEY_VALUE_LEN_SIZE, currKeyLen).slice();
766     }
767 
768     @Override
769     public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
770       return comparator.compareFlatKey(key, offset, length, blockBuffer.array(),
771           blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen);
772     }
773 
774     @Override
775     public ByteBuffer getValue() {
776       assertSeeked();
777       return ByteBuffer.wrap(
778           blockBuffer.array(),
779           blockBuffer.arrayOffset() + blockBuffer.position()
780               + KEY_VALUE_LEN_SIZE + currKeyLen, currValueLen).slice();
781     }
782 
783     protected void setNonSeekedState() {
784       block = null;
785       blockBuffer = null;
786       currKeyLen = 0;
787       currValueLen = 0;
788       currMemstoreTS = 0;
789       currMemstoreTSLen = 0;
790     }
791 
792     /**
793      * Go to the next key/value in the block section. Loads the next block if
794      * necessary. If successful, {@link #getKey()} and {@link #getValue()} can
795      * be called.
796      *
797      * @return true if successfully navigated to the next key/value
798      */
799     @Override
800     public boolean next() throws IOException {
801       assertSeeked();
802 
803       try {
804         blockBuffer.position(getNextCellStartPosition());
805       } catch (IllegalArgumentException e) {
806         LOG.error("Current pos = " + blockBuffer.position()
807             + "; currKeyLen = " + currKeyLen + "; currValLen = "
808             + currValueLen + "; block limit = " + blockBuffer.limit()
809             + "; HFile name = " + reader.getName()
810             + "; currBlock currBlockOffset = " + block.getOffset());
811         throw e;
812       }
813 
814       if (blockBuffer.remaining() <= 0) {
815         long lastDataBlockOffset =
816             reader.getTrailer().getLastDataBlockOffset();
817 
818         if (block.getOffset() >= lastDataBlockOffset) {
819           setNonSeekedState();
820           return false;
821         }
822 
823         // read the next block
824         HFileBlock nextBlock = readNextDataBlock();
825         if (nextBlock == null) {
826           setNonSeekedState();
827           return false;
828         }
829 
830         updateCurrBlock(nextBlock);
831         return true;
832       }
833 
834       // We are still in the same block.
835       readKeyValueLen();
836       return true;
837     }
838 
839     protected int getNextCellStartPosition() {
840       return blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen
841           + currMemstoreTSLen;
842     }
843 
844     /**
845      * Positions this scanner at the start of the file.
846      *
847      * @return false if empty file; i.e. a call to next would return false and
848      *         the current key and value are undefined.
849      * @throws IOException
850      */
851     @Override
852     public boolean seekTo() throws IOException {
853       if (reader == null) {
854         return false;
855       }
856 
857       if (reader.getTrailer().getEntryCount() == 0) {
858         // No data blocks.
859         return false;
860       }
861 
862       long firstDataBlockOffset =
863           reader.getTrailer().getFirstDataBlockOffset();
864       if (block != null && block.getOffset() == firstDataBlockOffset) {
865         blockBuffer.rewind();
866         readKeyValueLen();
867         return true;
868       }
869 
870       block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
871           isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
872       if (block.getOffset() < 0) {
873         throw new IOException("Invalid block offset: " + block.getOffset());
874       }
875       updateCurrBlock(block);
876       return true;
877     }
878 
879     @Override
880     protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, Cell nextIndexedKey,
881         boolean rewind, Cell key, boolean seekBefore) throws IOException {
882       if (block == null || block.getOffset() != seekToBlock.getOffset()) {
883         updateCurrBlock(seekToBlock);
884       } else if (rewind) {
885         blockBuffer.rewind();
886       }
887 
888       // Update the nextIndexedKey
889       this.nextIndexedKey = nextIndexedKey;
890       return blockSeek(key, seekBefore);
891     }
892 
893     /**
894      * Updates the current block to be the given {@link HFileBlock}. Seeks to
895      * the the first key/value pair.
896      *
897      * @param newBlock the block to make current
898      */
899     protected void updateCurrBlock(HFileBlock newBlock) {
900       block = newBlock;
901 
902       // sanity check
903       if (block.getBlockType() != BlockType.DATA) {
904         throw new IllegalStateException("ScannerV2 works only on data " +
905             "blocks, got " + block.getBlockType() + "; " +
906             "fileName=" + reader.name + ", " +
907             "dataBlockEncoder=" + reader.dataBlockEncoder + ", " +
908             "isCompaction=" + isCompaction);
909       }
910 
911       blockBuffer = block.getBufferWithoutHeader();
912       readKeyValueLen();
913       blockFetches++;
914 
915       // Reset the next indexed key
916       this.nextIndexedKey = null;
917     }
918 
919     protected void readKeyValueLen() {
920       blockBuffer.mark();
921       currKeyLen = blockBuffer.getInt();
922       currValueLen = blockBuffer.getInt();
923       ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
924       readMvccVersion();
925       if (currKeyLen < 0 || currValueLen < 0
926           || currKeyLen > blockBuffer.limit()
927           || currValueLen > blockBuffer.limit()) {
928         throw new IllegalStateException("Invalid currKeyLen " + currKeyLen
929             + " or currValueLen " + currValueLen + ". Block offset: "
930             + block.getOffset() + ", block length: " + blockBuffer.limit()
931             + ", position: " + blockBuffer.position() + " (without header).");
932       }
933       blockBuffer.reset();
934     }
935 
936     protected void readMvccVersion() {
937       if (this.reader.shouldIncludeMemstoreTS()) {
938         if (this.reader.decodeMemstoreTS) {
939           try {
940             currMemstoreTS = Bytes.readVLong(blockBuffer.array(), blockBuffer.arrayOffset()
941                 + blockBuffer.position());
942             currMemstoreTSLen = WritableUtils.getVIntSize(currMemstoreTS);
943           } catch (Exception e) {
944             throw new RuntimeException("Error reading memstore timestamp", e);
945           }
946         } else {
947           currMemstoreTS = 0;
948           currMemstoreTSLen = 1;
949         }
950       }
951     }
952 
953     /**
954      * Within a loaded block, seek looking for the last key that is smaller than
955      * (or equal to?) the key we are interested in.
956      *
957      * A note on the seekBefore: if you have seekBefore = true, AND the first
958      * key in the block = key, then you'll get thrown exceptions. The caller has
959      * to check for that case and load the previous block as appropriate.
960      *
961      * @param key
962      *          the key to find
963      * @param seekBefore
964      *          find the key before the given key in case of exact match.
965      * @return 0 in case of an exact key match, 1 in case of an inexact match,
966      *         -2 in case of an inexact match and furthermore, the input key
967      *         less than the first key of current block(e.g. using a faked index
968      *         key)
969      */
970     protected int blockSeek(Cell key, boolean seekBefore) {
971       int klen, vlen;
972       long memstoreTS = 0;
973       int memstoreTSLen = 0;
974       int lastKeyValueSize = -1;
975       KeyValue.KeyOnlyKeyValue keyOnlykv = new KeyValue.KeyOnlyKeyValue();
976       do {
977         blockBuffer.mark();
978         klen = blockBuffer.getInt();
979         vlen = blockBuffer.getInt();
980         blockBuffer.reset();
981         if (this.reader.shouldIncludeMemstoreTS()) {
982           if (this.reader.decodeMemstoreTS) {
983             try {
984               int memstoreTSOffset = blockBuffer.arrayOffset() + blockBuffer.position()
985                   + KEY_VALUE_LEN_SIZE + klen + vlen;
986               memstoreTS = Bytes.readVLong(blockBuffer.array(), memstoreTSOffset);
987               memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
988             } catch (Exception e) {
989               throw new RuntimeException("Error reading memstore timestamp", e);
990             }
991           } else {
992             memstoreTS = 0;
993             memstoreTSLen = 1;
994           }
995         }
996 
997         int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE;
998         keyOnlykv.setKey(blockBuffer.array(), keyOffset, klen);
999         int comp = reader.getComparator().compareOnlyKeyPortion(key, keyOnlykv);
1000 
1001         if (comp == 0) {
1002           if (seekBefore) {
1003             if (lastKeyValueSize < 0) {
1004               throw new IllegalStateException("blockSeek with seekBefore "
1005                   + "at the first key of the block: key="
1006                   + CellUtil.getCellKeyAsString(key)
1007                   + ", blockOffset=" + block.getOffset() + ", onDiskSize="
1008                   + block.getOnDiskSizeWithHeader());
1009             }
1010             blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1011             readKeyValueLen();
1012             return 1; // non exact match.
1013           }
1014           currKeyLen = klen;
1015           currValueLen = vlen;
1016           if (this.reader.shouldIncludeMemstoreTS()) {
1017             currMemstoreTS = memstoreTS;
1018             currMemstoreTSLen = memstoreTSLen;
1019           }
1020           return 0; // indicate exact match
1021         } else if (comp < 0) {
1022           if (lastKeyValueSize > 0)
1023             blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1024           readKeyValueLen();
1025           if (lastKeyValueSize == -1 && blockBuffer.position() == 0
1026               && this.reader.trailer.getMinorVersion() >= MINOR_VERSION_WITH_FAKED_KEY) {
1027             return HConstants.INDEX_KEY_MAGIC;
1028           }
1029           return 1;
1030         }
1031 
1032         // The size of this key/value tuple, including key/value length fields.
1033         lastKeyValueSize = klen + vlen + memstoreTSLen + KEY_VALUE_LEN_SIZE;
1034         blockBuffer.position(blockBuffer.position() + lastKeyValueSize);
1035       } while (blockBuffer.remaining() > 0);
1036 
1037       // Seek to the last key we successfully read. This will happen if this is
1038       // the last key/value pair in the file, in which case the following call
1039       // to next() has to return false.
1040       blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
1041       readKeyValueLen();
1042       return 1; // didn't exactly find it.
1043     }
1044 
1045     @Override
1046     protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
1047       ByteBuffer buffer = curBlock.getBufferWithoutHeader();
1048       // It is safe to manipulate this buffer because we own the buffer object.
1049       buffer.rewind();
1050       int klen = buffer.getInt();
1051       buffer.getInt();
1052       ByteBuffer keyBuff = buffer.slice();
1053       keyBuff.limit(klen);
1054       keyBuff.rewind();
1055       return keyBuff;
1056     }
1057 
1058     @Override
1059     public String getKeyString() {
1060       return Bytes.toStringBinary(blockBuffer.array(),
1061           blockBuffer.arrayOffset() + blockBuffer.position()
1062               + KEY_VALUE_LEN_SIZE, currKeyLen);
1063     }
1064 
1065     @Override
1066     public String getValueString() {
1067       return Bytes.toString(blockBuffer.array(), blockBuffer.arrayOffset()
1068           + blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen,
1069           currValueLen);
1070     }
1071 
1072     @Override
1073     public int compareKey(KVComparator comparator, Cell key) {
1074       return comparator.compareOnlyKeyPortion(
1075           key,
1076           new KeyValue.KeyOnlyKeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
1077               + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen));
1078     }
1079   }
1080 
1081   /**
1082    * ScannerV2 that operates on encoded data blocks.
1083    */
1084   protected static class EncodedScannerV2 extends AbstractScannerV2 {
1085     private final HFileBlockDecodingContext decodingCtx;
1086     private final DataBlockEncoder.EncodedSeeker seeker;
1087     private final DataBlockEncoder dataBlockEncoder;
1088     protected final HFileContext meta;
1089 
1090     public EncodedScannerV2(HFileReaderV2 reader, boolean cacheBlocks,
1091         boolean pread, boolean isCompaction, HFileContext meta) {
1092       super(reader, cacheBlocks, pread, isCompaction);
1093       DataBlockEncoding encoding = reader.dataBlockEncoder.getDataBlockEncoding();
1094       dataBlockEncoder = encoding.getEncoder();
1095       decodingCtx = dataBlockEncoder.newDataBlockDecodingContext(meta);
1096       seeker = dataBlockEncoder.createSeeker(
1097         reader.getComparator(), decodingCtx);
1098       this.meta = meta;
1099     }
1100 
1101     @Override
1102     public boolean isSeeked(){
1103       return this.block != null;
1104     }
1105 
1106     /**
1107      * Updates the current block to be the given {@link HFileBlock}. Seeks to
1108      * the the first key/value pair.
1109      *
1110      * @param newBlock the block to make current
1111      * @throws CorruptHFileException
1112      */
1113     private void updateCurrentBlock(HFileBlock newBlock) throws CorruptHFileException {
1114       block = newBlock;
1115 
1116       // sanity checks
1117       if (block.getBlockType() != BlockType.ENCODED_DATA) {
1118         throw new IllegalStateException(
1119             "EncodedScanner works only on encoded data blocks");
1120       }
1121       short dataBlockEncoderId = block.getDataBlockEncodingId();
1122       if (!DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) {
1123         String encoderCls = dataBlockEncoder.getClass().getName();
1124         throw new CorruptHFileException("Encoder " + encoderCls
1125           + " doesn't support data block encoding "
1126           + DataBlockEncoding.getNameFromId(dataBlockEncoderId));
1127       }
1128 
1129       seeker.setCurrentBuffer(getEncodedBuffer(newBlock));
1130       blockFetches++;
1131 
1132       // Reset the next indexed key
1133       this.nextIndexedKey = null;
1134     }
1135 
1136     private ByteBuffer getEncodedBuffer(HFileBlock newBlock) {
1137       ByteBuffer origBlock = newBlock.getBufferReadOnly();
1138       ByteBuffer encodedBlock = ByteBuffer.wrap(origBlock.array(),
1139           origBlock.arrayOffset() + newBlock.headerSize() +
1140           DataBlockEncoding.ID_SIZE,
1141           newBlock.getUncompressedSizeWithoutHeader() -
1142           DataBlockEncoding.ID_SIZE).slice();
1143       return encodedBlock;
1144     }
1145 
1146     @Override
1147     public boolean seekTo() throws IOException {
1148       if (reader == null) {
1149         return false;
1150       }
1151 
1152       if (reader.getTrailer().getEntryCount() == 0) {
1153         // No data blocks.
1154         return false;
1155       }
1156 
1157       long firstDataBlockOffset =
1158           reader.getTrailer().getFirstDataBlockOffset();
1159       if (block != null && block.getOffset() == firstDataBlockOffset) {
1160         seeker.rewind();
1161         return true;
1162       }
1163 
1164       block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
1165           isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
1166       if (block.getOffset() < 0) {
1167         throw new IOException("Invalid block offset: " + block.getOffset());
1168       }
1169       updateCurrentBlock(block);
1170       return true;
1171     }
1172 
1173     @Override
1174     public boolean next() throws IOException {
1175       boolean isValid = seeker.next();
1176       if (!isValid) {
1177         block = readNextDataBlock();
1178         isValid = block != null;
1179         if (isValid) {
1180           updateCurrentBlock(block);
1181         }
1182       }
1183       return isValid;
1184     }
1185 
1186     @Override
1187     public ByteBuffer getKey() {
1188       assertValidSeek();
1189       return seeker.getKeyDeepCopy();
1190     }
1191 
1192     @Override
1193     public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
1194       return seeker.compareKey(comparator, key, offset, length);
1195     }
1196 
1197     @Override
1198     public ByteBuffer getValue() {
1199       assertValidSeek();
1200       return seeker.getValueShallowCopy();
1201     }
1202 
1203     @Override
1204     public Cell getKeyValue() {
1205       if (block == null) {
1206         return null;
1207       }
1208       return seeker.getKeyValue();
1209     }
1210 
1211     @Override
1212     public String getKeyString() {
1213       ByteBuffer keyBuffer = getKey();
1214       return Bytes.toStringBinary(keyBuffer.array(),
1215           keyBuffer.arrayOffset(), keyBuffer.limit());
1216     }
1217 
1218     @Override
1219     public String getValueString() {
1220       ByteBuffer valueBuffer = getValue();
1221       return Bytes.toStringBinary(valueBuffer.array(),
1222           valueBuffer.arrayOffset(), valueBuffer.limit());
1223     }
1224 
1225     private void assertValidSeek() {
1226       if (block == null) {
1227         throw new NotSeekedException();
1228       }
1229     }
1230 
1231     @Override
1232     protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
1233       return dataBlockEncoder.getFirstKeyInBlock(getEncodedBuffer(curBlock));
1234     }
1235 
1236     @Override
1237     protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, Cell nextIndexedKey,
1238         boolean rewind, Cell key, boolean seekBefore) throws IOException {
1239       if (block == null || block.getOffset() != seekToBlock.getOffset()) {
1240         updateCurrentBlock(seekToBlock);
1241       } else if (rewind) {
1242         seeker.rewind();
1243       }
1244       this.nextIndexedKey = nextIndexedKey;
1245       return seeker.seekToKeyInBlock(key, seekBefore);
1246     }
1247 
1248     @Override
1249     public int compareKey(KVComparator comparator, Cell key) {
1250       return seeker.compareKey(comparator, key);
1251     }
1252   }
1253 
1254   /**
1255    * Returns a buffer with the Bloom filter metadata. The caller takes
1256    * ownership of the buffer.
1257    */
1258   @Override
1259   public DataInput getGeneralBloomFilterMetadata() throws IOException {
1260     return this.getBloomFilterMetadata(BlockType.GENERAL_BLOOM_META);
1261   }
1262 
1263   @Override
1264   public DataInput getDeleteBloomFilterMetadata() throws IOException {
1265     return this.getBloomFilterMetadata(BlockType.DELETE_FAMILY_BLOOM_META);
1266   }
1267 
1268   private DataInput getBloomFilterMetadata(BlockType blockType)
1269   throws IOException {
1270     if (blockType != BlockType.GENERAL_BLOOM_META &&
1271         blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
1272       throw new RuntimeException("Block Type: " + blockType.toString() +
1273           " is not supported") ;
1274     }
1275 
1276     for (HFileBlock b : loadOnOpenBlocks)
1277       if (b.getBlockType() == blockType)
1278         return b.getByteStream();
1279     return null;
1280   }
1281 
1282   @Override
1283   public boolean isFileInfoLoaded() {
1284     return true; // We load file info in constructor in version 2.
1285   }
1286 
1287   /**
1288    * Validates that the minor version is within acceptable limits.
1289    * Otherwise throws an Runtime exception
1290    */
1291   private void validateMinorVersion(Path path, int minorVersion) {
1292     if (minorVersion < MIN_MINOR_VERSION ||
1293         minorVersion > MAX_MINOR_VERSION) {
1294       String msg = "Minor version for path " + path +
1295                    " is expected to be between " +
1296                    MIN_MINOR_VERSION + " and " + MAX_MINOR_VERSION +
1297                    " but is found to be " + minorVersion;
1298       LOG.error(msg);
1299       throw new RuntimeException(msg);
1300     }
1301   }
1302 
1303   @Override
1304   public int getMajorVersion() {
1305     return 2;
1306   }
1307 
1308   @Override
1309   public HFileContext getFileContext() {
1310     return hfileContext;
1311   }
1312 
1313   /**
1314    * Returns false if block prefetching was requested for this file and has
1315    * not completed, true otherwise
1316    */
1317   @VisibleForTesting
1318   boolean prefetchComplete() {
1319     return PrefetchExecutor.isCompleted(path);
1320   }
1321 }