View Javadoc

1     /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.io.IOException;
22  import java.nio.ByteBuffer;
23  
24  import org.apache.hadoop.classification.InterfaceAudience;
25  import org.apache.hadoop.fs.FSDataInputStream;
26  import org.apache.hadoop.fs.Path;
27  import org.apache.hadoop.hbase.KeyValue;
28  import org.apache.hadoop.hbase.fs.HFileSystem;
29  import org.apache.hadoop.hbase.io.compress.Compression;
30  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
31  import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
32  import org.apache.hadoop.io.RawComparator;
33  
34  /**
35   * Common functionality needed by all versions of {@link HFile} readers.
36   */
37  @InterfaceAudience.Private
38  public abstract class AbstractHFileReader implements HFile.Reader {
39  
40    /** Filesystem-level block reader for this HFile format version. */
41    protected HFileBlock.FSReader fsBlockReader;
42  
43    /** Stream to read from. Does checksum verifications in file system */
44    protected FSDataInputStream istream;
45  
46    /** The file system stream of the underlying {@link HFile} that
47     * does not do checksum verification in the file system */
48    protected FSDataInputStream istreamNoFsChecksum;
49  
50    /**
51     * True if we should close the input stream when done. We don't close it if we
52     * didn't open it.
53     */
54    protected final boolean closeIStream;
55  
56    /** Data block index reader keeping the root data index in memory */
57    protected HFileBlockIndex.BlockIndexReader dataBlockIndexReader;
58  
59    /** Meta block index reader -- always single level */
60    protected HFileBlockIndex.BlockIndexReader metaBlockIndexReader;
61  
62    protected final FixedFileTrailer trailer;
63  
64    /** Filled when we read in the trailer. */
65    protected final Compression.Algorithm compressAlgo;
66  
67    /**
68     * What kind of data block encoding should be used while reading, writing,
69     * and handling cache.
70     */
71    protected HFileDataBlockEncoder dataBlockEncoder =
72        NoOpDataBlockEncoder.INSTANCE;
73  
74    /** Last key in the file. Filled in when we read in the file info */
75    protected byte [] lastKey = null;
76  
77    /** Average key length read from file info */
78    protected int avgKeyLen = -1;
79  
80    /** Average value length read from file info */
81    protected int avgValueLen = -1;
82  
83    /** Key comparator */
84    protected RawComparator<byte []> comparator;
85  
86    /** Size of this file. */
87    protected final long fileSize;
88  
89    /** Block cache configuration. */
90    protected final CacheConfig cacheConf;
91  
92    /** Path of file */
93    protected final Path path;
94  
95    /** File name to be used for block names */
96    protected final String name;
97  
98    protected FileInfo fileInfo;
99  
100   /** The filesystem used for accesing data */
101   protected HFileSystem hfs;
102 
103   protected AbstractHFileReader(Path path, FixedFileTrailer trailer,
104       final FSDataInputStream fsdis, final long fileSize,
105       final boolean closeIStream,
106       final CacheConfig cacheConf) {
107     this(path, trailer, fsdis, fsdis, fileSize, closeIStream, cacheConf, null);
108   }
109 
110   protected AbstractHFileReader(Path path, FixedFileTrailer trailer,
111       final FSDataInputStream fsdis, final FSDataInputStream fsdisNoFsChecksum,
112       final long fileSize,
113       final boolean closeIStream,
114       final CacheConfig cacheConf, final HFileSystem hfs) {
115     this.trailer = trailer;
116     this.compressAlgo = trailer.getCompressionCodec();
117     this.cacheConf = cacheConf;
118     this.fileSize = fileSize;
119     this.istream = fsdis;
120     this.closeIStream = closeIStream;
121     this.path = path;
122     this.name = path.getName();
123     this.hfs = hfs;
124     this.istreamNoFsChecksum = fsdisNoFsChecksum;
125   }
126 
127   @SuppressWarnings("serial")
128   public static class BlockIndexNotLoadedException
129       extends IllegalStateException {
130     public BlockIndexNotLoadedException() {
131       // Add a message in case anyone relies on it as opposed to class name.
132       super("Block index not loaded");
133     }
134   }
135 
136   protected String toStringFirstKey() {
137     return KeyValue.keyToString(getFirstKey());
138   }
139 
140   protected String toStringLastKey() {
141     return KeyValue.keyToString(getLastKey());
142   }
143 
144   public abstract boolean isFileInfoLoaded();
145 
146   @Override
147   public String toString() {
148     return "reader=" + path.toString() +
149         (!isFileInfoLoaded()? "":
150           ", compression=" + compressAlgo.getName() +
151           ", cacheConf=" + cacheConf +
152           ", firstKey=" + toStringFirstKey() +
153           ", lastKey=" + toStringLastKey()) +
154           ", avgKeyLen=" + avgKeyLen +
155           ", avgValueLen=" + avgValueLen +
156           ", entries=" + trailer.getEntryCount() +
157           ", length=" + fileSize;
158   }
159 
160   @Override
161   public long length() {
162     return fileSize;
163   }
164 
165   /**
166    * Create a Scanner on this file. No seeks or reads are done on creation. Call
167    * {@link HFileScanner#seekTo(byte[])} to position an start the read. There is
168    * nothing to clean up in a Scanner. Letting go of your references to the
169    * scanner is sufficient. NOTE: Do not use this overload of getScanner for
170    * compactions.
171    *
172    * @param cacheBlocks True if we should cache blocks read in by this scanner.
173    * @param pread Use positional read rather than seek+read if true (pread is
174    *          better for random reads, seek+read is better scanning).
175    * @return Scanner on this file.
176    */
177   @Override
178   public HFileScanner getScanner(boolean cacheBlocks, final boolean pread) {
179     return getScanner(cacheBlocks, pread, false);
180   }
181 
182   /**
183    * @return the first key in the file. May be null if file has no entries. Note
184    *         that this is not the first row key, but rather the byte form of the
185    *         first KeyValue.
186    */
187   @Override
188   public byte [] getFirstKey() {
189     if (dataBlockIndexReader == null) {
190       throw new BlockIndexNotLoadedException();
191     }
192     return dataBlockIndexReader.isEmpty() ? null
193         : dataBlockIndexReader.getRootBlockKey(0);
194   }
195 
196   /**
197    * TODO left from {@link HFile} version 1: move this to StoreFile after Ryan's
198    * patch goes in to eliminate {@link KeyValue} here.
199    *
200    * @return the first row key, or null if the file is empty.
201    */
202   @Override
203   public byte[] getFirstRowKey() {
204     byte[] firstKey = getFirstKey();
205     if (firstKey == null)
206       return null;
207     return KeyValue.createKeyValueFromKey(firstKey).getRow();
208   }
209 
210   /**
211    * TODO left from {@link HFile} version 1: move this to StoreFile after
212    * Ryan's patch goes in to eliminate {@link KeyValue} here.
213    *
214    * @return the last row key, or null if the file is empty.
215    */
216   @Override
217   public byte[] getLastRowKey() {
218     byte[] lastKey = getLastKey();
219     if (lastKey == null)
220       return null;
221     return KeyValue.createKeyValueFromKey(lastKey).getRow();
222   }
223 
224   /** @return number of KV entries in this HFile */
225   @Override
226   public long getEntries() {
227     return trailer.getEntryCount();
228   }
229 
230   /** @return comparator */
231   @Override
232   public RawComparator<byte []> getComparator() {
233     return comparator;
234   }
235 
236   /** @return compression algorithm */
237   @Override
238   public Compression.Algorithm getCompressionAlgorithm() {
239     return compressAlgo;
240   }
241 
242   /**
243    * @return the total heap size of data and meta block indexes in bytes. Does
244    *         not take into account non-root blocks of a multilevel data index.
245    */
246   public long indexSize() {
247     return (dataBlockIndexReader != null ? dataBlockIndexReader.heapSize() : 0)
248         + ((metaBlockIndexReader != null) ? metaBlockIndexReader.heapSize()
249             : 0);
250   }
251 
252   @Override
253   public String getName() {
254     return name;
255   }
256 
257   @Override
258   public HFileBlockIndex.BlockIndexReader getDataBlockIndexReader() {
259     return dataBlockIndexReader;
260   }
261 
262   @Override
263   public FixedFileTrailer getTrailer() {
264     return trailer;
265   }
266 
267   @Override
268   public FileInfo loadFileInfo() throws IOException {
269     return fileInfo;
270   }
271 
272   /**
273    * An exception thrown when an operation requiring a scanner to be seeked
274    * is invoked on a scanner that is not seeked.
275    */
276   @SuppressWarnings("serial")
277   public static class NotSeekedException extends IllegalStateException {
278     public NotSeekedException() {
279       super("Not seeked to a key/value");
280     }
281   }
282 
283   protected static abstract class Scanner implements HFileScanner {
284     protected ByteBuffer blockBuffer;
285 
286     protected boolean cacheBlocks;
287     protected final boolean pread;
288     protected final boolean isCompaction;
289 
290     protected int currKeyLen;
291     protected int currValueLen;
292     protected int currMemstoreTSLen;
293     protected long currMemstoreTS;
294 
295     protected int blockFetches;
296 
297     protected final HFile.Reader reader;
298 
299     public Scanner(final HFile.Reader reader, final boolean cacheBlocks,
300         final boolean pread, final boolean isCompaction) {
301       this.reader = reader;
302       this.cacheBlocks = cacheBlocks;
303       this.pread = pread;
304       this.isCompaction = isCompaction;
305     }
306 
307     @Override
308     public boolean isSeeked(){
309       return blockBuffer != null;
310     }
311 
312     @Override
313     public String toString() {
314       return "HFileScanner for reader " + String.valueOf(getReader());
315     }
316 
317     protected void assertSeeked() {
318       if (!isSeeked())
319         throw new NotSeekedException();
320     }
321 
322     @Override
323     public int seekTo(byte[] key) throws IOException {
324       return seekTo(key, 0, key.length);
325     }
326     
327     @Override
328     public boolean seekBefore(byte[] key) throws IOException {
329       return seekBefore(key, 0, key.length);
330     }
331     
332     @Override
333     public int reseekTo(byte[] key) throws IOException {
334       return reseekTo(key, 0, key.length);
335     }
336 
337     @Override
338     public HFile.Reader getReader() {
339       return reader;
340     }
341   }
342 
343   /** For testing */
344   HFileBlock.FSReader getUncachedBlockReader() {
345     return fsBlockReader;
346   }
347 
348   public Path getPath() {
349     return path;
350   }
351 
352   @Override
353   public DataBlockEncoding getEncodingOnDisk() {
354     return dataBlockEncoder.getEncodingOnDisk();
355   }
356 }