View Javadoc

1     /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.io.IOException;
22  import java.nio.ByteBuffer;
23  
24  import org.apache.hadoop.classification.InterfaceAudience;
25  import org.apache.hadoop.fs.FSDataInputStream;
26  import org.apache.hadoop.fs.Path;
27  import org.apache.hadoop.hbase.KeyValue;
28  import org.apache.hadoop.hbase.KeyValue.KVComparator;
29  import org.apache.hadoop.hbase.fs.HFileSystem;
30  import org.apache.hadoop.hbase.io.compress.Compression;
31  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
32  import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
33  
34  /**
35   * Common functionality needed by all versions of {@link HFile} readers.
36   */
37  @InterfaceAudience.Private
38  public abstract class AbstractHFileReader implements HFile.Reader {
39    /** Stream to read from. Does checksum verifications in file system */
40    protected FSDataInputStream istream;
41  
42    /** The file system stream of the underlying {@link HFile} that
43     * does not do checksum verification in the file system */
44    protected FSDataInputStream istreamNoFsChecksum;
45  
46    /** Data block index reader keeping the root data index in memory */
47    protected HFileBlockIndex.BlockIndexReader dataBlockIndexReader;
48  
49    /** Meta block index reader -- always single level */
50    protected HFileBlockIndex.BlockIndexReader metaBlockIndexReader;
51  
52    protected final FixedFileTrailer trailer;
53  
54    /** Filled when we read in the trailer. */
55    protected final Compression.Algorithm compressAlgo;
56  
57    /**
58     * What kind of data block encoding should be used while reading, writing,
59     * and handling cache.
60     */
61    protected HFileDataBlockEncoder dataBlockEncoder =
62        NoOpDataBlockEncoder.INSTANCE;
63  
64    /** Last key in the file. Filled in when we read in the file info */
65    protected byte [] lastKey = null;
66  
67    /** Average key length read from file info */
68    protected int avgKeyLen = -1;
69  
70    /** Average value length read from file info */
71    protected int avgValueLen = -1;
72  
73    /** Key comparator */
74    protected KVComparator comparator;
75  
76    /** Size of this file. */
77    protected final long fileSize;
78  
79    /** Block cache configuration. */
80    protected final CacheConfig cacheConf;
81  
82    /** Path of file */
83    protected final Path path;
84  
85    /** File name to be used for block names */
86    protected final String name;
87  
88    protected FileInfo fileInfo;
89  
90    /** The filesystem used for accesing data */
91    protected HFileSystem hfs;
92  
93    protected AbstractHFileReader(Path path, FixedFileTrailer trailer,
94        final long fileSize, final CacheConfig cacheConf, final HFileSystem hfs) {
95      this.trailer = trailer;
96      this.compressAlgo = trailer.getCompressionCodec();
97      this.cacheConf = cacheConf;
98      this.fileSize = fileSize;
99      this.path = path;
100     this.name = path.getName();
101     this.hfs = hfs;
102   }
103 
104   @SuppressWarnings("serial")
105   public static class BlockIndexNotLoadedException
106       extends IllegalStateException {
107     public BlockIndexNotLoadedException() {
108       // Add a message in case anyone relies on it as opposed to class name.
109       super("Block index not loaded");
110     }
111   }
112 
113   protected String toStringFirstKey() {
114     return KeyValue.keyToString(getFirstKey());
115   }
116 
117   protected String toStringLastKey() {
118     return KeyValue.keyToString(getLastKey());
119   }
120 
121   public abstract boolean isFileInfoLoaded();
122 
123   @Override
124   public String toString() {
125     return "reader=" + path.toString() +
126         (!isFileInfoLoaded()? "":
127           ", compression=" + compressAlgo.getName() +
128           ", cacheConf=" + cacheConf +
129           ", firstKey=" + toStringFirstKey() +
130           ", lastKey=" + toStringLastKey()) +
131           ", avgKeyLen=" + avgKeyLen +
132           ", avgValueLen=" + avgValueLen +
133           ", entries=" + trailer.getEntryCount() +
134           ", length=" + fileSize;
135   }
136 
137   @Override
138   public long length() {
139     return fileSize;
140   }
141 
142   /**
143    * Create a Scanner on this file. No seeks or reads are done on creation. Call
144    * {@link HFileScanner#seekTo(byte[])} to position an start the read. There is
145    * nothing to clean up in a Scanner. Letting go of your references to the
146    * scanner is sufficient. NOTE: Do not use this overload of getScanner for
147    * compactions.
148    *
149    * @param cacheBlocks True if we should cache blocks read in by this scanner.
150    * @param pread Use positional read rather than seek+read if true (pread is
151    *          better for random reads, seek+read is better scanning).
152    * @return Scanner on this file.
153    */
154   @Override
155   public HFileScanner getScanner(boolean cacheBlocks, final boolean pread) {
156     return getScanner(cacheBlocks, pread, false);
157   }
158 
159   /**
160    * @return the first key in the file. May be null if file has no entries. Note
161    *         that this is not the first row key, but rather the byte form of the
162    *         first KeyValue.
163    */
164   @Override
165   public byte [] getFirstKey() {
166     if (dataBlockIndexReader == null) {
167       throw new BlockIndexNotLoadedException();
168     }
169     return dataBlockIndexReader.isEmpty() ? null
170         : dataBlockIndexReader.getRootBlockKey(0);
171   }
172 
173   /**
174    * TODO left from {@link HFile} version 1: move this to StoreFile after Ryan's
175    * patch goes in to eliminate {@link KeyValue} here.
176    *
177    * @return the first row key, or null if the file is empty.
178    */
179   @Override
180   public byte[] getFirstRowKey() {
181     byte[] firstKey = getFirstKey();
182     if (firstKey == null)
183       return null;
184     return KeyValue.createKeyValueFromKey(firstKey).getRow();
185   }
186 
187   /**
188    * TODO left from {@link HFile} version 1: move this to StoreFile after
189    * Ryan's patch goes in to eliminate {@link KeyValue} here.
190    *
191    * @return the last row key, or null if the file is empty.
192    */
193   @Override
194   public byte[] getLastRowKey() {
195     byte[] lastKey = getLastKey();
196     if (lastKey == null)
197       return null;
198     return KeyValue.createKeyValueFromKey(lastKey).getRow();
199   }
200 
201   /** @return number of KV entries in this HFile */
202   @Override
203   public long getEntries() {
204     return trailer.getEntryCount();
205   }
206 
207   /** @return comparator */
208   @Override
209   public KVComparator getComparator() {
210     return comparator;
211   }
212 
213   /** @return compression algorithm */
214   @Override
215   public Compression.Algorithm getCompressionAlgorithm() {
216     return compressAlgo;
217   }
218 
219   /**
220    * @return the total heap size of data and meta block indexes in bytes. Does
221    *         not take into account non-root blocks of a multilevel data index.
222    */
223   public long indexSize() {
224     return (dataBlockIndexReader != null ? dataBlockIndexReader.heapSize() : 0)
225         + ((metaBlockIndexReader != null) ? metaBlockIndexReader.heapSize()
226             : 0);
227   }
228 
229   @Override
230   public String getName() {
231     return name;
232   }
233 
234   @Override
235   public HFileBlockIndex.BlockIndexReader getDataBlockIndexReader() {
236     return dataBlockIndexReader;
237   }
238 
239   @Override
240   public FixedFileTrailer getTrailer() {
241     return trailer;
242   }
243 
244   @Override
245   public FileInfo loadFileInfo() throws IOException {
246     return fileInfo;
247   }
248 
249   /**
250    * An exception thrown when an operation requiring a scanner to be seeked
251    * is invoked on a scanner that is not seeked.
252    */
253   @SuppressWarnings("serial")
254   public static class NotSeekedException extends IllegalStateException {
255     public NotSeekedException() {
256       super("Not seeked to a key/value");
257     }
258   }
259 
260   protected static abstract class Scanner implements HFileScanner {
261     protected ByteBuffer blockBuffer;
262 
263     protected boolean cacheBlocks;
264     protected final boolean pread;
265     protected final boolean isCompaction;
266 
267     protected int currKeyLen;
268     protected int currValueLen;
269     protected int currMemstoreTSLen;
270     protected long currMemstoreTS;
271 
272     protected int blockFetches;
273 
274     protected final HFile.Reader reader;
275 
276     public Scanner(final HFile.Reader reader, final boolean cacheBlocks,
277         final boolean pread, final boolean isCompaction) {
278       this.reader = reader;
279       this.cacheBlocks = cacheBlocks;
280       this.pread = pread;
281       this.isCompaction = isCompaction;
282     }
283 
284     @Override
285     public boolean isSeeked(){
286       return blockBuffer != null;
287     }
288 
289     @Override
290     public String toString() {
291       return "HFileScanner for reader " + String.valueOf(getReader());
292     }
293 
294     protected void assertSeeked() {
295       if (!isSeeked())
296         throw new NotSeekedException();
297     }
298 
299     @Override
300     public int seekTo(byte[] key) throws IOException {
301       return seekTo(key, 0, key.length);
302     }
303     
304     @Override
305     public boolean seekBefore(byte[] key) throws IOException {
306       return seekBefore(key, 0, key.length);
307     }
308     
309     @Override
310     public int reseekTo(byte[] key) throws IOException {
311       return reseekTo(key, 0, key.length);
312     }
313 
314     @Override
315     public HFile.Reader getReader() {
316       return reader;
317     }
318   }
319 
320   /** For testing */
321   abstract HFileBlock.FSReader getUncachedBlockReader();
322 
323   public Path getPath() {
324     return path;
325   }
326 
327   @Override
328   public DataBlockEncoding getDataBlockEncoding() {
329     return dataBlockEncoder.getDataBlockEncoding();
330   }
331 }