View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import java.io.IOException;
21  import java.security.Key;
22  import java.security.KeyException;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.hbase.classification.InterfaceAudience;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.HConstants;
30  import org.apache.hadoop.hbase.KeyValue;
31  import org.apache.hadoop.hbase.fs.HFileSystem;
32  import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
33  import org.apache.hadoop.hbase.io.crypto.Cipher;
34  import org.apache.hadoop.hbase.io.crypto.Encryption;
35  import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
36  import org.apache.hadoop.hbase.security.EncryptionUtil;
37  import org.apache.hadoop.hbase.security.User;
38  import org.apache.hadoop.hbase.util.ByteBufferUtils;
39  import org.apache.hadoop.hbase.util.Bytes;
40  import org.apache.hadoop.io.WritableUtils;
41  
42  /**
43   * {@link HFile} reader for version 3.
44   */
45  @InterfaceAudience.Private
46  public class HFileReaderV3 extends HFileReaderV2 {
47  
48    private static final Log LOG = LogFactory.getLog(HFileReaderV3.class);
49  
50    public static final int MAX_MINOR_VERSION = 0;
51  
52    /**
53     * Opens a HFile. You must load the index before you can use it by calling
54     * {@link #loadFileInfo()}.
55     * @param path
56     *          Path to HFile.
57     * @param trailer
58     *          File trailer.
59     * @param fsdis
60     *          input stream.
61     * @param size
62     *          Length of the stream.
63     * @param cacheConf
64     *          Cache configuration.
65     * @param hfs
66     *          The file system.
67     * @param conf
68     *          Configuration
69     */
70    public HFileReaderV3(final Path path, FixedFileTrailer trailer, final FSDataInputStreamWrapper fsdis,
71        final long size, final CacheConfig cacheConf, final HFileSystem hfs,
72        final Configuration conf) throws IOException {
73      super(path, trailer, fsdis, size, cacheConf, hfs, conf);
74      byte[] tmp = fileInfo.get(FileInfo.MAX_TAGS_LEN);
75      // max tag length is not present in the HFile means tags were not at all written to file.
76      if (tmp != null) {
77        hfileContext.setIncludesTags(true);
78        tmp = fileInfo.get(FileInfo.TAGS_COMPRESSED);
79        if (tmp != null && Bytes.toBoolean(tmp)) {
80          hfileContext.setCompressTags(true);
81        }
82      }
83    }
84  
85    @Override
86    protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
87        HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
88      trailer.expectMajorVersion(3);
89      HFileContextBuilder builder = new HFileContextBuilder()
90        .withIncludesMvcc(this.includesMemstoreTS)
91        .withHBaseCheckSum(true)
92        .withCompression(this.compressAlgo);
93  
94      // Check for any key material available
95      byte[] keyBytes = trailer.getEncryptionKey();
96      if (keyBytes != null) {
97        Encryption.Context cryptoContext = Encryption.newContext(conf);
98        Key key;
99        String masterKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
100         User.getCurrent().getShortName());
101       try {
102         // First try the master key
103         key = EncryptionUtil.unwrapKey(conf, masterKeyName, keyBytes);
104       } catch (KeyException e) {
105         // If the current master key fails to unwrap, try the alternate, if
106         // one is configured
107         if (LOG.isDebugEnabled()) {
108           LOG.debug("Unable to unwrap key with current master key '" + masterKeyName + "'");
109         }
110         String alternateKeyName =
111           conf.get(HConstants.CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY);
112         if (alternateKeyName != null) {
113           try {
114             key = EncryptionUtil.unwrapKey(conf, alternateKeyName, keyBytes);
115           } catch (KeyException ex) {
116             throw new IOException(ex);
117           }
118         } else {
119           throw new IOException(e);
120         }
121       }
122       // Use the algorithm the key wants
123       Cipher cipher = Encryption.getCipher(conf, key.getAlgorithm());
124       if (cipher == null) {
125         throw new IOException("Cipher '" + key.getAlgorithm() + "' is not available");
126       }
127       cryptoContext.setCipher(cipher);
128       cryptoContext.setKey(key);
129       builder.withEncryptionContext(cryptoContext);
130     }
131 
132     HFileContext context = builder.build();
133 
134     if (LOG.isTraceEnabled()) {
135       LOG.trace("Reader" + (path != null ? " for " + path : "" ) +
136         " initialized with cacheConf: " + cacheConf +
137         " comparator: " + comparator.getClass().getSimpleName() +
138         " fileContext: " + context);
139     }
140 
141     return context;
142   }
143 
144   /**
145    * Create a Scanner on this file. No seeks or reads are done on creation. Call
146    * {@link HFileScanner#seekTo(byte[])} to position an start the read. There is
147    * nothing to clean up in a Scanner. Letting go of your references to the
148    * scanner is sufficient.
149    * @param cacheBlocks
150    *          True if we should cache blocks read in by this scanner.
151    * @param pread
152    *          Use positional read rather than seek+read if true (pread is better
153    *          for random reads, seek+read is better scanning).
154    * @param isCompaction
155    *          is scanner being used for a compaction?
156    * @return Scanner on this file.
157    */
158   @Override
159   public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
160       final boolean isCompaction) {
161     if (dataBlockEncoder.useEncodedScanner()) {
162       return new EncodedScannerV3(this, cacheBlocks, pread, isCompaction, this.hfileContext);
163     }
164     return new ScannerV3(this, cacheBlocks, pread, isCompaction);
165   }
166 
167   /**
168    * Implementation of {@link HFileScanner} interface.
169    */
170   protected static class ScannerV3 extends ScannerV2 {
171 
172     private HFileReaderV3 reader;
173     private int currTagsLen;
174 
175     public ScannerV3(HFileReaderV3 r, boolean cacheBlocks, final boolean pread,
176         final boolean isCompaction) {
177       super(r, cacheBlocks, pread, isCompaction);
178       this.reader = r;
179     }
180 
181     @Override
182     protected int getCellBufSize() {
183       int kvBufSize = super.getCellBufSize();
184       if (reader.hfileContext.isIncludesTags()) {
185         kvBufSize += Bytes.SIZEOF_SHORT + currTagsLen;
186       }
187       return kvBufSize;
188     }
189 
190     @Override
191     public KeyValue getKeyValue() {
192       if (!isSeeked())
193         return null;
194       if (currTagsLen == 0) {
195         // There are no Tags in this KeyValue. Just return NoTagsKeyValue
196         return formNoTagsKeyValue();
197       }
198       KeyValue ret = new KeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
199           + blockBuffer.position(), getCellBufSize());
200       if (this.reader.shouldIncludeMemstoreTS()) {
201         ret.setMvccVersion(currMemstoreTS);
202       }
203       return ret;
204     }
205 
206     protected void setNonSeekedState() {
207       super.setNonSeekedState();
208       currTagsLen = 0;
209     }
210 
211     @Override
212     protected int getNextCellStartPosition() {
213       int nextKvPos = super.getNextCellStartPosition();
214       if (reader.hfileContext.isIncludesTags()) {
215         nextKvPos += Bytes.SIZEOF_SHORT + currTagsLen;
216       }
217       return nextKvPos;
218     }
219 
220     protected void readKeyValueLen() {
221       blockBuffer.mark();
222       currKeyLen = blockBuffer.getInt();
223       currValueLen = blockBuffer.getInt();
224       if (currKeyLen < 0 || currValueLen < 0 || currKeyLen > blockBuffer.limit()
225           || currValueLen > blockBuffer.limit()) {
226         throw new IllegalStateException("Invalid currKeyLen " + currKeyLen + " or currValueLen "
227             + currValueLen + ". Block offset: "
228             + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
229             + blockBuffer.position() + " (without header).");
230       }
231       ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
232       if (reader.hfileContext.isIncludesTags()) {
233         // Read short as unsigned, high byte first
234         currTagsLen = ((blockBuffer.get() & 0xff) << 8) ^ (blockBuffer.get() & 0xff);
235         if (currTagsLen < 0 || currTagsLen > blockBuffer.limit()) {
236           throw new IllegalStateException("Invalid currTagsLen " + currTagsLen + ". Block offset: "
237               + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
238               + blockBuffer.position() + " (without header).");
239         }
240         ByteBufferUtils.skip(blockBuffer, currTagsLen);
241       }
242       readMvccVersion();
243       blockBuffer.reset();
244     }
245 
246     /**
247      * Within a loaded block, seek looking for the last key that is smaller than
248      * (or equal to?) the key we are interested in.
249      * A note on the seekBefore: if you have seekBefore = true, AND the first
250      * key in the block = key, then you'll get thrown exceptions. The caller has
251      * to check for that case and load the previous block as appropriate.
252      * @param key
253      *          the key to find
254      * @param seekBefore
255      *          find the key before the given key in case of exact match.
256      * @param offset
257      *          Offset to find the key in the given bytebuffer
258      * @param length
259      *          Length of the key to be found
260      * @return 0 in case of an exact key match, 1 in case of an inexact match,
261      *         -2 in case of an inexact match and furthermore, the input key
262      *         less than the first key of current block(e.g. using a faked index
263      *         key)
264      */
265     protected int blockSeek(byte[] key, int offset, int length, boolean seekBefore) {
266       int klen, vlen, tlen = 0;
267       long memstoreTS = 0;
268       int memstoreTSLen = 0;
269       int lastKeyValueSize = -1;
270       do {
271         blockBuffer.mark();
272         klen = blockBuffer.getInt();
273         vlen = blockBuffer.getInt();
274         if (klen < 0 || vlen < 0 || klen > blockBuffer.limit()
275             || vlen > blockBuffer.limit()) {
276           throw new IllegalStateException("Invalid klen " + klen + " or vlen "
277               + vlen + ". Block offset: "
278               + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
279               + blockBuffer.position() + " (without header).");
280         }
281         ByteBufferUtils.skip(blockBuffer, klen + vlen);
282         if (reader.hfileContext.isIncludesTags()) {
283           // Read short as unsigned, high byte first
284           tlen = ((blockBuffer.get() & 0xff) << 8) ^ (blockBuffer.get() & 0xff);
285           if (tlen < 0 || tlen > blockBuffer.limit()) {
286             throw new IllegalStateException("Invalid tlen " + tlen + ". Block offset: "
287                 + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
288                 + blockBuffer.position() + " (without header).");
289           }
290           ByteBufferUtils.skip(blockBuffer, tlen);
291         }
292         if (this.reader.shouldIncludeMemstoreTS()) {
293           if (this.reader.decodeMemstoreTS) {
294             memstoreTS = Bytes.readAsVLong(blockBuffer.array(), blockBuffer.arrayOffset()
295                 + blockBuffer.position());
296             memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
297           } else {
298             memstoreTS = 0;
299             memstoreTSLen = 1;
300           }
301         }
302         blockBuffer.reset();
303         int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position() + (Bytes.SIZEOF_INT * 2);
304         int comp = reader.getComparator().compare(key, offset, length, blockBuffer.array(),
305             keyOffset, klen);
306 
307         if (comp == 0) {
308           if (seekBefore) {
309             if (lastKeyValueSize < 0) {
310               throw new IllegalStateException("blockSeek with seekBefore "
311                   + "at the first key of the block: key=" + Bytes.toStringBinary(key)
312                   + ", blockOffset=" + block.getOffset() + ", onDiskSize="
313                   + block.getOnDiskSizeWithHeader());
314             }
315             blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
316             readKeyValueLen();
317             return 1; // non exact match.
318           }
319           currKeyLen = klen;
320           currValueLen = vlen;
321           currTagsLen = tlen;
322           if (this.reader.shouldIncludeMemstoreTS()) {
323             currMemstoreTS = memstoreTS;
324             currMemstoreTSLen = memstoreTSLen;
325           }
326           return 0; // indicate exact match
327         } else if (comp < 0) {
328           if (lastKeyValueSize > 0)
329             blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
330           readKeyValueLen();
331           if (lastKeyValueSize == -1 && blockBuffer.position() == 0) {
332             return HConstants.INDEX_KEY_MAGIC;
333           }
334           return 1;
335         }
336 
337         // The size of this key/value tuple, including key/value length fields.
338         lastKeyValueSize = klen + vlen + memstoreTSLen + KEY_VALUE_LEN_SIZE;
339         // include tag length also if tags included with KV
340         if (reader.hfileContext.isIncludesTags()) {
341           lastKeyValueSize += tlen + Bytes.SIZEOF_SHORT;
342         }
343         blockBuffer.position(blockBuffer.position() + lastKeyValueSize);
344       } while (blockBuffer.remaining() > 0);
345 
346       // Seek to the last key we successfully read. This will happen if this is
347       // the last key/value pair in the file, in which case the following call
348       // to next() has to return false.
349       blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
350       readKeyValueLen();
351       return 1; // didn't exactly find it.
352     }
353 
354   }
355 
356   /**
357    * ScannerV3 that operates on encoded data blocks.
358    */
359   protected static class EncodedScannerV3 extends EncodedScannerV2 {
360     public EncodedScannerV3(HFileReaderV3 reader, boolean cacheBlocks, boolean pread,
361         boolean isCompaction, HFileContext context) {
362       super(reader, cacheBlocks, pread, isCompaction, context);
363     }
364   }
365 
366   @Override
367   public int getMajorVersion() {
368     return 3;
369   }
370 }