View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations
15   * under the License.
16   */
17  package org.apache.hadoop.hbase.io.encoding;
18  
19  import java.io.DataInputStream;
20  import java.io.DataOutputStream;
21  import java.io.IOException;
22  import java.nio.ByteBuffer;
23  
24  import org.apache.hadoop.hbase.classification.InterfaceAudience;
25  import org.apache.hadoop.hbase.HConstants;
26  import org.apache.hadoop.hbase.KeyValue;
27  import org.apache.hadoop.hbase.KeyValue.KVComparator;
28  import org.apache.hadoop.hbase.KeyValue.SamePrefixComparator;
29  import org.apache.hadoop.hbase.NoTagsKeyValue;
30  import org.apache.hadoop.hbase.io.TagCompressionContext;
31  import org.apache.hadoop.hbase.io.hfile.BlockType;
32  import org.apache.hadoop.hbase.io.hfile.HFileContext;
33  import org.apache.hadoop.hbase.io.util.LRUDictionary;
34  import org.apache.hadoop.hbase.util.ByteBufferUtils;
35  import org.apache.hadoop.hbase.util.Bytes;
36  import org.apache.hadoop.io.WritableUtils;
37  
38  /**
39   * Base class for all data block encoders that use a buffer.
40   */
41  @InterfaceAudience.Private
42  abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
43  
44    private static int INITIAL_KEY_BUFFER_SIZE = 512;
45  
46    @Override
47    public ByteBuffer decodeKeyValues(DataInputStream source,
48        HFileBlockDecodingContext blkDecodingCtx) throws IOException {
49      if (blkDecodingCtx.getClass() != HFileBlockDefaultDecodingContext.class) {
50        throw new IOException(this.getClass().getName() + " only accepts "
51            + HFileBlockDefaultDecodingContext.class.getName() + " as the decoding context.");
52      }
53  
54      HFileBlockDefaultDecodingContext decodingCtx =
55          (HFileBlockDefaultDecodingContext) blkDecodingCtx;
56      if (decodingCtx.getHFileContext().isIncludesTags()
57          && decodingCtx.getHFileContext().isCompressTags()) {
58        if (decodingCtx.getTagCompressionContext() != null) {
59          // It will be overhead to create the TagCompressionContext again and again for every block
60          // decoding.
61          decodingCtx.getTagCompressionContext().clear();
62        } else {
63          try {
64            TagCompressionContext tagCompressionContext = new TagCompressionContext(
65                LRUDictionary.class, Byte.MAX_VALUE);
66            decodingCtx.setTagCompressionContext(tagCompressionContext);
67          } catch (Exception e) {
68            throw new IOException("Failed to initialize TagCompressionContext", e);
69          }
70        }
71      }
72      return internalDecodeKeyValues(source, 0, 0, decodingCtx);
73    }
74  
75    protected static class SeekerState {
76      protected int valueOffset = -1;
77      protected int keyLength;
78      protected int valueLength;
79      protected int lastCommonPrefix;
80      protected int tagsLength = 0;
81      protected int tagsOffset = -1;
82      protected int tagsCompressedLength = 0;
83      protected boolean uncompressTags = true;
84  
85      /** We need to store a copy of the key. */
86      protected byte[] keyBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
87      protected byte[] tagsBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
88  
89      protected long memstoreTS;
90      protected int nextKvOffset;
91  
92      protected boolean isValid() {
93        return valueOffset != -1;
94      }
95  
96      protected void invalidate() {
97        valueOffset = -1;
98        tagsCompressedLength = 0;
99        uncompressTags = true;
100     }
101 
102     protected void ensureSpaceForKey() {
103       if (keyLength > keyBuffer.length) {
104         // rare case, but we need to handle arbitrary length of key
105         int newKeyBufferLength = Math.max(keyBuffer.length, 1) * 2;
106         while (keyLength > newKeyBufferLength) {
107           newKeyBufferLength *= 2;
108         }
109         byte[] newKeyBuffer = new byte[newKeyBufferLength];
110         System.arraycopy(keyBuffer, 0, newKeyBuffer, 0, keyBuffer.length);
111         keyBuffer = newKeyBuffer;
112       }
113     }
114 
115     protected void ensureSpaceForTags() {
116       if (tagsLength > tagsBuffer.length) {
117         // rare case, but we need to handle arbitrary length of tags
118         int newTagsBufferLength = Math.max(tagsBuffer.length, 1) * 2;
119         while (tagsLength > newTagsBufferLength) {
120           newTagsBufferLength *= 2;
121         }
122         byte[] newTagsBuffer = new byte[newTagsBufferLength];
123         System.arraycopy(tagsBuffer, 0, newTagsBuffer, 0, tagsBuffer.length);
124         tagsBuffer = newTagsBuffer;
125       }
126     }
127 
128     /**
129      * Copy the state from the next one into this instance (the previous state
130      * placeholder). Used to save the previous state when we are advancing the
131      * seeker to the next key/value.
132      */
133     protected void copyFromNext(SeekerState nextState) {
134       if (keyBuffer.length != nextState.keyBuffer.length) {
135         keyBuffer = nextState.keyBuffer.clone();
136       } else if (!isValid()) {
137         // Note: we can only call isValid before we override our state, so this
138         // comes before all the assignments at the end of this method.
139         System.arraycopy(nextState.keyBuffer, 0, keyBuffer, 0,
140              nextState.keyLength);
141       } else {
142         // don't copy the common prefix between this key and the previous one
143         System.arraycopy(nextState.keyBuffer, nextState.lastCommonPrefix,
144             keyBuffer, nextState.lastCommonPrefix, nextState.keyLength
145                 - nextState.lastCommonPrefix);
146       }
147 
148       valueOffset = nextState.valueOffset;
149       keyLength = nextState.keyLength;
150       valueLength = nextState.valueLength;
151       lastCommonPrefix = nextState.lastCommonPrefix;
152       nextKvOffset = nextState.nextKvOffset;
153       memstoreTS = nextState.memstoreTS;
154     }
155 
156   }
157 
158   protected abstract static class
159       BufferedEncodedSeeker<STATE extends SeekerState>
160       implements EncodedSeeker {
161     protected HFileBlockDecodingContext decodingCtx;
162     protected final KVComparator comparator;
163     protected final SamePrefixComparator<byte[]> samePrefixComparator;
164     protected ByteBuffer currentBuffer;
165     protected STATE current = createSeekerState(); // always valid
166     protected STATE previous = createSeekerState(); // may not be valid
167     protected TagCompressionContext tagCompressionContext = null;
168 
169     public BufferedEncodedSeeker(KVComparator comparator,
170         HFileBlockDecodingContext decodingCtx) {
171       this.comparator = comparator;
172       this.samePrefixComparator = comparator;
173       this.decodingCtx = decodingCtx;
174       if (decodingCtx.getHFileContext().isCompressTags()) {
175         try {
176           tagCompressionContext = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE);
177         } catch (Exception e) {
178           throw new RuntimeException("Failed to initialize TagCompressionContext", e);
179         }
180       }
181     }
182     
183     protected boolean includesMvcc() {
184       return this.decodingCtx.getHFileContext().isIncludesMvcc();
185     }
186 
187     protected boolean includesTags() {
188       return this.decodingCtx.getHFileContext().isIncludesTags();
189     }
190 
191     @Override
192     public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
193       return comparator.compareFlatKey(key, offset, length,
194           current.keyBuffer, 0, current.keyLength);
195     }
196 
197     @Override
198     public void setCurrentBuffer(ByteBuffer buffer) {
199       if (this.tagCompressionContext != null) {
200         this.tagCompressionContext.clear();
201       }
202       currentBuffer = buffer;
203       decodeFirst();
204       previous.invalidate();
205     }
206 
207     @Override
208     public ByteBuffer getKeyDeepCopy() {
209       ByteBuffer keyBuffer = ByteBuffer.allocate(current.keyLength);
210       keyBuffer.put(current.keyBuffer, 0, current.keyLength);
211       return keyBuffer;
212     }
213 
214     @Override
215     public ByteBuffer getValueShallowCopy() {
216       return ByteBuffer.wrap(currentBuffer.array(),
217           currentBuffer.arrayOffset() + current.valueOffset,
218           current.valueLength);
219     }
220 
221     @Override
222     public ByteBuffer getKeyValueBuffer() {
223       ByteBuffer kvBuffer = createKVBuffer();
224       kvBuffer.putInt(current.keyLength);
225       kvBuffer.putInt(current.valueLength);
226       kvBuffer.put(current.keyBuffer, 0, current.keyLength);
227       kvBuffer.put(currentBuffer.array(),
228           currentBuffer.arrayOffset() + current.valueOffset,
229           current.valueLength);
230       if (current.tagsLength > 0) {
231         // Put short as unsigned
232         kvBuffer.put((byte)(current.tagsLength >> 8 & 0xff));
233         kvBuffer.put((byte)(current.tagsLength & 0xff));
234         if (current.tagsOffset != -1) {
235           // the offset of the tags bytes in the underlying buffer is marked. So the temp
236           // buffer,tagsBuffer was not been used.
237           kvBuffer.put(currentBuffer.array(), currentBuffer.arrayOffset() + current.tagsOffset,
238               current.tagsLength);
239         } else {
240           // When tagsOffset is marked as -1, tag compression was present and so the tags were
241           // uncompressed into temp buffer, tagsBuffer. Let us copy it from there
242           kvBuffer.put(current.tagsBuffer, 0, current.tagsLength);
243         }
244       }
245       return kvBuffer;
246     }
247 
248     protected ByteBuffer createKVBuffer() {
249       int kvBufSize = (int) KeyValue.getKeyValueDataStructureSize(current.keyLength,
250           current.valueLength, current.tagsLength);
251       ByteBuffer kvBuffer = ByteBuffer.allocate(kvBufSize);
252       return kvBuffer;
253     }
254 
255     @Override
256     public KeyValue getKeyValue() {
257       ByteBuffer kvBuf = getKeyValueBuffer();
258       KeyValue kv;
259       if (current.tagsLength == 0) {
260         kv = new NoTagsKeyValue(kvBuf.array(), kvBuf.arrayOffset(), kvBuf.array().length
261             - kvBuf.arrayOffset());
262       } else {
263         kv = new KeyValue(kvBuf.array(), kvBuf.arrayOffset(), kvBuf.array().length
264             - kvBuf.arrayOffset());
265       }
266       kv.setMvccVersion(current.memstoreTS);
267       return kv;
268     }
269 
270     @Override
271     public void rewind() {
272       currentBuffer.rewind();
273       if (tagCompressionContext != null) {
274         tagCompressionContext.clear();
275       }
276       decodeFirst();
277       previous.invalidate();
278     }
279 
280     @Override
281     public boolean next() {
282       if (!currentBuffer.hasRemaining()) {
283         return false;
284       }
285       decodeNext();
286       previous.invalidate();
287       return true;
288     }
289 
290     protected void decodeTags() {
291       current.tagsLength = ByteBufferUtils.readCompressedInt(currentBuffer);
292       if (tagCompressionContext != null) {
293         if (current.uncompressTags) {
294           // Tag compression is been used. uncompress it into tagsBuffer
295           current.ensureSpaceForTags();
296           try {
297             current.tagsCompressedLength = tagCompressionContext.uncompressTags(currentBuffer,
298                 current.tagsBuffer, 0, current.tagsLength);
299           } catch (IOException e) {
300             throw new RuntimeException("Exception while uncompressing tags", e);
301           }
302         } else {
303           ByteBufferUtils.skip(currentBuffer, current.tagsCompressedLength);
304           current.uncompressTags = true;// Reset this.
305         }
306         current.tagsOffset = -1;
307       } else {
308         // When tag compress is not used, let us not do temp copying of tags bytes into tagsBuffer.
309         // Just mark the tags Offset so as to create the KV buffer later in getKeyValueBuffer()
310         current.tagsOffset = currentBuffer.position();
311         ByteBufferUtils.skip(currentBuffer, current.tagsLength);
312       }
313     }
314 
315     @Override
316     public int seekToKeyInBlock(byte[] key, int offset, int length,
317         boolean seekBefore) {
318       int commonPrefix = 0;
319       previous.invalidate();
320       do {
321         int comp;
322         if (samePrefixComparator != null) {
323           commonPrefix = Math.min(commonPrefix, current.lastCommonPrefix);
324 
325           // extend commonPrefix
326           commonPrefix += ByteBufferUtils.findCommonPrefix(
327               key, offset + commonPrefix, length - commonPrefix,
328               current.keyBuffer, commonPrefix,
329               current.keyLength - commonPrefix);
330 
331           comp = samePrefixComparator.compareIgnoringPrefix(commonPrefix, key,
332               offset, length, current.keyBuffer, 0, current.keyLength);
333         } else {
334           comp = comparator.compareFlatKey(key, offset, length,
335               current.keyBuffer, 0, current.keyLength);
336         }
337 
338         if (comp == 0) { // exact match
339           if (seekBefore) {
340             if (!previous.isValid()) {
341               // The caller (seekBefore) has to ensure that we are not at the
342               // first key in the block.
343               throw new IllegalStateException("Cannot seekBefore if " +
344                   "positioned at the first key in the block: key=" +
345                   Bytes.toStringBinary(key, offset, length));
346             }
347             moveToPrevious();
348             return 1;
349           }
350           return 0;
351         }
352 
353         if (comp < 0) { // already too large, check previous
354           if (previous.isValid()) {
355             moveToPrevious();
356           } else {
357             return HConstants.INDEX_KEY_MAGIC; // using optimized index key
358           }
359           return 1;
360         }
361 
362         // move to next, if more data is available
363         if (currentBuffer.hasRemaining()) {
364           previous.copyFromNext(current);
365           decodeNext();
366         } else {
367           break;
368         }
369       } while (true);
370 
371       // we hit the end of the block, not an exact match
372       return 1;
373     }
374 
375     private void moveToPrevious() {
376       if (!previous.isValid()) {
377         throw new IllegalStateException(
378             "Can move back only once and not in first key in the block.");
379       }
380 
381       STATE tmp = previous;
382       previous = current;
383       current = tmp;
384 
385       // move after last key value
386       currentBuffer.position(current.nextKvOffset);
387       // Already decoded the tag bytes. We cache this tags into current state and also the total
388       // compressed length of the tags bytes. For the next time decodeNext() we don't need to decode
389       // the tags again. This might pollute the Data Dictionary what we use for the compression.
390       // When current.uncompressTags is false, we will just reuse the current.tagsBuffer and skip
391       // 'tagsCompressedLength' bytes of source stream.
392       // See in decodeTags()
393       current.tagsBuffer = previous.tagsBuffer;
394       current.tagsCompressedLength = previous.tagsCompressedLength;
395       current.uncompressTags = false;
396       previous.invalidate();
397     }
398 
399     @SuppressWarnings("unchecked")
400     protected STATE createSeekerState() {
401       // This will fail for non-default seeker state if the subclass does not
402       // override this method.
403       return (STATE) new SeekerState();
404     }
405 
406     abstract protected void decodeFirst();
407     abstract protected void decodeNext();
408   }
409 
410   protected final void afterEncodingKeyValue(ByteBuffer in,
411       DataOutputStream out, HFileBlockDefaultEncodingContext encodingCtx) throws IOException {
412     if (encodingCtx.getHFileContext().isIncludesTags()) {
413       // Read short as unsigned, high byte first
414       int tagsLength = ((in.get() & 0xff) << 8) ^ (in.get() & 0xff);
415       ByteBufferUtils.putCompressedInt(out, tagsLength);
416       // There are some tags to be written
417       if (tagsLength > 0) {
418         TagCompressionContext tagCompressionContext = encodingCtx.getTagCompressionContext();
419         // When tag compression is enabled, tagCompressionContext will have a not null value. Write
420         // the tags using Dictionary compression in such a case
421         if (tagCompressionContext != null) {
422           tagCompressionContext.compressTags(out, in, tagsLength);
423         } else {
424           ByteBufferUtils.moveBufferToStream(out, in, tagsLength);
425         }
426       }
427     }
428     if (encodingCtx.getHFileContext().isIncludesMvcc()) {
429       // Copy memstore timestamp from the byte buffer to the output stream.
430       long memstoreTS = -1;
431       try {
432         memstoreTS = ByteBufferUtils.readVLong(in);
433         WritableUtils.writeVLong(out, memstoreTS);
434       } catch (IOException ex) {
435         throw new RuntimeException("Unable to copy memstore timestamp " +
436             memstoreTS + " after encoding a key/value");
437       }
438     }
439   }
440 
441   protected final void afterDecodingKeyValue(DataInputStream source,
442       ByteBuffer dest, HFileBlockDefaultDecodingContext decodingCtx) throws IOException {
443     if (decodingCtx.getHFileContext().isIncludesTags()) {
444       int tagsLength = ByteBufferUtils.readCompressedInt(source);
445       // Put as unsigned short
446       dest.put((byte)((tagsLength >> 8) & 0xff));
447       dest.put((byte)(tagsLength & 0xff));
448       if (tagsLength > 0) {
449         TagCompressionContext tagCompressionContext = decodingCtx.getTagCompressionContext();
450         // When tag compression is been used in this file, tagCompressionContext will have a not
451         // null value passed.
452         if (tagCompressionContext != null) {
453           tagCompressionContext.uncompressTags(source, dest, tagsLength);
454         } else {
455           ByteBufferUtils.copyFromStreamToBuffer(dest, source, tagsLength);
456         }
457       }
458     }
459     if (decodingCtx.getHFileContext().isIncludesMvcc()) {
460       long memstoreTS = -1;
461       try {
462         // Copy memstore timestamp from the data input stream to the byte
463         // buffer.
464         memstoreTS = WritableUtils.readVLong(source);
465         ByteBufferUtils.writeVLong(dest, memstoreTS);
466       } catch (IOException ex) {
467         throw new RuntimeException("Unable to copy memstore timestamp " +
468             memstoreTS + " after decoding a key/value");
469       }
470     }
471   }
472 
473   @Override
474   public HFileBlockEncodingContext newDataBlockEncodingContext(DataBlockEncoding encoding,
475       byte[] header, HFileContext meta) {
476     return new HFileBlockDefaultEncodingContext(encoding, header, meta);
477   }
478 
479   @Override
480   public HFileBlockDecodingContext newDataBlockDecodingContext(HFileContext meta) {
481     return new HFileBlockDefaultDecodingContext(meta);
482   }
483 
484   /**
485    * Compress KeyValues and write them to output buffer.
486    * @param out Where to write compressed data.
487    * @param in Source of KeyValue for compression.
488    * @param encodingCtx use the Encoding ctx associated with the current block
489    * @throws IOException If there is an error writing to output stream.
490    */
491   public abstract void internalEncodeKeyValues(DataOutputStream out,
492       ByteBuffer in, HFileBlockDefaultEncodingContext encodingCtx) throws IOException;
493 
494   protected abstract ByteBuffer internalDecodeKeyValues(DataInputStream source,
495       int allocateHeaderLength, int skipLastBytes, HFileBlockDefaultDecodingContext decodingCtx)
496       throws IOException;
497 
498   @Override
499   public void encodeKeyValues(ByteBuffer in,
500       HFileBlockEncodingContext blkEncodingCtx) throws IOException {
501     if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
502       throw new IOException (this.getClass().getName() + " only accepts "
503           + HFileBlockDefaultEncodingContext.class.getName() + " as the " +
504           "encoding context.");
505     }
506 
507     HFileBlockDefaultEncodingContext encodingCtx =
508         (HFileBlockDefaultEncodingContext) blkEncodingCtx;
509     encodingCtx.prepareEncoding();
510     DataOutputStream dataOut = encodingCtx.getOutputStreamForEncoder();
511     if (encodingCtx.getHFileContext().isIncludesTags()
512         && encodingCtx.getHFileContext().isCompressTags()) {
513       if (encodingCtx.getTagCompressionContext() != null) {
514         // It will be overhead to create the TagCompressionContext again and again for every block
515         // encoding.
516         encodingCtx.getTagCompressionContext().clear();
517       } else {
518         try {
519           TagCompressionContext tagCompressionContext = new TagCompressionContext(
520               LRUDictionary.class, Byte.MAX_VALUE);
521           encodingCtx.setTagCompressionContext(tagCompressionContext);
522         } catch (Exception e) {
523           throw new IOException("Failed to initialize TagCompressionContext", e);
524         }
525       }
526     }
527     internalEncodeKeyValues(dataOut, in, encodingCtx);
528     if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) {
529       encodingCtx.postEncoding(BlockType.ENCODED_DATA);
530     } else {
531       encodingCtx.postEncoding(BlockType.DATA);
532     }
533   }
534 
535   /**
536    * Asserts that there is at least the given amount of unfilled space
537    * remaining in the given buffer.
538    * @param out typically, the buffer we are writing to
539    * @param length the required space in the buffer
540    * @throws EncoderBufferTooSmallException If there are no enough bytes.
541    */
542   protected static void ensureSpace(ByteBuffer out, int length)
543       throws EncoderBufferTooSmallException {
544     if (out.position() + length > out.limit()) {
545       throw new EncoderBufferTooSmallException(
546           "Buffer position=" + out.position() +
547           ", buffer limit=" + out.limit() +
548           ", length to be written=" + length);
549     }
550   }
551 
552 }