View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations
15   * under the License.
16   */
17  package org.apache.hadoop.hbase.io.encoding;
18  
19  import java.io.DataInputStream;
20  import java.io.DataOutputStream;
21  import java.io.IOException;
22  import java.nio.ByteBuffer;
23  
24  import org.apache.hadoop.classification.InterfaceAudience;
25  import org.apache.hadoop.hbase.HConstants;
26  import org.apache.hadoop.hbase.KeyValue;
27  import org.apache.hadoop.hbase.KeyValue.KVComparator;
28  import org.apache.hadoop.hbase.KeyValue.SamePrefixComparator;
29  import org.apache.hadoop.hbase.io.TagCompressionContext;
30  import org.apache.hadoop.hbase.io.hfile.BlockType;
31  import org.apache.hadoop.hbase.io.hfile.HFileContext;
32  import org.apache.hadoop.hbase.io.util.LRUDictionary;
33  import org.apache.hadoop.hbase.util.ByteBufferUtils;
34  import org.apache.hadoop.hbase.util.Bytes;
35  import org.apache.hadoop.io.WritableUtils;
36  
37  /**
38   * Base class for all data block encoders that use a buffer.
39   */
40  @InterfaceAudience.Private
41  abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
42  
43    private static int INITIAL_KEY_BUFFER_SIZE = 512;
44  
45    @Override
46    public ByteBuffer decodeKeyValues(DataInputStream source,
47        HFileBlockDecodingContext blkDecodingCtx) throws IOException {
48      if (blkDecodingCtx.getClass() != HFileBlockDefaultDecodingContext.class) {
49        throw new IOException(this.getClass().getName() + " only accepts "
50            + HFileBlockDefaultDecodingContext.class.getName() + " as the decoding context.");
51      }
52  
53      HFileBlockDefaultDecodingContext decodingCtx =
54          (HFileBlockDefaultDecodingContext) blkDecodingCtx;
55      if (decodingCtx.getHFileContext().isCompressTags()) {
56        try {
57          TagCompressionContext tagCompressionContext = new TagCompressionContext(LRUDictionary.class);
58          decodingCtx.setTagCompressionContext(tagCompressionContext);
59        } catch (Exception e) {
60          throw new IOException("Failed to initialize TagCompressionContext", e);
61        }
62      }
63      return internalDecodeKeyValues(source, 0, 0, decodingCtx);
64    }
65  
66    protected static class SeekerState {
67      protected int valueOffset = -1;
68      protected int keyLength;
69      protected int valueLength;
70      protected int lastCommonPrefix;
71      protected int tagsLength = 0;
72      protected int tagsOffset = -1;
73  
74      /** We need to store a copy of the key. */
75      protected byte[] keyBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
76      protected byte[] tagsBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
77  
78      protected long memstoreTS;
79      protected int nextKvOffset;
80  
81      protected boolean isValid() {
82        return valueOffset != -1;
83      }
84  
85      protected void invalidate() {
86        valueOffset = -1;
87      }
88  
89      protected void ensureSpaceForKey() {
90        if (keyLength > keyBuffer.length) {
91          // rare case, but we need to handle arbitrary length of key
92          int newKeyBufferLength = Math.max(keyBuffer.length, 1) * 2;
93          while (keyLength > newKeyBufferLength) {
94            newKeyBufferLength *= 2;
95          }
96          byte[] newKeyBuffer = new byte[newKeyBufferLength];
97          System.arraycopy(keyBuffer, 0, newKeyBuffer, 0, keyBuffer.length);
98          keyBuffer = newKeyBuffer;
99        }
100     }
101 
102     protected void ensureSpaceForTags() {
103       if (tagsLength > tagsBuffer.length) {
104         // rare case, but we need to handle arbitrary length of tags
105         int newTagsBufferLength = Math.max(tagsBuffer.length, 1) * 2;
106         while (tagsLength > newTagsBufferLength) {
107           newTagsBufferLength *= 2;
108         }
109         byte[] newTagsBuffer = new byte[newTagsBufferLength];
110         System.arraycopy(tagsBuffer, 0, newTagsBuffer, 0, tagsBuffer.length);
111         tagsBuffer = newTagsBuffer;
112       }
113     }
114 
115     /**
116      * Copy the state from the next one into this instance (the previous state
117      * placeholder). Used to save the previous state when we are advancing the
118      * seeker to the next key/value.
119      */
120     protected void copyFromNext(SeekerState nextState) {
121       if (keyBuffer.length != nextState.keyBuffer.length) {
122         keyBuffer = nextState.keyBuffer.clone();
123       } else if (!isValid()) {
124         // Note: we can only call isValid before we override our state, so this
125         // comes before all the assignments at the end of this method.
126         System.arraycopy(nextState.keyBuffer, 0, keyBuffer, 0,
127              nextState.keyLength);
128       } else {
129         // don't copy the common prefix between this key and the previous one
130         System.arraycopy(nextState.keyBuffer, nextState.lastCommonPrefix,
131             keyBuffer, nextState.lastCommonPrefix, nextState.keyLength
132                 - nextState.lastCommonPrefix);
133       }
134 
135       valueOffset = nextState.valueOffset;
136       keyLength = nextState.keyLength;
137       valueLength = nextState.valueLength;
138       lastCommonPrefix = nextState.lastCommonPrefix;
139       nextKvOffset = nextState.nextKvOffset;
140       memstoreTS = nextState.memstoreTS;
141     }
142 
143   }
144 
145   protected abstract static class
146       BufferedEncodedSeeker<STATE extends SeekerState>
147       implements EncodedSeeker {
148     protected HFileBlockDecodingContext decodingCtx;
149     protected final KVComparator comparator;
150     protected final SamePrefixComparator<byte[]> samePrefixComparator;
151     protected ByteBuffer currentBuffer;
152     protected STATE current = createSeekerState(); // always valid
153     protected STATE previous = createSeekerState(); // may not be valid
154     protected TagCompressionContext tagCompressionContext = null;
155 
156     public BufferedEncodedSeeker(KVComparator comparator,
157         HFileBlockDecodingContext decodingCtx) {
158       this.comparator = comparator;
159       this.samePrefixComparator = comparator;
160       this.decodingCtx = decodingCtx;
161       if (decodingCtx.getHFileContext().isCompressTags()) {
162         try {
163           tagCompressionContext = new TagCompressionContext(LRUDictionary.class);
164         } catch (Exception e) {
165           throw new RuntimeException("Failed to initialize TagCompressionContext", e);
166         }
167       }
168     }
169     
170     protected boolean includesMvcc() {
171       return this.decodingCtx.getHFileContext().isIncludesMvcc();
172     }
173 
174     protected boolean includesTags() {
175       return this.decodingCtx.getHFileContext().isIncludesTags();
176     }
177 
178     @Override
179     public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
180       return comparator.compareFlatKey(key, offset, length,
181           current.keyBuffer, 0, current.keyLength);
182     }
183 
184     @Override
185     public void setCurrentBuffer(ByteBuffer buffer) {
186       if (this.tagCompressionContext != null) {
187         this.tagCompressionContext.clear();
188       }
189       currentBuffer = buffer;
190       decodeFirst();
191       previous.invalidate();
192     }
193 
194     @Override
195     public ByteBuffer getKeyDeepCopy() {
196       ByteBuffer keyBuffer = ByteBuffer.allocate(current.keyLength);
197       keyBuffer.put(current.keyBuffer, 0, current.keyLength);
198       return keyBuffer;
199     }
200 
201     @Override
202     public ByteBuffer getValueShallowCopy() {
203       return ByteBuffer.wrap(currentBuffer.array(),
204           currentBuffer.arrayOffset() + current.valueOffset,
205           current.valueLength);
206     }
207 
208     @Override
209     public ByteBuffer getKeyValueBuffer() {
210       ByteBuffer kvBuffer = createKVBuffer();
211       kvBuffer.putInt(current.keyLength);
212       kvBuffer.putInt(current.valueLength);
213       kvBuffer.put(current.keyBuffer, 0, current.keyLength);
214       kvBuffer.put(currentBuffer.array(),
215           currentBuffer.arrayOffset() + current.valueOffset,
216           current.valueLength);
217       if (current.tagsLength > 0) {
218         kvBuffer.putShort((short) current.tagsLength);
219         if (current.tagsOffset != -1) {
220           // the offset of the tags bytes in the underlying buffer is marked. So the temp
221           // buffer,tagsBuffer was not been used.
222           kvBuffer.put(currentBuffer.array(), currentBuffer.arrayOffset() + current.tagsOffset,
223               current.tagsLength);
224         } else {
225           // When tagsOffset is marked as -1, tag compression was present and so the tags were
226           // uncompressed into temp buffer, tagsBuffer. Let us copy it from there
227           kvBuffer.put(current.tagsBuffer, 0, current.tagsLength);
228         }
229       }
230       return kvBuffer;
231     }
232 
233     protected ByteBuffer createKVBuffer() {
234       int kvBufSize = (int) KeyValue.getKeyValueDataStructureSize(current.keyLength,
235           current.valueLength, current.tagsLength);
236       ByteBuffer kvBuffer = ByteBuffer.allocate(kvBufSize);
237       return kvBuffer;
238     }
239 
240     @Override
241     public KeyValue getKeyValue() {
242       ByteBuffer kvBuf = getKeyValueBuffer();
243       KeyValue kv = new KeyValue(kvBuf.array(), kvBuf.arrayOffset(), kvBuf.array().length
244           - kvBuf.arrayOffset());
245       kv.setMvccVersion(current.memstoreTS);
246       return kv;
247     }
248 
249     @Override
250     public void rewind() {
251       currentBuffer.rewind();
252       decodeFirst();
253       previous.invalidate();
254     }
255 
256     @Override
257     public boolean next() {
258       if (!currentBuffer.hasRemaining()) {
259         return false;
260       }
261       decodeNext();
262       previous.invalidate();
263       return true;
264     }
265 
266     protected void decodeTags() {
267       current.tagsLength = ByteBufferUtils.readCompressedInt(currentBuffer);
268       if (tagCompressionContext != null) {
269         // Tag compression is been used. uncompress it into tagsBuffer
270         current.ensureSpaceForTags();
271         try {
272           tagCompressionContext.uncompressTags(currentBuffer, current.tagsBuffer, 0,
273               current.tagsLength);
274         } catch (IOException e) {
275           throw new RuntimeException("Exception while uncompressing tags", e);
276         }
277         current.tagsOffset = -1;
278       } else {
279         // When tag compress is not used, let us not do temp copying of tags bytes into tagsBuffer.
280         // Just mark the tags Offset so as to create the KV buffer later in getKeyValueBuffer()
281         current.tagsOffset = currentBuffer.position();
282         ByteBufferUtils.skip(currentBuffer, current.tagsLength);
283       }
284     }
285 
286     @Override
287     public int seekToKeyInBlock(byte[] key, int offset, int length,
288         boolean seekBefore) {
289       int commonPrefix = 0;
290       previous.invalidate();
291       do {
292         int comp;
293         if (samePrefixComparator != null) {
294           commonPrefix = Math.min(commonPrefix, current.lastCommonPrefix);
295 
296           // extend commonPrefix
297           commonPrefix += ByteBufferUtils.findCommonPrefix(
298               key, offset + commonPrefix, length - commonPrefix,
299               current.keyBuffer, commonPrefix,
300               current.keyLength - commonPrefix);
301 
302           comp = samePrefixComparator.compareIgnoringPrefix(commonPrefix, key,
303               offset, length, current.keyBuffer, 0, current.keyLength);
304         } else {
305           comp = comparator.compareFlatKey(key, offset, length,
306               current.keyBuffer, 0, current.keyLength);
307         }
308 
309         if (comp == 0) { // exact match
310           if (seekBefore) {
311             if (!previous.isValid()) {
312               // The caller (seekBefore) has to ensure that we are not at the
313               // first key in the block.
314               throw new IllegalStateException("Cannot seekBefore if " +
315                   "positioned at the first key in the block: key=" +
316                   Bytes.toStringBinary(key, offset, length));
317             }
318             moveToPrevious();
319             return 1;
320           }
321           return 0;
322         }
323 
324         if (comp < 0) { // already too large, check previous
325           if (previous.isValid()) {
326             moveToPrevious();
327           } else {
328             return HConstants.INDEX_KEY_MAGIC; // using optimized index key
329           }
330           return 1;
331         }
332 
333         // move to next, if more data is available
334         if (currentBuffer.hasRemaining()) {
335           previous.copyFromNext(current);
336           decodeNext();
337         } else {
338           break;
339         }
340       } while (true);
341 
342       // we hit the end of the block, not an exact match
343       return 1;
344     }
345 
346     private void moveToPrevious() {
347       if (!previous.isValid()) {
348         throw new IllegalStateException(
349             "Can move back only once and not in first key in the block.");
350       }
351 
352       STATE tmp = previous;
353       previous = current;
354       current = tmp;
355 
356       // move after last key value
357       currentBuffer.position(current.nextKvOffset);
358 
359       previous.invalidate();
360     }
361 
362     @SuppressWarnings("unchecked")
363     protected STATE createSeekerState() {
364       // This will fail for non-default seeker state if the subclass does not
365       // override this method.
366       return (STATE) new SeekerState();
367     }
368 
369     abstract protected void decodeFirst();
370     abstract protected void decodeNext();
371   }
372 
373   protected final void afterEncodingKeyValue(ByteBuffer in,
374       DataOutputStream out, HFileBlockDefaultEncodingContext encodingCtx) throws IOException {
375     if (encodingCtx.getHFileContext().isIncludesTags()) {
376       short tagsLength = in.getShort();
377       ByteBufferUtils.putCompressedInt(out, tagsLength);
378       // There are some tags to be written
379       if (tagsLength > 0) {
380         TagCompressionContext tagCompressionContext = encodingCtx.getTagCompressionContext();
381         // When tag compression is enabled, tagCompressionContext will have a not null value. Write
382         // the tags using Dictionary compression in such a case
383         if (tagCompressionContext != null) {
384           tagCompressionContext.compressTags(out, in, tagsLength);
385         } else {
386           ByteBufferUtils.moveBufferToStream(out, in, tagsLength);
387         }
388       }
389     }
390     if (encodingCtx.getHFileContext().isIncludesMvcc()) {
391       // Copy memstore timestamp from the byte buffer to the output stream.
392       long memstoreTS = -1;
393       try {
394         memstoreTS = ByteBufferUtils.readVLong(in);
395         WritableUtils.writeVLong(out, memstoreTS);
396       } catch (IOException ex) {
397         throw new RuntimeException("Unable to copy memstore timestamp " +
398             memstoreTS + " after encoding a key/value");
399       }
400     }
401   }
402 
403   protected final void afterDecodingKeyValue(DataInputStream source,
404       ByteBuffer dest, HFileBlockDefaultDecodingContext decodingCtx) throws IOException {
405     if (decodingCtx.getHFileContext().isIncludesTags()) {
406       short tagsLength = (short) ByteBufferUtils.readCompressedInt(source);
407       dest.putShort(tagsLength);
408       if (tagsLength > 0) {
409         TagCompressionContext tagCompressionContext = decodingCtx.getTagCompressionContext();
410         // When tag compression is been used in this file, tagCompressionContext will have a not
411         // null value passed.
412         if (tagCompressionContext != null) {
413           tagCompressionContext.uncompressTags(source, dest, tagsLength);
414         } else {
415           ByteBufferUtils.copyFromStreamToBuffer(dest, source, tagsLength);
416         }
417       }
418     }
419     if (decodingCtx.getHFileContext().isIncludesMvcc()) {
420       long memstoreTS = -1;
421       try {
422         // Copy memstore timestamp from the data input stream to the byte
423         // buffer.
424         memstoreTS = WritableUtils.readVLong(source);
425         ByteBufferUtils.writeVLong(dest, memstoreTS);
426       } catch (IOException ex) {
427         throw new RuntimeException("Unable to copy memstore timestamp " +
428             memstoreTS + " after decoding a key/value");
429       }
430     }
431   }
432 
433   @Override
434   public HFileBlockEncodingContext newDataBlockEncodingContext(DataBlockEncoding encoding,
435       byte[] header, HFileContext meta) {
436     return new HFileBlockDefaultEncodingContext(encoding, header, meta);
437   }
438 
439   @Override
440   public HFileBlockDecodingContext newDataBlockDecodingContext(HFileContext meta) {
441     return new HFileBlockDefaultDecodingContext(meta);
442   }
443 
444   /**
445    * Compress KeyValues and write them to output buffer.
446    * @param out Where to write compressed data.
447    * @param in Source of KeyValue for compression.
448    * @param encodingCtx use the Encoding ctx associated with the current block
449    * @throws IOException If there is an error writing to output stream.
450    */
451   public abstract void internalEncodeKeyValues(DataOutputStream out,
452       ByteBuffer in, HFileBlockDefaultEncodingContext encodingCtx) throws IOException;
453 
454   protected abstract ByteBuffer internalDecodeKeyValues(DataInputStream source,
455       int allocateHeaderLength, int skipLastBytes, HFileBlockDefaultDecodingContext decodingCtx)
456       throws IOException;
457 
458   @Override
459   public void encodeKeyValues(ByteBuffer in,
460       HFileBlockEncodingContext blkEncodingCtx) throws IOException {
461     if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
462       throw new IOException (this.getClass().getName() + " only accepts "
463           + HFileBlockDefaultEncodingContext.class.getName() + " as the " +
464           "encoding context.");
465     }
466 
467     HFileBlockDefaultEncodingContext encodingCtx =
468         (HFileBlockDefaultEncodingContext) blkEncodingCtx;
469     encodingCtx.prepareEncoding();
470     DataOutputStream dataOut = encodingCtx.getOutputStreamForEncoder();
471     if (encodingCtx.getHFileContext().isCompressTags()) {
472       try {
473         TagCompressionContext tagCompressionContext = new TagCompressionContext(LRUDictionary.class);
474         encodingCtx.setTagCompressionContext(tagCompressionContext);
475       } catch (Exception e) {
476         throw new IOException("Failed to initialize TagCompressionContext", e);
477       }
478     }
479     internalEncodeKeyValues(dataOut, in, encodingCtx);
480     if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) {
481       encodingCtx.postEncoding(BlockType.ENCODED_DATA);
482     } else {
483       encodingCtx.postEncoding(BlockType.DATA);
484     }
485   }
486 
487   /**
488    * Asserts that there is at least the given amount of unfilled space
489    * remaining in the given buffer.
490    * @param out typically, the buffer we are writing to
491    * @param length the required space in the buffer
492    * @throws EncoderBufferTooSmallException If there are no enough bytes.
493    */
494   protected static void ensureSpace(ByteBuffer out, int length)
495       throws EncoderBufferTooSmallException {
496     if (out.position() + length > out.limit()) {
497       throw new EncoderBufferTooSmallException(
498           "Buffer position=" + out.position() +
499           ", buffer limit=" + out.limit() +
500           ", length to be written=" + length);
501     }
502   }
503 
504 }