View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations
15   * under the License.
16   */
17  package org.apache.hadoop.hbase.io.encoding;
18  
19  import java.io.DataInputStream;
20  import java.io.DataOutputStream;
21  import java.io.IOException;
22  import java.nio.ByteBuffer;
23  
24  import org.apache.hadoop.classification.InterfaceAudience;
25  import org.apache.hadoop.hbase.HConstants;
26  import org.apache.hadoop.hbase.KeyValue;
27  import org.apache.hadoop.hbase.KeyValue.KVComparator;
28  import org.apache.hadoop.hbase.KeyValue.SamePrefixComparator;
29  import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
30  import org.apache.hadoop.hbase.io.hfile.BlockType;
31  import org.apache.hadoop.hbase.util.ByteBufferUtils;
32  import org.apache.hadoop.hbase.util.Bytes;
33  import org.apache.hadoop.io.WritableUtils;
34  
35  /**
36   * Base class for all data block encoders that use a buffer.
37   */
38  @InterfaceAudience.Private
39  abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
40  
41    private static int INITIAL_KEY_BUFFER_SIZE = 512;
42  
43    @Override
44    public ByteBuffer decodeKeyValues(DataInputStream source,
45        boolean includesMemstoreTS) throws IOException {
46      return decodeKeyValues(source, 0, 0, includesMemstoreTS);
47    }
48  
49    protected static class SeekerState {
50      protected int valueOffset = -1;
51      protected int keyLength;
52      protected int valueLength;
53      protected int lastCommonPrefix;
54  
55      /** We need to store a copy of the key. */
56      protected byte[] keyBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
57  
58      protected long memstoreTS;
59      protected int nextKvOffset;
60  
61      protected boolean isValid() {
62        return valueOffset != -1;
63      }
64  
65      protected void invalidate() {
66        valueOffset = -1;
67      }
68  
69      protected void ensureSpaceForKey() {
70        if (keyLength > keyBuffer.length) {
71          // rare case, but we need to handle arbitrary length of key
72          int newKeyBufferLength = Math.max(keyBuffer.length, 1) * 2;
73          while (keyLength > newKeyBufferLength) {
74            newKeyBufferLength *= 2;
75          }
76          byte[] newKeyBuffer = new byte[newKeyBufferLength];
77          System.arraycopy(keyBuffer, 0, newKeyBuffer, 0, keyBuffer.length);
78          keyBuffer = newKeyBuffer;
79        }
80      }
81  
82      /**
83       * Copy the state from the next one into this instance (the previous state
84       * placeholder). Used to save the previous state when we are advancing the
85       * seeker to the next key/value.
86       */
87      protected void copyFromNext(SeekerState nextState) {
88        if (keyBuffer.length != nextState.keyBuffer.length) {
89          keyBuffer = nextState.keyBuffer.clone();
90        } else if (!isValid()) {
91          // Note: we can only call isValid before we override our state, so this
92          // comes before all the assignments at the end of this method.
93          System.arraycopy(nextState.keyBuffer, 0, keyBuffer, 0,
94               nextState.keyLength);
95        } else {
96          // don't copy the common prefix between this key and the previous one
97          System.arraycopy(nextState.keyBuffer, nextState.lastCommonPrefix,
98              keyBuffer, nextState.lastCommonPrefix, nextState.keyLength
99                  - nextState.lastCommonPrefix);
100       }
101 
102       valueOffset = nextState.valueOffset;
103       keyLength = nextState.keyLength;
104       valueLength = nextState.valueLength;
105       lastCommonPrefix = nextState.lastCommonPrefix;
106       nextKvOffset = nextState.nextKvOffset;
107       memstoreTS = nextState.memstoreTS;
108     }
109 
110   }
111 
112   protected abstract static class
113       BufferedEncodedSeeker<STATE extends SeekerState>
114       implements EncodedSeeker {
115 
116     protected final KVComparator comparator;
117     protected final SamePrefixComparator<byte[]> samePrefixComparator;
118     protected ByteBuffer currentBuffer;
119     protected STATE current = createSeekerState(); // always valid
120     protected STATE previous = createSeekerState(); // may not be valid
121 
122     @SuppressWarnings("unchecked")
123     public BufferedEncodedSeeker(KVComparator comparator) {
124       this.comparator = comparator;
125       if (comparator instanceof SamePrefixComparator) {
126         this.samePrefixComparator = (SamePrefixComparator<byte[]>) comparator;
127       } else {
128         this.samePrefixComparator = null;
129       }
130     }
131 
132     @Override
133     public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
134       return comparator.compareFlatKey(key, offset, length,
135           current.keyBuffer, 0, current.keyLength);
136     }
137 
138     @Override
139     public void setCurrentBuffer(ByteBuffer buffer) {
140       currentBuffer = buffer;
141       decodeFirst();
142       previous.invalidate();
143     }
144 
145     @Override
146     public ByteBuffer getKeyDeepCopy() {
147       ByteBuffer keyBuffer = ByteBuffer.allocate(current.keyLength);
148       keyBuffer.put(current.keyBuffer, 0, current.keyLength);
149       return keyBuffer;
150     }
151 
152     @Override
153     public ByteBuffer getValueShallowCopy() {
154       return ByteBuffer.wrap(currentBuffer.array(),
155           currentBuffer.arrayOffset() + current.valueOffset,
156           current.valueLength);
157     }
158 
159     @Override
160     public ByteBuffer getKeyValueBuffer() {
161       ByteBuffer kvBuffer = ByteBuffer.allocate(
162           2 * Bytes.SIZEOF_INT + current.keyLength + current.valueLength);
163       kvBuffer.putInt(current.keyLength);
164       kvBuffer.putInt(current.valueLength);
165       kvBuffer.put(current.keyBuffer, 0, current.keyLength);
166       kvBuffer.put(currentBuffer.array(),
167           currentBuffer.arrayOffset() + current.valueOffset,
168           current.valueLength);
169       return kvBuffer;
170     }
171 
172     @Override
173     public KeyValue getKeyValue() {
174       ByteBuffer kvBuf = getKeyValueBuffer();
175       KeyValue kv = new KeyValue(kvBuf.array(), kvBuf.arrayOffset());
176       kv.setMvccVersion(current.memstoreTS);
177       return kv;
178     }
179 
180     @Override
181     public void rewind() {
182       currentBuffer.rewind();
183       decodeFirst();
184       previous.invalidate();
185     }
186 
187     @Override
188     public boolean next() {
189       if (!currentBuffer.hasRemaining()) {
190         return false;
191       }
192       decodeNext();
193       previous.invalidate();
194       return true;
195     }
196 
197     @Override
198     public int seekToKeyInBlock(byte[] key, int offset, int length,
199         boolean seekBefore) {
200       int commonPrefix = 0;
201       previous.invalidate();
202       do {
203         int comp;
204         if (samePrefixComparator != null) {
205           commonPrefix = Math.min(commonPrefix, current.lastCommonPrefix);
206 
207           // extend commonPrefix
208           commonPrefix += ByteBufferUtils.findCommonPrefix(
209               key, offset + commonPrefix, length - commonPrefix,
210               current.keyBuffer, commonPrefix,
211               current.keyLength - commonPrefix);
212 
213           comp = samePrefixComparator.compareIgnoringPrefix(commonPrefix, key,
214               offset, length, current.keyBuffer, 0, current.keyLength);
215         } else {
216           comp = comparator.compareFlatKey(key, offset, length,
217               current.keyBuffer, 0, current.keyLength);
218         }
219 
220         if (comp == 0) { // exact match
221           if (seekBefore) {
222             if (!previous.isValid()) {
223               // The caller (seekBefore) has to ensure that we are not at the
224               // first key in the block.
225               throw new IllegalStateException("Cannot seekBefore if " +
226                   "positioned at the first key in the block: key=" +
227                   Bytes.toStringBinary(key, offset, length));
228             }
229             moveToPrevious();
230             return 1;
231           }
232           return 0;
233         }
234 
235         if (comp < 0) { // already too large, check previous
236           if (previous.isValid()) {
237             moveToPrevious();
238           } else {
239             return HConstants.INDEX_KEY_MAGIC; // using optimized index key
240           }
241           return 1;
242         }
243 
244         // move to next, if more data is available
245         if (currentBuffer.hasRemaining()) {
246           previous.copyFromNext(current);
247           decodeNext();
248         } else {
249           break;
250         }
251       } while (true);
252 
253       // we hit the end of the block, not an exact match
254       return 1;
255     }
256 
257     private void moveToPrevious() {
258       if (!previous.isValid()) {
259         throw new IllegalStateException(
260             "Can move back only once and not in first key in the block.");
261       }
262 
263       STATE tmp = previous;
264       previous = current;
265       current = tmp;
266 
267       // move after last key value
268       currentBuffer.position(current.nextKvOffset);
269 
270       previous.invalidate();
271     }
272 
273     @SuppressWarnings("unchecked")
274     protected STATE createSeekerState() {
275       // This will fail for non-default seeker state if the subclass does not
276       // override this method.
277       return (STATE) new SeekerState();
278     }
279 
280     abstract protected void decodeFirst();
281     abstract protected void decodeNext();
282   }
283 
284   protected final void afterEncodingKeyValue(ByteBuffer in,
285       DataOutputStream out, boolean includesMemstoreTS) {
286     if (includesMemstoreTS) {
287       // Copy memstore timestamp from the byte buffer to the output stream.
288       long memstoreTS = -1;
289       try {
290         memstoreTS = ByteBufferUtils.readVLong(in);
291         WritableUtils.writeVLong(out, memstoreTS);
292       } catch (IOException ex) {
293         throw new RuntimeException("Unable to copy memstore timestamp " +
294             memstoreTS + " after encoding a key/value");
295       }
296     }
297   }
298 
299   protected final void afterDecodingKeyValue(DataInputStream source,
300       ByteBuffer dest, boolean includesMemstoreTS) {
301     if (includesMemstoreTS) {
302       long memstoreTS = -1;
303       try {
304         // Copy memstore timestamp from the data input stream to the byte
305         // buffer.
306         memstoreTS = WritableUtils.readVLong(source);
307         ByteBufferUtils.writeVLong(dest, memstoreTS);
308       } catch (IOException ex) {
309         throw new RuntimeException("Unable to copy memstore timestamp " +
310             memstoreTS + " after decoding a key/value");
311       }
312     }
313   }
314 
315   @Override
316   public HFileBlockEncodingContext newDataBlockEncodingContext(
317       Algorithm compressionAlgorithm,
318       DataBlockEncoding encoding, byte[] header) {
319     return new HFileBlockDefaultEncodingContext(
320         compressionAlgorithm, encoding, header);
321   }
322 
323   @Override
324   public HFileBlockDecodingContext newDataBlockDecodingContext(
325       Algorithm compressionAlgorithm) {
326     return new HFileBlockDefaultDecodingContext(compressionAlgorithm);
327   }
328 
329   /**
330    * Compress KeyValues and write them to output buffer.
331    * @param out Where to write compressed data.
332    * @param in Source of KeyValue for compression.
333    * @param includesMemstoreTS true if including memstore timestamp after every
334    *          key-value pair
335    * @throws IOException If there is an error writing to output stream.
336    */
337   public abstract void internalEncodeKeyValues(DataOutputStream out,
338       ByteBuffer in, boolean includesMemstoreTS) throws IOException;
339 
340   @Override
341   public void encodeKeyValues(ByteBuffer in,
342       boolean includesMemstoreTS,
343       HFileBlockEncodingContext blkEncodingCtx) throws IOException {
344     if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
345       throw new IOException (this.getClass().getName() + " only accepts "
346           + HFileBlockDefaultEncodingContext.class.getName() + " as the " +
347           "encoding context.");
348     }
349 
350     HFileBlockDefaultEncodingContext encodingCtx =
351         (HFileBlockDefaultEncodingContext) blkEncodingCtx;
352     encodingCtx.prepareEncoding();
353     DataOutputStream dataOut =
354         ((HFileBlockDefaultEncodingContext) encodingCtx)
355         .getOutputStreamForEncoder();
356     internalEncodeKeyValues(dataOut, in, includesMemstoreTS);
357     if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) {
358       encodingCtx.postEncoding(BlockType.ENCODED_DATA);
359     } else {
360       encodingCtx.postEncoding(BlockType.DATA);
361     }
362   }
363 
364   /**
365    * Asserts that there is at least the given amount of unfilled space
366    * remaining in the given buffer.
367    * @param out typically, the buffer we are writing to
368    * @param length the required space in the buffer
369    * @throws EncoderBufferTooSmallException If there are no enough bytes.
370    */
371   protected static void ensureSpace(ByteBuffer out, int length)
372       throws EncoderBufferTooSmallException {
373     if (out.position() + length > out.limit()) {
374       throw new EncoderBufferTooSmallException(
375           "Buffer position=" + out.position() +
376           ", buffer limit=" + out.limit() +
377           ", length to be written=" + length);
378     }
379   }
380 
381 }