View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations
15   * under the License.
16   */
17  package org.apache.hadoop.hbase.io.encoding;
18  
19  import java.io.DataInputStream;
20  import java.io.DataOutputStream;
21  import java.io.IOException;
22  import java.nio.ByteBuffer;
23  
24  import org.apache.hadoop.classification.InterfaceAudience;
25  import org.apache.hadoop.hbase.HConstants;
26  import org.apache.hadoop.hbase.KeyValue;
27  import org.apache.hadoop.hbase.KeyValue.SamePrefixComparator;
28  import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
29  import org.apache.hadoop.hbase.io.hfile.BlockType;
30  import org.apache.hadoop.hbase.util.ByteBufferUtils;
31  import org.apache.hadoop.hbase.util.Bytes;
32  import org.apache.hadoop.io.RawComparator;
33  import org.apache.hadoop.io.WritableUtils;
34  
35  /**
36   * Base class for all data block encoders that use a buffer.
37   */
38  @InterfaceAudience.Private
39  abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
40  
41    private static int INITIAL_KEY_BUFFER_SIZE = 512;
42  
43    @Override
44    public ByteBuffer decodeKeyValues(DataInputStream source,
45        boolean includesMemstoreTS) throws IOException {
46      return decodeKeyValues(source, 0, 0, includesMemstoreTS);
47    }
48  
49    protected static class SeekerState {
50      protected int valueOffset = -1;
51      protected int keyLength;
52      protected int valueLength;
53      protected int lastCommonPrefix;
54  
55      /** We need to store a copy of the key. */
56      protected byte[] keyBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
57  
58      protected long memstoreTS;
59      protected int nextKvOffset;
60  
61      protected boolean isValid() {
62        return valueOffset != -1;
63      }
64  
65      protected void invalidate() {
66        valueOffset = -1;
67      }
68  
69      protected void ensureSpaceForKey() {
70        if (keyLength > keyBuffer.length) {
71          // rare case, but we need to handle arbitrary length of key
72          int newKeyBufferLength = Math.max(keyBuffer.length, 1) * 2;
73          while (keyLength > newKeyBufferLength) {
74            newKeyBufferLength *= 2;
75          }
76          byte[] newKeyBuffer = new byte[newKeyBufferLength];
77          System.arraycopy(keyBuffer, 0, newKeyBuffer, 0, keyBuffer.length);
78          keyBuffer = newKeyBuffer;
79        }
80      }
81  
82      /**
83       * Copy the state from the next one into this instance (the previous state
84       * placeholder). Used to save the previous state when we are advancing the
85       * seeker to the next key/value.
86       */
87      protected void copyFromNext(SeekerState nextState) {
88        if (keyBuffer.length != nextState.keyBuffer.length) {
89          keyBuffer = nextState.keyBuffer.clone();
90        } else if (!isValid()) {
91          // Note: we can only call isValid before we override our state, so this
92          // comes before all the assignments at the end of this method.
93          System.arraycopy(nextState.keyBuffer, 0, keyBuffer, 0,
94               nextState.keyLength);
95        } else {
96          // don't copy the common prefix between this key and the previous one
97          System.arraycopy(nextState.keyBuffer, nextState.lastCommonPrefix,
98              keyBuffer, nextState.lastCommonPrefix, nextState.keyLength
99                  - nextState.lastCommonPrefix);
100       }
101 
102       valueOffset = nextState.valueOffset;
103       keyLength = nextState.keyLength;
104       valueLength = nextState.valueLength;
105       lastCommonPrefix = nextState.lastCommonPrefix;
106       nextKvOffset = nextState.nextKvOffset;
107       memstoreTS = nextState.memstoreTS;
108     }
109 
110   }
111 
112   protected abstract static class
113       BufferedEncodedSeeker<STATE extends SeekerState>
114       implements EncodedSeeker {
115 
116     protected final RawComparator<byte[]> comparator;
117     protected final SamePrefixComparator<byte[]> samePrefixComparator;
118     protected ByteBuffer currentBuffer;
119     protected STATE current = createSeekerState(); // always valid
120     protected STATE previous = createSeekerState(); // may not be valid
121 
122     @SuppressWarnings("unchecked")
123     public BufferedEncodedSeeker(RawComparator<byte[]> comparator) {
124       this.comparator = comparator;
125       if (comparator instanceof SamePrefixComparator) {
126         this.samePrefixComparator = (SamePrefixComparator<byte[]>) comparator;
127       } else {
128         this.samePrefixComparator = null;
129       }
130     }
131 
132     @Override
133     public void setCurrentBuffer(ByteBuffer buffer) {
134       currentBuffer = buffer;
135       decodeFirst();
136       previous.invalidate();
137     }
138 
139     @Override
140     public ByteBuffer getKeyDeepCopy() {
141       ByteBuffer keyBuffer = ByteBuffer.allocate(current.keyLength);
142       keyBuffer.put(current.keyBuffer, 0, current.keyLength);
143       return keyBuffer;
144     }
145 
146     @Override
147     public ByteBuffer getValueShallowCopy() {
148       return ByteBuffer.wrap(currentBuffer.array(),
149           currentBuffer.arrayOffset() + current.valueOffset,
150           current.valueLength);
151     }
152 
153     @Override
154     public ByteBuffer getKeyValueBuffer() {
155       ByteBuffer kvBuffer = ByteBuffer.allocate(
156           2 * Bytes.SIZEOF_INT + current.keyLength + current.valueLength);
157       kvBuffer.putInt(current.keyLength);
158       kvBuffer.putInt(current.valueLength);
159       kvBuffer.put(current.keyBuffer, 0, current.keyLength);
160       kvBuffer.put(currentBuffer.array(),
161           currentBuffer.arrayOffset() + current.valueOffset,
162           current.valueLength);
163       return kvBuffer;
164     }
165 
166     @Override
167     public KeyValue getKeyValue() {
168       ByteBuffer kvBuf = getKeyValueBuffer();
169       KeyValue kv = new KeyValue(kvBuf.array(), kvBuf.arrayOffset());
170       kv.setMvccVersion(current.memstoreTS);
171       return kv;
172     }
173 
174     @Override
175     public void rewind() {
176       currentBuffer.rewind();
177       decodeFirst();
178       previous.invalidate();
179     }
180 
181     @Override
182     public boolean next() {
183       if (!currentBuffer.hasRemaining()) {
184         return false;
185       }
186       decodeNext();
187       previous.invalidate();
188       return true;
189     }
190 
191     @Override
192     public int seekToKeyInBlock(byte[] key, int offset, int length,
193         boolean seekBefore) {
194       int commonPrefix = 0;
195       previous.invalidate();
196       do {
197         int comp;
198         if (samePrefixComparator != null) {
199           commonPrefix = Math.min(commonPrefix, current.lastCommonPrefix);
200 
201           // extend commonPrefix
202           commonPrefix += ByteBufferUtils.findCommonPrefix(
203               key, offset + commonPrefix, length - commonPrefix,
204               current.keyBuffer, commonPrefix,
205               current.keyLength - commonPrefix);
206 
207           comp = samePrefixComparator.compareIgnoringPrefix(commonPrefix, key,
208               offset, length, current.keyBuffer, 0, current.keyLength);
209         } else {
210           comp = comparator.compare(key, offset, length,
211               current.keyBuffer, 0, current.keyLength);
212         }
213 
214         if (comp == 0) { // exact match
215           if (seekBefore) {
216             if (!previous.isValid()) {
217               // The caller (seekBefore) has to ensure that we are not at the
218               // first key in the block.
219               throw new IllegalStateException("Cannot seekBefore if " +
220                   "positioned at the first key in the block: key=" +
221                   Bytes.toStringBinary(key, offset, length));
222             }
223             moveToPrevious();
224             return 1;
225           }
226           return 0;
227         }
228 
229         if (comp < 0) { // already too large, check previous
230           if (previous.isValid()) {
231             moveToPrevious();
232           } else {
233             return HConstants.INDEX_KEY_MAGIC; // using optimized index key
234           }
235           return 1;
236         }
237 
238         // move to next, if more data is available
239         if (currentBuffer.hasRemaining()) {
240           previous.copyFromNext(current);
241           decodeNext();
242         } else {
243           break;
244         }
245       } while (true);
246 
247       // we hit the end of the block, not an exact match
248       return 1;
249     }
250 
251     private void moveToPrevious() {
252       if (!previous.isValid()) {
253         throw new IllegalStateException(
254             "Can move back only once and not in first key in the block.");
255       }
256 
257       STATE tmp = previous;
258       previous = current;
259       current = tmp;
260 
261       // move after last key value
262       currentBuffer.position(current.nextKvOffset);
263 
264       previous.invalidate();
265     }
266 
267     @SuppressWarnings("unchecked")
268     protected STATE createSeekerState() {
269       // This will fail for non-default seeker state if the subclass does not
270       // override this method.
271       return (STATE) new SeekerState();
272     }
273 
274     abstract protected void decodeFirst();
275     abstract protected void decodeNext();
276   }
277 
278   protected final void afterEncodingKeyValue(ByteBuffer in,
279       DataOutputStream out, boolean includesMemstoreTS) {
280     if (includesMemstoreTS) {
281       // Copy memstore timestamp from the byte buffer to the output stream.
282       long memstoreTS = -1;
283       try {
284         memstoreTS = ByteBufferUtils.readVLong(in);
285         WritableUtils.writeVLong(out, memstoreTS);
286       } catch (IOException ex) {
287         throw new RuntimeException("Unable to copy memstore timestamp " +
288             memstoreTS + " after encoding a key/value");
289       }
290     }
291   }
292 
293   protected final void afterDecodingKeyValue(DataInputStream source,
294       ByteBuffer dest, boolean includesMemstoreTS) {
295     if (includesMemstoreTS) {
296       long memstoreTS = -1;
297       try {
298         // Copy memstore timestamp from the data input stream to the byte
299         // buffer.
300         memstoreTS = WritableUtils.readVLong(source);
301         ByteBufferUtils.writeVLong(dest, memstoreTS);
302       } catch (IOException ex) {
303         throw new RuntimeException("Unable to copy memstore timestamp " +
304             memstoreTS + " after decoding a key/value");
305       }
306     }
307   }
308 
309   @Override
310   public HFileBlockEncodingContext newDataBlockEncodingContext(
311       Algorithm compressionAlgorithm,
312       DataBlockEncoding encoding, byte[] header) {
313     return new HFileBlockDefaultEncodingContext(
314         compressionAlgorithm, encoding, header);
315   }
316 
317   @Override
318   public HFileBlockDecodingContext newDataBlockDecodingContext(
319       Algorithm compressionAlgorithm) {
320     return new HFileBlockDefaultDecodingContext(compressionAlgorithm);
321   }
322 
323   /**
324    * Compress KeyValues and write them to output buffer.
325    * @param out Where to write compressed data.
326    * @param in Source of KeyValue for compression.
327    * @param includesMemstoreTS true if including memstore timestamp after every
328    *          key-value pair
329    * @throws IOException If there is an error writing to output stream.
330    */
331   public abstract void internalEncodeKeyValues(DataOutputStream out,
332       ByteBuffer in, boolean includesMemstoreTS) throws IOException;
333 
334   @Override
335   public void encodeKeyValues(ByteBuffer in,
336       boolean includesMemstoreTS,
337       HFileBlockEncodingContext blkEncodingCtx) throws IOException {
338     if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
339       throw new IOException (this.getClass().getName() + " only accepts "
340           + HFileBlockDefaultEncodingContext.class.getName() + " as the " +
341           "encoding context.");
342     }
343 
344     HFileBlockDefaultEncodingContext encodingCtx =
345         (HFileBlockDefaultEncodingContext) blkEncodingCtx;
346     encodingCtx.prepareEncoding();
347     DataOutputStream dataOut =
348         ((HFileBlockDefaultEncodingContext) encodingCtx)
349         .getOutputStreamForEncoder();
350     internalEncodeKeyValues(dataOut, in, includesMemstoreTS);
351     if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) {
352       encodingCtx.postEncoding(BlockType.ENCODED_DATA);
353     } else {
354       encodingCtx.postEncoding(BlockType.DATA);
355     }
356   }
357 
358   /**
359    * Asserts that there is at least the given amount of unfilled space
360    * remaining in the given buffer.
361    * @param out typically, the buffer we are writing to
362    * @param length the required space in the buffer
363    * @throws EncoderBufferTooSmallException If there are no enough bytes.
364    */
365   protected static void ensureSpace(ByteBuffer out, int length)
366       throws EncoderBufferTooSmallException {
367     if (out.position() + length > out.limit()) {
368       throw new EncoderBufferTooSmallException(
369           "Buffer position=" + out.position() +
370           ", buffer limit=" + out.limit() +
371           ", length to be written=" + length);
372     }
373   }
374 
375 }