View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations
15   * under the License.
16   */
17  package org.apache.hadoop.hbase.io.encoding;
18  
19  import java.io.DataInputStream;
20  import java.io.DataOutputStream;
21  import java.io.IOException;
22  import java.nio.ByteBuffer;
23  
24  import org.apache.hadoop.hbase.KeyValue;
25  import org.apache.hadoop.hbase.KeyValue.SamePrefixComparator;
26  import org.apache.hadoop.hbase.util.ByteBufferUtils;
27  import org.apache.hadoop.hbase.util.Bytes;
28  import org.apache.hadoop.io.RawComparator;
29  import org.apache.hadoop.io.WritableUtils;
30  
31  /**
32   * Base class for all data block encoders that use a buffer.
33   */
34  abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
35  
36    private static int INITIAL_KEY_BUFFER_SIZE = 512;
37  
38    @Override
39    public ByteBuffer uncompressKeyValues(DataInputStream source,
40        boolean includesMemstoreTS) throws IOException {
41      return uncompressKeyValues(source, 0, 0, includesMemstoreTS);
42    }
43  
44    protected static class SeekerState {
45      protected int valueOffset = -1;
46      protected int keyLength;
47      protected int valueLength;
48      protected int lastCommonPrefix;
49  
50      /** We need to store a copy of the key. */
51      protected byte[] keyBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
52  
53      protected long memstoreTS;
54      protected int nextKvOffset;
55  
56      protected boolean isValid() {
57        return valueOffset != -1;
58      }
59  
60      protected void invalidate() {
61        valueOffset = -1;
62      }
63  
64      protected void ensureSpaceForKey() {
65        if (keyLength > keyBuffer.length) {
66          // rare case, but we need to handle arbitrary length of key
67          int newKeyBufferLength = Math.max(keyBuffer.length, 1) * 2;
68          while (keyLength > newKeyBufferLength) {
69            newKeyBufferLength *= 2;
70          }
71          byte[] newKeyBuffer = new byte[newKeyBufferLength];
72          System.arraycopy(keyBuffer, 0, newKeyBuffer, 0, keyBuffer.length);
73          keyBuffer = newKeyBuffer;
74        }
75      }
76  
77      /**
78       * Copy the state from the next one into this instance (the previous state
79       * placeholder). Used to save the previous state when we are advancing the
80       * seeker to the next key/value.
81       */
82      protected void copyFromNext(SeekerState nextState) {
83        if (keyBuffer.length != nextState.keyBuffer.length) {
84          keyBuffer = nextState.keyBuffer.clone();
85        } else if (!isValid()) {
86          // Note: we can only call isValid before we override our state, so this
87          // comes before all the assignments at the end of this method.
88          System.arraycopy(nextState.keyBuffer, 0, keyBuffer, 0,
89               nextState.keyLength);
90        } else {
91          // don't copy the common prefix between this key and the previous one
92          System.arraycopy(nextState.keyBuffer, nextState.lastCommonPrefix,
93              keyBuffer, nextState.lastCommonPrefix, nextState.keyLength
94                  - nextState.lastCommonPrefix);
95        }
96  
97        valueOffset = nextState.valueOffset;
98        keyLength = nextState.keyLength;
99        valueLength = nextState.valueLength;
100       lastCommonPrefix = nextState.lastCommonPrefix;
101       nextKvOffset = nextState.nextKvOffset;
102       memstoreTS = nextState.memstoreTS;
103     }
104 
105   }
106 
107   protected abstract static class
108       BufferedEncodedSeeker<STATE extends SeekerState>
109       implements EncodedSeeker {
110 
111     protected final RawComparator<byte[]> comparator;
112     protected final SamePrefixComparator<byte[]> samePrefixComparator;
113     protected ByteBuffer currentBuffer;
114     protected STATE current = createSeekerState(); // always valid
115     protected STATE previous = createSeekerState(); // may not be valid
116 
117     @SuppressWarnings("unchecked")
118     public BufferedEncodedSeeker(RawComparator<byte[]> comparator) {
119       this.comparator = comparator;
120       if (comparator instanceof SamePrefixComparator) {
121         this.samePrefixComparator = (SamePrefixComparator<byte[]>) comparator;
122       } else {
123         this.samePrefixComparator = null;
124       }
125     }
126 
127     @Override
128     public void setCurrentBuffer(ByteBuffer buffer) {
129       currentBuffer = buffer;
130       decodeFirst();
131       previous.invalidate();
132     }
133 
134     @Override
135     public ByteBuffer getKeyDeepCopy() {
136       ByteBuffer keyBuffer = ByteBuffer.allocate(current.keyLength);
137       keyBuffer.put(current.keyBuffer, 0, current.keyLength);
138       return keyBuffer;
139     }
140 
141     @Override
142     public ByteBuffer getValueShallowCopy() {
143       return ByteBuffer.wrap(currentBuffer.array(),
144           currentBuffer.arrayOffset() + current.valueOffset,
145           current.valueLength);
146     }
147 
148     @Override
149     public ByteBuffer getKeyValueBuffer() {
150       ByteBuffer kvBuffer = ByteBuffer.allocate(
151           2 * Bytes.SIZEOF_INT + current.keyLength + current.valueLength);
152       kvBuffer.putInt(current.keyLength);
153       kvBuffer.putInt(current.valueLength);
154       kvBuffer.put(current.keyBuffer, 0, current.keyLength);
155       kvBuffer.put(currentBuffer.array(),
156           currentBuffer.arrayOffset() + current.valueOffset,
157           current.valueLength);
158       return kvBuffer;
159     }
160 
161     @Override
162     public KeyValue getKeyValue() {
163       ByteBuffer kvBuf = getKeyValueBuffer();
164       KeyValue kv = new KeyValue(kvBuf.array(), kvBuf.arrayOffset());
165       kv.setMemstoreTS(current.memstoreTS);
166       return kv;
167     }
168 
169     @Override
170     public void rewind() {
171       currentBuffer.rewind();
172       decodeFirst();
173       previous.invalidate();
174     }
175 
176     @Override
177     public boolean next() {
178       if (!currentBuffer.hasRemaining()) {
179         return false;
180       }
181       decodeNext();
182       previous.invalidate();
183       return true;
184     }
185 
186     @Override
187     public int seekToKeyInBlock(byte[] key, int offset, int length,
188         boolean seekBefore) {
189       int commonPrefix = 0;
190       previous.invalidate();
191       do {
192         int comp;
193         if (samePrefixComparator != null) {
194           commonPrefix = Math.min(commonPrefix, current.lastCommonPrefix);
195 
196           // extend commonPrefix
197           commonPrefix += ByteBufferUtils.findCommonPrefix(
198               key, offset + commonPrefix, length - commonPrefix,
199               current.keyBuffer, commonPrefix,
200               current.keyLength - commonPrefix);
201 
202           comp = samePrefixComparator.compareIgnoringPrefix(commonPrefix, key,
203               offset, length, current.keyBuffer, 0, current.keyLength);
204         } else {
205           comp = comparator.compare(key, offset, length,
206               current.keyBuffer, 0, current.keyLength);
207         }
208 
209         if (comp == 0) { // exact match
210           if (seekBefore) {
211             if (!previous.isValid()) {
212               // The caller (seekBefore) has to ensure that we are not at the
213               // first key in the block.
214               throw new IllegalStateException("Cannot seekBefore if " +
215                   "positioned at the first key in the block: key=" +
216                   Bytes.toStringBinary(key, offset, length));
217             }
218             moveToPrevious();
219             return 1;
220           }
221           return 0;
222         }
223 
224         if (comp < 0) { // already too large, check previous
225           if (previous.isValid()) {
226             moveToPrevious();
227           }
228           return 1;
229         }
230 
231         // move to next, if more data is available
232         if (currentBuffer.hasRemaining()) {
233           previous.copyFromNext(current);
234           decodeNext();
235         } else {
236           break;
237         }
238       } while (true);
239 
240       // we hit the end of the block, not an exact match
241       return 1;
242     }
243 
244     private void moveToPrevious() {
245       if (!previous.isValid()) {
246         throw new IllegalStateException(
247             "Can move back only once and not in first key in the block.");
248       }
249 
250       STATE tmp = previous;
251       previous = current;
252       current = tmp;
253 
254       // move after last key value
255       currentBuffer.position(current.nextKvOffset);
256 
257       previous.invalidate();
258     }
259 
260     @SuppressWarnings("unchecked")
261     protected STATE createSeekerState() {
262       // This will fail for non-default seeker state if the subclass does not
263       // override this method.
264       return (STATE) new SeekerState();
265     }
266 
267     abstract protected void decodeFirst();
268     abstract protected void decodeNext();
269   }
270 
271   protected final void afterEncodingKeyValue(ByteBuffer in,
272       DataOutputStream out, boolean includesMemstoreTS) {
273     if (includesMemstoreTS) {
274       // Copy memstore timestamp from the byte buffer to the output stream.
275       long memstoreTS = -1;
276       try {
277         memstoreTS = ByteBufferUtils.readVLong(in);
278         WritableUtils.writeVLong(out, memstoreTS);
279       } catch (IOException ex) {
280         throw new RuntimeException("Unable to copy memstore timestamp " +
281             memstoreTS + " after encoding a key/value");
282       }
283     }
284   }
285 
286   protected final void afterDecodingKeyValue(DataInputStream source,
287       ByteBuffer dest, boolean includesMemstoreTS) {
288     if (includesMemstoreTS) {
289       long memstoreTS = -1;
290       try {
291         // Copy memstore timestamp from the data input stream to the byte
292         // buffer.
293         memstoreTS = WritableUtils.readVLong(source);
294         ByteBufferUtils.writeVLong(dest, memstoreTS);
295       } catch (IOException ex) {
296         throw new RuntimeException("Unable to copy memstore timestamp " +
297             memstoreTS + " after decoding a key/value");
298       }
299     }
300   }
301 
302 }