1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.io.encoding;
18
19 import java.io.DataInputStream;
20 import java.io.DataOutputStream;
21 import java.io.IOException;
22 import java.nio.ByteBuffer;
23
24 import org.apache.hadoop.classification.InterfaceAudience;
25 import org.apache.hadoop.hbase.HConstants;
26 import org.apache.hadoop.hbase.KeyValue;
27 import org.apache.hadoop.hbase.KeyValue.KVComparator;
28 import org.apache.hadoop.hbase.KeyValue.SamePrefixComparator;
29 import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
30 import org.apache.hadoop.hbase.io.hfile.BlockType;
31 import org.apache.hadoop.hbase.util.ByteBufferUtils;
32 import org.apache.hadoop.hbase.util.Bytes;
33 import org.apache.hadoop.io.WritableUtils;
34
35
36
37
38 @InterfaceAudience.Private
39 abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
40
41 private static int INITIAL_KEY_BUFFER_SIZE = 512;
42
43 @Override
44 public ByteBuffer decodeKeyValues(DataInputStream source,
45 boolean includesMemstoreTS) throws IOException {
46 return decodeKeyValues(source, 0, 0, includesMemstoreTS);
47 }
48
49 protected static class SeekerState {
50 protected int valueOffset = -1;
51 protected int keyLength;
52 protected int valueLength;
53 protected int lastCommonPrefix;
54
55
56 protected byte[] keyBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
57
58 protected long memstoreTS;
59 protected int nextKvOffset;
60
61 protected boolean isValid() {
62 return valueOffset != -1;
63 }
64
65 protected void invalidate() {
66 valueOffset = -1;
67 }
68
69 protected void ensureSpaceForKey() {
70 if (keyLength > keyBuffer.length) {
71
72 int newKeyBufferLength = Math.max(keyBuffer.length, 1) * 2;
73 while (keyLength > newKeyBufferLength) {
74 newKeyBufferLength *= 2;
75 }
76 byte[] newKeyBuffer = new byte[newKeyBufferLength];
77 System.arraycopy(keyBuffer, 0, newKeyBuffer, 0, keyBuffer.length);
78 keyBuffer = newKeyBuffer;
79 }
80 }
81
82
83
84
85
86
87 protected void copyFromNext(SeekerState nextState) {
88 if (keyBuffer.length != nextState.keyBuffer.length) {
89 keyBuffer = nextState.keyBuffer.clone();
90 } else if (!isValid()) {
91
92
93 System.arraycopy(nextState.keyBuffer, 0, keyBuffer, 0,
94 nextState.keyLength);
95 } else {
96
97 System.arraycopy(nextState.keyBuffer, nextState.lastCommonPrefix,
98 keyBuffer, nextState.lastCommonPrefix, nextState.keyLength
99 - nextState.lastCommonPrefix);
100 }
101
102 valueOffset = nextState.valueOffset;
103 keyLength = nextState.keyLength;
104 valueLength = nextState.valueLength;
105 lastCommonPrefix = nextState.lastCommonPrefix;
106 nextKvOffset = nextState.nextKvOffset;
107 memstoreTS = nextState.memstoreTS;
108 }
109
110 }
111
112 protected abstract static class
113 BufferedEncodedSeeker<STATE extends SeekerState>
114 implements EncodedSeeker {
115
116 protected final KVComparator comparator;
117 protected final SamePrefixComparator<byte[]> samePrefixComparator;
118 protected ByteBuffer currentBuffer;
119 protected STATE current = createSeekerState();
120 protected STATE previous = createSeekerState();
121
122 @SuppressWarnings("unchecked")
123 public BufferedEncodedSeeker(KVComparator comparator) {
124 this.comparator = comparator;
125 if (comparator instanceof SamePrefixComparator) {
126 this.samePrefixComparator = (SamePrefixComparator<byte[]>) comparator;
127 } else {
128 this.samePrefixComparator = null;
129 }
130 }
131
132 @Override
133 public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
134 return comparator.compareFlatKey(key, offset, length,
135 current.keyBuffer, 0, current.keyLength);
136 }
137
138 @Override
139 public void setCurrentBuffer(ByteBuffer buffer) {
140 currentBuffer = buffer;
141 decodeFirst();
142 previous.invalidate();
143 }
144
145 @Override
146 public ByteBuffer getKeyDeepCopy() {
147 ByteBuffer keyBuffer = ByteBuffer.allocate(current.keyLength);
148 keyBuffer.put(current.keyBuffer, 0, current.keyLength);
149 return keyBuffer;
150 }
151
152 @Override
153 public ByteBuffer getValueShallowCopy() {
154 return ByteBuffer.wrap(currentBuffer.array(),
155 currentBuffer.arrayOffset() + current.valueOffset,
156 current.valueLength);
157 }
158
159 @Override
160 public ByteBuffer getKeyValueBuffer() {
161 ByteBuffer kvBuffer = ByteBuffer.allocate(
162 2 * Bytes.SIZEOF_INT + current.keyLength + current.valueLength);
163 kvBuffer.putInt(current.keyLength);
164 kvBuffer.putInt(current.valueLength);
165 kvBuffer.put(current.keyBuffer, 0, current.keyLength);
166 kvBuffer.put(currentBuffer.array(),
167 currentBuffer.arrayOffset() + current.valueOffset,
168 current.valueLength);
169 return kvBuffer;
170 }
171
172 @Override
173 public KeyValue getKeyValue() {
174 ByteBuffer kvBuf = getKeyValueBuffer();
175 KeyValue kv = new KeyValue(kvBuf.array(), kvBuf.arrayOffset());
176 kv.setMvccVersion(current.memstoreTS);
177 return kv;
178 }
179
180 @Override
181 public void rewind() {
182 currentBuffer.rewind();
183 decodeFirst();
184 previous.invalidate();
185 }
186
187 @Override
188 public boolean next() {
189 if (!currentBuffer.hasRemaining()) {
190 return false;
191 }
192 decodeNext();
193 previous.invalidate();
194 return true;
195 }
196
197 @Override
198 public int seekToKeyInBlock(byte[] key, int offset, int length,
199 boolean seekBefore) {
200 int commonPrefix = 0;
201 previous.invalidate();
202 do {
203 int comp;
204 if (samePrefixComparator != null) {
205 commonPrefix = Math.min(commonPrefix, current.lastCommonPrefix);
206
207
208 commonPrefix += ByteBufferUtils.findCommonPrefix(
209 key, offset + commonPrefix, length - commonPrefix,
210 current.keyBuffer, commonPrefix,
211 current.keyLength - commonPrefix);
212
213 comp = samePrefixComparator.compareIgnoringPrefix(commonPrefix, key,
214 offset, length, current.keyBuffer, 0, current.keyLength);
215 } else {
216 comp = comparator.compareFlatKey(key, offset, length,
217 current.keyBuffer, 0, current.keyLength);
218 }
219
220 if (comp == 0) {
221 if (seekBefore) {
222 if (!previous.isValid()) {
223
224
225 throw new IllegalStateException("Cannot seekBefore if " +
226 "positioned at the first key in the block: key=" +
227 Bytes.toStringBinary(key, offset, length));
228 }
229 moveToPrevious();
230 return 1;
231 }
232 return 0;
233 }
234
235 if (comp < 0) {
236 if (previous.isValid()) {
237 moveToPrevious();
238 } else {
239 return HConstants.INDEX_KEY_MAGIC;
240 }
241 return 1;
242 }
243
244
245 if (currentBuffer.hasRemaining()) {
246 previous.copyFromNext(current);
247 decodeNext();
248 } else {
249 break;
250 }
251 } while (true);
252
253
254 return 1;
255 }
256
257 private void moveToPrevious() {
258 if (!previous.isValid()) {
259 throw new IllegalStateException(
260 "Can move back only once and not in first key in the block.");
261 }
262
263 STATE tmp = previous;
264 previous = current;
265 current = tmp;
266
267
268 currentBuffer.position(current.nextKvOffset);
269
270 previous.invalidate();
271 }
272
273 @SuppressWarnings("unchecked")
274 protected STATE createSeekerState() {
275
276
277 return (STATE) new SeekerState();
278 }
279
280 abstract protected void decodeFirst();
281 abstract protected void decodeNext();
282 }
283
284 protected final void afterEncodingKeyValue(ByteBuffer in,
285 DataOutputStream out, boolean includesMemstoreTS) {
286 if (includesMemstoreTS) {
287
288 long memstoreTS = -1;
289 try {
290 memstoreTS = ByteBufferUtils.readVLong(in);
291 WritableUtils.writeVLong(out, memstoreTS);
292 } catch (IOException ex) {
293 throw new RuntimeException("Unable to copy memstore timestamp " +
294 memstoreTS + " after encoding a key/value");
295 }
296 }
297 }
298
299 protected final void afterDecodingKeyValue(DataInputStream source,
300 ByteBuffer dest, boolean includesMemstoreTS) {
301 if (includesMemstoreTS) {
302 long memstoreTS = -1;
303 try {
304
305
306 memstoreTS = WritableUtils.readVLong(source);
307 ByteBufferUtils.writeVLong(dest, memstoreTS);
308 } catch (IOException ex) {
309 throw new RuntimeException("Unable to copy memstore timestamp " +
310 memstoreTS + " after decoding a key/value");
311 }
312 }
313 }
314
315 @Override
316 public HFileBlockEncodingContext newDataBlockEncodingContext(
317 Algorithm compressionAlgorithm,
318 DataBlockEncoding encoding, byte[] header) {
319 return new HFileBlockDefaultEncodingContext(
320 compressionAlgorithm, encoding, header);
321 }
322
323 @Override
324 public HFileBlockDecodingContext newDataBlockDecodingContext(
325 Algorithm compressionAlgorithm) {
326 return new HFileBlockDefaultDecodingContext(compressionAlgorithm);
327 }
328
329
330
331
332
333
334
335
336
337 public abstract void internalEncodeKeyValues(DataOutputStream out,
338 ByteBuffer in, boolean includesMemstoreTS) throws IOException;
339
340 @Override
341 public void encodeKeyValues(ByteBuffer in,
342 boolean includesMemstoreTS,
343 HFileBlockEncodingContext blkEncodingCtx) throws IOException {
344 if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
345 throw new IOException (this.getClass().getName() + " only accepts "
346 + HFileBlockDefaultEncodingContext.class.getName() + " as the " +
347 "encoding context.");
348 }
349
350 HFileBlockDefaultEncodingContext encodingCtx =
351 (HFileBlockDefaultEncodingContext) blkEncodingCtx;
352 encodingCtx.prepareEncoding();
353 DataOutputStream dataOut =
354 ((HFileBlockDefaultEncodingContext) encodingCtx)
355 .getOutputStreamForEncoder();
356 internalEncodeKeyValues(dataOut, in, includesMemstoreTS);
357 if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) {
358 encodingCtx.postEncoding(BlockType.ENCODED_DATA);
359 } else {
360 encodingCtx.postEncoding(BlockType.DATA);
361 }
362 }
363
364
365
366
367
368
369
370
371 protected static void ensureSpace(ByteBuffer out, int length)
372 throws EncoderBufferTooSmallException {
373 if (out.position() + length > out.limit()) {
374 throw new EncoderBufferTooSmallException(
375 "Buffer position=" + out.position() +
376 ", buffer limit=" + out.limit() +
377 ", length to be written=" + length);
378 }
379 }
380
381 }