1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.io.encoding;
18
19 import java.io.DataInputStream;
20 import java.io.DataOutputStream;
21 import java.io.IOException;
22 import java.nio.ByteBuffer;
23
24 import org.apache.hadoop.hbase.classification.InterfaceAudience;
25 import org.apache.hadoop.hbase.HConstants;
26 import org.apache.hadoop.hbase.KeyValue;
27 import org.apache.hadoop.hbase.KeyValue.KVComparator;
28 import org.apache.hadoop.hbase.KeyValue.SamePrefixComparator;
29 import org.apache.hadoop.hbase.NoTagsKeyValue;
30 import org.apache.hadoop.hbase.io.TagCompressionContext;
31 import org.apache.hadoop.hbase.io.hfile.BlockType;
32 import org.apache.hadoop.hbase.io.hfile.HFileContext;
33 import org.apache.hadoop.hbase.io.util.LRUDictionary;
34 import org.apache.hadoop.hbase.util.ByteBufferUtils;
35 import org.apache.hadoop.hbase.util.Bytes;
36 import org.apache.hadoop.io.WritableUtils;
37
38
39
40
41 @InterfaceAudience.Private
42 abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
43
44 private static int INITIAL_KEY_BUFFER_SIZE = 512;
45
46 @Override
47 public ByteBuffer decodeKeyValues(DataInputStream source,
48 HFileBlockDecodingContext blkDecodingCtx) throws IOException {
49 if (blkDecodingCtx.getClass() != HFileBlockDefaultDecodingContext.class) {
50 throw new IOException(this.getClass().getName() + " only accepts "
51 + HFileBlockDefaultDecodingContext.class.getName() + " as the decoding context.");
52 }
53
54 HFileBlockDefaultDecodingContext decodingCtx =
55 (HFileBlockDefaultDecodingContext) blkDecodingCtx;
56 if (decodingCtx.getHFileContext().isIncludesTags()
57 && decodingCtx.getHFileContext().isCompressTags()) {
58 if (decodingCtx.getTagCompressionContext() != null) {
59
60
61 decodingCtx.getTagCompressionContext().clear();
62 } else {
63 try {
64 TagCompressionContext tagCompressionContext = new TagCompressionContext(
65 LRUDictionary.class, Byte.MAX_VALUE);
66 decodingCtx.setTagCompressionContext(tagCompressionContext);
67 } catch (Exception e) {
68 throw new IOException("Failed to initialize TagCompressionContext", e);
69 }
70 }
71 }
72 return internalDecodeKeyValues(source, 0, 0, decodingCtx);
73 }
74
75 protected static class SeekerState {
76 protected int valueOffset = -1;
77 protected int keyLength;
78 protected int valueLength;
79 protected int lastCommonPrefix;
80 protected int tagsLength = 0;
81 protected int tagsOffset = -1;
82 protected int tagsCompressedLength = 0;
83 protected boolean uncompressTags = true;
84
85
86 protected byte[] keyBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
87 protected byte[] tagsBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
88
89 protected long memstoreTS;
90 protected int nextKvOffset;
91
92 protected boolean isValid() {
93 return valueOffset != -1;
94 }
95
96 protected void invalidate() {
97 valueOffset = -1;
98 tagsCompressedLength = 0;
99 uncompressTags = true;
100 }
101
102 protected void ensureSpaceForKey() {
103 if (keyLength > keyBuffer.length) {
104
105 int newKeyBufferLength = Math.max(keyBuffer.length, 1) * 2;
106 while (keyLength > newKeyBufferLength) {
107 newKeyBufferLength *= 2;
108 }
109 byte[] newKeyBuffer = new byte[newKeyBufferLength];
110 System.arraycopy(keyBuffer, 0, newKeyBuffer, 0, keyBuffer.length);
111 keyBuffer = newKeyBuffer;
112 }
113 }
114
115 protected void ensureSpaceForTags() {
116 if (tagsLength > tagsBuffer.length) {
117
118 int newTagsBufferLength = Math.max(tagsBuffer.length, 1) * 2;
119 while (tagsLength > newTagsBufferLength) {
120 newTagsBufferLength *= 2;
121 }
122 byte[] newTagsBuffer = new byte[newTagsBufferLength];
123 System.arraycopy(tagsBuffer, 0, newTagsBuffer, 0, tagsBuffer.length);
124 tagsBuffer = newTagsBuffer;
125 }
126 }
127
128
129
130
131
132
133 protected void copyFromNext(SeekerState nextState) {
134 if (keyBuffer.length != nextState.keyBuffer.length) {
135 keyBuffer = nextState.keyBuffer.clone();
136 } else if (!isValid()) {
137
138
139 System.arraycopy(nextState.keyBuffer, 0, keyBuffer, 0,
140 nextState.keyLength);
141 } else {
142
143 System.arraycopy(nextState.keyBuffer, nextState.lastCommonPrefix,
144 keyBuffer, nextState.lastCommonPrefix, nextState.keyLength
145 - nextState.lastCommonPrefix);
146 }
147
148 valueOffset = nextState.valueOffset;
149 keyLength = nextState.keyLength;
150 valueLength = nextState.valueLength;
151 lastCommonPrefix = nextState.lastCommonPrefix;
152 nextKvOffset = nextState.nextKvOffset;
153 memstoreTS = nextState.memstoreTS;
154 }
155
156 }
157
158 protected abstract static class
159 BufferedEncodedSeeker<STATE extends SeekerState>
160 implements EncodedSeeker {
161 protected HFileBlockDecodingContext decodingCtx;
162 protected final KVComparator comparator;
163 protected final SamePrefixComparator<byte[]> samePrefixComparator;
164 protected ByteBuffer currentBuffer;
165 protected STATE current = createSeekerState();
166 protected STATE previous = createSeekerState();
167 protected TagCompressionContext tagCompressionContext = null;
168
169 public BufferedEncodedSeeker(KVComparator comparator,
170 HFileBlockDecodingContext decodingCtx) {
171 this.comparator = comparator;
172 this.samePrefixComparator = comparator;
173 this.decodingCtx = decodingCtx;
174 if (decodingCtx.getHFileContext().isCompressTags()) {
175 try {
176 tagCompressionContext = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE);
177 } catch (Exception e) {
178 throw new RuntimeException("Failed to initialize TagCompressionContext", e);
179 }
180 }
181 }
182
183 protected boolean includesMvcc() {
184 return this.decodingCtx.getHFileContext().isIncludesMvcc();
185 }
186
187 protected boolean includesTags() {
188 return this.decodingCtx.getHFileContext().isIncludesTags();
189 }
190
191 @Override
192 public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
193 return comparator.compareFlatKey(key, offset, length,
194 current.keyBuffer, 0, current.keyLength);
195 }
196
197 @Override
198 public void setCurrentBuffer(ByteBuffer buffer) {
199 if (this.tagCompressionContext != null) {
200 this.tagCompressionContext.clear();
201 }
202 currentBuffer = buffer;
203 decodeFirst();
204 previous.invalidate();
205 }
206
207 @Override
208 public ByteBuffer getKeyDeepCopy() {
209 ByteBuffer keyBuffer = ByteBuffer.allocate(current.keyLength);
210 keyBuffer.put(current.keyBuffer, 0, current.keyLength);
211 return keyBuffer;
212 }
213
214 @Override
215 public ByteBuffer getValueShallowCopy() {
216 return ByteBuffer.wrap(currentBuffer.array(),
217 currentBuffer.arrayOffset() + current.valueOffset,
218 current.valueLength);
219 }
220
221 @Override
222 public ByteBuffer getKeyValueBuffer() {
223 ByteBuffer kvBuffer = createKVBuffer();
224 kvBuffer.putInt(current.keyLength);
225 kvBuffer.putInt(current.valueLength);
226 kvBuffer.put(current.keyBuffer, 0, current.keyLength);
227 kvBuffer.put(currentBuffer.array(),
228 currentBuffer.arrayOffset() + current.valueOffset,
229 current.valueLength);
230 if (current.tagsLength > 0) {
231
232 kvBuffer.put((byte)(current.tagsLength >> 8 & 0xff));
233 kvBuffer.put((byte)(current.tagsLength & 0xff));
234 if (current.tagsOffset != -1) {
235
236
237 kvBuffer.put(currentBuffer.array(), currentBuffer.arrayOffset() + current.tagsOffset,
238 current.tagsLength);
239 } else {
240
241
242 kvBuffer.put(current.tagsBuffer, 0, current.tagsLength);
243 }
244 }
245 return kvBuffer;
246 }
247
248 protected ByteBuffer createKVBuffer() {
249 int kvBufSize = (int) KeyValue.getKeyValueDataStructureSize(current.keyLength,
250 current.valueLength, current.tagsLength);
251 ByteBuffer kvBuffer = ByteBuffer.allocate(kvBufSize);
252 return kvBuffer;
253 }
254
255 @Override
256 public KeyValue getKeyValue() {
257 ByteBuffer kvBuf = getKeyValueBuffer();
258 KeyValue kv;
259 if (current.tagsLength == 0) {
260 kv = new NoTagsKeyValue(kvBuf.array(), kvBuf.arrayOffset(), kvBuf.array().length
261 - kvBuf.arrayOffset());
262 } else {
263 kv = new KeyValue(kvBuf.array(), kvBuf.arrayOffset(), kvBuf.array().length
264 - kvBuf.arrayOffset());
265 }
266 kv.setMvccVersion(current.memstoreTS);
267 return kv;
268 }
269
270 @Override
271 public void rewind() {
272 currentBuffer.rewind();
273 if (tagCompressionContext != null) {
274 tagCompressionContext.clear();
275 }
276 decodeFirst();
277 previous.invalidate();
278 }
279
280 @Override
281 public boolean next() {
282 if (!currentBuffer.hasRemaining()) {
283 return false;
284 }
285 decodeNext();
286 previous.invalidate();
287 return true;
288 }
289
290 protected void decodeTags() {
291 current.tagsLength = ByteBufferUtils.readCompressedInt(currentBuffer);
292 if (tagCompressionContext != null) {
293 if (current.uncompressTags) {
294
295 current.ensureSpaceForTags();
296 try {
297 current.tagsCompressedLength = tagCompressionContext.uncompressTags(currentBuffer,
298 current.tagsBuffer, 0, current.tagsLength);
299 } catch (IOException e) {
300 throw new RuntimeException("Exception while uncompressing tags", e);
301 }
302 } else {
303 ByteBufferUtils.skip(currentBuffer, current.tagsCompressedLength);
304 current.uncompressTags = true;
305 }
306 current.tagsOffset = -1;
307 } else {
308
309
310 current.tagsOffset = currentBuffer.position();
311 ByteBufferUtils.skip(currentBuffer, current.tagsLength);
312 }
313 }
314
315 @Override
316 public int seekToKeyInBlock(byte[] key, int offset, int length,
317 boolean seekBefore) {
318 int commonPrefix = 0;
319 previous.invalidate();
320 do {
321 int comp;
322 if (samePrefixComparator != null) {
323 commonPrefix = Math.min(commonPrefix, current.lastCommonPrefix);
324
325
326 commonPrefix += ByteBufferUtils.findCommonPrefix(
327 key, offset + commonPrefix, length - commonPrefix,
328 current.keyBuffer, commonPrefix,
329 current.keyLength - commonPrefix);
330
331 comp = samePrefixComparator.compareIgnoringPrefix(commonPrefix, key,
332 offset, length, current.keyBuffer, 0, current.keyLength);
333 } else {
334 comp = comparator.compareFlatKey(key, offset, length,
335 current.keyBuffer, 0, current.keyLength);
336 }
337
338 if (comp == 0) {
339 if (seekBefore) {
340 if (!previous.isValid()) {
341
342
343 throw new IllegalStateException("Cannot seekBefore if " +
344 "positioned at the first key in the block: key=" +
345 Bytes.toStringBinary(key, offset, length));
346 }
347 moveToPrevious();
348 return 1;
349 }
350 return 0;
351 }
352
353 if (comp < 0) {
354 if (previous.isValid()) {
355 moveToPrevious();
356 } else {
357 return HConstants.INDEX_KEY_MAGIC;
358 }
359 return 1;
360 }
361
362
363 if (currentBuffer.hasRemaining()) {
364 previous.copyFromNext(current);
365 decodeNext();
366 } else {
367 break;
368 }
369 } while (true);
370
371
372 return 1;
373 }
374
375 private void moveToPrevious() {
376 if (!previous.isValid()) {
377 throw new IllegalStateException(
378 "Can move back only once and not in first key in the block.");
379 }
380
381 STATE tmp = previous;
382 previous = current;
383 current = tmp;
384
385
386 currentBuffer.position(current.nextKvOffset);
387
388
389
390
391
392
393 current.tagsBuffer = previous.tagsBuffer;
394 current.tagsCompressedLength = previous.tagsCompressedLength;
395 current.uncompressTags = false;
396 previous.invalidate();
397 }
398
399 @SuppressWarnings("unchecked")
400 protected STATE createSeekerState() {
401
402
403 return (STATE) new SeekerState();
404 }
405
406 abstract protected void decodeFirst();
407 abstract protected void decodeNext();
408 }
409
410 protected final void afterEncodingKeyValue(ByteBuffer in,
411 DataOutputStream out, HFileBlockDefaultEncodingContext encodingCtx) throws IOException {
412 if (encodingCtx.getHFileContext().isIncludesTags()) {
413
414 int tagsLength = ((in.get() & 0xff) << 8) ^ (in.get() & 0xff);
415 ByteBufferUtils.putCompressedInt(out, tagsLength);
416
417 if (tagsLength > 0) {
418 TagCompressionContext tagCompressionContext = encodingCtx.getTagCompressionContext();
419
420
421 if (tagCompressionContext != null) {
422 tagCompressionContext.compressTags(out, in, tagsLength);
423 } else {
424 ByteBufferUtils.moveBufferToStream(out, in, tagsLength);
425 }
426 }
427 }
428 if (encodingCtx.getHFileContext().isIncludesMvcc()) {
429
430 long memstoreTS = -1;
431 try {
432 memstoreTS = ByteBufferUtils.readVLong(in);
433 WritableUtils.writeVLong(out, memstoreTS);
434 } catch (IOException ex) {
435 throw new RuntimeException("Unable to copy memstore timestamp " +
436 memstoreTS + " after encoding a key/value");
437 }
438 }
439 }
440
441 protected final void afterDecodingKeyValue(DataInputStream source,
442 ByteBuffer dest, HFileBlockDefaultDecodingContext decodingCtx) throws IOException {
443 if (decodingCtx.getHFileContext().isIncludesTags()) {
444 int tagsLength = ByteBufferUtils.readCompressedInt(source);
445
446 dest.put((byte)((tagsLength >> 8) & 0xff));
447 dest.put((byte)(tagsLength & 0xff));
448 if (tagsLength > 0) {
449 TagCompressionContext tagCompressionContext = decodingCtx.getTagCompressionContext();
450
451
452 if (tagCompressionContext != null) {
453 tagCompressionContext.uncompressTags(source, dest, tagsLength);
454 } else {
455 ByteBufferUtils.copyFromStreamToBuffer(dest, source, tagsLength);
456 }
457 }
458 }
459 if (decodingCtx.getHFileContext().isIncludesMvcc()) {
460 long memstoreTS = -1;
461 try {
462
463
464 memstoreTS = WritableUtils.readVLong(source);
465 ByteBufferUtils.writeVLong(dest, memstoreTS);
466 } catch (IOException ex) {
467 throw new RuntimeException("Unable to copy memstore timestamp " +
468 memstoreTS + " after decoding a key/value");
469 }
470 }
471 }
472
473 @Override
474 public HFileBlockEncodingContext newDataBlockEncodingContext(DataBlockEncoding encoding,
475 byte[] header, HFileContext meta) {
476 return new HFileBlockDefaultEncodingContext(encoding, header, meta);
477 }
478
479 @Override
480 public HFileBlockDecodingContext newDataBlockDecodingContext(HFileContext meta) {
481 return new HFileBlockDefaultDecodingContext(meta);
482 }
483
484
485
486
487
488
489
490
491 public abstract void internalEncodeKeyValues(DataOutputStream out,
492 ByteBuffer in, HFileBlockDefaultEncodingContext encodingCtx) throws IOException;
493
494 protected abstract ByteBuffer internalDecodeKeyValues(DataInputStream source,
495 int allocateHeaderLength, int skipLastBytes, HFileBlockDefaultDecodingContext decodingCtx)
496 throws IOException;
497
498 @Override
499 public void encodeKeyValues(ByteBuffer in,
500 HFileBlockEncodingContext blkEncodingCtx) throws IOException {
501 if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
502 throw new IOException (this.getClass().getName() + " only accepts "
503 + HFileBlockDefaultEncodingContext.class.getName() + " as the " +
504 "encoding context.");
505 }
506
507 HFileBlockDefaultEncodingContext encodingCtx =
508 (HFileBlockDefaultEncodingContext) blkEncodingCtx;
509 encodingCtx.prepareEncoding();
510 DataOutputStream dataOut = encodingCtx.getOutputStreamForEncoder();
511 if (encodingCtx.getHFileContext().isIncludesTags()
512 && encodingCtx.getHFileContext().isCompressTags()) {
513 if (encodingCtx.getTagCompressionContext() != null) {
514
515
516 encodingCtx.getTagCompressionContext().clear();
517 } else {
518 try {
519 TagCompressionContext tagCompressionContext = new TagCompressionContext(
520 LRUDictionary.class, Byte.MAX_VALUE);
521 encodingCtx.setTagCompressionContext(tagCompressionContext);
522 } catch (Exception e) {
523 throw new IOException("Failed to initialize TagCompressionContext", e);
524 }
525 }
526 }
527 internalEncodeKeyValues(dataOut, in, encodingCtx);
528 if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) {
529 encodingCtx.postEncoding(BlockType.ENCODED_DATA);
530 } else {
531 encodingCtx.postEncoding(BlockType.DATA);
532 }
533 }
534
535
536
537
538
539
540
541
542 protected static void ensureSpace(ByteBuffer out, int length)
543 throws EncoderBufferTooSmallException {
544 if (out.position() + length > out.limit()) {
545 throw new EncoderBufferTooSmallException(
546 "Buffer position=" + out.position() +
547 ", buffer limit=" + out.limit() +
548 ", length to be written=" + length);
549 }
550 }
551
552 }