1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.io.encoding;
18
19 import java.io.DataInputStream;
20 import java.io.DataOutputStream;
21 import java.io.IOException;
22 import java.nio.ByteBuffer;
23
24 import org.apache.hadoop.classification.InterfaceAudience;
25 import org.apache.hadoop.hbase.HConstants;
26 import org.apache.hadoop.hbase.KeyValue;
27 import org.apache.hadoop.hbase.KeyValue.KVComparator;
28 import org.apache.hadoop.hbase.KeyValue.SamePrefixComparator;
29 import org.apache.hadoop.hbase.io.TagCompressionContext;
30 import org.apache.hadoop.hbase.io.hfile.BlockType;
31 import org.apache.hadoop.hbase.io.hfile.HFileContext;
32 import org.apache.hadoop.hbase.io.util.LRUDictionary;
33 import org.apache.hadoop.hbase.util.ByteBufferUtils;
34 import org.apache.hadoop.hbase.util.Bytes;
35 import org.apache.hadoop.io.WritableUtils;
36
37
38
39
40 @InterfaceAudience.Private
41 abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
42
43 private static int INITIAL_KEY_BUFFER_SIZE = 512;
44
45 @Override
46 public ByteBuffer decodeKeyValues(DataInputStream source,
47 HFileBlockDecodingContext blkDecodingCtx) throws IOException {
48 if (blkDecodingCtx.getClass() != HFileBlockDefaultDecodingContext.class) {
49 throw new IOException(this.getClass().getName() + " only accepts "
50 + HFileBlockDefaultDecodingContext.class.getName() + " as the decoding context.");
51 }
52
53 HFileBlockDefaultDecodingContext decodingCtx =
54 (HFileBlockDefaultDecodingContext) blkDecodingCtx;
55 if (decodingCtx.getHFileContext().isCompressTags()) {
56 try {
57 TagCompressionContext tagCompressionContext = new TagCompressionContext(LRUDictionary.class);
58 decodingCtx.setTagCompressionContext(tagCompressionContext);
59 } catch (Exception e) {
60 throw new IOException("Failed to initialize TagCompressionContext", e);
61 }
62 }
63 return internalDecodeKeyValues(source, 0, 0, decodingCtx);
64 }
65
66 protected static class SeekerState {
67 protected int valueOffset = -1;
68 protected int keyLength;
69 protected int valueLength;
70 protected int lastCommonPrefix;
71 protected int tagsLength = 0;
72 protected int tagsOffset = -1;
73
74
75 protected byte[] keyBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
76 protected byte[] tagsBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
77
78 protected long memstoreTS;
79 protected int nextKvOffset;
80
81 protected boolean isValid() {
82 return valueOffset != -1;
83 }
84
85 protected void invalidate() {
86 valueOffset = -1;
87 }
88
89 protected void ensureSpaceForKey() {
90 if (keyLength > keyBuffer.length) {
91
92 int newKeyBufferLength = Math.max(keyBuffer.length, 1) * 2;
93 while (keyLength > newKeyBufferLength) {
94 newKeyBufferLength *= 2;
95 }
96 byte[] newKeyBuffer = new byte[newKeyBufferLength];
97 System.arraycopy(keyBuffer, 0, newKeyBuffer, 0, keyBuffer.length);
98 keyBuffer = newKeyBuffer;
99 }
100 }
101
102 protected void ensureSpaceForTags() {
103 if (tagsLength > tagsBuffer.length) {
104
105 int newTagsBufferLength = Math.max(tagsBuffer.length, 1) * 2;
106 while (tagsLength > newTagsBufferLength) {
107 newTagsBufferLength *= 2;
108 }
109 byte[] newTagsBuffer = new byte[newTagsBufferLength];
110 System.arraycopy(tagsBuffer, 0, newTagsBuffer, 0, tagsBuffer.length);
111 tagsBuffer = newTagsBuffer;
112 }
113 }
114
115
116
117
118
119
120 protected void copyFromNext(SeekerState nextState) {
121 if (keyBuffer.length != nextState.keyBuffer.length) {
122 keyBuffer = nextState.keyBuffer.clone();
123 } else if (!isValid()) {
124
125
126 System.arraycopy(nextState.keyBuffer, 0, keyBuffer, 0,
127 nextState.keyLength);
128 } else {
129
130 System.arraycopy(nextState.keyBuffer, nextState.lastCommonPrefix,
131 keyBuffer, nextState.lastCommonPrefix, nextState.keyLength
132 - nextState.lastCommonPrefix);
133 }
134
135 valueOffset = nextState.valueOffset;
136 keyLength = nextState.keyLength;
137 valueLength = nextState.valueLength;
138 lastCommonPrefix = nextState.lastCommonPrefix;
139 nextKvOffset = nextState.nextKvOffset;
140 memstoreTS = nextState.memstoreTS;
141 }
142
143 }
144
145 protected abstract static class
146 BufferedEncodedSeeker<STATE extends SeekerState>
147 implements EncodedSeeker {
148 protected HFileBlockDecodingContext decodingCtx;
149 protected final KVComparator comparator;
150 protected final SamePrefixComparator<byte[]> samePrefixComparator;
151 protected ByteBuffer currentBuffer;
152 protected STATE current = createSeekerState();
153 protected STATE previous = createSeekerState();
154 protected TagCompressionContext tagCompressionContext = null;
155
156 public BufferedEncodedSeeker(KVComparator comparator,
157 HFileBlockDecodingContext decodingCtx) {
158 this.comparator = comparator;
159 this.samePrefixComparator = comparator;
160 this.decodingCtx = decodingCtx;
161 if (decodingCtx.getHFileContext().isCompressTags()) {
162 try {
163 tagCompressionContext = new TagCompressionContext(LRUDictionary.class);
164 } catch (Exception e) {
165 throw new RuntimeException("Failed to initialize TagCompressionContext", e);
166 }
167 }
168 }
169
170 protected boolean includesMvcc() {
171 return this.decodingCtx.getHFileContext().isIncludesMvcc();
172 }
173
174 protected boolean includesTags() {
175 return this.decodingCtx.getHFileContext().isIncludesTags();
176 }
177
178 @Override
179 public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
180 return comparator.compareFlatKey(key, offset, length,
181 current.keyBuffer, 0, current.keyLength);
182 }
183
184 @Override
185 public void setCurrentBuffer(ByteBuffer buffer) {
186 if (this.tagCompressionContext != null) {
187 this.tagCompressionContext.clear();
188 }
189 currentBuffer = buffer;
190 decodeFirst();
191 previous.invalidate();
192 }
193
194 @Override
195 public ByteBuffer getKeyDeepCopy() {
196 ByteBuffer keyBuffer = ByteBuffer.allocate(current.keyLength);
197 keyBuffer.put(current.keyBuffer, 0, current.keyLength);
198 return keyBuffer;
199 }
200
201 @Override
202 public ByteBuffer getValueShallowCopy() {
203 return ByteBuffer.wrap(currentBuffer.array(),
204 currentBuffer.arrayOffset() + current.valueOffset,
205 current.valueLength);
206 }
207
208 @Override
209 public ByteBuffer getKeyValueBuffer() {
210 ByteBuffer kvBuffer = createKVBuffer();
211 kvBuffer.putInt(current.keyLength);
212 kvBuffer.putInt(current.valueLength);
213 kvBuffer.put(current.keyBuffer, 0, current.keyLength);
214 kvBuffer.put(currentBuffer.array(),
215 currentBuffer.arrayOffset() + current.valueOffset,
216 current.valueLength);
217 if (current.tagsLength > 0) {
218 kvBuffer.putShort((short) current.tagsLength);
219 if (current.tagsOffset != -1) {
220
221
222 kvBuffer.put(currentBuffer.array(), currentBuffer.arrayOffset() + current.tagsOffset,
223 current.tagsLength);
224 } else {
225
226
227 kvBuffer.put(current.tagsBuffer, 0, current.tagsLength);
228 }
229 }
230 return kvBuffer;
231 }
232
233 protected ByteBuffer createKVBuffer() {
234 int kvBufSize = (int) KeyValue.getKeyValueDataStructureSize(current.keyLength,
235 current.valueLength, current.tagsLength);
236 ByteBuffer kvBuffer = ByteBuffer.allocate(kvBufSize);
237 return kvBuffer;
238 }
239
240 @Override
241 public KeyValue getKeyValue() {
242 ByteBuffer kvBuf = getKeyValueBuffer();
243 KeyValue kv = new KeyValue(kvBuf.array(), kvBuf.arrayOffset(), kvBuf.array().length
244 - kvBuf.arrayOffset());
245 kv.setMvccVersion(current.memstoreTS);
246 return kv;
247 }
248
249 @Override
250 public void rewind() {
251 currentBuffer.rewind();
252 decodeFirst();
253 previous.invalidate();
254 }
255
256 @Override
257 public boolean next() {
258 if (!currentBuffer.hasRemaining()) {
259 return false;
260 }
261 decodeNext();
262 previous.invalidate();
263 return true;
264 }
265
266 protected void decodeTags() {
267 current.tagsLength = ByteBufferUtils.readCompressedInt(currentBuffer);
268 if (tagCompressionContext != null) {
269
270 current.ensureSpaceForTags();
271 try {
272 tagCompressionContext.uncompressTags(currentBuffer, current.tagsBuffer, 0,
273 current.tagsLength);
274 } catch (IOException e) {
275 throw new RuntimeException("Exception while uncompressing tags", e);
276 }
277 current.tagsOffset = -1;
278 } else {
279
280
281 current.tagsOffset = currentBuffer.position();
282 ByteBufferUtils.skip(currentBuffer, current.tagsLength);
283 }
284 }
285
286 @Override
287 public int seekToKeyInBlock(byte[] key, int offset, int length,
288 boolean seekBefore) {
289 int commonPrefix = 0;
290 previous.invalidate();
291 do {
292 int comp;
293 if (samePrefixComparator != null) {
294 commonPrefix = Math.min(commonPrefix, current.lastCommonPrefix);
295
296
297 commonPrefix += ByteBufferUtils.findCommonPrefix(
298 key, offset + commonPrefix, length - commonPrefix,
299 current.keyBuffer, commonPrefix,
300 current.keyLength - commonPrefix);
301
302 comp = samePrefixComparator.compareIgnoringPrefix(commonPrefix, key,
303 offset, length, current.keyBuffer, 0, current.keyLength);
304 } else {
305 comp = comparator.compareFlatKey(key, offset, length,
306 current.keyBuffer, 0, current.keyLength);
307 }
308
309 if (comp == 0) {
310 if (seekBefore) {
311 if (!previous.isValid()) {
312
313
314 throw new IllegalStateException("Cannot seekBefore if " +
315 "positioned at the first key in the block: key=" +
316 Bytes.toStringBinary(key, offset, length));
317 }
318 moveToPrevious();
319 return 1;
320 }
321 return 0;
322 }
323
324 if (comp < 0) {
325 if (previous.isValid()) {
326 moveToPrevious();
327 } else {
328 return HConstants.INDEX_KEY_MAGIC;
329 }
330 return 1;
331 }
332
333
334 if (currentBuffer.hasRemaining()) {
335 previous.copyFromNext(current);
336 decodeNext();
337 } else {
338 break;
339 }
340 } while (true);
341
342
343 return 1;
344 }
345
346 private void moveToPrevious() {
347 if (!previous.isValid()) {
348 throw new IllegalStateException(
349 "Can move back only once and not in first key in the block.");
350 }
351
352 STATE tmp = previous;
353 previous = current;
354 current = tmp;
355
356
357 currentBuffer.position(current.nextKvOffset);
358
359 previous.invalidate();
360 }
361
362 @SuppressWarnings("unchecked")
363 protected STATE createSeekerState() {
364
365
366 return (STATE) new SeekerState();
367 }
368
369 abstract protected void decodeFirst();
370 abstract protected void decodeNext();
371 }
372
373 protected final void afterEncodingKeyValue(ByteBuffer in,
374 DataOutputStream out, HFileBlockDefaultEncodingContext encodingCtx) throws IOException {
375 if (encodingCtx.getHFileContext().isIncludesTags()) {
376 short tagsLength = in.getShort();
377 ByteBufferUtils.putCompressedInt(out, tagsLength);
378
379 if (tagsLength > 0) {
380 TagCompressionContext tagCompressionContext = encodingCtx.getTagCompressionContext();
381
382
383 if (tagCompressionContext != null) {
384 tagCompressionContext.compressTags(out, in, tagsLength);
385 } else {
386 ByteBufferUtils.moveBufferToStream(out, in, tagsLength);
387 }
388 }
389 }
390 if (encodingCtx.getHFileContext().isIncludesMvcc()) {
391
392 long memstoreTS = -1;
393 try {
394 memstoreTS = ByteBufferUtils.readVLong(in);
395 WritableUtils.writeVLong(out, memstoreTS);
396 } catch (IOException ex) {
397 throw new RuntimeException("Unable to copy memstore timestamp " +
398 memstoreTS + " after encoding a key/value");
399 }
400 }
401 }
402
403 protected final void afterDecodingKeyValue(DataInputStream source,
404 ByteBuffer dest, HFileBlockDefaultDecodingContext decodingCtx) throws IOException {
405 if (decodingCtx.getHFileContext().isIncludesTags()) {
406 short tagsLength = (short) ByteBufferUtils.readCompressedInt(source);
407 dest.putShort(tagsLength);
408 if (tagsLength > 0) {
409 TagCompressionContext tagCompressionContext = decodingCtx.getTagCompressionContext();
410
411
412 if (tagCompressionContext != null) {
413 tagCompressionContext.uncompressTags(source, dest, tagsLength);
414 } else {
415 ByteBufferUtils.copyFromStreamToBuffer(dest, source, tagsLength);
416 }
417 }
418 }
419 if (decodingCtx.getHFileContext().isIncludesMvcc()) {
420 long memstoreTS = -1;
421 try {
422
423
424 memstoreTS = WritableUtils.readVLong(source);
425 ByteBufferUtils.writeVLong(dest, memstoreTS);
426 } catch (IOException ex) {
427 throw new RuntimeException("Unable to copy memstore timestamp " +
428 memstoreTS + " after decoding a key/value");
429 }
430 }
431 }
432
433 @Override
434 public HFileBlockEncodingContext newDataBlockEncodingContext(DataBlockEncoding encoding,
435 byte[] header, HFileContext meta) {
436 return new HFileBlockDefaultEncodingContext(encoding, header, meta);
437 }
438
439 @Override
440 public HFileBlockDecodingContext newDataBlockDecodingContext(HFileContext meta) {
441 return new HFileBlockDefaultDecodingContext(meta);
442 }
443
444
445
446
447
448
449
450
451 public abstract void internalEncodeKeyValues(DataOutputStream out,
452 ByteBuffer in, HFileBlockDefaultEncodingContext encodingCtx) throws IOException;
453
454 protected abstract ByteBuffer internalDecodeKeyValues(DataInputStream source,
455 int allocateHeaderLength, int skipLastBytes, HFileBlockDefaultDecodingContext decodingCtx)
456 throws IOException;
457
458 @Override
459 public void encodeKeyValues(ByteBuffer in,
460 HFileBlockEncodingContext blkEncodingCtx) throws IOException {
461 if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
462 throw new IOException (this.getClass().getName() + " only accepts "
463 + HFileBlockDefaultEncodingContext.class.getName() + " as the " +
464 "encoding context.");
465 }
466
467 HFileBlockDefaultEncodingContext encodingCtx =
468 (HFileBlockDefaultEncodingContext) blkEncodingCtx;
469 encodingCtx.prepareEncoding();
470 DataOutputStream dataOut = encodingCtx.getOutputStreamForEncoder();
471 if (encodingCtx.getHFileContext().isCompressTags()) {
472 try {
473 TagCompressionContext tagCompressionContext = new TagCompressionContext(LRUDictionary.class);
474 encodingCtx.setTagCompressionContext(tagCompressionContext);
475 } catch (Exception e) {
476 throw new IOException("Failed to initialize TagCompressionContext", e);
477 }
478 }
479 internalEncodeKeyValues(dataOut, in, encodingCtx);
480 if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) {
481 encodingCtx.postEncoding(BlockType.ENCODED_DATA);
482 } else {
483 encodingCtx.postEncoding(BlockType.DATA);
484 }
485 }
486
487
488
489
490
491
492
493
494 protected static void ensureSpace(ByteBuffer out, int length)
495 throws EncoderBufferTooSmallException {
496 if (out.position() + length > out.limit()) {
497 throw new EncoderBufferTooSmallException(
498 "Buffer position=" + out.position() +
499 ", buffer limit=" + out.limit() +
500 ", length to be written=" + length);
501 }
502 }
503
504 }