1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import java.io.IOException;
22 import java.io.OutputStream;
23 import java.nio.ByteBuffer;
24 import java.util.ArrayList;
25 import java.util.List;
26
27 import org.apache.hadoop.hbase.classification.InterfaceAudience;
28 import org.apache.hadoop.hbase.KeyValue.Type;
29 import org.apache.hadoop.hbase.io.util.StreamUtils;
30 import org.apache.hadoop.hbase.util.ByteBufferUtils;
31 import org.apache.hadoop.hbase.util.Bytes;
32 import org.apache.hadoop.hbase.util.IterableUtils;
33 import org.apache.hadoop.hbase.util.SimpleMutableByteRange;
34 import org.apache.hadoop.io.WritableUtils;
35
36 import com.google.common.base.Function;
37 import com.google.common.collect.Lists;
38
39
40
41
42 @InterfaceAudience.Private
43 public class KeyValueUtil {
44
45
46
47
48
49
50
51
52 public static int length(final Cell cell) {
53 return length(cell.getRowLength(), cell.getFamilyLength(), cell.getQualifierLength(),
54 cell.getValueLength(), cell.getTagsLength(), true);
55 }
56
57 private static int length(short rlen, byte flen, int qlen, int vlen, int tlen, boolean withTags) {
58 if (withTags) {
59 return (int) (KeyValue.getKeyValueDataStructureSize(rlen, flen, qlen, vlen, tlen));
60 }
61 return (int) (KeyValue.getKeyValueDataStructureSize(rlen, flen, qlen, vlen));
62 }
63
64
65
66
67
68
69
70 public static int keyLength(final Cell cell) {
71 return keyLength(cell.getRowLength(), cell.getFamilyLength(), cell.getQualifierLength());
72 }
73
74 private static int keyLength(short rlen, byte flen, int qlen) {
75 return (int) KeyValue.getKeyDataStructureSize(rlen, flen, qlen);
76 }
77
78 public static int lengthWithMvccVersion(final KeyValue kv, final boolean includeMvccVersion) {
79 int length = kv.getLength();
80 if (includeMvccVersion) {
81 length += WritableUtils.getVIntSize(kv.getMvccVersion());
82 }
83 return length;
84 }
85
86 public static int totalLengthWithMvccVersion(final Iterable<? extends KeyValue> kvs,
87 final boolean includeMvccVersion) {
88 int length = 0;
89 for (KeyValue kv : IterableUtils.nullSafe(kvs)) {
90 length += lengthWithMvccVersion(kv, includeMvccVersion);
91 }
92 return length;
93 }
94
95
96
97
98 public static KeyValue copyToNewKeyValue(final Cell cell) {
99 byte[] bytes = copyToNewByteArray(cell);
100 KeyValue kvCell = new KeyValue(bytes, 0, bytes.length);
101 kvCell.setSequenceId(cell.getMvccVersion());
102 return kvCell;
103 }
104
105 public static ByteBuffer copyKeyToNewByteBuffer(final Cell cell) {
106 byte[] bytes = new byte[keyLength(cell)];
107 appendKeyTo(cell, bytes, 0);
108 ByteBuffer buffer = ByteBuffer.wrap(bytes);
109 buffer.position(buffer.limit());
110 return buffer;
111 }
112
113 public static byte[] copyToNewByteArray(final Cell cell) {
114 int v1Length = length(cell);
115 byte[] backingBytes = new byte[v1Length];
116 appendToByteArray(cell, backingBytes, 0);
117 return backingBytes;
118 }
119
120 public static int appendKeyTo(final Cell cell, final byte[] output,
121 final int offset) {
122 int nextOffset = offset;
123 nextOffset = Bytes.putShort(output, nextOffset, cell.getRowLength());
124 nextOffset = CellUtil.copyRowTo(cell, output, nextOffset);
125 nextOffset = Bytes.putByte(output, nextOffset, cell.getFamilyLength());
126 nextOffset = CellUtil.copyFamilyTo(cell, output, nextOffset);
127 nextOffset = CellUtil.copyQualifierTo(cell, output, nextOffset);
128 nextOffset = Bytes.putLong(output, nextOffset, cell.getTimestamp());
129 nextOffset = Bytes.putByte(output, nextOffset, cell.getTypeByte());
130 return nextOffset;
131 }
132
133
134
135
136 public static int appendToByteArray(final Cell cell, final byte[] output, final int offset) {
137
138
139 int pos = offset;
140 pos = Bytes.putInt(output, pos, keyLength(cell));
141 pos = Bytes.putInt(output, pos, cell.getValueLength());
142 pos = appendKeyTo(cell, output, pos);
143 pos = CellUtil.copyValueTo(cell, output, pos);
144 if ((cell.getTagsLength() > 0)) {
145 pos = Bytes.putAsShort(output, pos, cell.getTagsLength());
146 pos = CellUtil.copyTagTo(cell, output, pos);
147 }
148 return pos;
149 }
150
151 public static ByteBuffer copyToNewByteBuffer(final Cell cell) {
152 byte[] bytes = new byte[length(cell)];
153 appendToByteArray(cell, bytes, 0);
154 ByteBuffer buffer = ByteBuffer.wrap(bytes);
155 buffer.position(buffer.limit());
156 return buffer;
157 }
158
159 public static void appendToByteBuffer(final ByteBuffer bb, final KeyValue kv,
160 final boolean includeMvccVersion) {
161
162 bb.limit(bb.position() + kv.getLength());
163 bb.put(kv.getBuffer(), kv.getOffset(), kv.getLength());
164 if (includeMvccVersion) {
165 int numMvccVersionBytes = WritableUtils.getVIntSize(kv.getMvccVersion());
166 ByteBufferUtils.extendLimit(bb, numMvccVersionBytes);
167 ByteBufferUtils.writeVLong(bb, kv.getMvccVersion());
168 }
169 }
170
171
172
173
174
175
176
177
178
179
180
181 public static KeyValue nextShallowCopy(final ByteBuffer bb, final boolean includesMvccVersion,
182 boolean includesTags) {
183 if (bb.isDirect()) {
184 throw new IllegalArgumentException("only supports heap buffers");
185 }
186 if (bb.remaining() < 1) {
187 return null;
188 }
189 KeyValue keyValue = null;
190 int underlyingArrayOffset = bb.arrayOffset() + bb.position();
191 int keyLength = bb.getInt();
192 int valueLength = bb.getInt();
193 ByteBufferUtils.skip(bb, keyLength + valueLength);
194 int tagsLength = 0;
195 if (includesTags) {
196
197 tagsLength = ((bb.get() & 0xff) << 8) ^ (bb.get() & 0xff);
198 ByteBufferUtils.skip(bb, tagsLength);
199 }
200 int kvLength = (int) KeyValue.getKeyValueDataStructureSize(keyLength, valueLength, tagsLength);
201 keyValue = new KeyValue(bb.array(), underlyingArrayOffset, kvLength);
202 if (includesMvccVersion) {
203 long mvccVersion = ByteBufferUtils.readVLong(bb);
204 keyValue.setSequenceId(mvccVersion);
205 }
206 return keyValue;
207 }
208
209
210
211
212
213
214
215 public static KeyValue createFirstKeyInNextRow(final Cell in){
216 byte[] nextRow = new byte[in.getRowLength() + 1];
217 System.arraycopy(in.getRowArray(), in.getRowOffset(), nextRow, 0, in.getRowLength());
218 nextRow[nextRow.length - 1] = 0;
219 return createFirstOnRow(nextRow);
220 }
221
222
223
224
225 public static KeyValue createFirstKeyInIncrementedRow(final Cell in){
226 byte[] thisRow = new SimpleMutableByteRange(in.getRowArray(), in.getRowOffset(),
227 in.getRowLength()).deepCopyToNewArray();
228 byte[] nextRow = Bytes.unsignedCopyAndIncrement(thisRow);
229 return createFirstOnRow(nextRow);
230 }
231
232
233
234
235
236
237
238
239 public static KeyValue previousKey(final KeyValue in) {
240 return createFirstOnRow(CellUtil.cloneRow(in), CellUtil.cloneFamily(in),
241 CellUtil.cloneQualifier(in), in.getTimestamp() - 1);
242 }
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270 public static KeyValue createLastOnRow(final byte[] row, final int roffset, final int rlength,
271 final byte[] family, final int foffset, final int flength, final byte[] qualifier,
272 final int qoffset, final int qlength) {
273 return new KeyValue(row, roffset, rlength, family, foffset, flength, qualifier, qoffset,
274 qlength, HConstants.OLDEST_TIMESTAMP, Type.Minimum, null, 0, 0);
275 }
276
277
278
279
280
281
282
283 public static KeyValue createLastOnRow(Cell kv) {
284 return createLastOnRow(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), null, 0, 0,
285 null, 0, 0);
286 }
287
288
289
290
291
292
293
294
295
296
297
298
299 public static KeyValue createLastOnRowCol(Cell kv) {
300 return new KeyValue(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
301 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), kv.getQualifierArray(),
302 kv.getQualifierOffset(), kv.getQualifierLength(), HConstants.OLDEST_TIMESTAMP,
303 Type.Minimum, null, 0, 0);
304 }
305
306
307
308
309
310
311
312
313
314
315
316 public static KeyValue createFirstOnRowColTS(Cell kv, long ts) {
317 return new KeyValue(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
318 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), kv.getQualifierArray(),
319 kv.getQualifierOffset(), kv.getQualifierLength(), ts, Type.Maximum, kv.getValueArray(),
320 kv.getValueOffset(), kv.getValueLength());
321 }
322
323
324
325
326
327
328
329
330
331 public static KeyValue createFirstOnRow(final byte [] row, int roffset, short rlength) {
332 return new KeyValue(row, roffset, rlength,
333 null, 0, 0, null, 0, 0, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0);
334 }
335
336
337
338
339
340
341
342
343
344 public static KeyValue createLastOnRow(final byte[] row) {
345 return new KeyValue(row, null, null, HConstants.LATEST_TIMESTAMP, Type.Minimum);
346 }
347
348
349
350
351
352
353
354
355
356 public static KeyValue createFirstOnRow(final byte [] row) {
357 return createFirstOnRow(row, HConstants.LATEST_TIMESTAMP);
358 }
359
360
361
362
363
364
365
366
367 public static KeyValue createFirstOnRow(final byte [] row,
368 final long ts) {
369 return new KeyValue(row, null, null, ts, Type.Maximum);
370 }
371
372
373
374
375
376
377
378
379
380
381 public static KeyValue createFirstOnRow(final byte [] row, final byte [] family,
382 final byte [] qualifier) {
383 return new KeyValue(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Maximum);
384 }
385
386
387
388
389
390
391
392
393
394
395 public static KeyValue createFirstDeleteFamilyOnRow(final byte [] row,
396 final byte [] family) {
397 return new KeyValue(row, family, null, HConstants.LATEST_TIMESTAMP,
398 Type.DeleteFamily);
399 }
400
401
402
403
404
405
406
407
408 public static KeyValue createFirstOnRow(final byte [] row, final byte [] f,
409 final byte [] q, final long ts) {
410 return new KeyValue(row, f, q, ts, Type.Maximum);
411 }
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429 public static KeyValue createFirstOnRow(final byte [] row,
430 final int roffset, final int rlength, final byte [] family,
431 final int foffset, final int flength, final byte [] qualifier,
432 final int qoffset, final int qlength) {
433 return new KeyValue(row, roffset, rlength, family,
434 foffset, flength, qualifier, qoffset, qlength,
435 HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0);
436 }
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454 public static KeyValue createFirstOnRow(byte [] buffer, final byte [] row,
455 final byte [] family, final byte [] qualifier)
456 throws IllegalArgumentException {
457 return createFirstOnRow(buffer, 0, row, 0, row.length,
458 family, 0, family.length,
459 qualifier, 0, qualifier.length);
460 }
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485 public static KeyValue createFirstOnRow(byte[] buffer, final int boffset, final byte[] row,
486 final int roffset, final int rlength, final byte[] family, final int foffset,
487 final int flength, final byte[] qualifier, final int qoffset, final int qlength)
488 throws IllegalArgumentException {
489
490 long lLength = KeyValue.getKeyValueDataStructureSize(rlength, flength, qlength, 0);
491
492 if (lLength > Integer.MAX_VALUE) {
493 throw new IllegalArgumentException("KeyValue length " + lLength + " > " + Integer.MAX_VALUE);
494 }
495 int iLength = (int) lLength;
496 if (buffer.length - boffset < iLength) {
497 throw new IllegalArgumentException("Buffer size " + (buffer.length - boffset) + " < "
498 + iLength);
499 }
500
501 int len = KeyValue.writeByteArray(buffer, boffset, row, roffset, rlength, family, foffset,
502 flength, qualifier, qoffset, qlength, HConstants.LATEST_TIMESTAMP, KeyValue.Type.Maximum,
503 null, 0, 0, null);
504 return new KeyValue(buffer, boffset, len);
505 }
506
507
508
509
510
511
512
513
514 public static KeyValue createFirstOnRowColTS(KeyValue kv, long ts) {
515 return new KeyValue(
516 kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
517 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(),
518 kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength(),
519 ts, Type.Maximum, kv.getValueArray(), kv.getValueOffset(), kv.getValueLength());
520 }
521
522
523
524
525
526
527
528
529 @Deprecated
530 public static KeyValue ensureKeyValue(final Cell cell) {
531 if (cell == null) return null;
532 return cell instanceof KeyValue? (KeyValue)cell: copyToNewKeyValue(cell);
533 }
534
535 @Deprecated
536 public static List<KeyValue> ensureKeyValues(List<Cell> cells) {
537 List<KeyValue> lazyList = Lists.transform(cells, new Function<Cell, KeyValue>() {
538 public KeyValue apply(Cell arg0) {
539 return KeyValueUtil.ensureKeyValue(arg0);
540 }
541 });
542 return new ArrayList<KeyValue>(lazyList);
543 }
544
545 public static void oswrite(final Cell cell, final OutputStream out, final boolean withTags)
546 throws IOException {
547 if (cell instanceof KeyValue) {
548 KeyValue.oswrite((KeyValue) cell, out, withTags);
549 } else {
550 short rlen = cell.getRowLength();
551 byte flen = cell.getFamilyLength();
552 int qlen = cell.getQualifierLength();
553 int vlen = cell.getValueLength();
554 int tlen = cell.getTagsLength();
555
556
557 StreamUtils.writeInt(out, length(rlen, flen, qlen, vlen, tlen, withTags));
558
559 StreamUtils.writeInt(out, keyLength(rlen, flen, qlen));
560
561 StreamUtils.writeInt(out, vlen);
562
563 StreamUtils.writeShort(out, rlen);
564 out.write(cell.getRowArray(), cell.getRowOffset(), rlen);
565
566 out.write(flen);
567 out.write(cell.getFamilyArray(), cell.getFamilyOffset(), flen);
568
569 out.write(cell.getQualifierArray(), cell.getQualifierOffset(), qlen);
570
571 StreamUtils.writeLong(out, cell.getTimestamp());
572
573 out.write(cell.getTypeByte());
574
575 out.write(cell.getValueArray(), cell.getValueOffset(), vlen);
576
577 if (withTags) {
578
579
580
581 out.write((byte) (0xff & (tlen >> 8)));
582 out.write((byte) (0xff & tlen));
583 out.write(cell.getTagsArray(), cell.getTagsOffset(), tlen);
584 }
585 }
586 }
587 }