1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import java.nio.ByteBuffer;
22 import java.util.ArrayList;
23 import java.util.List;
24
25 import org.apache.hadoop.hbase.classification.InterfaceAudience;
26 import org.apache.hadoop.hbase.KeyValue.Type;
27 import org.apache.hadoop.hbase.util.ByteBufferUtils;
28 import org.apache.hadoop.hbase.util.Bytes;
29 import org.apache.hadoop.hbase.util.IterableUtils;
30 import org.apache.hadoop.hbase.util.SimpleByteRange;
31 import org.apache.hadoop.io.WritableUtils;
32
33 import com.google.common.base.Function;
34 import com.google.common.collect.Lists;
35
36
37
38
39 @InterfaceAudience.Private
40 public class KeyValueUtil {
41
42
43
44 public static int length(final Cell cell) {
45 return (int) (KeyValue.getKeyValueDataStructureSize(cell.getRowLength(),
46 cell.getFamilyLength(), cell.getQualifierLength(), cell.getValueLength(),
47 cell.getTagsLengthUnsigned()));
48 }
49
50 public static int keyLength(final Cell cell) {
51 return (int)KeyValue.getKeyDataStructureSize(cell.getRowLength(), cell.getFamilyLength(),
52 cell.getQualifierLength());
53 }
54
55 public static int lengthWithMvccVersion(final KeyValue kv, final boolean includeMvccVersion) {
56 int length = kv.getLength();
57 if (includeMvccVersion) {
58 length += WritableUtils.getVIntSize(kv.getMvccVersion());
59 }
60 return length;
61 }
62
63 public static int totalLengthWithMvccVersion(final Iterable<? extends KeyValue> kvs,
64 final boolean includeMvccVersion) {
65 int length = 0;
66 for (KeyValue kv : IterableUtils.nullSafe(kvs)) {
67 length += lengthWithMvccVersion(kv, includeMvccVersion);
68 }
69 return length;
70 }
71
72
73
74
75 public static KeyValue copyToNewKeyValue(final Cell cell) {
76 byte[] bytes = copyToNewByteArray(cell);
77 KeyValue kvCell = new KeyValue(bytes, 0, bytes.length);
78 kvCell.setMvccVersion(cell.getMvccVersion());
79 return kvCell;
80 }
81
82 public static ByteBuffer copyKeyToNewByteBuffer(final Cell cell) {
83 byte[] bytes = new byte[keyLength(cell)];
84 appendKeyToByteArrayWithoutValue(cell, bytes, 0);
85 ByteBuffer buffer = ByteBuffer.wrap(bytes);
86 return buffer;
87 }
88
89 public static byte[] copyToNewByteArray(final Cell cell) {
90 int v1Length = length(cell);
91 byte[] backingBytes = new byte[v1Length];
92 appendToByteArray(cell, backingBytes, 0);
93 return backingBytes;
94 }
95
96 protected static int appendKeyToByteArrayWithoutValue(final Cell cell, final byte[] output,
97 final int offset) {
98 int nextOffset = offset;
99 nextOffset = Bytes.putShort(output, nextOffset, cell.getRowLength());
100 nextOffset = CellUtil.copyRowTo(cell, output, nextOffset);
101 nextOffset = Bytes.putByte(output, nextOffset, cell.getFamilyLength());
102 nextOffset = CellUtil.copyFamilyTo(cell, output, nextOffset);
103 nextOffset = CellUtil.copyQualifierTo(cell, output, nextOffset);
104 nextOffset = Bytes.putLong(output, nextOffset, cell.getTimestamp());
105 nextOffset = Bytes.putByte(output, nextOffset, cell.getTypeByte());
106 return nextOffset;
107 }
108
109
110
111
112 public static int appendToByteArray(final Cell cell, final byte[] output, final int offset) {
113 int pos = offset;
114 pos = Bytes.putInt(output, pos, keyLength(cell));
115 pos = Bytes.putInt(output, pos, cell.getValueLength());
116 pos = appendKeyToByteArrayWithoutValue(cell, output, pos);
117 pos = CellUtil.copyValueTo(cell, output, pos);
118 if ((cell.getTagsLengthUnsigned() > 0)) {
119 pos = Bytes.putAsShort(output, pos, cell.getTagsLengthUnsigned());
120 pos = CellUtil.copyTagTo(cell, output, pos);
121 }
122 return pos;
123 }
124
125 public static ByteBuffer copyToNewByteBuffer(final Cell cell) {
126 byte[] bytes = new byte[length(cell)];
127 appendToByteArray(cell, bytes, 0);
128 ByteBuffer buffer = ByteBuffer.wrap(bytes);
129 return buffer;
130 }
131
132 public static void appendToByteBuffer(final ByteBuffer bb, final KeyValue kv,
133 final boolean includeMvccVersion) {
134
135 bb.limit(bb.position() + kv.getLength());
136 bb.put(kv.getBuffer(), kv.getOffset(), kv.getLength());
137 if (includeMvccVersion) {
138 int numMvccVersionBytes = WritableUtils.getVIntSize(kv.getMvccVersion());
139 ByteBufferUtils.extendLimit(bb, numMvccVersionBytes);
140 ByteBufferUtils.writeVLong(bb, kv.getMvccVersion());
141 }
142 }
143
144
145
146
147
148
149
150
151
152
153
154 public static KeyValue nextShallowCopy(final ByteBuffer bb, final boolean includesMvccVersion,
155 boolean includesTags) {
156 if (bb.isDirect()) {
157 throw new IllegalArgumentException("only supports heap buffers");
158 }
159 if (bb.remaining() < 1) {
160 return null;
161 }
162 KeyValue keyValue = null;
163 int underlyingArrayOffset = bb.arrayOffset() + bb.position();
164 int keyLength = bb.getInt();
165 int valueLength = bb.getInt();
166 ByteBufferUtils.skip(bb, keyLength + valueLength);
167 int tagsLength = 0;
168 if (includesTags) {
169
170 tagsLength = ((bb.get() & 0xff) << 8) ^ (bb.get() & 0xff);
171 ByteBufferUtils.skip(bb, tagsLength);
172 }
173 int kvLength = (int) KeyValue.getKeyValueDataStructureSize(keyLength, valueLength, tagsLength);
174 keyValue = new KeyValue(bb.array(), underlyingArrayOffset, kvLength);
175 if (includesMvccVersion) {
176 long mvccVersion = ByteBufferUtils.readVLong(bb);
177 keyValue.setMvccVersion(mvccVersion);
178 }
179 return keyValue;
180 }
181
182
183
184
185
186
187
188
189
190 public static KeyValue createFirstKeyInNextRow(final Cell in){
191 byte[] nextRow = new byte[in.getRowLength() + 1];
192 System.arraycopy(in.getRowArray(), in.getRowOffset(), nextRow, 0, in.getRowLength());
193 nextRow[nextRow.length - 1] = 0;
194 return createFirstOnRow(nextRow);
195 }
196
197
198
199
200 public static KeyValue createFirstKeyInIncrementedRow(final Cell in){
201 byte[] thisRow = new SimpleByteRange(in.getRowArray(), in.getRowOffset(),
202 in.getRowLength()).deepCopyToNewArray();
203 byte[] nextRow = Bytes.unsignedCopyAndIncrement(thisRow);
204 return createFirstOnRow(nextRow);
205 }
206
207
208
209
210
211
212
213
214 public static KeyValue previousKey(final KeyValue in) {
215 return createFirstOnRow(CellUtil.cloneRow(in), CellUtil.cloneFamily(in),
216 CellUtil.cloneQualifier(in), in.getTimestamp() - 1);
217 }
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245 public static KeyValue createLastOnRow(final byte[] row, final int roffset, final int rlength,
246 final byte[] family, final int foffset, final int flength, final byte[] qualifier,
247 final int qoffset, final int qlength) {
248 return new KeyValue(row, roffset, rlength, family, foffset, flength, qualifier, qoffset,
249 qlength, HConstants.OLDEST_TIMESTAMP, Type.Minimum, null, 0, 0);
250 }
251
252
253
254
255
256
257
258 public static KeyValue createLastOnRow(Cell kv) {
259 return createLastOnRow(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), null, 0, 0,
260 null, 0, 0);
261 }
262
263
264
265
266
267
268
269
270
271
272
273
274 public static KeyValue createLastOnRowCol(Cell kv) {
275 return new KeyValue(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
276 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), kv.getQualifierArray(),
277 kv.getQualifierOffset(), kv.getQualifierLength(), HConstants.OLDEST_TIMESTAMP,
278 Type.Minimum, null, 0, 0);
279 }
280
281
282
283
284
285
286
287
288
289
290
291 public static KeyValue createFirstOnRowColTS(Cell kv, long ts) {
292 return new KeyValue(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
293 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), kv.getQualifierArray(),
294 kv.getQualifierOffset(), kv.getQualifierLength(), ts, Type.Maximum, kv.getValueArray(),
295 kv.getValueOffset(), kv.getValueLength());
296 }
297
298
299
300
301
302
303
304
305
306 public static KeyValue createFirstOnRow(final byte [] row, int roffset, short rlength) {
307 return new KeyValue(row, roffset, rlength,
308 null, 0, 0, null, 0, 0, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0);
309 }
310
311
312
313
314
315
316
317
318
319 public static KeyValue createLastOnRow(final byte[] row) {
320 return new KeyValue(row, null, null, HConstants.LATEST_TIMESTAMP, Type.Minimum);
321 }
322
323
324
325
326
327
328
329
330
331 public static KeyValue createFirstOnRow(final byte [] row) {
332 return createFirstOnRow(row, HConstants.LATEST_TIMESTAMP);
333 }
334
335
336
337
338
339
340
341
342 public static KeyValue createFirstOnRow(final byte [] row,
343 final long ts) {
344 return new KeyValue(row, null, null, ts, Type.Maximum);
345 }
346
347
348
349
350
351
352
353
354
355
356 public static KeyValue createFirstOnRow(final byte [] row, final byte [] family,
357 final byte [] qualifier) {
358 return new KeyValue(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Maximum);
359 }
360
361
362
363
364
365
366
367
368
369
370 public static KeyValue createFirstDeleteFamilyOnRow(final byte [] row,
371 final byte [] family) {
372 return new KeyValue(row, family, null, HConstants.LATEST_TIMESTAMP,
373 Type.DeleteFamily);
374 }
375
376
377
378
379
380
381
382
383 public static KeyValue createFirstOnRow(final byte [] row, final byte [] f,
384 final byte [] q, final long ts) {
385 return new KeyValue(row, f, q, ts, Type.Maximum);
386 }
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404 public static KeyValue createFirstOnRow(final byte [] row,
405 final int roffset, final int rlength, final byte [] family,
406 final int foffset, final int flength, final byte [] qualifier,
407 final int qoffset, final int qlength) {
408 return new KeyValue(row, roffset, rlength, family,
409 foffset, flength, qualifier, qoffset, qlength,
410 HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0);
411 }
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429 public static KeyValue createFirstOnRow(byte [] buffer, final byte [] row,
430 final byte [] family, final byte [] qualifier)
431 throws IllegalArgumentException {
432 return createFirstOnRow(buffer, 0, row, 0, row.length,
433 family, 0, family.length,
434 qualifier, 0, qualifier.length);
435 }
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460 public static KeyValue createFirstOnRow(byte[] buffer, final int boffset, final byte[] row,
461 final int roffset, final int rlength, final byte[] family, final int foffset,
462 final int flength, final byte[] qualifier, final int qoffset, final int qlength)
463 throws IllegalArgumentException {
464
465 long lLength = KeyValue.getKeyValueDataStructureSize(rlength, flength, qlength, 0);
466
467 if (lLength > Integer.MAX_VALUE) {
468 throw new IllegalArgumentException("KeyValue length " + lLength + " > " + Integer.MAX_VALUE);
469 }
470 int iLength = (int) lLength;
471 if (buffer.length - boffset < iLength) {
472 throw new IllegalArgumentException("Buffer size " + (buffer.length - boffset) + " < "
473 + iLength);
474 }
475
476 int len = KeyValue.writeByteArray(buffer, boffset, row, roffset, rlength, family, foffset,
477 flength, qualifier, qoffset, qlength, HConstants.LATEST_TIMESTAMP, KeyValue.Type.Maximum,
478 null, 0, 0, null);
479 return new KeyValue(buffer, boffset, len);
480 }
481
482
483
484
485
486
487
488
489 public static KeyValue createFirstOnRowColTS(KeyValue kv, long ts) {
490 return new KeyValue(
491 kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
492 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(),
493 kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength(),
494 ts, Type.Maximum, kv.getValueArray(), kv.getValueOffset(), kv.getValueLength());
495 }
496
497
498
499
500
501
502
503 public static KeyValue ensureKeyValue(final Cell cell) {
504 if (cell == null) return null;
505 return cell instanceof KeyValue? (KeyValue)cell: copyToNewKeyValue(cell);
506 }
507
508 public static List<KeyValue> ensureKeyValues(List<Cell> cells) {
509 List<KeyValue> lazyList = Lists.transform(cells, new Function<Cell, KeyValue>() {
510 @Override
511 public KeyValue apply(Cell arg0) {
512 return KeyValueUtil.ensureKeyValue(arg0);
513 }
514 });
515 return new ArrayList<KeyValue>(lazyList);
516 }
517 }