1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.util;
19
20 import static com.google.common.base.Preconditions.checkArgument;
21 import static com.google.common.base.Preconditions.checkPositionIndex;
22 import static com.google.common.base.Preconditions.checkNotNull;
23
24 import java.io.DataInput;
25 import java.io.DataOutput;
26 import java.io.IOException;
27 import java.lang.reflect.Field;
28 import java.math.BigDecimal;
29 import java.math.BigInteger;
30 import java.nio.ByteBuffer;
31 import java.nio.ByteOrder;
32 import java.security.AccessController;
33 import java.security.PrivilegedAction;
34 import java.security.SecureRandom;
35 import java.util.Arrays;
36 import java.util.Collection;
37 import java.util.Comparator;
38 import java.util.Iterator;
39 import java.util.List;
40
41 import org.apache.commons.logging.Log;
42 import org.apache.commons.logging.LogFactory;
43 import org.apache.hadoop.classification.InterfaceAudience;
44 import org.apache.hadoop.classification.InterfaceStability;
45 import org.apache.hadoop.hbase.HConstants;
46 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
47 import org.apache.hadoop.io.RawComparator;
48 import org.apache.hadoop.io.WritableComparator;
49 import org.apache.hadoop.io.WritableUtils;
50
51 import sun.misc.Unsafe;
52
53 import com.google.common.annotations.VisibleForTesting;
54 import com.google.common.collect.Lists;
55
56
57
58
59
60
61 @InterfaceAudience.Public
62 @InterfaceStability.Stable
63 public class Bytes {
64
65 private static final Log LOG = LogFactory.getLog(Bytes.class);
66
67
68
69
70 public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE;
71
72
73
74
75 public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN;
76
77
78
79
80 public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE;
81
82
83
84
85 public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE;
86
87
88
89
90 public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE;
91
92
93
94
95 public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE;
96
97
98
99
100 public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE;
101
102
103
104
105 public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE;
106
107
108
109
110
111
112
113
114 public static final int ESTIMATED_HEAP_TAX = 16;
115
116
117
118
119 public static class ByteArrayComparator implements RawComparator<byte []> {
120
121
122
123 public ByteArrayComparator() {
124 super();
125 }
126 public int compare(byte [] left, byte [] right) {
127 return compareTo(left, right);
128 }
129 public int compare(byte [] b1, int s1, int l1, byte [] b2, int s2, int l2) {
130 return LexicographicalComparerHolder.BEST_COMPARER.
131 compareTo(b1, s1, l1, b2, s2, l2);
132 }
133 }
134
135
136
137
138 public static Comparator<byte []> BYTES_COMPARATOR =
139 new ByteArrayComparator();
140
141
142
143
144 public static RawComparator<byte []> BYTES_RAWCOMPARATOR =
145 new ByteArrayComparator();
146
147
148
149
150
151
152
153 public static byte [] readByteArray(final DataInput in)
154 throws IOException {
155 int len = WritableUtils.readVInt(in);
156 if (len < 0) {
157 throw new NegativeArraySizeException(Integer.toString(len));
158 }
159 byte [] result = new byte[len];
160 in.readFully(result, 0, len);
161 return result;
162 }
163
164
165
166
167
168
169
170 public static byte [] readByteArrayThrowsRuntime(final DataInput in) {
171 try {
172 return readByteArray(in);
173 } catch (Exception e) {
174 throw new RuntimeException(e);
175 }
176 }
177
178
179
180
181
182
183
184 public static void writeByteArray(final DataOutput out, final byte [] b)
185 throws IOException {
186 if(b == null) {
187 WritableUtils.writeVInt(out, 0);
188 } else {
189 writeByteArray(out, b, 0, b.length);
190 }
191 }
192
193
194
195
196
197
198
199
200
201 public static void writeByteArray(final DataOutput out, final byte [] b,
202 final int offset, final int length)
203 throws IOException {
204 WritableUtils.writeVInt(out, length);
205 out.write(b, offset, length);
206 }
207
208
209
210
211
212
213
214
215
216
217 public static int writeByteArray(final byte [] tgt, final int tgtOffset,
218 final byte [] src, final int srcOffset, final int srcLength) {
219 byte [] vint = vintToBytes(srcLength);
220 System.arraycopy(vint, 0, tgt, tgtOffset, vint.length);
221 int offset = tgtOffset + vint.length;
222 System.arraycopy(src, srcOffset, tgt, offset, srcLength);
223 return offset + srcLength;
224 }
225
226
227
228
229
230
231
232
233
234
235 public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes,
236 int srcOffset, int srcLength) {
237 System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength);
238 return tgtOffset + srcLength;
239 }
240
241
242
243
244
245
246
247
248 public static int putByte(byte[] bytes, int offset, byte b) {
249 bytes[offset] = b;
250 return offset + 1;
251 }
252
253
254
255
256
257
258
259
260
261
262
263 public static byte[] toBytes(ByteBuffer buf) {
264 ByteBuffer dup = buf.duplicate();
265 dup.position(0);
266 return readBytes(dup);
267 }
268
269 private static byte[] readBytes(ByteBuffer buf) {
270 byte [] result = new byte[buf.remaining()];
271 buf.get(result);
272 return result;
273 }
274
275
276
277
278
279 public static String toString(final byte [] b) {
280 if (b == null) {
281 return null;
282 }
283 return toString(b, 0, b.length);
284 }
285
286
287
288
289
290
291
292 public static String toString(final byte [] b1,
293 String sep,
294 final byte [] b2) {
295 return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length);
296 }
297
298
299
300
301
302
303
304
305
306
307 public static String toString(final byte [] b, int off, int len) {
308 if (b == null) {
309 return null;
310 }
311 if (len == 0) {
312 return "";
313 }
314 return new String(b, off, len, HConstants.UTF8_CHARSET);
315 }
316
317
318
319
320
321
322
323
324 public static String toStringBinary(final byte [] b) {
325 if (b == null)
326 return "null";
327 return toStringBinary(b, 0, b.length);
328 }
329
330
331
332
333
334
335
336
337
338
339
340
341 public static String toStringBinary(ByteBuffer buf) {
342 if (buf == null)
343 return "null";
344 if (buf.hasArray()) {
345 return toStringBinary(buf.array(), buf.arrayOffset(), buf.limit());
346 }
347 return toStringBinary(toBytes(buf));
348 }
349
350
351
352
353
354
355
356
357
358
359
360 public static String toStringBinary(final byte [] b, int off, int len) {
361 StringBuilder result = new StringBuilder();
362
363 if (off >= b.length) return result.toString();
364 if (off + len > b.length) len = b.length - off;
365 for (int i = off; i < off + len ; ++i ) {
366 int ch = b[i] & 0xFF;
367 if ( (ch >= '0' && ch <= '9')
368 || (ch >= 'A' && ch <= 'Z')
369 || (ch >= 'a' && ch <= 'z')
370 || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0 ) {
371 result.append((char)ch);
372 } else {
373 result.append(String.format("\\x%02X", ch));
374 }
375 }
376 return result.toString();
377 }
378
379 private static boolean isHexDigit(char c) {
380 return
381 (c >= 'A' && c <= 'F') ||
382 (c >= '0' && c <= '9');
383 }
384
385
386
387
388
389
390
391 public static byte toBinaryFromHex(byte ch) {
392 if ( ch >= 'A' && ch <= 'F' )
393 return (byte) ((byte)10 + (byte) (ch - 'A'));
394
395 return (byte) (ch - '0');
396 }
397
398 public static byte [] toBytesBinary(String in) {
399
400 byte [] b = new byte[in.length()];
401 int size = 0;
402 for (int i = 0; i < in.length(); ++i) {
403 char ch = in.charAt(i);
404 if (ch == '\\' && in.length() > i+1 && in.charAt(i+1) == 'x') {
405
406 char hd1 = in.charAt(i+2);
407 char hd2 = in.charAt(i+3);
408
409
410 if (!isHexDigit(hd1) ||
411 !isHexDigit(hd2)) {
412
413 continue;
414 }
415
416 byte d = (byte) ((toBinaryFromHex((byte)hd1) << 4) + toBinaryFromHex((byte)hd2));
417
418 b[size++] = d;
419 i += 3;
420 } else {
421 b[size++] = (byte) ch;
422 }
423 }
424
425 byte [] b2 = new byte[size];
426 System.arraycopy(b, 0, b2, 0, size);
427 return b2;
428 }
429
430
431
432
433
434
435 public static byte[] toBytes(String s) {
436 return s.getBytes(HConstants.UTF8_CHARSET);
437 }
438
439
440
441
442
443
444
445
446 public static byte [] toBytes(final boolean b) {
447 return new byte[] { b ? (byte) -1 : (byte) 0 };
448 }
449
450
451
452
453
454
455 public static boolean toBoolean(final byte [] b) {
456 if (b.length != 1) {
457 throw new IllegalArgumentException("Array has wrong size: " + b.length);
458 }
459 return b[0] != (byte) 0;
460 }
461
462
463
464
465
466
467
468 public static byte[] toBytes(long val) {
469 byte [] b = new byte[8];
470 for (int i = 7; i > 0; i--) {
471 b[i] = (byte) val;
472 val >>>= 8;
473 }
474 b[0] = (byte) val;
475 return b;
476 }
477
478
479
480
481
482
483
484 public static long toLong(byte[] bytes) {
485 return toLong(bytes, 0, SIZEOF_LONG);
486 }
487
488
489
490
491
492
493
494
495
496 public static long toLong(byte[] bytes, int offset) {
497 return toLong(bytes, offset, SIZEOF_LONG);
498 }
499
500
501
502
503
504
505
506
507
508
509
510 public static long toLong(byte[] bytes, int offset, final int length) {
511 if (length != SIZEOF_LONG || offset + length > bytes.length) {
512 throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
513 }
514 long l = 0;
515 for(int i = offset; i < offset + length; i++) {
516 l <<= 8;
517 l ^= bytes[i] & 0xFF;
518 }
519 return l;
520 }
521
522 private static IllegalArgumentException
523 explainWrongLengthOrOffset(final byte[] bytes,
524 final int offset,
525 final int length,
526 final int expectedLength) {
527 String reason;
528 if (length != expectedLength) {
529 reason = "Wrong length: " + length + ", expected " + expectedLength;
530 } else {
531 reason = "offset (" + offset + ") + length (" + length + ") exceed the"
532 + " capacity of the array: " + bytes.length;
533 }
534 return new IllegalArgumentException(reason);
535 }
536
537
538
539
540
541
542
543
544
545
546 public static int putLong(byte[] bytes, int offset, long val) {
547 if (bytes.length - offset < SIZEOF_LONG) {
548 throw new IllegalArgumentException("Not enough room to put a long at"
549 + " offset " + offset + " in a " + bytes.length + " byte array");
550 }
551 for(int i = offset + 7; i > offset; i--) {
552 bytes[i] = (byte) val;
553 val >>>= 8;
554 }
555 bytes[offset] = (byte) val;
556 return offset + SIZEOF_LONG;
557 }
558
559
560
561
562
563
564 public static float toFloat(byte [] bytes) {
565 return toFloat(bytes, 0);
566 }
567
568
569
570
571
572
573
574 public static float toFloat(byte [] bytes, int offset) {
575 return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
576 }
577
578
579
580
581
582
583
584 public static int putFloat(byte [] bytes, int offset, float f) {
585 return putInt(bytes, offset, Float.floatToRawIntBits(f));
586 }
587
588
589
590
591
592 public static byte [] toBytes(final float f) {
593
594 return Bytes.toBytes(Float.floatToRawIntBits(f));
595 }
596
597
598
599
600
601 public static double toDouble(final byte [] bytes) {
602 return toDouble(bytes, 0);
603 }
604
605
606
607
608
609
610 public static double toDouble(final byte [] bytes, final int offset) {
611 return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
612 }
613
614
615
616
617
618
619
620 public static int putDouble(byte [] bytes, int offset, double d) {
621 return putLong(bytes, offset, Double.doubleToLongBits(d));
622 }
623
624
625
626
627
628
629
630
631 public static byte [] toBytes(final double d) {
632
633 return Bytes.toBytes(Double.doubleToRawLongBits(d));
634 }
635
636
637
638
639
640
641
642
643 public static byte[] toBytes(int val) {
644 byte [] b = new byte[4];
645 for(int i = 3; i > 0; i--) {
646 b[i] = (byte) val;
647 val >>>= 8;
648 }
649 b[0] = (byte) val;
650 return b;
651 }
652
653
654
655
656
657
658 public static int toInt(byte[] bytes) {
659 return toInt(bytes, 0, SIZEOF_INT);
660 }
661
662
663
664
665
666
667
668 public static int toInt(byte[] bytes, int offset) {
669 return toInt(bytes, offset, SIZEOF_INT);
670 }
671
672
673
674
675
676
677
678
679
680
681 public static int toInt(byte[] bytes, int offset, final int length) {
682 if (length != SIZEOF_INT || offset + length > bytes.length) {
683 throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
684 }
685 int n = 0;
686 for(int i = offset; i < (offset + length); i++) {
687 n <<= 8;
688 n ^= bytes[i] & 0xFF;
689 }
690 return n;
691 }
692
693
694
695
696
697
698
699
700
701
702 public static int putInt(byte[] bytes, int offset, int val) {
703 if (bytes.length - offset < SIZEOF_INT) {
704 throw new IllegalArgumentException("Not enough room to put an int at"
705 + " offset " + offset + " in a " + bytes.length + " byte array");
706 }
707 for(int i= offset + 3; i > offset; i--) {
708 bytes[i] = (byte) val;
709 val >>>= 8;
710 }
711 bytes[offset] = (byte) val;
712 return offset + SIZEOF_INT;
713 }
714
715
716
717
718
719
720 public static byte[] toBytes(short val) {
721 byte[] b = new byte[SIZEOF_SHORT];
722 b[1] = (byte) val;
723 val >>= 8;
724 b[0] = (byte) val;
725 return b;
726 }
727
728
729
730
731
732
733 public static short toShort(byte[] bytes) {
734 return toShort(bytes, 0, SIZEOF_SHORT);
735 }
736
737
738
739
740
741
742
743 public static short toShort(byte[] bytes, int offset) {
744 return toShort(bytes, offset, SIZEOF_SHORT);
745 }
746
747
748
749
750
751
752
753
754
755
756 public static short toShort(byte[] bytes, int offset, final int length) {
757 if (length != SIZEOF_SHORT || offset + length > bytes.length) {
758 throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
759 }
760 short n = 0;
761 n ^= bytes[offset] & 0xFF;
762 n <<= 8;
763 n ^= bytes[offset+1] & 0xFF;
764 return n;
765 }
766
767
768
769
770
771
772
773
774
775
776 public static byte[] getBytes(ByteBuffer buf) {
777 return readBytes(buf.duplicate());
778 }
779
780
781
782
783
784
785
786
787
788
789 public static int putShort(byte[] bytes, int offset, short val) {
790 if (bytes.length - offset < SIZEOF_SHORT) {
791 throw new IllegalArgumentException("Not enough room to put a short at"
792 + " offset " + offset + " in a " + bytes.length + " byte array");
793 }
794 bytes[offset+1] = (byte) val;
795 val >>= 8;
796 bytes[offset] = (byte) val;
797 return offset + SIZEOF_SHORT;
798 }
799
800
801
802
803
804
805
806 public static byte[] toBytes(BigDecimal val) {
807 byte[] valueBytes = val.unscaledValue().toByteArray();
808 byte[] result = new byte[valueBytes.length + SIZEOF_INT];
809 int offset = putInt(result, 0, val.scale());
810 putBytes(result, offset, valueBytes, 0, valueBytes.length);
811 return result;
812 }
813
814
815
816
817
818
819
820
821 public static BigDecimal toBigDecimal(byte[] bytes) {
822 return toBigDecimal(bytes, 0, bytes.length);
823 }
824
825
826
827
828
829
830
831
832
833 public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) {
834 if (bytes == null || length < SIZEOF_INT + 1 ||
835 (offset + length > bytes.length)) {
836 return null;
837 }
838
839 int scale = toInt(bytes, offset);
840 byte[] tcBytes = new byte[length - SIZEOF_INT];
841 System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT);
842 return new BigDecimal(new BigInteger(tcBytes), scale);
843 }
844
845
846
847
848
849
850
851
852
853 public static int putBigDecimal(byte[] bytes, int offset, BigDecimal val) {
854 if (bytes == null) {
855 return offset;
856 }
857
858 byte[] valueBytes = val.unscaledValue().toByteArray();
859 byte[] result = new byte[valueBytes.length + SIZEOF_INT];
860 offset = putInt(result, offset, val.scale());
861 return putBytes(result, offset, valueBytes, 0, valueBytes.length);
862 }
863
864
865
866
867
868 public static byte [] vintToBytes(final long vint) {
869 long i = vint;
870 int size = WritableUtils.getVIntSize(i);
871 byte [] result = new byte[size];
872 int offset = 0;
873 if (i >= -112 && i <= 127) {
874 result[offset] = (byte) i;
875 return result;
876 }
877
878 int len = -112;
879 if (i < 0) {
880 i ^= -1L;
881 len = -120;
882 }
883
884 long tmp = i;
885 while (tmp != 0) {
886 tmp = tmp >> 8;
887 len--;
888 }
889
890 result[offset++] = (byte) len;
891
892 len = (len < -120) ? -(len + 120) : -(len + 112);
893
894 for (int idx = len; idx != 0; idx--) {
895 int shiftbits = (idx - 1) * 8;
896 long mask = 0xFFL << shiftbits;
897 result[offset++] = (byte)((i & mask) >> shiftbits);
898 }
899 return result;
900 }
901
902
903
904
905
906 public static long bytesToVint(final byte [] buffer) {
907 int offset = 0;
908 byte firstByte = buffer[offset++];
909 int len = WritableUtils.decodeVIntSize(firstByte);
910 if (len == 1) {
911 return firstByte;
912 }
913 long i = 0;
914 for (int idx = 0; idx < len-1; idx++) {
915 byte b = buffer[offset++];
916 i = i << 8;
917 i = i | (b & 0xFF);
918 }
919 return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
920 }
921
922
923
924
925
926
927
928
929 public static long readVLong(final byte [] buffer, final int offset)
930 throws IOException {
931 byte firstByte = buffer[offset];
932 int len = WritableUtils.decodeVIntSize(firstByte);
933 if (len == 1) {
934 return firstByte;
935 }
936 long i = 0;
937 for (int idx = 0; idx < len-1; idx++) {
938 byte b = buffer[offset + 1 + idx];
939 i = i << 8;
940 i = i | (b & 0xFF);
941 }
942 return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
943 }
944
945
946
947
948
949
950 public static int compareTo(final byte [] left, final byte [] right) {
951 return LexicographicalComparerHolder.BEST_COMPARER.
952 compareTo(left, 0, left.length, right, 0, right.length);
953 }
954
955
956
957
958
959
960
961
962
963
964
965
966 public static int compareTo(byte[] buffer1, int offset1, int length1,
967 byte[] buffer2, int offset2, int length2) {
968 return LexicographicalComparerHolder.BEST_COMPARER.
969 compareTo(buffer1, offset1, length1, buffer2, offset2, length2);
970 }
971
972 interface Comparer<T> {
973 abstract public int compareTo(T buffer1, int offset1, int length1,
974 T buffer2, int offset2, int length2);
975 }
976
977 @VisibleForTesting
978 static Comparer<byte[]> lexicographicalComparerJavaImpl() {
979 return LexicographicalComparerHolder.PureJavaComparer.INSTANCE;
980 }
981
982
983
984
985
986
987
988
989 @VisibleForTesting
990 static class LexicographicalComparerHolder {
991 static final String UNSAFE_COMPARER_NAME =
992 LexicographicalComparerHolder.class.getName() + "$UnsafeComparer";
993
994 static final Comparer<byte[]> BEST_COMPARER = getBestComparer();
995
996
997
998
999 static Comparer<byte[]> getBestComparer() {
1000 try {
1001 Class<?> theClass = Class.forName(UNSAFE_COMPARER_NAME);
1002
1003
1004 @SuppressWarnings("unchecked")
1005 Comparer<byte[]> comparer =
1006 (Comparer<byte[]>) theClass.getEnumConstants()[0];
1007 return comparer;
1008 } catch (Throwable t) {
1009 return lexicographicalComparerJavaImpl();
1010 }
1011 }
1012
1013 enum PureJavaComparer implements Comparer<byte[]> {
1014 INSTANCE;
1015
1016 @Override
1017 public int compareTo(byte[] buffer1, int offset1, int length1,
1018 byte[] buffer2, int offset2, int length2) {
1019
1020 if (buffer1 == buffer2 &&
1021 offset1 == offset2 &&
1022 length1 == length2) {
1023 return 0;
1024 }
1025
1026 int end1 = offset1 + length1;
1027 int end2 = offset2 + length2;
1028 for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) {
1029 int a = (buffer1[i] & 0xff);
1030 int b = (buffer2[j] & 0xff);
1031 if (a != b) {
1032 return a - b;
1033 }
1034 }
1035 return length1 - length2;
1036 }
1037 }
1038
1039 @VisibleForTesting
1040 enum UnsafeComparer implements Comparer<byte[]> {
1041 INSTANCE;
1042
1043 static final Unsafe theUnsafe;
1044
1045
1046 static final int BYTE_ARRAY_BASE_OFFSET;
1047
1048 static {
1049 theUnsafe = (Unsafe) AccessController.doPrivileged(
1050 new PrivilegedAction<Object>() {
1051 @Override
1052 public Object run() {
1053 try {
1054 Field f = Unsafe.class.getDeclaredField("theUnsafe");
1055 f.setAccessible(true);
1056 return f.get(null);
1057 } catch (NoSuchFieldException e) {
1058
1059
1060 throw new Error();
1061 } catch (IllegalAccessException e) {
1062 throw new Error();
1063 }
1064 }
1065 });
1066
1067 BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class);
1068
1069
1070 if (theUnsafe.arrayIndexScale(byte[].class) != 1) {
1071 throw new AssertionError();
1072 }
1073 }
1074
1075 static final boolean littleEndian =
1076 ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN);
1077
1078
1079
1080
1081
1082 static boolean lessThanUnsigned(long x1, long x2) {
1083 return (x1 + Long.MIN_VALUE) < (x2 + Long.MIN_VALUE);
1084 }
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097 @Override
1098 public int compareTo(byte[] buffer1, int offset1, int length1,
1099 byte[] buffer2, int offset2, int length2) {
1100
1101 if (buffer1 == buffer2 &&
1102 offset1 == offset2 &&
1103 length1 == length2) {
1104 return 0;
1105 }
1106 int minLength = Math.min(length1, length2);
1107 int minWords = minLength / SIZEOF_LONG;
1108 int offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET;
1109 int offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET;
1110
1111
1112
1113
1114
1115
1116 for (int i = 0; i < minWords * SIZEOF_LONG; i += SIZEOF_LONG) {
1117 long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i);
1118 long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i);
1119 long diff = lw ^ rw;
1120
1121 if (diff != 0) {
1122 if (!littleEndian) {
1123 return lessThanUnsigned(lw, rw) ? -1 : 1;
1124 }
1125
1126
1127 int n = 0;
1128 int y;
1129 int x = (int) diff;
1130 if (x == 0) {
1131 x = (int) (diff >>> 32);
1132 n = 32;
1133 }
1134
1135 y = x << 16;
1136 if (y == 0) {
1137 n += 16;
1138 } else {
1139 x = y;
1140 }
1141
1142 y = x << 8;
1143 if (y == 0) {
1144 n += 8;
1145 }
1146 return (int) (((lw >>> n) & 0xFFL) - ((rw >>> n) & 0xFFL));
1147 }
1148 }
1149
1150
1151 for (int i = minWords * SIZEOF_LONG; i < minLength; i++) {
1152 int a = (buffer1[offset1 + i] & 0xff);
1153 int b = (buffer2[offset2 + i] & 0xff);
1154 if (a != b) {
1155 return a - b;
1156 }
1157 }
1158 return length1 - length2;
1159 }
1160 }
1161 }
1162
1163
1164
1165
1166
1167
1168 public static boolean equals(final byte [] left, final byte [] right) {
1169
1170
1171 if (left == right) return true;
1172 if (left == null || right == null) return false;
1173 if (left.length != right.length) return false;
1174 if (left.length == 0) return true;
1175
1176
1177
1178
1179 if (left[left.length - 1] != right[right.length - 1]) return false;
1180
1181 return compareTo(left, right) == 0;
1182 }
1183
1184 public static boolean equals(final byte[] left, int leftOffset, int leftLen,
1185 final byte[] right, int rightOffset, int rightLen) {
1186
1187 if (left == right &&
1188 leftOffset == rightOffset &&
1189 leftLen == rightLen) {
1190 return true;
1191 }
1192
1193 if (leftLen != rightLen) {
1194 return false;
1195 }
1196 if (leftLen == 0) {
1197 return true;
1198 }
1199
1200
1201
1202
1203 if (left[leftOffset + leftLen - 1] != right[rightOffset + rightLen - 1]) return false;
1204
1205 return LexicographicalComparerHolder.BEST_COMPARER.
1206 compareTo(left, leftOffset, leftLen, right, rightOffset, rightLen) == 0;
1207 }
1208
1209
1210
1211
1212
1213
1214 public static boolean startsWith(byte[] bytes, byte[] prefix) {
1215 return bytes != null && prefix != null &&
1216 bytes.length >= prefix.length &&
1217 LexicographicalComparerHolder.BEST_COMPARER.
1218 compareTo(bytes, 0, prefix.length, prefix, 0, prefix.length) == 0;
1219 }
1220
1221
1222
1223
1224
1225
1226
1227 public static int hashCode(final byte [] b) {
1228 return hashCode(b, b.length);
1229 }
1230
1231
1232
1233
1234
1235
1236
1237
1238 public static int hashCode(final byte [] b, final int length) {
1239 return WritableComparator.hashBytes(b, length);
1240 }
1241
1242
1243
1244
1245
1246
1247 public static Integer mapKey(final byte [] b) {
1248 return hashCode(b);
1249 }
1250
1251
1252
1253
1254
1255
1256
1257 public static Integer mapKey(final byte [] b, final int length) {
1258 return hashCode(b, length);
1259 }
1260
1261
1262
1263
1264
1265
1266 public static byte [] add(final byte [] a, final byte [] b) {
1267 return add(a, b, HConstants.EMPTY_BYTE_ARRAY);
1268 }
1269
1270
1271
1272
1273
1274
1275
1276 public static byte [] add(final byte [] a, final byte [] b, final byte [] c) {
1277 byte [] result = new byte[a.length + b.length + c.length];
1278 System.arraycopy(a, 0, result, 0, a.length);
1279 System.arraycopy(b, 0, result, a.length, b.length);
1280 System.arraycopy(c, 0, result, a.length + b.length, c.length);
1281 return result;
1282 }
1283
1284
1285
1286
1287
1288
1289 public static byte [] head(final byte [] a, final int length) {
1290 if (a.length < length) {
1291 return null;
1292 }
1293 byte [] result = new byte[length];
1294 System.arraycopy(a, 0, result, 0, length);
1295 return result;
1296 }
1297
1298
1299
1300
1301
1302
1303 public static byte [] tail(final byte [] a, final int length) {
1304 if (a.length < length) {
1305 return null;
1306 }
1307 byte [] result = new byte[length];
1308 System.arraycopy(a, a.length - length, result, 0, length);
1309 return result;
1310 }
1311
1312
1313
1314
1315
1316
1317 public static byte [] padHead(final byte [] a, final int length) {
1318 byte [] padding = new byte[length];
1319 for (int i = 0; i < length; i++) {
1320 padding[i] = 0;
1321 }
1322 return add(padding,a);
1323 }
1324
1325
1326
1327
1328
1329
1330 public static byte [] padTail(final byte [] a, final int length) {
1331 byte [] padding = new byte[length];
1332 for (int i = 0; i < length; i++) {
1333 padding[i] = 0;
1334 }
1335 return add(a,padding);
1336 }
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347 public static byte [][] split(final byte [] a, final byte [] b, final int num) {
1348 return split(a, b, false, num);
1349 }
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363 public static byte[][] split(final byte[] a, final byte[] b,
1364 boolean inclusive, final int num) {
1365 byte[][] ret = new byte[num + 2][];
1366 int i = 0;
1367 Iterable<byte[]> iter = iterateOnSplits(a, b, inclusive, num);
1368 if (iter == null)
1369 return null;
1370 for (byte[] elem : iter) {
1371 ret[i++] = elem;
1372 }
1373 return ret;
1374 }
1375
1376
1377
1378
1379 public static Iterable<byte[]> iterateOnSplits(final byte[] a,
1380 final byte[] b, final int num)
1381 {
1382 return iterateOnSplits(a, b, false, num);
1383 }
1384
1385
1386
1387
1388 public static Iterable<byte[]> iterateOnSplits(
1389 final byte[] a, final byte[]b, boolean inclusive, final int num)
1390 {
1391 byte [] aPadded;
1392 byte [] bPadded;
1393 if (a.length < b.length) {
1394 aPadded = padTail(a, b.length - a.length);
1395 bPadded = b;
1396 } else if (b.length < a.length) {
1397 aPadded = a;
1398 bPadded = padTail(b, a.length - b.length);
1399 } else {
1400 aPadded = a;
1401 bPadded = b;
1402 }
1403 if (compareTo(aPadded,bPadded) >= 0) {
1404 throw new IllegalArgumentException("b <= a");
1405 }
1406 if (num <= 0) {
1407 throw new IllegalArgumentException("num cannot be < 0");
1408 }
1409 byte [] prependHeader = {1, 0};
1410 final BigInteger startBI = new BigInteger(add(prependHeader, aPadded));
1411 final BigInteger stopBI = new BigInteger(add(prependHeader, bPadded));
1412 BigInteger diffBI = stopBI.subtract(startBI);
1413 if (inclusive) {
1414 diffBI = diffBI.add(BigInteger.ONE);
1415 }
1416 final BigInteger splitsBI = BigInteger.valueOf(num + 1);
1417 if(diffBI.compareTo(splitsBI) < 0) {
1418 return null;
1419 }
1420 final BigInteger intervalBI;
1421 try {
1422 intervalBI = diffBI.divide(splitsBI);
1423 } catch(Exception e) {
1424 LOG.error("Exception caught during division", e);
1425 return null;
1426 }
1427
1428 final Iterator<byte[]> iterator = new Iterator<byte[]>() {
1429 private int i = -1;
1430
1431 @Override
1432 public boolean hasNext() {
1433 return i < num+1;
1434 }
1435
1436 @Override
1437 public byte[] next() {
1438 i++;
1439 if (i == 0) return a;
1440 if (i == num + 1) return b;
1441
1442 BigInteger curBI = startBI.add(intervalBI.multiply(BigInteger.valueOf(i)));
1443 byte [] padded = curBI.toByteArray();
1444 if (padded[1] == 0)
1445 padded = tail(padded, padded.length - 2);
1446 else
1447 padded = tail(padded, padded.length - 1);
1448 return padded;
1449 }
1450
1451 @Override
1452 public void remove() {
1453 throw new UnsupportedOperationException();
1454 }
1455
1456 };
1457
1458 return new Iterable<byte[]>() {
1459 @Override
1460 public Iterator<byte[]> iterator() {
1461 return iterator;
1462 }
1463 };
1464 }
1465
1466
1467
1468
1469
1470
1471 public static int hashCode(byte[] bytes, int offset, int length) {
1472 int hash = 1;
1473 for (int i = offset; i < offset + length; i++)
1474 hash = (31 * hash) + (int) bytes[i];
1475 return hash;
1476 }
1477
1478
1479
1480
1481
1482 public static byte [][] toByteArrays(final String [] t) {
1483 byte [][] result = new byte[t.length][];
1484 for (int i = 0; i < t.length; i++) {
1485 result[i] = Bytes.toBytes(t[i]);
1486 }
1487 return result;
1488 }
1489
1490
1491
1492
1493
1494
1495 public static byte [][] toByteArrays(final String column) {
1496 return toByteArrays(toBytes(column));
1497 }
1498
1499
1500
1501
1502
1503
1504 public static byte [][] toByteArrays(final byte [] column) {
1505 byte [][] result = new byte[1][];
1506 result[0] = column;
1507 return result;
1508 }
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525 public static int binarySearch(byte [][]arr, byte []key, int offset,
1526 int length, RawComparator<byte []> comparator) {
1527 int low = 0;
1528 int high = arr.length - 1;
1529
1530 while (low <= high) {
1531 int mid = (low+high) >>> 1;
1532
1533
1534 int cmp = comparator.compare(key, offset, length,
1535 arr[mid], 0, arr[mid].length);
1536
1537 if (cmp > 0)
1538 low = mid + 1;
1539
1540 else if (cmp < 0)
1541 high = mid - 1;
1542
1543 else
1544 return mid;
1545 }
1546 return - (low+1);
1547 }
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557 public static byte [] incrementBytes(byte[] value, long amount)
1558 {
1559 byte[] val = value;
1560 if (val.length < SIZEOF_LONG) {
1561
1562 byte [] newvalue;
1563 if (val[0] < 0) {
1564 newvalue = new byte[]{-1, -1, -1, -1, -1, -1, -1, -1};
1565 } else {
1566 newvalue = new byte[SIZEOF_LONG];
1567 }
1568 System.arraycopy(val, 0, newvalue, newvalue.length - val.length,
1569 val.length);
1570 val = newvalue;
1571 } else if (val.length > SIZEOF_LONG) {
1572 throw new IllegalArgumentException("Increment Bytes - value too big: " +
1573 val.length);
1574 }
1575 if(amount == 0) return val;
1576 if(val[0] < 0){
1577 return binaryIncrementNeg(val, amount);
1578 }
1579 return binaryIncrementPos(val, amount);
1580 }
1581
1582
1583 private static byte [] binaryIncrementPos(byte [] value, long amount) {
1584 long amo = amount;
1585 int sign = 1;
1586 if (amount < 0) {
1587 amo = -amount;
1588 sign = -1;
1589 }
1590 for(int i=0;i<value.length;i++) {
1591 int cur = ((int)amo % 256) * sign;
1592 amo = (amo >> 8);
1593 int val = value[value.length-i-1] & 0x0ff;
1594 int total = val + cur;
1595 if(total > 255) {
1596 amo += sign;
1597 total %= 256;
1598 } else if (total < 0) {
1599 amo -= sign;
1600 }
1601 value[value.length-i-1] = (byte)total;
1602 if (amo == 0) return value;
1603 }
1604 return value;
1605 }
1606
1607
1608 private static byte [] binaryIncrementNeg(byte [] value, long amount) {
1609 long amo = amount;
1610 int sign = 1;
1611 if (amount < 0) {
1612 amo = -amount;
1613 sign = -1;
1614 }
1615 for(int i=0;i<value.length;i++) {
1616 int cur = ((int)amo % 256) * sign;
1617 amo = (amo >> 8);
1618 int val = ((~value[value.length-i-1]) & 0x0ff) + 1;
1619 int total = cur - val;
1620 if(total >= 0) {
1621 amo += sign;
1622 } else if (total < -256) {
1623 amo -= sign;
1624 total %= 256;
1625 }
1626 value[value.length-i-1] = (byte)total;
1627 if (amo == 0) return value;
1628 }
1629 return value;
1630 }
1631
1632
1633
1634
1635 public static void writeStringFixedSize(final DataOutput out, String s,
1636 int size) throws IOException {
1637 byte[] b = toBytes(s);
1638 if (b.length > size) {
1639 throw new IOException("Trying to write " + b.length + " bytes (" +
1640 toStringBinary(b) + ") into a field of length " + size);
1641 }
1642
1643 out.writeBytes(s);
1644 for (int i = 0; i < size - s.length(); ++i)
1645 out.writeByte(0);
1646 }
1647
1648
1649
1650
1651 public static String readStringFixedSize(final DataInput in, int size)
1652 throws IOException {
1653 byte[] b = new byte[size];
1654 in.readFully(b);
1655 int n = b.length;
1656 while (n > 0 && b[n - 1] == 0)
1657 --n;
1658
1659 return toString(b, 0, n);
1660 }
1661
1662
1663
1664
1665
1666
1667
1668 public static byte [] copy(byte [] bytes) {
1669 if (bytes == null) return null;
1670 byte [] result = new byte[bytes.length];
1671 System.arraycopy(bytes, 0, result, 0, bytes.length);
1672 return result;
1673 }
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683 public static byte [] copy(byte [] bytes, final int offset, final int length) {
1684 if (bytes == null) return null;
1685 byte [] result = new byte[length];
1686 System.arraycopy(bytes, offset, result, 0, length);
1687 return result;
1688 }
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700 public static int unsignedBinarySearch(byte[] a, int fromIndex, int toIndex, byte key) {
1701 int unsignedKey = key & 0xff;
1702 int low = fromIndex;
1703 int high = toIndex - 1;
1704
1705 while (low <= high) {
1706 int mid = (low + high) >>> 1;
1707 int midVal = a[mid] & 0xff;
1708
1709 if (midVal < unsignedKey) {
1710 low = mid + 1;
1711 } else if (midVal > unsignedKey) {
1712 high = mid - 1;
1713 } else {
1714 return mid;
1715 }
1716 }
1717 return -(low + 1);
1718 }
1719
1720
1721
1722
1723
1724
1725
1726
1727 public static byte[] unsignedCopyAndIncrement(final byte[] input) {
1728 byte[] copy = copy(input);
1729 if (copy == null) {
1730 throw new IllegalArgumentException("cannot increment null array");
1731 }
1732 for (int i = copy.length - 1; i >= 0; --i) {
1733 if (copy[i] == -1) {
1734 copy[i] = 0;
1735 } else {
1736 ++copy[i];
1737 return copy;
1738 }
1739 }
1740
1741 byte[] out = new byte[copy.length + 1];
1742 out[0] = 1;
1743 System.arraycopy(copy, 0, out, 1, copy.length);
1744 return out;
1745 }
1746
1747 public static boolean equals(List<byte[]> a, List<byte[]> b) {
1748 if (a == null) {
1749 if (b == null) {
1750 return true;
1751 }
1752 return false;
1753 }
1754 if (b == null) {
1755 return false;
1756 }
1757 if (a.size() != b.size()) {
1758 return false;
1759 }
1760 for (int i = 0; i < a.size(); ++i) {
1761 if (!Bytes.equals(a.get(i), b.get(i))) {
1762 return false;
1763 }
1764 }
1765 return true;
1766 }
1767
1768 public static boolean isSorted(Collection<byte[]> arrays) {
1769 byte[] previous = new byte[0];
1770 for (byte[] array : IterableUtils.nullSafe(arrays)) {
1771 if (Bytes.compareTo(previous, array) > 0) {
1772 return false;
1773 }
1774 previous = array;
1775 }
1776 return true;
1777 }
1778
1779 public static List<byte[]> getUtf8ByteArrays(List<String> strings) {
1780 List<byte[]> byteArrays = Lists.newArrayListWithCapacity(CollectionUtils.nullSafeSize(strings));
1781 for (String s : IterableUtils.nullSafe(strings)) {
1782 byteArrays.add(Bytes.toBytes(s));
1783 }
1784 return byteArrays;
1785 }
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796 public static int indexOf(byte[] array, byte target) {
1797 for (int i = 0; i < array.length; i++) {
1798 if (array[i] == target) {
1799 return i;
1800 }
1801 }
1802 return -1;
1803 }
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816 public static int indexOf(byte[] array, byte[] target) {
1817 checkNotNull(array, "array");
1818 checkNotNull(target, "target");
1819 if (target.length == 0) {
1820 return 0;
1821 }
1822
1823 outer:
1824 for (int i = 0; i < array.length - target.length + 1; i++) {
1825 for (int j = 0; j < target.length; j++) {
1826 if (array[i + j] != target[j]) {
1827 continue outer;
1828 }
1829 }
1830 return i;
1831 }
1832 return -1;
1833 }
1834
1835
1836
1837
1838
1839
1840 public static boolean contains(byte[] array, byte target) {
1841 return indexOf(array, target) > -1;
1842 }
1843
1844
1845
1846
1847
1848
1849 public static boolean contains(byte[] array, byte[] target) {
1850 return indexOf(array, target) > -1;
1851 }
1852
1853
1854
1855
1856
1857 public static void zero(byte[] b) {
1858 zero(b, 0, b.length);
1859 }
1860
1861
1862
1863
1864
1865
1866
1867 public static void zero(byte[] b, int offset, int length) {
1868 checkPositionIndex(offset, b.length, "offset");
1869 checkArgument(length > 0, "length must be greater than 0");
1870 checkPositionIndex(offset + length, b.length, "offset + length");
1871 Arrays.fill(b, offset, offset + length, (byte) 0);
1872 }
1873
1874 private static final SecureRandom RNG = new SecureRandom();
1875
1876
1877
1878
1879
1880 public static void random(byte[] b) {
1881 RNG.nextBytes(b);
1882 }
1883
1884
1885
1886
1887
1888
1889
1890 public static void random(byte[] b, int offset, int length) {
1891 checkPositionIndex(offset, b.length, "offset");
1892 checkArgument(length > 0, "length must be greater than 0");
1893 checkPositionIndex(offset + length, b.length, "offset + length");
1894 byte[] buf = new byte[length];
1895 RNG.nextBytes(buf);
1896 System.arraycopy(buf, 0, b, offset, length);
1897 }
1898 }