1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.util;
19
20 import static com.google.common.base.Preconditions.checkArgument;
21 import static com.google.common.base.Preconditions.checkNotNull;
22 import static com.google.common.base.Preconditions.checkPositionIndex;
23
24 import java.io.DataInput;
25 import java.io.DataOutput;
26 import java.io.IOException;
27 import java.lang.reflect.Field;
28 import java.math.BigDecimal;
29 import java.math.BigInteger;
30 import java.nio.ByteBuffer;
31 import java.nio.ByteOrder;
32 import java.nio.charset.Charset;
33 import java.security.AccessController;
34 import java.security.PrivilegedAction;
35 import java.security.SecureRandom;
36 import java.util.Arrays;
37 import java.util.Collection;
38 import java.util.Comparator;
39 import java.util.Iterator;
40 import java.util.List;
41
42 import org.apache.commons.logging.Log;
43 import org.apache.commons.logging.LogFactory;
44 import org.apache.hadoop.classification.InterfaceAudience;
45 import org.apache.hadoop.classification.InterfaceStability;
46 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
47 import org.apache.hadoop.io.RawComparator;
48 import org.apache.hadoop.io.WritableComparator;
49 import org.apache.hadoop.io.WritableUtils;
50
51 import sun.misc.Unsafe;
52
53 import com.google.common.annotations.VisibleForTesting;
54 import com.google.common.collect.Lists;
55
56
57
58
59
60
61 @InterfaceAudience.Public
62 @InterfaceStability.Stable
63 public class Bytes {
64
65
66 private static final String UTF8_ENCODING = "UTF-8";
67
68
69
70 private static final Charset UTF8_CHARSET = Charset.forName(UTF8_ENCODING);
71
72
73 private static final byte [] EMPTY_BYTE_ARRAY = new byte [0];
74
75 private static final Log LOG = LogFactory.getLog(Bytes.class);
76
77
78
79
80 public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE;
81
82
83
84
85 public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN;
86
87
88
89
90 public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE;
91
92
93
94
95 public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE;
96
97
98
99
100 public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE;
101
102
103
104
105 public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE;
106
107
108
109
110 public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE;
111
112
113
114
115 public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE;
116
117
118
119
120
121
122
123
124 public static final int ESTIMATED_HEAP_TAX = 16;
125
126
127
128
129 public static class ByteArrayComparator implements RawComparator<byte []> {
130
131
132
133 public ByteArrayComparator() {
134 super();
135 }
136 @Override
137 public int compare(byte [] left, byte [] right) {
138 return compareTo(left, right);
139 }
140 @Override
141 public int compare(byte [] b1, int s1, int l1, byte [] b2, int s2, int l2) {
142 return LexicographicalComparerHolder.BEST_COMPARER.
143 compareTo(b1, s1, l1, b2, s2, l2);
144 }
145 }
146
147
148
149
150
151
152
153
154
155 public static class RowEndKeyComparator extends ByteArrayComparator {
156 @Override
157 public int compare(byte[] left, byte[] right) {
158 return compare(left, 0, left.length, right, 0, right.length);
159 }
160 @Override
161 public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
162 if (b1 == b2 && s1 == s2 && l1 == l2) {
163 return 0;
164 }
165 if (l1 == 0) {
166 return l2;
167 }
168 if (l2 == 0) {
169 return -1;
170 }
171 return super.compare(b1, s1, l1, b2, s2, l2);
172 }
173 }
174
175
176
177
178 public static Comparator<byte []> BYTES_COMPARATOR =
179 new ByteArrayComparator();
180
181
182
183
184 public static RawComparator<byte []> BYTES_RAWCOMPARATOR =
185 new ByteArrayComparator();
186
187
188
189
190
191
192
193 public static byte [] readByteArray(final DataInput in)
194 throws IOException {
195 int len = WritableUtils.readVInt(in);
196 if (len < 0) {
197 throw new NegativeArraySizeException(Integer.toString(len));
198 }
199 byte [] result = new byte[len];
200 in.readFully(result, 0, len);
201 return result;
202 }
203
204
205
206
207
208
209
210 public static byte [] readByteArrayThrowsRuntime(final DataInput in) {
211 try {
212 return readByteArray(in);
213 } catch (Exception e) {
214 throw new RuntimeException(e);
215 }
216 }
217
218
219
220
221
222
223
224 public static void writeByteArray(final DataOutput out, final byte [] b)
225 throws IOException {
226 if(b == null) {
227 WritableUtils.writeVInt(out, 0);
228 } else {
229 writeByteArray(out, b, 0, b.length);
230 }
231 }
232
233
234
235
236
237
238
239
240
241 public static void writeByteArray(final DataOutput out, final byte [] b,
242 final int offset, final int length)
243 throws IOException {
244 WritableUtils.writeVInt(out, length);
245 out.write(b, offset, length);
246 }
247
248
249
250
251
252
253
254
255
256
257 public static int writeByteArray(final byte [] tgt, final int tgtOffset,
258 final byte [] src, final int srcOffset, final int srcLength) {
259 byte [] vint = vintToBytes(srcLength);
260 System.arraycopy(vint, 0, tgt, tgtOffset, vint.length);
261 int offset = tgtOffset + vint.length;
262 System.arraycopy(src, srcOffset, tgt, offset, srcLength);
263 return offset + srcLength;
264 }
265
266
267
268
269
270
271
272
273
274
275 public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes,
276 int srcOffset, int srcLength) {
277 System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength);
278 return tgtOffset + srcLength;
279 }
280
281
282
283
284
285
286
287
288 public static int putByte(byte[] bytes, int offset, byte b) {
289 bytes[offset] = b;
290 return offset + 1;
291 }
292
293
294
295
296
297
298
299
300
301
302
303 public static byte[] toBytes(ByteBuffer buf) {
304 ByteBuffer dup = buf.duplicate();
305 dup.position(0);
306 return readBytes(dup);
307 }
308
309 private static byte[] readBytes(ByteBuffer buf) {
310 byte [] result = new byte[buf.remaining()];
311 buf.get(result);
312 return result;
313 }
314
315
316
317
318
319 public static String toString(final byte [] b) {
320 if (b == null) {
321 return null;
322 }
323 return toString(b, 0, b.length);
324 }
325
326
327
328
329
330
331
332 public static String toString(final byte [] b1,
333 String sep,
334 final byte [] b2) {
335 return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length);
336 }
337
338
339
340
341
342
343
344
345
346
347 public static String toString(final byte [] b, int off, int len) {
348 if (b == null) {
349 return null;
350 }
351 if (len == 0) {
352 return "";
353 }
354 return new String(b, off, len, UTF8_CHARSET);
355 }
356
357
358
359
360
361
362
363
364 public static String toStringBinary(final byte [] b) {
365 if (b == null)
366 return "null";
367 return toStringBinary(b, 0, b.length);
368 }
369
370
371
372
373
374
375
376
377
378
379
380
381 public static String toStringBinary(ByteBuffer buf) {
382 if (buf == null)
383 return "null";
384 if (buf.hasArray()) {
385 return toStringBinary(buf.array(), buf.arrayOffset(), buf.limit());
386 }
387 return toStringBinary(toBytes(buf));
388 }
389
390
391
392
393
394
395
396
397
398
399
400 public static String toStringBinary(final byte [] b, int off, int len) {
401 StringBuilder result = new StringBuilder();
402
403 if (off >= b.length) return result.toString();
404 if (off + len > b.length) len = b.length - off;
405 for (int i = off; i < off + len ; ++i ) {
406 int ch = b[i] & 0xFF;
407 if ( (ch >= '0' && ch <= '9')
408 || (ch >= 'A' && ch <= 'Z')
409 || (ch >= 'a' && ch <= 'z')
410 || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0 ) {
411 result.append((char)ch);
412 } else {
413 result.append(String.format("\\x%02X", ch));
414 }
415 }
416 return result.toString();
417 }
418
419 private static boolean isHexDigit(char c) {
420 return
421 (c >= 'A' && c <= 'F') ||
422 (c >= '0' && c <= '9');
423 }
424
425
426
427
428
429
430
431 public static byte toBinaryFromHex(byte ch) {
432 if ( ch >= 'A' && ch <= 'F' )
433 return (byte) ((byte)10 + (byte) (ch - 'A'));
434
435 return (byte) (ch - '0');
436 }
437
438 public static byte [] toBytesBinary(String in) {
439
440 byte [] b = new byte[in.length()];
441 int size = 0;
442 for (int i = 0; i < in.length(); ++i) {
443 char ch = in.charAt(i);
444 if (ch == '\\' && in.length() > i+1 && in.charAt(i+1) == 'x') {
445
446 char hd1 = in.charAt(i+2);
447 char hd2 = in.charAt(i+3);
448
449
450 if (!isHexDigit(hd1) ||
451 !isHexDigit(hd2)) {
452
453 continue;
454 }
455
456 byte d = (byte) ((toBinaryFromHex((byte)hd1) << 4) + toBinaryFromHex((byte)hd2));
457
458 b[size++] = d;
459 i += 3;
460 } else {
461 b[size++] = (byte) ch;
462 }
463 }
464
465 byte [] b2 = new byte[size];
466 System.arraycopy(b, 0, b2, 0, size);
467 return b2;
468 }
469
470
471
472
473
474
475 public static byte[] toBytes(String s) {
476 return s.getBytes(UTF8_CHARSET);
477 }
478
479
480
481
482
483
484
485
486 public static byte [] toBytes(final boolean b) {
487 return new byte[] { b ? (byte) -1 : (byte) 0 };
488 }
489
490
491
492
493
494
495 public static boolean toBoolean(final byte [] b) {
496 if (b.length != 1) {
497 throw new IllegalArgumentException("Array has wrong size: " + b.length);
498 }
499 return b[0] != (byte) 0;
500 }
501
502
503
504
505
506
507
508 public static byte[] toBytes(long val) {
509 byte [] b = new byte[8];
510 for (int i = 7; i > 0; i--) {
511 b[i] = (byte) val;
512 val >>>= 8;
513 }
514 b[0] = (byte) val;
515 return b;
516 }
517
518
519
520
521
522
523
524 public static long toLong(byte[] bytes) {
525 return toLong(bytes, 0, SIZEOF_LONG);
526 }
527
528
529
530
531
532
533
534
535
536 public static long toLong(byte[] bytes, int offset) {
537 return toLong(bytes, offset, SIZEOF_LONG);
538 }
539
540
541
542
543
544
545
546
547
548
549
550 public static long toLong(byte[] bytes, int offset, final int length) {
551 if (length != SIZEOF_LONG || offset + length > bytes.length) {
552 throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
553 }
554 long l = 0;
555 for(int i = offset; i < offset + length; i++) {
556 l <<= 8;
557 l ^= bytes[i] & 0xFF;
558 }
559 return l;
560 }
561
562 private static IllegalArgumentException
563 explainWrongLengthOrOffset(final byte[] bytes,
564 final int offset,
565 final int length,
566 final int expectedLength) {
567 String reason;
568 if (length != expectedLength) {
569 reason = "Wrong length: " + length + ", expected " + expectedLength;
570 } else {
571 reason = "offset (" + offset + ") + length (" + length + ") exceed the"
572 + " capacity of the array: " + bytes.length;
573 }
574 return new IllegalArgumentException(reason);
575 }
576
577
578
579
580
581
582
583
584
585
586 public static int putLong(byte[] bytes, int offset, long val) {
587 if (bytes.length - offset < SIZEOF_LONG) {
588 throw new IllegalArgumentException("Not enough room to put a long at"
589 + " offset " + offset + " in a " + bytes.length + " byte array");
590 }
591 for(int i = offset + 7; i > offset; i--) {
592 bytes[i] = (byte) val;
593 val >>>= 8;
594 }
595 bytes[offset] = (byte) val;
596 return offset + SIZEOF_LONG;
597 }
598
599
600
601
602
603
604 public static float toFloat(byte [] bytes) {
605 return toFloat(bytes, 0);
606 }
607
608
609
610
611
612
613
614 public static float toFloat(byte [] bytes, int offset) {
615 return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
616 }
617
618
619
620
621
622
623
624 public static int putFloat(byte [] bytes, int offset, float f) {
625 return putInt(bytes, offset, Float.floatToRawIntBits(f));
626 }
627
628
629
630
631
632 public static byte [] toBytes(final float f) {
633
634 return Bytes.toBytes(Float.floatToRawIntBits(f));
635 }
636
637
638
639
640
641 public static double toDouble(final byte [] bytes) {
642 return toDouble(bytes, 0);
643 }
644
645
646
647
648
649
650 public static double toDouble(final byte [] bytes, final int offset) {
651 return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
652 }
653
654
655
656
657
658
659
660 public static int putDouble(byte [] bytes, int offset, double d) {
661 return putLong(bytes, offset, Double.doubleToLongBits(d));
662 }
663
664
665
666
667
668
669
670
671 public static byte [] toBytes(final double d) {
672
673 return Bytes.toBytes(Double.doubleToRawLongBits(d));
674 }
675
676
677
678
679
680
681
682
683 public static byte[] toBytes(int val) {
684 byte [] b = new byte[4];
685 for(int i = 3; i > 0; i--) {
686 b[i] = (byte) val;
687 val >>>= 8;
688 }
689 b[0] = (byte) val;
690 return b;
691 }
692
693
694
695
696
697
698 public static int toInt(byte[] bytes) {
699 return toInt(bytes, 0, SIZEOF_INT);
700 }
701
702
703
704
705
706
707
708 public static int toInt(byte[] bytes, int offset) {
709 return toInt(bytes, offset, SIZEOF_INT);
710 }
711
712
713
714
715
716
717
718
719
720
721 public static int toInt(byte[] bytes, int offset, final int length) {
722 if (length != SIZEOF_INT || offset + length > bytes.length) {
723 throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
724 }
725 int n = 0;
726 for(int i = offset; i < (offset + length); i++) {
727 n <<= 8;
728 n ^= bytes[i] & 0xFF;
729 }
730 return n;
731 }
732
733
734
735
736
737
738
739
740
741
742 public static int putInt(byte[] bytes, int offset, int val) {
743 if (bytes.length - offset < SIZEOF_INT) {
744 throw new IllegalArgumentException("Not enough room to put an int at"
745 + " offset " + offset + " in a " + bytes.length + " byte array");
746 }
747 for(int i= offset + 3; i > offset; i--) {
748 bytes[i] = (byte) val;
749 val >>>= 8;
750 }
751 bytes[offset] = (byte) val;
752 return offset + SIZEOF_INT;
753 }
754
755
756
757
758
759
760 public static byte[] toBytes(short val) {
761 byte[] b = new byte[SIZEOF_SHORT];
762 b[1] = (byte) val;
763 val >>= 8;
764 b[0] = (byte) val;
765 return b;
766 }
767
768
769
770
771
772
773 public static short toShort(byte[] bytes) {
774 return toShort(bytes, 0, SIZEOF_SHORT);
775 }
776
777
778
779
780
781
782
783 public static short toShort(byte[] bytes, int offset) {
784 return toShort(bytes, offset, SIZEOF_SHORT);
785 }
786
787
788
789
790
791
792
793
794
795
796 public static short toShort(byte[] bytes, int offset, final int length) {
797 if (length != SIZEOF_SHORT || offset + length > bytes.length) {
798 throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
799 }
800 short n = 0;
801 n ^= bytes[offset] & 0xFF;
802 n <<= 8;
803 n ^= bytes[offset+1] & 0xFF;
804 return n;
805 }
806
807
808
809
810
811
812
813
814
815
816 public static byte[] getBytes(ByteBuffer buf) {
817 return readBytes(buf.duplicate());
818 }
819
820
821
822
823
824
825
826
827
828
829 public static int putShort(byte[] bytes, int offset, short val) {
830 if (bytes.length - offset < SIZEOF_SHORT) {
831 throw new IllegalArgumentException("Not enough room to put a short at"
832 + " offset " + offset + " in a " + bytes.length + " byte array");
833 }
834 bytes[offset+1] = (byte) val;
835 val >>= 8;
836 bytes[offset] = (byte) val;
837 return offset + SIZEOF_SHORT;
838 }
839
840
841
842
843
844
845
846 public static byte[] toBytes(BigDecimal val) {
847 byte[] valueBytes = val.unscaledValue().toByteArray();
848 byte[] result = new byte[valueBytes.length + SIZEOF_INT];
849 int offset = putInt(result, 0, val.scale());
850 putBytes(result, offset, valueBytes, 0, valueBytes.length);
851 return result;
852 }
853
854
855
856
857
858
859
860
861 public static BigDecimal toBigDecimal(byte[] bytes) {
862 return toBigDecimal(bytes, 0, bytes.length);
863 }
864
865
866
867
868
869
870
871
872
873 public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) {
874 if (bytes == null || length < SIZEOF_INT + 1 ||
875 (offset + length > bytes.length)) {
876 return null;
877 }
878
879 int scale = toInt(bytes, offset);
880 byte[] tcBytes = new byte[length - SIZEOF_INT];
881 System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT);
882 return new BigDecimal(new BigInteger(tcBytes), scale);
883 }
884
885
886
887
888
889
890
891
892
893 public static int putBigDecimal(byte[] bytes, int offset, BigDecimal val) {
894 if (bytes == null) {
895 return offset;
896 }
897
898 byte[] valueBytes = val.unscaledValue().toByteArray();
899 byte[] result = new byte[valueBytes.length + SIZEOF_INT];
900 offset = putInt(result, offset, val.scale());
901 return putBytes(result, offset, valueBytes, 0, valueBytes.length);
902 }
903
904
905
906
907
908 public static byte [] vintToBytes(final long vint) {
909 long i = vint;
910 int size = WritableUtils.getVIntSize(i);
911 byte [] result = new byte[size];
912 int offset = 0;
913 if (i >= -112 && i <= 127) {
914 result[offset] = (byte) i;
915 return result;
916 }
917
918 int len = -112;
919 if (i < 0) {
920 i ^= -1L;
921 len = -120;
922 }
923
924 long tmp = i;
925 while (tmp != 0) {
926 tmp = tmp >> 8;
927 len--;
928 }
929
930 result[offset++] = (byte) len;
931
932 len = (len < -120) ? -(len + 120) : -(len + 112);
933
934 for (int idx = len; idx != 0; idx--) {
935 int shiftbits = (idx - 1) * 8;
936 long mask = 0xFFL << shiftbits;
937 result[offset++] = (byte)((i & mask) >> shiftbits);
938 }
939 return result;
940 }
941
942
943
944
945
946 public static long bytesToVint(final byte [] buffer) {
947 int offset = 0;
948 byte firstByte = buffer[offset++];
949 int len = WritableUtils.decodeVIntSize(firstByte);
950 if (len == 1) {
951 return firstByte;
952 }
953 long i = 0;
954 for (int idx = 0; idx < len-1; idx++) {
955 byte b = buffer[offset++];
956 i = i << 8;
957 i = i | (b & 0xFF);
958 }
959 return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
960 }
961
962
963
964
965
966
967
968
969 public static long readVLong(final byte [] buffer, final int offset)
970 throws IOException {
971 byte firstByte = buffer[offset];
972 int len = WritableUtils.decodeVIntSize(firstByte);
973 if (len == 1) {
974 return firstByte;
975 }
976 long i = 0;
977 for (int idx = 0; idx < len-1; idx++) {
978 byte b = buffer[offset + 1 + idx];
979 i = i << 8;
980 i = i | (b & 0xFF);
981 }
982 return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
983 }
984
985
986
987
988
989
990 public static int compareTo(final byte [] left, final byte [] right) {
991 return LexicographicalComparerHolder.BEST_COMPARER.
992 compareTo(left, 0, left.length, right, 0, right.length);
993 }
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006 public static int compareTo(byte[] buffer1, int offset1, int length1,
1007 byte[] buffer2, int offset2, int length2) {
1008 return LexicographicalComparerHolder.BEST_COMPARER.
1009 compareTo(buffer1, offset1, length1, buffer2, offset2, length2);
1010 }
1011
1012 interface Comparer<T> {
1013 int compareTo(
1014 T buffer1, int offset1, int length1, T buffer2, int offset2, int length2
1015 );
1016 }
1017
1018 @VisibleForTesting
1019 static Comparer<byte[]> lexicographicalComparerJavaImpl() {
1020 return LexicographicalComparerHolder.PureJavaComparer.INSTANCE;
1021 }
1022
1023
1024
1025
1026
1027
1028
1029
1030 @VisibleForTesting
1031 static class LexicographicalComparerHolder {
1032 static final String UNSAFE_COMPARER_NAME =
1033 LexicographicalComparerHolder.class.getName() + "$UnsafeComparer";
1034
1035 static final Comparer<byte[]> BEST_COMPARER = getBestComparer();
1036
1037
1038
1039
1040 static Comparer<byte[]> getBestComparer() {
1041 try {
1042 Class<?> theClass = Class.forName(UNSAFE_COMPARER_NAME);
1043
1044
1045 @SuppressWarnings("unchecked")
1046 Comparer<byte[]> comparer =
1047 (Comparer<byte[]>) theClass.getEnumConstants()[0];
1048 return comparer;
1049 } catch (Throwable t) {
1050 return lexicographicalComparerJavaImpl();
1051 }
1052 }
1053
1054 enum PureJavaComparer implements Comparer<byte[]> {
1055 INSTANCE;
1056
1057 @Override
1058 public int compareTo(byte[] buffer1, int offset1, int length1,
1059 byte[] buffer2, int offset2, int length2) {
1060
1061 if (buffer1 == buffer2 &&
1062 offset1 == offset2 &&
1063 length1 == length2) {
1064 return 0;
1065 }
1066
1067 int end1 = offset1 + length1;
1068 int end2 = offset2 + length2;
1069 for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) {
1070 int a = (buffer1[i] & 0xff);
1071 int b = (buffer2[j] & 0xff);
1072 if (a != b) {
1073 return a - b;
1074 }
1075 }
1076 return length1 - length2;
1077 }
1078 }
1079
1080 @VisibleForTesting
1081 enum UnsafeComparer implements Comparer<byte[]> {
1082 INSTANCE;
1083
1084 static final Unsafe theUnsafe;
1085
1086
1087 static final int BYTE_ARRAY_BASE_OFFSET;
1088
1089 static {
1090 theUnsafe = (Unsafe) AccessController.doPrivileged(
1091 new PrivilegedAction<Object>() {
1092 @Override
1093 public Object run() {
1094 try {
1095 Field f = Unsafe.class.getDeclaredField("theUnsafe");
1096 f.setAccessible(true);
1097 return f.get(null);
1098 } catch (NoSuchFieldException e) {
1099
1100
1101 throw new Error();
1102 } catch (IllegalAccessException e) {
1103 throw new Error();
1104 }
1105 }
1106 });
1107
1108 BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class);
1109
1110
1111 if (theUnsafe.arrayIndexScale(byte[].class) != 1) {
1112 throw new AssertionError();
1113 }
1114 }
1115
1116 static final boolean littleEndian =
1117 ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN);
1118
1119
1120
1121
1122
1123 static boolean lessThanUnsigned(long x1, long x2) {
1124 return (x1 + Long.MIN_VALUE) < (x2 + Long.MIN_VALUE);
1125 }
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138 @Override
1139 public int compareTo(byte[] buffer1, int offset1, int length1,
1140 byte[] buffer2, int offset2, int length2) {
1141
1142 if (buffer1 == buffer2 &&
1143 offset1 == offset2 &&
1144 length1 == length2) {
1145 return 0;
1146 }
1147 int minLength = Math.min(length1, length2);
1148 int minWords = minLength / SIZEOF_LONG;
1149 int offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET;
1150 int offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET;
1151
1152
1153
1154
1155
1156
1157 for (int i = 0; i < minWords * SIZEOF_LONG; i += SIZEOF_LONG) {
1158 long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i);
1159 long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i);
1160 long diff = lw ^ rw;
1161
1162 if (diff != 0) {
1163 if (!littleEndian) {
1164 return lessThanUnsigned(lw, rw) ? -1 : 1;
1165 }
1166
1167
1168 int n = 0;
1169 int y;
1170 int x = (int) diff;
1171 if (x == 0) {
1172 x = (int) (diff >>> 32);
1173 n = 32;
1174 }
1175
1176 y = x << 16;
1177 if (y == 0) {
1178 n += 16;
1179 } else {
1180 x = y;
1181 }
1182
1183 y = x << 8;
1184 if (y == 0) {
1185 n += 8;
1186 }
1187 return (int) (((lw >>> n) & 0xFFL) - ((rw >>> n) & 0xFFL));
1188 }
1189 }
1190
1191
1192 for (int i = minWords * SIZEOF_LONG; i < minLength; i++) {
1193 int a = (buffer1[offset1 + i] & 0xff);
1194 int b = (buffer2[offset2 + i] & 0xff);
1195 if (a != b) {
1196 return a - b;
1197 }
1198 }
1199 return length1 - length2;
1200 }
1201 }
1202 }
1203
1204
1205
1206
1207
1208
1209 public static boolean equals(final byte [] left, final byte [] right) {
1210
1211
1212 if (left == right) return true;
1213 if (left == null || right == null) return false;
1214 if (left.length != right.length) return false;
1215 if (left.length == 0) return true;
1216
1217
1218
1219
1220 if (left[left.length - 1] != right[right.length - 1]) return false;
1221
1222 return compareTo(left, right) == 0;
1223 }
1224
1225 public static boolean equals(final byte[] left, int leftOffset, int leftLen,
1226 final byte[] right, int rightOffset, int rightLen) {
1227
1228 if (left == right &&
1229 leftOffset == rightOffset &&
1230 leftLen == rightLen) {
1231 return true;
1232 }
1233
1234 if (leftLen != rightLen) {
1235 return false;
1236 }
1237 if (leftLen == 0) {
1238 return true;
1239 }
1240
1241
1242
1243
1244 if (left[leftOffset + leftLen - 1] != right[rightOffset + rightLen - 1]) return false;
1245
1246 return LexicographicalComparerHolder.BEST_COMPARER.
1247 compareTo(left, leftOffset, leftLen, right, rightOffset, rightLen) == 0;
1248 }
1249
1250
1251
1252
1253
1254
1255 public static boolean startsWith(byte[] bytes, byte[] prefix) {
1256 return bytes != null && prefix != null &&
1257 bytes.length >= prefix.length &&
1258 LexicographicalComparerHolder.BEST_COMPARER.
1259 compareTo(bytes, 0, prefix.length, prefix, 0, prefix.length) == 0;
1260 }
1261
1262
1263
1264
1265
1266
1267
1268 public static int hashCode(final byte [] b) {
1269 return hashCode(b, b.length);
1270 }
1271
1272
1273
1274
1275
1276
1277
1278
1279 public static int hashCode(final byte [] b, final int length) {
1280 return WritableComparator.hashBytes(b, length);
1281 }
1282
1283
1284
1285
1286
1287
1288 public static Integer mapKey(final byte [] b) {
1289 return hashCode(b);
1290 }
1291
1292
1293
1294
1295
1296
1297
1298 public static Integer mapKey(final byte [] b, final int length) {
1299 return hashCode(b, length);
1300 }
1301
1302
1303
1304
1305
1306
1307 public static byte [] add(final byte [] a, final byte [] b) {
1308 return add(a, b, EMPTY_BYTE_ARRAY);
1309 }
1310
1311
1312
1313
1314
1315
1316
1317 public static byte [] add(final byte [] a, final byte [] b, final byte [] c) {
1318 byte [] result = new byte[a.length + b.length + c.length];
1319 System.arraycopy(a, 0, result, 0, a.length);
1320 System.arraycopy(b, 0, result, a.length, b.length);
1321 System.arraycopy(c, 0, result, a.length + b.length, c.length);
1322 return result;
1323 }
1324
1325
1326
1327
1328
1329
1330 public static byte [] head(final byte [] a, final int length) {
1331 if (a.length < length) {
1332 return null;
1333 }
1334 byte [] result = new byte[length];
1335 System.arraycopy(a, 0, result, 0, length);
1336 return result;
1337 }
1338
1339
1340
1341
1342
1343
1344 public static byte [] tail(final byte [] a, final int length) {
1345 if (a.length < length) {
1346 return null;
1347 }
1348 byte [] result = new byte[length];
1349 System.arraycopy(a, a.length - length, result, 0, length);
1350 return result;
1351 }
1352
1353
1354
1355
1356
1357
1358 public static byte [] padHead(final byte [] a, final int length) {
1359 byte [] padding = new byte[length];
1360 for (int i = 0; i < length; i++) {
1361 padding[i] = 0;
1362 }
1363 return add(padding,a);
1364 }
1365
1366
1367
1368
1369
1370
1371 public static byte [] padTail(final byte [] a, final int length) {
1372 byte [] padding = new byte[length];
1373 for (int i = 0; i < length; i++) {
1374 padding[i] = 0;
1375 }
1376 return add(a,padding);
1377 }
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388 public static byte [][] split(final byte [] a, final byte [] b, final int num) {
1389 return split(a, b, false, num);
1390 }
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404 public static byte[][] split(final byte[] a, final byte[] b,
1405 boolean inclusive, final int num) {
1406 byte[][] ret = new byte[num + 2][];
1407 int i = 0;
1408 Iterable<byte[]> iter = iterateOnSplits(a, b, inclusive, num);
1409 if (iter == null)
1410 return null;
1411 for (byte[] elem : iter) {
1412 ret[i++] = elem;
1413 }
1414 return ret;
1415 }
1416
1417
1418
1419
1420 public static Iterable<byte[]> iterateOnSplits(final byte[] a,
1421 final byte[] b, final int num)
1422 {
1423 return iterateOnSplits(a, b, false, num);
1424 }
1425
1426
1427
1428
1429 public static Iterable<byte[]> iterateOnSplits(
1430 final byte[] a, final byte[]b, boolean inclusive, final int num)
1431 {
1432 byte [] aPadded;
1433 byte [] bPadded;
1434 if (a.length < b.length) {
1435 aPadded = padTail(a, b.length - a.length);
1436 bPadded = b;
1437 } else if (b.length < a.length) {
1438 aPadded = a;
1439 bPadded = padTail(b, a.length - b.length);
1440 } else {
1441 aPadded = a;
1442 bPadded = b;
1443 }
1444 if (compareTo(aPadded,bPadded) >= 0) {
1445 throw new IllegalArgumentException("b <= a");
1446 }
1447 if (num <= 0) {
1448 throw new IllegalArgumentException("num cannot be < 0");
1449 }
1450 byte [] prependHeader = {1, 0};
1451 final BigInteger startBI = new BigInteger(add(prependHeader, aPadded));
1452 final BigInteger stopBI = new BigInteger(add(prependHeader, bPadded));
1453 BigInteger diffBI = stopBI.subtract(startBI);
1454 if (inclusive) {
1455 diffBI = diffBI.add(BigInteger.ONE);
1456 }
1457 final BigInteger splitsBI = BigInteger.valueOf(num + 1);
1458 if(diffBI.compareTo(splitsBI) < 0) {
1459 return null;
1460 }
1461 final BigInteger intervalBI;
1462 try {
1463 intervalBI = diffBI.divide(splitsBI);
1464 } catch(Exception e) {
1465 LOG.error("Exception caught during division", e);
1466 return null;
1467 }
1468
1469 final Iterator<byte[]> iterator = new Iterator<byte[]>() {
1470 private int i = -1;
1471
1472 @Override
1473 public boolean hasNext() {
1474 return i < num+1;
1475 }
1476
1477 @Override
1478 public byte[] next() {
1479 i++;
1480 if (i == 0) return a;
1481 if (i == num + 1) return b;
1482
1483 BigInteger curBI = startBI.add(intervalBI.multiply(BigInteger.valueOf(i)));
1484 byte [] padded = curBI.toByteArray();
1485 if (padded[1] == 0)
1486 padded = tail(padded, padded.length - 2);
1487 else
1488 padded = tail(padded, padded.length - 1);
1489 return padded;
1490 }
1491
1492 @Override
1493 public void remove() {
1494 throw new UnsupportedOperationException();
1495 }
1496
1497 };
1498
1499 return new Iterable<byte[]>() {
1500 @Override
1501 public Iterator<byte[]> iterator() {
1502 return iterator;
1503 }
1504 };
1505 }
1506
1507
1508
1509
1510
1511
1512 public static int hashCode(byte[] bytes, int offset, int length) {
1513 int hash = 1;
1514 for (int i = offset; i < offset + length; i++)
1515 hash = (31 * hash) + (int) bytes[i];
1516 return hash;
1517 }
1518
1519
1520
1521
1522
1523 public static byte [][] toByteArrays(final String [] t) {
1524 byte [][] result = new byte[t.length][];
1525 for (int i = 0; i < t.length; i++) {
1526 result[i] = Bytes.toBytes(t[i]);
1527 }
1528 return result;
1529 }
1530
1531
1532
1533
1534
1535
1536 public static byte [][] toByteArrays(final String column) {
1537 return toByteArrays(toBytes(column));
1538 }
1539
1540
1541
1542
1543
1544
1545 public static byte [][] toByteArrays(final byte [] column) {
1546 byte [][] result = new byte[1][];
1547 result[0] = column;
1548 return result;
1549 }
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566 public static int binarySearch(byte [][]arr, byte []key, int offset,
1567 int length, RawComparator<byte []> comparator) {
1568 int low = 0;
1569 int high = arr.length - 1;
1570
1571 while (low <= high) {
1572 int mid = (low+high) >>> 1;
1573
1574
1575 int cmp = comparator.compare(key, offset, length,
1576 arr[mid], 0, arr[mid].length);
1577
1578 if (cmp > 0)
1579 low = mid + 1;
1580
1581 else if (cmp < 0)
1582 high = mid - 1;
1583
1584 else
1585 return mid;
1586 }
1587 return - (low+1);
1588 }
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598 public static byte [] incrementBytes(byte[] value, long amount)
1599 {
1600 byte[] val = value;
1601 if (val.length < SIZEOF_LONG) {
1602
1603 byte [] newvalue;
1604 if (val[0] < 0) {
1605 newvalue = new byte[]{-1, -1, -1, -1, -1, -1, -1, -1};
1606 } else {
1607 newvalue = new byte[SIZEOF_LONG];
1608 }
1609 System.arraycopy(val, 0, newvalue, newvalue.length - val.length,
1610 val.length);
1611 val = newvalue;
1612 } else if (val.length > SIZEOF_LONG) {
1613 throw new IllegalArgumentException("Increment Bytes - value too big: " +
1614 val.length);
1615 }
1616 if(amount == 0) return val;
1617 if(val[0] < 0){
1618 return binaryIncrementNeg(val, amount);
1619 }
1620 return binaryIncrementPos(val, amount);
1621 }
1622
1623
1624 private static byte [] binaryIncrementPos(byte [] value, long amount) {
1625 long amo = amount;
1626 int sign = 1;
1627 if (amount < 0) {
1628 amo = -amount;
1629 sign = -1;
1630 }
1631 for(int i=0;i<value.length;i++) {
1632 int cur = ((int)amo % 256) * sign;
1633 amo = (amo >> 8);
1634 int val = value[value.length-i-1] & 0x0ff;
1635 int total = val + cur;
1636 if(total > 255) {
1637 amo += sign;
1638 total %= 256;
1639 } else if (total < 0) {
1640 amo -= sign;
1641 }
1642 value[value.length-i-1] = (byte)total;
1643 if (amo == 0) return value;
1644 }
1645 return value;
1646 }
1647
1648
1649 private static byte [] binaryIncrementNeg(byte [] value, long amount) {
1650 long amo = amount;
1651 int sign = 1;
1652 if (amount < 0) {
1653 amo = -amount;
1654 sign = -1;
1655 }
1656 for(int i=0;i<value.length;i++) {
1657 int cur = ((int)amo % 256) * sign;
1658 amo = (amo >> 8);
1659 int val = ((~value[value.length-i-1]) & 0x0ff) + 1;
1660 int total = cur - val;
1661 if(total >= 0) {
1662 amo += sign;
1663 } else if (total < -256) {
1664 amo -= sign;
1665 total %= 256;
1666 }
1667 value[value.length-i-1] = (byte)total;
1668 if (amo == 0) return value;
1669 }
1670 return value;
1671 }
1672
1673
1674
1675
1676 public static void writeStringFixedSize(final DataOutput out, String s,
1677 int size) throws IOException {
1678 byte[] b = toBytes(s);
1679 if (b.length > size) {
1680 throw new IOException("Trying to write " + b.length + " bytes (" +
1681 toStringBinary(b) + ") into a field of length " + size);
1682 }
1683
1684 out.writeBytes(s);
1685 for (int i = 0; i < size - s.length(); ++i)
1686 out.writeByte(0);
1687 }
1688
1689
1690
1691
1692 public static String readStringFixedSize(final DataInput in, int size)
1693 throws IOException {
1694 byte[] b = new byte[size];
1695 in.readFully(b);
1696 int n = b.length;
1697 while (n > 0 && b[n - 1] == 0)
1698 --n;
1699
1700 return toString(b, 0, n);
1701 }
1702
1703
1704
1705
1706
1707
1708
1709 public static byte [] copy(byte [] bytes) {
1710 if (bytes == null) return null;
1711 byte [] result = new byte[bytes.length];
1712 System.arraycopy(bytes, 0, result, 0, bytes.length);
1713 return result;
1714 }
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724 public static byte [] copy(byte [] bytes, final int offset, final int length) {
1725 if (bytes == null) return null;
1726 byte [] result = new byte[length];
1727 System.arraycopy(bytes, offset, result, 0, length);
1728 return result;
1729 }
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741 public static int unsignedBinarySearch(byte[] a, int fromIndex, int toIndex, byte key) {
1742 int unsignedKey = key & 0xff;
1743 int low = fromIndex;
1744 int high = toIndex - 1;
1745
1746 while (low <= high) {
1747 int mid = (low + high) >>> 1;
1748 int midVal = a[mid] & 0xff;
1749
1750 if (midVal < unsignedKey) {
1751 low = mid + 1;
1752 } else if (midVal > unsignedKey) {
1753 high = mid - 1;
1754 } else {
1755 return mid;
1756 }
1757 }
1758 return -(low + 1);
1759 }
1760
1761
1762
1763
1764
1765
1766
1767
1768 public static byte[] unsignedCopyAndIncrement(final byte[] input) {
1769 byte[] copy = copy(input);
1770 if (copy == null) {
1771 throw new IllegalArgumentException("cannot increment null array");
1772 }
1773 for (int i = copy.length - 1; i >= 0; --i) {
1774 if (copy[i] == -1) {
1775 copy[i] = 0;
1776 } else {
1777 ++copy[i];
1778 return copy;
1779 }
1780 }
1781
1782 byte[] out = new byte[copy.length + 1];
1783 out[0] = 1;
1784 System.arraycopy(copy, 0, out, 1, copy.length);
1785 return out;
1786 }
1787
1788 public static boolean equals(List<byte[]> a, List<byte[]> b) {
1789 if (a == null) {
1790 if (b == null) {
1791 return true;
1792 }
1793 return false;
1794 }
1795 if (b == null) {
1796 return false;
1797 }
1798 if (a.size() != b.size()) {
1799 return false;
1800 }
1801 for (int i = 0; i < a.size(); ++i) {
1802 if (!Bytes.equals(a.get(i), b.get(i))) {
1803 return false;
1804 }
1805 }
1806 return true;
1807 }
1808
1809 public static boolean isSorted(Collection<byte[]> arrays) {
1810 byte[] previous = new byte[0];
1811 for (byte[] array : IterableUtils.nullSafe(arrays)) {
1812 if (Bytes.compareTo(previous, array) > 0) {
1813 return false;
1814 }
1815 previous = array;
1816 }
1817 return true;
1818 }
1819
1820 public static List<byte[]> getUtf8ByteArrays(List<String> strings) {
1821 List<byte[]> byteArrays = Lists.newArrayListWithCapacity(CollectionUtils.nullSafeSize(strings));
1822 for (String s : IterableUtils.nullSafe(strings)) {
1823 byteArrays.add(Bytes.toBytes(s));
1824 }
1825 return byteArrays;
1826 }
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837 public static int indexOf(byte[] array, byte target) {
1838 for (int i = 0; i < array.length; i++) {
1839 if (array[i] == target) {
1840 return i;
1841 }
1842 }
1843 return -1;
1844 }
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857 public static int indexOf(byte[] array, byte[] target) {
1858 checkNotNull(array, "array");
1859 checkNotNull(target, "target");
1860 if (target.length == 0) {
1861 return 0;
1862 }
1863
1864 outer:
1865 for (int i = 0; i < array.length - target.length + 1; i++) {
1866 for (int j = 0; j < target.length; j++) {
1867 if (array[i + j] != target[j]) {
1868 continue outer;
1869 }
1870 }
1871 return i;
1872 }
1873 return -1;
1874 }
1875
1876
1877
1878
1879
1880
1881 public static boolean contains(byte[] array, byte target) {
1882 return indexOf(array, target) > -1;
1883 }
1884
1885
1886
1887
1888
1889
1890 public static boolean contains(byte[] array, byte[] target) {
1891 return indexOf(array, target) > -1;
1892 }
1893
1894
1895
1896
1897
1898 public static void zero(byte[] b) {
1899 zero(b, 0, b.length);
1900 }
1901
1902
1903
1904
1905
1906
1907
1908 public static void zero(byte[] b, int offset, int length) {
1909 checkPositionIndex(offset, b.length, "offset");
1910 checkArgument(length > 0, "length must be greater than 0");
1911 checkPositionIndex(offset + length, b.length, "offset + length");
1912 Arrays.fill(b, offset, offset + length, (byte) 0);
1913 }
1914
1915 private static final SecureRandom RNG = new SecureRandom();
1916
1917
1918
1919
1920
1921 public static void random(byte[] b) {
1922 RNG.nextBytes(b);
1923 }
1924
1925
1926
1927
1928
1929
1930
1931 public static void random(byte[] b, int offset, int length) {
1932 checkPositionIndex(offset, b.length, "offset");
1933 checkArgument(length > 0, "length must be greater than 0");
1934 checkPositionIndex(offset + length, b.length, "offset + length");
1935 byte[] buf = new byte[length];
1936 RNG.nextBytes(buf);
1937 System.arraycopy(buf, 0, b, offset, length);
1938 }
1939 }