1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.util;
19
20 import static com.google.common.base.Preconditions.checkArgument;
21 import static com.google.common.base.Preconditions.checkNotNull;
22 import static com.google.common.base.Preconditions.checkPositionIndex;
23
24 import java.io.DataInput;
25 import java.io.DataOutput;
26 import java.io.IOException;
27 import java.lang.reflect.Field;
28 import java.math.BigDecimal;
29 import java.math.BigInteger;
30 import java.nio.ByteBuffer;
31 import java.nio.ByteOrder;
32 import java.nio.charset.Charset;
33 import java.security.AccessController;
34 import java.security.PrivilegedAction;
35 import java.security.SecureRandom;
36 import java.util.Arrays;
37 import java.util.Collection;
38 import java.util.Comparator;
39 import java.util.Iterator;
40 import java.util.List;
41
42 import org.apache.commons.logging.Log;
43 import org.apache.commons.logging.LogFactory;
44 import org.apache.hadoop.classification.InterfaceAudience;
45 import org.apache.hadoop.classification.InterfaceStability;
46 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
47 import org.apache.hadoop.io.RawComparator;
48 import org.apache.hadoop.io.WritableComparator;
49 import org.apache.hadoop.io.WritableUtils;
50
51 import sun.misc.Unsafe;
52
53 import com.google.common.annotations.VisibleForTesting;
54 import com.google.common.collect.Lists;
55
56
57
58
59
60
61 @InterfaceAudience.Public
62 @InterfaceStability.Stable
63 public class Bytes {
64
65
66 private static final String UTF8_ENCODING = "UTF-8";
67
68
69
70 private static final Charset UTF8_CHARSET = Charset.forName(UTF8_ENCODING);
71
72
73 private static final byte [] EMPTY_BYTE_ARRAY = new byte [0];
74
75 private static final Log LOG = LogFactory.getLog(Bytes.class);
76
77
78
79
80 public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE;
81
82
83
84
85 public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN;
86
87
88
89
90 public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE;
91
92
93
94
95 public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE;
96
97
98
99
100 public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE;
101
102
103
104
105 public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE;
106
107
108
109
110 public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE;
111
112
113
114
115 public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE;
116
117
118
119
120
121
122
123
124 public static final int ESTIMATED_HEAP_TAX = 16;
125
126
127
128
129
130
131
132
133 final public static int len(byte[] b) {
134 return b == null ? 0 : b.length;
135 }
136
137
138
139
140 @InterfaceAudience.Public
141 @InterfaceStability.Stable
142 public static class ByteArrayComparator implements RawComparator<byte []> {
143
144
145
146 public ByteArrayComparator() {
147 super();
148 }
149 @Override
150 public int compare(byte [] left, byte [] right) {
151 return compareTo(left, right);
152 }
153 @Override
154 public int compare(byte [] b1, int s1, int l1, byte [] b2, int s2, int l2) {
155 return LexicographicalComparerHolder.BEST_COMPARER.
156 compareTo(b1, s1, l1, b2, s2, l2);
157 }
158 }
159
160
161
162
163
164
165
166
167
168 @InterfaceAudience.Public
169 @InterfaceStability.Stable
170 public static class RowEndKeyComparator extends ByteArrayComparator {
171 @Override
172 public int compare(byte[] left, byte[] right) {
173 return compare(left, 0, left.length, right, 0, right.length);
174 }
175 @Override
176 public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
177 if (b1 == b2 && s1 == s2 && l1 == l2) {
178 return 0;
179 }
180 if (l1 == 0) {
181 return l2;
182 }
183 if (l2 == 0) {
184 return -1;
185 }
186 return super.compare(b1, s1, l1, b2, s2, l2);
187 }
188 }
189
190
191
192
193 public static Comparator<byte []> BYTES_COMPARATOR =
194 new ByteArrayComparator();
195
196
197
198
199 public static RawComparator<byte []> BYTES_RAWCOMPARATOR =
200 new ByteArrayComparator();
201
202
203
204
205
206
207
208 public static byte [] readByteArray(final DataInput in)
209 throws IOException {
210 int len = WritableUtils.readVInt(in);
211 if (len < 0) {
212 throw new NegativeArraySizeException(Integer.toString(len));
213 }
214 byte [] result = new byte[len];
215 in.readFully(result, 0, len);
216 return result;
217 }
218
219
220
221
222
223
224
225 public static byte [] readByteArrayThrowsRuntime(final DataInput in) {
226 try {
227 return readByteArray(in);
228 } catch (Exception e) {
229 throw new RuntimeException(e);
230 }
231 }
232
233
234
235
236
237
238
239 public static void writeByteArray(final DataOutput out, final byte [] b)
240 throws IOException {
241 if(b == null) {
242 WritableUtils.writeVInt(out, 0);
243 } else {
244 writeByteArray(out, b, 0, b.length);
245 }
246 }
247
248
249
250
251
252
253
254
255
256 public static void writeByteArray(final DataOutput out, final byte [] b,
257 final int offset, final int length)
258 throws IOException {
259 WritableUtils.writeVInt(out, length);
260 out.write(b, offset, length);
261 }
262
263
264
265
266
267
268
269
270
271
272 public static int writeByteArray(final byte [] tgt, final int tgtOffset,
273 final byte [] src, final int srcOffset, final int srcLength) {
274 byte [] vint = vintToBytes(srcLength);
275 System.arraycopy(vint, 0, tgt, tgtOffset, vint.length);
276 int offset = tgtOffset + vint.length;
277 System.arraycopy(src, srcOffset, tgt, offset, srcLength);
278 return offset + srcLength;
279 }
280
281
282
283
284
285
286
287
288
289
290 public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes,
291 int srcOffset, int srcLength) {
292 System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength);
293 return tgtOffset + srcLength;
294 }
295
296
297
298
299
300
301
302
303 public static int putByte(byte[] bytes, int offset, byte b) {
304 bytes[offset] = b;
305 return offset + 1;
306 }
307
308
309
310
311
312
313
314
315 public static int putByteBuffer(byte[] bytes, int offset, ByteBuffer buf) {
316 int len = buf.remaining();
317 buf.get(bytes, offset, len);
318 return offset + len;
319 }
320
321
322
323
324
325
326
327
328
329
330
331 public static byte[] toBytes(ByteBuffer buf) {
332 ByteBuffer dup = buf.duplicate();
333 dup.position(0);
334 return readBytes(dup);
335 }
336
337 private static byte[] readBytes(ByteBuffer buf) {
338 byte [] result = new byte[buf.remaining()];
339 buf.get(result);
340 return result;
341 }
342
343
344
345
346
347 public static String toString(final byte [] b) {
348 if (b == null) {
349 return null;
350 }
351 return toString(b, 0, b.length);
352 }
353
354
355
356
357
358
359
360 public static String toString(final byte [] b1,
361 String sep,
362 final byte [] b2) {
363 return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length);
364 }
365
366
367
368
369
370
371
372
373
374
375 public static String toString(final byte [] b, int off, int len) {
376 if (b == null) {
377 return null;
378 }
379 if (len == 0) {
380 return "";
381 }
382 return new String(b, off, len, UTF8_CHARSET);
383 }
384
385
386
387
388
389
390
391
392 public static String toStringBinary(final byte [] b) {
393 if (b == null)
394 return "null";
395 return toStringBinary(b, 0, b.length);
396 }
397
398
399
400
401
402
403
404
405
406
407
408
409 public static String toStringBinary(ByteBuffer buf) {
410 if (buf == null)
411 return "null";
412 if (buf.hasArray()) {
413 return toStringBinary(buf.array(), buf.arrayOffset(), buf.limit());
414 }
415 return toStringBinary(toBytes(buf));
416 }
417
418
419
420
421
422
423
424
425
426
427
428 public static String toStringBinary(final byte [] b, int off, int len) {
429 StringBuilder result = new StringBuilder();
430
431 if (off >= b.length) return result.toString();
432 if (off + len > b.length) len = b.length - off;
433 for (int i = off; i < off + len ; ++i ) {
434 int ch = b[i] & 0xFF;
435 if ( (ch >= '0' && ch <= '9')
436 || (ch >= 'A' && ch <= 'Z')
437 || (ch >= 'a' && ch <= 'z')
438 || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0 ) {
439 result.append((char)ch);
440 } else {
441 result.append(String.format("\\x%02X", ch));
442 }
443 }
444 return result.toString();
445 }
446
447 private static boolean isHexDigit(char c) {
448 return
449 (c >= 'A' && c <= 'F') ||
450 (c >= '0' && c <= '9');
451 }
452
453
454
455
456
457
458
459 public static byte toBinaryFromHex(byte ch) {
460 if ( ch >= 'A' && ch <= 'F' )
461 return (byte) ((byte)10 + (byte) (ch - 'A'));
462
463 return (byte) (ch - '0');
464 }
465
466 public static byte [] toBytesBinary(String in) {
467
468 byte [] b = new byte[in.length()];
469 int size = 0;
470 for (int i = 0; i < in.length(); ++i) {
471 char ch = in.charAt(i);
472 if (ch == '\\' && in.length() > i+1 && in.charAt(i+1) == 'x') {
473
474 char hd1 = in.charAt(i+2);
475 char hd2 = in.charAt(i+3);
476
477
478 if (!isHexDigit(hd1) ||
479 !isHexDigit(hd2)) {
480
481 continue;
482 }
483
484 byte d = (byte) ((toBinaryFromHex((byte)hd1) << 4) + toBinaryFromHex((byte)hd2));
485
486 b[size++] = d;
487 i += 3;
488 } else {
489 b[size++] = (byte) ch;
490 }
491 }
492
493 byte [] b2 = new byte[size];
494 System.arraycopy(b, 0, b2, 0, size);
495 return b2;
496 }
497
498
499
500
501
502
503 public static byte[] toBytes(String s) {
504 return s.getBytes(UTF8_CHARSET);
505 }
506
507
508
509
510
511
512
513
514 public static byte [] toBytes(final boolean b) {
515 return new byte[] { b ? (byte) -1 : (byte) 0 };
516 }
517
518
519
520
521
522
523 public static boolean toBoolean(final byte [] b) {
524 if (b.length != 1) {
525 throw new IllegalArgumentException("Array has wrong size: " + b.length);
526 }
527 return b[0] != (byte) 0;
528 }
529
530
531
532
533
534
535
536 public static byte[] toBytes(long val) {
537 byte [] b = new byte[8];
538 for (int i = 7; i > 0; i--) {
539 b[i] = (byte) val;
540 val >>>= 8;
541 }
542 b[0] = (byte) val;
543 return b;
544 }
545
546
547
548
549
550
551
552 public static long toLong(byte[] bytes) {
553 return toLong(bytes, 0, SIZEOF_LONG);
554 }
555
556
557
558
559
560
561
562
563
564 public static long toLong(byte[] bytes, int offset) {
565 return toLong(bytes, offset, SIZEOF_LONG);
566 }
567
568
569
570
571
572
573
574
575
576
577
578 public static long toLong(byte[] bytes, int offset, final int length) {
579 if (length != SIZEOF_LONG || offset + length > bytes.length) {
580 throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
581 }
582 long l = 0;
583 for(int i = offset; i < offset + length; i++) {
584 l <<= 8;
585 l ^= bytes[i] & 0xFF;
586 }
587 return l;
588 }
589
590 private static IllegalArgumentException
591 explainWrongLengthOrOffset(final byte[] bytes,
592 final int offset,
593 final int length,
594 final int expectedLength) {
595 String reason;
596 if (length != expectedLength) {
597 reason = "Wrong length: " + length + ", expected " + expectedLength;
598 } else {
599 reason = "offset (" + offset + ") + length (" + length + ") exceed the"
600 + " capacity of the array: " + bytes.length;
601 }
602 return new IllegalArgumentException(reason);
603 }
604
605
606
607
608
609
610
611
612
613
614 public static int putLong(byte[] bytes, int offset, long val) {
615 if (bytes.length - offset < SIZEOF_LONG) {
616 throw new IllegalArgumentException("Not enough room to put a long at"
617 + " offset " + offset + " in a " + bytes.length + " byte array");
618 }
619 for(int i = offset + 7; i > offset; i--) {
620 bytes[i] = (byte) val;
621 val >>>= 8;
622 }
623 bytes[offset] = (byte) val;
624 return offset + SIZEOF_LONG;
625 }
626
627
628
629
630
631
632 public static float toFloat(byte [] bytes) {
633 return toFloat(bytes, 0);
634 }
635
636
637
638
639
640
641
642 public static float toFloat(byte [] bytes, int offset) {
643 return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
644 }
645
646
647
648
649
650
651
652 public static int putFloat(byte [] bytes, int offset, float f) {
653 return putInt(bytes, offset, Float.floatToRawIntBits(f));
654 }
655
656
657
658
659
660 public static byte [] toBytes(final float f) {
661
662 return Bytes.toBytes(Float.floatToRawIntBits(f));
663 }
664
665
666
667
668
669 public static double toDouble(final byte [] bytes) {
670 return toDouble(bytes, 0);
671 }
672
673
674
675
676
677
678 public static double toDouble(final byte [] bytes, final int offset) {
679 return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
680 }
681
682
683
684
685
686
687
688 public static int putDouble(byte [] bytes, int offset, double d) {
689 return putLong(bytes, offset, Double.doubleToLongBits(d));
690 }
691
692
693
694
695
696
697
698
699 public static byte [] toBytes(final double d) {
700
701 return Bytes.toBytes(Double.doubleToRawLongBits(d));
702 }
703
704
705
706
707
708
709
710
711 public static byte[] toBytes(int val) {
712 byte [] b = new byte[4];
713 for(int i = 3; i > 0; i--) {
714 b[i] = (byte) val;
715 val >>>= 8;
716 }
717 b[0] = (byte) val;
718 return b;
719 }
720
721
722
723
724
725
726 public static int toInt(byte[] bytes) {
727 return toInt(bytes, 0, SIZEOF_INT);
728 }
729
730
731
732
733
734
735
736 public static int toInt(byte[] bytes, int offset) {
737 return toInt(bytes, offset, SIZEOF_INT);
738 }
739
740
741
742
743
744
745
746
747
748
749 public static int toInt(byte[] bytes, int offset, final int length) {
750 if (length != SIZEOF_INT || offset + length > bytes.length) {
751 throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
752 }
753 int n = 0;
754 for(int i = offset; i < (offset + length); i++) {
755 n <<= 8;
756 n ^= bytes[i] & 0xFF;
757 }
758 return n;
759 }
760
761
762
763
764
765
766
767
768
769
770 public static int putInt(byte[] bytes, int offset, int val) {
771 if (bytes.length - offset < SIZEOF_INT) {
772 throw new IllegalArgumentException("Not enough room to put an int at"
773 + " offset " + offset + " in a " + bytes.length + " byte array");
774 }
775 for(int i= offset + 3; i > offset; i--) {
776 bytes[i] = (byte) val;
777 val >>>= 8;
778 }
779 bytes[offset] = (byte) val;
780 return offset + SIZEOF_INT;
781 }
782
783
784
785
786
787
788 public static byte[] toBytes(short val) {
789 byte[] b = new byte[SIZEOF_SHORT];
790 b[1] = (byte) val;
791 val >>= 8;
792 b[0] = (byte) val;
793 return b;
794 }
795
796
797
798
799
800
801 public static short toShort(byte[] bytes) {
802 return toShort(bytes, 0, SIZEOF_SHORT);
803 }
804
805
806
807
808
809
810
811 public static short toShort(byte[] bytes, int offset) {
812 return toShort(bytes, offset, SIZEOF_SHORT);
813 }
814
815
816
817
818
819
820
821
822
823
824 public static short toShort(byte[] bytes, int offset, final int length) {
825 if (length != SIZEOF_SHORT || offset + length > bytes.length) {
826 throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
827 }
828 short n = 0;
829 n ^= bytes[offset] & 0xFF;
830 n <<= 8;
831 n ^= bytes[offset+1] & 0xFF;
832 return n;
833 }
834
835
836
837
838
839
840
841
842
843
844 public static byte[] getBytes(ByteBuffer buf) {
845 return readBytes(buf.duplicate());
846 }
847
848
849
850
851
852
853
854
855
856
857 public static int putShort(byte[] bytes, int offset, short val) {
858 if (bytes.length - offset < SIZEOF_SHORT) {
859 throw new IllegalArgumentException("Not enough room to put a short at"
860 + " offset " + offset + " in a " + bytes.length + " byte array");
861 }
862 bytes[offset+1] = (byte) val;
863 val >>= 8;
864 bytes[offset] = (byte) val;
865 return offset + SIZEOF_SHORT;
866 }
867
868
869
870
871
872
873
874 public static byte[] toBytes(BigDecimal val) {
875 byte[] valueBytes = val.unscaledValue().toByteArray();
876 byte[] result = new byte[valueBytes.length + SIZEOF_INT];
877 int offset = putInt(result, 0, val.scale());
878 putBytes(result, offset, valueBytes, 0, valueBytes.length);
879 return result;
880 }
881
882
883
884
885
886
887
888
889 public static BigDecimal toBigDecimal(byte[] bytes) {
890 return toBigDecimal(bytes, 0, bytes.length);
891 }
892
893
894
895
896
897
898
899
900
901 public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) {
902 if (bytes == null || length < SIZEOF_INT + 1 ||
903 (offset + length > bytes.length)) {
904 return null;
905 }
906
907 int scale = toInt(bytes, offset);
908 byte[] tcBytes = new byte[length - SIZEOF_INT];
909 System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT);
910 return new BigDecimal(new BigInteger(tcBytes), scale);
911 }
912
913
914
915
916
917
918
919
920
921 public static int putBigDecimal(byte[] bytes, int offset, BigDecimal val) {
922 if (bytes == null) {
923 return offset;
924 }
925
926 byte[] valueBytes = val.unscaledValue().toByteArray();
927 byte[] result = new byte[valueBytes.length + SIZEOF_INT];
928 offset = putInt(result, offset, val.scale());
929 return putBytes(result, offset, valueBytes, 0, valueBytes.length);
930 }
931
932
933
934
935
936 public static byte [] vintToBytes(final long vint) {
937 long i = vint;
938 int size = WritableUtils.getVIntSize(i);
939 byte [] result = new byte[size];
940 int offset = 0;
941 if (i >= -112 && i <= 127) {
942 result[offset] = (byte) i;
943 return result;
944 }
945
946 int len = -112;
947 if (i < 0) {
948 i ^= -1L;
949 len = -120;
950 }
951
952 long tmp = i;
953 while (tmp != 0) {
954 tmp = tmp >> 8;
955 len--;
956 }
957
958 result[offset++] = (byte) len;
959
960 len = (len < -120) ? -(len + 120) : -(len + 112);
961
962 for (int idx = len; idx != 0; idx--) {
963 int shiftbits = (idx - 1) * 8;
964 long mask = 0xFFL << shiftbits;
965 result[offset++] = (byte)((i & mask) >> shiftbits);
966 }
967 return result;
968 }
969
970
971
972
973
974 public static long bytesToVint(final byte [] buffer) {
975 int offset = 0;
976 byte firstByte = buffer[offset++];
977 int len = WritableUtils.decodeVIntSize(firstByte);
978 if (len == 1) {
979 return firstByte;
980 }
981 long i = 0;
982 for (int idx = 0; idx < len-1; idx++) {
983 byte b = buffer[offset++];
984 i = i << 8;
985 i = i | (b & 0xFF);
986 }
987 return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
988 }
989
990
991
992
993
994
995
996
997 public static long readVLong(final byte [] buffer, final int offset)
998 throws IOException {
999 byte firstByte = buffer[offset];
1000 int len = WritableUtils.decodeVIntSize(firstByte);
1001 if (len == 1) {
1002 return firstByte;
1003 }
1004 long i = 0;
1005 for (int idx = 0; idx < len-1; idx++) {
1006 byte b = buffer[offset + 1 + idx];
1007 i = i << 8;
1008 i = i | (b & 0xFF);
1009 }
1010 return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
1011 }
1012
1013
1014
1015
1016
1017
1018 public static int compareTo(final byte [] left, final byte [] right) {
1019 return LexicographicalComparerHolder.BEST_COMPARER.
1020 compareTo(left, 0, left.length, right, 0, right.length);
1021 }
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034 public static int compareTo(byte[] buffer1, int offset1, int length1,
1035 byte[] buffer2, int offset2, int length2) {
1036 return LexicographicalComparerHolder.BEST_COMPARER.
1037 compareTo(buffer1, offset1, length1, buffer2, offset2, length2);
1038 }
1039
1040 interface Comparer<T> {
1041 int compareTo(
1042 T buffer1, int offset1, int length1, T buffer2, int offset2, int length2
1043 );
1044 }
1045
1046 @VisibleForTesting
1047 static Comparer<byte[]> lexicographicalComparerJavaImpl() {
1048 return LexicographicalComparerHolder.PureJavaComparer.INSTANCE;
1049 }
1050
1051
1052
1053
1054
1055
1056
1057
1058 @VisibleForTesting
1059 static class LexicographicalComparerHolder {
1060 static final String UNSAFE_COMPARER_NAME =
1061 LexicographicalComparerHolder.class.getName() + "$UnsafeComparer";
1062
1063 static final Comparer<byte[]> BEST_COMPARER = getBestComparer();
1064
1065
1066
1067
1068 static Comparer<byte[]> getBestComparer() {
1069 try {
1070 Class<?> theClass = Class.forName(UNSAFE_COMPARER_NAME);
1071
1072
1073 @SuppressWarnings("unchecked")
1074 Comparer<byte[]> comparer =
1075 (Comparer<byte[]>) theClass.getEnumConstants()[0];
1076 return comparer;
1077 } catch (Throwable t) {
1078 return lexicographicalComparerJavaImpl();
1079 }
1080 }
1081
1082 enum PureJavaComparer implements Comparer<byte[]> {
1083 INSTANCE;
1084
1085 @Override
1086 public int compareTo(byte[] buffer1, int offset1, int length1,
1087 byte[] buffer2, int offset2, int length2) {
1088
1089 if (buffer1 == buffer2 &&
1090 offset1 == offset2 &&
1091 length1 == length2) {
1092 return 0;
1093 }
1094
1095 int end1 = offset1 + length1;
1096 int end2 = offset2 + length2;
1097 for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) {
1098 int a = (buffer1[i] & 0xff);
1099 int b = (buffer2[j] & 0xff);
1100 if (a != b) {
1101 return a - b;
1102 }
1103 }
1104 return length1 - length2;
1105 }
1106 }
1107
1108 @VisibleForTesting
1109 enum UnsafeComparer implements Comparer<byte[]> {
1110 INSTANCE;
1111
1112 static final Unsafe theUnsafe;
1113
1114
1115 static final int BYTE_ARRAY_BASE_OFFSET;
1116
1117 static {
1118 theUnsafe = (Unsafe) AccessController.doPrivileged(
1119 new PrivilegedAction<Object>() {
1120 @Override
1121 public Object run() {
1122 try {
1123 Field f = Unsafe.class.getDeclaredField("theUnsafe");
1124 f.setAccessible(true);
1125 return f.get(null);
1126 } catch (NoSuchFieldException e) {
1127
1128
1129 throw new Error();
1130 } catch (IllegalAccessException e) {
1131 throw new Error();
1132 }
1133 }
1134 });
1135
1136 BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class);
1137
1138
1139 if (theUnsafe.arrayIndexScale(byte[].class) != 1) {
1140 throw new AssertionError();
1141 }
1142 }
1143
1144 static final boolean littleEndian =
1145 ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN);
1146
1147
1148
1149
1150
1151 static boolean lessThanUnsigned(long x1, long x2) {
1152 return (x1 + Long.MIN_VALUE) < (x2 + Long.MIN_VALUE);
1153 }
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166 @Override
1167 public int compareTo(byte[] buffer1, int offset1, int length1,
1168 byte[] buffer2, int offset2, int length2) {
1169
1170 if (buffer1 == buffer2 &&
1171 offset1 == offset2 &&
1172 length1 == length2) {
1173 return 0;
1174 }
1175 int minLength = Math.min(length1, length2);
1176 int minWords = minLength / SIZEOF_LONG;
1177 int offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET;
1178 int offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET;
1179
1180
1181
1182
1183
1184
1185 for (int i = 0; i < minWords * SIZEOF_LONG; i += SIZEOF_LONG) {
1186 long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i);
1187 long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i);
1188 long diff = lw ^ rw;
1189
1190 if (diff != 0) {
1191 if (!littleEndian) {
1192 return lessThanUnsigned(lw, rw) ? -1 : 1;
1193 }
1194
1195
1196 int n = 0;
1197 int y;
1198 int x = (int) diff;
1199 if (x == 0) {
1200 x = (int) (diff >>> 32);
1201 n = 32;
1202 }
1203
1204 y = x << 16;
1205 if (y == 0) {
1206 n += 16;
1207 } else {
1208 x = y;
1209 }
1210
1211 y = x << 8;
1212 if (y == 0) {
1213 n += 8;
1214 }
1215 return (int) (((lw >>> n) & 0xFFL) - ((rw >>> n) & 0xFFL));
1216 }
1217 }
1218
1219
1220 for (int i = minWords * SIZEOF_LONG; i < minLength; i++) {
1221 int a = (buffer1[offset1 + i] & 0xff);
1222 int b = (buffer2[offset2 + i] & 0xff);
1223 if (a != b) {
1224 return a - b;
1225 }
1226 }
1227 return length1 - length2;
1228 }
1229 }
1230 }
1231
1232
1233
1234
1235
1236
1237 public static boolean equals(final byte [] left, final byte [] right) {
1238
1239
1240 if (left == right) return true;
1241 if (left == null || right == null) return false;
1242 if (left.length != right.length) return false;
1243 if (left.length == 0) return true;
1244
1245
1246
1247
1248 if (left[left.length - 1] != right[right.length - 1]) return false;
1249
1250 return compareTo(left, right) == 0;
1251 }
1252
1253 public static boolean equals(final byte[] left, int leftOffset, int leftLen,
1254 final byte[] right, int rightOffset, int rightLen) {
1255
1256 if (left == right &&
1257 leftOffset == rightOffset &&
1258 leftLen == rightLen) {
1259 return true;
1260 }
1261
1262 if (leftLen != rightLen) {
1263 return false;
1264 }
1265 if (leftLen == 0) {
1266 return true;
1267 }
1268
1269
1270
1271
1272 if (left[leftOffset + leftLen - 1] != right[rightOffset + rightLen - 1]) return false;
1273
1274 return LexicographicalComparerHolder.BEST_COMPARER.
1275 compareTo(left, leftOffset, leftLen, right, rightOffset, rightLen) == 0;
1276 }
1277
1278
1279
1280
1281
1282
1283
1284 public static boolean equals(byte[] a, ByteBuffer buf) {
1285 if (a == null) return buf == null;
1286 if (buf == null) return false;
1287 if (a.length != buf.remaining()) return false;
1288
1289
1290 ByteBuffer b = buf.duplicate();
1291 for (byte anA : a) {
1292 if (anA != b.get()) {
1293 return false;
1294 }
1295 }
1296 return true;
1297 }
1298
1299
1300
1301
1302
1303
1304 public static boolean startsWith(byte[] bytes, byte[] prefix) {
1305 return bytes != null && prefix != null &&
1306 bytes.length >= prefix.length &&
1307 LexicographicalComparerHolder.BEST_COMPARER.
1308 compareTo(bytes, 0, prefix.length, prefix, 0, prefix.length) == 0;
1309 }
1310
1311
1312
1313
1314
1315
1316
1317 public static int hashCode(final byte [] b) {
1318 return hashCode(b, b.length);
1319 }
1320
1321
1322
1323
1324
1325
1326
1327
1328 public static int hashCode(final byte [] b, final int length) {
1329 return WritableComparator.hashBytes(b, length);
1330 }
1331
1332
1333
1334
1335
1336
1337 public static Integer mapKey(final byte [] b) {
1338 return hashCode(b);
1339 }
1340
1341
1342
1343
1344
1345
1346
1347 public static Integer mapKey(final byte [] b, final int length) {
1348 return hashCode(b, length);
1349 }
1350
1351
1352
1353
1354
1355
1356 public static byte [] add(final byte [] a, final byte [] b) {
1357 return add(a, b, EMPTY_BYTE_ARRAY);
1358 }
1359
1360
1361
1362
1363
1364
1365
1366 public static byte [] add(final byte [] a, final byte [] b, final byte [] c) {
1367 byte [] result = new byte[a.length + b.length + c.length];
1368 System.arraycopy(a, 0, result, 0, a.length);
1369 System.arraycopy(b, 0, result, a.length, b.length);
1370 System.arraycopy(c, 0, result, a.length + b.length, c.length);
1371 return result;
1372 }
1373
1374
1375
1376
1377
1378
1379 public static byte [] head(final byte [] a, final int length) {
1380 if (a.length < length) {
1381 return null;
1382 }
1383 byte [] result = new byte[length];
1384 System.arraycopy(a, 0, result, 0, length);
1385 return result;
1386 }
1387
1388
1389
1390
1391
1392
1393 public static byte [] tail(final byte [] a, final int length) {
1394 if (a.length < length) {
1395 return null;
1396 }
1397 byte [] result = new byte[length];
1398 System.arraycopy(a, a.length - length, result, 0, length);
1399 return result;
1400 }
1401
1402
1403
1404
1405
1406
1407 public static byte [] padHead(final byte [] a, final int length) {
1408 byte [] padding = new byte[length];
1409 for (int i = 0; i < length; i++) {
1410 padding[i] = 0;
1411 }
1412 return add(padding,a);
1413 }
1414
1415
1416
1417
1418
1419
1420 public static byte [] padTail(final byte [] a, final int length) {
1421 byte [] padding = new byte[length];
1422 for (int i = 0; i < length; i++) {
1423 padding[i] = 0;
1424 }
1425 return add(a,padding);
1426 }
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437 public static byte [][] split(final byte [] a, final byte [] b, final int num) {
1438 return split(a, b, false, num);
1439 }
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453 public static byte[][] split(final byte[] a, final byte[] b,
1454 boolean inclusive, final int num) {
1455 byte[][] ret = new byte[num + 2][];
1456 int i = 0;
1457 Iterable<byte[]> iter = iterateOnSplits(a, b, inclusive, num);
1458 if (iter == null)
1459 return null;
1460 for (byte[] elem : iter) {
1461 ret[i++] = elem;
1462 }
1463 return ret;
1464 }
1465
1466
1467
1468
1469 public static Iterable<byte[]> iterateOnSplits(final byte[] a,
1470 final byte[] b, final int num)
1471 {
1472 return iterateOnSplits(a, b, false, num);
1473 }
1474
1475
1476
1477
1478 public static Iterable<byte[]> iterateOnSplits(
1479 final byte[] a, final byte[]b, boolean inclusive, final int num)
1480 {
1481 byte [] aPadded;
1482 byte [] bPadded;
1483 if (a.length < b.length) {
1484 aPadded = padTail(a, b.length - a.length);
1485 bPadded = b;
1486 } else if (b.length < a.length) {
1487 aPadded = a;
1488 bPadded = padTail(b, a.length - b.length);
1489 } else {
1490 aPadded = a;
1491 bPadded = b;
1492 }
1493 if (compareTo(aPadded,bPadded) >= 0) {
1494 throw new IllegalArgumentException("b <= a");
1495 }
1496 if (num <= 0) {
1497 throw new IllegalArgumentException("num cannot be < 0");
1498 }
1499 byte [] prependHeader = {1, 0};
1500 final BigInteger startBI = new BigInteger(add(prependHeader, aPadded));
1501 final BigInteger stopBI = new BigInteger(add(prependHeader, bPadded));
1502 BigInteger diffBI = stopBI.subtract(startBI);
1503 if (inclusive) {
1504 diffBI = diffBI.add(BigInteger.ONE);
1505 }
1506 final BigInteger splitsBI = BigInteger.valueOf(num + 1);
1507 if(diffBI.compareTo(splitsBI) < 0) {
1508 return null;
1509 }
1510 final BigInteger intervalBI;
1511 try {
1512 intervalBI = diffBI.divide(splitsBI);
1513 } catch(Exception e) {
1514 LOG.error("Exception caught during division", e);
1515 return null;
1516 }
1517
1518 final Iterator<byte[]> iterator = new Iterator<byte[]>() {
1519 private int i = -1;
1520
1521 @Override
1522 public boolean hasNext() {
1523 return i < num+1;
1524 }
1525
1526 @Override
1527 public byte[] next() {
1528 i++;
1529 if (i == 0) return a;
1530 if (i == num + 1) return b;
1531
1532 BigInteger curBI = startBI.add(intervalBI.multiply(BigInteger.valueOf(i)));
1533 byte [] padded = curBI.toByteArray();
1534 if (padded[1] == 0)
1535 padded = tail(padded, padded.length - 2);
1536 else
1537 padded = tail(padded, padded.length - 1);
1538 return padded;
1539 }
1540
1541 @Override
1542 public void remove() {
1543 throw new UnsupportedOperationException();
1544 }
1545
1546 };
1547
1548 return new Iterable<byte[]>() {
1549 @Override
1550 public Iterator<byte[]> iterator() {
1551 return iterator;
1552 }
1553 };
1554 }
1555
1556
1557
1558
1559
1560
1561 public static int hashCode(byte[] bytes, int offset, int length) {
1562 int hash = 1;
1563 for (int i = offset; i < offset + length; i++)
1564 hash = (31 * hash) + (int) bytes[i];
1565 return hash;
1566 }
1567
1568
1569
1570
1571
1572 public static byte [][] toByteArrays(final String [] t) {
1573 byte [][] result = new byte[t.length][];
1574 for (int i = 0; i < t.length; i++) {
1575 result[i] = Bytes.toBytes(t[i]);
1576 }
1577 return result;
1578 }
1579
1580
1581
1582
1583
1584
1585 public static byte [][] toByteArrays(final String column) {
1586 return toByteArrays(toBytes(column));
1587 }
1588
1589
1590
1591
1592
1593
1594 public static byte [][] toByteArrays(final byte [] column) {
1595 byte [][] result = new byte[1][];
1596 result[0] = column;
1597 return result;
1598 }
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615 public static int binarySearch(byte [][]arr, byte []key, int offset,
1616 int length, RawComparator<?> comparator) {
1617 int low = 0;
1618 int high = arr.length - 1;
1619
1620 while (low <= high) {
1621 int mid = (low+high) >>> 1;
1622
1623
1624 int cmp = comparator.compare(key, offset, length,
1625 arr[mid], 0, arr[mid].length);
1626
1627 if (cmp > 0)
1628 low = mid + 1;
1629
1630 else if (cmp < 0)
1631 high = mid - 1;
1632
1633 else
1634 return mid;
1635 }
1636 return - (low+1);
1637 }
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647 public static byte [] incrementBytes(byte[] value, long amount)
1648 {
1649 byte[] val = value;
1650 if (val.length < SIZEOF_LONG) {
1651
1652 byte [] newvalue;
1653 if (val[0] < 0) {
1654 newvalue = new byte[]{-1, -1, -1, -1, -1, -1, -1, -1};
1655 } else {
1656 newvalue = new byte[SIZEOF_LONG];
1657 }
1658 System.arraycopy(val, 0, newvalue, newvalue.length - val.length,
1659 val.length);
1660 val = newvalue;
1661 } else if (val.length > SIZEOF_LONG) {
1662 throw new IllegalArgumentException("Increment Bytes - value too big: " +
1663 val.length);
1664 }
1665 if(amount == 0) return val;
1666 if(val[0] < 0){
1667 return binaryIncrementNeg(val, amount);
1668 }
1669 return binaryIncrementPos(val, amount);
1670 }
1671
1672
1673 private static byte [] binaryIncrementPos(byte [] value, long amount) {
1674 long amo = amount;
1675 int sign = 1;
1676 if (amount < 0) {
1677 amo = -amount;
1678 sign = -1;
1679 }
1680 for(int i=0;i<value.length;i++) {
1681 int cur = ((int)amo % 256) * sign;
1682 amo = (amo >> 8);
1683 int val = value[value.length-i-1] & 0x0ff;
1684 int total = val + cur;
1685 if(total > 255) {
1686 amo += sign;
1687 total %= 256;
1688 } else if (total < 0) {
1689 amo -= sign;
1690 }
1691 value[value.length-i-1] = (byte)total;
1692 if (amo == 0) return value;
1693 }
1694 return value;
1695 }
1696
1697
1698 private static byte [] binaryIncrementNeg(byte [] value, long amount) {
1699 long amo = amount;
1700 int sign = 1;
1701 if (amount < 0) {
1702 amo = -amount;
1703 sign = -1;
1704 }
1705 for(int i=0;i<value.length;i++) {
1706 int cur = ((int)amo % 256) * sign;
1707 amo = (amo >> 8);
1708 int val = ((~value[value.length-i-1]) & 0x0ff) + 1;
1709 int total = cur - val;
1710 if(total >= 0) {
1711 amo += sign;
1712 } else if (total < -256) {
1713 amo -= sign;
1714 total %= 256;
1715 }
1716 value[value.length-i-1] = (byte)total;
1717 if (amo == 0) return value;
1718 }
1719 return value;
1720 }
1721
1722
1723
1724
1725 public static void writeStringFixedSize(final DataOutput out, String s,
1726 int size) throws IOException {
1727 byte[] b = toBytes(s);
1728 if (b.length > size) {
1729 throw new IOException("Trying to write " + b.length + " bytes (" +
1730 toStringBinary(b) + ") into a field of length " + size);
1731 }
1732
1733 out.writeBytes(s);
1734 for (int i = 0; i < size - s.length(); ++i)
1735 out.writeByte(0);
1736 }
1737
1738
1739
1740
1741 public static String readStringFixedSize(final DataInput in, int size)
1742 throws IOException {
1743 byte[] b = new byte[size];
1744 in.readFully(b);
1745 int n = b.length;
1746 while (n > 0 && b[n - 1] == 0)
1747 --n;
1748
1749 return toString(b, 0, n);
1750 }
1751
1752
1753
1754
1755
1756
1757
1758 public static byte [] copy(byte [] bytes) {
1759 if (bytes == null) return null;
1760 byte [] result = new byte[bytes.length];
1761 System.arraycopy(bytes, 0, result, 0, bytes.length);
1762 return result;
1763 }
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773 public static byte [] copy(byte [] bytes, final int offset, final int length) {
1774 if (bytes == null) return null;
1775 byte [] result = new byte[length];
1776 System.arraycopy(bytes, offset, result, 0, length);
1777 return result;
1778 }
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790 public static int unsignedBinarySearch(byte[] a, int fromIndex, int toIndex, byte key) {
1791 int unsignedKey = key & 0xff;
1792 int low = fromIndex;
1793 int high = toIndex - 1;
1794
1795 while (low <= high) {
1796 int mid = (low + high) >>> 1;
1797 int midVal = a[mid] & 0xff;
1798
1799 if (midVal < unsignedKey) {
1800 low = mid + 1;
1801 } else if (midVal > unsignedKey) {
1802 high = mid - 1;
1803 } else {
1804 return mid;
1805 }
1806 }
1807 return -(low + 1);
1808 }
1809
1810
1811
1812
1813
1814
1815
1816
1817 public static byte[] unsignedCopyAndIncrement(final byte[] input) {
1818 byte[] copy = copy(input);
1819 if (copy == null) {
1820 throw new IllegalArgumentException("cannot increment null array");
1821 }
1822 for (int i = copy.length - 1; i >= 0; --i) {
1823 if (copy[i] == -1) {
1824 copy[i] = 0;
1825 } else {
1826 ++copy[i];
1827 return copy;
1828 }
1829 }
1830
1831 byte[] out = new byte[copy.length + 1];
1832 out[0] = 1;
1833 System.arraycopy(copy, 0, out, 1, copy.length);
1834 return out;
1835 }
1836
1837 public static boolean equals(List<byte[]> a, List<byte[]> b) {
1838 if (a == null) {
1839 if (b == null) {
1840 return true;
1841 }
1842 return false;
1843 }
1844 if (b == null) {
1845 return false;
1846 }
1847 if (a.size() != b.size()) {
1848 return false;
1849 }
1850 for (int i = 0; i < a.size(); ++i) {
1851 if (!Bytes.equals(a.get(i), b.get(i))) {
1852 return false;
1853 }
1854 }
1855 return true;
1856 }
1857
1858 public static boolean isSorted(Collection<byte[]> arrays) {
1859 byte[] previous = new byte[0];
1860 for (byte[] array : IterableUtils.nullSafe(arrays)) {
1861 if (Bytes.compareTo(previous, array) > 0) {
1862 return false;
1863 }
1864 previous = array;
1865 }
1866 return true;
1867 }
1868
1869 public static List<byte[]> getUtf8ByteArrays(List<String> strings) {
1870 List<byte[]> byteArrays = Lists.newArrayListWithCapacity(CollectionUtils.nullSafeSize(strings));
1871 for (String s : IterableUtils.nullSafe(strings)) {
1872 byteArrays.add(Bytes.toBytes(s));
1873 }
1874 return byteArrays;
1875 }
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886 public static int indexOf(byte[] array, byte target) {
1887 for (int i = 0; i < array.length; i++) {
1888 if (array[i] == target) {
1889 return i;
1890 }
1891 }
1892 return -1;
1893 }
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906 public static int indexOf(byte[] array, byte[] target) {
1907 checkNotNull(array, "array");
1908 checkNotNull(target, "target");
1909 if (target.length == 0) {
1910 return 0;
1911 }
1912
1913 outer:
1914 for (int i = 0; i < array.length - target.length + 1; i++) {
1915 for (int j = 0; j < target.length; j++) {
1916 if (array[i + j] != target[j]) {
1917 continue outer;
1918 }
1919 }
1920 return i;
1921 }
1922 return -1;
1923 }
1924
1925
1926
1927
1928
1929
1930 public static boolean contains(byte[] array, byte target) {
1931 return indexOf(array, target) > -1;
1932 }
1933
1934
1935
1936
1937
1938
1939 public static boolean contains(byte[] array, byte[] target) {
1940 return indexOf(array, target) > -1;
1941 }
1942
1943
1944
1945
1946
1947 public static void zero(byte[] b) {
1948 zero(b, 0, b.length);
1949 }
1950
1951
1952
1953
1954
1955
1956
1957 public static void zero(byte[] b, int offset, int length) {
1958 checkPositionIndex(offset, b.length, "offset");
1959 checkArgument(length > 0, "length must be greater than 0");
1960 checkPositionIndex(offset + length, b.length, "offset + length");
1961 Arrays.fill(b, offset, offset + length, (byte) 0);
1962 }
1963
1964 private static final SecureRandom RNG = new SecureRandom();
1965
1966
1967
1968
1969
1970 public static void random(byte[] b) {
1971 RNG.nextBytes(b);
1972 }
1973
1974
1975
1976
1977
1978
1979
1980 public static void random(byte[] b, int offset, int length) {
1981 checkPositionIndex(offset, b.length, "offset");
1982 checkArgument(length > 0, "length must be greater than 0");
1983 checkPositionIndex(offset + length, b.length, "offset + length");
1984 byte[] buf = new byte[length];
1985 RNG.nextBytes(buf);
1986 System.arraycopy(buf, 0, b, offset, length);
1987 }
1988 }