1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.util;
21
22 import java.io.DataInput;
23 import java.io.DataOutput;
24 import java.io.IOException;
25 import java.io.UnsupportedEncodingException;
26 import java.lang.reflect.Field;
27 import java.math.BigDecimal;
28 import java.math.BigInteger;
29 import java.nio.ByteBuffer;
30 import java.nio.ByteOrder;
31 import java.security.AccessController;
32 import java.security.PrivilegedAction;
33 import java.util.Comparator;
34 import java.util.Iterator;
35
36 import org.apache.commons.logging.Log;
37 import org.apache.commons.logging.LogFactory;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
40 import org.apache.hadoop.io.RawComparator;
41 import org.apache.hadoop.io.WritableComparator;
42 import org.apache.hadoop.io.WritableUtils;
43
44 import com.google.common.annotations.VisibleForTesting;
45 import org.apache.hadoop.hbase.util.Bytes.LexicographicalComparerHolder.UnsafeComparer;
46
47 import sun.misc.Unsafe;
48
49
50
51
52
53
54 public class Bytes {
55
56 private static final Log LOG = LogFactory.getLog(Bytes.class);
57
58
59
60
61 public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE;
62
63
64
65
66 public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN;
67
68
69
70
71 public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE;
72
73
74
75
76 public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE;
77
78
79
80
81 public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE;
82
83
84
85
86 public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE;
87
88
89
90
91 public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE;
92
93
94
95
96 public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE;
97
98
99
100
101
102
103
104
105 public static final int ESTIMATED_HEAP_TAX = 16;
106
107
108
109
110 public static class ByteArrayComparator implements RawComparator<byte []> {
111
112
113
114 public ByteArrayComparator() {
115 super();
116 }
117 public int compare(byte [] left, byte [] right) {
118 return compareTo(left, right);
119 }
120 public int compare(byte [] b1, int s1, int l1, byte [] b2, int s2, int l2) {
121 return LexicographicalComparerHolder.BEST_COMPARER.
122 compareTo(b1, s1, l1, b2, s2, l2);
123 }
124 }
125
126
127
128
129
130
131
132
133
134 public static class RowEndKeyComparator extends ByteArrayComparator {
135 @Override
136 public int compare(byte[] left, byte[] right) {
137 return compare(left, 0, left.length, right, 0, right.length);
138 }
139 @Override
140 public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
141 if (b1 == b2 && s1 == s2 && l1 == l2) {
142 return 0;
143 }
144 if (l1 == 0) {
145 return l2;
146 }
147 if (l2 == 0) {
148 return -1;
149 }
150 return super.compare(b1, s1, l1, b2, s2, l2);
151 }
152 }
153
154
155
156
157 public static Comparator<byte []> BYTES_COMPARATOR =
158 new ByteArrayComparator();
159
160
161
162
163 public static RawComparator<byte []> BYTES_RAWCOMPARATOR =
164 new ByteArrayComparator();
165
166
167
168
169
170
171
172 public static byte [] readByteArray(final DataInput in)
173 throws IOException {
174 int len = WritableUtils.readVInt(in);
175 if (len < 0) {
176 throw new NegativeArraySizeException(Integer.toString(len));
177 }
178 byte [] result = new byte[len];
179 in.readFully(result, 0, len);
180 return result;
181 }
182
183
184
185
186
187
188
189 public static byte [] readByteArrayThrowsRuntime(final DataInput in) {
190 try {
191 return readByteArray(in);
192 } catch (Exception e) {
193 throw new RuntimeException(e);
194 }
195 }
196
197
198
199
200
201
202
203 public static void writeByteArray(final DataOutput out, final byte [] b)
204 throws IOException {
205 if(b == null) {
206 WritableUtils.writeVInt(out, 0);
207 } else {
208 writeByteArray(out, b, 0, b.length);
209 }
210 }
211
212
213
214
215
216
217
218
219
220 public static void writeByteArray(final DataOutput out, final byte [] b,
221 final int offset, final int length)
222 throws IOException {
223 WritableUtils.writeVInt(out, length);
224 out.write(b, offset, length);
225 }
226
227
228
229
230
231
232
233
234
235
236 public static int writeByteArray(final byte [] tgt, final int tgtOffset,
237 final byte [] src, final int srcOffset, final int srcLength) {
238 byte [] vint = vintToBytes(srcLength);
239 System.arraycopy(vint, 0, tgt, tgtOffset, vint.length);
240 int offset = tgtOffset + vint.length;
241 System.arraycopy(src, srcOffset, tgt, offset, srcLength);
242 return offset + srcLength;
243 }
244
245
246
247
248
249
250
251
252
253
254 public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes,
255 int srcOffset, int srcLength) {
256 System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength);
257 return tgtOffset + srcLength;
258 }
259
260
261
262
263
264
265
266
267 public static int putByte(byte[] bytes, int offset, byte b) {
268 bytes[offset] = b;
269 return offset + 1;
270 }
271
272
273
274
275
276
277 public static byte[] toBytes(ByteBuffer bb) {
278 int length = bb.limit();
279 byte [] result = new byte[length];
280 System.arraycopy(bb.array(), bb.arrayOffset(), result, 0, length);
281 return result;
282 }
283
284
285
286
287
288 public static String toString(final byte [] b) {
289 if (b == null) {
290 return null;
291 }
292 return toString(b, 0, b.length);
293 }
294
295
296
297
298
299
300
301 public static String toString(final byte [] b1,
302 String sep,
303 final byte [] b2) {
304 return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length);
305 }
306
307
308
309
310
311
312
313
314
315
316
317 public static String toString(final byte [] b, int off, int len) {
318 if (b == null) {
319 return null;
320 }
321 if (len == 0) {
322 return "";
323 }
324 try {
325 return new String(b, off, len, HConstants.UTF8_ENCODING);
326 } catch (UnsupportedEncodingException e) {
327 LOG.error("UTF-8 not supported?", e);
328 return null;
329 }
330 }
331
332
333
334
335
336
337
338
339 public static String toStringBinary(final byte [] b) {
340 if (b == null)
341 return "null";
342 return toStringBinary(b, 0, b.length);
343 }
344
345
346
347
348
349
350
351
352 public static String toStringBinary(ByteBuffer buf) {
353 if (buf == null)
354 return "null";
355 return toStringBinary(buf.array(), buf.arrayOffset(), buf.limit());
356 }
357
358
359
360
361
362
363
364
365
366
367
368 public static String toStringBinary(final byte [] b, int off, int len) {
369 StringBuilder result = new StringBuilder();
370 for (int i = off; i < off + len ; ++i ) {
371 int ch = b[i] & 0xFF;
372 if ( (ch >= '0' && ch <= '9')
373 || (ch >= 'A' && ch <= 'Z')
374 || (ch >= 'a' && ch <= 'z')
375 || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0 ) {
376 result.append((char)ch);
377 } else {
378 result.append(String.format("\\x%02X", ch));
379 }
380 }
381 return result.toString();
382 }
383
384 private static boolean isHexDigit(char c) {
385 return
386 (c >= 'A' && c <= 'F') ||
387 (c >= '0' && c <= '9');
388 }
389
390
391
392
393
394
395
396 public static byte toBinaryFromHex(byte ch) {
397 if ( ch >= 'A' && ch <= 'F' )
398 return (byte) ((byte)10 + (byte) (ch - 'A'));
399
400 return (byte) (ch - '0');
401 }
402
403 public static byte [] toBytesBinary(String in) {
404
405 byte [] b = new byte[in.length()];
406 int size = 0;
407 for (int i = 0; i < in.length(); ++i) {
408 char ch = in.charAt(i);
409 if (ch == '\\' && in.length() > i+1 && in.charAt(i+1) == 'x') {
410
411 char hd1 = in.charAt(i+2);
412 char hd2 = in.charAt(i+3);
413
414
415 if (!isHexDigit(hd1) ||
416 !isHexDigit(hd2)) {
417
418 continue;
419 }
420
421 byte d = (byte) ((toBinaryFromHex((byte)hd1) << 4) + toBinaryFromHex((byte)hd2));
422
423 b[size++] = d;
424 i += 3;
425 } else {
426 b[size++] = (byte) ch;
427 }
428 }
429
430 byte [] b2 = new byte[size];
431 System.arraycopy(b, 0, b2, 0, size);
432 return b2;
433 }
434
435
436
437
438
439
440 public static byte[] toBytes(String s) {
441 try {
442 return s.getBytes(HConstants.UTF8_ENCODING);
443 } catch (UnsupportedEncodingException e) {
444 LOG.error("UTF-8 not supported?", e);
445 return null;
446 }
447 }
448
449
450
451
452
453
454
455
456 public static byte [] toBytes(final boolean b) {
457 return new byte[] { b ? (byte) -1 : (byte) 0 };
458 }
459
460
461
462
463
464
465 public static boolean toBoolean(final byte [] b) {
466 if (b.length != 1) {
467 throw new IllegalArgumentException("Array has wrong size: " + b.length);
468 }
469 return b[0] != (byte) 0;
470 }
471
472
473
474
475
476
477
478 public static byte[] toBytes(long val) {
479 byte [] b = new byte[8];
480 for (int i = 7; i > 0; i--) {
481 b[i] = (byte) val;
482 val >>>= 8;
483 }
484 b[0] = (byte) val;
485 return b;
486 }
487
488
489
490
491
492
493
494 public static long toLong(byte[] bytes) {
495 return toLong(bytes, 0, SIZEOF_LONG);
496 }
497
498
499
500
501
502
503
504
505
506 public static long toLong(byte[] bytes, int offset) {
507 return toLong(bytes, offset, SIZEOF_LONG);
508 }
509
510
511
512
513
514
515
516
517
518
519
520 public static long toLong(byte[] bytes, int offset, final int length) {
521 if (length != SIZEOF_LONG || offset + length > bytes.length) {
522 throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
523 }
524 if (UnsafeComparer.isAvailable()) {
525 return toLongUnsafe(bytes, offset);
526 } else {
527 long l = 0;
528 for(int i = offset; i < offset + length; i++) {
529 l <<= 8;
530 l ^= bytes[i] & 0xFF;
531 }
532 return l;
533 }
534 }
535
536 private static IllegalArgumentException
537 explainWrongLengthOrOffset(final byte[] bytes,
538 final int offset,
539 final int length,
540 final int expectedLength) {
541 String reason;
542 if (length != expectedLength) {
543 reason = "Wrong length: " + length + ", expected " + expectedLength;
544 } else {
545 reason = "offset (" + offset + ") + length (" + length + ") exceed the"
546 + " capacity of the array: " + bytes.length;
547 }
548 return new IllegalArgumentException(reason);
549 }
550
551
552
553
554
555
556
557
558
559
560 public static int putLong(byte[] bytes, int offset, long val) {
561 if (bytes.length - offset < SIZEOF_LONG) {
562 throw new IllegalArgumentException("Not enough room to put a long at"
563 + " offset " + offset + " in a " + bytes.length + " byte array");
564 }
565 if (UnsafeComparer.isAvailable()) {
566 return putLongUnsafe(bytes, offset, val);
567 } else {
568 for(int i = offset + 7; i > offset; i--) {
569 bytes[i] = (byte) val;
570 val >>>= 8;
571 }
572 bytes[offset] = (byte) val;
573 return offset + SIZEOF_LONG;
574 }
575 }
576
577
578
579
580
581
582
583
584 public static int putLongUnsafe(byte[] bytes, int offset, long val)
585 {
586 if (UnsafeComparer.littleEndian) {
587 val = Long.reverseBytes(val);
588 }
589 UnsafeComparer.theUnsafe.putLong(bytes, (long) offset +
590 UnsafeComparer.BYTE_ARRAY_BASE_OFFSET , val);
591 return offset + SIZEOF_LONG;
592 }
593
594
595
596
597
598
599 public static float toFloat(byte [] bytes) {
600 return toFloat(bytes, 0);
601 }
602
603
604
605
606
607
608
609 public static float toFloat(byte [] bytes, int offset) {
610 return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
611 }
612
613
614
615
616
617
618
619 public static int putFloat(byte [] bytes, int offset, float f) {
620 return putInt(bytes, offset, Float.floatToRawIntBits(f));
621 }
622
623
624
625
626
627 public static byte [] toBytes(final float f) {
628
629 return Bytes.toBytes(Float.floatToRawIntBits(f));
630 }
631
632
633
634
635
636 public static double toDouble(final byte [] bytes) {
637 return toDouble(bytes, 0);
638 }
639
640
641
642
643
644
645 public static double toDouble(final byte [] bytes, final int offset) {
646 return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
647 }
648
649
650
651
652
653
654
655 public static int putDouble(byte [] bytes, int offset, double d) {
656 return putLong(bytes, offset, Double.doubleToLongBits(d));
657 }
658
659
660
661
662
663
664
665
666 public static byte [] toBytes(final double d) {
667
668 return Bytes.toBytes(Double.doubleToRawLongBits(d));
669 }
670
671
672
673
674
675
676 public static byte[] toBytes(int val) {
677 byte [] b = new byte[4];
678 for(int i = 3; i > 0; i--) {
679 b[i] = (byte) val;
680 val >>>= 8;
681 }
682 b[0] = (byte) val;
683 return b;
684 }
685
686
687
688
689
690
691 public static int toInt(byte[] bytes) {
692 return toInt(bytes, 0, SIZEOF_INT);
693 }
694
695
696
697
698
699
700
701 public static int toInt(byte[] bytes, int offset) {
702 return toInt(bytes, offset, SIZEOF_INT);
703 }
704
705
706
707
708
709
710
711
712
713
714 public static int toInt(byte[] bytes, int offset, final int length) {
715 if (length != SIZEOF_INT || offset + length > bytes.length) {
716 throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
717 }
718 if (UnsafeComparer.isAvailable()) {
719 return toIntUnsafe(bytes, offset);
720 } else {
721 int n = 0;
722 for(int i = offset; i < (offset + length); i++) {
723 n <<= 8;
724 n ^= bytes[i] & 0xFF;
725 }
726 return n;
727 }
728 }
729
730
731
732
733
734
735
736 public static int toIntUnsafe(byte[] bytes, int offset) {
737 if (UnsafeComparer.littleEndian) {
738 return Integer.reverseBytes(UnsafeComparer.theUnsafe.getInt(bytes,
739 (long) offset + UnsafeComparer.BYTE_ARRAY_BASE_OFFSET));
740 } else {
741 return UnsafeComparer.theUnsafe.getInt(bytes,
742 (long) offset + UnsafeComparer.BYTE_ARRAY_BASE_OFFSET);
743 }
744 }
745
746
747
748
749
750
751
752 public static short toShortUnsafe(byte[] bytes, int offset) {
753 if (UnsafeComparer.littleEndian) {
754 return Short.reverseBytes(UnsafeComparer.theUnsafe.getShort(bytes,
755 (long) offset + UnsafeComparer.BYTE_ARRAY_BASE_OFFSET));
756 } else {
757 return UnsafeComparer.theUnsafe.getShort(bytes,
758 (long) offset + UnsafeComparer.BYTE_ARRAY_BASE_OFFSET);
759 }
760 }
761
762
763
764
765
766
767
768 public static long toLongUnsafe(byte[] bytes, int offset) {
769 if (UnsafeComparer.littleEndian) {
770 return Long.reverseBytes(UnsafeComparer.theUnsafe.getLong(bytes,
771 (long) offset + UnsafeComparer.BYTE_ARRAY_BASE_OFFSET));
772 } else {
773 return UnsafeComparer.theUnsafe.getLong(bytes,
774 (long) offset + UnsafeComparer.BYTE_ARRAY_BASE_OFFSET);
775 }
776 }
777
778
779
780
781
782
783
784
785
786
787 public static int putInt(byte[] bytes, int offset, int val) {
788 if (bytes.length - offset < SIZEOF_INT) {
789 throw new IllegalArgumentException("Not enough room to put an int at"
790 + " offset " + offset + " in a " + bytes.length + " byte array");
791 }
792 if (UnsafeComparer.isAvailable()) {
793 return putIntUnsafe(bytes, offset, val);
794 } else {
795 for(int i= offset + 3; i > offset; i--) {
796 bytes[i] = (byte) val;
797 val >>>= 8;
798 }
799 bytes[offset] = (byte) val;
800 return offset + SIZEOF_INT;
801 }
802 }
803
804
805
806
807
808
809
810
811 public static int putIntUnsafe(byte[] bytes, int offset, int val)
812 {
813 if (UnsafeComparer.littleEndian) {
814 val = Integer.reverseBytes(val);
815 }
816 UnsafeComparer.theUnsafe.putInt(bytes, (long) offset +
817 UnsafeComparer.BYTE_ARRAY_BASE_OFFSET , val);
818 return offset + SIZEOF_INT;
819 }
820
821
822
823
824
825
826 public static byte[] toBytes(short val) {
827 byte[] b = new byte[SIZEOF_SHORT];
828 b[1] = (byte) val;
829 val >>= 8;
830 b[0] = (byte) val;
831 return b;
832 }
833
834
835
836
837
838
839 public static short toShort(byte[] bytes) {
840 return toShort(bytes, 0, SIZEOF_SHORT);
841 }
842
843
844
845
846
847
848
849 public static short toShort(byte[] bytes, int offset) {
850 return toShort(bytes, offset, SIZEOF_SHORT);
851 }
852
853
854
855
856
857
858
859
860
861
862 public static short toShort(byte[] bytes, int offset, final int length) {
863 if (length != SIZEOF_SHORT || offset + length > bytes.length) {
864 throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
865 }
866 if (UnsafeComparer.isAvailable()) {
867 return toShortUnsafe(bytes, offset);
868 } else {
869 short n = 0;
870 n ^= bytes[offset] & 0xFF;
871 n <<= 8;
872 n ^= bytes[offset+1] & 0xFF;
873 return n;
874 }
875 }
876
877
878
879
880
881
882
883 public static byte[] getBytes(ByteBuffer buf) {
884 int savedPos = buf.position();
885 byte [] newBytes = new byte[buf.remaining()];
886 buf.get(newBytes);
887 buf.position(savedPos);
888 return newBytes;
889 }
890
891
892
893
894
895
896
897
898
899
900 public static int putShort(byte[] bytes, int offset, short val) {
901 if (bytes.length - offset < SIZEOF_SHORT) {
902 throw new IllegalArgumentException("Not enough room to put a short at"
903 + " offset " + offset + " in a " + bytes.length + " byte array");
904 }
905 if (UnsafeComparer.isAvailable()) {
906 return putShortUnsafe(bytes, offset, val);
907 } else {
908 bytes[offset+1] = (byte) val;
909 val >>= 8;
910 bytes[offset] = (byte) val;
911 return offset + SIZEOF_SHORT;
912 }
913 }
914
915
916
917
918
919
920
921
922 public static int putShortUnsafe(byte[] bytes, int offset, short val)
923 {
924 if (UnsafeComparer.littleEndian) {
925 val = Short.reverseBytes(val);
926 }
927 UnsafeComparer.theUnsafe.putShort(bytes, (long) offset +
928 UnsafeComparer.BYTE_ARRAY_BASE_OFFSET , val);
929 return offset + SIZEOF_SHORT;
930 }
931
932
933
934
935
936
937
938 public static byte[] toBytes(BigDecimal val) {
939 byte[] valueBytes = val.unscaledValue().toByteArray();
940 byte[] result = new byte[valueBytes.length + SIZEOF_INT];
941 int offset = putInt(result, 0, val.scale());
942 putBytes(result, offset, valueBytes, 0, valueBytes.length);
943 return result;
944 }
945
946
947
948
949
950
951
952
953 public static BigDecimal toBigDecimal(byte[] bytes) {
954 return toBigDecimal(bytes, 0, bytes.length);
955 }
956
957
958
959
960
961
962
963
964
965 public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) {
966 if (bytes == null || length < SIZEOF_INT + 1 ||
967 (offset + length > bytes.length)) {
968 return null;
969 }
970
971 int scale = toInt(bytes, offset);
972 byte[] tcBytes = new byte[length - SIZEOF_INT];
973 System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT);
974 return new BigDecimal(new BigInteger(tcBytes), scale);
975 }
976
977
978
979
980
981
982
983
984
985 public static int putBigDecimal(byte[] bytes, int offset, BigDecimal val) {
986 if (bytes == null) {
987 return offset;
988 }
989
990 byte[] valueBytes = val.unscaledValue().toByteArray();
991 byte[] result = new byte[valueBytes.length + SIZEOF_INT];
992 offset = putInt(result, offset, val.scale());
993 return putBytes(result, offset, valueBytes, 0, valueBytes.length);
994 }
995
996
997
998
999
1000 public static byte [] vintToBytes(final long vint) {
1001 long i = vint;
1002 int size = WritableUtils.getVIntSize(i);
1003 byte [] result = new byte[size];
1004 int offset = 0;
1005 if (i >= -112 && i <= 127) {
1006 result[offset] = (byte) i;
1007 return result;
1008 }
1009
1010 int len = -112;
1011 if (i < 0) {
1012 i ^= -1L;
1013 len = -120;
1014 }
1015
1016 long tmp = i;
1017 while (tmp != 0) {
1018 tmp = tmp >> 8;
1019 len--;
1020 }
1021
1022 result[offset++] = (byte) len;
1023
1024 len = (len < -120) ? -(len + 120) : -(len + 112);
1025
1026 for (int idx = len; idx != 0; idx--) {
1027 int shiftbits = (idx - 1) * 8;
1028 long mask = 0xFFL << shiftbits;
1029 result[offset++] = (byte)((i & mask) >> shiftbits);
1030 }
1031 return result;
1032 }
1033
1034
1035
1036
1037
1038 public static long bytesToVint(final byte [] buffer) {
1039 int offset = 0;
1040 byte firstByte = buffer[offset++];
1041 int len = WritableUtils.decodeVIntSize(firstByte);
1042 if (len == 1) {
1043 return firstByte;
1044 }
1045 long i = 0;
1046 for (int idx = 0; idx < len-1; idx++) {
1047 byte b = buffer[offset++];
1048 i = i << 8;
1049 i = i | (b & 0xFF);
1050 }
1051 return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
1052 }
1053
1054
1055
1056
1057
1058
1059
1060
1061 public static long readVLong(final byte [] buffer, final int offset)
1062 throws IOException {
1063 byte firstByte = buffer[offset];
1064 int len = WritableUtils.decodeVIntSize(firstByte);
1065 if (len == 1) {
1066 return firstByte;
1067 }
1068 long i = 0;
1069 for (int idx = 0; idx < len-1; idx++) {
1070 byte b = buffer[offset + 1 + idx];
1071 i = i << 8;
1072 i = i | (b & 0xFF);
1073 }
1074 return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
1075 }
1076
1077
1078
1079
1080
1081
1082 public static int compareTo(final byte [] left, final byte [] right) {
1083 return LexicographicalComparerHolder.BEST_COMPARER.
1084 compareTo(left, 0, left.length, right, 0, right.length);
1085 }
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098 public static int compareTo(byte[] buffer1, int offset1, int length1,
1099 byte[] buffer2, int offset2, int length2) {
1100 return LexicographicalComparerHolder.BEST_COMPARER.
1101 compareTo(buffer1, offset1, length1, buffer2, offset2, length2);
1102 }
1103
1104 interface Comparer<T> {
1105 abstract public int compareTo(T buffer1, int offset1, int length1,
1106 T buffer2, int offset2, int length2);
1107 }
1108
1109 @VisibleForTesting
1110 static Comparer<byte[]> lexicographicalComparerJavaImpl() {
1111 return LexicographicalComparerHolder.PureJavaComparer.INSTANCE;
1112 }
1113
1114
1115
1116
1117
1118
1119
1120
1121 @VisibleForTesting
1122 static class LexicographicalComparerHolder {
1123 static final String UNSAFE_COMPARER_NAME =
1124 LexicographicalComparerHolder.class.getName() + "$UnsafeComparer";
1125
1126 static final Comparer<byte[]> BEST_COMPARER = getBestComparer();
1127
1128
1129
1130
1131 static Comparer<byte[]> getBestComparer() {
1132 try {
1133 Class<?> theClass = Class.forName(UNSAFE_COMPARER_NAME);
1134
1135
1136 @SuppressWarnings("unchecked")
1137 Comparer<byte[]> comparer =
1138 (Comparer<byte[]>) theClass.getEnumConstants()[0];
1139 return comparer;
1140 } catch (Throwable t) {
1141 return lexicographicalComparerJavaImpl();
1142 }
1143 }
1144
1145 enum PureJavaComparer implements Comparer<byte[]> {
1146 INSTANCE;
1147
1148 @Override
1149 public int compareTo(byte[] buffer1, int offset1, int length1,
1150 byte[] buffer2, int offset2, int length2) {
1151
1152 if (buffer1 == buffer2 &&
1153 offset1 == offset2 &&
1154 length1 == length2) {
1155 return 0;
1156 }
1157
1158 int end1 = offset1 + length1;
1159 int end2 = offset2 + length2;
1160 for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) {
1161 int a = (buffer1[i] & 0xff);
1162 int b = (buffer2[j] & 0xff);
1163 if (a != b) {
1164 return a - b;
1165 }
1166 }
1167 return length1 - length2;
1168 }
1169 }
1170
1171 @VisibleForTesting
1172 enum UnsafeComparer implements Comparer<byte[]> {
1173 INSTANCE;
1174
1175 static final Unsafe theUnsafe;
1176
1177
1178 static final int BYTE_ARRAY_BASE_OFFSET;
1179
1180 static {
1181 theUnsafe = (Unsafe) AccessController.doPrivileged(
1182 new PrivilegedAction<Object>() {
1183 @Override
1184 public Object run() {
1185 try {
1186 Field f = Unsafe.class.getDeclaredField("theUnsafe");
1187 f.setAccessible(true);
1188 return f.get(null);
1189 } catch (NoSuchFieldException e) {
1190
1191
1192 throw new Error();
1193 } catch (IllegalAccessException e) {
1194 throw new Error();
1195 }
1196 }
1197 });
1198
1199 BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class);
1200
1201
1202 if (theUnsafe.arrayIndexScale(byte[].class) != 1) {
1203 throw new AssertionError();
1204 }
1205 }
1206
1207 static final boolean littleEndian =
1208 ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN);
1209
1210
1211
1212
1213
1214 static boolean lessThanUnsignedLong(long x1, long x2) {
1215 return (x1 + Long.MIN_VALUE) < (x2 + Long.MIN_VALUE);
1216 }
1217
1218
1219
1220
1221
1222 static boolean lessThanUnsignedInt(int x1, int x2) {
1223 return (x1 & 0xffffffffL) < (x2 & 0xffffffffL);
1224 }
1225
1226
1227
1228
1229
1230 static boolean lessThanUnsignedShort(short x1, short x2) {
1231 return (x1 & 0xffff) < (x2 & 0xffff);
1232 }
1233
1234
1235
1236
1237
1238 public static boolean isAvailable()
1239 {
1240 return theUnsafe != null;
1241 }
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254 @Override
1255 public int compareTo(byte[] buffer1, int offset1, int length1,
1256 byte[] buffer2, int offset2, int length2) {
1257
1258
1259 if (buffer1 == buffer2 &&
1260 offset1 == offset2 &&
1261 length1 == length2) {
1262 return 0;
1263 }
1264 final int minLength = Math.min(length1, length2);
1265 final int minWords = minLength / SIZEOF_LONG;
1266 final long offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET;
1267 final long offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET;
1268
1269
1270
1271
1272
1273
1274 for (int i = 0; i < minWords * SIZEOF_LONG; i += SIZEOF_LONG) {
1275 long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i);
1276 long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i);
1277 long diff = lw ^ rw;
1278 if(littleEndian){
1279 lw = Long.reverseBytes(lw);
1280 rw = Long.reverseBytes(rw);
1281 }
1282 if (diff != 0) {
1283 return lessThanUnsignedLong(lw, rw) ? -1 : 1;
1284 }
1285 }
1286 int offset = minWords * SIZEOF_LONG;
1287
1288 if (minLength - offset >= SIZEOF_INT) {
1289 int il = theUnsafe.getInt(buffer1, offset1Adj + offset);
1290 int ir = theUnsafe.getInt(buffer2, offset2Adj + offset);
1291 if(littleEndian){
1292 il = Integer.reverseBytes(il);
1293 ir = Integer.reverseBytes(ir);
1294 }
1295 if(il != ir){
1296 return lessThanUnsignedInt(il, ir) ? -1: 1;
1297 }
1298 offset += SIZEOF_INT;
1299 }
1300 if (minLength - offset >= SIZEOF_SHORT) {
1301 short sl = theUnsafe.getShort(buffer1, offset1Adj + offset);
1302 short sr = theUnsafe.getShort(buffer2, offset2Adj + offset);
1303 if(littleEndian){
1304 sl = Short.reverseBytes(sl);
1305 sr = Short.reverseBytes(sr);
1306 }
1307 if(sl != sr){
1308 return lessThanUnsignedShort(sl, sr) ? -1: 1;
1309 }
1310 offset += SIZEOF_SHORT;
1311 }
1312 if (minLength - offset == 1) {
1313 int a = (buffer1[(int)(offset1 + offset)] & 0xff);
1314 int b = (buffer2[(int)(offset2 + offset)] & 0xff);
1315 if (a != b) {
1316 return a - b;
1317 }
1318 }
1319 return length1 - length2;
1320 }
1321 }
1322 }
1323
1324
1325
1326
1327
1328
1329 public static boolean equals(final byte [] left, final byte [] right) {
1330
1331
1332 if (left == right) return true;
1333 if (left == null || right == null) return false;
1334 if (left.length != right.length) return false;
1335 if (left.length == 0) return true;
1336
1337
1338
1339
1340 if (left[left.length - 1] != right[right.length - 1]) return false;
1341
1342 return compareTo(left, right) == 0;
1343 }
1344
1345 public static boolean equals(final byte[] left, int leftOffset, int leftLen,
1346 final byte[] right, int rightOffset, int rightLen) {
1347
1348 if (left == right &&
1349 leftOffset == rightOffset &&
1350 leftLen == rightLen) {
1351 return true;
1352 }
1353
1354 if (leftLen != rightLen) {
1355 return false;
1356 }
1357 if (leftLen == 0) {
1358 return true;
1359 }
1360
1361
1362
1363
1364 if (left[leftOffset + leftLen - 1] != right[rightOffset + rightLen - 1]) return false;
1365
1366 return LexicographicalComparerHolder.BEST_COMPARER.
1367 compareTo(left, leftOffset, leftLen, right, rightOffset, rightLen) == 0;
1368 }
1369
1370
1371
1372
1373
1374
1375 public static boolean startsWith(byte[] bytes, byte[] prefix) {
1376 return bytes != null && prefix != null &&
1377 bytes.length >= prefix.length &&
1378 LexicographicalComparerHolder.BEST_COMPARER.
1379 compareTo(bytes, 0, prefix.length, prefix, 0, prefix.length) == 0;
1380 }
1381
1382
1383
1384
1385
1386
1387
1388 public static int hashCode(final byte [] b) {
1389 return hashCode(b, b.length);
1390 }
1391
1392
1393
1394
1395
1396
1397
1398
1399 public static int hashCode(final byte [] b, final int length) {
1400 return WritableComparator.hashBytes(b, length);
1401 }
1402
1403
1404
1405
1406
1407
1408 public static Integer mapKey(final byte [] b) {
1409 return hashCode(b);
1410 }
1411
1412
1413
1414
1415
1416
1417
1418 public static Integer mapKey(final byte [] b, final int length) {
1419 return hashCode(b, length);
1420 }
1421
1422
1423
1424
1425
1426
1427 public static byte [] add(final byte [] a, final byte [] b) {
1428 return add(a, b, HConstants.EMPTY_BYTE_ARRAY);
1429 }
1430
1431
1432
1433
1434
1435
1436
1437 public static byte [] add(final byte [] a, final byte [] b, final byte [] c) {
1438 byte [] result = new byte[a.length + b.length + c.length];
1439 System.arraycopy(a, 0, result, 0, a.length);
1440 System.arraycopy(b, 0, result, a.length, b.length);
1441 System.arraycopy(c, 0, result, a.length + b.length, c.length);
1442 return result;
1443 }
1444
1445
1446
1447
1448
1449
1450 public static byte [] head(final byte [] a, final int length) {
1451 if (a.length < length) {
1452 return null;
1453 }
1454 byte [] result = new byte[length];
1455 System.arraycopy(a, 0, result, 0, length);
1456 return result;
1457 }
1458
1459
1460
1461
1462
1463
1464 public static byte [] tail(final byte [] a, final int length) {
1465 if (a.length < length) {
1466 return null;
1467 }
1468 byte [] result = new byte[length];
1469 System.arraycopy(a, a.length - length, result, 0, length);
1470 return result;
1471 }
1472
1473
1474
1475
1476
1477
1478 public static byte [] padHead(final byte [] a, final int length) {
1479 byte [] padding = new byte[length];
1480 for (int i = 0; i < length; i++) {
1481 padding[i] = 0;
1482 }
1483 return add(padding,a);
1484 }
1485
1486
1487
1488
1489
1490
1491 public static byte [] padTail(final byte [] a, final int length) {
1492 byte [] padding = new byte[length];
1493 for (int i = 0; i < length; i++) {
1494 padding[i] = 0;
1495 }
1496 return add(a,padding);
1497 }
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508 public static byte [][] split(final byte [] a, final byte [] b, final int num) {
1509 return split(a, b, false, num);
1510 }
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524 public static byte[][] split(final byte[] a, final byte[] b,
1525 boolean inclusive, final int num) {
1526 byte[][] ret = new byte[num + 2][];
1527 int i = 0;
1528 Iterable<byte[]> iter = iterateOnSplits(a, b, inclusive, num);
1529 if (iter == null)
1530 return null;
1531 for (byte[] elem : iter) {
1532 ret[i++] = elem;
1533 }
1534 return ret;
1535 }
1536
1537
1538
1539
1540 public static Iterable<byte[]> iterateOnSplits(final byte[] a,
1541 final byte[] b, final int num)
1542 {
1543 return iterateOnSplits(a, b, false, num);
1544 }
1545
1546
1547
1548
1549 public static Iterable<byte[]> iterateOnSplits(
1550 final byte[] a, final byte[]b, boolean inclusive, final int num)
1551 {
1552 byte [] aPadded;
1553 byte [] bPadded;
1554 if (a.length < b.length) {
1555 aPadded = padTail(a, b.length - a.length);
1556 bPadded = b;
1557 } else if (b.length < a.length) {
1558 aPadded = a;
1559 bPadded = padTail(b, a.length - b.length);
1560 } else {
1561 aPadded = a;
1562 bPadded = b;
1563 }
1564 if (compareTo(aPadded,bPadded) >= 0) {
1565 throw new IllegalArgumentException("b <= a");
1566 }
1567 if (num <= 0) {
1568 throw new IllegalArgumentException("num cannot be < 0");
1569 }
1570 byte [] prependHeader = {1, 0};
1571 final BigInteger startBI = new BigInteger(add(prependHeader, aPadded));
1572 final BigInteger stopBI = new BigInteger(add(prependHeader, bPadded));
1573 BigInteger diffBI = stopBI.subtract(startBI);
1574 if (inclusive) {
1575 diffBI = diffBI.add(BigInteger.ONE);
1576 }
1577 final BigInteger splitsBI = BigInteger.valueOf(num + 1);
1578 if(diffBI.compareTo(splitsBI) < 0) {
1579 return null;
1580 }
1581 final BigInteger intervalBI;
1582 try {
1583 intervalBI = diffBI.divide(splitsBI);
1584 } catch(Exception e) {
1585 LOG.error("Exception caught during division", e);
1586 return null;
1587 }
1588
1589 final Iterator<byte[]> iterator = new Iterator<byte[]>() {
1590 private int i = -1;
1591
1592 @Override
1593 public boolean hasNext() {
1594 return i < num+1;
1595 }
1596
1597 @Override
1598 public byte[] next() {
1599 i++;
1600 if (i == 0) return a;
1601 if (i == num + 1) return b;
1602
1603 BigInteger curBI = startBI.add(intervalBI.multiply(BigInteger.valueOf(i)));
1604 byte [] padded = curBI.toByteArray();
1605 if (padded[1] == 0)
1606 padded = tail(padded, padded.length - 2);
1607 else
1608 padded = tail(padded, padded.length - 1);
1609 return padded;
1610 }
1611
1612 @Override
1613 public void remove() {
1614 throw new UnsupportedOperationException();
1615 }
1616
1617 };
1618
1619 return new Iterable<byte[]>() {
1620 @Override
1621 public Iterator<byte[]> iterator() {
1622 return iterator;
1623 }
1624 };
1625 }
1626
1627
1628
1629
1630
1631
1632 public static int hashCode(byte[] bytes, int offset, int length) {
1633 int hash = 1;
1634 for (int i = offset; i < offset + length; i++)
1635 hash = (31 * hash) + (int) bytes[i];
1636 return hash;
1637 }
1638
1639
1640
1641
1642
1643 public static byte [][] toByteArrays(final String [] t) {
1644 byte [][] result = new byte[t.length][];
1645 for (int i = 0; i < t.length; i++) {
1646 result[i] = Bytes.toBytes(t[i]);
1647 }
1648 return result;
1649 }
1650
1651
1652
1653
1654
1655
1656 public static byte [][] toByteArrays(final String column) {
1657 return toByteArrays(toBytes(column));
1658 }
1659
1660
1661
1662
1663
1664
1665 public static byte [][] toByteArrays(final byte [] column) {
1666 byte [][] result = new byte[1][];
1667 result[0] = column;
1668 return result;
1669 }
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686 public static int binarySearch(byte [][]arr, byte []key, int offset,
1687 int length, RawComparator<byte []> comparator) {
1688 int low = 0;
1689 int high = arr.length - 1;
1690
1691 while (low <= high) {
1692 int mid = (low+high) >>> 1;
1693
1694
1695 int cmp = comparator.compare(key, offset, length,
1696 arr[mid], 0, arr[mid].length);
1697
1698 if (cmp > 0)
1699 low = mid + 1;
1700
1701 else if (cmp < 0)
1702 high = mid - 1;
1703
1704 else
1705 return mid;
1706 }
1707 return - (low+1);
1708 }
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718 public static byte [] incrementBytes(byte[] value, long amount)
1719 {
1720 byte[] val = value;
1721 if (val.length < SIZEOF_LONG) {
1722
1723 byte [] newvalue;
1724 if (val[0] < 0) {
1725 newvalue = new byte[]{-1, -1, -1, -1, -1, -1, -1, -1};
1726 } else {
1727 newvalue = new byte[SIZEOF_LONG];
1728 }
1729 System.arraycopy(val, 0, newvalue, newvalue.length - val.length,
1730 val.length);
1731 val = newvalue;
1732 } else if (val.length > SIZEOF_LONG) {
1733 throw new IllegalArgumentException("Increment Bytes - value too big: " +
1734 val.length);
1735 }
1736 if(amount == 0) return val;
1737 if(val[0] < 0){
1738 return binaryIncrementNeg(val, amount);
1739 }
1740 return binaryIncrementPos(val, amount);
1741 }
1742
1743
1744 private static byte [] binaryIncrementPos(byte [] value, long amount) {
1745 long amo = amount;
1746 int sign = 1;
1747 if (amount < 0) {
1748 amo = -amount;
1749 sign = -1;
1750 }
1751 for(int i=0;i<value.length;i++) {
1752 int cur = ((int)amo % 256) * sign;
1753 amo = (amo >> 8);
1754 int val = value[value.length-i-1] & 0x0ff;
1755 int total = val + cur;
1756 if(total > 255) {
1757 amo += sign;
1758 total %= 256;
1759 } else if (total < 0) {
1760 amo -= sign;
1761 }
1762 value[value.length-i-1] = (byte)total;
1763 if (amo == 0) return value;
1764 }
1765 return value;
1766 }
1767
1768
1769 private static byte [] binaryIncrementNeg(byte [] value, long amount) {
1770 long amo = amount;
1771 int sign = 1;
1772 if (amount < 0) {
1773 amo = -amount;
1774 sign = -1;
1775 }
1776 for(int i=0;i<value.length;i++) {
1777 int cur = ((int)amo % 256) * sign;
1778 amo = (amo >> 8);
1779 int val = ((~value[value.length-i-1]) & 0x0ff) + 1;
1780 int total = cur - val;
1781 if(total >= 0) {
1782 amo += sign;
1783 } else if (total < -256) {
1784 amo -= sign;
1785 total %= 256;
1786 }
1787 value[value.length-i-1] = (byte)total;
1788 if (amo == 0) return value;
1789 }
1790 return value;
1791 }
1792
1793
1794
1795
1796 public static void writeStringFixedSize(final DataOutput out, String s,
1797 int size) throws IOException {
1798 byte[] b = toBytes(s);
1799 if (b.length > size) {
1800 throw new IOException("Trying to write " + b.length + " bytes (" +
1801 toStringBinary(b) + ") into a field of length " + size);
1802 }
1803
1804 out.writeBytes(s);
1805 for (int i = 0; i < size - s.length(); ++i)
1806 out.writeByte(0);
1807 }
1808
1809
1810
1811
1812 public static String readStringFixedSize(final DataInput in, int size)
1813 throws IOException {
1814 byte[] b = new byte[size];
1815 in.readFully(b);
1816 int n = b.length;
1817 while (n > 0 && b[n - 1] == 0)
1818 --n;
1819
1820 return toString(b, 0, n);
1821 }
1822
1823 }