View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import static com.google.common.base.Preconditions.checkArgument;
21  import static com.google.common.base.Preconditions.checkPositionIndex;
22  import static com.google.common.base.Preconditions.checkNotNull;
23  
24  import java.io.DataInput;
25  import java.io.DataOutput;
26  import java.io.IOException;
27  import java.lang.reflect.Field;
28  import java.math.BigDecimal;
29  import java.math.BigInteger;
30  import java.nio.ByteBuffer;
31  import java.nio.ByteOrder;
32  import java.security.AccessController;
33  import java.security.PrivilegedAction;
34  import java.security.SecureRandom;
35  import java.util.Arrays;
36  import java.util.Collection;
37  import java.util.Comparator;
38  import java.util.Iterator;
39  import java.util.List;
40  
41  import org.apache.commons.logging.Log;
42  import org.apache.commons.logging.LogFactory;
43  import org.apache.hadoop.classification.InterfaceAudience;
44  import org.apache.hadoop.classification.InterfaceStability;
45  import org.apache.hadoop.hbase.HConstants;
46  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
47  import org.apache.hadoop.io.RawComparator;
48  import org.apache.hadoop.io.WritableComparator;
49  import org.apache.hadoop.io.WritableUtils;
50  
51  import sun.misc.Unsafe;
52  
53  import com.google.common.annotations.VisibleForTesting;
54  import com.google.common.collect.Lists;
55  
56  /**
57   * Utility class that handles byte arrays, conversions to/from other types,
58   * comparisons, hash code generation, manufacturing keys for HashMaps or
59   * HashSets, etc.
60   */
61  @InterfaceAudience.Public
62  @InterfaceStability.Stable
63  public class Bytes {
64  
65    private static final Log LOG = LogFactory.getLog(Bytes.class);
66  
67    /**
68     * Size of boolean in bytes
69     */
70    public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE;
71  
72    /**
73     * Size of byte in bytes
74     */
75    public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN;
76  
77    /**
78     * Size of char in bytes
79     */
80    public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE;
81  
82    /**
83     * Size of double in bytes
84     */
85    public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE;
86  
87    /**
88     * Size of float in bytes
89     */
90    public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE;
91  
92    /**
93     * Size of int in bytes
94     */
95    public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE;
96  
97    /**
98     * Size of long in bytes
99     */
100   public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE;
101 
102   /**
103    * Size of short in bytes
104    */
105   public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE;
106 
107 
108   /**
109    * Estimate of size cost to pay beyond payload in jvm for instance of byte [].
110    * Estimate based on study of jhat and jprofiler numbers.
111    */
112   // JHat says BU is 56 bytes.
113   // SizeOf which uses java.lang.instrument says 24 bytes. (3 longs?)
114   public static final int ESTIMATED_HEAP_TAX = 16;
115 
116   /**
117    * Byte array comparator class.
118    */
119   public static class ByteArrayComparator implements RawComparator<byte []> {
120     /**
121      * Constructor
122      */
123     public ByteArrayComparator() {
124       super();
125     }
126     public int compare(byte [] left, byte [] right) {
127       return compareTo(left, right);
128     }
129     public int compare(byte [] b1, int s1, int l1, byte [] b2, int s2, int l2) {
130       return LexicographicalComparerHolder.BEST_COMPARER.
131         compareTo(b1, s1, l1, b2, s2, l2);
132     }
133   }
134 
135   /**
136    * Pass this to TreeMaps where byte [] are keys.
137    */
138   public static Comparator<byte []> BYTES_COMPARATOR =
139     new ByteArrayComparator();
140 
141   /**
142    * Use comparing byte arrays, byte-by-byte
143    */
144   public static RawComparator<byte []> BYTES_RAWCOMPARATOR =
145     new ByteArrayComparator();
146 
147   /**
148    * Read byte-array written with a WritableableUtils.vint prefix.
149    * @param in Input to read from.
150    * @return byte array read off <code>in</code>
151    * @throws IOException e
152    */
153   public static byte [] readByteArray(final DataInput in)
154   throws IOException {
155     int len = WritableUtils.readVInt(in);
156     if (len < 0) {
157       throw new NegativeArraySizeException(Integer.toString(len));
158     }
159     byte [] result = new byte[len];
160     in.readFully(result, 0, len);
161     return result;
162   }
163 
164   /**
165    * Read byte-array written with a WritableableUtils.vint prefix.
166    * IOException is converted to a RuntimeException.
167    * @param in Input to read from.
168    * @return byte array read off <code>in</code>
169    */
170   public static byte [] readByteArrayThrowsRuntime(final DataInput in) {
171     try {
172       return readByteArray(in);
173     } catch (Exception e) {
174       throw new RuntimeException(e);
175     }
176   }
177 
178   /**
179    * Write byte-array with a WritableableUtils.vint prefix.
180    * @param out output stream to be written to
181    * @param b array to write
182    * @throws IOException e
183    */
184   public static void writeByteArray(final DataOutput out, final byte [] b)
185   throws IOException {
186     if(b == null) {
187       WritableUtils.writeVInt(out, 0);
188     } else {
189       writeByteArray(out, b, 0, b.length);
190     }
191   }
192 
193   /**
194    * Write byte-array to out with a vint length prefix.
195    * @param out output stream
196    * @param b array
197    * @param offset offset into array
198    * @param length length past offset
199    * @throws IOException e
200    */
201   public static void writeByteArray(final DataOutput out, final byte [] b,
202       final int offset, final int length)
203   throws IOException {
204     WritableUtils.writeVInt(out, length);
205     out.write(b, offset, length);
206   }
207 
208   /**
209    * Write byte-array from src to tgt with a vint length prefix.
210    * @param tgt target array
211    * @param tgtOffset offset into target array
212    * @param src source array
213    * @param srcOffset source offset
214    * @param srcLength source length
215    * @return New offset in src array.
216    */
217   public static int writeByteArray(final byte [] tgt, final int tgtOffset,
218       final byte [] src, final int srcOffset, final int srcLength) {
219     byte [] vint = vintToBytes(srcLength);
220     System.arraycopy(vint, 0, tgt, tgtOffset, vint.length);
221     int offset = tgtOffset + vint.length;
222     System.arraycopy(src, srcOffset, tgt, offset, srcLength);
223     return offset + srcLength;
224   }
225 
226   /**
227    * Put bytes at the specified byte array position.
228    * @param tgtBytes the byte array
229    * @param tgtOffset position in the array
230    * @param srcBytes array to write out
231    * @param srcOffset source offset
232    * @param srcLength source length
233    * @return incremented offset
234    */
235   public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes,
236       int srcOffset, int srcLength) {
237     System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength);
238     return tgtOffset + srcLength;
239   }
240 
241   /**
242    * Write a single byte out to the specified byte array position.
243    * @param bytes the byte array
244    * @param offset position in the array
245    * @param b byte to write out
246    * @return incremented offset
247    */
248   public static int putByte(byte[] bytes, int offset, byte b) {
249     bytes[offset] = b;
250     return offset + 1;
251   }
252 
253   /**
254    * Returns a new byte array, copied from the given {@code buf},
255    * from the index 0 (inclusive) to the limit (exclusive),
256    * regardless of the current position.
257    * The position and the other index parameters are not changed.
258    *
259    * @param buf a byte buffer
260    * @return the byte array
261    * @see #getBytes(ByteBuffer)
262    */
263   public static byte[] toBytes(ByteBuffer buf) {
264     ByteBuffer dup = buf.duplicate();
265     dup.position(0);
266     return readBytes(dup);
267   }
268 
269   private static byte[] readBytes(ByteBuffer buf) {
270     byte [] result = new byte[buf.remaining()];
271     buf.get(result);
272     return result;
273   }
274 
275   /**
276    * @param b Presumed UTF-8 encoded byte array.
277    * @return String made from <code>b</code>
278    */
279   public static String toString(final byte [] b) {
280     if (b == null) {
281       return null;
282     }
283     return toString(b, 0, b.length);
284   }
285 
286   /**
287    * Joins two byte arrays together using a separator.
288    * @param b1 The first byte array.
289    * @param sep The separator to use.
290    * @param b2 The second byte array.
291    */
292   public static String toString(final byte [] b1,
293                                 String sep,
294                                 final byte [] b2) {
295     return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length);
296   }
297 
298   /**
299    * This method will convert utf8 encoded bytes into a string. If
300    * the given byte array is null, this method will return null.
301    *
302    * @param b Presumed UTF-8 encoded byte array.
303    * @param off offset into array
304    * @param len length of utf-8 sequence
305    * @return String made from <code>b</code> or null
306    */
307   public static String toString(final byte [] b, int off, int len) {
308     if (b == null) {
309       return null;
310     }
311     if (len == 0) {
312       return "";
313     }
314     return new String(b, off, len, HConstants.UTF8_CHARSET);
315   }
316 
317   /**
318    * Write a printable representation of a byte array.
319    *
320    * @param b byte array
321    * @return string
322    * @see #toStringBinary(byte[], int, int)
323    */
324   public static String toStringBinary(final byte [] b) {
325     if (b == null)
326       return "null";
327     return toStringBinary(b, 0, b.length);
328   }
329 
330   /**
331    * Converts the given byte buffer to a printable representation,
332    * from the index 0 (inclusive) to the limit (exclusive),
333    * regardless of the current position.
334    * The position and the other index parameters are not changed.
335    *
336    * @param buf a byte buffer
337    * @return a string representation of the buffer's binary contents
338    * @see #toBytes(ByteBuffer)
339    * @see #getBytes(ByteBuffer)
340    */
341   public static String toStringBinary(ByteBuffer buf) {
342     if (buf == null)
343       return "null";
344     if (buf.hasArray()) {
345       return toStringBinary(buf.array(), buf.arrayOffset(), buf.limit());
346     }
347     return toStringBinary(toBytes(buf));
348   }
349 
350   /**
351    * Write a printable representation of a byte array. Non-printable
352    * characters are hex escaped in the format \\x%02X, eg:
353    * \x00 \x05 etc
354    *
355    * @param b array to write out
356    * @param off offset to start at
357    * @param len length to write
358    * @return string output
359    */
360   public static String toStringBinary(final byte [] b, int off, int len) {
361     StringBuilder result = new StringBuilder();
362     // Just in case we are passed a 'len' that is > buffer length...
363     if (off >= b.length) return result.toString();
364     if (off + len > b.length) len = b.length - off;
365     for (int i = off; i < off + len ; ++i ) {
366       int ch = b[i] & 0xFF;
367       if ( (ch >= '0' && ch <= '9')
368           || (ch >= 'A' && ch <= 'Z')
369           || (ch >= 'a' && ch <= 'z')
370           || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0 ) {
371         result.append((char)ch);
372       } else {
373         result.append(String.format("\\x%02X", ch));
374       }
375     }
376     return result.toString();
377   }
378 
379   private static boolean isHexDigit(char c) {
380     return
381         (c >= 'A' && c <= 'F') ||
382         (c >= '0' && c <= '9');
383   }
384 
385   /**
386    * Takes a ASCII digit in the range A-F0-9 and returns
387    * the corresponding integer/ordinal value.
388    * @param ch  The hex digit.
389    * @return The converted hex value as a byte.
390    */
391   public static byte toBinaryFromHex(byte ch) {
392     if ( ch >= 'A' && ch <= 'F' )
393       return (byte) ((byte)10 + (byte) (ch - 'A'));
394     // else
395     return (byte) (ch - '0');
396   }
397 
398   public static byte [] toBytesBinary(String in) {
399     // this may be bigger than we need, but let's be safe.
400     byte [] b = new byte[in.length()];
401     int size = 0;
402     for (int i = 0; i < in.length(); ++i) {
403       char ch = in.charAt(i);
404       if (ch == '\\' && in.length() > i+1 && in.charAt(i+1) == 'x') {
405         // ok, take next 2 hex digits.
406         char hd1 = in.charAt(i+2);
407         char hd2 = in.charAt(i+3);
408 
409         // they need to be A-F0-9:
410         if (!isHexDigit(hd1) ||
411             !isHexDigit(hd2)) {
412           // bogus escape code, ignore:
413           continue;
414         }
415         // turn hex ASCII digit -> number
416         byte d = (byte) ((toBinaryFromHex((byte)hd1) << 4) + toBinaryFromHex((byte)hd2));
417 
418         b[size++] = d;
419         i += 3; // skip 3
420       } else {
421         b[size++] = (byte) ch;
422       }
423     }
424     // resize:
425     byte [] b2 = new byte[size];
426     System.arraycopy(b, 0, b2, 0, size);
427     return b2;
428   }
429 
430   /**
431    * Converts a string to a UTF-8 byte array.
432    * @param s string
433    * @return the byte array
434    */
435   public static byte[] toBytes(String s) {
436     return s.getBytes(HConstants.UTF8_CHARSET);
437   }
438 
439   /**
440    * Convert a boolean to a byte array. True becomes -1
441    * and false becomes 0.
442    *
443    * @param b value
444    * @return <code>b</code> encoded in a byte array.
445    */
446   public static byte [] toBytes(final boolean b) {
447     return new byte[] { b ? (byte) -1 : (byte) 0 };
448   }
449 
450   /**
451    * Reverses {@link #toBytes(boolean)}
452    * @param b array
453    * @return True or false.
454    */
455   public static boolean toBoolean(final byte [] b) {
456     if (b.length != 1) {
457       throw new IllegalArgumentException("Array has wrong size: " + b.length);
458     }
459     return b[0] != (byte) 0;
460   }
461 
462   /**
463    * Convert a long value to a byte array using big-endian.
464    *
465    * @param val value to convert
466    * @return the byte array
467    */
468   public static byte[] toBytes(long val) {
469     byte [] b = new byte[8];
470     for (int i = 7; i > 0; i--) {
471       b[i] = (byte) val;
472       val >>>= 8;
473     }
474     b[0] = (byte) val;
475     return b;
476   }
477 
478   /**
479    * Converts a byte array to a long value. Reverses
480    * {@link #toBytes(long)}
481    * @param bytes array
482    * @return the long value
483    */
484   public static long toLong(byte[] bytes) {
485     return toLong(bytes, 0, SIZEOF_LONG);
486   }
487 
488   /**
489    * Converts a byte array to a long value. Assumes there will be
490    * {@link #SIZEOF_LONG} bytes available.
491    *
492    * @param bytes bytes
493    * @param offset offset
494    * @return the long value
495    */
496   public static long toLong(byte[] bytes, int offset) {
497     return toLong(bytes, offset, SIZEOF_LONG);
498   }
499 
500   /**
501    * Converts a byte array to a long value.
502    *
503    * @param bytes array of bytes
504    * @param offset offset into array
505    * @param length length of data (must be {@link #SIZEOF_LONG})
506    * @return the long value
507    * @throws IllegalArgumentException if length is not {@link #SIZEOF_LONG} or
508    * if there's not enough room in the array at the offset indicated.
509    */
510   public static long toLong(byte[] bytes, int offset, final int length) {
511     if (length != SIZEOF_LONG || offset + length > bytes.length) {
512       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
513     }
514     long l = 0;
515     for(int i = offset; i < offset + length; i++) {
516       l <<= 8;
517       l ^= bytes[i] & 0xFF;
518     }
519     return l;
520   }
521 
522   private static IllegalArgumentException
523     explainWrongLengthOrOffset(final byte[] bytes,
524                                final int offset,
525                                final int length,
526                                final int expectedLength) {
527     String reason;
528     if (length != expectedLength) {
529       reason = "Wrong length: " + length + ", expected " + expectedLength;
530     } else {
531      reason = "offset (" + offset + ") + length (" + length + ") exceed the"
532         + " capacity of the array: " + bytes.length;
533     }
534     return new IllegalArgumentException(reason);
535   }
536 
537   /**
538    * Put a long value out to the specified byte array position.
539    * @param bytes the byte array
540    * @param offset position in the array
541    * @param val long to write out
542    * @return incremented offset
543    * @throws IllegalArgumentException if the byte array given doesn't have
544    * enough room at the offset specified.
545    */
546   public static int putLong(byte[] bytes, int offset, long val) {
547     if (bytes.length - offset < SIZEOF_LONG) {
548       throw new IllegalArgumentException("Not enough room to put a long at"
549           + " offset " + offset + " in a " + bytes.length + " byte array");
550     }
551     for(int i = offset + 7; i > offset; i--) {
552       bytes[i] = (byte) val;
553       val >>>= 8;
554     }
555     bytes[offset] = (byte) val;
556     return offset + SIZEOF_LONG;
557   }
558 
559   /**
560    * Presumes float encoded as IEEE 754 floating-point "single format"
561    * @param bytes byte array
562    * @return Float made from passed byte array.
563    */
564   public static float toFloat(byte [] bytes) {
565     return toFloat(bytes, 0);
566   }
567 
568   /**
569    * Presumes float encoded as IEEE 754 floating-point "single format"
570    * @param bytes array to convert
571    * @param offset offset into array
572    * @return Float made from passed byte array.
573    */
574   public static float toFloat(byte [] bytes, int offset) {
575     return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
576   }
577 
578   /**
579    * @param bytes byte array
580    * @param offset offset to write to
581    * @param f float value
582    * @return New offset in <code>bytes</code>
583    */
584   public static int putFloat(byte [] bytes, int offset, float f) {
585     return putInt(bytes, offset, Float.floatToRawIntBits(f));
586   }
587 
588   /**
589    * @param f float value
590    * @return the float represented as byte []
591    */
592   public static byte [] toBytes(final float f) {
593     // Encode it as int
594     return Bytes.toBytes(Float.floatToRawIntBits(f));
595   }
596 
597   /**
598    * @param bytes byte array
599    * @return Return double made from passed bytes.
600    */
601   public static double toDouble(final byte [] bytes) {
602     return toDouble(bytes, 0);
603   }
604 
605   /**
606    * @param bytes byte array
607    * @param offset offset where double is
608    * @return Return double made from passed bytes.
609    */
610   public static double toDouble(final byte [] bytes, final int offset) {
611     return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
612   }
613 
614   /**
615    * @param bytes byte array
616    * @param offset offset to write to
617    * @param d value
618    * @return New offset into array <code>bytes</code>
619    */
620   public static int putDouble(byte [] bytes, int offset, double d) {
621     return putLong(bytes, offset, Double.doubleToLongBits(d));
622   }
623 
624   /**
625    * Serialize a double as the IEEE 754 double format output. The resultant
626    * array will be 8 bytes long.
627    *
628    * @param d value
629    * @return the double represented as byte []
630    */
631   public static byte [] toBytes(final double d) {
632     // Encode it as a long
633     return Bytes.toBytes(Double.doubleToRawLongBits(d));
634   }
635 
636   /**
637    * Convert an int value to a byte array.  Big-endian.  Same as what DataOutputStream.writeInt
638    * does.
639    *
640    * @param val value
641    * @return the byte array
642    */
643   public static byte[] toBytes(int val) {
644     byte [] b = new byte[4];
645     for(int i = 3; i > 0; i--) {
646       b[i] = (byte) val;
647       val >>>= 8;
648     }
649     b[0] = (byte) val;
650     return b;
651   }
652 
653   /**
654    * Converts a byte array to an int value
655    * @param bytes byte array
656    * @return the int value
657    */
658   public static int toInt(byte[] bytes) {
659     return toInt(bytes, 0, SIZEOF_INT);
660   }
661 
662   /**
663    * Converts a byte array to an int value
664    * @param bytes byte array
665    * @param offset offset into array
666    * @return the int value
667    */
668   public static int toInt(byte[] bytes, int offset) {
669     return toInt(bytes, offset, SIZEOF_INT);
670   }
671 
672   /**
673    * Converts a byte array to an int value
674    * @param bytes byte array
675    * @param offset offset into array
676    * @param length length of int (has to be {@link #SIZEOF_INT})
677    * @return the int value
678    * @throws IllegalArgumentException if length is not {@link #SIZEOF_INT} or
679    * if there's not enough room in the array at the offset indicated.
680    */
681   public static int toInt(byte[] bytes, int offset, final int length) {
682     if (length != SIZEOF_INT || offset + length > bytes.length) {
683       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
684     }
685     int n = 0;
686     for(int i = offset; i < (offset + length); i++) {
687       n <<= 8;
688       n ^= bytes[i] & 0xFF;
689     }
690     return n;
691   }
692 
693   /**
694    * Put an int value out to the specified byte array position.
695    * @param bytes the byte array
696    * @param offset position in the array
697    * @param val int to write out
698    * @return incremented offset
699    * @throws IllegalArgumentException if the byte array given doesn't have
700    * enough room at the offset specified.
701    */
702   public static int putInt(byte[] bytes, int offset, int val) {
703     if (bytes.length - offset < SIZEOF_INT) {
704       throw new IllegalArgumentException("Not enough room to put an int at"
705           + " offset " + offset + " in a " + bytes.length + " byte array");
706     }
707     for(int i= offset + 3; i > offset; i--) {
708       bytes[i] = (byte) val;
709       val >>>= 8;
710     }
711     bytes[offset] = (byte) val;
712     return offset + SIZEOF_INT;
713   }
714 
715   /**
716    * Convert a short value to a byte array of {@link #SIZEOF_SHORT} bytes long.
717    * @param val value
718    * @return the byte array
719    */
720   public static byte[] toBytes(short val) {
721     byte[] b = new byte[SIZEOF_SHORT];
722     b[1] = (byte) val;
723     val >>= 8;
724     b[0] = (byte) val;
725     return b;
726   }
727 
728   /**
729    * Converts a byte array to a short value
730    * @param bytes byte array
731    * @return the short value
732    */
733   public static short toShort(byte[] bytes) {
734     return toShort(bytes, 0, SIZEOF_SHORT);
735   }
736 
737   /**
738    * Converts a byte array to a short value
739    * @param bytes byte array
740    * @param offset offset into array
741    * @return the short value
742    */
743   public static short toShort(byte[] bytes, int offset) {
744     return toShort(bytes, offset, SIZEOF_SHORT);
745   }
746 
747   /**
748    * Converts a byte array to a short value
749    * @param bytes byte array
750    * @param offset offset into array
751    * @param length length, has to be {@link #SIZEOF_SHORT}
752    * @return the short value
753    * @throws IllegalArgumentException if length is not {@link #SIZEOF_SHORT}
754    * or if there's not enough room in the array at the offset indicated.
755    */
756   public static short toShort(byte[] bytes, int offset, final int length) {
757     if (length != SIZEOF_SHORT || offset + length > bytes.length) {
758       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
759     }
760     short n = 0;
761     n ^= bytes[offset] & 0xFF;
762     n <<= 8;
763     n ^= bytes[offset+1] & 0xFF;
764     return n;
765   }
766 
767   /**
768    * Returns a new byte array, copied from the given {@code buf},
769    * from the position (inclusive) to the limit (exclusive).
770    * The position and the other index parameters are not changed.
771    *
772    * @param buf a byte buffer
773    * @return the byte array
774    * @see #toBytes(ByteBuffer)
775    */
776   public static byte[] getBytes(ByteBuffer buf) {
777     return readBytes(buf.duplicate());
778   }
779 
780   /**
781    * Put a short value out to the specified byte array position.
782    * @param bytes the byte array
783    * @param offset position in the array
784    * @param val short to write out
785    * @return incremented offset
786    * @throws IllegalArgumentException if the byte array given doesn't have
787    * enough room at the offset specified.
788    */
789   public static int putShort(byte[] bytes, int offset, short val) {
790     if (bytes.length - offset < SIZEOF_SHORT) {
791       throw new IllegalArgumentException("Not enough room to put a short at"
792           + " offset " + offset + " in a " + bytes.length + " byte array");
793     }
794     bytes[offset+1] = (byte) val;
795     val >>= 8;
796     bytes[offset] = (byte) val;
797     return offset + SIZEOF_SHORT;
798   }
799 
800   /**
801    * Convert a BigDecimal value to a byte array
802    *
803    * @param val
804    * @return the byte array
805    */
806   public static byte[] toBytes(BigDecimal val) {
807     byte[] valueBytes = val.unscaledValue().toByteArray();
808     byte[] result = new byte[valueBytes.length + SIZEOF_INT];
809     int offset = putInt(result, 0, val.scale());
810     putBytes(result, offset, valueBytes, 0, valueBytes.length);
811     return result;
812   }
813 
814 
815   /**
816    * Converts a byte array to a BigDecimal
817    *
818    * @param bytes
819    * @return the char value
820    */
821   public static BigDecimal toBigDecimal(byte[] bytes) {
822     return toBigDecimal(bytes, 0, bytes.length);
823   }
824 
825   /**
826    * Converts a byte array to a BigDecimal value
827    *
828    * @param bytes
829    * @param offset
830    * @param length
831    * @return the char value
832    */
833   public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) {
834     if (bytes == null || length < SIZEOF_INT + 1 ||
835       (offset + length > bytes.length)) {
836       return null;
837     }
838 
839     int scale = toInt(bytes, offset);
840     byte[] tcBytes = new byte[length - SIZEOF_INT];
841     System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT);
842     return new BigDecimal(new BigInteger(tcBytes), scale);
843   }
844 
845   /**
846    * Put a BigDecimal value out to the specified byte array position.
847    *
848    * @param bytes  the byte array
849    * @param offset position in the array
850    * @param val    BigDecimal to write out
851    * @return incremented offset
852    */
853   public static int putBigDecimal(byte[] bytes, int offset, BigDecimal val) {
854     if (bytes == null) {
855       return offset;
856     }
857 
858     byte[] valueBytes = val.unscaledValue().toByteArray();
859     byte[] result = new byte[valueBytes.length + SIZEOF_INT];
860     offset = putInt(result, offset, val.scale());
861     return putBytes(result, offset, valueBytes, 0, valueBytes.length);
862   }
863 
864   /**
865    * @param vint Integer to make a vint of.
866    * @return Vint as bytes array.
867    */
868   public static byte [] vintToBytes(final long vint) {
869     long i = vint;
870     int size = WritableUtils.getVIntSize(i);
871     byte [] result = new byte[size];
872     int offset = 0;
873     if (i >= -112 && i <= 127) {
874       result[offset] = (byte) i;
875       return result;
876     }
877 
878     int len = -112;
879     if (i < 0) {
880       i ^= -1L; // take one's complement'
881       len = -120;
882     }
883 
884     long tmp = i;
885     while (tmp != 0) {
886       tmp = tmp >> 8;
887       len--;
888     }
889 
890     result[offset++] = (byte) len;
891 
892     len = (len < -120) ? -(len + 120) : -(len + 112);
893 
894     for (int idx = len; idx != 0; idx--) {
895       int shiftbits = (idx - 1) * 8;
896       long mask = 0xFFL << shiftbits;
897       result[offset++] = (byte)((i & mask) >> shiftbits);
898     }
899     return result;
900   }
901 
902   /**
903    * @param buffer buffer to convert
904    * @return vint bytes as an integer.
905    */
906   public static long bytesToVint(final byte [] buffer) {
907     int offset = 0;
908     byte firstByte = buffer[offset++];
909     int len = WritableUtils.decodeVIntSize(firstByte);
910     if (len == 1) {
911       return firstByte;
912     }
913     long i = 0;
914     for (int idx = 0; idx < len-1; idx++) {
915       byte b = buffer[offset++];
916       i = i << 8;
917       i = i | (b & 0xFF);
918     }
919     return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
920   }
921 
922   /**
923    * Reads a zero-compressed encoded long from input stream and returns it.
924    * @param buffer Binary array
925    * @param offset Offset into array at which vint begins.
926    * @throws java.io.IOException e
927    * @return deserialized long from stream.
928    */
929   public static long readVLong(final byte [] buffer, final int offset)
930   throws IOException {
931     byte firstByte = buffer[offset];
932     int len = WritableUtils.decodeVIntSize(firstByte);
933     if (len == 1) {
934       return firstByte;
935     }
936     long i = 0;
937     for (int idx = 0; idx < len-1; idx++) {
938       byte b = buffer[offset + 1 + idx];
939       i = i << 8;
940       i = i | (b & 0xFF);
941     }
942     return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
943   }
944 
945   /**
946    * @param left left operand
947    * @param right right operand
948    * @return 0 if equal, < 0 if left is less than right, etc.
949    */
950   public static int compareTo(final byte [] left, final byte [] right) {
951     return LexicographicalComparerHolder.BEST_COMPARER.
952       compareTo(left, 0, left.length, right, 0, right.length);
953   }
954 
955   /**
956    * Lexicographically compare two arrays.
957    *
958    * @param buffer1 left operand
959    * @param buffer2 right operand
960    * @param offset1 Where to start comparing in the left buffer
961    * @param offset2 Where to start comparing in the right buffer
962    * @param length1 How much to compare from the left buffer
963    * @param length2 How much to compare from the right buffer
964    * @return 0 if equal, < 0 if left is less than right, etc.
965    */
966   public static int compareTo(byte[] buffer1, int offset1, int length1,
967       byte[] buffer2, int offset2, int length2) {
968     return LexicographicalComparerHolder.BEST_COMPARER.
969       compareTo(buffer1, offset1, length1, buffer2, offset2, length2);
970   }
971 
972   interface Comparer<T> {
973     abstract public int compareTo(T buffer1, int offset1, int length1,
974         T buffer2, int offset2, int length2);
975   }
976 
977   @VisibleForTesting
978   static Comparer<byte[]> lexicographicalComparerJavaImpl() {
979     return LexicographicalComparerHolder.PureJavaComparer.INSTANCE;
980   }
981 
982   /**
983    * Provides a lexicographical comparer implementation; either a Java
984    * implementation or a faster implementation based on {@link Unsafe}.
985    *
986    * <p>Uses reflection to gracefully fall back to the Java implementation if
987    * {@code Unsafe} isn't available.
988    */
989   @VisibleForTesting
990   static class LexicographicalComparerHolder {
991     static final String UNSAFE_COMPARER_NAME =
992         LexicographicalComparerHolder.class.getName() + "$UnsafeComparer";
993 
994     static final Comparer<byte[]> BEST_COMPARER = getBestComparer();
995     /**
996      * Returns the Unsafe-using Comparer, or falls back to the pure-Java
997      * implementation if unable to do so.
998      */
999     static Comparer<byte[]> getBestComparer() {
1000       try {
1001         Class<?> theClass = Class.forName(UNSAFE_COMPARER_NAME);
1002 
1003         // yes, UnsafeComparer does implement Comparer<byte[]>
1004         @SuppressWarnings("unchecked")
1005         Comparer<byte[]> comparer =
1006           (Comparer<byte[]>) theClass.getEnumConstants()[0];
1007         return comparer;
1008       } catch (Throwable t) { // ensure we really catch *everything*
1009         return lexicographicalComparerJavaImpl();
1010       }
1011     }
1012 
1013     enum PureJavaComparer implements Comparer<byte[]> {
1014       INSTANCE;
1015 
1016       @Override
1017       public int compareTo(byte[] buffer1, int offset1, int length1,
1018           byte[] buffer2, int offset2, int length2) {
1019         // Short circuit equal case
1020         if (buffer1 == buffer2 &&
1021             offset1 == offset2 &&
1022             length1 == length2) {
1023           return 0;
1024         }
1025         // Bring WritableComparator code local
1026         int end1 = offset1 + length1;
1027         int end2 = offset2 + length2;
1028         for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) {
1029           int a = (buffer1[i] & 0xff);
1030           int b = (buffer2[j] & 0xff);
1031           if (a != b) {
1032             return a - b;
1033           }
1034         }
1035         return length1 - length2;
1036       }
1037     }
1038 
1039     @VisibleForTesting
1040     enum UnsafeComparer implements Comparer<byte[]> {
1041       INSTANCE;
1042 
1043       static final Unsafe theUnsafe;
1044 
1045       /** The offset to the first element in a byte array. */
1046       static final int BYTE_ARRAY_BASE_OFFSET;
1047 
1048       static {
1049         theUnsafe = (Unsafe) AccessController.doPrivileged(
1050             new PrivilegedAction<Object>() {
1051               @Override
1052               public Object run() {
1053                 try {
1054                   Field f = Unsafe.class.getDeclaredField("theUnsafe");
1055                   f.setAccessible(true);
1056                   return f.get(null);
1057                 } catch (NoSuchFieldException e) {
1058                   // It doesn't matter what we throw;
1059                   // it's swallowed in getBestComparer().
1060                   throw new Error();
1061                 } catch (IllegalAccessException e) {
1062                   throw new Error();
1063                 }
1064               }
1065             });
1066 
1067         BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class);
1068 
1069         // sanity check - this should never fail
1070         if (theUnsafe.arrayIndexScale(byte[].class) != 1) {
1071           throw new AssertionError();
1072         }
1073       }
1074 
1075       static final boolean littleEndian =
1076         ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN);
1077 
1078       /**
1079        * Returns true if x1 is less than x2, when both values are treated as
1080        * unsigned.
1081        */
1082       static boolean lessThanUnsigned(long x1, long x2) {
1083         return (x1 + Long.MIN_VALUE) < (x2 + Long.MIN_VALUE);
1084       }
1085 
1086       /**
1087        * Lexicographically compare two arrays.
1088        *
1089        * @param buffer1 left operand
1090        * @param buffer2 right operand
1091        * @param offset1 Where to start comparing in the left buffer
1092        * @param offset2 Where to start comparing in the right buffer
1093        * @param length1 How much to compare from the left buffer
1094        * @param length2 How much to compare from the right buffer
1095        * @return 0 if equal, < 0 if left is less than right, etc.
1096        */
1097       @Override
1098       public int compareTo(byte[] buffer1, int offset1, int length1,
1099           byte[] buffer2, int offset2, int length2) {
1100         // Short circuit equal case
1101         if (buffer1 == buffer2 &&
1102             offset1 == offset2 &&
1103             length1 == length2) {
1104           return 0;
1105         }
1106         int minLength = Math.min(length1, length2);
1107         int minWords = minLength / SIZEOF_LONG;
1108         int offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET;
1109         int offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET;
1110 
1111         /*
1112          * Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a
1113          * time is no slower than comparing 4 bytes at a time even on 32-bit.
1114          * On the other hand, it is substantially faster on 64-bit.
1115          */
1116         for (int i = 0; i < minWords * SIZEOF_LONG; i += SIZEOF_LONG) {
1117           long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i);
1118           long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i);
1119           long diff = lw ^ rw;
1120 
1121           if (diff != 0) {
1122             if (!littleEndian) {
1123               return lessThanUnsigned(lw, rw) ? -1 : 1;
1124             }
1125 
1126             // Use binary search
1127             int n = 0;
1128             int y;
1129             int x = (int) diff;
1130             if (x == 0) {
1131               x = (int) (diff >>> 32);
1132               n = 32;
1133             }
1134 
1135             y = x << 16;
1136             if (y == 0) {
1137               n += 16;
1138             } else {
1139               x = y;
1140             }
1141 
1142             y = x << 8;
1143             if (y == 0) {
1144               n += 8;
1145             }
1146             return (int) (((lw >>> n) & 0xFFL) - ((rw >>> n) & 0xFFL));
1147           }
1148         }
1149 
1150         // The epilogue to cover the last (minLength % 8) elements.
1151         for (int i = minWords * SIZEOF_LONG; i < minLength; i++) {
1152           int a = (buffer1[offset1 + i] & 0xff);
1153           int b = (buffer2[offset2 + i] & 0xff);
1154           if (a != b) {
1155             return a - b;
1156           }
1157         }
1158         return length1 - length2;
1159       }
1160     }
1161   }
1162 
1163   /**
1164    * @param left left operand
1165    * @param right right operand
1166    * @return True if equal
1167    */
1168   public static boolean equals(final byte [] left, final byte [] right) {
1169     // Could use Arrays.equals?
1170     //noinspection SimplifiableConditionalExpression
1171     if (left == right) return true;
1172     if (left == null || right == null) return false;
1173     if (left.length != right.length) return false;
1174     if (left.length == 0) return true;
1175 
1176     // Since we're often comparing adjacent sorted data,
1177     // it's usual to have equal arrays except for the very last byte
1178     // so check that first
1179     if (left[left.length - 1] != right[right.length - 1]) return false;
1180 
1181     return compareTo(left, right) == 0;
1182   }
1183 
1184   public static boolean equals(final byte[] left, int leftOffset, int leftLen,
1185                                final byte[] right, int rightOffset, int rightLen) {
1186     // short circuit case
1187     if (left == right &&
1188         leftOffset == rightOffset &&
1189         leftLen == rightLen) {
1190       return true;
1191     }
1192     // different lengths fast check
1193     if (leftLen != rightLen) {
1194       return false;
1195     }
1196     if (leftLen == 0) {
1197       return true;
1198     }
1199 
1200     // Since we're often comparing adjacent sorted data,
1201     // it's usual to have equal arrays except for the very last byte
1202     // so check that first
1203     if (left[leftOffset + leftLen - 1] != right[rightOffset + rightLen - 1]) return false;
1204 
1205     return LexicographicalComparerHolder.BEST_COMPARER.
1206       compareTo(left, leftOffset, leftLen, right, rightOffset, rightLen) == 0;
1207   }
1208 
1209 
1210   /**
1211    * Return true if the byte array on the right is a prefix of the byte
1212    * array on the left.
1213    */
1214   public static boolean startsWith(byte[] bytes, byte[] prefix) {
1215     return bytes != null && prefix != null &&
1216       bytes.length >= prefix.length &&
1217       LexicographicalComparerHolder.BEST_COMPARER.
1218         compareTo(bytes, 0, prefix.length, prefix, 0, prefix.length) == 0;
1219   }
1220 
1221   /**
1222    * @param b bytes to hash
1223    * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the
1224    * passed in array.  This method is what {@link org.apache.hadoop.io.Text} and
1225    * {@link ImmutableBytesWritable} use calculating hash code.
1226    */
1227   public static int hashCode(final byte [] b) {
1228     return hashCode(b, b.length);
1229   }
1230 
1231   /**
1232    * @param b value
1233    * @param length length of the value
1234    * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the
1235    * passed in array.  This method is what {@link org.apache.hadoop.io.Text} and
1236    * {@link ImmutableBytesWritable} use calculating hash code.
1237    */
1238   public static int hashCode(final byte [] b, final int length) {
1239     return WritableComparator.hashBytes(b, length);
1240   }
1241 
1242   /**
1243    * @param b bytes to hash
1244    * @return A hash of <code>b</code> as an Integer that can be used as key in
1245    * Maps.
1246    */
1247   public static Integer mapKey(final byte [] b) {
1248     return hashCode(b);
1249   }
1250 
1251   /**
1252    * @param b bytes to hash
1253    * @param length length to hash
1254    * @return A hash of <code>b</code> as an Integer that can be used as key in
1255    * Maps.
1256    */
1257   public static Integer mapKey(final byte [] b, final int length) {
1258     return hashCode(b, length);
1259   }
1260 
1261   /**
1262    * @param a lower half
1263    * @param b upper half
1264    * @return New array that has a in lower half and b in upper half.
1265    */
1266   public static byte [] add(final byte [] a, final byte [] b) {
1267     return add(a, b, HConstants.EMPTY_BYTE_ARRAY);
1268   }
1269 
1270   /**
1271    * @param a first third
1272    * @param b second third
1273    * @param c third third
1274    * @return New array made from a, b and c
1275    */
1276   public static byte [] add(final byte [] a, final byte [] b, final byte [] c) {
1277     byte [] result = new byte[a.length + b.length + c.length];
1278     System.arraycopy(a, 0, result, 0, a.length);
1279     System.arraycopy(b, 0, result, a.length, b.length);
1280     System.arraycopy(c, 0, result, a.length + b.length, c.length);
1281     return result;
1282   }
1283 
1284   /**
1285    * @param a array
1286    * @param length amount of bytes to grab
1287    * @return First <code>length</code> bytes from <code>a</code>
1288    */
1289   public static byte [] head(final byte [] a, final int length) {
1290     if (a.length < length) {
1291       return null;
1292     }
1293     byte [] result = new byte[length];
1294     System.arraycopy(a, 0, result, 0, length);
1295     return result;
1296   }
1297 
1298   /**
1299    * @param a array
1300    * @param length amount of bytes to snarf
1301    * @return Last <code>length</code> bytes from <code>a</code>
1302    */
1303   public static byte [] tail(final byte [] a, final int length) {
1304     if (a.length < length) {
1305       return null;
1306     }
1307     byte [] result = new byte[length];
1308     System.arraycopy(a, a.length - length, result, 0, length);
1309     return result;
1310   }
1311 
1312   /**
1313    * @param a array
1314    * @param length new array size
1315    * @return Value in <code>a</code> plus <code>length</code> prepended 0 bytes
1316    */
1317   public static byte [] padHead(final byte [] a, final int length) {
1318     byte [] padding = new byte[length];
1319     for (int i = 0; i < length; i++) {
1320       padding[i] = 0;
1321     }
1322     return add(padding,a);
1323   }
1324 
1325   /**
1326    * @param a array
1327    * @param length new array size
1328    * @return Value in <code>a</code> plus <code>length</code> appended 0 bytes
1329    */
1330   public static byte [] padTail(final byte [] a, final int length) {
1331     byte [] padding = new byte[length];
1332     for (int i = 0; i < length; i++) {
1333       padding[i] = 0;
1334     }
1335     return add(a,padding);
1336   }
1337 
1338   /**
1339    * Split passed range.  Expensive operation relatively.  Uses BigInteger math.
1340    * Useful splitting ranges for MapReduce jobs.
1341    * @param a Beginning of range
1342    * @param b End of range
1343    * @param num Number of times to split range.  Pass 1 if you want to split
1344    * the range in two; i.e. one split.
1345    * @return Array of dividing values
1346    */
1347   public static byte [][] split(final byte [] a, final byte [] b, final int num) {
1348     return split(a, b, false, num);
1349   }
1350 
1351   /**
1352    * Split passed range.  Expensive operation relatively.  Uses BigInteger math.
1353    * Useful splitting ranges for MapReduce jobs.
1354    * @param a Beginning of range
1355    * @param b End of range
1356    * @param inclusive Whether the end of range is prefix-inclusive or is
1357    * considered an exclusive boundary.  Automatic splits are generally exclusive
1358    * and manual splits with an explicit range utilize an inclusive end of range.
1359    * @param num Number of times to split range.  Pass 1 if you want to split
1360    * the range in two; i.e. one split.
1361    * @return Array of dividing values
1362    */
1363   public static byte[][] split(final byte[] a, final byte[] b,
1364       boolean inclusive, final int num) {
1365     byte[][] ret = new byte[num + 2][];
1366     int i = 0;
1367     Iterable<byte[]> iter = iterateOnSplits(a, b, inclusive, num);
1368     if (iter == null)
1369       return null;
1370     for (byte[] elem : iter) {
1371       ret[i++] = elem;
1372     }
1373     return ret;
1374   }
1375 
1376   /**
1377    * Iterate over keys within the passed range, splitting at an [a,b) boundary.
1378    */
1379   public static Iterable<byte[]> iterateOnSplits(final byte[] a,
1380       final byte[] b, final int num)
1381   {
1382     return iterateOnSplits(a, b, false, num);
1383   }
1384 
1385   /**
1386    * Iterate over keys within the passed range.
1387    */
1388   public static Iterable<byte[]> iterateOnSplits(
1389       final byte[] a, final byte[]b, boolean inclusive, final int num)
1390   {
1391     byte [] aPadded;
1392     byte [] bPadded;
1393     if (a.length < b.length) {
1394       aPadded = padTail(a, b.length - a.length);
1395       bPadded = b;
1396     } else if (b.length < a.length) {
1397       aPadded = a;
1398       bPadded = padTail(b, a.length - b.length);
1399     } else {
1400       aPadded = a;
1401       bPadded = b;
1402     }
1403     if (compareTo(aPadded,bPadded) >= 0) {
1404       throw new IllegalArgumentException("b <= a");
1405     }
1406     if (num <= 0) {
1407       throw new IllegalArgumentException("num cannot be < 0");
1408     }
1409     byte [] prependHeader = {1, 0};
1410     final BigInteger startBI = new BigInteger(add(prependHeader, aPadded));
1411     final BigInteger stopBI = new BigInteger(add(prependHeader, bPadded));
1412     BigInteger diffBI = stopBI.subtract(startBI);
1413     if (inclusive) {
1414       diffBI = diffBI.add(BigInteger.ONE);
1415     }
1416     final BigInteger splitsBI = BigInteger.valueOf(num + 1);
1417     if(diffBI.compareTo(splitsBI) < 0) {
1418       return null;
1419     }
1420     final BigInteger intervalBI;
1421     try {
1422       intervalBI = diffBI.divide(splitsBI);
1423     } catch(Exception e) {
1424       LOG.error("Exception caught during division", e);
1425       return null;
1426     }
1427 
1428     final Iterator<byte[]> iterator = new Iterator<byte[]>() {
1429       private int i = -1;
1430 
1431       @Override
1432       public boolean hasNext() {
1433         return i < num+1;
1434       }
1435 
1436       @Override
1437       public byte[] next() {
1438         i++;
1439         if (i == 0) return a;
1440         if (i == num + 1) return b;
1441 
1442         BigInteger curBI = startBI.add(intervalBI.multiply(BigInteger.valueOf(i)));
1443         byte [] padded = curBI.toByteArray();
1444         if (padded[1] == 0)
1445           padded = tail(padded, padded.length - 2);
1446         else
1447           padded = tail(padded, padded.length - 1);
1448         return padded;
1449       }
1450 
1451       @Override
1452       public void remove() {
1453         throw new UnsupportedOperationException();
1454       }
1455 
1456     };
1457 
1458     return new Iterable<byte[]>() {
1459       @Override
1460       public Iterator<byte[]> iterator() {
1461         return iterator;
1462       }
1463     };
1464   }
1465 
1466   /**
1467    * @param bytes array to hash
1468    * @param offset offset to start from
1469    * @param length length to hash
1470    * */
1471   public static int hashCode(byte[] bytes, int offset, int length) {
1472     int hash = 1;
1473     for (int i = offset; i < offset + length; i++)
1474       hash = (31 * hash) + (int) bytes[i];
1475     return hash;
1476   }
1477 
1478   /**
1479    * @param t operands
1480    * @return Array of byte arrays made from passed array of Text
1481    */
1482   public static byte [][] toByteArrays(final String [] t) {
1483     byte [][] result = new byte[t.length][];
1484     for (int i = 0; i < t.length; i++) {
1485       result[i] = Bytes.toBytes(t[i]);
1486     }
1487     return result;
1488   }
1489 
1490   /**
1491    * @param column operand
1492    * @return A byte array of a byte array where first and only entry is
1493    * <code>column</code>
1494    */
1495   public static byte [][] toByteArrays(final String column) {
1496     return toByteArrays(toBytes(column));
1497   }
1498 
1499   /**
1500    * @param column operand
1501    * @return A byte array of a byte array where first and only entry is
1502    * <code>column</code>
1503    */
1504   public static byte [][] toByteArrays(final byte [] column) {
1505     byte [][] result = new byte[1][];
1506     result[0] = column;
1507     return result;
1508   }
1509 
1510   /**
1511    * Binary search for keys in indexes.
1512    *
1513    * @param arr array of byte arrays to search for
1514    * @param key the key you want to find
1515    * @param offset the offset in the key you want to find
1516    * @param length the length of the key
1517    * @param comparator a comparator to compare.
1518    * @return zero-based index of the key, if the key is present in the array.
1519    *         Otherwise, a value -(i + 1) such that the key is between arr[i -
1520    *         1] and arr[i] non-inclusively, where i is in [0, i], if we define
1521    *         arr[-1] = -Inf and arr[N] = Inf for an N-element array. The above
1522    *         means that this function can return 2N + 1 different values
1523    *         ranging from -(N + 1) to N - 1.
1524    */
1525   public static int binarySearch(byte [][]arr, byte []key, int offset,
1526       int length, RawComparator<byte []> comparator) {
1527     int low = 0;
1528     int high = arr.length - 1;
1529 
1530     while (low <= high) {
1531       int mid = (low+high) >>> 1;
1532       // we have to compare in this order, because the comparator order
1533       // has special logic when the 'left side' is a special key.
1534       int cmp = comparator.compare(key, offset, length,
1535           arr[mid], 0, arr[mid].length);
1536       // key lives above the midpoint
1537       if (cmp > 0)
1538         low = mid + 1;
1539       // key lives below the midpoint
1540       else if (cmp < 0)
1541         high = mid - 1;
1542       // BAM. how often does this really happen?
1543       else
1544         return mid;
1545     }
1546     return - (low+1);
1547   }
1548 
1549   /**
1550    * Bytewise binary increment/deincrement of long contained in byte array
1551    * on given amount.
1552    *
1553    * @param value - array of bytes containing long (length <= SIZEOF_LONG)
1554    * @param amount value will be incremented on (deincremented if negative)
1555    * @return array of bytes containing incremented long (length == SIZEOF_LONG)
1556    */
1557   public static byte [] incrementBytes(byte[] value, long amount)
1558   {
1559     byte[] val = value;
1560     if (val.length < SIZEOF_LONG) {
1561       // Hopefully this doesn't happen too often.
1562       byte [] newvalue;
1563       if (val[0] < 0) {
1564         newvalue = new byte[]{-1, -1, -1, -1, -1, -1, -1, -1};
1565       } else {
1566         newvalue = new byte[SIZEOF_LONG];
1567       }
1568       System.arraycopy(val, 0, newvalue, newvalue.length - val.length,
1569         val.length);
1570       val = newvalue;
1571     } else if (val.length > SIZEOF_LONG) {
1572       throw new IllegalArgumentException("Increment Bytes - value too big: " +
1573         val.length);
1574     }
1575     if(amount == 0) return val;
1576     if(val[0] < 0){
1577       return binaryIncrementNeg(val, amount);
1578     }
1579     return binaryIncrementPos(val, amount);
1580   }
1581 
1582   /* increment/deincrement for positive value */
1583   private static byte [] binaryIncrementPos(byte [] value, long amount) {
1584     long amo = amount;
1585     int sign = 1;
1586     if (amount < 0) {
1587       amo = -amount;
1588       sign = -1;
1589     }
1590     for(int i=0;i<value.length;i++) {
1591       int cur = ((int)amo % 256) * sign;
1592       amo = (amo >> 8);
1593       int val = value[value.length-i-1] & 0x0ff;
1594       int total = val + cur;
1595       if(total > 255) {
1596         amo += sign;
1597         total %= 256;
1598       } else if (total < 0) {
1599         amo -= sign;
1600       }
1601       value[value.length-i-1] = (byte)total;
1602       if (amo == 0) return value;
1603     }
1604     return value;
1605   }
1606 
1607   /* increment/deincrement for negative value */
1608   private static byte [] binaryIncrementNeg(byte [] value, long amount) {
1609     long amo = amount;
1610     int sign = 1;
1611     if (amount < 0) {
1612       amo = -amount;
1613       sign = -1;
1614     }
1615     for(int i=0;i<value.length;i++) {
1616       int cur = ((int)amo % 256) * sign;
1617       amo = (amo >> 8);
1618       int val = ((~value[value.length-i-1]) & 0x0ff) + 1;
1619       int total = cur - val;
1620       if(total >= 0) {
1621         amo += sign;
1622       } else if (total < -256) {
1623         amo -= sign;
1624         total %= 256;
1625       }
1626       value[value.length-i-1] = (byte)total;
1627       if (amo == 0) return value;
1628     }
1629     return value;
1630   }
1631 
1632   /**
1633    * Writes a string as a fixed-size field, padded with zeros.
1634    */
1635   public static void writeStringFixedSize(final DataOutput out, String s,
1636       int size) throws IOException {
1637     byte[] b = toBytes(s);
1638     if (b.length > size) {
1639       throw new IOException("Trying to write " + b.length + " bytes (" +
1640           toStringBinary(b) + ") into a field of length " + size);
1641     }
1642 
1643     out.writeBytes(s);
1644     for (int i = 0; i < size - s.length(); ++i)
1645       out.writeByte(0);
1646   }
1647 
1648   /**
1649    * Reads a fixed-size field and interprets it as a string padded with zeros.
1650    */
1651   public static String readStringFixedSize(final DataInput in, int size)
1652       throws IOException {
1653     byte[] b = new byte[size];
1654     in.readFully(b);
1655     int n = b.length;
1656     while (n > 0 && b[n - 1] == 0)
1657       --n;
1658 
1659     return toString(b, 0, n);
1660   }
1661 
1662   /**
1663    * Copy the byte array given in parameter and return an instance 
1664    * of a new byte array with the same length and the same content.
1665    * @param bytes the byte array to duplicate
1666    * @return a copy of the given byte array 
1667    */
1668   public static byte [] copy(byte [] bytes) {
1669     if (bytes == null) return null;
1670     byte [] result = new byte[bytes.length];
1671     System.arraycopy(bytes, 0, result, 0, bytes.length);	  
1672     return result;
1673   }
1674 
1675   /**
1676    * Copy the byte array given in parameter and return an instance 
1677    * of a new byte array with the same length and the same content.
1678    * @param bytes the byte array to copy from
1679    * @return a copy of the given designated byte array 
1680    * @param offset
1681    * @param length
1682    */
1683   public static byte [] copy(byte [] bytes, final int offset, final int length) {
1684     if (bytes == null) return null;
1685     byte [] result = new byte[length];
1686     System.arraycopy(bytes, offset, result, 0, length);
1687     return result;
1688   }
1689 
1690   /**
1691    * Search sorted array "a" for byte "key". I can't remember if I wrote this or copied it from
1692    * somewhere. (mcorgan)
1693    * @param a Array to search. Entries must be sorted and unique.
1694    * @param fromIndex First index inclusive of "a" to include in the search.
1695    * @param toIndex Last index exclusive of "a" to include in the search.
1696    * @param key The byte to search for.
1697    * @return The index of key if found. If not found, return -(index + 1), where negative indicates
1698    *         "not found" and the "index + 1" handles the "-0" case.
1699    */
1700   public static int unsignedBinarySearch(byte[] a, int fromIndex, int toIndex, byte key) {
1701     int unsignedKey = key & 0xff;
1702     int low = fromIndex;
1703     int high = toIndex - 1;
1704 
1705     while (low <= high) {
1706       int mid = (low + high) >>> 1;
1707       int midVal = a[mid] & 0xff;
1708 
1709       if (midVal < unsignedKey) {
1710         low = mid + 1;
1711       } else if (midVal > unsignedKey) {
1712         high = mid - 1;
1713       } else {
1714         return mid; // key found
1715       }
1716     }
1717     return -(low + 1); // key not found.
1718   }
1719 
1720   /**
1721    * Treat the byte[] as an unsigned series of bytes, most significant bits first.  Start by adding
1722    * 1 to the rightmost bit/byte and carry over all overflows to the more significant bits/bytes.
1723    *
1724    * @param input The byte[] to increment.
1725    * @return The incremented copy of "in".  May be same length or 1 byte longer.
1726    */
1727   public static byte[] unsignedCopyAndIncrement(final byte[] input) {
1728     byte[] copy = copy(input);
1729     if (copy == null) {
1730       throw new IllegalArgumentException("cannot increment null array");
1731     }
1732     for (int i = copy.length - 1; i >= 0; --i) {
1733       if (copy[i] == -1) {// -1 is all 1-bits, which is the unsigned maximum
1734         copy[i] = 0;
1735       } else {
1736         ++copy[i];
1737         return copy;
1738       }
1739     }
1740     // we maxed out the array
1741     byte[] out = new byte[copy.length + 1];
1742     out[0] = 1;
1743     System.arraycopy(copy, 0, out, 1, copy.length);
1744     return out;
1745   }
1746 
1747   public static boolean equals(List<byte[]> a, List<byte[]> b) {
1748     if (a == null) {
1749       if (b == null) {
1750         return true;
1751       }
1752       return false;
1753     }
1754     if (b == null) {
1755       return false;
1756     }
1757     if (a.size() != b.size()) {
1758       return false;
1759     }
1760     for (int i = 0; i < a.size(); ++i) {
1761       if (!Bytes.equals(a.get(i), b.get(i))) {
1762         return false;
1763       }
1764     }
1765     return true;
1766   }
1767 
1768   public static boolean isSorted(Collection<byte[]> arrays) {
1769     byte[] previous = new byte[0];
1770     for (byte[] array : IterableUtils.nullSafe(arrays)) {
1771       if (Bytes.compareTo(previous, array) > 0) {
1772         return false;
1773       }
1774       previous = array;
1775     }
1776     return true;
1777   }
1778 
1779   public static List<byte[]> getUtf8ByteArrays(List<String> strings) {
1780     List<byte[]> byteArrays = Lists.newArrayListWithCapacity(CollectionUtils.nullSafeSize(strings));
1781     for (String s : IterableUtils.nullSafe(strings)) {
1782       byteArrays.add(Bytes.toBytes(s));
1783     }
1784     return byteArrays;
1785   }
1786 
1787   /**
1788    * Returns the index of the first appearance of the value {@code target} in
1789    * {@code array}.
1790    *
1791    * @param array an array of {@code byte} values, possibly empty
1792    * @param target a primitive {@code byte} value
1793    * @return the least index {@code i} for which {@code array[i] == target}, or
1794    *     {@code -1} if no such index exists.
1795    */
1796   public static int indexOf(byte[] array, byte target) {
1797     for (int i = 0; i < array.length; i++) {
1798       if (array[i] == target) {
1799         return i;
1800       }
1801     }
1802     return -1;
1803   }
1804   
1805   /**
1806    * Returns the start position of the first occurrence of the specified {@code
1807    * target} within {@code array}, or {@code -1} if there is no such occurrence.
1808    *
1809    * <p>More formally, returns the lowest index {@code i} such that {@code
1810    * java.util.Arrays.copyOfRange(array, i, i + target.length)} contains exactly
1811    * the same elements as {@code target}.
1812    *
1813    * @param array the array to search for the sequence {@code target}
1814    * @param target the array to search for as a sub-sequence of {@code array}
1815    */
1816   public static int indexOf(byte[] array, byte[] target) {
1817     checkNotNull(array, "array");
1818     checkNotNull(target, "target");
1819     if (target.length == 0) {
1820       return 0;
1821     }
1822 
1823     outer:
1824     for (int i = 0; i < array.length - target.length + 1; i++) {
1825       for (int j = 0; j < target.length; j++) {
1826         if (array[i + j] != target[j]) {
1827           continue outer;
1828         }
1829       }
1830       return i;
1831     }
1832     return -1;
1833   }
1834   
1835   /**
1836    * @param array an array of {@code byte} values, possibly empty
1837    * @param target a primitive {@code byte} value
1838    * @return {@code true} if {@code target} is present as an element anywhere in {@code array}.
1839    */
1840   public static boolean contains(byte[] array, byte target) {
1841     return indexOf(array, target) > -1;
1842   }
1843   
1844   /**
1845    * @param array an array of {@code byte} values, possibly empty
1846    * @param target an array of {@code byte}
1847    * @return {@code true} if {@code target} is present anywhere in {@code array}
1848    */
1849   public static boolean contains(byte[] array, byte[] target) {
1850     return indexOf(array, target) > -1;
1851   }
1852   
1853   /**
1854    * Fill given array with zeros.
1855    * @param b array which needs to be filled with zeros
1856    */
1857   public static void zero(byte[] b) {
1858     zero(b, 0, b.length);
1859   }
1860 
1861   /**
1862    * Fill given array with zeros at the specified position.
1863    * @param b
1864    * @param offset
1865    * @param length
1866    */
1867   public static void zero(byte[] b, int offset, int length) {
1868     checkPositionIndex(offset, b.length, "offset");
1869     checkArgument(length > 0, "length must be greater than 0");
1870     checkPositionIndex(offset + length, b.length, "offset + length");
1871     Arrays.fill(b, offset, offset + length, (byte) 0);
1872   }
1873 
1874   private static final SecureRandom RNG = new SecureRandom();
1875 
1876   /**
1877    * Fill given array with random bytes.
1878    * @param b array which needs to be filled with random bytes
1879    */
1880   public static void random(byte[] b) {
1881     RNG.nextBytes(b);
1882   }
1883 
1884   /**
1885    * Fill given array with random bytes at the specified position.
1886    * @param b
1887    * @param offset
1888    * @param length
1889    */
1890   public static void random(byte[] b, int offset, int length) {
1891     checkPositionIndex(offset, b.length, "offset");
1892     checkArgument(length > 0, "length must be greater than 0");
1893     checkPositionIndex(offset + length, b.length, "offset + length");
1894     byte[] buf = new byte[length];
1895     RNG.nextBytes(buf);
1896     System.arraycopy(buf, 0, b, offset, length);
1897   }
1898 }