View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import static com.google.common.base.Preconditions.checkArgument;
21  import static com.google.common.base.Preconditions.checkNotNull;
22  import static com.google.common.base.Preconditions.checkPositionIndex;
23  
24  import java.io.DataInput;
25  import java.io.DataOutput;
26  import java.io.IOException;
27  import java.lang.reflect.Field;
28  import java.math.BigDecimal;
29  import java.math.BigInteger;
30  import java.nio.ByteBuffer;
31  import java.nio.ByteOrder;
32  import java.nio.charset.Charset;
33  import java.security.AccessController;
34  import java.security.PrivilegedAction;
35  import java.security.SecureRandom;
36  import java.util.Arrays;
37  import java.util.Collection;
38  import java.util.Comparator;
39  import java.util.Iterator;
40  import java.util.List;
41  
42  import org.apache.commons.logging.Log;
43  import org.apache.commons.logging.LogFactory;
44  import org.apache.hadoop.classification.InterfaceAudience;
45  import org.apache.hadoop.classification.InterfaceStability;
46  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
47  import org.apache.hadoop.io.RawComparator;
48  import org.apache.hadoop.io.WritableComparator;
49  import org.apache.hadoop.io.WritableUtils;
50  
51  import sun.misc.Unsafe;
52  
53  import com.google.common.annotations.VisibleForTesting;
54  import com.google.common.collect.Lists;
55  
56  /**
57   * Utility class that handles byte arrays, conversions to/from other types,
58   * comparisons, hash code generation, manufacturing keys for HashMaps or
59   * HashSets, etc.
60   */
61  @InterfaceAudience.Public
62  @InterfaceStability.Stable
63  public class Bytes {
64    //HConstants.UTF8_ENCODING should be updated if this changed
65    /** When we encode strings, we always specify UTF8 encoding */
66    private static final String UTF8_ENCODING = "UTF-8";
67  
68    //HConstants.UTF8_CHARSET should be updated if this changed
69    /** When we encode strings, we always specify UTF8 encoding */
70    private static final Charset UTF8_CHARSET = Charset.forName(UTF8_ENCODING);
71  
72    //HConstants.EMPTY_BYTE_ARRAY should be updated if this changed
73    private static final byte [] EMPTY_BYTE_ARRAY = new byte [0];
74  
75    private static final Log LOG = LogFactory.getLog(Bytes.class);
76  
77    /**
78     * Size of boolean in bytes
79     */
80    public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE;
81  
82    /**
83     * Size of byte in bytes
84     */
85    public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN;
86  
87    /**
88     * Size of char in bytes
89     */
90    public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE;
91  
92    /**
93     * Size of double in bytes
94     */
95    public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE;
96  
97    /**
98     * Size of float in bytes
99     */
100   public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE;
101 
102   /**
103    * Size of int in bytes
104    */
105   public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE;
106 
107   /**
108    * Size of long in bytes
109    */
110   public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE;
111 
112   /**
113    * Size of short in bytes
114    */
115   public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE;
116 
117 
118   /**
119    * Estimate of size cost to pay beyond payload in jvm for instance of byte [].
120    * Estimate based on study of jhat and jprofiler numbers.
121    */
122   // JHat says BU is 56 bytes.
123   // SizeOf which uses java.lang.instrument says 24 bytes. (3 longs?)
124   public static final int ESTIMATED_HEAP_TAX = 16;
125 
126   /**
127    * Byte array comparator class.
128    */
129   public static class ByteArrayComparator implements RawComparator<byte []> {
130     /**
131      * Constructor
132      */
133     public ByteArrayComparator() {
134       super();
135     }
136     @Override
137     public int compare(byte [] left, byte [] right) {
138       return compareTo(left, right);
139     }
140     @Override
141     public int compare(byte [] b1, int s1, int l1, byte [] b2, int s2, int l2) {
142       return LexicographicalComparerHolder.BEST_COMPARER.
143         compareTo(b1, s1, l1, b2, s2, l2);
144     }
145   }
146 
147   /**
148    * A {@link ByteArrayComparator} that treats the empty array as the largest value.
149    * This is useful for comparing row end keys for regions.
150    */
151   // TODO: unfortunately, HBase uses byte[0] as both start and end keys for region
152   // boundaries. Thus semantically, we should treat empty byte array as the smallest value
153   // while comparing row keys, start keys etc; but as the largest value for comparing
154   // region boundaries for endKeys.
155   public static class RowEndKeyComparator extends ByteArrayComparator {
156     @Override
157     public int compare(byte[] left, byte[] right) {
158       return compare(left, 0, left.length, right, 0, right.length);
159     }
160     @Override
161     public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
162       if (b1 == b2 && s1 == s2 && l1 == l2) {
163         return 0;
164       }
165       if (l1 == 0) {
166         return l2; //0 or positive
167       }
168       if (l2 == 0) {
169         return -1;
170       }
171       return super.compare(b1, s1, l1, b2, s2, l2);
172     }
173   }
174 
175   /**
176    * Pass this to TreeMaps where byte [] are keys.
177    */
178   public static Comparator<byte []> BYTES_COMPARATOR =
179     new ByteArrayComparator();
180 
181   /**
182    * Use comparing byte arrays, byte-by-byte
183    */
184   public static RawComparator<byte []> BYTES_RAWCOMPARATOR =
185     new ByteArrayComparator();
186 
187   /**
188    * Read byte-array written with a WritableableUtils.vint prefix.
189    * @param in Input to read from.
190    * @return byte array read off <code>in</code>
191    * @throws IOException e
192    */
193   public static byte [] readByteArray(final DataInput in)
194   throws IOException {
195     int len = WritableUtils.readVInt(in);
196     if (len < 0) {
197       throw new NegativeArraySizeException(Integer.toString(len));
198     }
199     byte [] result = new byte[len];
200     in.readFully(result, 0, len);
201     return result;
202   }
203 
204   /**
205    * Read byte-array written with a WritableableUtils.vint prefix.
206    * IOException is converted to a RuntimeException.
207    * @param in Input to read from.
208    * @return byte array read off <code>in</code>
209    */
210   public static byte [] readByteArrayThrowsRuntime(final DataInput in) {
211     try {
212       return readByteArray(in);
213     } catch (Exception e) {
214       throw new RuntimeException(e);
215     }
216   }
217 
218   /**
219    * Write byte-array with a WritableableUtils.vint prefix.
220    * @param out output stream to be written to
221    * @param b array to write
222    * @throws IOException e
223    */
224   public static void writeByteArray(final DataOutput out, final byte [] b)
225   throws IOException {
226     if(b == null) {
227       WritableUtils.writeVInt(out, 0);
228     } else {
229       writeByteArray(out, b, 0, b.length);
230     }
231   }
232 
233   /**
234    * Write byte-array to out with a vint length prefix.
235    * @param out output stream
236    * @param b array
237    * @param offset offset into array
238    * @param length length past offset
239    * @throws IOException e
240    */
241   public static void writeByteArray(final DataOutput out, final byte [] b,
242       final int offset, final int length)
243   throws IOException {
244     WritableUtils.writeVInt(out, length);
245     out.write(b, offset, length);
246   }
247 
248   /**
249    * Write byte-array from src to tgt with a vint length prefix.
250    * @param tgt target array
251    * @param tgtOffset offset into target array
252    * @param src source array
253    * @param srcOffset source offset
254    * @param srcLength source length
255    * @return New offset in src array.
256    */
257   public static int writeByteArray(final byte [] tgt, final int tgtOffset,
258       final byte [] src, final int srcOffset, final int srcLength) {
259     byte [] vint = vintToBytes(srcLength);
260     System.arraycopy(vint, 0, tgt, tgtOffset, vint.length);
261     int offset = tgtOffset + vint.length;
262     System.arraycopy(src, srcOffset, tgt, offset, srcLength);
263     return offset + srcLength;
264   }
265 
266   /**
267    * Put bytes at the specified byte array position.
268    * @param tgtBytes the byte array
269    * @param tgtOffset position in the array
270    * @param srcBytes array to write out
271    * @param srcOffset source offset
272    * @param srcLength source length
273    * @return incremented offset
274    */
275   public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes,
276       int srcOffset, int srcLength) {
277     System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength);
278     return tgtOffset + srcLength;
279   }
280 
281   /**
282    * Write a single byte out to the specified byte array position.
283    * @param bytes the byte array
284    * @param offset position in the array
285    * @param b byte to write out
286    * @return incremented offset
287    */
288   public static int putByte(byte[] bytes, int offset, byte b) {
289     bytes[offset] = b;
290     return offset + 1;
291   }
292 
293   /**
294    * Returns a new byte array, copied from the given {@code buf},
295    * from the index 0 (inclusive) to the limit (exclusive),
296    * regardless of the current position.
297    * The position and the other index parameters are not changed.
298    *
299    * @param buf a byte buffer
300    * @return the byte array
301    * @see #getBytes(ByteBuffer)
302    */
303   public static byte[] toBytes(ByteBuffer buf) {
304     ByteBuffer dup = buf.duplicate();
305     dup.position(0);
306     return readBytes(dup);
307   }
308 
309   private static byte[] readBytes(ByteBuffer buf) {
310     byte [] result = new byte[buf.remaining()];
311     buf.get(result);
312     return result;
313   }
314 
315   /**
316    * @param b Presumed UTF-8 encoded byte array.
317    * @return String made from <code>b</code>
318    */
319   public static String toString(final byte [] b) {
320     if (b == null) {
321       return null;
322     }
323     return toString(b, 0, b.length);
324   }
325 
326   /**
327    * Joins two byte arrays together using a separator.
328    * @param b1 The first byte array.
329    * @param sep The separator to use.
330    * @param b2 The second byte array.
331    */
332   public static String toString(final byte [] b1,
333                                 String sep,
334                                 final byte [] b2) {
335     return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length);
336   }
337 
338   /**
339    * This method will convert utf8 encoded bytes into a string. If
340    * the given byte array is null, this method will return null.
341    *
342    * @param b Presumed UTF-8 encoded byte array.
343    * @param off offset into array
344    * @param len length of utf-8 sequence
345    * @return String made from <code>b</code> or null
346    */
347   public static String toString(final byte [] b, int off, int len) {
348     if (b == null) {
349       return null;
350     }
351     if (len == 0) {
352       return "";
353     }
354     return new String(b, off, len, UTF8_CHARSET);
355   }
356 
357   /**
358    * Write a printable representation of a byte array.
359    *
360    * @param b byte array
361    * @return string
362    * @see #toStringBinary(byte[], int, int)
363    */
364   public static String toStringBinary(final byte [] b) {
365     if (b == null)
366       return "null";
367     return toStringBinary(b, 0, b.length);
368   }
369 
370   /**
371    * Converts the given byte buffer to a printable representation,
372    * from the index 0 (inclusive) to the limit (exclusive),
373    * regardless of the current position.
374    * The position and the other index parameters are not changed.
375    *
376    * @param buf a byte buffer
377    * @return a string representation of the buffer's binary contents
378    * @see #toBytes(ByteBuffer)
379    * @see #getBytes(ByteBuffer)
380    */
381   public static String toStringBinary(ByteBuffer buf) {
382     if (buf == null)
383       return "null";
384     if (buf.hasArray()) {
385       return toStringBinary(buf.array(), buf.arrayOffset(), buf.limit());
386     }
387     return toStringBinary(toBytes(buf));
388   }
389 
390   /**
391    * Write a printable representation of a byte array. Non-printable
392    * characters are hex escaped in the format \\x%02X, eg:
393    * \x00 \x05 etc
394    *
395    * @param b array to write out
396    * @param off offset to start at
397    * @param len length to write
398    * @return string output
399    */
400   public static String toStringBinary(final byte [] b, int off, int len) {
401     StringBuilder result = new StringBuilder();
402     // Just in case we are passed a 'len' that is > buffer length...
403     if (off >= b.length) return result.toString();
404     if (off + len > b.length) len = b.length - off;
405     for (int i = off; i < off + len ; ++i ) {
406       int ch = b[i] & 0xFF;
407       if ( (ch >= '0' && ch <= '9')
408           || (ch >= 'A' && ch <= 'Z')
409           || (ch >= 'a' && ch <= 'z')
410           || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0 ) {
411         result.append((char)ch);
412       } else {
413         result.append(String.format("\\x%02X", ch));
414       }
415     }
416     return result.toString();
417   }
418 
419   private static boolean isHexDigit(char c) {
420     return
421         (c >= 'A' && c <= 'F') ||
422         (c >= '0' && c <= '9');
423   }
424 
425   /**
426    * Takes a ASCII digit in the range A-F0-9 and returns
427    * the corresponding integer/ordinal value.
428    * @param ch  The hex digit.
429    * @return The converted hex value as a byte.
430    */
431   public static byte toBinaryFromHex(byte ch) {
432     if ( ch >= 'A' && ch <= 'F' )
433       return (byte) ((byte)10 + (byte) (ch - 'A'));
434     // else
435     return (byte) (ch - '0');
436   }
437 
438   public static byte [] toBytesBinary(String in) {
439     // this may be bigger than we need, but let's be safe.
440     byte [] b = new byte[in.length()];
441     int size = 0;
442     for (int i = 0; i < in.length(); ++i) {
443       char ch = in.charAt(i);
444       if (ch == '\\' && in.length() > i+1 && in.charAt(i+1) == 'x') {
445         // ok, take next 2 hex digits.
446         char hd1 = in.charAt(i+2);
447         char hd2 = in.charAt(i+3);
448 
449         // they need to be A-F0-9:
450         if (!isHexDigit(hd1) ||
451             !isHexDigit(hd2)) {
452           // bogus escape code, ignore:
453           continue;
454         }
455         // turn hex ASCII digit -> number
456         byte d = (byte) ((toBinaryFromHex((byte)hd1) << 4) + toBinaryFromHex((byte)hd2));
457 
458         b[size++] = d;
459         i += 3; // skip 3
460       } else {
461         b[size++] = (byte) ch;
462       }
463     }
464     // resize:
465     byte [] b2 = new byte[size];
466     System.arraycopy(b, 0, b2, 0, size);
467     return b2;
468   }
469 
470   /**
471    * Converts a string to a UTF-8 byte array.
472    * @param s string
473    * @return the byte array
474    */
475   public static byte[] toBytes(String s) {
476     return s.getBytes(UTF8_CHARSET);
477   }
478 
479   /**
480    * Convert a boolean to a byte array. True becomes -1
481    * and false becomes 0.
482    *
483    * @param b value
484    * @return <code>b</code> encoded in a byte array.
485    */
486   public static byte [] toBytes(final boolean b) {
487     return new byte[] { b ? (byte) -1 : (byte) 0 };
488   }
489 
490   /**
491    * Reverses {@link #toBytes(boolean)}
492    * @param b array
493    * @return True or false.
494    */
495   public static boolean toBoolean(final byte [] b) {
496     if (b.length != 1) {
497       throw new IllegalArgumentException("Array has wrong size: " + b.length);
498     }
499     return b[0] != (byte) 0;
500   }
501 
502   /**
503    * Convert a long value to a byte array using big-endian.
504    *
505    * @param val value to convert
506    * @return the byte array
507    */
508   public static byte[] toBytes(long val) {
509     byte [] b = new byte[8];
510     for (int i = 7; i > 0; i--) {
511       b[i] = (byte) val;
512       val >>>= 8;
513     }
514     b[0] = (byte) val;
515     return b;
516   }
517 
518   /**
519    * Converts a byte array to a long value. Reverses
520    * {@link #toBytes(long)}
521    * @param bytes array
522    * @return the long value
523    */
524   public static long toLong(byte[] bytes) {
525     return toLong(bytes, 0, SIZEOF_LONG);
526   }
527 
528   /**
529    * Converts a byte array to a long value. Assumes there will be
530    * {@link #SIZEOF_LONG} bytes available.
531    *
532    * @param bytes bytes
533    * @param offset offset
534    * @return the long value
535    */
536   public static long toLong(byte[] bytes, int offset) {
537     return toLong(bytes, offset, SIZEOF_LONG);
538   }
539 
540   /**
541    * Converts a byte array to a long value.
542    *
543    * @param bytes array of bytes
544    * @param offset offset into array
545    * @param length length of data (must be {@link #SIZEOF_LONG})
546    * @return the long value
547    * @throws IllegalArgumentException if length is not {@link #SIZEOF_LONG} or
548    * if there's not enough room in the array at the offset indicated.
549    */
550   public static long toLong(byte[] bytes, int offset, final int length) {
551     if (length != SIZEOF_LONG || offset + length > bytes.length) {
552       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
553     }
554     long l = 0;
555     for(int i = offset; i < offset + length; i++) {
556       l <<= 8;
557       l ^= bytes[i] & 0xFF;
558     }
559     return l;
560   }
561 
562   private static IllegalArgumentException
563     explainWrongLengthOrOffset(final byte[] bytes,
564                                final int offset,
565                                final int length,
566                                final int expectedLength) {
567     String reason;
568     if (length != expectedLength) {
569       reason = "Wrong length: " + length + ", expected " + expectedLength;
570     } else {
571      reason = "offset (" + offset + ") + length (" + length + ") exceed the"
572         + " capacity of the array: " + bytes.length;
573     }
574     return new IllegalArgumentException(reason);
575   }
576 
577   /**
578    * Put a long value out to the specified byte array position.
579    * @param bytes the byte array
580    * @param offset position in the array
581    * @param val long to write out
582    * @return incremented offset
583    * @throws IllegalArgumentException if the byte array given doesn't have
584    * enough room at the offset specified.
585    */
586   public static int putLong(byte[] bytes, int offset, long val) {
587     if (bytes.length - offset < SIZEOF_LONG) {
588       throw new IllegalArgumentException("Not enough room to put a long at"
589           + " offset " + offset + " in a " + bytes.length + " byte array");
590     }
591     for(int i = offset + 7; i > offset; i--) {
592       bytes[i] = (byte) val;
593       val >>>= 8;
594     }
595     bytes[offset] = (byte) val;
596     return offset + SIZEOF_LONG;
597   }
598 
599   /**
600    * Presumes float encoded as IEEE 754 floating-point "single format"
601    * @param bytes byte array
602    * @return Float made from passed byte array.
603    */
604   public static float toFloat(byte [] bytes) {
605     return toFloat(bytes, 0);
606   }
607 
608   /**
609    * Presumes float encoded as IEEE 754 floating-point "single format"
610    * @param bytes array to convert
611    * @param offset offset into array
612    * @return Float made from passed byte array.
613    */
614   public static float toFloat(byte [] bytes, int offset) {
615     return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
616   }
617 
618   /**
619    * @param bytes byte array
620    * @param offset offset to write to
621    * @param f float value
622    * @return New offset in <code>bytes</code>
623    */
624   public static int putFloat(byte [] bytes, int offset, float f) {
625     return putInt(bytes, offset, Float.floatToRawIntBits(f));
626   }
627 
628   /**
629    * @param f float value
630    * @return the float represented as byte []
631    */
632   public static byte [] toBytes(final float f) {
633     // Encode it as int
634     return Bytes.toBytes(Float.floatToRawIntBits(f));
635   }
636 
637   /**
638    * @param bytes byte array
639    * @return Return double made from passed bytes.
640    */
641   public static double toDouble(final byte [] bytes) {
642     return toDouble(bytes, 0);
643   }
644 
645   /**
646    * @param bytes byte array
647    * @param offset offset where double is
648    * @return Return double made from passed bytes.
649    */
650   public static double toDouble(final byte [] bytes, final int offset) {
651     return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
652   }
653 
654   /**
655    * @param bytes byte array
656    * @param offset offset to write to
657    * @param d value
658    * @return New offset into array <code>bytes</code>
659    */
660   public static int putDouble(byte [] bytes, int offset, double d) {
661     return putLong(bytes, offset, Double.doubleToLongBits(d));
662   }
663 
664   /**
665    * Serialize a double as the IEEE 754 double format output. The resultant
666    * array will be 8 bytes long.
667    *
668    * @param d value
669    * @return the double represented as byte []
670    */
671   public static byte [] toBytes(final double d) {
672     // Encode it as a long
673     return Bytes.toBytes(Double.doubleToRawLongBits(d));
674   }
675 
676   /**
677    * Convert an int value to a byte array.  Big-endian.  Same as what DataOutputStream.writeInt
678    * does.
679    *
680    * @param val value
681    * @return the byte array
682    */
683   public static byte[] toBytes(int val) {
684     byte [] b = new byte[4];
685     for(int i = 3; i > 0; i--) {
686       b[i] = (byte) val;
687       val >>>= 8;
688     }
689     b[0] = (byte) val;
690     return b;
691   }
692 
693   /**
694    * Converts a byte array to an int value
695    * @param bytes byte array
696    * @return the int value
697    */
698   public static int toInt(byte[] bytes) {
699     return toInt(bytes, 0, SIZEOF_INT);
700   }
701 
702   /**
703    * Converts a byte array to an int value
704    * @param bytes byte array
705    * @param offset offset into array
706    * @return the int value
707    */
708   public static int toInt(byte[] bytes, int offset) {
709     return toInt(bytes, offset, SIZEOF_INT);
710   }
711 
712   /**
713    * Converts a byte array to an int value
714    * @param bytes byte array
715    * @param offset offset into array
716    * @param length length of int (has to be {@link #SIZEOF_INT})
717    * @return the int value
718    * @throws IllegalArgumentException if length is not {@link #SIZEOF_INT} or
719    * if there's not enough room in the array at the offset indicated.
720    */
721   public static int toInt(byte[] bytes, int offset, final int length) {
722     if (length != SIZEOF_INT || offset + length > bytes.length) {
723       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
724     }
725     int n = 0;
726     for(int i = offset; i < (offset + length); i++) {
727       n <<= 8;
728       n ^= bytes[i] & 0xFF;
729     }
730     return n;
731   }
732 
733   /**
734    * Put an int value out to the specified byte array position.
735    * @param bytes the byte array
736    * @param offset position in the array
737    * @param val int to write out
738    * @return incremented offset
739    * @throws IllegalArgumentException if the byte array given doesn't have
740    * enough room at the offset specified.
741    */
742   public static int putInt(byte[] bytes, int offset, int val) {
743     if (bytes.length - offset < SIZEOF_INT) {
744       throw new IllegalArgumentException("Not enough room to put an int at"
745           + " offset " + offset + " in a " + bytes.length + " byte array");
746     }
747     for(int i= offset + 3; i > offset; i--) {
748       bytes[i] = (byte) val;
749       val >>>= 8;
750     }
751     bytes[offset] = (byte) val;
752     return offset + SIZEOF_INT;
753   }
754 
755   /**
756    * Convert a short value to a byte array of {@link #SIZEOF_SHORT} bytes long.
757    * @param val value
758    * @return the byte array
759    */
760   public static byte[] toBytes(short val) {
761     byte[] b = new byte[SIZEOF_SHORT];
762     b[1] = (byte) val;
763     val >>= 8;
764     b[0] = (byte) val;
765     return b;
766   }
767 
768   /**
769    * Converts a byte array to a short value
770    * @param bytes byte array
771    * @return the short value
772    */
773   public static short toShort(byte[] bytes) {
774     return toShort(bytes, 0, SIZEOF_SHORT);
775   }
776 
777   /**
778    * Converts a byte array to a short value
779    * @param bytes byte array
780    * @param offset offset into array
781    * @return the short value
782    */
783   public static short toShort(byte[] bytes, int offset) {
784     return toShort(bytes, offset, SIZEOF_SHORT);
785   }
786 
787   /**
788    * Converts a byte array to a short value
789    * @param bytes byte array
790    * @param offset offset into array
791    * @param length length, has to be {@link #SIZEOF_SHORT}
792    * @return the short value
793    * @throws IllegalArgumentException if length is not {@link #SIZEOF_SHORT}
794    * or if there's not enough room in the array at the offset indicated.
795    */
796   public static short toShort(byte[] bytes, int offset, final int length) {
797     if (length != SIZEOF_SHORT || offset + length > bytes.length) {
798       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
799     }
800     short n = 0;
801     n ^= bytes[offset] & 0xFF;
802     n <<= 8;
803     n ^= bytes[offset+1] & 0xFF;
804     return n;
805   }
806 
807   /**
808    * Returns a new byte array, copied from the given {@code buf},
809    * from the position (inclusive) to the limit (exclusive).
810    * The position and the other index parameters are not changed.
811    *
812    * @param buf a byte buffer
813    * @return the byte array
814    * @see #toBytes(ByteBuffer)
815    */
816   public static byte[] getBytes(ByteBuffer buf) {
817     return readBytes(buf.duplicate());
818   }
819 
820   /**
821    * Put a short value out to the specified byte array position.
822    * @param bytes the byte array
823    * @param offset position in the array
824    * @param val short to write out
825    * @return incremented offset
826    * @throws IllegalArgumentException if the byte array given doesn't have
827    * enough room at the offset specified.
828    */
829   public static int putShort(byte[] bytes, int offset, short val) {
830     if (bytes.length - offset < SIZEOF_SHORT) {
831       throw new IllegalArgumentException("Not enough room to put a short at"
832           + " offset " + offset + " in a " + bytes.length + " byte array");
833     }
834     bytes[offset+1] = (byte) val;
835     val >>= 8;
836     bytes[offset] = (byte) val;
837     return offset + SIZEOF_SHORT;
838   }
839 
840   /**
841    * Convert a BigDecimal value to a byte array
842    *
843    * @param val
844    * @return the byte array
845    */
846   public static byte[] toBytes(BigDecimal val) {
847     byte[] valueBytes = val.unscaledValue().toByteArray();
848     byte[] result = new byte[valueBytes.length + SIZEOF_INT];
849     int offset = putInt(result, 0, val.scale());
850     putBytes(result, offset, valueBytes, 0, valueBytes.length);
851     return result;
852   }
853 
854 
855   /**
856    * Converts a byte array to a BigDecimal
857    *
858    * @param bytes
859    * @return the char value
860    */
861   public static BigDecimal toBigDecimal(byte[] bytes) {
862     return toBigDecimal(bytes, 0, bytes.length);
863   }
864 
865   /**
866    * Converts a byte array to a BigDecimal value
867    *
868    * @param bytes
869    * @param offset
870    * @param length
871    * @return the char value
872    */
873   public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) {
874     if (bytes == null || length < SIZEOF_INT + 1 ||
875       (offset + length > bytes.length)) {
876       return null;
877     }
878 
879     int scale = toInt(bytes, offset);
880     byte[] tcBytes = new byte[length - SIZEOF_INT];
881     System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT);
882     return new BigDecimal(new BigInteger(tcBytes), scale);
883   }
884 
885   /**
886    * Put a BigDecimal value out to the specified byte array position.
887    *
888    * @param bytes  the byte array
889    * @param offset position in the array
890    * @param val    BigDecimal to write out
891    * @return incremented offset
892    */
893   public static int putBigDecimal(byte[] bytes, int offset, BigDecimal val) {
894     if (bytes == null) {
895       return offset;
896     }
897 
898     byte[] valueBytes = val.unscaledValue().toByteArray();
899     byte[] result = new byte[valueBytes.length + SIZEOF_INT];
900     offset = putInt(result, offset, val.scale());
901     return putBytes(result, offset, valueBytes, 0, valueBytes.length);
902   }
903 
904   /**
905    * @param vint Integer to make a vint of.
906    * @return Vint as bytes array.
907    */
908   public static byte [] vintToBytes(final long vint) {
909     long i = vint;
910     int size = WritableUtils.getVIntSize(i);
911     byte [] result = new byte[size];
912     int offset = 0;
913     if (i >= -112 && i <= 127) {
914       result[offset] = (byte) i;
915       return result;
916     }
917 
918     int len = -112;
919     if (i < 0) {
920       i ^= -1L; // take one's complement'
921       len = -120;
922     }
923 
924     long tmp = i;
925     while (tmp != 0) {
926       tmp = tmp >> 8;
927       len--;
928     }
929 
930     result[offset++] = (byte) len;
931 
932     len = (len < -120) ? -(len + 120) : -(len + 112);
933 
934     for (int idx = len; idx != 0; idx--) {
935       int shiftbits = (idx - 1) * 8;
936       long mask = 0xFFL << shiftbits;
937       result[offset++] = (byte)((i & mask) >> shiftbits);
938     }
939     return result;
940   }
941 
942   /**
943    * @param buffer buffer to convert
944    * @return vint bytes as an integer.
945    */
946   public static long bytesToVint(final byte [] buffer) {
947     int offset = 0;
948     byte firstByte = buffer[offset++];
949     int len = WritableUtils.decodeVIntSize(firstByte);
950     if (len == 1) {
951       return firstByte;
952     }
953     long i = 0;
954     for (int idx = 0; idx < len-1; idx++) {
955       byte b = buffer[offset++];
956       i = i << 8;
957       i = i | (b & 0xFF);
958     }
959     return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
960   }
961 
962   /**
963    * Reads a zero-compressed encoded long from input stream and returns it.
964    * @param buffer Binary array
965    * @param offset Offset into array at which vint begins.
966    * @throws java.io.IOException e
967    * @return deserialized long from stream.
968    */
969   public static long readVLong(final byte [] buffer, final int offset)
970   throws IOException {
971     byte firstByte = buffer[offset];
972     int len = WritableUtils.decodeVIntSize(firstByte);
973     if (len == 1) {
974       return firstByte;
975     }
976     long i = 0;
977     for (int idx = 0; idx < len-1; idx++) {
978       byte b = buffer[offset + 1 + idx];
979       i = i << 8;
980       i = i | (b & 0xFF);
981     }
982     return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
983   }
984 
985   /**
986    * @param left left operand
987    * @param right right operand
988    * @return 0 if equal, < 0 if left is less than right, etc.
989    */
990   public static int compareTo(final byte [] left, final byte [] right) {
991     return LexicographicalComparerHolder.BEST_COMPARER.
992       compareTo(left, 0, left.length, right, 0, right.length);
993   }
994 
995   /**
996    * Lexicographically compare two arrays.
997    *
998    * @param buffer1 left operand
999    * @param buffer2 right operand
1000    * @param offset1 Where to start comparing in the left buffer
1001    * @param offset2 Where to start comparing in the right buffer
1002    * @param length1 How much to compare from the left buffer
1003    * @param length2 How much to compare from the right buffer
1004    * @return 0 if equal, < 0 if left is less than right, etc.
1005    */
1006   public static int compareTo(byte[] buffer1, int offset1, int length1,
1007       byte[] buffer2, int offset2, int length2) {
1008     return LexicographicalComparerHolder.BEST_COMPARER.
1009       compareTo(buffer1, offset1, length1, buffer2, offset2, length2);
1010   }
1011 
1012   interface Comparer<T> {
1013     int compareTo(
1014       T buffer1, int offset1, int length1, T buffer2, int offset2, int length2
1015     );
1016   }
1017 
1018   @VisibleForTesting
1019   static Comparer<byte[]> lexicographicalComparerJavaImpl() {
1020     return LexicographicalComparerHolder.PureJavaComparer.INSTANCE;
1021   }
1022 
1023   /**
1024    * Provides a lexicographical comparer implementation; either a Java
1025    * implementation or a faster implementation based on {@link Unsafe}.
1026    *
1027    * <p>Uses reflection to gracefully fall back to the Java implementation if
1028    * {@code Unsafe} isn't available.
1029    */
1030   @VisibleForTesting
1031   static class LexicographicalComparerHolder {
1032     static final String UNSAFE_COMPARER_NAME =
1033         LexicographicalComparerHolder.class.getName() + "$UnsafeComparer";
1034 
1035     static final Comparer<byte[]> BEST_COMPARER = getBestComparer();
1036     /**
1037      * Returns the Unsafe-using Comparer, or falls back to the pure-Java
1038      * implementation if unable to do so.
1039      */
1040     static Comparer<byte[]> getBestComparer() {
1041       try {
1042         Class<?> theClass = Class.forName(UNSAFE_COMPARER_NAME);
1043 
1044         // yes, UnsafeComparer does implement Comparer<byte[]>
1045         @SuppressWarnings("unchecked")
1046         Comparer<byte[]> comparer =
1047           (Comparer<byte[]>) theClass.getEnumConstants()[0];
1048         return comparer;
1049       } catch (Throwable t) { // ensure we really catch *everything*
1050         return lexicographicalComparerJavaImpl();
1051       }
1052     }
1053 
1054     enum PureJavaComparer implements Comparer<byte[]> {
1055       INSTANCE;
1056 
1057       @Override
1058       public int compareTo(byte[] buffer1, int offset1, int length1,
1059           byte[] buffer2, int offset2, int length2) {
1060         // Short circuit equal case
1061         if (buffer1 == buffer2 &&
1062             offset1 == offset2 &&
1063             length1 == length2) {
1064           return 0;
1065         }
1066         // Bring WritableComparator code local
1067         int end1 = offset1 + length1;
1068         int end2 = offset2 + length2;
1069         for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) {
1070           int a = (buffer1[i] & 0xff);
1071           int b = (buffer2[j] & 0xff);
1072           if (a != b) {
1073             return a - b;
1074           }
1075         }
1076         return length1 - length2;
1077       }
1078     }
1079 
1080     @VisibleForTesting
1081     enum UnsafeComparer implements Comparer<byte[]> {
1082       INSTANCE;
1083 
1084       static final Unsafe theUnsafe;
1085 
1086       /** The offset to the first element in a byte array. */
1087       static final int BYTE_ARRAY_BASE_OFFSET;
1088 
1089       static {
1090         theUnsafe = (Unsafe) AccessController.doPrivileged(
1091             new PrivilegedAction<Object>() {
1092               @Override
1093               public Object run() {
1094                 try {
1095                   Field f = Unsafe.class.getDeclaredField("theUnsafe");
1096                   f.setAccessible(true);
1097                   return f.get(null);
1098                 } catch (NoSuchFieldException e) {
1099                   // It doesn't matter what we throw;
1100                   // it's swallowed in getBestComparer().
1101                   throw new Error();
1102                 } catch (IllegalAccessException e) {
1103                   throw new Error();
1104                 }
1105               }
1106             });
1107 
1108         BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class);
1109 
1110         // sanity check - this should never fail
1111         if (theUnsafe.arrayIndexScale(byte[].class) != 1) {
1112           throw new AssertionError();
1113         }
1114       }
1115 
1116       static final boolean littleEndian =
1117         ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN);
1118 
1119       /**
1120        * Returns true if x1 is less than x2, when both values are treated as
1121        * unsigned.
1122        */
1123       static boolean lessThanUnsigned(long x1, long x2) {
1124         return (x1 + Long.MIN_VALUE) < (x2 + Long.MIN_VALUE);
1125       }
1126 
1127       /**
1128        * Lexicographically compare two arrays.
1129        *
1130        * @param buffer1 left operand
1131        * @param buffer2 right operand
1132        * @param offset1 Where to start comparing in the left buffer
1133        * @param offset2 Where to start comparing in the right buffer
1134        * @param length1 How much to compare from the left buffer
1135        * @param length2 How much to compare from the right buffer
1136        * @return 0 if equal, < 0 if left is less than right, etc.
1137        */
1138       @Override
1139       public int compareTo(byte[] buffer1, int offset1, int length1,
1140           byte[] buffer2, int offset2, int length2) {
1141         // Short circuit equal case
1142         if (buffer1 == buffer2 &&
1143             offset1 == offset2 &&
1144             length1 == length2) {
1145           return 0;
1146         }
1147         int minLength = Math.min(length1, length2);
1148         int minWords = minLength / SIZEOF_LONG;
1149         int offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET;
1150         int offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET;
1151 
1152         /*
1153          * Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a
1154          * time is no slower than comparing 4 bytes at a time even on 32-bit.
1155          * On the other hand, it is substantially faster on 64-bit.
1156          */
1157         for (int i = 0; i < minWords * SIZEOF_LONG; i += SIZEOF_LONG) {
1158           long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i);
1159           long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i);
1160           long diff = lw ^ rw;
1161 
1162           if (diff != 0) {
1163             if (!littleEndian) {
1164               return lessThanUnsigned(lw, rw) ? -1 : 1;
1165             }
1166 
1167             // Use binary search
1168             int n = 0;
1169             int y;
1170             int x = (int) diff;
1171             if (x == 0) {
1172               x = (int) (diff >>> 32);
1173               n = 32;
1174             }
1175 
1176             y = x << 16;
1177             if (y == 0) {
1178               n += 16;
1179             } else {
1180               x = y;
1181             }
1182 
1183             y = x << 8;
1184             if (y == 0) {
1185               n += 8;
1186             }
1187             return (int) (((lw >>> n) & 0xFFL) - ((rw >>> n) & 0xFFL));
1188           }
1189         }
1190 
1191         // The epilogue to cover the last (minLength % 8) elements.
1192         for (int i = minWords * SIZEOF_LONG; i < minLength; i++) {
1193           int a = (buffer1[offset1 + i] & 0xff);
1194           int b = (buffer2[offset2 + i] & 0xff);
1195           if (a != b) {
1196             return a - b;
1197           }
1198         }
1199         return length1 - length2;
1200       }
1201     }
1202   }
1203 
1204   /**
1205    * @param left left operand
1206    * @param right right operand
1207    * @return True if equal
1208    */
1209   public static boolean equals(final byte [] left, final byte [] right) {
1210     // Could use Arrays.equals?
1211     //noinspection SimplifiableConditionalExpression
1212     if (left == right) return true;
1213     if (left == null || right == null) return false;
1214     if (left.length != right.length) return false;
1215     if (left.length == 0) return true;
1216 
1217     // Since we're often comparing adjacent sorted data,
1218     // it's usual to have equal arrays except for the very last byte
1219     // so check that first
1220     if (left[left.length - 1] != right[right.length - 1]) return false;
1221 
1222     return compareTo(left, right) == 0;
1223   }
1224 
1225   public static boolean equals(final byte[] left, int leftOffset, int leftLen,
1226                                final byte[] right, int rightOffset, int rightLen) {
1227     // short circuit case
1228     if (left == right &&
1229         leftOffset == rightOffset &&
1230         leftLen == rightLen) {
1231       return true;
1232     }
1233     // different lengths fast check
1234     if (leftLen != rightLen) {
1235       return false;
1236     }
1237     if (leftLen == 0) {
1238       return true;
1239     }
1240 
1241     // Since we're often comparing adjacent sorted data,
1242     // it's usual to have equal arrays except for the very last byte
1243     // so check that first
1244     if (left[leftOffset + leftLen - 1] != right[rightOffset + rightLen - 1]) return false;
1245 
1246     return LexicographicalComparerHolder.BEST_COMPARER.
1247       compareTo(left, leftOffset, leftLen, right, rightOffset, rightLen) == 0;
1248   }
1249 
1250 
1251   /**
1252    * Return true if the byte array on the right is a prefix of the byte
1253    * array on the left.
1254    */
1255   public static boolean startsWith(byte[] bytes, byte[] prefix) {
1256     return bytes != null && prefix != null &&
1257       bytes.length >= prefix.length &&
1258       LexicographicalComparerHolder.BEST_COMPARER.
1259         compareTo(bytes, 0, prefix.length, prefix, 0, prefix.length) == 0;
1260   }
1261 
1262   /**
1263    * @param b bytes to hash
1264    * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the
1265    * passed in array.  This method is what {@link org.apache.hadoop.io.Text} and
1266    * {@link ImmutableBytesWritable} use calculating hash code.
1267    */
1268   public static int hashCode(final byte [] b) {
1269     return hashCode(b, b.length);
1270   }
1271 
1272   /**
1273    * @param b value
1274    * @param length length of the value
1275    * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the
1276    * passed in array.  This method is what {@link org.apache.hadoop.io.Text} and
1277    * {@link ImmutableBytesWritable} use calculating hash code.
1278    */
1279   public static int hashCode(final byte [] b, final int length) {
1280     return WritableComparator.hashBytes(b, length);
1281   }
1282 
1283   /**
1284    * @param b bytes to hash
1285    * @return A hash of <code>b</code> as an Integer that can be used as key in
1286    * Maps.
1287    */
1288   public static Integer mapKey(final byte [] b) {
1289     return hashCode(b);
1290   }
1291 
1292   /**
1293    * @param b bytes to hash
1294    * @param length length to hash
1295    * @return A hash of <code>b</code> as an Integer that can be used as key in
1296    * Maps.
1297    */
1298   public static Integer mapKey(final byte [] b, final int length) {
1299     return hashCode(b, length);
1300   }
1301 
1302   /**
1303    * @param a lower half
1304    * @param b upper half
1305    * @return New array that has a in lower half and b in upper half.
1306    */
1307   public static byte [] add(final byte [] a, final byte [] b) {
1308     return add(a, b, EMPTY_BYTE_ARRAY);
1309   }
1310 
1311   /**
1312    * @param a first third
1313    * @param b second third
1314    * @param c third third
1315    * @return New array made from a, b and c
1316    */
1317   public static byte [] add(final byte [] a, final byte [] b, final byte [] c) {
1318     byte [] result = new byte[a.length + b.length + c.length];
1319     System.arraycopy(a, 0, result, 0, a.length);
1320     System.arraycopy(b, 0, result, a.length, b.length);
1321     System.arraycopy(c, 0, result, a.length + b.length, c.length);
1322     return result;
1323   }
1324 
1325   /**
1326    * @param a array
1327    * @param length amount of bytes to grab
1328    * @return First <code>length</code> bytes from <code>a</code>
1329    */
1330   public static byte [] head(final byte [] a, final int length) {
1331     if (a.length < length) {
1332       return null;
1333     }
1334     byte [] result = new byte[length];
1335     System.arraycopy(a, 0, result, 0, length);
1336     return result;
1337   }
1338 
1339   /**
1340    * @param a array
1341    * @param length amount of bytes to snarf
1342    * @return Last <code>length</code> bytes from <code>a</code>
1343    */
1344   public static byte [] tail(final byte [] a, final int length) {
1345     if (a.length < length) {
1346       return null;
1347     }
1348     byte [] result = new byte[length];
1349     System.arraycopy(a, a.length - length, result, 0, length);
1350     return result;
1351   }
1352 
1353   /**
1354    * @param a array
1355    * @param length new array size
1356    * @return Value in <code>a</code> plus <code>length</code> prepended 0 bytes
1357    */
1358   public static byte [] padHead(final byte [] a, final int length) {
1359     byte [] padding = new byte[length];
1360     for (int i = 0; i < length; i++) {
1361       padding[i] = 0;
1362     }
1363     return add(padding,a);
1364   }
1365 
1366   /**
1367    * @param a array
1368    * @param length new array size
1369    * @return Value in <code>a</code> plus <code>length</code> appended 0 bytes
1370    */
1371   public static byte [] padTail(final byte [] a, final int length) {
1372     byte [] padding = new byte[length];
1373     for (int i = 0; i < length; i++) {
1374       padding[i] = 0;
1375     }
1376     return add(a,padding);
1377   }
1378 
1379   /**
1380    * Split passed range.  Expensive operation relatively.  Uses BigInteger math.
1381    * Useful splitting ranges for MapReduce jobs.
1382    * @param a Beginning of range
1383    * @param b End of range
1384    * @param num Number of times to split range.  Pass 1 if you want to split
1385    * the range in two; i.e. one split.
1386    * @return Array of dividing values
1387    */
1388   public static byte [][] split(final byte [] a, final byte [] b, final int num) {
1389     return split(a, b, false, num);
1390   }
1391 
1392   /**
1393    * Split passed range.  Expensive operation relatively.  Uses BigInteger math.
1394    * Useful splitting ranges for MapReduce jobs.
1395    * @param a Beginning of range
1396    * @param b End of range
1397    * @param inclusive Whether the end of range is prefix-inclusive or is
1398    * considered an exclusive boundary.  Automatic splits are generally exclusive
1399    * and manual splits with an explicit range utilize an inclusive end of range.
1400    * @param num Number of times to split range.  Pass 1 if you want to split
1401    * the range in two; i.e. one split.
1402    * @return Array of dividing values
1403    */
1404   public static byte[][] split(final byte[] a, final byte[] b,
1405       boolean inclusive, final int num) {
1406     byte[][] ret = new byte[num + 2][];
1407     int i = 0;
1408     Iterable<byte[]> iter = iterateOnSplits(a, b, inclusive, num);
1409     if (iter == null)
1410       return null;
1411     for (byte[] elem : iter) {
1412       ret[i++] = elem;
1413     }
1414     return ret;
1415   }
1416 
1417   /**
1418    * Iterate over keys within the passed range, splitting at an [a,b) boundary.
1419    */
1420   public static Iterable<byte[]> iterateOnSplits(final byte[] a,
1421       final byte[] b, final int num)
1422   {
1423     return iterateOnSplits(a, b, false, num);
1424   }
1425 
1426   /**
1427    * Iterate over keys within the passed range.
1428    */
1429   public static Iterable<byte[]> iterateOnSplits(
1430       final byte[] a, final byte[]b, boolean inclusive, final int num)
1431   {
1432     byte [] aPadded;
1433     byte [] bPadded;
1434     if (a.length < b.length) {
1435       aPadded = padTail(a, b.length - a.length);
1436       bPadded = b;
1437     } else if (b.length < a.length) {
1438       aPadded = a;
1439       bPadded = padTail(b, a.length - b.length);
1440     } else {
1441       aPadded = a;
1442       bPadded = b;
1443     }
1444     if (compareTo(aPadded,bPadded) >= 0) {
1445       throw new IllegalArgumentException("b <= a");
1446     }
1447     if (num <= 0) {
1448       throw new IllegalArgumentException("num cannot be < 0");
1449     }
1450     byte [] prependHeader = {1, 0};
1451     final BigInteger startBI = new BigInteger(add(prependHeader, aPadded));
1452     final BigInteger stopBI = new BigInteger(add(prependHeader, bPadded));
1453     BigInteger diffBI = stopBI.subtract(startBI);
1454     if (inclusive) {
1455       diffBI = diffBI.add(BigInteger.ONE);
1456     }
1457     final BigInteger splitsBI = BigInteger.valueOf(num + 1);
1458     if(diffBI.compareTo(splitsBI) < 0) {
1459       return null;
1460     }
1461     final BigInteger intervalBI;
1462     try {
1463       intervalBI = diffBI.divide(splitsBI);
1464     } catch(Exception e) {
1465       LOG.error("Exception caught during division", e);
1466       return null;
1467     }
1468 
1469     final Iterator<byte[]> iterator = new Iterator<byte[]>() {
1470       private int i = -1;
1471 
1472       @Override
1473       public boolean hasNext() {
1474         return i < num+1;
1475       }
1476 
1477       @Override
1478       public byte[] next() {
1479         i++;
1480         if (i == 0) return a;
1481         if (i == num + 1) return b;
1482 
1483         BigInteger curBI = startBI.add(intervalBI.multiply(BigInteger.valueOf(i)));
1484         byte [] padded = curBI.toByteArray();
1485         if (padded[1] == 0)
1486           padded = tail(padded, padded.length - 2);
1487         else
1488           padded = tail(padded, padded.length - 1);
1489         return padded;
1490       }
1491 
1492       @Override
1493       public void remove() {
1494         throw new UnsupportedOperationException();
1495       }
1496 
1497     };
1498 
1499     return new Iterable<byte[]>() {
1500       @Override
1501       public Iterator<byte[]> iterator() {
1502         return iterator;
1503       }
1504     };
1505   }
1506 
1507   /**
1508    * @param bytes array to hash
1509    * @param offset offset to start from
1510    * @param length length to hash
1511    * */
1512   public static int hashCode(byte[] bytes, int offset, int length) {
1513     int hash = 1;
1514     for (int i = offset; i < offset + length; i++)
1515       hash = (31 * hash) + (int) bytes[i];
1516     return hash;
1517   }
1518 
1519   /**
1520    * @param t operands
1521    * @return Array of byte arrays made from passed array of Text
1522    */
1523   public static byte [][] toByteArrays(final String [] t) {
1524     byte [][] result = new byte[t.length][];
1525     for (int i = 0; i < t.length; i++) {
1526       result[i] = Bytes.toBytes(t[i]);
1527     }
1528     return result;
1529   }
1530 
1531   /**
1532    * @param column operand
1533    * @return A byte array of a byte array where first and only entry is
1534    * <code>column</code>
1535    */
1536   public static byte [][] toByteArrays(final String column) {
1537     return toByteArrays(toBytes(column));
1538   }
1539 
1540   /**
1541    * @param column operand
1542    * @return A byte array of a byte array where first and only entry is
1543    * <code>column</code>
1544    */
1545   public static byte [][] toByteArrays(final byte [] column) {
1546     byte [][] result = new byte[1][];
1547     result[0] = column;
1548     return result;
1549   }
1550 
1551   /**
1552    * Binary search for keys in indexes.
1553    *
1554    * @param arr array of byte arrays to search for
1555    * @param key the key you want to find
1556    * @param offset the offset in the key you want to find
1557    * @param length the length of the key
1558    * @param comparator a comparator to compare.
1559    * @return zero-based index of the key, if the key is present in the array.
1560    *         Otherwise, a value -(i + 1) such that the key is between arr[i -
1561    *         1] and arr[i] non-inclusively, where i is in [0, i], if we define
1562    *         arr[-1] = -Inf and arr[N] = Inf for an N-element array. The above
1563    *         means that this function can return 2N + 1 different values
1564    *         ranging from -(N + 1) to N - 1.
1565    */
1566   public static int binarySearch(byte [][]arr, byte []key, int offset,
1567       int length, RawComparator<byte []> comparator) {
1568     int low = 0;
1569     int high = arr.length - 1;
1570 
1571     while (low <= high) {
1572       int mid = (low+high) >>> 1;
1573       // we have to compare in this order, because the comparator order
1574       // has special logic when the 'left side' is a special key.
1575       int cmp = comparator.compare(key, offset, length,
1576           arr[mid], 0, arr[mid].length);
1577       // key lives above the midpoint
1578       if (cmp > 0)
1579         low = mid + 1;
1580       // key lives below the midpoint
1581       else if (cmp < 0)
1582         high = mid - 1;
1583       // BAM. how often does this really happen?
1584       else
1585         return mid;
1586     }
1587     return - (low+1);
1588   }
1589 
1590   /**
1591    * Bytewise binary increment/deincrement of long contained in byte array
1592    * on given amount.
1593    *
1594    * @param value - array of bytes containing long (length <= SIZEOF_LONG)
1595    * @param amount value will be incremented on (deincremented if negative)
1596    * @return array of bytes containing incremented long (length == SIZEOF_LONG)
1597    */
1598   public static byte [] incrementBytes(byte[] value, long amount)
1599   {
1600     byte[] val = value;
1601     if (val.length < SIZEOF_LONG) {
1602       // Hopefully this doesn't happen too often.
1603       byte [] newvalue;
1604       if (val[0] < 0) {
1605         newvalue = new byte[]{-1, -1, -1, -1, -1, -1, -1, -1};
1606       } else {
1607         newvalue = new byte[SIZEOF_LONG];
1608       }
1609       System.arraycopy(val, 0, newvalue, newvalue.length - val.length,
1610         val.length);
1611       val = newvalue;
1612     } else if (val.length > SIZEOF_LONG) {
1613       throw new IllegalArgumentException("Increment Bytes - value too big: " +
1614         val.length);
1615     }
1616     if(amount == 0) return val;
1617     if(val[0] < 0){
1618       return binaryIncrementNeg(val, amount);
1619     }
1620     return binaryIncrementPos(val, amount);
1621   }
1622 
1623   /* increment/deincrement for positive value */
1624   private static byte [] binaryIncrementPos(byte [] value, long amount) {
1625     long amo = amount;
1626     int sign = 1;
1627     if (amount < 0) {
1628       amo = -amount;
1629       sign = -1;
1630     }
1631     for(int i=0;i<value.length;i++) {
1632       int cur = ((int)amo % 256) * sign;
1633       amo = (amo >> 8);
1634       int val = value[value.length-i-1] & 0x0ff;
1635       int total = val + cur;
1636       if(total > 255) {
1637         amo += sign;
1638         total %= 256;
1639       } else if (total < 0) {
1640         amo -= sign;
1641       }
1642       value[value.length-i-1] = (byte)total;
1643       if (amo == 0) return value;
1644     }
1645     return value;
1646   }
1647 
1648   /* increment/deincrement for negative value */
1649   private static byte [] binaryIncrementNeg(byte [] value, long amount) {
1650     long amo = amount;
1651     int sign = 1;
1652     if (amount < 0) {
1653       amo = -amount;
1654       sign = -1;
1655     }
1656     for(int i=0;i<value.length;i++) {
1657       int cur = ((int)amo % 256) * sign;
1658       amo = (amo >> 8);
1659       int val = ((~value[value.length-i-1]) & 0x0ff) + 1;
1660       int total = cur - val;
1661       if(total >= 0) {
1662         amo += sign;
1663       } else if (total < -256) {
1664         amo -= sign;
1665         total %= 256;
1666       }
1667       value[value.length-i-1] = (byte)total;
1668       if (amo == 0) return value;
1669     }
1670     return value;
1671   }
1672 
1673   /**
1674    * Writes a string as a fixed-size field, padded with zeros.
1675    */
1676   public static void writeStringFixedSize(final DataOutput out, String s,
1677       int size) throws IOException {
1678     byte[] b = toBytes(s);
1679     if (b.length > size) {
1680       throw new IOException("Trying to write " + b.length + " bytes (" +
1681           toStringBinary(b) + ") into a field of length " + size);
1682     }
1683 
1684     out.writeBytes(s);
1685     for (int i = 0; i < size - s.length(); ++i)
1686       out.writeByte(0);
1687   }
1688 
1689   /**
1690    * Reads a fixed-size field and interprets it as a string padded with zeros.
1691    */
1692   public static String readStringFixedSize(final DataInput in, int size)
1693       throws IOException {
1694     byte[] b = new byte[size];
1695     in.readFully(b);
1696     int n = b.length;
1697     while (n > 0 && b[n - 1] == 0)
1698       --n;
1699 
1700     return toString(b, 0, n);
1701   }
1702 
1703   /**
1704    * Copy the byte array given in parameter and return an instance
1705    * of a new byte array with the same length and the same content.
1706    * @param bytes the byte array to duplicate
1707    * @return a copy of the given byte array
1708    */
1709   public static byte [] copy(byte [] bytes) {
1710     if (bytes == null) return null;
1711     byte [] result = new byte[bytes.length];
1712     System.arraycopy(bytes, 0, result, 0, bytes.length);
1713     return result;
1714   }
1715 
1716   /**
1717    * Copy the byte array given in parameter and return an instance
1718    * of a new byte array with the same length and the same content.
1719    * @param bytes the byte array to copy from
1720    * @return a copy of the given designated byte array
1721    * @param offset
1722    * @param length
1723    */
1724   public static byte [] copy(byte [] bytes, final int offset, final int length) {
1725     if (bytes == null) return null;
1726     byte [] result = new byte[length];
1727     System.arraycopy(bytes, offset, result, 0, length);
1728     return result;
1729   }
1730 
1731   /**
1732    * Search sorted array "a" for byte "key". I can't remember if I wrote this or copied it from
1733    * somewhere. (mcorgan)
1734    * @param a Array to search. Entries must be sorted and unique.
1735    * @param fromIndex First index inclusive of "a" to include in the search.
1736    * @param toIndex Last index exclusive of "a" to include in the search.
1737    * @param key The byte to search for.
1738    * @return The index of key if found. If not found, return -(index + 1), where negative indicates
1739    *         "not found" and the "index + 1" handles the "-0" case.
1740    */
1741   public static int unsignedBinarySearch(byte[] a, int fromIndex, int toIndex, byte key) {
1742     int unsignedKey = key & 0xff;
1743     int low = fromIndex;
1744     int high = toIndex - 1;
1745 
1746     while (low <= high) {
1747       int mid = (low + high) >>> 1;
1748       int midVal = a[mid] & 0xff;
1749 
1750       if (midVal < unsignedKey) {
1751         low = mid + 1;
1752       } else if (midVal > unsignedKey) {
1753         high = mid - 1;
1754       } else {
1755         return mid; // key found
1756       }
1757     }
1758     return -(low + 1); // key not found.
1759   }
1760 
1761   /**
1762    * Treat the byte[] as an unsigned series of bytes, most significant bits first.  Start by adding
1763    * 1 to the rightmost bit/byte and carry over all overflows to the more significant bits/bytes.
1764    *
1765    * @param input The byte[] to increment.
1766    * @return The incremented copy of "in".  May be same length or 1 byte longer.
1767    */
1768   public static byte[] unsignedCopyAndIncrement(final byte[] input) {
1769     byte[] copy = copy(input);
1770     if (copy == null) {
1771       throw new IllegalArgumentException("cannot increment null array");
1772     }
1773     for (int i = copy.length - 1; i >= 0; --i) {
1774       if (copy[i] == -1) {// -1 is all 1-bits, which is the unsigned maximum
1775         copy[i] = 0;
1776       } else {
1777         ++copy[i];
1778         return copy;
1779       }
1780     }
1781     // we maxed out the array
1782     byte[] out = new byte[copy.length + 1];
1783     out[0] = 1;
1784     System.arraycopy(copy, 0, out, 1, copy.length);
1785     return out;
1786   }
1787 
1788   public static boolean equals(List<byte[]> a, List<byte[]> b) {
1789     if (a == null) {
1790       if (b == null) {
1791         return true;
1792       }
1793       return false;
1794     }
1795     if (b == null) {
1796       return false;
1797     }
1798     if (a.size() != b.size()) {
1799       return false;
1800     }
1801     for (int i = 0; i < a.size(); ++i) {
1802       if (!Bytes.equals(a.get(i), b.get(i))) {
1803         return false;
1804       }
1805     }
1806     return true;
1807   }
1808 
1809   public static boolean isSorted(Collection<byte[]> arrays) {
1810     byte[] previous = new byte[0];
1811     for (byte[] array : IterableUtils.nullSafe(arrays)) {
1812       if (Bytes.compareTo(previous, array) > 0) {
1813         return false;
1814       }
1815       previous = array;
1816     }
1817     return true;
1818   }
1819 
1820   public static List<byte[]> getUtf8ByteArrays(List<String> strings) {
1821     List<byte[]> byteArrays = Lists.newArrayListWithCapacity(CollectionUtils.nullSafeSize(strings));
1822     for (String s : IterableUtils.nullSafe(strings)) {
1823       byteArrays.add(Bytes.toBytes(s));
1824     }
1825     return byteArrays;
1826   }
1827 
1828   /**
1829    * Returns the index of the first appearance of the value {@code target} in
1830    * {@code array}.
1831    *
1832    * @param array an array of {@code byte} values, possibly empty
1833    * @param target a primitive {@code byte} value
1834    * @return the least index {@code i} for which {@code array[i] == target}, or
1835    *     {@code -1} if no such index exists.
1836    */
1837   public static int indexOf(byte[] array, byte target) {
1838     for (int i = 0; i < array.length; i++) {
1839       if (array[i] == target) {
1840         return i;
1841       }
1842     }
1843     return -1;
1844   }
1845 
1846   /**
1847    * Returns the start position of the first occurrence of the specified {@code
1848    * target} within {@code array}, or {@code -1} if there is no such occurrence.
1849    *
1850    * <p>More formally, returns the lowest index {@code i} such that {@code
1851    * java.util.Arrays.copyOfRange(array, i, i + target.length)} contains exactly
1852    * the same elements as {@code target}.
1853    *
1854    * @param array the array to search for the sequence {@code target}
1855    * @param target the array to search for as a sub-sequence of {@code array}
1856    */
1857   public static int indexOf(byte[] array, byte[] target) {
1858     checkNotNull(array, "array");
1859     checkNotNull(target, "target");
1860     if (target.length == 0) {
1861       return 0;
1862     }
1863 
1864     outer:
1865     for (int i = 0; i < array.length - target.length + 1; i++) {
1866       for (int j = 0; j < target.length; j++) {
1867         if (array[i + j] != target[j]) {
1868           continue outer;
1869         }
1870       }
1871       return i;
1872     }
1873     return -1;
1874   }
1875 
1876   /**
1877    * @param array an array of {@code byte} values, possibly empty
1878    * @param target a primitive {@code byte} value
1879    * @return {@code true} if {@code target} is present as an element anywhere in {@code array}.
1880    */
1881   public static boolean contains(byte[] array, byte target) {
1882     return indexOf(array, target) > -1;
1883   }
1884 
1885   /**
1886    * @param array an array of {@code byte} values, possibly empty
1887    * @param target an array of {@code byte}
1888    * @return {@code true} if {@code target} is present anywhere in {@code array}
1889    */
1890   public static boolean contains(byte[] array, byte[] target) {
1891     return indexOf(array, target) > -1;
1892   }
1893 
1894   /**
1895    * Fill given array with zeros.
1896    * @param b array which needs to be filled with zeros
1897    */
1898   public static void zero(byte[] b) {
1899     zero(b, 0, b.length);
1900   }
1901 
1902   /**
1903    * Fill given array with zeros at the specified position.
1904    * @param b
1905    * @param offset
1906    * @param length
1907    */
1908   public static void zero(byte[] b, int offset, int length) {
1909     checkPositionIndex(offset, b.length, "offset");
1910     checkArgument(length > 0, "length must be greater than 0");
1911     checkPositionIndex(offset + length, b.length, "offset + length");
1912     Arrays.fill(b, offset, offset + length, (byte) 0);
1913   }
1914 
1915   private static final SecureRandom RNG = new SecureRandom();
1916 
1917   /**
1918    * Fill given array with random bytes.
1919    * @param b array which needs to be filled with random bytes
1920    */
1921   public static void random(byte[] b) {
1922     RNG.nextBytes(b);
1923   }
1924 
1925   /**
1926    * Fill given array with random bytes at the specified position.
1927    * @param b
1928    * @param offset
1929    * @param length
1930    */
1931   public static void random(byte[] b, int offset, int length) {
1932     checkPositionIndex(offset, b.length, "offset");
1933     checkArgument(length > 0, "length must be greater than 0");
1934     checkPositionIndex(offset + length, b.length, "offset + length");
1935     byte[] buf = new byte[length];
1936     RNG.nextBytes(buf);
1937     System.arraycopy(buf, 0, b, offset, length);
1938   }
1939 }