View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.util;
21  
22  import java.io.DataInput;
23  import java.io.DataOutput;
24  import java.io.IOException;
25  import java.io.UnsupportedEncodingException;
26  import java.lang.reflect.Field;
27  import java.math.BigDecimal;
28  import java.math.BigInteger;
29  import java.nio.ByteBuffer;
30  import java.nio.ByteOrder;
31  import java.security.AccessController;
32  import java.security.PrivilegedAction;
33  import java.util.Comparator;
34  import java.util.Iterator;
35  
36  import org.apache.commons.lang.StringUtils;
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.hadoop.hbase.HConstants;
40  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
41  import org.apache.hadoop.io.RawComparator;
42  import org.apache.hadoop.io.WritableComparator;
43  import org.apache.hadoop.io.WritableUtils;
44  
45  import sun.misc.Unsafe;
46  
47  import com.google.common.annotations.VisibleForTesting;
48  
49  /**
50   * Utility class that handles byte arrays, conversions to/from other types,
51   * comparisons, hash code generation, manufacturing keys for HashMaps or
52   * HashSets, etc.
53   */
54  public class Bytes {
55  
56    private static final Log LOG = LogFactory.getLog(Bytes.class);
57  
58    /**
59     * Size of boolean in bytes
60     */
61    public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE;
62  
63    /**
64     * Size of byte in bytes
65     */
66    public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN;
67  
68    /**
69     * Size of char in bytes
70     */
71    public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE;
72  
73    /**
74     * Size of double in bytes
75     */
76    public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE;
77  
78    /**
79     * Size of float in bytes
80     */
81    public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE;
82  
83    /**
84     * Size of int in bytes
85     */
86    public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE;
87  
88    /**
89     * Size of long in bytes
90     */
91    public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE;
92  
93    /**
94     * Size of short in bytes
95     */
96    public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE;
97  
98  
99    /**
100    * Estimate of size cost to pay beyond payload in jvm for instance of byte [].
101    * Estimate based on study of jhat and jprofiler numbers.
102    */
103   // JHat says BU is 56 bytes.
104   // SizeOf which uses java.lang.instrument says 24 bytes. (3 longs?)
105   public static final int ESTIMATED_HEAP_TAX = 16;
106 
107   /**
108    * Byte array comparator class.
109    */
110   public static class ByteArrayComparator implements RawComparator<byte []> {
111     /**
112      * Constructor
113      */
114     public ByteArrayComparator() {
115       super();
116     }
117     public int compare(byte [] left, byte [] right) {
118       return compareTo(left, right);
119     }
120     public int compare(byte [] b1, int s1, int l1, byte [] b2, int s2, int l2) {
121       return LexicographicalComparerHolder.BEST_COMPARER.
122         compareTo(b1, s1, l1, b2, s2, l2);
123     }
124   }
125 
126   /**
127    * Pass this to TreeMaps where byte [] are keys.
128    */
129   public static Comparator<byte []> BYTES_COMPARATOR =
130     new ByteArrayComparator();
131 
132   /**
133    * Use comparing byte arrays, byte-by-byte
134    */
135   public static RawComparator<byte []> BYTES_RAWCOMPARATOR =
136     new ByteArrayComparator();
137 
138   /**
139    * Read byte-array written with a WritableableUtils.vint prefix.
140    * @param in Input to read from.
141    * @return byte array read off <code>in</code>
142    * @throws IOException e
143    */
144   public static byte [] readByteArray(final DataInput in)
145   throws IOException {
146     int len = WritableUtils.readVInt(in);
147     if (len < 0) {
148       throw new NegativeArraySizeException(Integer.toString(len));
149     }
150     byte [] result = new byte[len];
151     in.readFully(result, 0, len);
152     return result;
153   }
154 
155   /**
156    * Read byte-array written with a WritableableUtils.vint prefix.
157    * IOException is converted to a RuntimeException.
158    * @param in Input to read from.
159    * @return byte array read off <code>in</code>
160    */
161   public static byte [] readByteArrayThrowsRuntime(final DataInput in) {
162     try {
163       return readByteArray(in);
164     } catch (Exception e) {
165       throw new RuntimeException(e);
166     }
167   }
168 
169   /**
170    * Write byte-array with a WritableableUtils.vint prefix.
171    * @param out output stream to be written to
172    * @param b array to write
173    * @throws IOException e
174    */
175   public static void writeByteArray(final DataOutput out, final byte [] b)
176   throws IOException {
177     if(b == null) {
178       WritableUtils.writeVInt(out, 0);
179     } else {
180       writeByteArray(out, b, 0, b.length);
181     }
182   }
183 
184   /**
185    * Write byte-array to out with a vint length prefix.
186    * @param out output stream
187    * @param b array
188    * @param offset offset into array
189    * @param length length past offset
190    * @throws IOException e
191    */
192   public static void writeByteArray(final DataOutput out, final byte [] b,
193       final int offset, final int length)
194   throws IOException {
195     WritableUtils.writeVInt(out, length);
196     out.write(b, offset, length);
197   }
198 
199   /**
200    * Write byte-array from src to tgt with a vint length prefix.
201    * @param tgt target array
202    * @param tgtOffset offset into target array
203    * @param src source array
204    * @param srcOffset source offset
205    * @param srcLength source length
206    * @return New offset in src array.
207    */
208   public static int writeByteArray(final byte [] tgt, final int tgtOffset,
209       final byte [] src, final int srcOffset, final int srcLength) {
210     byte [] vint = vintToBytes(srcLength);
211     System.arraycopy(vint, 0, tgt, tgtOffset, vint.length);
212     int offset = tgtOffset + vint.length;
213     System.arraycopy(src, srcOffset, tgt, offset, srcLength);
214     return offset + srcLength;
215   }
216 
217   /**
218    * Put bytes at the specified byte array position.
219    * @param tgtBytes the byte array
220    * @param tgtOffset position in the array
221    * @param srcBytes array to write out
222    * @param srcOffset source offset
223    * @param srcLength source length
224    * @return incremented offset
225    */
226   public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes,
227       int srcOffset, int srcLength) {
228     System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength);
229     return tgtOffset + srcLength;
230   }
231 
232   /**
233    * Write a single byte out to the specified byte array position.
234    * @param bytes the byte array
235    * @param offset position in the array
236    * @param b byte to write out
237    * @return incremented offset
238    */
239   public static int putByte(byte[] bytes, int offset, byte b) {
240     bytes[offset] = b;
241     return offset + 1;
242   }
243 
244   /**
245    * Returns a new byte array, copied from the passed ByteBuffer.
246    * @param bb A ByteBuffer
247    * @return the byte array
248    */
249   public static byte[] toBytes(ByteBuffer bb) {
250     int length = bb.limit();
251     byte [] result = new byte[length];
252     System.arraycopy(bb.array(), bb.arrayOffset(), result, 0, length);
253     return result;
254   }
255 
256   /**
257    * @param b Presumed UTF-8 encoded byte array.
258    * @return String made from <code>b</code>
259    */
260   public static String toString(final byte [] b) {
261     if (b == null) {
262       return null;
263     }
264     return toString(b, 0, b.length);
265   }
266 
267   /**
268    * Joins two byte arrays together using a separator.
269    * @param b1 The first byte array.
270    * @param sep The separator to use.
271    * @param b2 The second byte array.
272    */
273   public static String toString(final byte [] b1,
274                                 String sep,
275                                 final byte [] b2) {
276     return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length);
277   }
278 
279   /**
280    * This method will convert utf8 encoded bytes into a string. If
281    * an UnsupportedEncodingException occurs, this method will eat it
282    * and return null instead.
283    *
284    * @param b Presumed UTF-8 encoded byte array.
285    * @param off offset into array
286    * @param len length of utf-8 sequence
287    * @return String made from <code>b</code> or null
288    */
289   public static String toString(final byte [] b, int off, int len) {
290     if (b == null) {
291       return null;
292     }
293     if (len == 0) {
294       return "";
295     }
296     try {
297       return new String(b, off, len, HConstants.UTF8_ENCODING);
298     } catch (UnsupportedEncodingException e) {
299       LOG.error("UTF-8 not supported?", e);
300       return null;
301     }
302   }
303 
304   /**
305    * Write a printable representation of a byte array.
306    *
307    * @param b byte array
308    * @return string
309    * @see #toStringBinary(byte[], int, int)
310    */
311   public static String toStringBinary(final byte [] b) {
312     if (b == null)
313       return "null";
314     return toStringBinary(b, 0, b.length);
315   }
316 
317   /**
318    * Converts the given byte buffer, from its array offset to its limit, to
319    * a string. The position and the mark are ignored.
320    *
321    * @param buf a byte buffer
322    * @return a string representation of the buffer's binary contents
323    */
324   public static String toStringBinary(ByteBuffer buf) {
325     if (buf == null)
326       return "null";
327     return toStringBinary(buf.array(), buf.arrayOffset(), buf.limit());
328   }
329 
330   /**
331    * Write a printable representation of a byte array. Non-printable
332    * characters are hex escaped in the format \\x%02X, eg:
333    * \x00 \x05 etc
334    *
335    * @param b array to write out
336    * @param off offset to start at
337    * @param len length to write
338    * @return string output
339    */
340   public static String toStringBinary(final byte [] b, int off, int len) {
341     StringBuilder result = new StringBuilder();
342     for (int i = off; i < off + len ; ++i ) { 
343       int ch = b[i] & 0xFF;
344       if ( (ch >= '0' && ch <= '9')      
345           || (ch >= 'A' && ch <= 'Z')
346           || (ch >= 'a' && ch <= 'z')
347           || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0 ) {
348           result.append((char)ch);
349       } else {
350         result.append(String.format("\\x%02X", ch));
351       }
352     }
353     return result.toString();
354   }
355 
356   private static boolean isHexDigit(char c) {
357     return
358         (c >= 'A' && c <= 'F') ||
359         (c >= '0' && c <= '9');
360   }
361 
362   /**
363    * Takes a ASCII digit in the range A-F0-9 and returns
364    * the corresponding integer/ordinal value.
365    * @param ch  The hex digit.
366    * @return The converted hex value as a byte.
367    */
368   public static byte toBinaryFromHex(byte ch) {
369     if ( ch >= 'A' && ch <= 'F' )
370       return (byte) ((byte)10 + (byte) (ch - 'A'));
371     // else
372     return (byte) (ch - '0');
373   }
374 
375   public static byte [] toBytesBinary(String in) {
376     // this may be bigger than we need, but let's be safe.
377     byte [] b = new byte[in.length()];
378     int size = 0;
379     for (int i = 0; i < in.length(); ++i) {
380       char ch = in.charAt(i);
381       if (ch == '\\' && in.length() > i+1 && in.charAt(i+1) == 'x') {
382         // ok, take next 2 hex digits.
383         char hd1 = in.charAt(i+2);
384         char hd2 = in.charAt(i+3);
385 
386         // they need to be A-F0-9:
387         if (!isHexDigit(hd1) ||
388             !isHexDigit(hd2)) {
389           // bogus escape code, ignore:
390           continue;
391         }
392         // turn hex ASCII digit -> number
393         byte d = (byte) ((toBinaryFromHex((byte)hd1) << 4) + toBinaryFromHex((byte)hd2));
394 
395         b[size++] = d;
396         i += 3; // skip 3
397       } else {
398         b[size++] = (byte) ch;
399       }
400     }
401     // resize:
402     byte [] b2 = new byte[size];
403     System.arraycopy(b, 0, b2, 0, size);
404     return b2;
405   }
406 
407   /**
408    * Converts a string to a UTF-8 byte array.
409    * @param s string
410    * @return the byte array
411    */
412   public static byte[] toBytes(String s) {
413     try {
414       return s.getBytes(HConstants.UTF8_ENCODING);
415     } catch (UnsupportedEncodingException e) {
416       LOG.error("UTF-8 not supported?", e);
417       return null;
418     }
419   }
420 
421   /**
422    * Convert a boolean to a byte array. True becomes -1
423    * and false becomes 0.
424    *
425    * @param b value
426    * @return <code>b</code> encoded in a byte array.
427    */
428   public static byte [] toBytes(final boolean b) {
429     return new byte[] { b ? (byte) -1 : (byte) 0 };
430   }
431 
432   /**
433    * Reverses {@link #toBytes(boolean)}
434    * @param b array
435    * @return True or false.
436    */
437   public static boolean toBoolean(final byte [] b) {
438     if (b.length != 1) {
439       throw new IllegalArgumentException("Array has wrong size: " + b.length);
440     }
441     return b[0] != (byte) 0;
442   }
443 
444   /**
445    * Convert a long value to a byte array using big-endian.
446    *
447    * @param val value to convert
448    * @return the byte array
449    */
450   public static byte[] toBytes(long val) {
451     byte [] b = new byte[8];
452     for (int i = 7; i > 0; i--) {
453       b[i] = (byte) val;
454       val >>>= 8;
455     }
456     b[0] = (byte) val;
457     return b;
458   }
459 
460   /**
461    * Converts a byte array to a long value. Reverses
462    * {@link #toBytes(long)}
463    * @param bytes array
464    * @return the long value
465    */
466   public static long toLong(byte[] bytes) {
467     return toLong(bytes, 0, SIZEOF_LONG);
468   }
469 
470   /**
471    * Converts a byte array to a long value. Assumes there will be
472    * {@link #SIZEOF_LONG} bytes available.
473    *
474    * @param bytes bytes
475    * @param offset offset
476    * @return the long value
477    */
478   public static long toLong(byte[] bytes, int offset) {
479     return toLong(bytes, offset, SIZEOF_LONG);
480   }
481 
482   /**
483    * Converts a byte array to a long value.
484    *
485    * @param bytes array of bytes
486    * @param offset offset into array
487    * @param length length of data (must be {@link #SIZEOF_LONG})
488    * @return the long value
489    * @throws IllegalArgumentException if length is not {@link #SIZEOF_LONG} or
490    * if there's not enough room in the array at the offset indicated.
491    */
492   public static long toLong(byte[] bytes, int offset, final int length) {
493     if (length != SIZEOF_LONG || offset + length > bytes.length) {
494       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
495     }
496     long l = 0;
497     for(int i = offset; i < offset + length; i++) {
498       l <<= 8;
499       l ^= bytes[i] & 0xFF;
500     }
501     return l;
502   }
503 
504   private static IllegalArgumentException
505     explainWrongLengthOrOffset(final byte[] bytes,
506                                final int offset,
507                                final int length,
508                                final int expectedLength) {
509     String reason;
510     if (length != expectedLength) {
511       reason = "Wrong length: " + length + ", expected " + expectedLength;
512     } else {
513      reason = "offset (" + offset + ") + length (" + length + ") exceed the"
514         + " capacity of the array: " + bytes.length;
515     }
516     return new IllegalArgumentException(reason);
517   }
518 
519   /**
520    * Put a long value out to the specified byte array position.
521    * @param bytes the byte array
522    * @param offset position in the array
523    * @param val long to write out
524    * @return incremented offset
525    * @throws IllegalArgumentException if the byte array given doesn't have
526    * enough room at the offset specified.
527    */
528   public static int putLong(byte[] bytes, int offset, long val) {
529     if (bytes.length - offset < SIZEOF_LONG) {
530       throw new IllegalArgumentException("Not enough room to put a long at"
531           + " offset " + offset + " in a " + bytes.length + " byte array");
532     }
533     for(int i = offset + 7; i > offset; i--) {
534       bytes[i] = (byte) val;
535       val >>>= 8;
536     }
537     bytes[offset] = (byte) val;
538     return offset + SIZEOF_LONG;
539   }
540 
541   /**
542    * Presumes float encoded as IEEE 754 floating-point "single format"
543    * @param bytes byte array
544    * @return Float made from passed byte array.
545    */
546   public static float toFloat(byte [] bytes) {
547     return toFloat(bytes, 0);
548   }
549 
550   /**
551    * Presumes float encoded as IEEE 754 floating-point "single format"
552    * @param bytes array to convert
553    * @param offset offset into array
554    * @return Float made from passed byte array.
555    */
556   public static float toFloat(byte [] bytes, int offset) {
557     return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
558   }
559 
560   /**
561    * @param bytes byte array
562    * @param offset offset to write to
563    * @param f float value
564    * @return New offset in <code>bytes</code>
565    */
566   public static int putFloat(byte [] bytes, int offset, float f) {
567     return putInt(bytes, offset, Float.floatToRawIntBits(f));
568   }
569 
570   /**
571    * @param f float value
572    * @return the float represented as byte []
573    */
574   public static byte [] toBytes(final float f) {
575     // Encode it as int
576     return Bytes.toBytes(Float.floatToRawIntBits(f));
577   }
578 
579   /**
580    * @param bytes byte array
581    * @return Return double made from passed bytes.
582    */
583   public static double toDouble(final byte [] bytes) {
584     return toDouble(bytes, 0);
585   }
586 
587   /**
588    * @param bytes byte array
589    * @param offset offset where double is
590    * @return Return double made from passed bytes.
591    */
592   public static double toDouble(final byte [] bytes, final int offset) {
593     return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
594   }
595 
596   /**
597    * @param bytes byte array
598    * @param offset offset to write to
599    * @param d value
600    * @return New offset into array <code>bytes</code>
601    */
602   public static int putDouble(byte [] bytes, int offset, double d) {
603     return putLong(bytes, offset, Double.doubleToLongBits(d));
604   }
605 
606   /**
607    * Serialize a double as the IEEE 754 double format output. The resultant
608    * array will be 8 bytes long.
609    *
610    * @param d value
611    * @return the double represented as byte []
612    */
613   public static byte [] toBytes(final double d) {
614     // Encode it as a long
615     return Bytes.toBytes(Double.doubleToRawLongBits(d));
616   }
617 
618   /**
619    * Convert an int value to a byte array
620    * @param val value
621    * @return the byte array
622    */
623   public static byte[] toBytes(int val) {
624     byte [] b = new byte[4];
625     for(int i = 3; i > 0; i--) {
626       b[i] = (byte) val;
627       val >>>= 8;
628     }
629     b[0] = (byte) val;
630     return b;
631   }
632 
633   /**
634    * Converts a byte array to an int value
635    * @param bytes byte array
636    * @return the int value
637    */
638   public static int toInt(byte[] bytes) {
639     return toInt(bytes, 0, SIZEOF_INT);
640   }
641 
642   /**
643    * Converts a byte array to an int value
644    * @param bytes byte array
645    * @param offset offset into array
646    * @return the int value
647    */
648   public static int toInt(byte[] bytes, int offset) {
649     return toInt(bytes, offset, SIZEOF_INT);
650   }
651 
652   /**
653    * Converts a byte array to an int value
654    * @param bytes byte array
655    * @param offset offset into array
656    * @param length length of int (has to be {@link #SIZEOF_INT})
657    * @return the int value
658    * @throws IllegalArgumentException if length is not {@link #SIZEOF_INT} or
659    * if there's not enough room in the array at the offset indicated.
660    */
661   public static int toInt(byte[] bytes, int offset, final int length) {
662     if (length != SIZEOF_INT || offset + length > bytes.length) {
663       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
664     }
665     int n = 0;
666     for(int i = offset; i < (offset + length); i++) {
667       n <<= 8;
668       n ^= bytes[i] & 0xFF;
669     }
670     return n;
671   }
672 
673   /**
674    * Put an int value out to the specified byte array position.
675    * @param bytes the byte array
676    * @param offset position in the array
677    * @param val int to write out
678    * @return incremented offset
679    * @throws IllegalArgumentException if the byte array given doesn't have
680    * enough room at the offset specified.
681    */
682   public static int putInt(byte[] bytes, int offset, int val) {
683     if (bytes.length - offset < SIZEOF_INT) {
684       throw new IllegalArgumentException("Not enough room to put an int at"
685           + " offset " + offset + " in a " + bytes.length + " byte array");
686     }
687     for(int i= offset + 3; i > offset; i--) {
688       bytes[i] = (byte) val;
689       val >>>= 8;
690     }
691     bytes[offset] = (byte) val;
692     return offset + SIZEOF_INT;
693   }
694 
695   /**
696    * Convert a short value to a byte array of {@link #SIZEOF_SHORT} bytes long.
697    * @param val value
698    * @return the byte array
699    */
700   public static byte[] toBytes(short val) {
701     byte[] b = new byte[SIZEOF_SHORT];
702     b[1] = (byte) val;
703     val >>= 8;
704     b[0] = (byte) val;
705     return b;
706   }
707 
708   /**
709    * Converts a byte array to a short value
710    * @param bytes byte array
711    * @return the short value
712    */
713   public static short toShort(byte[] bytes) {
714     return toShort(bytes, 0, SIZEOF_SHORT);
715   }
716 
717   /**
718    * Converts a byte array to a short value
719    * @param bytes byte array
720    * @param offset offset into array
721    * @return the short value
722    */
723   public static short toShort(byte[] bytes, int offset) {
724     return toShort(bytes, offset, SIZEOF_SHORT);
725   }
726 
727   /**
728    * Converts a byte array to a short value
729    * @param bytes byte array
730    * @param offset offset into array
731    * @param length length, has to be {@link #SIZEOF_SHORT}
732    * @return the short value
733    * @throws IllegalArgumentException if length is not {@link #SIZEOF_SHORT}
734    * or if there's not enough room in the array at the offset indicated.
735    */
736   public static short toShort(byte[] bytes, int offset, final int length) {
737     if (length != SIZEOF_SHORT || offset + length > bytes.length) {
738       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
739     }
740     short n = 0;
741     n ^= bytes[offset] & 0xFF;
742     n <<= 8;
743     n ^= bytes[offset+1] & 0xFF;
744     return n;
745   }
746 
747   /**
748    * This method will get a sequence of bytes from pos -> limit,
749    * but will restore pos after.
750    * @param buf
751    * @return byte array
752    */
753   public static byte[] getBytes(ByteBuffer buf) {
754     int savedPos = buf.position();
755     byte [] newBytes = new byte[buf.remaining()];
756     buf.get(newBytes);
757     buf.position(savedPos);
758     return newBytes;
759   }
760 
761   /**
762    * Put a short value out to the specified byte array position.
763    * @param bytes the byte array
764    * @param offset position in the array
765    * @param val short to write out
766    * @return incremented offset
767    * @throws IllegalArgumentException if the byte array given doesn't have
768    * enough room at the offset specified.
769    */
770   public static int putShort(byte[] bytes, int offset, short val) {
771     if (bytes.length - offset < SIZEOF_SHORT) {
772       throw new IllegalArgumentException("Not enough room to put a short at"
773           + " offset " + offset + " in a " + bytes.length + " byte array");
774     }
775     bytes[offset+1] = (byte) val;
776     val >>= 8;
777     bytes[offset] = (byte) val;
778     return offset + SIZEOF_SHORT;
779   }
780 
781   /**
782    * Convert a BigDecimal value to a byte array
783    *
784    * @param val
785    * @return the byte array
786    */
787   public static byte[] toBytes(BigDecimal val) {
788     byte[] valueBytes = val.unscaledValue().toByteArray();
789     byte[] result = new byte[valueBytes.length + SIZEOF_INT];
790     int offset = putInt(result, 0, val.scale());
791     putBytes(result, offset, valueBytes, 0, valueBytes.length);
792     return result;
793   }
794 
795 
796   /**
797    * Converts a byte array to a BigDecimal
798    *
799    * @param bytes
800    * @return the char value
801    */
802   public static BigDecimal toBigDecimal(byte[] bytes) {
803     return toBigDecimal(bytes, 0, bytes.length);
804   }
805 
806   /**
807    * Converts a byte array to a BigDecimal value
808    *
809    * @param bytes
810    * @param offset
811    * @param length
812    * @return the char value
813    */
814   public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) {
815     if (bytes == null || length < SIZEOF_INT + 1 ||
816       (offset + length > bytes.length)) {
817       return null;
818     }
819 
820     int scale = toInt(bytes, offset);
821     byte[] tcBytes = new byte[length - SIZEOF_INT];
822     System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT);
823     return new BigDecimal(new BigInteger(tcBytes), scale);
824   }
825 
826   /**
827    * Put a BigDecimal value out to the specified byte array position.
828    *
829    * @param bytes  the byte array
830    * @param offset position in the array
831    * @param val    BigDecimal to write out
832    * @return incremented offset
833    */
834   public static int putBigDecimal(byte[] bytes, int offset, BigDecimal val) {
835     if (bytes == null) {
836       return offset;
837     }
838 
839     byte[] valueBytes = val.unscaledValue().toByteArray();
840     byte[] result = new byte[valueBytes.length + SIZEOF_INT];
841     offset = putInt(result, offset, val.scale());
842     return putBytes(result, offset, valueBytes, 0, valueBytes.length);
843   }
844 
845   /**
846    * @param vint Integer to make a vint of.
847    * @return Vint as bytes array.
848    */
849   public static byte [] vintToBytes(final long vint) {
850     long i = vint;
851     int size = WritableUtils.getVIntSize(i);
852     byte [] result = new byte[size];
853     int offset = 0;
854     if (i >= -112 && i <= 127) {
855       result[offset] = (byte) i;
856       return result;
857     }
858 
859     int len = -112;
860     if (i < 0) {
861       i ^= -1L; // take one's complement'
862       len = -120;
863     }
864 
865     long tmp = i;
866     while (tmp != 0) {
867       tmp = tmp >> 8;
868       len--;
869     }
870 
871     result[offset++] = (byte) len;
872 
873     len = (len < -120) ? -(len + 120) : -(len + 112);
874 
875     for (int idx = len; idx != 0; idx--) {
876       int shiftbits = (idx - 1) * 8;
877       long mask = 0xFFL << shiftbits;
878       result[offset++] = (byte)((i & mask) >> shiftbits);
879     }
880     return result;
881   }
882 
883   /**
884    * @param buffer buffer to convert
885    * @return vint bytes as an integer.
886    */
887   public static long bytesToVint(final byte [] buffer) {
888     int offset = 0;
889     byte firstByte = buffer[offset++];
890     int len = WritableUtils.decodeVIntSize(firstByte);
891     if (len == 1) {
892       return firstByte;
893     }
894     long i = 0;
895     for (int idx = 0; idx < len-1; idx++) {
896       byte b = buffer[offset++];
897       i = i << 8;
898       i = i | (b & 0xFF);
899     }
900     return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
901   }
902 
903   /**
904    * Reads a zero-compressed encoded long from input stream and returns it.
905    * @param buffer Binary array
906    * @param offset Offset into array at which vint begins.
907    * @throws java.io.IOException e
908    * @return deserialized long from stream.
909    */
910   public static long readVLong(final byte [] buffer, final int offset)
911   throws IOException {
912     byte firstByte = buffer[offset];
913     int len = WritableUtils.decodeVIntSize(firstByte);
914     if (len == 1) {
915       return firstByte;
916     }
917     long i = 0;
918     for (int idx = 0; idx < len-1; idx++) {
919       byte b = buffer[offset + 1 + idx];
920       i = i << 8;
921       i = i | (b & 0xFF);
922     }
923     return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
924   }
925 
926   /**
927    * @param left left operand
928    * @param right right operand
929    * @return 0 if equal, < 0 if left is less than right, etc.
930    */
931   public static int compareTo(final byte [] left, final byte [] right) {
932     return LexicographicalComparerHolder.BEST_COMPARER.
933       compareTo(left, 0, left.length, right, 0, right.length);
934   }
935 
936   /**
937    * Lexicographically compare two arrays.
938    *
939    * @param buffer1 left operand
940    * @param buffer2 right operand
941    * @param offset1 Where to start comparing in the left buffer
942    * @param offset2 Where to start comparing in the right buffer
943    * @param length1 How much to compare from the left buffer
944    * @param length2 How much to compare from the right buffer
945    * @return 0 if equal, < 0 if left is less than right, etc.
946    */
947   public static int compareTo(byte[] buffer1, int offset1, int length1,
948       byte[] buffer2, int offset2, int length2) {
949     return LexicographicalComparerHolder.BEST_COMPARER.
950       compareTo(buffer1, offset1, length1, buffer2, offset2, length2);
951   }
952 
953   interface Comparer<T> {
954     abstract public int compareTo(T buffer1, int offset1, int length1,
955         T buffer2, int offset2, int length2);
956   }
957 
958   @VisibleForTesting
959   static Comparer<byte[]> lexicographicalComparerJavaImpl() {
960     return LexicographicalComparerHolder.PureJavaComparer.INSTANCE;
961   }
962 
963   /**
964    * Provides a lexicographical comparer implementation; either a Java
965    * implementation or a faster implementation based on {@link Unsafe}.
966    *
967    * <p>Uses reflection to gracefully fall back to the Java implementation if
968    * {@code Unsafe} isn't available.
969    */
970   @VisibleForTesting
971   static class LexicographicalComparerHolder {
972     static final String UNSAFE_COMPARER_NAME =
973         LexicographicalComparerHolder.class.getName() + "$UnsafeComparer";
974 
975     static final Comparer<byte[]> BEST_COMPARER = getBestComparer();
976     /**
977      * Returns the Unsafe-using Comparer, or falls back to the pure-Java
978      * implementation if unable to do so.
979      */
980     static Comparer<byte[]> getBestComparer() {
981       try {
982         Class<?> theClass = Class.forName(UNSAFE_COMPARER_NAME);
983 
984         // yes, UnsafeComparer does implement Comparer<byte[]>
985         @SuppressWarnings("unchecked")
986         Comparer<byte[]> comparer =
987           (Comparer<byte[]>) theClass.getEnumConstants()[0];
988         return comparer;
989       } catch (Throwable t) { // ensure we really catch *everything*
990         return lexicographicalComparerJavaImpl();
991       }
992     }
993 
994     enum PureJavaComparer implements Comparer<byte[]> {
995       INSTANCE;
996 
997       @Override
998       public int compareTo(byte[] buffer1, int offset1, int length1,
999           byte[] buffer2, int offset2, int length2) {
1000         // Short circuit equal case
1001         if (buffer1 == buffer2 &&
1002             offset1 == offset2 &&
1003             length1 == length2) {
1004           return 0;
1005         }
1006         // Bring WritableComparator code local
1007         int end1 = offset1 + length1;
1008         int end2 = offset2 + length2;
1009         for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) {
1010           int a = (buffer1[i] & 0xff);
1011           int b = (buffer2[j] & 0xff);
1012           if (a != b) {
1013             return a - b;
1014           }
1015         }
1016         return length1 - length2;
1017       }
1018     }
1019 
1020     @VisibleForTesting
1021     enum UnsafeComparer implements Comparer<byte[]> {
1022       INSTANCE;
1023 
1024       static final Unsafe theUnsafe;
1025 
1026       /** The offset to the first element in a byte array. */
1027       static final int BYTE_ARRAY_BASE_OFFSET;
1028 
1029       static {
1030         theUnsafe = (Unsafe) AccessController.doPrivileged(
1031             new PrivilegedAction<Object>() {
1032               @Override
1033               public Object run() {
1034                 try {
1035                   Field f = Unsafe.class.getDeclaredField("theUnsafe");
1036                   f.setAccessible(true);
1037                   return f.get(null);
1038                 } catch (NoSuchFieldException e) {
1039                   // It doesn't matter what we throw;
1040                   // it's swallowed in getBestComparer().
1041                   throw new Error();
1042                 } catch (IllegalAccessException e) {
1043                   throw new Error();
1044                 }
1045               }
1046             });
1047 
1048         BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class);
1049 
1050         // sanity check - this should never fail
1051         if (theUnsafe.arrayIndexScale(byte[].class) != 1) {
1052           throw new AssertionError();
1053         }
1054       }
1055 
1056       static final boolean littleEndian =
1057         ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN);
1058 
1059       /**
1060        * Returns true if x1 is less than x2, when both values are treated as
1061        * unsigned.
1062        */
1063       static boolean lessThanUnsigned(long x1, long x2) {
1064         return (x1 + Long.MIN_VALUE) < (x2 + Long.MIN_VALUE);
1065       }
1066 
1067       /**
1068        * Lexicographically compare two arrays.
1069        *
1070        * @param buffer1 left operand
1071        * @param buffer2 right operand
1072        * @param offset1 Where to start comparing in the left buffer
1073        * @param offset2 Where to start comparing in the right buffer
1074        * @param length1 How much to compare from the left buffer
1075        * @param length2 How much to compare from the right buffer
1076        * @return 0 if equal, < 0 if left is less than right, etc.
1077        */
1078       @Override
1079       public int compareTo(byte[] buffer1, int offset1, int length1,
1080           byte[] buffer2, int offset2, int length2) {
1081         // Short circuit equal case
1082         if (buffer1 == buffer2 &&
1083             offset1 == offset2 &&
1084             length1 == length2) {
1085           return 0;
1086         }
1087         int minLength = Math.min(length1, length2);
1088         int minWords = minLength / SIZEOF_LONG;
1089         int offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET;
1090         int offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET;
1091 
1092         /*
1093          * Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a
1094          * time is no slower than comparing 4 bytes at a time even on 32-bit.
1095          * On the other hand, it is substantially faster on 64-bit.
1096          */
1097         for (int i = 0; i < minWords * SIZEOF_LONG; i += SIZEOF_LONG) {
1098           long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i);
1099           long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i);
1100           long diff = lw ^ rw;
1101 
1102           if (diff != 0) {
1103             if (!littleEndian) {
1104               return lessThanUnsigned(lw, rw) ? -1 : 1;
1105             }
1106 
1107             // Use binary search
1108             int n = 0;
1109             int y;
1110             int x = (int) diff;
1111             if (x == 0) {
1112               x = (int) (diff >>> 32);
1113               n = 32;
1114             }
1115 
1116             y = x << 16;
1117             if (y == 0) {
1118               n += 16;
1119             } else {
1120               x = y;
1121             }
1122 
1123             y = x << 8;
1124             if (y == 0) {
1125               n += 8;
1126             }
1127             return (int) (((lw >>> n) & 0xFFL) - ((rw >>> n) & 0xFFL));
1128           }
1129         }
1130 
1131         // The epilogue to cover the last (minLength % 8) elements.
1132         for (int i = minWords * SIZEOF_LONG; i < minLength; i++) {
1133           int a = (buffer1[offset1 + i] & 0xff);
1134           int b = (buffer2[offset2 + i] & 0xff);
1135           if (a != b) {
1136             return a - b;
1137           }
1138         }
1139         return length1 - length2;
1140       }
1141     }
1142   }
1143 
1144   /**
1145    * @param left left operand
1146    * @param right right operand
1147    * @return True if equal
1148    */
1149   public static boolean equals(final byte [] left, final byte [] right) {
1150     // Could use Arrays.equals?
1151     //noinspection SimplifiableConditionalExpression
1152     if (left == right) return true;
1153     if (left == null || right == null) return false;
1154     if (left.length != right.length) return false;
1155     if (left.length == 0) return true;
1156 
1157     // Since we're often comparing adjacent sorted data,
1158     // it's usual to have equal arrays except for the very last byte
1159     // so check that first
1160     if (left[left.length - 1] != right[right.length - 1]) return false;
1161 
1162     return compareTo(left, right) == 0;
1163   }
1164 
1165   public static boolean equals(final byte[] left, int leftOffset, int leftLen,
1166                                final byte[] right, int rightOffset, int rightLen) {
1167     // short circuit case
1168     if (left == right &&
1169         leftOffset == rightOffset &&
1170         leftLen == rightLen) {
1171       return true;
1172     }
1173     // different lengths fast check
1174     if (leftLen != rightLen) {
1175       return false;
1176     }
1177     if (leftLen == 0) {
1178       return true;
1179     }
1180 
1181     // Since we're often comparing adjacent sorted data,
1182     // it's usual to have equal arrays except for the very last byte
1183     // so check that first
1184     if (left[leftOffset + leftLen - 1] != right[rightOffset + rightLen - 1]) return false;
1185 
1186     return LexicographicalComparerHolder.BEST_COMPARER.
1187       compareTo(left, leftOffset, leftLen, right, rightOffset, rightLen) == 0;
1188   }
1189 
1190 
1191   /**
1192    * Return true if the byte array on the right is a prefix of the byte
1193    * array on the left.
1194    */
1195   public static boolean startsWith(byte[] bytes, byte[] prefix) {
1196     return bytes != null && prefix != null &&
1197       bytes.length >= prefix.length &&
1198       LexicographicalComparerHolder.BEST_COMPARER.
1199         compareTo(bytes, 0, prefix.length, prefix, 0, prefix.length) == 0;
1200   }
1201 
1202   /**
1203    * @param b bytes to hash
1204    * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the
1205    * passed in array.  This method is what {@link org.apache.hadoop.io.Text} and
1206    * {@link ImmutableBytesWritable} use calculating hash code.
1207    */
1208   public static int hashCode(final byte [] b) {
1209     return hashCode(b, b.length);
1210   }
1211 
1212   /**
1213    * @param b value
1214    * @param length length of the value
1215    * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the
1216    * passed in array.  This method is what {@link org.apache.hadoop.io.Text} and
1217    * {@link ImmutableBytesWritable} use calculating hash code.
1218    */
1219   public static int hashCode(final byte [] b, final int length) {
1220     return WritableComparator.hashBytes(b, length);
1221   }
1222 
1223   /**
1224    * @param b bytes to hash
1225    * @return A hash of <code>b</code> as an Integer that can be used as key in
1226    * Maps.
1227    */
1228   public static Integer mapKey(final byte [] b) {
1229     return hashCode(b);
1230   }
1231 
1232   /**
1233    * @param b bytes to hash
1234    * @param length length to hash
1235    * @return A hash of <code>b</code> as an Integer that can be used as key in
1236    * Maps.
1237    */
1238   public static Integer mapKey(final byte [] b, final int length) {
1239     return hashCode(b, length);
1240   }
1241 
1242   /**
1243    * @param a lower half
1244    * @param b upper half
1245    * @return New array that has a in lower half and b in upper half.
1246    */
1247   public static byte [] add(final byte [] a, final byte [] b) {
1248     return add(a, b, HConstants.EMPTY_BYTE_ARRAY);
1249   }
1250 
1251   /**
1252    * @param a first third
1253    * @param b second third
1254    * @param c third third
1255    * @return New array made from a, b and c
1256    */
1257   public static byte [] add(final byte [] a, final byte [] b, final byte [] c) {
1258     byte [] result = new byte[a.length + b.length + c.length];
1259     System.arraycopy(a, 0, result, 0, a.length);
1260     System.arraycopy(b, 0, result, a.length, b.length);
1261     System.arraycopy(c, 0, result, a.length + b.length, c.length);
1262     return result;
1263   }
1264 
1265   /**
1266    * @param a array
1267    * @param length amount of bytes to grab
1268    * @return First <code>length</code> bytes from <code>a</code>
1269    */
1270   public static byte [] head(final byte [] a, final int length) {
1271     if (a.length < length) {
1272       return null;
1273     }
1274     byte [] result = new byte[length];
1275     System.arraycopy(a, 0, result, 0, length);
1276     return result;
1277   }
1278 
1279   /**
1280    * @param a array
1281    * @param length amount of bytes to snarf
1282    * @return Last <code>length</code> bytes from <code>a</code>
1283    */
1284   public static byte [] tail(final byte [] a, final int length) {
1285     if (a.length < length) {
1286       return null;
1287     }
1288     byte [] result = new byte[length];
1289     System.arraycopy(a, a.length - length, result, 0, length);
1290     return result;
1291   }
1292 
1293   /**
1294    * @param a array
1295    * @param length new array size
1296    * @return Value in <code>a</code> plus <code>length</code> prepended 0 bytes
1297    */
1298   public static byte [] padHead(final byte [] a, final int length) {
1299     byte [] padding = new byte[length];
1300     for (int i = 0; i < length; i++) {
1301       padding[i] = 0;
1302     }
1303     return add(padding,a);
1304   }
1305 
1306   /**
1307    * @param a array
1308    * @param length new array size
1309    * @return Value in <code>a</code> plus <code>length</code> appended 0 bytes
1310    */
1311   public static byte [] padTail(final byte [] a, final int length) {
1312     byte [] padding = new byte[length];
1313     for (int i = 0; i < length; i++) {
1314       padding[i] = 0;
1315     }
1316     return add(a,padding);
1317   }
1318 
1319   /**
1320    * Split passed range.  Expensive operation relatively.  Uses BigInteger math.
1321    * Useful splitting ranges for MapReduce jobs.
1322    * @param a Beginning of range
1323    * @param b End of range
1324    * @param num Number of times to split range.  Pass 1 if you want to split
1325    * the range in two; i.e. one split.
1326    * @return Array of dividing values
1327    */
1328   public static byte [][] split(final byte [] a, final byte [] b, final int num) {
1329     return split(a, b, false, num);
1330   }
1331 
1332   /**
1333    * Split passed range.  Expensive operation relatively.  Uses BigInteger math.
1334    * Useful splitting ranges for MapReduce jobs.
1335    * @param a Beginning of range
1336    * @param b End of range
1337    * @param inclusive Whether the end of range is prefix-inclusive or is
1338    * considered an exclusive boundary.  Automatic splits are generally exclusive
1339    * and manual splits with an explicit range utilize an inclusive end of range.
1340    * @param num Number of times to split range.  Pass 1 if you want to split
1341    * the range in two; i.e. one split.
1342    * @return Array of dividing values
1343    */
1344   public static byte[][] split(final byte[] a, final byte[] b,
1345       boolean inclusive, final int num) {
1346     byte[][] ret = new byte[num + 2][];
1347     int i = 0;
1348     Iterable<byte[]> iter = iterateOnSplits(a, b, inclusive, num);
1349     if (iter == null)
1350       return null;
1351     for (byte[] elem : iter) {
1352       ret[i++] = elem;
1353     }
1354     return ret;
1355   }
1356 
1357   /**
1358    * Iterate over keys within the passed range, splitting at an [a,b) boundary.
1359    */
1360   public static Iterable<byte[]> iterateOnSplits(final byte[] a,
1361       final byte[] b, final int num)
1362   {
1363     return iterateOnSplits(a, b, false, num);
1364   }
1365 
1366   /**
1367    * Iterate over keys within the passed range.
1368    */
1369   public static Iterable<byte[]> iterateOnSplits(
1370       final byte[] a, final byte[]b, boolean inclusive, final int num)
1371   {
1372     byte [] aPadded;
1373     byte [] bPadded;
1374     if (a.length < b.length) {
1375       aPadded = padTail(a, b.length - a.length);
1376       bPadded = b;
1377     } else if (b.length < a.length) {
1378       aPadded = a;
1379       bPadded = padTail(b, a.length - b.length);
1380     } else {
1381       aPadded = a;
1382       bPadded = b;
1383     }
1384     if (compareTo(aPadded,bPadded) >= 0) {
1385       throw new IllegalArgumentException("b <= a");
1386     }
1387     if (num <= 0) {
1388       throw new IllegalArgumentException("num cannot be < 0");
1389     }
1390     byte [] prependHeader = {1, 0};
1391     final BigInteger startBI = new BigInteger(add(prependHeader, aPadded));
1392     final BigInteger stopBI = new BigInteger(add(prependHeader, bPadded));
1393     BigInteger diffBI = stopBI.subtract(startBI);
1394     if (inclusive) {
1395       diffBI = diffBI.add(BigInteger.ONE);
1396     }
1397     final BigInteger splitsBI = BigInteger.valueOf(num + 1);
1398     if(diffBI.compareTo(splitsBI) < 0) {
1399       return null;
1400     }
1401     final BigInteger intervalBI;
1402     try {
1403       intervalBI = diffBI.divide(splitsBI);
1404     } catch(Exception e) {
1405       LOG.error("Exception caught during division", e);
1406       return null;
1407     }
1408 
1409     final Iterator<byte[]> iterator = new Iterator<byte[]>() {
1410       private int i = -1;
1411 
1412       @Override
1413       public boolean hasNext() {
1414         return i < num+1;
1415       }
1416 
1417       @Override
1418       public byte[] next() {
1419         i++;
1420         if (i == 0) return a;
1421         if (i == num + 1) return b;
1422 
1423         BigInteger curBI = startBI.add(intervalBI.multiply(BigInteger.valueOf(i)));
1424         byte [] padded = curBI.toByteArray();
1425         if (padded[1] == 0)
1426           padded = tail(padded, padded.length - 2);
1427         else
1428           padded = tail(padded, padded.length - 1);
1429         return padded;
1430       }
1431 
1432       @Override
1433       public void remove() {
1434         throw new UnsupportedOperationException();
1435       }
1436 
1437     };
1438 
1439     return new Iterable<byte[]>() {
1440       @Override
1441       public Iterator<byte[]> iterator() {
1442         return iterator;
1443       }
1444     };
1445   }
1446 
1447   /**
1448    * @param bytes array to hash
1449    * @param offset offset to start from
1450    * @param length length to hash
1451    * */
1452   public static int hashCode(byte[] bytes, int offset, int length) {
1453     int hash = 1;
1454     for (int i = offset; i < offset + length; i++)
1455       hash = (31 * hash) + (int) bytes[i];
1456     return hash;
1457   }
1458 
1459   /**
1460    * @param t operands
1461    * @return Array of byte arrays made from passed array of Text
1462    */
1463   public static byte [][] toByteArrays(final String [] t) {
1464     byte [][] result = new byte[t.length][];
1465     for (int i = 0; i < t.length; i++) {
1466       result[i] = Bytes.toBytes(t[i]);
1467     }
1468     return result;
1469   }
1470 
1471   /**
1472    * @param column operand
1473    * @return A byte array of a byte array where first and only entry is
1474    * <code>column</code>
1475    */
1476   public static byte [][] toByteArrays(final String column) {
1477     return toByteArrays(toBytes(column));
1478   }
1479 
1480   /**
1481    * @param column operand
1482    * @return A byte array of a byte array where first and only entry is
1483    * <code>column</code>
1484    */
1485   public static byte [][] toByteArrays(final byte [] column) {
1486     byte [][] result = new byte[1][];
1487     result[0] = column;
1488     return result;
1489   }
1490 
1491   /**
1492    * Binary search for keys in indexes.
1493    *
1494    * @param arr array of byte arrays to search for
1495    * @param key the key you want to find
1496    * @param offset the offset in the key you want to find
1497    * @param length the length of the key
1498    * @param comparator a comparator to compare.
1499    * @return zero-based index of the key, if the key is present in the array.
1500    *         Otherwise, a value -(i + 1) such that the key is between arr[i -
1501    *         1] and arr[i] non-inclusively, where i is in [0, i], if we define
1502    *         arr[-1] = -Inf and arr[N] = Inf for an N-element array. The above
1503    *         means that this function can return 2N + 1 different values
1504    *         ranging from -(N + 1) to N - 1.
1505    */
1506   public static int binarySearch(byte [][]arr, byte []key, int offset,
1507       int length, RawComparator<byte []> comparator) {
1508     int low = 0;
1509     int high = arr.length - 1;
1510 
1511     while (low <= high) {
1512       int mid = (low+high) >>> 1;
1513       // we have to compare in this order, because the comparator order
1514       // has special logic when the 'left side' is a special key.
1515       int cmp = comparator.compare(key, offset, length,
1516           arr[mid], 0, arr[mid].length);
1517       // key lives above the midpoint
1518       if (cmp > 0)
1519         low = mid + 1;
1520       // key lives below the midpoint
1521       else if (cmp < 0)
1522         high = mid - 1;
1523       // BAM. how often does this really happen?
1524       else
1525         return mid;
1526     }
1527     return - (low+1);
1528   }
1529 
1530   /**
1531    * Bytewise binary increment/deincrement of long contained in byte array
1532    * on given amount.
1533    *
1534    * @param value - array of bytes containing long (length <= SIZEOF_LONG)
1535    * @param amount value will be incremented on (deincremented if negative)
1536    * @return array of bytes containing incremented long (length == SIZEOF_LONG)
1537    */
1538   public static byte [] incrementBytes(byte[] value, long amount)
1539   {
1540     byte[] val = value;
1541     if (val.length < SIZEOF_LONG) {
1542       // Hopefully this doesn't happen too often.
1543       byte [] newvalue;
1544       if (val[0] < 0) {
1545         newvalue = new byte[]{-1, -1, -1, -1, -1, -1, -1, -1};
1546       } else {
1547         newvalue = new byte[SIZEOF_LONG];
1548       }
1549       System.arraycopy(val, 0, newvalue, newvalue.length - val.length,
1550         val.length);
1551       val = newvalue;
1552     } else if (val.length > SIZEOF_LONG) {
1553       throw new IllegalArgumentException("Increment Bytes - value too big: " +
1554         val.length);
1555     }
1556     if(amount == 0) return val;
1557     if(val[0] < 0){
1558       return binaryIncrementNeg(val, amount);
1559     }
1560     return binaryIncrementPos(val, amount);
1561   }
1562 
1563   /* increment/deincrement for positive value */
1564   private static byte [] binaryIncrementPos(byte [] value, long amount) {
1565     long amo = amount;
1566     int sign = 1;
1567     if (amount < 0) {
1568       amo = -amount;
1569       sign = -1;
1570     }
1571     for(int i=0;i<value.length;i++) {
1572       int cur = ((int)amo % 256) * sign;
1573       amo = (amo >> 8);
1574       int val = value[value.length-i-1] & 0x0ff;
1575       int total = val + cur;
1576       if(total > 255) {
1577         amo += sign;
1578         total %= 256;
1579       } else if (total < 0) {
1580         amo -= sign;
1581       }
1582       value[value.length-i-1] = (byte)total;
1583       if (amo == 0) return value;
1584     }
1585     return value;
1586   }
1587 
1588   /* increment/deincrement for negative value */
1589   private static byte [] binaryIncrementNeg(byte [] value, long amount) {
1590     long amo = amount;
1591     int sign = 1;
1592     if (amount < 0) {
1593       amo = -amount;
1594       sign = -1;
1595     }
1596     for(int i=0;i<value.length;i++) {
1597       int cur = ((int)amo % 256) * sign;
1598       amo = (amo >> 8);
1599       int val = ((~value[value.length-i-1]) & 0x0ff) + 1;
1600       int total = cur - val;
1601       if(total >= 0) {
1602         amo += sign;
1603       } else if (total < -256) {
1604         amo -= sign;
1605         total %= 256;
1606       }
1607       value[value.length-i-1] = (byte)total;
1608       if (amo == 0) return value;
1609     }
1610     return value;
1611   }
1612 
1613   /**
1614    * Writes a string as a fixed-size field, padded with zeros.
1615    */
1616   public static void writeStringFixedSize(final DataOutput out, String s,
1617       int size) throws IOException {
1618     byte[] b = toBytes(s);
1619     if (b.length > size) {
1620       throw new IOException("Trying to write " + b.length + " bytes (" +
1621           toStringBinary(b) + ") into a field of length " + size);
1622     }
1623 
1624     out.writeBytes(s);
1625     for (int i = 0; i < size - s.length(); ++i)
1626       out.writeByte(0);
1627   }
1628 
1629   /**
1630    * Reads a fixed-size field and interprets it as a string padded with zeros.
1631    */
1632   public static String readStringFixedSize(final DataInput in, int size)
1633       throws IOException {
1634     byte[] b = new byte[size];
1635     in.readFully(b);
1636     int n = b.length;
1637     while (n > 0 && b[n - 1] == 0)
1638       --n;
1639 
1640     return toString(b, 0, n);
1641   }
1642 
1643 }