View Javadoc

1   /**
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase;
21  
22  import static org.apache.hadoop.hbase.util.Bytes.len;
23  
24  import java.io.DataInput;
25  import java.io.DataOutput;
26  import java.io.IOException;
27  import java.io.InputStream;
28  import java.io.OutputStream;
29  import java.nio.ByteBuffer;
30  import java.util.ArrayList;
31  import java.util.Arrays;
32  import java.util.Comparator;
33  import java.util.HashMap;
34  import java.util.List;
35  import java.util.Map;
36  
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.hadoop.hbase.classification.InterfaceAudience;
40  import org.apache.hadoop.hbase.io.HeapSize;
41  import org.apache.hadoop.hbase.util.Bytes;
42  import org.apache.hadoop.hbase.util.ClassSize;
43  import org.apache.hadoop.io.IOUtils;
44  import org.apache.hadoop.io.RawComparator;
45  
46  import com.google.common.primitives.Longs;
47  
48  /**
49   * An HBase Key/Value. This is the fundamental HBase Type.  
50   * <p>
51   * HBase applications and users should use the Cell interface and avoid directly using KeyValue
52   * and member functions not defined in Cell.
53   * <p>
54   * If being used client-side, the primary methods to access individual fields are {@link #getRow()},
55   * {@link #getFamily()}, {@link #getQualifier()}, {@link #getTimestamp()}, and {@link #getValue()}.
56   * These methods allocate new byte arrays and return copies. Avoid their use server-side.
57   * <p>
58   * Instances of this class are immutable. They do not implement Comparable but Comparators are
59   * provided. Comparators change with context, whether user table or a catalog table comparison. Its
60   * critical you use the appropriate comparator. There are Comparators for normal HFiles, Meta's
61   * Hfiles, and bloom filter keys.
62   * <p>
63   * KeyValue wraps a byte array and takes offsets and lengths into passed array at where to start
64   * interpreting the content as KeyValue. The KeyValue format inside a byte array is:
65   * <code>&lt;keylength> &lt;valuelength> &lt;key> &lt;value></code> Key is further decomposed as:
66   * <code>&lt;rowlength> &lt;row> &lt;columnfamilylength> &lt;columnfamily> &lt;columnqualifier>
67   * &lt;timestamp> &lt;keytype></code>
68   * The <code>rowlength</code> maximum is <code>Short.MAX_SIZE</code>, column family length maximum
69   * is <code>Byte.MAX_SIZE</code>, and column qualifier + key length must be <
70   * <code>Integer.MAX_SIZE</code>. The column does not contain the family/qualifier delimiter,
71   * {@link #COLUMN_FAMILY_DELIMITER}<br>
72   * KeyValue can optionally contain Tags. When it contains tags, it is added in the byte array after
73   * the value part. The format for this part is: <code>&lt;tagslength>&lt;tagsbytes></code>.
74   * <code>tagslength</code> maximum is <code>Short.MAX_SIZE</code>. The <code>tagsbytes</code>
75   * contain one or more tags where as each tag is of the form
76   * <code>&lt;taglength>&lt;tagtype>&lt;tagbytes></code>.  <code>tagtype</code> is one byte and
77   * <code>taglength</code> maximum is <code>Short.MAX_SIZE</code> and it includes 1 byte type length
78   * and actual tag bytes length.
79   */
80  @InterfaceAudience.Private
81  public class KeyValue implements Cell, HeapSize, Cloneable {
82    private static final ArrayList<Tag> EMPTY_ARRAY_LIST = new ArrayList<Tag>();
83  
84    static final Log LOG = LogFactory.getLog(KeyValue.class);
85  
86    /**
87     * Colon character in UTF-8
88     */
89    public static final char COLUMN_FAMILY_DELIMITER = ':';
90  
91    public static final byte[] COLUMN_FAMILY_DELIM_ARRAY =
92      new byte[]{COLUMN_FAMILY_DELIMITER};
93  
94    /**
95     * Comparator for plain key/values; i.e. non-catalog table key/values. Works on Key portion
96     * of KeyValue only.
97     */
98    public static final KVComparator COMPARATOR = new KVComparator();
99    /**
100    * A {@link KVComparator} for <code>hbase:meta</code> catalog table
101    * {@link KeyValue}s.
102    */
103   public static final KVComparator META_COMPARATOR = new MetaComparator();
104 
105   /**
106    * Needed for Bloom Filters.
107    */
108   public static final KVComparator RAW_COMPARATOR = new RawBytesComparator();
109 
110   /** Size of the key length field in bytes*/
111   public static final int KEY_LENGTH_SIZE = Bytes.SIZEOF_INT;
112 
113   /** Size of the key type field in bytes */
114   public static final int TYPE_SIZE = Bytes.SIZEOF_BYTE;
115 
116   /** Size of the row length field in bytes */
117   public static final int ROW_LENGTH_SIZE = Bytes.SIZEOF_SHORT;
118 
119   /** Size of the family length field in bytes */
120   public static final int FAMILY_LENGTH_SIZE = Bytes.SIZEOF_BYTE;
121 
122   /** Size of the timestamp field in bytes */
123   public static final int TIMESTAMP_SIZE = Bytes.SIZEOF_LONG;
124 
125   // Size of the timestamp and type byte on end of a key -- a long + a byte.
126   public static final int TIMESTAMP_TYPE_SIZE = TIMESTAMP_SIZE + TYPE_SIZE;
127 
128   // Size of the length shorts and bytes in key.
129   public static final int KEY_INFRASTRUCTURE_SIZE = ROW_LENGTH_SIZE
130       + FAMILY_LENGTH_SIZE + TIMESTAMP_TYPE_SIZE;
131 
132   // How far into the key the row starts at. First thing to read is the short
133   // that says how long the row is.
134   public static final int ROW_OFFSET =
135     Bytes.SIZEOF_INT /*keylength*/ +
136     Bytes.SIZEOF_INT /*valuelength*/;
137 
138   // Size of the length ints in a KeyValue datastructure.
139   public static final int KEYVALUE_INFRASTRUCTURE_SIZE = ROW_OFFSET;
140 
141   /** Size of the tags length field in bytes */
142   public static final int TAGS_LENGTH_SIZE = Bytes.SIZEOF_SHORT;
143 
144   public static final int KEYVALUE_WITH_TAGS_INFRASTRUCTURE_SIZE = ROW_OFFSET + TAGS_LENGTH_SIZE;
145 
146   private static final int MAX_TAGS_LENGTH = (2 * Short.MAX_VALUE) + 1;
147 
148   /**
149    * Computes the number of bytes that a <code>KeyValue</code> instance with the provided
150    * characteristics would take up for its underlying data structure.
151    *
152    * @param rlength row length
153    * @param flength family length
154    * @param qlength qualifier length
155    * @param vlength value length
156    *
157    * @return the <code>KeyValue</code> data structure length
158    */
159   public static long getKeyValueDataStructureSize(int rlength,
160       int flength, int qlength, int vlength) {
161     return KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE
162         + getKeyDataStructureSize(rlength, flength, qlength) + vlength;
163   }
164 
165   /**
166    * Computes the number of bytes that a <code>KeyValue</code> instance with the provided
167    * characteristics would take up for its underlying data structure.
168    *
169    * @param rlength row length
170    * @param flength family length
171    * @param qlength qualifier length
172    * @param vlength value length
173    * @param tagsLength total length of the tags
174    *
175    * @return the <code>KeyValue</code> data structure length
176    */
177   public static long getKeyValueDataStructureSize(int rlength, int flength, int qlength,
178       int vlength, int tagsLength) {
179     if (tagsLength == 0) {
180       return getKeyValueDataStructureSize(rlength, flength, qlength, vlength);
181     }
182     return KeyValue.KEYVALUE_WITH_TAGS_INFRASTRUCTURE_SIZE
183         + getKeyDataStructureSize(rlength, flength, qlength) + vlength + tagsLength;
184   }
185 
186   /**
187    * Computes the number of bytes that a <code>KeyValue</code> instance with the provided
188    * characteristics would take up for its underlying data structure.
189    *
190    * @param klength key length
191    * @param vlength value length
192    * @param tagsLength total length of the tags
193    *
194    * @return the <code>KeyValue</code> data structure length
195    */
196   public static long getKeyValueDataStructureSize(int klength, int vlength, int tagsLength) {
197     if (tagsLength == 0) {
198       return KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE + klength + vlength;
199     }
200     return KeyValue.KEYVALUE_WITH_TAGS_INFRASTRUCTURE_SIZE + klength + vlength + tagsLength;
201   }
202 
203   /**
204    * Computes the number of bytes that a <code>KeyValue</code> instance with the provided
205    * characteristics would take up in its underlying data structure for the key.
206    *
207    * @param rlength row length
208    * @param flength family length
209    * @param qlength qualifier length
210    *
211    * @return the key data structure length
212    */
213   public static long getKeyDataStructureSize(int rlength, int flength, int qlength) {
214     return KeyValue.KEY_INFRASTRUCTURE_SIZE + rlength + flength + qlength;
215   }
216 
217   /**
218    * Key type.
219    * Has space for other key types to be added later.  Cannot rely on
220    * enum ordinals . They change if item is removed or moved.  Do our own codes.
221    */
222   public static enum Type {
223     Minimum((byte)0),
224     Put((byte)4),
225 
226     Delete((byte)8),
227     DeleteFamilyVersion((byte)10),
228     DeleteColumn((byte)12),
229     DeleteFamily((byte)14),
230 
231     // Maximum is used when searching; you look from maximum on down.
232     Maximum((byte)255);
233 
234     private final byte code;
235 
236     Type(final byte c) {
237       this.code = c;
238     }
239 
240     public byte getCode() {
241       return this.code;
242     }
243 
244     /**
245      * Cannot rely on enum ordinals . They change if item is removed or moved.
246      * Do our own codes.
247      * @param b
248      * @return Type associated with passed code.
249      */
250     public static Type codeToType(final byte b) {
251       for (Type t : Type.values()) {
252         if (t.getCode() == b) {
253           return t;
254         }
255       }
256       throw new RuntimeException("Unknown code " + b);
257     }
258   }
259 
260   /**
261    * Lowest possible key.
262    * Makes a Key with highest possible Timestamp, empty row and column.  No
263    * key can be equal or lower than this one in memstore or in store file.
264    */
265   public static final KeyValue LOWESTKEY =
266     new KeyValue(HConstants.EMPTY_BYTE_ARRAY, HConstants.LATEST_TIMESTAMP);
267 
268   ////
269   // KeyValue core instance fields.
270   private byte [] bytes = null;  // an immutable byte array that contains the KV
271   private int offset = 0;  // offset into bytes buffer KV starts at
272   private int length = 0;  // length of the KV starting from offset.
273 
274   /**
275    * @return True if a delete type, a {@link KeyValue.Type#Delete} or
276    * a {KeyValue.Type#DeleteFamily} or a {@link KeyValue.Type#DeleteColumn}
277    * KeyValue type.
278    */
279   public static boolean isDelete(byte t) {
280     return Type.Delete.getCode() <= t && t <= Type.DeleteFamily.getCode();
281   }
282 
283   /** Here be dragons **/
284 
285   // used to achieve atomic operations in the memstore.
286   @Override
287   public long getMvccVersion() {
288     return mvcc;
289   }
290 
291   public void setMvccVersion(long mvccVersion){
292     this.mvcc = mvccVersion;
293   }
294 
295   // multi-version concurrency control version.  default value is 0, aka do not care.
296   private long mvcc = 0;  // this value is not part of a serialized KeyValue (not in HFiles)
297 
298   /** Dragon time over, return to normal business */
299 
300 
301   /** Writable Constructor -- DO NOT USE */
302   public KeyValue() {}
303 
304   /**
305    * Creates a KeyValue from the start of the specified byte array.
306    * Presumes <code>bytes</code> content is formatted as a KeyValue blob.
307    * @param bytes byte array
308    */
309   public KeyValue(final byte [] bytes) {
310     this(bytes, 0);
311   }
312 
313   /**
314    * Creates a KeyValue from the specified byte array and offset.
315    * Presumes <code>bytes</code> content starting at <code>offset</code> is
316    * formatted as a KeyValue blob.
317    * @param bytes byte array
318    * @param offset offset to start of KeyValue
319    */
320   public KeyValue(final byte [] bytes, final int offset) {
321     this(bytes, offset, getLength(bytes, offset));
322   }
323 
324   /**
325    * Creates a KeyValue from the specified byte array, starting at offset, and
326    * for length <code>length</code>.
327    * @param bytes byte array
328    * @param offset offset to start of the KeyValue
329    * @param length length of the KeyValue
330    */
331   public KeyValue(final byte [] bytes, final int offset, final int length) {
332     this.bytes = bytes;
333     this.offset = offset;
334     this.length = length;
335   }
336 
337   /**
338    * Creates a KeyValue from the specified byte array, starting at offset, and
339    * for length <code>length</code>.
340    *
341    * @param bytes  byte array
342    * @param offset offset to start of the KeyValue
343    * @param length length of the KeyValue
344    * @param ts
345    */
346   public KeyValue(final byte[] bytes, final int offset, final int length, long ts) {
347     this(bytes, offset, length, null, 0, 0, null, 0, 0, ts, Type.Maximum, null, 0, 0, null);
348   }
349 
350   /** Constructors that build a new backing byte array from fields */
351 
352   /**
353    * Constructs KeyValue structure filled with null value.
354    * Sets type to {@link KeyValue.Type#Maximum}
355    * @param row - row key (arbitrary byte array)
356    * @param timestamp
357    */
358   public KeyValue(final byte [] row, final long timestamp) {
359     this(row, null, null, timestamp, Type.Maximum, null);
360   }
361 
362   /**
363    * Constructs KeyValue structure filled with null value.
364    * @param row - row key (arbitrary byte array)
365    * @param timestamp
366    */
367   public KeyValue(final byte [] row, final long timestamp, Type type) {
368     this(row, null, null, timestamp, type, null);
369   }
370 
371   /**
372    * Constructs KeyValue structure filled with null value.
373    * Sets type to {@link KeyValue.Type#Maximum}
374    * @param row - row key (arbitrary byte array)
375    * @param family family name
376    * @param qualifier column qualifier
377    */
378   public KeyValue(final byte [] row, final byte [] family,
379       final byte [] qualifier) {
380     this(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Maximum);
381   }
382 
383   /**
384    * Constructs KeyValue structure filled with null value.
385    * @param row - row key (arbitrary byte array)
386    * @param family family name
387    * @param qualifier column qualifier
388    */
389   public KeyValue(final byte [] row, final byte [] family,
390       final byte [] qualifier, final byte [] value) {
391     this(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Put, value);
392   }
393 
394   /**
395    * Constructs KeyValue structure filled with specified values.
396    * @param row row key
397    * @param family family name
398    * @param qualifier column qualifier
399    * @param timestamp version timestamp
400    * @param type key type
401    * @throws IllegalArgumentException
402    */
403   public KeyValue(final byte[] row, final byte[] family,
404       final byte[] qualifier, final long timestamp, Type type) {
405     this(row, family, qualifier, timestamp, type, null);
406   }
407 
408   /**
409    * Constructs KeyValue structure filled with specified values.
410    * @param row row key
411    * @param family family name
412    * @param qualifier column qualifier
413    * @param timestamp version timestamp
414    * @param value column value
415    * @throws IllegalArgumentException
416    */
417   public KeyValue(final byte[] row, final byte[] family,
418       final byte[] qualifier, final long timestamp, final byte[] value) {
419     this(row, family, qualifier, timestamp, Type.Put, value);
420   }
421 
422   /**
423    * Constructs KeyValue structure filled with specified values.
424    * @param row row key
425    * @param family family name
426    * @param qualifier column qualifier
427    * @param timestamp version timestamp
428    * @param value column value
429    * @param tags tags
430    * @throws IllegalArgumentException
431    */
432   public KeyValue(final byte[] row, final byte[] family,
433       final byte[] qualifier, final long timestamp, final byte[] value,
434       final Tag[] tags) {
435     this(row, family, qualifier, timestamp, value, tags != null ? Arrays.asList(tags) : null);
436   }
437 
438   /**
439    * Constructs KeyValue structure filled with specified values.
440    * @param row row key
441    * @param family family name
442    * @param qualifier column qualifier
443    * @param timestamp version timestamp
444    * @param value column value
445    * @param tags tags non-empty list of tags or null
446    * @throws IllegalArgumentException
447    */
448   public KeyValue(final byte[] row, final byte[] family,
449       final byte[] qualifier, final long timestamp, final byte[] value,
450       final List<Tag> tags) {
451     this(row, 0, row==null ? 0 : row.length,
452       family, 0, family==null ? 0 : family.length,
453       qualifier, 0, qualifier==null ? 0 : qualifier.length,
454       timestamp, Type.Put,
455       value, 0, value==null ? 0 : value.length, tags);
456   }
457 
458   /**
459    * Constructs KeyValue structure filled with specified values.
460    * @param row row key
461    * @param family family name
462    * @param qualifier column qualifier
463    * @param timestamp version timestamp
464    * @param type key type
465    * @param value column value
466    * @throws IllegalArgumentException
467    */
468   public KeyValue(final byte[] row, final byte[] family,
469       final byte[] qualifier, final long timestamp, Type type,
470       final byte[] value) {
471     this(row, 0, len(row),   family, 0, len(family),   qualifier, 0, len(qualifier),
472         timestamp, type,   value, 0, len(value));
473   }
474 
475   /**
476    * Constructs KeyValue structure filled with specified values.
477    * <p>
478    * Column is split into two fields, family and qualifier.
479    * @param row row key
480    * @param family family name
481    * @param qualifier column qualifier
482    * @param timestamp version timestamp
483    * @param type key type
484    * @param value column value
485    * @throws IllegalArgumentException
486    */
487   public KeyValue(final byte[] row, final byte[] family,
488       final byte[] qualifier, final long timestamp, Type type,
489       final byte[] value, final List<Tag> tags) {
490     this(row, family, qualifier, 0, qualifier==null ? 0 : qualifier.length,
491         timestamp, type, value, 0, value==null ? 0 : value.length, tags);
492   }
493 
494   /**
495    * Constructs KeyValue structure filled with specified values.
496    * @param row row key
497    * @param family family name
498    * @param qualifier column qualifier
499    * @param timestamp version timestamp
500    * @param type key type
501    * @param value column value
502    * @throws IllegalArgumentException
503    */
504   public KeyValue(final byte[] row, final byte[] family,
505       final byte[] qualifier, final long timestamp, Type type,
506       final byte[] value, final byte[] tags) {
507     this(row, family, qualifier, 0, qualifier==null ? 0 : qualifier.length,
508         timestamp, type, value, 0, value==null ? 0 : value.length, tags);
509   }
510 
511   /**
512    * Constructs KeyValue structure filled with specified values.
513    * @param row row key
514    * @param family family name
515    * @param qualifier column qualifier
516    * @param qoffset qualifier offset
517    * @param qlength qualifier length
518    * @param timestamp version timestamp
519    * @param type key type
520    * @param value column value
521    * @param voffset value offset
522    * @param vlength value length
523    * @throws IllegalArgumentException
524    */
525   public KeyValue(byte [] row, byte [] family,
526       byte [] qualifier, int qoffset, int qlength, long timestamp, Type type,
527       byte [] value, int voffset, int vlength, List<Tag> tags) {
528     this(row, 0, row==null ? 0 : row.length,
529         family, 0, family==null ? 0 : family.length,
530         qualifier, qoffset, qlength, timestamp, type,
531         value, voffset, vlength, tags);
532   }
533 
534   /**
535    * @param row
536    * @param family
537    * @param qualifier
538    * @param qoffset
539    * @param qlength
540    * @param timestamp
541    * @param type
542    * @param value
543    * @param voffset
544    * @param vlength
545    * @param tags
546    */
547   public KeyValue(byte [] row, byte [] family,
548       byte [] qualifier, int qoffset, int qlength, long timestamp, Type type,
549       byte [] value, int voffset, int vlength, byte[] tags) {
550     this(row, 0, row==null ? 0 : row.length,
551         family, 0, family==null ? 0 : family.length,
552         qualifier, qoffset, qlength, timestamp, type,
553         value, voffset, vlength, tags, 0, tags==null ? 0 : tags.length);
554   }
555 
556   /**
557    * Constructs KeyValue structure filled with specified values.
558    * <p>
559    * Column is split into two fields, family and qualifier.
560    * @param row row key
561    * @throws IllegalArgumentException
562    */
563   public KeyValue(final byte [] row, final int roffset, final int rlength,
564       final byte [] family, final int foffset, final int flength,
565       final byte [] qualifier, final int qoffset, final int qlength,
566       final long timestamp, final Type type,
567       final byte [] value, final int voffset, final int vlength) {
568     this(row, roffset, rlength, family, foffset, flength, qualifier, qoffset,
569       qlength, timestamp, type, value, voffset, vlength, null);
570   }
571   
572   /**
573    * Constructs KeyValue structure filled with specified values. Uses the provided buffer as the
574    * data buffer.
575    * <p>
576    * Column is split into two fields, family and qualifier.
577    *
578    * @param buffer the bytes buffer to use
579    * @param boffset buffer offset
580    * @param row row key
581    * @param roffset row offset
582    * @param rlength row length
583    * @param family family name
584    * @param foffset family offset
585    * @param flength family length
586    * @param qualifier column qualifier
587    * @param qoffset qualifier offset
588    * @param qlength qualifier length
589    * @param timestamp version timestamp
590    * @param type key type
591    * @param value column value
592    * @param voffset value offset
593    * @param vlength value length
594    * @param tags non-empty list of tags or null
595    * @throws IllegalArgumentException an illegal value was passed or there is insufficient space
596    * remaining in the buffer
597    */
598   public KeyValue(byte [] buffer, final int boffset,
599       final byte [] row, final int roffset, final int rlength,
600       final byte [] family, final int foffset, final int flength,
601       final byte [] qualifier, final int qoffset, final int qlength,
602       final long timestamp, final Type type,
603       final byte [] value, final int voffset, final int vlength,
604       final Tag[] tags) {
605      this.bytes  = buffer;
606      this.length = writeByteArray(buffer, boffset,
607          row, roffset, rlength,
608          family, foffset, flength, qualifier, qoffset, qlength,
609         timestamp, type, value, voffset, vlength, tags);
610      this.offset = boffset;
611    }
612 
613   /**
614    * Constructs KeyValue structure filled with specified values.
615    * <p>
616    * Column is split into two fields, family and qualifier.
617    * @param row row key
618    * @param roffset row offset
619    * @param rlength row length
620    * @param family family name
621    * @param foffset family offset
622    * @param flength family length
623    * @param qualifier column qualifier
624    * @param qoffset qualifier offset
625    * @param qlength qualifier length
626    * @param timestamp version timestamp
627    * @param type key type
628    * @param value column value
629    * @param voffset value offset
630    * @param vlength value length
631    * @param tags tags
632    * @throws IllegalArgumentException
633    */
634   public KeyValue(final byte [] row, final int roffset, final int rlength,
635       final byte [] family, final int foffset, final int flength,
636       final byte [] qualifier, final int qoffset, final int qlength,
637       final long timestamp, final Type type,
638       final byte [] value, final int voffset, final int vlength,
639       final List<Tag> tags) {
640     this.bytes = createByteArray(row, roffset, rlength,
641         family, foffset, flength, qualifier, qoffset, qlength,
642         timestamp, type, value, voffset, vlength, tags);
643     this.length = bytes.length;
644     this.offset = 0;
645   }
646 
647   /**
648    * @param row
649    * @param roffset
650    * @param rlength
651    * @param family
652    * @param foffset
653    * @param flength
654    * @param qualifier
655    * @param qoffset
656    * @param qlength
657    * @param timestamp
658    * @param type
659    * @param value
660    * @param voffset
661    * @param vlength
662    * @param tags
663    */
664   public KeyValue(final byte [] row, final int roffset, final int rlength,
665       final byte [] family, final int foffset, final int flength,
666       final byte [] qualifier, final int qoffset, final int qlength,
667       final long timestamp, final Type type,
668       final byte [] value, final int voffset, final int vlength,
669       final byte[] tags, final int tagsOffset, final int tagsLength) {
670     this.bytes = createByteArray(row, roffset, rlength,
671         family, foffset, flength, qualifier, qoffset, qlength,
672         timestamp, type, value, voffset, vlength, tags, tagsOffset, tagsLength);
673     this.length = bytes.length;
674     this.offset = 0;
675   }
676 
677   /**
678    * Constructs an empty KeyValue structure, with specified sizes.
679    * This can be used to partially fill up KeyValues.
680    * <p>
681    * Column is split into two fields, family and qualifier.
682    * @param rlength row length
683    * @param flength family length
684    * @param qlength qualifier length
685    * @param timestamp version timestamp
686    * @param type key type
687    * @param vlength value length
688    * @throws IllegalArgumentException
689    */
690   public KeyValue(final int rlength,
691       final int flength,
692       final int qlength,
693       final long timestamp, final Type type,
694       final int vlength) {
695     this(rlength, flength, qlength, timestamp, type, vlength, 0);
696   }
697 
698   /**
699    * Constructs an empty KeyValue structure, with specified sizes.
700    * This can be used to partially fill up KeyValues.
701    * <p>
702    * Column is split into two fields, family and qualifier.
703    * @param rlength row length
704    * @param flength family length
705    * @param qlength qualifier length
706    * @param timestamp version timestamp
707    * @param type key type
708    * @param vlength value length
709    * @param tagsLength
710    * @throws IllegalArgumentException
711    */
712   public KeyValue(final int rlength,
713       final int flength,
714       final int qlength,
715       final long timestamp, final Type type,
716       final int vlength, final int tagsLength) {
717     this.bytes = createEmptyByteArray(rlength, flength, qlength, timestamp, type, vlength,
718         tagsLength);
719     this.length = bytes.length;
720     this.offset = 0;
721   }
722 
723 
724   public KeyValue(byte[] row, int roffset, int rlength,
725                   byte[] family, int foffset, int flength,
726                   ByteBuffer qualifier, long ts, Type type, ByteBuffer value, List<Tag> tags) {
727     this.bytes = createByteArray(row, roffset, rlength, family, foffset, flength,
728         qualifier, 0, qualifier == null ? 0 : qualifier.remaining(), ts, type,
729         value, 0, value == null ? 0 : value.remaining(), tags);
730     this.length = bytes.length;
731     this.offset = 0;
732   }
733 
734   public KeyValue(Cell c) {
735     this(c.getRowArray(), c.getRowOffset(), (int)c.getRowLength(),
736         c.getFamilyArray(), c.getFamilyOffset(), (int)c.getFamilyLength(), 
737         c.getQualifierArray(), c.getQualifierOffset(), (int) c.getQualifierLength(), 
738         c.getTimestamp(), Type.codeToType(c.getTypeByte()), c.getValueArray(), c.getValueOffset(), 
739         c.getValueLength(), c.getTagsArray(), c.getTagsOffset(), c.getTagsLengthUnsigned());
740   }
741 
742   /**
743    * Create an empty byte[] representing a KeyValue
744    * All lengths are preset and can be filled in later.
745    * @param rlength
746    * @param flength
747    * @param qlength
748    * @param timestamp
749    * @param type
750    * @param vlength
751    * @return The newly created byte array.
752    */
753   private static byte[] createEmptyByteArray(final int rlength, int flength,
754       int qlength, final long timestamp, final Type type, int vlength, int tagsLength) {
755     if (rlength > Short.MAX_VALUE) {
756       throw new IllegalArgumentException("Row > " + Short.MAX_VALUE);
757     }
758     if (flength > Byte.MAX_VALUE) {
759       throw new IllegalArgumentException("Family > " + Byte.MAX_VALUE);
760     }
761     // Qualifier length
762     if (qlength > Integer.MAX_VALUE - rlength - flength) {
763       throw new IllegalArgumentException("Qualifier > " + Integer.MAX_VALUE);
764     }
765     checkForTagsLength(tagsLength);
766     // Key length
767     long longkeylength = getKeyDataStructureSize(rlength, flength, qlength);
768     if (longkeylength > Integer.MAX_VALUE) {
769       throw new IllegalArgumentException("keylength " + longkeylength + " > " +
770         Integer.MAX_VALUE);
771     }
772     int keylength = (int)longkeylength;
773     // Value length
774     if (vlength > HConstants.MAXIMUM_VALUE_LENGTH) { // FindBugs INT_VACUOUS_COMPARISON
775       throw new IllegalArgumentException("Valuer > " +
776           HConstants.MAXIMUM_VALUE_LENGTH);
777     }
778 
779     // Allocate right-sized byte array.
780     byte[] bytes= new byte[(int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength,
781         tagsLength)];
782     // Write the correct size markers
783     int pos = 0;
784     pos = Bytes.putInt(bytes, pos, keylength);
785     pos = Bytes.putInt(bytes, pos, vlength);
786     pos = Bytes.putShort(bytes, pos, (short)(rlength & 0x0000ffff));
787     pos += rlength;
788     pos = Bytes.putByte(bytes, pos, (byte)(flength & 0x0000ff));
789     pos += flength + qlength;
790     pos = Bytes.putLong(bytes, pos, timestamp);
791     pos = Bytes.putByte(bytes, pos, type.getCode());
792     pos += vlength;
793     if (tagsLength > 0) {
794       pos = Bytes.putAsShort(bytes, pos, tagsLength);
795     }
796     return bytes;
797   }
798 
799   /**
800    * Checks the parameters passed to a constructor.
801    *
802    * @param row row key
803    * @param rlength row length
804    * @param family family name
805    * @param flength family length
806    * @param qlength qualifier length
807    * @param vlength value length
808    *
809    * @throws IllegalArgumentException an illegal value was passed
810    */
811   private static void checkParameters(final byte [] row, final int rlength,
812       final byte [] family, int flength, int qlength, int vlength)
813           throws IllegalArgumentException {
814     if (rlength > Short.MAX_VALUE) {
815       throw new IllegalArgumentException("Row > " + Short.MAX_VALUE);
816     }
817     if (row == null) {
818       throw new IllegalArgumentException("Row is null");
819     }
820     // Family length
821     flength = family == null ? 0 : flength;
822     if (flength > Byte.MAX_VALUE) {
823       throw new IllegalArgumentException("Family > " + Byte.MAX_VALUE);
824     }
825     // Qualifier length
826     if (qlength > Integer.MAX_VALUE - rlength - flength) {
827       throw new IllegalArgumentException("Qualifier > " + Integer.MAX_VALUE);
828     }
829     // Key length
830     long longKeyLength = getKeyDataStructureSize(rlength, flength, qlength);
831     if (longKeyLength > Integer.MAX_VALUE) {
832       throw new IllegalArgumentException("keylength " + longKeyLength + " > " +
833           Integer.MAX_VALUE);
834     }
835     // Value length
836     if (vlength > HConstants.MAXIMUM_VALUE_LENGTH) { // FindBugs INT_VACUOUS_COMPARISON
837       throw new IllegalArgumentException("Value length " + vlength + " > " +
838           HConstants.MAXIMUM_VALUE_LENGTH);
839     }
840   }
841 
842   /**
843    * Write KeyValue format into the provided byte array.
844    *
845    * @param buffer the bytes buffer to use
846    * @param boffset buffer offset
847    * @param row row key
848    * @param roffset row offset
849    * @param rlength row length
850    * @param family family name
851    * @param foffset family offset
852    * @param flength family length
853    * @param qualifier column qualifier
854    * @param qoffset qualifier offset
855    * @param qlength qualifier length
856    * @param timestamp version timestamp
857    * @param type key type
858    * @param value column value
859    * @param voffset value offset
860    * @param vlength value length
861    *
862    * @return The number of useful bytes in the buffer.
863    *
864    * @throws IllegalArgumentException an illegal value was passed or there is insufficient space
865    * remaining in the buffer
866    */
867   private static int writeByteArray(byte [] buffer, final int boffset,
868       final byte [] row, final int roffset, final int rlength,
869       final byte [] family, final int foffset, int flength,
870       final byte [] qualifier, final int qoffset, int qlength,
871       final long timestamp, final Type type,
872       final byte [] value, final int voffset, int vlength, Tag[] tags) {
873 
874     checkParameters(row, rlength, family, flength, qlength, vlength);
875 
876     // Calculate length of tags area
877     int tagsLength = 0;
878     if (tags != null && tags.length > 0) {
879       for (Tag t: tags) {
880         tagsLength += t.getLength();
881       }
882     }
883     checkForTagsLength(tagsLength);
884     int keyLength = (int) getKeyDataStructureSize(rlength, flength, qlength);
885     int keyValueLength = (int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength,
886         tagsLength);
887     if (keyValueLength > buffer.length - boffset) {
888       throw new IllegalArgumentException("Buffer size " + (buffer.length - boffset) + " < " +
889           keyValueLength);
890     }
891 
892     // Write key, value and key row length.
893     int pos = boffset;
894     pos = Bytes.putInt(buffer, pos, keyLength);
895     pos = Bytes.putInt(buffer, pos, vlength);
896     pos = Bytes.putShort(buffer, pos, (short)(rlength & 0x0000ffff));
897     pos = Bytes.putBytes(buffer, pos, row, roffset, rlength);
898     pos = Bytes.putByte(buffer, pos, (byte) (flength & 0x0000ff));
899     if (flength != 0) {
900       pos = Bytes.putBytes(buffer, pos, family, foffset, flength);
901     }
902     if (qlength != 0) {
903       pos = Bytes.putBytes(buffer, pos, qualifier, qoffset, qlength);
904     }
905     pos = Bytes.putLong(buffer, pos, timestamp);
906     pos = Bytes.putByte(buffer, pos, type.getCode());
907     if (value != null && value.length > 0) {
908       pos = Bytes.putBytes(buffer, pos, value, voffset, vlength);
909     }
910     // Write the number of tags. If it is 0 then it means there are no tags.
911     if (tagsLength > 0) {
912       pos = Bytes.putAsShort(buffer, pos, tagsLength);
913       for (Tag t : tags) {
914         pos = Bytes.putBytes(buffer, pos, t.getBuffer(), t.getOffset(), t.getLength());
915       }
916     }
917     return keyValueLength;
918   }
919 
920   private static void checkForTagsLength(int tagsLength) {
921     if (tagsLength > MAX_TAGS_LENGTH) {
922       throw new IllegalArgumentException("tagslength "+ tagsLength + " > " + MAX_TAGS_LENGTH);
923     }
924   }
925 
926   /**
927    * Write KeyValue format into a byte array.
928    * @param row row key
929    * @param roffset row offset
930    * @param rlength row length
931    * @param family family name
932    * @param foffset family offset
933    * @param flength family length
934    * @param qualifier column qualifier
935    * @param qoffset qualifier offset
936    * @param qlength qualifier length
937    * @param timestamp version timestamp
938    * @param type key type
939    * @param value column value
940    * @param voffset value offset
941    * @param vlength value length
942    * @return The newly created byte array.
943    */
944   private static byte [] createByteArray(final byte [] row, final int roffset,
945       final int rlength, final byte [] family, final int foffset, int flength,
946       final byte [] qualifier, final int qoffset, int qlength,
947       final long timestamp, final Type type,
948       final byte [] value, final int voffset, 
949       int vlength, byte[] tags, int tagsOffset, int tagsLength) {
950 
951     checkParameters(row, rlength, family, flength, qlength, vlength);
952     checkForTagsLength(tagsLength);
953     // Allocate right-sized byte array.
954     int keyLength = (int) getKeyDataStructureSize(rlength, flength, qlength);
955     byte [] bytes =
956         new byte[(int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength, tagsLength)];
957     // Write key, value and key row length.
958     int pos = 0;
959     pos = Bytes.putInt(bytes, pos, keyLength);
960     pos = Bytes.putInt(bytes, pos, vlength);
961     pos = Bytes.putShort(bytes, pos, (short)(rlength & 0x0000ffff));
962     pos = Bytes.putBytes(bytes, pos, row, roffset, rlength);
963     pos = Bytes.putByte(bytes, pos, (byte)(flength & 0x0000ff));
964     if(flength != 0) {
965       pos = Bytes.putBytes(bytes, pos, family, foffset, flength);
966     }
967     if(qlength != 0) {
968       pos = Bytes.putBytes(bytes, pos, qualifier, qoffset, qlength);
969     }
970     pos = Bytes.putLong(bytes, pos, timestamp);
971     pos = Bytes.putByte(bytes, pos, type.getCode());
972     if (value != null && value.length > 0) {
973       pos = Bytes.putBytes(bytes, pos, value, voffset, vlength);
974     }
975     // Add the tags after the value part
976     if (tagsLength > 0) {
977       pos = Bytes.putAsShort(bytes, pos, tagsLength);
978       pos = Bytes.putBytes(bytes, pos, tags, tagsOffset, tagsLength);
979     }
980     return bytes;
981   }
982 
983   /**
984    * @param qualifier can be a ByteBuffer or a byte[], or null.
985    * @param value can be a ByteBuffer or a byte[], or null.
986    */
987   private static byte [] createByteArray(final byte [] row, final int roffset,
988       final int rlength, final byte [] family, final int foffset, int flength,
989       final Object qualifier, final int qoffset, int qlength,
990       final long timestamp, final Type type,
991       final Object value, final int voffset, int vlength, List<Tag> tags) {
992 
993     checkParameters(row, rlength, family, flength, qlength, vlength);
994 
995     // Calculate length of tags area
996     int tagsLength = 0;
997     if (tags != null && !tags.isEmpty()) {
998       for (Tag t : tags) {
999         tagsLength += t.getLength();
1000       }
1001     }
1002     checkForTagsLength(tagsLength);
1003     // Allocate right-sized byte array.
1004     int keyLength = (int) getKeyDataStructureSize(rlength, flength, qlength);
1005     byte[] bytes = new byte[(int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength,
1006         tagsLength)];
1007 
1008     // Write key, value and key row length.
1009     int pos = 0;
1010     pos = Bytes.putInt(bytes, pos, keyLength);
1011 
1012     pos = Bytes.putInt(bytes, pos, vlength);
1013     pos = Bytes.putShort(bytes, pos, (short)(rlength & 0x0000ffff));
1014     pos = Bytes.putBytes(bytes, pos, row, roffset, rlength);
1015     pos = Bytes.putByte(bytes, pos, (byte)(flength & 0x0000ff));
1016     if(flength != 0) {
1017       pos = Bytes.putBytes(bytes, pos, family, foffset, flength);
1018     }
1019     if (qlength > 0) {
1020       if (qualifier instanceof ByteBuffer) {
1021         pos = Bytes.putByteBuffer(bytes, pos, (ByteBuffer) qualifier);
1022       } else {
1023         pos = Bytes.putBytes(bytes, pos, (byte[]) qualifier, qoffset, qlength);
1024       }
1025     }
1026     pos = Bytes.putLong(bytes, pos, timestamp);
1027     pos = Bytes.putByte(bytes, pos, type.getCode());
1028     if (vlength > 0) {
1029       if (value instanceof ByteBuffer) {
1030         pos = Bytes.putByteBuffer(bytes, pos, (ByteBuffer) value);
1031       } else {
1032         pos = Bytes.putBytes(bytes, pos, (byte[]) value, voffset, vlength);
1033       }
1034     }
1035     // Add the tags after the value part
1036     if (tagsLength > 0) {
1037       pos = Bytes.putAsShort(bytes, pos, tagsLength);
1038       for (Tag t : tags) {
1039         pos = Bytes.putBytes(bytes, pos, t.getBuffer(), t.getOffset(), t.getLength());
1040       }
1041     }
1042     return bytes;
1043   }
1044 
1045   /**
1046    * Needed doing 'contains' on List.  Only compares the key portion, not the value.
1047    */
1048   @Override
1049   public boolean equals(Object other) {
1050     if (!(other instanceof Cell)) {
1051       return false;
1052     }
1053     return CellComparator.equals(this, (Cell)other);
1054   }
1055 
1056   /**
1057    * In line with {@link #equals(Object)}, only uses the key portion, not the value.
1058    */
1059   @Override
1060   public int hashCode() {
1061     return CellComparator.hashCodeIgnoreMvcc(this);
1062   }
1063 
1064   //---------------------------------------------------------------------------
1065   //
1066   //  KeyValue cloning
1067   //
1068   //---------------------------------------------------------------------------
1069 
1070   /**
1071    * Clones a KeyValue.  This creates a copy, re-allocating the buffer.
1072    * @return Fully copied clone of this KeyValue
1073    * @throws CloneNotSupportedException
1074    */
1075   @Override
1076   public KeyValue clone() throws CloneNotSupportedException {
1077     super.clone();
1078     byte [] b = new byte[this.length];
1079     System.arraycopy(this.bytes, this.offset, b, 0, this.length);
1080     KeyValue ret = new KeyValue(b, 0, b.length);
1081     // Important to clone the memstoreTS as well - otherwise memstore's
1082     // update-in-place methods (eg increment) will end up creating
1083     // new entries
1084     ret.setMvccVersion(mvcc);
1085     return ret;
1086   }
1087 
1088   /**
1089    * Creates a shallow copy of this KeyValue, reusing the data byte buffer.
1090    * http://en.wikipedia.org/wiki/Object_copy
1091    * @return Shallow copy of this KeyValue
1092    */
1093   public KeyValue shallowCopy() {
1094     KeyValue shallowCopy = new KeyValue(this.bytes, this.offset, this.length);
1095     shallowCopy.setMvccVersion(this.mvcc);
1096     return shallowCopy;
1097   }
1098 
1099   //---------------------------------------------------------------------------
1100   //
1101   //  String representation
1102   //
1103   //---------------------------------------------------------------------------
1104 
1105   public String toString() {
1106     if (this.bytes == null || this.bytes.length == 0) {
1107       return "empty";
1108     }
1109     return keyToString(this.bytes, this.offset + ROW_OFFSET, getKeyLength()) +
1110       "/vlen=" + getValueLength() + "/mvcc=" + mvcc;
1111   }
1112 
1113   /**
1114    * @param k Key portion of a KeyValue.
1115    * @return Key as a String, empty string if k is null. 
1116    */
1117   public static String keyToString(final byte [] k) {
1118     if (k == null) { 
1119       return "";
1120     }
1121     return keyToString(k, 0, k.length);
1122   }
1123 
1124   /**
1125    * Produces a string map for this key/value pair. Useful for programmatic use
1126    * and manipulation of the data stored in an HLogKey, for example, printing
1127    * as JSON. Values are left out due to their tendency to be large. If needed,
1128    * they can be added manually.
1129    *
1130    * @return the Map<String,?> containing data from this key
1131    */
1132   public Map<String, Object> toStringMap() {
1133     Map<String, Object> stringMap = new HashMap<String, Object>();
1134     stringMap.put("row", Bytes.toStringBinary(getRow()));
1135     stringMap.put("family", Bytes.toStringBinary(getFamily()));
1136     stringMap.put("qualifier", Bytes.toStringBinary(getQualifier()));
1137     stringMap.put("timestamp", getTimestamp());
1138     stringMap.put("vlen", getValueLength());
1139     List<Tag> tags = getTags();
1140     if (tags != null) {
1141       List<String> tagsString = new ArrayList<String>();
1142       for (Tag t : tags) {
1143         tagsString.add((t.getType()) + ":" +Bytes.toStringBinary(t.getValue()));
1144       }
1145       stringMap.put("tag", tagsString);
1146     }
1147     return stringMap;
1148   }
1149 
1150   /**
1151    * Use for logging.
1152    * @param b Key portion of a KeyValue.
1153    * @param o Offset to start of key
1154    * @param l Length of key.
1155    * @return Key as a String.
1156    */
1157   public static String keyToString(final byte [] b, final int o, final int l) {
1158     if (b == null) return "";
1159     int rowlength = Bytes.toShort(b, o);
1160     String row = Bytes.toStringBinary(b, o + Bytes.SIZEOF_SHORT, rowlength);
1161     int columnoffset = o + Bytes.SIZEOF_SHORT + 1 + rowlength;
1162     int familylength = b[columnoffset - 1];
1163     int columnlength = l - ((columnoffset - o) + TIMESTAMP_TYPE_SIZE);
1164     String family = familylength == 0? "":
1165       Bytes.toStringBinary(b, columnoffset, familylength);
1166     String qualifier = columnlength == 0? "":
1167       Bytes.toStringBinary(b, columnoffset + familylength,
1168       columnlength - familylength);
1169     long timestamp = Bytes.toLong(b, o + (l - TIMESTAMP_TYPE_SIZE));
1170     String timestampStr = humanReadableTimestamp(timestamp);
1171     byte type = b[o + l - 1];
1172     return row + "/" + family +
1173       (family != null && family.length() > 0? ":" :"") +
1174       qualifier + "/" + timestampStr + "/" + Type.codeToType(type);
1175   }
1176 
1177   public static String humanReadableTimestamp(final long timestamp) {
1178     if (timestamp == HConstants.LATEST_TIMESTAMP) {
1179       return "LATEST_TIMESTAMP";
1180     }
1181     if (timestamp == HConstants.OLDEST_TIMESTAMP) {
1182       return "OLDEST_TIMESTAMP";
1183     }
1184     return String.valueOf(timestamp);
1185   }
1186 
1187   //---------------------------------------------------------------------------
1188   //
1189   //  Public Member Accessors
1190   //
1191   //---------------------------------------------------------------------------
1192 
1193   /**
1194    * @return The byte array backing this KeyValue.
1195    * @deprecated Since 0.98.0.  Use Cell Interface instead.  Do not presume single backing buffer.
1196    */
1197   @Deprecated
1198   public byte [] getBuffer() {
1199     return this.bytes;
1200   }
1201 
1202   /**
1203    * @return Offset into {@link #getBuffer()} at which this KeyValue starts.
1204    */
1205   public int getOffset() {
1206     return this.offset;
1207   }
1208 
1209   /**
1210    * @return Length of bytes this KeyValue occupies in {@link #getBuffer()}.
1211    */
1212   public int getLength() {
1213     return length;
1214   }
1215 
1216   //---------------------------------------------------------------------------
1217   //
1218   //  Length and Offset Calculators
1219   //
1220   //---------------------------------------------------------------------------
1221 
1222   /**
1223    * Determines the total length of the KeyValue stored in the specified
1224    * byte array and offset.  Includes all headers.
1225    * @param bytes byte array
1226    * @param offset offset to start of the KeyValue
1227    * @return length of entire KeyValue, in bytes
1228    */
1229   private static int getLength(byte [] bytes, int offset) {
1230     int klength = ROW_OFFSET + Bytes.toInt(bytes, offset);
1231     int vlength = Bytes.toInt(bytes, offset + Bytes.SIZEOF_INT);
1232     return klength + vlength;
1233   }
1234 
1235   /**
1236    * @return Key offset in backing buffer..
1237    */
1238   public int getKeyOffset() {
1239     return this.offset + ROW_OFFSET;
1240   }
1241 
1242   public String getKeyString() {
1243     return Bytes.toStringBinary(getBuffer(), getKeyOffset(), getKeyLength());
1244   }
1245 
1246   /**
1247    * @return Length of key portion.
1248    */
1249   public int getKeyLength() {
1250     return Bytes.toInt(this.bytes, this.offset);
1251   }
1252 
1253   /**
1254    * @return the backing array of the entire KeyValue (all KeyValue fields are in a single array)
1255    */
1256   @Override
1257   public byte[] getValueArray() {
1258     return bytes;
1259   }
1260 
1261   /**
1262    * @return the value offset
1263    */
1264   @Override
1265   public int getValueOffset() {
1266     int voffset = getKeyOffset() + getKeyLength();
1267     return voffset;
1268   }
1269 
1270   /**
1271    * @return Value length
1272    */
1273   @Override
1274   public int getValueLength() {
1275     int vlength = Bytes.toInt(this.bytes, this.offset + Bytes.SIZEOF_INT);
1276     return vlength;
1277   }
1278 
1279   /**
1280    * @return the backing array of the entire KeyValue (all KeyValue fields are in a single array)
1281    */
1282   @Override
1283   public byte[] getRowArray() {
1284     return bytes;
1285   }
1286 
1287   /**
1288    * @return Row offset
1289    */
1290   @Override
1291   public int getRowOffset() {
1292     return getKeyOffset() + Bytes.SIZEOF_SHORT;
1293   }
1294 
1295   /**
1296    * @return Row length
1297    */
1298   @Override
1299   public short getRowLength() {
1300     return Bytes.toShort(this.bytes, getKeyOffset());
1301   }
1302 
1303   /**
1304    * @return the backing array of the entire KeyValue (all KeyValue fields are in a single array)
1305    */
1306   @Override
1307   public byte[] getFamilyArray() {
1308     return bytes;
1309   }
1310 
1311   /**
1312    * @return Family offset
1313    */
1314   @Override
1315   public int getFamilyOffset() {
1316     return getFamilyOffset(getRowLength());
1317   }
1318 
1319   /**
1320    * @return Family offset
1321    */
1322   private int getFamilyOffset(int rlength) {
1323     return this.offset + ROW_OFFSET + Bytes.SIZEOF_SHORT + rlength + Bytes.SIZEOF_BYTE;
1324   }
1325 
1326   /**
1327    * @return Family length
1328    */
1329   @Override
1330   public byte getFamilyLength() {
1331     return getFamilyLength(getFamilyOffset());
1332   }
1333 
1334   /**
1335    * @return Family length
1336    */
1337   public byte getFamilyLength(int foffset) {
1338     return this.bytes[foffset-1];
1339   }
1340 
1341   /**
1342    * @return the backing array of the entire KeyValue (all KeyValue fields are in a single array)
1343    */
1344   @Override
1345   public byte[] getQualifierArray() {
1346     return bytes;
1347   }
1348 
1349   /**
1350    * @return Qualifier offset
1351    */
1352   @Override
1353   public int getQualifierOffset() {
1354     return getQualifierOffset(getFamilyOffset());
1355   }
1356 
1357   /**
1358    * @return Qualifier offset
1359    */
1360   private int getQualifierOffset(int foffset) {
1361     return foffset + getFamilyLength(foffset);
1362   }
1363 
1364   /**
1365    * @return Qualifier length
1366    */
1367   @Override
1368   public int getQualifierLength() {
1369     return getQualifierLength(getRowLength(),getFamilyLength());
1370   }
1371 
1372   /**
1373    * @return Qualifier length
1374    */
1375   private int getQualifierLength(int rlength, int flength) {
1376     return getKeyLength() - (int) getKeyDataStructureSize(rlength, flength, 0);
1377   }
1378 
1379   /**
1380    * @return Column (family + qualifier) length
1381    */
1382   private int getTotalColumnLength(int rlength, int foffset) {
1383     int flength = getFamilyLength(foffset);
1384     int qlength = getQualifierLength(rlength,flength);
1385     return flength + qlength;
1386   }
1387 
1388   /**
1389    * @return Timestamp offset
1390    */
1391   public int getTimestampOffset() {
1392     return getTimestampOffset(getKeyLength());
1393   }
1394 
1395   /**
1396    * @param keylength Pass if you have it to save on a int creation.
1397    * @return Timestamp offset
1398    */
1399   private int getTimestampOffset(final int keylength) {
1400     return getKeyOffset() + keylength - TIMESTAMP_TYPE_SIZE;
1401   }
1402 
1403   /**
1404    * @return True if this KeyValue has a LATEST_TIMESTAMP timestamp.
1405    */
1406   public boolean isLatestTimestamp() {
1407     return Bytes.equals(getBuffer(), getTimestampOffset(), Bytes.SIZEOF_LONG,
1408       HConstants.LATEST_TIMESTAMP_BYTES, 0, Bytes.SIZEOF_LONG);
1409   }
1410 
1411   /**
1412    * @param now Time to set into <code>this</code> IFF timestamp ==
1413    * {@link HConstants#LATEST_TIMESTAMP} (else, its a noop).
1414    * @return True is we modified this.
1415    */
1416   public boolean updateLatestStamp(final byte [] now) {
1417     if (this.isLatestTimestamp()) {
1418       int tsOffset = getTimestampOffset();
1419       System.arraycopy(now, 0, this.bytes, tsOffset, Bytes.SIZEOF_LONG);
1420       // clear cache or else getTimestamp() possibly returns an old value
1421       return true;
1422     }
1423     return false;
1424   }
1425 
1426   //---------------------------------------------------------------------------
1427   //
1428   //  Methods that return copies of fields
1429   //
1430   //---------------------------------------------------------------------------
1431 
1432   /**
1433    * Do not use unless you have to.  Used internally for compacting and testing.
1434    *
1435    * Use {@link #getRow()}, {@link #getFamily()}, {@link #getQualifier()}, and
1436    * {@link #getValue()} if accessing a KeyValue client-side.
1437    * @return Copy of the key portion only.
1438    */
1439   public byte [] getKey() {
1440     int keylength = getKeyLength();
1441     byte [] key = new byte[keylength];
1442     System.arraycopy(getBuffer(), getKeyOffset(), key, 0, keylength);
1443     return key;
1444   }
1445 
1446   /**
1447    * Returns value in a new byte array.
1448    * Primarily for use client-side. If server-side, use
1449    * {@link #getBuffer()} with appropriate offsets and lengths instead to
1450    * save on allocations.
1451    * @return Value in a new byte array.
1452    */
1453   @Deprecated // use CellUtil.getValueArray()
1454   public byte [] getValue() {
1455     return CellUtil.cloneValue(this);
1456   }
1457 
1458   /**
1459    * Primarily for use client-side.  Returns the row of this KeyValue in a new
1460    * byte array.<p>
1461    *
1462    * If server-side, use {@link #getBuffer()} with appropriate offsets and
1463    * lengths instead.
1464    * @return Row in a new byte array.
1465    */
1466   @Deprecated // use CellUtil.getRowArray()
1467   public byte [] getRow() {
1468     return CellUtil.cloneRow(this);
1469   }
1470 
1471   /**
1472    *
1473    * @return Timestamp
1474    */
1475   @Override
1476   public long getTimestamp() {
1477     return getTimestamp(getKeyLength());
1478   }
1479 
1480   /**
1481    * @param keylength Pass if you have it to save on a int creation.
1482    * @return Timestamp
1483    */
1484   long getTimestamp(final int keylength) {
1485     int tsOffset = getTimestampOffset(keylength);
1486     return Bytes.toLong(this.bytes, tsOffset);
1487   }
1488 
1489   /**
1490    * @return Type of this KeyValue.
1491    */
1492   @Deprecated
1493   public byte getType() {
1494     return getTypeByte();
1495   }
1496 
1497   /**
1498    * @return KeyValue.TYPE byte representation
1499    */
1500   @Override
1501   public byte getTypeByte() {
1502     return this.bytes[this.offset + getKeyLength() - 1 + ROW_OFFSET];
1503   }
1504 
1505   /**
1506    * @return True if a delete type, a {@link KeyValue.Type#Delete} or
1507    * a {KeyValue.Type#DeleteFamily} or a {@link KeyValue.Type#DeleteColumn}
1508    * KeyValue type.
1509    */
1510   @Deprecated // use CellUtil#isDelete
1511   public boolean isDelete() {
1512     return KeyValue.isDelete(getType());
1513   }
1514 
1515   /**
1516    * @return True if this KV is a {@link KeyValue.Type#Delete} type.
1517    */
1518   public boolean isDeleteType() {
1519     // TODO: Fix this method name vis-a-vis isDelete!
1520     return getTypeByte() == Type.Delete.getCode();
1521   }
1522 
1523   /**
1524    * @return True if this KV is a delete family type.
1525    */
1526   public boolean isDeleteFamily() {
1527     return getTypeByte() == Type.DeleteFamily.getCode();
1528   }
1529 
1530   /**
1531    * @return True if this KV is a delete family-version type.
1532    */
1533   public boolean isDeleteFamilyVersion() {
1534     return getTypeByte() == Type.DeleteFamilyVersion.getCode();
1535   }
1536 
1537   /**
1538    *
1539    * @return True if this KV is a delete family or column type.
1540    */
1541   public boolean isDeleteColumnOrFamily() {
1542     int t = getTypeByte();
1543     return t == Type.DeleteColumn.getCode() || t == Type.DeleteFamily.getCode();
1544   }
1545 
1546   /**
1547    * Primarily for use client-side.  Returns the family of this KeyValue in a
1548    * new byte array.<p>
1549    *
1550    * If server-side, use {@link #getBuffer()} with appropriate offsets and
1551    * lengths instead.
1552    * @return Returns family. Makes a copy.
1553    */
1554   @Deprecated // use CellUtil.getFamilyArray
1555   public byte [] getFamily() {
1556     return CellUtil.cloneFamily(this);
1557   }
1558 
1559   /**
1560    * Primarily for use client-side.  Returns the column qualifier of this
1561    * KeyValue in a new byte array.<p>
1562    *
1563    * If server-side, use {@link #getBuffer()} with appropriate offsets and
1564    * lengths instead.
1565    * Use {@link #getBuffer()} with appropriate offsets and lengths instead.
1566    * @return Returns qualifier. Makes a copy.
1567    */
1568   @Deprecated // use CellUtil.getQualifierArray
1569   public byte [] getQualifier() {
1570     return CellUtil.cloneQualifier(this);
1571   }
1572 
1573   /**
1574    * This returns the offset where the tag actually starts.
1575    */
1576   @Override
1577   public int getTagsOffset() {
1578     int tagsLen = getTagsLengthUnsigned();
1579     if (tagsLen == 0) {
1580       return this.offset + this.length;
1581     }
1582     return this.offset + this.length - tagsLen;
1583   }
1584 
1585   /**
1586    * This returns the total length of the tag bytes
1587    */
1588   @Override
1589   @Deprecated
1590   public int getTagsLengthUnsigned() {
1591     int tagsLen = this.length - (getKeyLength() + getValueLength() + KEYVALUE_INFRASTRUCTURE_SIZE);
1592     if (tagsLen > 0) {
1593       // There are some Tag bytes in the byte[]. So reduce 2 bytes which is added to denote the tags
1594       // length
1595       tagsLen -= TAGS_LENGTH_SIZE;
1596     }
1597     return tagsLen;
1598   }
1599 
1600   @Override
1601   @Deprecated
1602   public short getTagsLength() {
1603     return (short) getTagsLengthUnsigned();
1604   }
1605 
1606   /**
1607    * Returns any tags embedded in the KeyValue.  Used in testcases.
1608    * @return The tags
1609    */
1610   public List<Tag> getTags() {
1611     int tagsLength = getTagsLengthUnsigned();
1612     if (tagsLength == 0) {
1613       return EMPTY_ARRAY_LIST;
1614     }
1615     return Tag.asList(getBuffer(), getTagsOffset(), tagsLength);
1616   }
1617 
1618   /**
1619    * @return the backing array of the entire KeyValue (all KeyValue fields are in a single array)
1620    */
1621   @Override
1622   public byte[] getTagsArray() {
1623     return bytes;
1624   }
1625 
1626   //---------------------------------------------------------------------------
1627   //
1628   //  Compare specified fields against those contained in this KeyValue
1629   //
1630   //---------------------------------------------------------------------------
1631 
1632   /**
1633    * @param family
1634    * @return True if matching families.
1635    */
1636   public boolean matchingFamily(final byte [] family) {
1637     if (this.length == 0 || this.bytes.length == 0) {
1638       return false;
1639     }
1640     return Bytes.equals(family, 0, family.length,
1641         this.bytes, getFamilyOffset(), getFamilyLength());
1642   }
1643 
1644   /**
1645    * @param qualifier
1646    * @return True if matching qualifiers.
1647    */
1648   public boolean matchingQualifier(final byte [] qualifier) {
1649     return matchingQualifier(qualifier, 0, qualifier.length);
1650   }
1651 
1652   public boolean matchingQualifier(final byte [] qualifier, int offset, int length) {
1653     return Bytes.equals(qualifier, offset, length,
1654         this.bytes, getQualifierOffset(), getQualifierLength());
1655   }
1656 
1657   public boolean matchingQualifier(final KeyValue other) {
1658     return matchingQualifier(other.getBuffer(), other.getQualifierOffset(),
1659         other.getQualifierLength());
1660   }
1661 
1662   public boolean matchingRow(final byte [] row) {
1663     return matchingRow(row, 0, row.length);
1664   }
1665 
1666   public boolean matchingRow(final byte[] row, int offset, int length) {
1667     return Bytes.equals(row, offset, length,
1668         this.bytes, getRowOffset(), getRowLength());
1669   }
1670 
1671   public boolean matchingRow(KeyValue other) {
1672     return matchingRow(other.getBuffer(), other.getRowOffset(),
1673         other.getRowLength());
1674   }
1675 
1676   /**
1677    *
1678    * @param family column family
1679    * @param qualifier column qualifier
1680    * @return True if column matches
1681    */
1682   public boolean matchingColumn(final byte[] family, final byte[] qualifier) {
1683     return matchingColumn(family, 0, len(family), qualifier, 0, len(qualifier));
1684   }
1685 
1686   /**
1687    * Checks if column matches.
1688    *
1689    * @param family family name
1690    * @param foffset family offset
1691    * @param flength family length
1692    * @param qualifier column qualifier
1693    * @param qoffset qualifier offset
1694    * @param qlength qualifier length
1695    *
1696    * @return True if column matches
1697    */
1698   public boolean matchingColumn(final byte [] family, final int foffset, final int flength,
1699       final byte [] qualifier, final int qoffset, final int qlength) {
1700     int rl = getRowLength();
1701     int o = getFamilyOffset(rl);
1702     int fl = getFamilyLength(o);
1703     if (!Bytes.equals(family, foffset, flength, this.bytes, o, fl)) {
1704       return false;
1705     }
1706 
1707     int ql = getQualifierLength(rl, fl);
1708     if (qualifier == null || qlength == 0) {
1709       return (ql == 0);
1710     }
1711     return Bytes.equals(qualifier, qoffset, qlength, this.bytes, o + fl, ql);
1712   }
1713 
1714   /**
1715    * Creates a new KeyValue that only contains the key portion (the value is
1716    * set to be null).
1717    *
1718    * TODO only used by KeyOnlyFilter -- move there.
1719    * @param lenAsVal replace value with the actual value length (false=empty)
1720    */
1721   public KeyValue createKeyOnly(boolean lenAsVal) {
1722     // KV format:  <keylen:4><valuelen:4><key:keylen><value:valuelen>
1723     // Rebuild as: <keylen:4><0:4><key:keylen>
1724     int dataLen = lenAsVal? Bytes.SIZEOF_INT : 0;
1725     byte [] newBuffer = new byte[getKeyLength() + ROW_OFFSET + dataLen];
1726     System.arraycopy(this.bytes, this.offset, newBuffer, 0,
1727         Math.min(newBuffer.length,this.length));
1728     Bytes.putInt(newBuffer, Bytes.SIZEOF_INT, dataLen);
1729     if (lenAsVal) {
1730       Bytes.putInt(newBuffer, newBuffer.length - dataLen, this.getValueLength());
1731     }
1732     return new KeyValue(newBuffer);
1733   }
1734 
1735   /**
1736    * Splits a column in {@code family:qualifier} form into separate byte arrays. An empty qualifier
1737    * (ie, {@code fam:}) is parsed as <code>{ fam, EMPTY_BYTE_ARRAY }</code> while no delimiter (ie,
1738    * {@code fam}) is parsed as an array of one element, <code>{ fam }</code>.
1739    * <p>
1740    * Don't forget, HBase DOES support empty qualifiers. (see HBASE-9549)
1741    * </p>
1742    * <p>
1743    * Not recommend to be used as this is old-style API.
1744    * </p>
1745    * @param c The column.
1746    * @return The parsed column.
1747    */
1748   public static byte [][] parseColumn(byte [] c) {
1749     final int index = getDelimiter(c, 0, c.length, COLUMN_FAMILY_DELIMITER);
1750     if (index == -1) {
1751       // If no delimiter, return array of size 1
1752       return new byte [][] { c };
1753     } else if(index == c.length - 1) {
1754       // family with empty qualifier, return array size 2
1755       byte [] family = new byte[c.length-1];
1756       System.arraycopy(c, 0, family, 0, family.length);
1757       return new byte [][] { family, HConstants.EMPTY_BYTE_ARRAY};
1758     }
1759     // Family and column, return array size 2
1760     final byte [][] result = new byte [2][];
1761     result[0] = new byte [index];
1762     System.arraycopy(c, 0, result[0], 0, index);
1763     final int len = c.length - (index + 1);
1764     result[1] = new byte[len];
1765     System.arraycopy(c, index + 1 /* Skip delimiter */, result[1], 0, len);
1766     return result;
1767   }
1768 
1769   /**
1770    * Makes a column in family:qualifier form from separate byte arrays.
1771    * <p>
1772    * Not recommended for usage as this is old-style API.
1773    * @param family
1774    * @param qualifier
1775    * @return family:qualifier
1776    */
1777   public static byte [] makeColumn(byte [] family, byte [] qualifier) {
1778     return Bytes.add(family, COLUMN_FAMILY_DELIM_ARRAY, qualifier);
1779   }
1780 
1781   /**
1782    * @param b
1783    * @param delimiter
1784    * @return Index of delimiter having started from start of <code>b</code>
1785    * moving rightward.
1786    */
1787   public static int getDelimiter(final byte [] b, int offset, final int length,
1788       final int delimiter) {
1789     if (b == null) {
1790       throw new IllegalArgumentException("Passed buffer is null");
1791     }
1792     int result = -1;
1793     for (int i = offset; i < length + offset; i++) {
1794       if (b[i] == delimiter) {
1795         result = i;
1796         break;
1797       }
1798     }
1799     return result;
1800   }
1801 
1802   /**
1803    * Find index of passed delimiter walking from end of buffer backwards.
1804    * @param b
1805    * @param delimiter
1806    * @return Index of delimiter
1807    */
1808   public static int getDelimiterInReverse(final byte [] b, final int offset,
1809       final int length, final int delimiter) {
1810     if (b == null) {
1811       throw new IllegalArgumentException("Passed buffer is null");
1812     }
1813     int result = -1;
1814     for (int i = (offset + length) - 1; i >= offset; i--) {
1815       if (b[i] == delimiter) {
1816         result = i;
1817         break;
1818       }
1819     }
1820     return result;
1821   }
1822 
1823   /**
1824    * A {@link KVComparator} for <code>hbase:meta</code> catalog table
1825    * {@link KeyValue}s.
1826    */
1827   public static class MetaComparator extends KVComparator {
1828     /**
1829      * Compare key portion of a {@link KeyValue} for keys in <code>hbase:meta</code>
1830      * table.
1831      */
1832     @Override
1833     public int compareRows(byte [] left, int loffset, int llength,
1834         byte [] right, int roffset, int rlength) {
1835       int leftDelimiter = getDelimiter(left, loffset, llength,
1836           HConstants.DELIMITER);
1837       int rightDelimiter = getDelimiter(right, roffset, rlength,
1838           HConstants.DELIMITER);
1839       // Compare up to the delimiter
1840       int lpart = (leftDelimiter < 0 ? llength :leftDelimiter - loffset);
1841       int rpart = (rightDelimiter < 0 ? rlength :rightDelimiter - roffset);
1842       int result = Bytes.compareTo(left, loffset, lpart, right, roffset, rpart);
1843       if (result != 0) {
1844         return result;
1845       } else {
1846         if (leftDelimiter < 0 && rightDelimiter >= 0) {
1847           return -1;
1848         } else if (rightDelimiter < 0 && leftDelimiter >= 0) {
1849           return 1;
1850         } else if (leftDelimiter < 0 && rightDelimiter < 0) {
1851           return 0;
1852         }
1853       }
1854       // Compare middle bit of the row.
1855       // Move past delimiter
1856       leftDelimiter++;
1857       rightDelimiter++;
1858       int leftFarDelimiter = getDelimiterInReverse(left, leftDelimiter,
1859           llength - (leftDelimiter - loffset), HConstants.DELIMITER);
1860       int rightFarDelimiter = getDelimiterInReverse(right,
1861           rightDelimiter, rlength - (rightDelimiter - roffset),
1862           HConstants.DELIMITER);
1863       // Now compare middlesection of row.
1864       lpart = (leftFarDelimiter < 0 ? llength + loffset: leftFarDelimiter) - leftDelimiter;
1865       rpart = (rightFarDelimiter < 0 ? rlength + roffset: rightFarDelimiter)- rightDelimiter;
1866       result = super.compareRows(left, leftDelimiter, lpart, right, rightDelimiter, rpart);
1867       if (result != 0) {
1868         return result;
1869       }  else {
1870         if (leftDelimiter < 0 && rightDelimiter >= 0) {
1871           return -1;
1872         } else if (rightDelimiter < 0 && leftDelimiter >= 0) {
1873           return 1;
1874         } else if (leftDelimiter < 0 && rightDelimiter < 0) {
1875           return 0;
1876         }
1877       }
1878       // Compare last part of row, the rowid.
1879       leftFarDelimiter++;
1880       rightFarDelimiter++;
1881       result = Bytes.compareTo(left, leftFarDelimiter, llength - (leftFarDelimiter - loffset),
1882           right, rightFarDelimiter, rlength - (rightFarDelimiter - roffset));
1883       return result;
1884     }
1885 
1886     /**
1887      * Don't do any fancy Block Index splitting tricks.
1888      */
1889     @Override
1890     public byte[] getShortMidpointKey(final byte[] leftKey, final byte[] rightKey) {
1891       return Arrays.copyOf(rightKey, rightKey.length);
1892     }
1893 
1894     /**
1895      * The HFileV2 file format's trailer contains this class name.  We reinterpret this and
1896      * instantiate the appropriate comparator.
1897      * TODO: With V3 consider removing this.
1898      * @return legacy class name for FileFileTrailer#comparatorClassName
1899      */
1900     @Override
1901     public String getLegacyKeyComparatorName() {
1902       return "org.apache.hadoop.hbase.KeyValue$MetaKeyComparator";
1903     }
1904 
1905     @Override
1906     protected Object clone() throws CloneNotSupportedException {
1907       return new MetaComparator();
1908     }
1909 
1910     /**
1911      * Override the row key comparison to parse and compare the meta row key parts.
1912      */
1913     @Override
1914     protected int compareRowKey(final Cell l, final Cell r) {
1915       byte[] left = l.getRowArray();
1916       int loffset = l.getRowOffset();
1917       int llength = l.getRowLength();
1918       byte[] right = r.getRowArray();
1919       int roffset = r.getRowOffset();
1920       int rlength = r.getRowLength();
1921       return compareRows(left, loffset, llength, right, roffset, rlength);
1922     }
1923   }
1924 
1925   /**
1926    * Compare KeyValues.  When we compare KeyValues, we only compare the Key
1927    * portion.  This means two KeyValues with same Key but different Values are
1928    * considered the same as far as this Comparator is concerned.
1929    */
1930   public static class KVComparator implements RawComparator<Cell>, SamePrefixComparator<byte[]> {
1931 
1932     /**
1933      * The HFileV2 file format's trailer contains this class name.  We reinterpret this and
1934      * instantiate the appropriate comparator.
1935      * TODO: With V3 consider removing this.
1936      * @return legacy class name for FileFileTrailer#comparatorClassName
1937      */
1938     public String getLegacyKeyComparatorName() {
1939       return "org.apache.hadoop.hbase.KeyValue$KeyComparator";
1940     }
1941 
1942     @Override // RawComparator
1943     public int compare(byte[] l, int loff, int llen, byte[] r, int roff, int rlen) {
1944       return compareFlatKey(l,loff,llen, r,roff,rlen);
1945     }
1946 
1947     
1948     /**
1949      * Compares the only the user specified portion of a Key.  This is overridden by MetaComparator.
1950      * @param left
1951      * @param right
1952      * @return 0 if equal, <0 if left smaller, >0 if right smaller
1953      */
1954     protected int compareRowKey(final Cell left, final Cell right) {
1955       return Bytes.compareTo(
1956           left.getRowArray(),  left.getRowOffset(),  left.getRowLength(),
1957           right.getRowArray(), right.getRowOffset(), right.getRowLength());
1958     }
1959 
1960     /**
1961      * Compares left to right assuming that left,loffset,llength and right,roffset,rlength are
1962      * full KVs laid out in a flat byte[]s.
1963      * @param left
1964      * @param loffset
1965      * @param llength
1966      * @param right
1967      * @param roffset
1968      * @param rlength
1969      * @return  0 if equal, <0 if left smaller, >0 if right smaller
1970      */
1971     public int compareFlatKey(byte[] left, int loffset, int llength,
1972         byte[] right, int roffset, int rlength) {
1973       // Compare row
1974       short lrowlength = Bytes.toShort(left, loffset);
1975       short rrowlength = Bytes.toShort(right, roffset);
1976       int compare = compareRows(left, loffset + Bytes.SIZEOF_SHORT,
1977           lrowlength, right, roffset + Bytes.SIZEOF_SHORT, rrowlength);
1978       if (compare != 0) {
1979         return compare;
1980       }
1981 
1982       // Compare the rest of the two KVs without making any assumptions about
1983       // the common prefix. This function will not compare rows anyway, so we
1984       // don't need to tell it that the common prefix includes the row.
1985       return compareWithoutRow(0, left, loffset, llength, right, roffset,
1986           rlength, rrowlength);
1987     }
1988 
1989     public int compareFlatKey(byte[] left, byte[] right) {
1990       return compareFlatKey(left, 0, left.length, right, 0, right.length);
1991     }
1992 
1993     // compare a key against a given row/fam/qual/ts/type
1994     public int compareKey(byte[] key, int koff, int klen, byte[] row, int roff, int rlen,
1995         byte[] fam, int foff, int flen, byte[] col, int coff, int clen, long ts, byte type) {
1996 
1997       short lrowlength = Bytes.toShort(key, koff);
1998       int compare = compareRows(key, koff + Bytes.SIZEOF_SHORT, lrowlength, row, roff, rlen);
1999       if (compare != 0) {
2000         return compare;
2001       }
2002       // See compareWithoutRows...
2003       // -------------------------
2004       int commonLength = ROW_LENGTH_SIZE + FAMILY_LENGTH_SIZE + rlen;
2005 
2006       // key's ColumnFamily + Qualifier length.
2007       int lcolumnlength = klen - TIMESTAMP_TYPE_SIZE + commonLength;
2008 
2009       byte ltype = key[koff + (klen - 1)];
2010 
2011       // If the column is not specified, the "minimum" key type appears the
2012       // latest in the sorted order, regardless of the timestamp. This is used
2013       // for specifying the last key/value in a given row, because there is no
2014       // "lexicographically last column" (it would be infinitely long). The
2015       // "maximum" key type does not need this behavior.
2016       if (lcolumnlength == 0 && ltype == Type.Minimum.getCode()) {
2017         // left is "bigger", i.e. it appears later in the sorted order
2018         return 1;
2019       }
2020       if (flen + clen == 0 && type == Type.Minimum.getCode()) {
2021         return -1;
2022       }
2023 
2024       int lfamilyoffset = commonLength + koff;
2025       int lfamilylength = key[lfamilyoffset - 1];
2026       compare = Bytes.compareTo(key, lfamilyoffset, lfamilylength, fam, foff, flen);
2027       if (compare != 0) {
2028         return compare;
2029       }
2030       int lColOffset = lfamilyoffset + lfamilylength;
2031       int lColLength = lcolumnlength - lfamilylength;
2032       compare = Bytes.compareTo(key, lColOffset, lColLength, col, coff, clen);
2033       if (compare != 0) {
2034         return compare;
2035       }
2036       // Next compare timestamps.
2037       long ltimestamp = Bytes.toLong(key, koff + (klen - TIMESTAMP_TYPE_SIZE));
2038       compare = compareTimestamps(ltimestamp, ts);
2039       if (compare != 0) {
2040         return compare;
2041       }
2042 
2043       // Compare types. Let the delete types sort ahead of puts; i.e. types
2044       // of higher numbers sort before those of lesser numbers. Maximum (255)
2045       // appears ahead of everything, and minimum (0) appears after
2046       // everything.
2047       return (0xff & type) - (0xff & ltype);
2048     }
2049 
2050     /**
2051      * Compares the Key of a cell -- with fields being more significant in this order:
2052      * rowkey, colfam/qual, timestamp, type, mvcc
2053      */
2054     public int compare(final Cell left, final Cell right) {
2055       // compare row
2056       int compare = compareRowKey(left, right);
2057       if (compare != 0) {
2058         return compare;
2059       }
2060 
2061       // compare vs minimum
2062       byte ltype = left.getTypeByte();
2063       byte rtype = right.getTypeByte();
2064       // If the column is not specified, the "minimum" key type appears the
2065       // latest in the sorted order, regardless of the timestamp. This is used
2066       // for specifying the last key/value in a given row, because there is no
2067       // "lexicographically last column" (it would be infinitely long). The
2068       // "maximum" key type does not need this behavior.
2069       int lcfqLen = left.getFamilyLength() + left.getQualifierLength() ;
2070       int rcfqLen = right.getFamilyLength() + right.getQualifierLength() ;
2071       if (lcfqLen == 0 && ltype == Type.Minimum.getCode()) {
2072         // left is "bigger", i.e. it appears later in the sorted order
2073         return 1;
2074       }
2075       if (rcfqLen == 0 && rtype == Type.Minimum.getCode()) {
2076         return -1;
2077       }
2078 
2079 
2080       // compare col family / col fam + qual
2081       // If left family size is not equal to right family size, we need not
2082       // compare the qualifiers.
2083       compare = Bytes.compareTo(
2084         left.getFamilyArray(),  left.getFamilyOffset(),  left.getFamilyLength(),
2085         right.getFamilyArray(), right.getFamilyOffset(), right.getFamilyLength());
2086       if (compare != 0) {
2087         return compare;
2088       }
2089 
2090       // Compare qualifier
2091       compare = Bytes.compareTo(
2092           left.getQualifierArray(), left.getQualifierOffset(), left.getQualifierLength(),
2093           right.getQualifierArray(), right.getQualifierOffset(), right.getQualifierLength());
2094       if (compare!= 0) {
2095         return compare;
2096       }
2097 
2098       // compare timestamp
2099       long ltimestamp = left.getTimestamp();
2100       long rtimestamp = right.getTimestamp();
2101       compare = compareTimestamps(ltimestamp, rtimestamp);
2102       if (compare != 0) {
2103         return compare;
2104       }
2105 
2106       // Compare types. Let the delete types sort ahead of puts; i.e. types
2107       // of higher numbers sort before those of lesser numbers. Maximum (255)
2108       // appears ahead of everything, and minimum (0) appears after
2109       // everything.
2110       compare = (0xff & rtype) - (0xff & ltype);
2111       if (compare != 0) {
2112         return compare;
2113       }
2114 
2115       // Negate following comparisons so later edits show up first
2116 
2117       // compare log replay tag value if there is any
2118       // when either keyvalue tagged with log replay sequence number, we need to compare them:
2119       // 1) when both keyvalues have the tag, then use the tag values for comparison
2120       // 2) when one has and the other doesn't have, the one without the log replay tag wins because
2121       // it means the edit isn't from recovery but new one coming from clients during recovery
2122       // 3) when both doesn't have, then skip to the next mvcc comparison
2123       long leftChangeSeqNum = getReplaySeqNum(left);
2124       long RightChangeSeqNum = getReplaySeqNum(right);
2125       if (leftChangeSeqNum != Long.MAX_VALUE || RightChangeSeqNum != Long.MAX_VALUE) {
2126         return Longs.compare(RightChangeSeqNum, leftChangeSeqNum);
2127       }
2128 
2129       // compare Mvcc Version
2130       return Longs.compare(right.getMvccVersion(), left.getMvccVersion());
2131     }
2132     
2133     /**
2134      * Return replay log sequence number for the cell
2135      * @param c
2136      * @return Long.MAX_VALUE if there is no LOG_REPLAY_TAG
2137      */
2138     private long getReplaySeqNum(final Cell c) {
2139       Tag tag = Tag.getTag(c.getTagsArray(), c.getTagsOffset(), c.getTagsLengthUnsigned(),
2140           TagType.LOG_REPLAY_TAG_TYPE);
2141 
2142       if(tag != null) {
2143         return Bytes.toLong(tag.getBuffer(), tag.getTagOffset(), tag.getTagLength());
2144       }
2145       return Long.MAX_VALUE;
2146     }
2147 
2148     public int compareTimestamps(final KeyValue left, final KeyValue right) {
2149       // Compare timestamps
2150       long ltimestamp = left.getTimestamp(left.getKeyLength());
2151       long rtimestamp = right.getTimestamp(right.getKeyLength());
2152       return compareTimestamps(ltimestamp, rtimestamp);
2153     }
2154 
2155     /**
2156      * @param left
2157      * @param right
2158      * @return Result comparing rows.
2159      */
2160     public int compareRows(final KeyValue left, final KeyValue right) {
2161       return compareRows(left.getBuffer(),left.getRowOffset(), left.getRowLength(),
2162       right.getBuffer(), right.getRowOffset(), right.getRowLength());
2163     }
2164 
2165     /**
2166      * Get the b[],o,l for left and right rowkey portions and compare.
2167      * @param left
2168      * @param loffset
2169      * @param llength
2170      * @param right
2171      * @param roffset
2172      * @param rlength
2173      * @return 0 if equal, <0 if left smaller, >0 if right smaller
2174      */
2175     public int compareRows(byte [] left, int loffset, int llength,
2176         byte [] right, int roffset, int rlength) {
2177       return Bytes.compareTo(left, loffset, llength, right, roffset, rlength);
2178     }
2179 
2180     int compareColumns(final KeyValue left, final short lrowlength,
2181         final KeyValue right, final short rrowlength) {
2182       int lfoffset = left.getFamilyOffset(lrowlength);
2183       int rfoffset = right.getFamilyOffset(rrowlength);
2184       int lclength = left.getTotalColumnLength(lrowlength,lfoffset);
2185       int rclength = right.getTotalColumnLength(rrowlength, rfoffset);
2186       int lfamilylength = left.getFamilyLength(lfoffset);
2187       int rfamilylength = right.getFamilyLength(rfoffset);
2188       return compareColumns(left.getBuffer(), lfoffset,
2189           lclength, lfamilylength,
2190         right.getBuffer(), rfoffset, rclength, rfamilylength);
2191     }
2192 
2193     protected int compareColumns(
2194         byte [] left, int loffset, int llength, final int lfamilylength,
2195         byte [] right, int roffset, int rlength, final int rfamilylength) {
2196       // Compare family portion first.
2197       int diff = Bytes.compareTo(left, loffset, lfamilylength,
2198         right, roffset, rfamilylength);
2199       if (diff != 0) {
2200         return diff;
2201       }
2202       // Compare qualifier portion
2203       return Bytes.compareTo(left, loffset + lfamilylength,
2204         llength - lfamilylength,
2205         right, roffset + rfamilylength, rlength - rfamilylength);
2206       }
2207 
2208     static int compareTimestamps(final long ltimestamp, final long rtimestamp) {
2209       // The below older timestamps sorting ahead of newer timestamps looks
2210       // wrong but it is intentional. This way, newer timestamps are first
2211       // found when we iterate over a memstore and newer versions are the
2212       // first we trip over when reading from a store file.
2213       if (ltimestamp < rtimestamp) {
2214         return 1;
2215       } else if (ltimestamp > rtimestamp) {
2216         return -1;
2217       }
2218       return 0;
2219     }
2220 
2221     /**
2222      * Overridden
2223      * @param commonPrefix
2224      * @param left
2225      * @param loffset
2226      * @param llength
2227      * @param right
2228      * @param roffset
2229      * @param rlength
2230      * @return 0 if equal, <0 if left smaller, >0 if right smaller
2231      */
2232     @Override // SamePrefixComparator
2233     public int compareIgnoringPrefix(int commonPrefix, byte[] left,
2234         int loffset, int llength, byte[] right, int roffset, int rlength) {
2235       // Compare row
2236       short lrowlength = Bytes.toShort(left, loffset);
2237       short rrowlength;
2238 
2239       int comparisonResult = 0;
2240       if (commonPrefix < ROW_LENGTH_SIZE) {
2241         // almost nothing in common
2242         rrowlength = Bytes.toShort(right, roffset);
2243         comparisonResult = compareRows(left, loffset + ROW_LENGTH_SIZE,
2244             lrowlength, right, roffset + ROW_LENGTH_SIZE, rrowlength);
2245       } else { // the row length is the same
2246         rrowlength = lrowlength;
2247         if (commonPrefix < ROW_LENGTH_SIZE + rrowlength) {
2248           // The rows are not the same. Exclude the common prefix and compare
2249           // the rest of the two rows.
2250           int common = commonPrefix - ROW_LENGTH_SIZE;
2251           comparisonResult = compareRows(
2252               left, loffset + common + ROW_LENGTH_SIZE, lrowlength - common,
2253               right, roffset + common + ROW_LENGTH_SIZE, rrowlength - common);
2254         }
2255       }
2256       if (comparisonResult != 0) {
2257         return comparisonResult;
2258       }
2259 
2260       assert lrowlength == rrowlength;
2261       return compareWithoutRow(commonPrefix, left, loffset, llength, right,
2262           roffset, rlength, lrowlength);
2263     }
2264 
2265     /**
2266      * Compare columnFamily, qualifier, timestamp, and key type (everything
2267      * except the row). This method is used both in the normal comparator and
2268      * the "same-prefix" comparator. Note that we are assuming that row portions
2269      * of both KVs have already been parsed and found identical, and we don't
2270      * validate that assumption here.
2271      * @param commonPrefix
2272      *          the length of the common prefix of the two key-values being
2273      *          compared, including row length and row
2274      */
2275     private int compareWithoutRow(int commonPrefix, byte[] left, int loffset,
2276         int llength, byte[] right, int roffset, int rlength, short rowlength) {
2277       /***
2278        * KeyValue Format and commonLength:
2279        * |_keyLen_|_valLen_|_rowLen_|_rowKey_|_famiLen_|_fami_|_Quali_|....
2280        * ------------------|-------commonLength--------|--------------
2281        */
2282       int commonLength = ROW_LENGTH_SIZE + FAMILY_LENGTH_SIZE + rowlength;
2283 
2284       // commonLength + TIMESTAMP_TYPE_SIZE
2285       int commonLengthWithTSAndType = TIMESTAMP_TYPE_SIZE + commonLength;
2286       // ColumnFamily + Qualifier length.
2287       int lcolumnlength = llength - commonLengthWithTSAndType;
2288       int rcolumnlength = rlength - commonLengthWithTSAndType;
2289 
2290       byte ltype = left[loffset + (llength - 1)];
2291       byte rtype = right[roffset + (rlength - 1)];
2292 
2293       // If the column is not specified, the "minimum" key type appears the
2294       // latest in the sorted order, regardless of the timestamp. This is used
2295       // for specifying the last key/value in a given row, because there is no
2296       // "lexicographically last column" (it would be infinitely long). The
2297       // "maximum" key type does not need this behavior.
2298       if (lcolumnlength == 0 && ltype == Type.Minimum.getCode()) {
2299         // left is "bigger", i.e. it appears later in the sorted order
2300         return 1;
2301       }
2302       if (rcolumnlength == 0 && rtype == Type.Minimum.getCode()) {
2303         return -1;
2304       }
2305 
2306       int lfamilyoffset = commonLength + loffset;
2307       int rfamilyoffset = commonLength + roffset;
2308 
2309       // Column family length.
2310       int lfamilylength = left[lfamilyoffset - 1];
2311       int rfamilylength = right[rfamilyoffset - 1];
2312       // If left family size is not equal to right family size, we need not
2313       // compare the qualifiers.
2314       boolean sameFamilySize = (lfamilylength == rfamilylength);
2315       int common = 0;
2316       if (commonPrefix > 0) {
2317         common = Math.max(0, commonPrefix - commonLength);
2318         if (!sameFamilySize) {
2319           // Common should not be larger than Math.min(lfamilylength,
2320           // rfamilylength).
2321           common = Math.min(common, Math.min(lfamilylength, rfamilylength));
2322         } else {
2323           common = Math.min(common, Math.min(lcolumnlength, rcolumnlength));
2324         }
2325       }
2326       if (!sameFamilySize) {
2327         // comparing column family is enough.
2328         return Bytes.compareTo(left, lfamilyoffset + common, lfamilylength
2329             - common, right, rfamilyoffset + common, rfamilylength - common);
2330       }
2331       // Compare family & qualifier together.
2332       final int comparison = Bytes.compareTo(left, lfamilyoffset + common,
2333           lcolumnlength - common, right, rfamilyoffset + common,
2334           rcolumnlength - common);
2335       if (comparison != 0) {
2336         return comparison;
2337       }
2338 
2339       ////
2340       // Next compare timestamps.
2341       long ltimestamp = Bytes.toLong(left,
2342           loffset + (llength - TIMESTAMP_TYPE_SIZE));
2343       long rtimestamp = Bytes.toLong(right,
2344           roffset + (rlength - TIMESTAMP_TYPE_SIZE));
2345       int compare = compareTimestamps(ltimestamp, rtimestamp);
2346       if (compare != 0) {
2347         return compare;
2348       }
2349 
2350       // Compare types. Let the delete types sort ahead of puts; i.e. types
2351       // of higher numbers sort before those of lesser numbers. Maximum (255)
2352       // appears ahead of everything, and minimum (0) appears after
2353       // everything.
2354       return (0xff & rtype) - (0xff & ltype);
2355     }
2356 
2357     /**
2358      * Compares the row and column of two keyvalues for equality
2359      * @param left
2360      * @param right
2361      * @return True if same row and column.
2362      */
2363     public boolean matchingRowColumn(final KeyValue left,
2364         final KeyValue right) {
2365       short lrowlength = left.getRowLength();
2366       short rrowlength = right.getRowLength();
2367 
2368       // TsOffset = end of column data. just comparing Row+CF length of each
2369       if ((left.getTimestampOffset() - left.getOffset()) !=
2370           (right.getTimestampOffset() - right.getOffset())) {
2371         return false;
2372       }
2373 
2374       if (!matchingRows(left, lrowlength, right, rrowlength)) {
2375         return false;
2376       }
2377 
2378       int lfoffset = left.getFamilyOffset(lrowlength);
2379       int rfoffset = right.getFamilyOffset(rrowlength);
2380       int lclength = left.getTotalColumnLength(lrowlength,lfoffset);
2381       int rclength = right.getTotalColumnLength(rrowlength, rfoffset);
2382       int lfamilylength = left.getFamilyLength(lfoffset);
2383       int rfamilylength = right.getFamilyLength(rfoffset);
2384       int ccRes = compareColumns(left.getBuffer(), lfoffset, lclength, lfamilylength,
2385           right.getBuffer(), rfoffset, rclength, rfamilylength);
2386       return ccRes == 0;
2387     }
2388 
2389     /**
2390      * Compares the row of two keyvalues for equality
2391      * @param left
2392      * @param right
2393      * @return True if rows match.
2394      */
2395     public boolean matchingRows(final KeyValue left, final KeyValue right) {
2396       short lrowlength = left.getRowLength();
2397       short rrowlength = right.getRowLength();
2398       return matchingRows(left, lrowlength, right, rrowlength);
2399     }
2400 
2401     /**
2402      * @param left
2403      * @param lrowlength
2404      * @param right
2405      * @param rrowlength
2406      * @return True if rows match.
2407      */
2408     private boolean matchingRows(final KeyValue left, final short lrowlength,
2409         final KeyValue right, final short rrowlength) {
2410       return lrowlength == rrowlength &&
2411           matchingRows(left.getBuffer(), left.getRowOffset(), lrowlength,
2412               right.getBuffer(), right.getRowOffset(), rrowlength);
2413     }
2414 
2415     /**
2416      * Compare rows. Just calls Bytes.equals, but it's good to have this encapsulated.
2417      * @param left Left row array.
2418      * @param loffset Left row offset.
2419      * @param llength Left row length.
2420      * @param right Right row array.
2421      * @param roffset Right row offset.
2422      * @param rlength Right row length.
2423      * @return Whether rows are the same row.
2424      */
2425     public boolean matchingRows(final byte [] left, final int loffset, final int llength,
2426         final byte [] right, final int roffset, final int rlength) {
2427       return Bytes.equals(left, loffset, llength, right, roffset, rlength);
2428     }
2429 
2430     public byte[] calcIndexKey(byte[] lastKeyOfPreviousBlock, byte[] firstKeyInBlock) {
2431       byte[] fakeKey = getShortMidpointKey(lastKeyOfPreviousBlock, firstKeyInBlock);
2432       if (compareFlatKey(fakeKey, firstKeyInBlock) > 0) {
2433         LOG.error("Unexpected getShortMidpointKey result, fakeKey:"
2434             + Bytes.toStringBinary(fakeKey) + ", firstKeyInBlock:"
2435             + Bytes.toStringBinary(firstKeyInBlock));
2436         return firstKeyInBlock;
2437       }
2438       if (lastKeyOfPreviousBlock != null && compareFlatKey(lastKeyOfPreviousBlock, fakeKey) >= 0) {
2439         LOG.error("Unexpected getShortMidpointKey result, lastKeyOfPreviousBlock:" +
2440             Bytes.toStringBinary(lastKeyOfPreviousBlock) + ", fakeKey:" +
2441             Bytes.toStringBinary(fakeKey));
2442         return firstKeyInBlock;
2443       }
2444       return fakeKey;
2445     }
2446 
2447     /**
2448      * This is a HFile block index key optimization.
2449      * @param leftKey
2450      * @param rightKey
2451      * @return 0 if equal, <0 if left smaller, >0 if right smaller
2452      */
2453     public byte[] getShortMidpointKey(final byte[] leftKey, final byte[] rightKey) {
2454       if (rightKey == null) {
2455         throw new IllegalArgumentException("rightKey can not be null");
2456       }
2457       if (leftKey == null) {
2458         return Arrays.copyOf(rightKey, rightKey.length);
2459       }
2460       if (compareFlatKey(leftKey, rightKey) >= 0) {
2461         throw new IllegalArgumentException("Unexpected input, leftKey:" + Bytes.toString(leftKey)
2462           + ", rightKey:" + Bytes.toString(rightKey));
2463       }
2464 
2465       short leftRowLength = Bytes.toShort(leftKey, 0);
2466       short rightRowLength = Bytes.toShort(rightKey, 0);
2467       int leftCommonLength = ROW_LENGTH_SIZE + FAMILY_LENGTH_SIZE + leftRowLength;
2468       int rightCommonLength = ROW_LENGTH_SIZE + FAMILY_LENGTH_SIZE + rightRowLength;
2469       int leftCommonLengthWithTSAndType = TIMESTAMP_TYPE_SIZE + leftCommonLength;
2470       int rightCommonLengthWithTSAndType = TIMESTAMP_TYPE_SIZE + rightCommonLength;
2471       int leftColumnLength = leftKey.length - leftCommonLengthWithTSAndType;
2472       int rightColumnLength = rightKey.length - rightCommonLengthWithTSAndType;
2473       // rows are equal
2474       if (leftRowLength == rightRowLength && compareRows(leftKey, ROW_LENGTH_SIZE, leftRowLength,
2475         rightKey, ROW_LENGTH_SIZE, rightRowLength) == 0) {
2476         // Compare family & qualifier together.
2477         int comparison = Bytes.compareTo(leftKey, leftCommonLength, leftColumnLength, rightKey,
2478           rightCommonLength, rightColumnLength);
2479         // same with "row + family + qualifier", return rightKey directly
2480         if (comparison == 0) {
2481           return Arrays.copyOf(rightKey, rightKey.length);
2482         }
2483         // "family + qualifier" are different, generate a faked key per rightKey
2484         byte[] newKey = Arrays.copyOf(rightKey, rightKey.length);
2485         Bytes.putLong(newKey, rightKey.length - TIMESTAMP_TYPE_SIZE, HConstants.LATEST_TIMESTAMP);
2486         Bytes.putByte(newKey, rightKey.length - TYPE_SIZE, Type.Maximum.getCode());
2487         return newKey;
2488       }
2489       // rows are different
2490       short minLength = leftRowLength < rightRowLength ? leftRowLength : rightRowLength;
2491       short diffIdx = 0;
2492       while (diffIdx < minLength
2493           && leftKey[ROW_LENGTH_SIZE + diffIdx] == rightKey[ROW_LENGTH_SIZE + diffIdx]) {
2494         diffIdx++;
2495       }
2496       byte[] newRowKey = null;
2497       if (diffIdx >= minLength) {
2498         // leftKey's row is prefix of rightKey's.
2499         newRowKey = new byte[diffIdx + 1];
2500         System.arraycopy(rightKey, ROW_LENGTH_SIZE, newRowKey, 0, diffIdx + 1);
2501       } else {
2502         int diffByte = leftKey[ROW_LENGTH_SIZE + diffIdx];
2503         if ((0xff & diffByte) < 0xff && (diffByte + 1) <
2504             (rightKey[ROW_LENGTH_SIZE + diffIdx] & 0xff)) {
2505           newRowKey = new byte[diffIdx + 1];
2506           System.arraycopy(leftKey, ROW_LENGTH_SIZE, newRowKey, 0, diffIdx);
2507           newRowKey[diffIdx] = (byte) (diffByte + 1);
2508         } else {
2509           newRowKey = new byte[diffIdx + 1];
2510           System.arraycopy(rightKey, ROW_LENGTH_SIZE, newRowKey, 0, diffIdx + 1);
2511         }
2512       }
2513       return new KeyValue(newRowKey, null, null, HConstants.LATEST_TIMESTAMP,
2514         Type.Maximum).getKey();
2515     }
2516 
2517     @Override
2518     protected Object clone() throws CloneNotSupportedException {
2519       return new KVComparator();
2520     }
2521 
2522   }
2523 
2524   /**
2525    * Creates a KeyValue that is last on the specified row id. That is,
2526    * every other possible KeyValue for the given row would compareTo()
2527    * less than the result of this call.
2528    * @param row row key
2529    * @return Last possible KeyValue on passed <code>row</code>
2530    */
2531   public static KeyValue createLastOnRow(final byte[] row) {
2532     return new KeyValue(row, null, null, HConstants.LATEST_TIMESTAMP, Type.Minimum);
2533   }
2534 
2535   /**
2536    * Create a KeyValue that is smaller than all other possible KeyValues
2537    * for the given row. That is any (valid) KeyValue on 'row' would sort
2538    * _after_ the result.
2539    *
2540    * @param row - row key (arbitrary byte array)
2541    * @return First possible KeyValue on passed <code>row</code>
2542    */
2543   public static KeyValue createFirstOnRow(final byte [] row) {
2544     return createFirstOnRow(row, HConstants.LATEST_TIMESTAMP);
2545   }
2546 
2547   /**
2548    * Create a KeyValue that is smaller than all other possible KeyValues
2549    * for the given row. That is any (valid) KeyValue on 'row' would sort
2550    * _after_ the result.
2551    *
2552    * @param row - row key (arbitrary byte array)
2553    * @return First possible KeyValue on passed <code>row</code>
2554    */
2555   public static KeyValue createFirstOnRow(final byte [] row, int roffset, short rlength) {
2556     return new KeyValue(row, roffset, rlength,
2557         null, 0, 0, null, 0, 0, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0);
2558   }
2559 
2560   /**
2561    * Creates a KeyValue that is smaller than all other KeyValues that
2562    * are older than the passed timestamp.
2563    * @param row - row key (arbitrary byte array)
2564    * @param ts - timestamp
2565    * @return First possible key on passed <code>row</code> and timestamp.
2566    */
2567   public static KeyValue createFirstOnRow(final byte [] row,
2568       final long ts) {
2569     return new KeyValue(row, null, null, ts, Type.Maximum);
2570   }
2571 
2572   /**
2573    * Create a KeyValue for the specified row, family and qualifier that would be
2574    * smaller than all other possible KeyValues that have the same row,family,qualifier.
2575    * Used for seeking.
2576    * @param row - row key (arbitrary byte array)
2577    * @param family - family name
2578    * @param qualifier - column qualifier
2579    * @return First possible key on passed <code>row</code>, and column.
2580    */
2581   public static KeyValue createFirstOnRow(final byte [] row, final byte [] family,
2582       final byte [] qualifier) {
2583     return new KeyValue(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Maximum);
2584   }
2585 
2586   /**
2587    * Create a Delete Family KeyValue for the specified row and family that would
2588    * be smaller than all other possible Delete Family KeyValues that have the
2589    * same row and family.
2590    * Used for seeking.
2591    * @param row - row key (arbitrary byte array)
2592    * @param family - family name
2593    * @return First Delete Family possible key on passed <code>row</code>.
2594    */
2595   public static KeyValue createFirstDeleteFamilyOnRow(final byte [] row,
2596       final byte [] family) {
2597     return new KeyValue(row, family, null, HConstants.LATEST_TIMESTAMP,
2598         Type.DeleteFamily);
2599   }
2600 
2601   /**
2602    * @param row - row key (arbitrary byte array)
2603    * @param f - family name
2604    * @param q - column qualifier
2605    * @param ts - timestamp
2606    * @return First possible key on passed <code>row</code>, column and timestamp
2607    */
2608   public static KeyValue createFirstOnRow(final byte [] row, final byte [] f,
2609       final byte [] q, final long ts) {
2610     return new KeyValue(row, f, q, ts, Type.Maximum);
2611   }
2612 
2613   /**
2614    * Create a KeyValue for the specified row, family and qualifier that would be
2615    * smaller than all other possible KeyValues that have the same row,
2616    * family, qualifier.
2617    * Used for seeking.
2618    * @param row row key
2619    * @param roffset row offset
2620    * @param rlength row length
2621    * @param family family name
2622    * @param foffset family offset
2623    * @param flength family length
2624    * @param qualifier column qualifier
2625    * @param qoffset qualifier offset
2626    * @param qlength qualifier length
2627    * @return First possible key on passed Row, Family, Qualifier.
2628    */
2629   public static KeyValue createFirstOnRow(final byte [] row,
2630       final int roffset, final int rlength, final byte [] family,
2631       final int foffset, final int flength, final byte [] qualifier,
2632       final int qoffset, final int qlength) {
2633     return new KeyValue(row, roffset, rlength, family,
2634         foffset, flength, qualifier, qoffset, qlength,
2635         HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0);
2636   }
2637 
2638   /**
2639    * Create a KeyValue for the specified row, family and qualifier that would be
2640    * smaller than all other possible KeyValues that have the same row,
2641    * family, qualifier.
2642    * Used for seeking.
2643    *
2644    * @param buffer the buffer to use for the new <code>KeyValue</code> object
2645    * @param row the value key
2646    * @param family family name
2647    * @param qualifier column qualifier
2648    *
2649    * @return First possible key on passed Row, Family, Qualifier.
2650    *
2651    * @throws IllegalArgumentException The resulting <code>KeyValue</code> object would be larger
2652    * than the provided buffer or than <code>Integer.MAX_VALUE</code>
2653    */
2654   public static KeyValue createFirstOnRow(byte [] buffer, final byte [] row,
2655       final byte [] family, final byte [] qualifier)
2656           throws IllegalArgumentException {
2657 
2658     return createFirstOnRow(buffer, 0, row, 0, row.length,
2659         family, 0, family.length,
2660         qualifier, 0, qualifier.length);
2661   }
2662 
2663   /**
2664    * Create a KeyValue for the specified row, family and qualifier that would be
2665    * smaller than all other possible KeyValues that have the same row,
2666    * family, qualifier.
2667    * Used for seeking.
2668    *
2669    * @param buffer the buffer to use for the new <code>KeyValue</code> object
2670    * @param boffset buffer offset
2671    * @param row the value key
2672    * @param roffset row offset
2673    * @param rlength row length
2674    * @param family family name
2675    * @param foffset family offset
2676    * @param flength family length
2677    * @param qualifier column qualifier
2678    * @param qoffset qualifier offset
2679    * @param qlength qualifier length
2680    *
2681    * @return First possible key on passed Row, Family, Qualifier.
2682    *
2683    * @throws IllegalArgumentException The resulting <code>KeyValue</code> object would be larger
2684    * than the provided buffer or than <code>Integer.MAX_VALUE</code>
2685    */
2686   public static KeyValue createFirstOnRow(byte [] buffer, final int boffset,
2687       final byte [] row, final int roffset, final int rlength,
2688       final byte [] family, final int foffset, final int flength,
2689       final byte [] qualifier, final int qoffset, final int qlength)
2690           throws IllegalArgumentException {
2691 
2692     long lLength = getKeyValueDataStructureSize(rlength, flength, qlength, 0);
2693 
2694     if (lLength > Integer.MAX_VALUE) {
2695       throw new IllegalArgumentException("KeyValue length " + lLength + " > " + Integer.MAX_VALUE);
2696     }
2697     int iLength = (int) lLength;
2698     if (buffer.length - boffset < iLength) {
2699       throw new IllegalArgumentException("Buffer size " + (buffer.length - boffset) + " < " +
2700           iLength);
2701     }
2702 
2703     int len = writeByteArray(buffer, boffset, row, roffset, rlength, family, foffset, flength,
2704         qualifier, qoffset, qlength, HConstants.LATEST_TIMESTAMP, KeyValue.Type.Maximum,
2705         null, 0, 0, null);
2706     return new KeyValue(buffer, boffset, len);
2707   }
2708 
2709   /**
2710    * Create a KeyValue for the specified row, family and qualifier that would be
2711    * larger than or equal to all other possible KeyValues that have the same
2712    * row, family, qualifier.
2713    * Used for reseeking.
2714    * @param row row key
2715    * @param roffset row offset
2716    * @param rlength row length
2717    * @param family family name
2718    * @param foffset family offset
2719    * @param flength family length
2720    * @param qualifier column qualifier
2721    * @param qoffset qualifier offset
2722    * @param qlength qualifier length
2723    * @return Last possible key on passed row, family, qualifier.
2724    */
2725   public static KeyValue createLastOnRow(final byte [] row,
2726       final int roffset, final int rlength, final byte [] family,
2727       final int foffset, final int flength, final byte [] qualifier,
2728       final int qoffset, final int qlength) {
2729     return new KeyValue(row, roffset, rlength, family,
2730         foffset, flength, qualifier, qoffset, qlength,
2731         HConstants.OLDEST_TIMESTAMP, Type.Minimum, null, 0, 0);
2732   }
2733 
2734   /**
2735    * Similar to {@link #createLastOnRow(byte[], int, int, byte[], int, int,
2736    * byte[], int, int)} but creates the last key on the row/column of this KV
2737    * (the value part of the returned KV is always empty). Used in creating
2738    * "fake keys" for the multi-column Bloom filter optimization to skip the
2739    * row/column we already know is not in the file.
2740    * @return the last key on the row/column of the given key-value pair
2741    */
2742   public KeyValue createLastOnRowCol() {
2743     return new KeyValue(
2744         bytes, getRowOffset(), getRowLength(),
2745         bytes, getFamilyOffset(), getFamilyLength(),
2746         bytes, getQualifierOffset(), getQualifierLength(),
2747         HConstants.OLDEST_TIMESTAMP, Type.Minimum, null, 0, 0);
2748   }
2749 
2750   /**
2751    * Creates the first KV with the row/family/qualifier of this KV and the
2752    * given timestamp. Uses the "maximum" KV type that guarantees that the new
2753    * KV is the lowest possible for this combination of row, family, qualifier,
2754    * and timestamp. This KV's own timestamp is ignored. While this function
2755    * copies the value from this KV, it is normally used on key-only KVs.
2756    */
2757   public KeyValue createFirstOnRowColTS(long ts) {
2758     return new KeyValue(
2759         bytes, getRowOffset(), getRowLength(),
2760         bytes, getFamilyOffset(), getFamilyLength(),
2761         bytes, getQualifierOffset(), getQualifierLength(),
2762         ts, Type.Maximum, bytes, getValueOffset(), getValueLength());
2763   }
2764 
2765   /**
2766    * @param b
2767    * @return A KeyValue made of a byte array that holds the key-only part.
2768    * Needed to convert hfile index members to KeyValues.
2769    */
2770   public static KeyValue createKeyValueFromKey(final byte [] b) {
2771     return createKeyValueFromKey(b, 0, b.length);
2772   }
2773 
2774   /**
2775    * @param bb
2776    * @return A KeyValue made of a byte buffer that holds the key-only part.
2777    * Needed to convert hfile index members to KeyValues.
2778    */
2779   public static KeyValue createKeyValueFromKey(final ByteBuffer bb) {
2780     return createKeyValueFromKey(bb.array(), bb.arrayOffset(), bb.limit());
2781   }
2782 
2783   /**
2784    * @param b
2785    * @param o
2786    * @param l
2787    * @return A KeyValue made of a byte array that holds the key-only part.
2788    * Needed to convert hfile index members to KeyValues.
2789    */
2790   public static KeyValue createKeyValueFromKey(final byte [] b, final int o,
2791       final int l) {
2792     byte [] newb = new byte[l + ROW_OFFSET];
2793     System.arraycopy(b, o, newb, ROW_OFFSET, l);
2794     Bytes.putInt(newb, 0, l);
2795     Bytes.putInt(newb, Bytes.SIZEOF_INT, 0);
2796     return new KeyValue(newb);
2797   }
2798 
2799   /**
2800    * @param in Where to read bytes from.  Creates a byte array to hold the KeyValue
2801    * backing bytes copied from the steam.
2802    * @return KeyValue created by deserializing from <code>in</code> OR if we find a length
2803    * of zero, we will return null which can be useful marking a stream as done.
2804    * @throws IOException
2805    */
2806   public static KeyValue create(final DataInput in) throws IOException {
2807     return create(in.readInt(), in);
2808   }
2809 
2810   /**
2811    * Create a KeyValue reading <code>length</code> from <code>in</code>
2812    * @param length
2813    * @param in
2814    * @return Created KeyValue OR if we find a length of zero, we will return null which
2815    * can be useful marking a stream as done.
2816    * @throws IOException
2817    */
2818   public static KeyValue create(int length, final DataInput in) throws IOException {
2819 
2820     if (length <= 0) {
2821       if (length == 0) return null;
2822       throw new IOException("Failed read " + length + " bytes, stream corrupt?");
2823     }
2824 
2825     // This is how the old Writables.readFrom used to deserialize.  Didn't even vint.
2826     byte [] bytes = new byte[length];
2827     in.readFully(bytes);
2828     return new KeyValue(bytes, 0, length);
2829   }
2830   
2831   /**
2832    * Create a new KeyValue by copying existing cell and adding new tags
2833    * @param c
2834    * @param newTags
2835    * @return a new KeyValue instance with new tags
2836    */
2837   public static KeyValue cloneAndAddTags(Cell c, List<Tag> newTags) {
2838     List<Tag> existingTags = null;
2839     if(c.getTagsLengthUnsigned() > 0) {
2840       existingTags = Tag.asList(c.getTagsArray(), c.getTagsOffset(), c.getTagsLengthUnsigned());
2841       existingTags.addAll(newTags);
2842     } else {
2843       existingTags = newTags;
2844     }
2845     return new KeyValue(c.getRowArray(), c.getRowOffset(), (int)c.getRowLength(),
2846       c.getFamilyArray(), c.getFamilyOffset(), (int)c.getFamilyLength(), 
2847       c.getQualifierArray(), c.getQualifierOffset(), (int) c.getQualifierLength(), 
2848       c.getTimestamp(), Type.codeToType(c.getTypeByte()), c.getValueArray(), c.getValueOffset(), 
2849       c.getValueLength(), existingTags);
2850   }
2851 
2852   /**
2853    * Create a KeyValue reading from the raw InputStream.
2854    * Named <code>iscreate</code> so doesn't clash with {@link #create(DataInput)}
2855    * @param in
2856    * @return Created KeyValue OR if we find a length of zero, we will return null which
2857    * can be useful marking a stream as done.
2858    * @throws IOException
2859    */
2860   public static KeyValue iscreate(final InputStream in) throws IOException {
2861     byte [] intBytes = new byte[Bytes.SIZEOF_INT];
2862     int bytesRead = 0;
2863     while (bytesRead < intBytes.length) {
2864       int n = in.read(intBytes, bytesRead, intBytes.length - bytesRead);
2865       if (n < 0) {
2866         if (bytesRead == 0) return null; // EOF at start is ok
2867         throw new IOException("Failed read of int, read " + bytesRead + " bytes");
2868       }
2869       bytesRead += n;
2870     }
2871     // TODO: perhaps some sanity check is needed here.
2872     byte [] bytes = new byte[Bytes.toInt(intBytes)];
2873     IOUtils.readFully(in, bytes, 0, bytes.length);
2874     return new KeyValue(bytes, 0, bytes.length);
2875   }
2876 
2877   /**
2878    * Write out a KeyValue in the manner in which we used to when KeyValue was a Writable.
2879    * @param kv
2880    * @param out
2881    * @return Length written on stream
2882    * @throws IOException
2883    * @see #create(DataInput) for the inverse function
2884    */
2885   public static long write(final KeyValue kv, final DataOutput out) throws IOException {
2886     // This is how the old Writables write used to serialize KVs.  Need to figure way to make it
2887     // work for all implementations.
2888     int length = kv.getLength();
2889     out.writeInt(length);
2890     out.write(kv.getBuffer(), kv.getOffset(), length);
2891     return length + Bytes.SIZEOF_INT;
2892   }
2893 
2894   /**
2895    * Write out a KeyValue in the manner in which we used to when KeyValue was a Writable but do
2896    * not require a {@link DataOutput}, just take plain {@link OutputStream}
2897    * Named <code>oswrite</code> so does not clash with {@link #write(KeyValue, DataOutput)}
2898    * @param kv
2899    * @param out
2900    * @return Length written on stream
2901    * @throws IOException
2902    * @see #create(DataInput) for the inverse function
2903    * @see #write(KeyValue, DataOutput)
2904    */
2905   @Deprecated
2906   public static long oswrite(final KeyValue kv, final OutputStream out)
2907       throws IOException {
2908     int length = kv.getLength();
2909     // This does same as DataOuput#writeInt (big-endian, etc.)
2910     out.write(Bytes.toBytes(length));
2911     out.write(kv.getBuffer(), kv.getOffset(), length);
2912     return length + Bytes.SIZEOF_INT;
2913   }
2914 
2915   /**
2916    * Write out a KeyValue in the manner in which we used to when KeyValue was a Writable but do
2917    * not require a {@link DataOutput}, just take plain {@link OutputStream}
2918    * Named <code>oswrite</code> so does not clash with {@link #write(KeyValue, DataOutput)}
2919    * @param kv
2920    * @param out
2921    * @param withTags
2922    * @return Length written on stream
2923    * @throws IOException
2924    * @see #create(DataInput) for the inverse function
2925    * @see #write(KeyValue, DataOutput)
2926    */
2927   public static long oswrite(final KeyValue kv, final OutputStream out, final boolean withTags)
2928       throws IOException {
2929     int length = kv.getLength();
2930     if (!withTags) {
2931       length = kv.getKeyLength() + kv.getValueLength() + KEYVALUE_INFRASTRUCTURE_SIZE;
2932     }
2933     // This does same as DataOuput#writeInt (big-endian, etc.)
2934     out.write(Bytes.toBytes(length));
2935     out.write(kv.getBuffer(), kv.getOffset(), length);
2936     return length + Bytes.SIZEOF_INT;
2937   }
2938 
2939   /**
2940    * Comparator that compares row component only of a KeyValue.
2941    */
2942   public static class RowOnlyComparator implements Comparator<KeyValue> {
2943     final KVComparator comparator;
2944 
2945     public RowOnlyComparator(final KVComparator c) {
2946       this.comparator = c;
2947     }
2948 
2949     public int compare(KeyValue left, KeyValue right) {
2950       return comparator.compareRows(left, right);
2951     }
2952   }
2953 
2954 
2955   /**
2956    * Avoids redundant comparisons for better performance.
2957    * 
2958    * TODO get rid of this wart
2959    */
2960   public interface SamePrefixComparator<T> {
2961     /**
2962      * Compare two keys assuming that the first n bytes are the same.
2963      * @param commonPrefix How many bytes are the same.
2964      */
2965     int compareIgnoringPrefix(
2966       int commonPrefix, byte[] left, int loffset, int llength, byte[] right, int roffset, int rlength
2967     );
2968   }
2969 
2970   /**
2971    * This is a TEST only Comparator used in TestSeekTo and TestReseekTo.
2972    */
2973   public static class RawBytesComparator extends KVComparator {
2974     /**
2975      * The HFileV2 file format's trailer contains this class name.  We reinterpret this and
2976      * instantiate the appropriate comparator.
2977      * TODO: With V3 consider removing this.
2978      * @return legacy class name for FileFileTrailer#comparatorClassName
2979      */
2980     public String getLegacyKeyComparatorName() {
2981       return "org.apache.hadoop.hbase.util.Bytes$ByteArrayComparator";
2982     }
2983 
2984     public int compareFlatKey(byte[] left, int loffset, int llength, byte[] right,
2985         int roffset, int rlength) {
2986       return Bytes.BYTES_RAWCOMPARATOR.compare(left,  loffset, llength, right, roffset, rlength);
2987     }
2988 
2989     public byte[] calcIndexKey(byte[] lastKeyOfPreviousBlock, byte[] firstKeyInBlock) {
2990       return firstKeyInBlock;
2991     }
2992 
2993   }
2994 
2995   /**
2996    * HeapSize implementation
2997    *
2998    * We do not count the bytes in the rowCache because it should be empty for a KeyValue in the
2999    * MemStore.
3000    */
3001   @Override
3002   public long heapSize() {
3003     int sum = 0;
3004     sum += ClassSize.OBJECT;// the KeyValue object itself
3005     sum += ClassSize.REFERENCE;// pointer to "bytes"
3006     sum += ClassSize.align(ClassSize.ARRAY);// "bytes"
3007     sum += ClassSize.align(length);// number of bytes of data in the "bytes" array
3008     sum += 2 * Bytes.SIZEOF_INT;// offset, length
3009     sum += Bytes.SIZEOF_LONG;// memstoreTS
3010     return ClassSize.align(sum);
3011   }
3012 
3013   /**
3014    * This is a hack that should be removed once we don't care about matching
3015    * up client- and server-side estimations of cell size. It needed to be
3016    * backwards compatible with estimations done by older clients. We need to
3017    * pretend that tags never exist and KeyValues aren't serialized with tag
3018    * length included. See HBASE-13262 and HBASE-13303
3019    */
3020   @Deprecated
3021   public long heapSizeWithoutTags() {
3022     int sum = 0;
3023     sum += ClassSize.OBJECT;// the KeyValue object itself
3024     sum += ClassSize.REFERENCE;// pointer to "bytes"
3025     sum += ClassSize.align(ClassSize.ARRAY);// "bytes"
3026     sum += KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE;
3027     sum += getKeyLength();
3028     sum += getValueLength();
3029     sum += 2 * Bytes.SIZEOF_INT;// offset, length
3030     sum += Bytes.SIZEOF_LONG;// memstoreTS
3031     return ClassSize.align(sum);
3032   }
3033 }