View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import java.io.ByteArrayInputStream;
22  import java.io.ByteArrayOutputStream;
23  import java.io.DataInputStream;
24  import java.io.DataOutputStream;
25  import java.io.IOException;
26  import java.util.Arrays;
27  import java.util.Collections;
28  import java.util.Iterator;
29  import java.util.List;
30  import java.util.Set;
31  import java.util.TreeSet;
32  
33  import junit.framework.TestCase;
34  
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.hbase.KeyValue.KVComparator;
38  import org.apache.hadoop.hbase.KeyValue.MetaComparator;
39  import org.apache.hadoop.hbase.KeyValue.Type;
40  import org.apache.hadoop.hbase.util.Bytes;
41  
42  public class TestKeyValue extends TestCase {
43    private final Log LOG = LogFactory.getLog(this.getClass().getName());
44  
45    public void testColumnCompare() throws Exception {
46      final byte [] a = Bytes.toBytes("aaa");
47      byte [] family1 = Bytes.toBytes("abc");
48      byte [] qualifier1 = Bytes.toBytes("def");
49      byte [] family2 = Bytes.toBytes("abcd");
50      byte [] qualifier2 = Bytes.toBytes("ef");
51  
52      KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, Type.Put, a);
53      assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2));
54      assertTrue(CellUtil.matchingColumn(aaa, family1, qualifier1));
55      aaa = new KeyValue(a, family2, qualifier2, 0L, Type.Put, a);
56      assertFalse(CellUtil.matchingColumn(aaa, family1, qualifier1));
57      assertTrue(CellUtil.matchingColumn(aaa, family2,qualifier2));
58      byte [] nullQualifier = new byte[0];
59      aaa = new KeyValue(a, family1, nullQualifier, 0L, Type.Put, a);
60      assertTrue(CellUtil.matchingColumn(aaa, family1,null));
61      assertFalse(CellUtil.matchingColumn(aaa, family2,qualifier2));
62    }
63  
64    /**
65     * Test a corner case when the family qualifier is a prefix of the
66     *  column qualifier.
67     */
68    public void testColumnCompare_prefix() throws Exception {
69      final byte [] a = Bytes.toBytes("aaa");
70      byte [] family1 = Bytes.toBytes("abc");
71      byte [] qualifier1 = Bytes.toBytes("def");
72      byte [] family2 = Bytes.toBytes("ab");
73      byte [] qualifier2 = Bytes.toBytes("def");
74  
75      KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, Type.Put, a);
76      assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2));
77    }
78  
79    public void testBasics() throws Exception {
80      LOG.info("LOWKEY: " + KeyValue.LOWESTKEY.toString());
81      check(Bytes.toBytes(getName()),
82        Bytes.toBytes(getName()), Bytes.toBytes(getName()), 1,
83        Bytes.toBytes(getName()));
84      // Test empty value and empty column -- both should work. (not empty fam)
85      check(Bytes.toBytes(getName()), Bytes.toBytes(getName()), null, 1, null);
86      check(HConstants.EMPTY_BYTE_ARRAY, Bytes.toBytes(getName()), null, 1, null);
87      // empty qual is equivalent to null qual
88      assertEquals(
89        new KeyValue(Bytes.toBytes("rk"), Bytes.toBytes("fam"), null, 1, (byte[]) null),
90        new KeyValue(Bytes.toBytes("rk"), Bytes.toBytes("fam"),
91          HConstants.EMPTY_BYTE_ARRAY, 1, (byte[]) null));
92    }
93  
94    private void check(final byte [] row, final byte [] family, byte [] qualifier,
95      final long timestamp, final byte [] value) {
96      KeyValue kv = new KeyValue(row, family, qualifier, timestamp, value);
97      assertTrue(Bytes.compareTo(kv.getRow(), row) == 0);
98      assertTrue(CellUtil.matchingColumn(kv, family, qualifier));
99      // Call toString to make sure it works.
100     LOG.info(kv.toString());
101   }
102 
103   public void testPlainCompare() throws Exception {
104     final byte [] a = Bytes.toBytes("aaa");
105     final byte [] b = Bytes.toBytes("bbb");
106     final byte [] fam = Bytes.toBytes("col");
107     final byte [] qf = Bytes.toBytes("umn");
108     KeyValue aaa = new KeyValue(a, fam, qf, a);
109     KeyValue bbb = new KeyValue(b, fam, qf, b);
110     assertTrue(KeyValue.COMPARATOR.compare(aaa, bbb) < 0);
111     assertTrue(KeyValue.COMPARATOR.compare(bbb, aaa) > 0);
112     // Compare breaks if passed same ByteBuffer as both left and right arguments.
113     assertTrue(KeyValue.COMPARATOR.compare(bbb, bbb) == 0);
114     assertTrue(KeyValue.COMPARATOR.compare(aaa, aaa) == 0);
115     // Do compare with different timestamps.
116     aaa = new KeyValue(a, fam, qf, 1, a);
117     bbb = new KeyValue(a, fam, qf, 2, a);
118     assertTrue(KeyValue.COMPARATOR.compare(aaa, bbb) > 0);
119     assertTrue(KeyValue.COMPARATOR.compare(bbb, aaa) < 0);
120     assertTrue(KeyValue.COMPARATOR.compare(aaa, aaa) == 0);
121     // Do compare with different types.  Higher numbered types -- Delete
122     // should sort ahead of lower numbers; i.e. Put
123     aaa = new KeyValue(a, fam, qf, 1, KeyValue.Type.Delete, a);
124     bbb = new KeyValue(a, fam, qf, 1, a);
125     assertTrue(KeyValue.COMPARATOR.compare(aaa, bbb) < 0);
126     assertTrue(KeyValue.COMPARATOR.compare(bbb, aaa) > 0);
127     assertTrue(KeyValue.COMPARATOR.compare(aaa, aaa) == 0);
128   }
129 
130   public void testMoreComparisons() throws Exception {
131     long now = System.currentTimeMillis();
132 
133     // Meta compares
134     KeyValue aaa = new KeyValue(
135         Bytes.toBytes("TestScanMultipleVersions,row_0500,1236020145502"), now);
136     KeyValue bbb = new KeyValue(
137         Bytes.toBytes("TestScanMultipleVersions,,99999999999999"), now);
138     KVComparator c = new KeyValue.MetaComparator();
139     assertTrue(c.compare(bbb, aaa) < 0);
140 
141     KeyValue aaaa = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,,1236023996656"),
142         Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236024396271L,
143         (byte[])null);
144     assertTrue(c.compare(aaaa, bbb) < 0);
145 
146     KeyValue x = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"),
147         Bytes.toBytes("info"), Bytes.toBytes(""), 9223372036854775807L,
148         (byte[])null);
149     KeyValue y = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"),
150         Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236034574912L,
151         (byte[])null);
152     assertTrue(c.compare(x, y) < 0);
153     comparisons(new KeyValue.MetaComparator());
154     comparisons(new KeyValue.KVComparator());
155     metacomparisons(new KeyValue.MetaComparator());
156   }
157 
158   public void testMetaComparatorTableKeysWithCommaOk() {
159     MetaComparator c = new KeyValue.MetaComparator();
160     long now = System.currentTimeMillis();
161     // meta keys values are not quite right.  A users can enter illegal values
162     // from shell when scanning meta.
163     KeyValue a = new KeyValue(Bytes.toBytes("table,key,with,commas1,1234"), now);
164     KeyValue b = new KeyValue(Bytes.toBytes("table,key,with,commas2,0123"), now);
165     assertTrue(c.compare(a, b) < 0);
166   }
167 
168   /**
169    * Tests cases where rows keys have characters below the ','.
170    * See HBASE-832
171    * @throws IOException
172    */
173   public void testKeyValueBorderCases() throws IOException {
174     // % sorts before , so if we don't do special comparator, rowB would
175     // come before rowA.
176     KeyValue rowA = new KeyValue(Bytes.toBytes("testtable,www.hbase.org/,1234"),
177       Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null);
178     KeyValue rowB = new KeyValue(Bytes.toBytes("testtable,www.hbase.org/%20,99999"),
179         Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null);
180     assertTrue(KeyValue.META_COMPARATOR.compare(rowA, rowB) < 0);
181 
182     rowA = new KeyValue(Bytes.toBytes("testtable,,1234"), Bytes.toBytes("fam"),
183         Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null);
184     rowB = new KeyValue(Bytes.toBytes("testtable,$www.hbase.org/,99999"),
185         Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null);
186     assertTrue(KeyValue.META_COMPARATOR.compare(rowA, rowB) < 0);
187 
188   }
189 
190   private void metacomparisons(final KeyValue.MetaComparator c) {
191     long now = System.currentTimeMillis();
192     assertTrue(c.compare(new KeyValue(
193         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,1"), now),
194       new KeyValue(
195           Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,1"), now)) == 0);
196     KeyValue a = new KeyValue(
197         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,1"), now);
198     KeyValue b = new KeyValue(
199         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,2"), now);
200     assertTrue(c.compare(a, b) < 0);
201     assertTrue(c.compare(new KeyValue(
202         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,2"), now),
203       new KeyValue(
204           Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,1"), now)) > 0);
205   }
206 
207   private void comparisons(final KeyValue.KVComparator c) {
208     long now = System.currentTimeMillis();
209     assertTrue(c.compare(new KeyValue(
210         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,1"), now),
211       new KeyValue(
212           Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,1"), now)) == 0);
213     assertTrue(c.compare(new KeyValue(
214         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,1"), now),
215       new KeyValue(
216           Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,2"), now)) < 0);
217     assertTrue(c.compare(new KeyValue(
218         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,2"), now),
219       new KeyValue(
220           Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,1"), now)) > 0);
221   }
222 
223   public void testBinaryKeys() throws Exception {
224     Set<KeyValue> set = new TreeSet<KeyValue>(KeyValue.COMPARATOR);
225     final byte [] fam = Bytes.toBytes("col");
226     final byte [] qf = Bytes.toBytes("umn");
227     final byte [] nb = new byte[0];
228     KeyValue [] keys = {new KeyValue(Bytes.toBytes("aaaaa,\u0000\u0000,2"), fam, qf, 2, nb),
229       new KeyValue(Bytes.toBytes("aaaaa,\u0001,3"), fam, qf, 3, nb),
230       new KeyValue(Bytes.toBytes("aaaaa,,1"), fam, qf, 1, nb),
231       new KeyValue(Bytes.toBytes("aaaaa,\u1000,5"), fam, qf, 5, nb),
232       new KeyValue(Bytes.toBytes("aaaaa,a,4"), fam, qf, 4, nb),
233       new KeyValue(Bytes.toBytes("a,a,0"), fam, qf, 0, nb),
234     };
235     // Add to set with bad comparator
236     Collections.addAll(set, keys);
237     // This will output the keys incorrectly.
238     boolean assertion = false;
239     int count = 0;
240     try {
241       for (KeyValue k: set) {
242         assertTrue(count++ == k.getTimestamp());
243       }
244     } catch (junit.framework.AssertionFailedError e) {
245       // Expected
246       assertion = true;
247     }
248     assertTrue(assertion);
249     // Make set with good comparator
250     set = new TreeSet<KeyValue>(new KeyValue.MetaComparator());
251     Collections.addAll(set, keys);
252     count = 0;
253     for (KeyValue k: set) {
254       assertTrue(count++ == k.getTimestamp());
255     }
256   }
257 
258   public void testStackedUpKeyValue() {
259     // Test multiple KeyValues in a single blob.
260 
261     // TODO actually write this test!
262 
263   }
264 
265   private final byte[] rowA = Bytes.toBytes("rowA");
266   private final byte[] rowB = Bytes.toBytes("rowB");
267 
268   private final byte[] family = Bytes.toBytes("family");
269   private final byte[] qualA = Bytes.toBytes("qfA");
270   private final byte[] qualB = Bytes.toBytes("qfB");
271 
272   private void assertKVLess(KeyValue.KVComparator c,
273                             KeyValue less,
274                             KeyValue greater) {
275     int cmp = c.compare(less,greater);
276     assertTrue(cmp < 0);
277     cmp = c.compare(greater,less);
278     assertTrue(cmp > 0);
279   }
280 
281   private void assertKVLessWithoutRow(KeyValue.KVComparator c, int common, KeyValue less,
282       KeyValue greater) {
283     int cmp = c.compareIgnoringPrefix(common, less.getBuffer(), less.getOffset()
284         + KeyValue.ROW_OFFSET, less.getKeyLength(), greater.getBuffer(),
285         greater.getOffset() + KeyValue.ROW_OFFSET, greater.getKeyLength());
286     assertTrue(cmp < 0);
287     cmp = c.compareIgnoringPrefix(common, greater.getBuffer(), greater.getOffset()
288         + KeyValue.ROW_OFFSET, greater.getKeyLength(), less.getBuffer(),
289         less.getOffset() + KeyValue.ROW_OFFSET, less.getKeyLength());
290     assertTrue(cmp > 0);
291   }
292 
293   public void testCompareWithoutRow() {
294     final KeyValue.KVComparator c = KeyValue.COMPARATOR;
295     byte[] row = Bytes.toBytes("row");
296 
297     byte[] fa = Bytes.toBytes("fa");
298     byte[] fami = Bytes.toBytes("fami");
299     byte[] fami1 = Bytes.toBytes("fami1");
300 
301     byte[] qual0 = Bytes.toBytes("");
302     byte[] qual1 = Bytes.toBytes("qf1");
303     byte[] qual2 = Bytes.toBytes("qf2");
304     long ts = 1;
305 
306     // 'fa:'
307     KeyValue kv_0 = new KeyValue(row, fa, qual0, ts, Type.Put);
308     // 'fami:'
309     KeyValue kv0_0 = new KeyValue(row, fami, qual0, ts, Type.Put);
310     // 'fami:qf1'
311     KeyValue kv0_1 = new KeyValue(row, fami, qual1, ts, Type.Put);
312     // 'fami:qf2'
313     KeyValue kv0_2 = new KeyValue(row, fami, qual2, ts, Type.Put);
314     // 'fami1:'
315     KeyValue kv1_0 = new KeyValue(row, fami1, qual0, ts, Type.Put);
316 
317     // 'fami:qf1' < 'fami:qf2'
318     assertKVLessWithoutRow(c, 0, kv0_1, kv0_2);
319     // 'fami:qf1' < 'fami1:'
320     assertKVLessWithoutRow(c, 0, kv0_1, kv1_0);
321 
322     // Test comparison by skipping the same prefix bytes.
323     /***
324      * KeyValue Format and commonLength:
325      * |_keyLen_|_valLen_|_rowLen_|_rowKey_|_famiLen_|_fami_|_Quali_|....
326      * ------------------|-------commonLength--------|--------------
327      */
328     int commonLength = KeyValue.ROW_LENGTH_SIZE + KeyValue.FAMILY_LENGTH_SIZE
329         + row.length;
330     // 'fa:' < 'fami:'. They have commonPrefix + 2 same prefix bytes.
331     assertKVLessWithoutRow(c, commonLength + 2, kv_0, kv0_0);
332     // 'fami:' < 'fami:qf1'. They have commonPrefix + 4 same prefix bytes.
333     assertKVLessWithoutRow(c, commonLength + 4, kv0_0, kv0_1);
334     // 'fami:qf1' < 'fami1:'. They have commonPrefix + 4 same prefix bytes.
335     assertKVLessWithoutRow(c, commonLength + 4, kv0_1, kv1_0);
336     // 'fami:qf1' < 'fami:qf2'. They have commonPrefix + 6 same prefix bytes.
337     assertKVLessWithoutRow(c, commonLength + 6, kv0_1, kv0_2);
338   }
339 
340   public void testFirstLastOnRow() {
341     final KVComparator c = KeyValue.COMPARATOR;
342     long ts = 1;
343     byte[] bufferA = new byte[128];
344     int offsetA = 0;
345     byte[] bufferB = new byte[128];
346     int offsetB = 7;
347 
348     // These are listed in sort order (ie: every one should be less
349     // than the one on the next line).
350     final KeyValue firstOnRowA = KeyValueUtil.createFirstOnRow(rowA);
351     final KeyValue firstOnRowABufferFamQual = KeyValueUtil.createFirstOnRow(bufferA, offsetA,
352         rowA, 0, rowA.length, family, 0, family.length, qualA, 0, qualA.length);
353     final KeyValue kvA_1 = new KeyValue(rowA, null, null, ts, Type.Put);
354     final KeyValue kvA_2 = new KeyValue(rowA, family, qualA, ts, Type.Put);
355 
356     final KeyValue lastOnRowA = KeyValueUtil.createLastOnRow(rowA);
357     final KeyValue firstOnRowB = KeyValueUtil.createFirstOnRow(rowB);
358     final KeyValue firstOnRowBBufferFam = KeyValueUtil.createFirstOnRow(bufferB, offsetB,
359         rowB, 0, rowB.length, family, 0, family.length, null, 0, 0);
360     final KeyValue kvB = new KeyValue(rowB, family, qualA, ts, Type.Put);
361 
362     assertKVLess(c, firstOnRowA, firstOnRowB);
363     assertKVLess(c, firstOnRowA, firstOnRowBBufferFam);
364     assertKVLess(c, firstOnRowABufferFamQual, firstOnRowB);
365     assertKVLess(c, firstOnRowA, kvA_1);
366     assertKVLess(c, firstOnRowA, kvA_2);
367     assertKVLess(c, firstOnRowABufferFamQual, kvA_2);
368     assertKVLess(c, kvA_1, kvA_2);
369     assertKVLess(c, kvA_2, firstOnRowB);
370     assertKVLess(c, kvA_1, firstOnRowB);
371     assertKVLess(c, kvA_2, firstOnRowBBufferFam);
372     assertKVLess(c, kvA_1, firstOnRowBBufferFam);
373 
374     assertKVLess(c, lastOnRowA, firstOnRowB);
375     assertKVLess(c, lastOnRowA, firstOnRowBBufferFam);
376     assertKVLess(c, firstOnRowB, kvB);
377     assertKVLess(c, firstOnRowBBufferFam, kvB);
378     assertKVLess(c, lastOnRowA, kvB);
379 
380     assertKVLess(c, kvA_2, lastOnRowA);
381     assertKVLess(c, kvA_1, lastOnRowA);
382     assertKVLess(c, firstOnRowA, lastOnRowA);
383     assertKVLess(c, firstOnRowABufferFamQual, lastOnRowA);
384   }
385 
386   public void testCreateKeyOnly() throws Exception {
387     long ts = 1;
388     byte [] value = Bytes.toBytes("a real value");
389     byte [] evalue = new byte[0]; // empty value
390 
391     for (byte[] val : new byte[][]{value, evalue}) {
392       for (boolean useLen : new boolean[]{false,true}) {
393         KeyValue kv1 = new KeyValue(rowA, family, qualA, ts, val);
394         KeyValue kv1ko = kv1.createKeyOnly(useLen);
395         // keys are still the same
396         assertTrue(kv1.equals(kv1ko));
397         // but values are not
398         assertTrue(kv1ko.getValue().length == (useLen?Bytes.SIZEOF_INT:0));
399         if (useLen) {
400           assertEquals(kv1.getValueLength(), Bytes.toInt(kv1ko.getValue()));
401         }
402       }
403     }
404   }
405 
406   public void testCreateKeyValueFromKey() {
407     KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),
408         Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("myValue"));
409     int initialPadding = 10;
410     int endingPadding = 20;
411     int keyLen = kv.getKeyLength();
412     byte[] tmpArr = new byte[initialPadding + endingPadding + keyLen];
413     System.arraycopy(kv.getBuffer(), kv.getKeyOffset(), tmpArr,
414         initialPadding, keyLen);
415     KeyValue kvFromKey = KeyValue.createKeyValueFromKey(tmpArr, initialPadding,
416         keyLen);
417     assertEquals(keyLen, kvFromKey.getKeyLength());
418     assertEquals(KeyValue.ROW_OFFSET + keyLen, kvFromKey.getBuffer().length);
419     System.err.println("kv=" + kv);
420     System.err.println("kvFromKey=" + kvFromKey);
421     assertEquals(kvFromKey.toString(),
422         kv.toString().replaceAll("=[0-9]+", "=0"));
423   }
424 
425   /**
426    * Tests that getTimestamp() does always return the proper timestamp, even after updating it.
427    * See HBASE-6265.
428    */
429   public void testGetTimestamp() {
430     KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),
431       Bytes.toBytes("myQualifier"), HConstants.LATEST_TIMESTAMP,
432       Bytes.toBytes("myValue"));
433     long time1 = kv.getTimestamp();
434     kv.updateLatestStamp(Bytes.toBytes(12345L));
435     long time2 = kv.getTimestamp();
436     assertEquals(HConstants.LATEST_TIMESTAMP, time1);
437     assertEquals(12345L, time2);
438   }
439 
440   /**
441    * See HBASE-7845
442    */
443   public void testGetShortMidpointKey() {
444     final KVComparator keyComparator = KeyValue.COMPARATOR;
445     //verify that faked shorter rowkey could be generated
446     long ts = 5;
447     KeyValue kv1 = new KeyValue(Bytes.toBytes("the quick brown fox"), family, qualA, ts, Type.Put);
448     KeyValue kv2 = new KeyValue(Bytes.toBytes("the who test text"), family, qualA, ts, Type.Put);
449     byte[] newKey = keyComparator.getShortMidpointKey(kv1.getKey(), kv2.getKey());
450     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), newKey) < 0);
451     assertTrue(keyComparator.compareFlatKey(newKey, kv2.getKey()) < 0);
452     short newRowLength = Bytes.toShort(newKey, 0);
453     byte[] expectedArray = Bytes.toBytes("the r");
454     Bytes.equals(newKey, KeyValue.ROW_LENGTH_SIZE, newRowLength, expectedArray, 0,
455       expectedArray.length);
456 
457     //verify: same with "row + family + qualifier", return rightKey directly
458     kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 5, Type.Put);
459     kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 0, Type.Put);
460     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), kv2.getKey()) < 0);
461     newKey = keyComparator.getShortMidpointKey(kv1.getKey(), kv2.getKey());
462     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), newKey) < 0);
463     assertTrue(keyComparator.compareFlatKey(newKey, kv2.getKey()) == 0);
464     kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, -5, Type.Put);
465     kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, -10, Type.Put);
466     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), kv2.getKey()) < 0);
467     newKey = keyComparator.getShortMidpointKey(kv1.getKey(), kv2.getKey());
468     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), newKey) < 0);
469     assertTrue(keyComparator.compareFlatKey(newKey, kv2.getKey()) == 0);
470 
471     // verify: same with row, different with qualifier
472     kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 5, Type.Put);
473     kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualB, 5, Type.Put);
474     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), kv2.getKey()) < 0);
475     newKey = keyComparator.getShortMidpointKey(kv1.getKey(), kv2.getKey());
476     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), newKey) < 0);
477     assertTrue(keyComparator.compareFlatKey(newKey, kv2.getKey()) < 0);
478     KeyValue newKeyValue = KeyValue.createKeyValueFromKey(newKey);
479     assertTrue(Arrays.equals(newKeyValue.getFamily(),family));
480     assertTrue(Arrays.equals(newKeyValue.getQualifier(),qualB));
481     assertTrue(newKeyValue.getTimestamp() == HConstants.LATEST_TIMESTAMP);
482     assertTrue(newKeyValue.getTypeByte() == Type.Maximum.getCode());
483 
484     //verify metaKeyComparator's getShortMidpointKey output
485     final KVComparator metaKeyComparator = KeyValue.META_COMPARATOR;
486     kv1 = new KeyValue(Bytes.toBytes("ilovehbase123"), family, qualA, 5, Type.Put);
487     kv2 = new KeyValue(Bytes.toBytes("ilovehbase234"), family, qualA, 0, Type.Put);
488     newKey = metaKeyComparator.getShortMidpointKey(kv1.getKey(), kv2.getKey());
489     assertTrue(metaKeyComparator.compareFlatKey(kv1.getKey(), newKey) < 0);
490     assertTrue(metaKeyComparator.compareFlatKey(newKey, kv2.getKey()) == 0);
491 
492     //verify common fix scenario
493     kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, ts, Type.Put);
494     kv2 = new KeyValue(Bytes.toBytes("ilovehbaseandhdfs"), family, qualA, ts, Type.Put);
495     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), kv2.getKey()) < 0);
496     newKey = keyComparator.getShortMidpointKey(kv1.getKey(), kv2.getKey());
497     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), newKey) < 0);
498     assertTrue(keyComparator.compareFlatKey(newKey, kv2.getKey()) < 0);
499     newRowLength = Bytes.toShort(newKey, 0);
500     expectedArray = Bytes.toBytes("ilovehbasea");
501     Bytes.equals(newKey, KeyValue.ROW_LENGTH_SIZE, newRowLength, expectedArray, 0,
502       expectedArray.length);
503     //verify only 1 offset scenario
504     kv1 = new KeyValue(Bytes.toBytes("100abcdefg"), family, qualA, ts, Type.Put);
505     kv2 = new KeyValue(Bytes.toBytes("101abcdefg"), family, qualA, ts, Type.Put);
506     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), kv2.getKey()) < 0);
507     newKey = keyComparator.getShortMidpointKey(kv1.getKey(), kv2.getKey());
508     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), newKey) < 0);
509     assertTrue(keyComparator.compareFlatKey(newKey, kv2.getKey()) < 0);
510     newRowLength = Bytes.toShort(newKey, 0);
511     expectedArray = Bytes.toBytes("101");
512     Bytes.equals(newKey, KeyValue.ROW_LENGTH_SIZE, newRowLength, expectedArray, 0,
513       expectedArray.length);
514   }
515 
516   public void testKVsWithTags() {
517     byte[] row = Bytes.toBytes("myRow");
518     byte[] cf = Bytes.toBytes("myCF");
519     byte[] q = Bytes.toBytes("myQualifier");
520     byte[] value = Bytes.toBytes("myValue");
521     byte[] metaValue1 = Bytes.toBytes("metaValue1");
522     byte[] metaValue2 = Bytes.toBytes("metaValue2");
523     KeyValue kv = new KeyValue(row, cf, q, HConstants.LATEST_TIMESTAMP, value, new Tag[] {
524         new Tag((byte) 1, metaValue1), new Tag((byte) 2, metaValue2) });
525     assertTrue(kv.getTagsLength() > 0);
526     assertTrue(Bytes.equals(kv.getRow(), row));
527     assertTrue(Bytes.equals(kv.getFamily(), cf));
528     assertTrue(Bytes.equals(kv.getQualifier(), q));
529     assertTrue(Bytes.equals(kv.getValue(), value));
530     List<Tag> tags = kv.getTags();
531     assertNotNull(tags);
532     assertEquals(2, tags.size());
533     boolean meta1Ok = false, meta2Ok = false;
534     for (Tag tag : tags) {
535       if (tag.getType() == (byte) 1) {
536         if (Bytes.equals(tag.getValue(), metaValue1)) {
537           meta1Ok = true;
538         }
539       } else {
540         if (Bytes.equals(tag.getValue(), metaValue2)) {
541           meta2Ok = true;
542         }
543       }
544     }
545     assertTrue(meta1Ok);
546     assertTrue(meta2Ok);
547     Iterator<Tag> tagItr = CellUtil.tagsIterator(kv.getTagsArray(), kv.getTagsOffset(),
548         kv.getTagsLength());
549     //Iterator<Tag> tagItr = kv.tagsIterator();
550     assertTrue(tagItr.hasNext());
551     Tag next = tagItr.next();
552     assertEquals(10, next.getTagLength());
553     assertEquals((byte) 1, next.getType());
554     Bytes.equals(next.getValue(), metaValue1);
555     assertTrue(tagItr.hasNext());
556     next = tagItr.next();
557     assertEquals(10, next.getTagLength());
558     assertEquals((byte) 2, next.getType());
559     Bytes.equals(next.getValue(), metaValue2);
560     assertFalse(tagItr.hasNext());
561 
562     tagItr = CellUtil.tagsIterator(kv.getTagsArray(), kv.getTagsOffset(),
563         kv.getTagsLength());
564     assertTrue(tagItr.hasNext());
565     next = tagItr.next();
566     assertEquals(10, next.getTagLength());
567     assertEquals((byte) 1, next.getType());
568     Bytes.equals(next.getValue(), metaValue1);
569     assertTrue(tagItr.hasNext());
570     next = tagItr.next();
571     assertEquals(10, next.getTagLength());
572     assertEquals((byte) 2, next.getType());
573     Bytes.equals(next.getValue(), metaValue2);
574     assertFalse(tagItr.hasNext());
575   }
576   
577   public void testMetaKeyComparator() {
578     MetaComparator c = new KeyValue.MetaComparator();
579     long now = System.currentTimeMillis();
580 
581     KeyValue a = new KeyValue(Bytes.toBytes("table1"), now);
582     KeyValue b = new KeyValue(Bytes.toBytes("table2"), now);
583     assertTrue(c.compare(a, b) < 0);
584     
585     a = new KeyValue(Bytes.toBytes("table1,111"), now);
586     b = new KeyValue(Bytes.toBytes("table2"), now);
587     assertTrue(c.compare(a, b) < 0);
588     
589     a = new KeyValue(Bytes.toBytes("table1"), now);
590     b = new KeyValue(Bytes.toBytes("table2,111"), now);
591     assertTrue(c.compare(a, b) < 0);
592     
593     a = new KeyValue(Bytes.toBytes("table,111"), now);
594     b = new KeyValue(Bytes.toBytes("table,2222"), now);
595     assertTrue(c.compare(a, b) < 0);
596     
597     a = new KeyValue(Bytes.toBytes("table,111,aaaa"), now);
598     b = new KeyValue(Bytes.toBytes("table,2222"), now);
599     assertTrue(c.compare(a, b) < 0);
600     
601     a = new KeyValue(Bytes.toBytes("table,111"), now);
602     b = new KeyValue(Bytes.toBytes("table,2222.bbb"), now);
603     assertTrue(c.compare(a, b) < 0);
604 
605     a = new KeyValue(Bytes.toBytes("table,,aaaa"), now);
606     b = new KeyValue(Bytes.toBytes("table,111,bbb"), now);
607     assertTrue(c.compare(a, b) < 0);
608     
609     a = new KeyValue(Bytes.toBytes("table,111,aaaa"), now);
610     b = new KeyValue(Bytes.toBytes("table,111,bbb"), now);
611     assertTrue(c.compare(a, b) < 0);
612 
613     a = new KeyValue(Bytes.toBytes("table,111,xxxx"), now);
614     b = new KeyValue(Bytes.toBytes("table,111,222,bbb"), now);
615     assertTrue(c.compare(a, b) < 0);
616     
617     a = new KeyValue(Bytes.toBytes("table,111,11,xxx"), now);
618     b = new KeyValue(Bytes.toBytes("table,111,222,bbb"), now);
619     assertTrue(c.compare(a, b) < 0);
620   }
621 
622   public void testKeyValueSerialization() throws Exception {
623     KeyValue kvA1 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
624         Bytes.toBytes("1"));
625     KeyValue kvA2 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
626         Bytes.toBytes("2"));
627     MockKeyValue mkvA1 = new MockKeyValue(kvA1);
628     MockKeyValue mkvA2 = new MockKeyValue(kvA2);
629     ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
630     DataOutputStream os = new DataOutputStream(byteArrayOutputStream);
631     KeyValueUtil.oswrite(mkvA1, os, true);
632     KeyValueUtil.oswrite(mkvA2, os, true);
633     DataInputStream is = new DataInputStream(new ByteArrayInputStream(
634         byteArrayOutputStream.toByteArray()));
635     KeyValue deSerKV1 = KeyValue.iscreate(is);
636     assertTrue(kvA1.equals(deSerKV1));
637     KeyValue deSerKV2 = KeyValue.iscreate(is);
638     assertTrue(kvA2.equals(deSerKV2));
639   }
640 
641   private class MockKeyValue implements Cell {
642     private final KeyValue kv;
643 
644     public MockKeyValue(KeyValue kv) {
645       this.kv = kv;
646     }
647 
648     /**
649      * This returns the offset where the tag actually starts.
650      */
651     @Override
652     public int getTagsOffset() {
653       return this.kv.getTagsOffset();
654     }
655 
656     // used to achieve atomic operations in the memstore.
657     @Override
658     public long getMvccVersion() {
659       return this.kv.getMvccVersion();
660     }
661 
662     /**
663      * used to achieve atomic operations in the memstore.
664      */
665     @Override
666     public long getSequenceId() {
667       return this.kv.getSequenceId();
668     }
669 
670     /**
671      * This returns the total length of the tag bytes
672      */
673     @Override
674     public int getTagsLength() {
675       return this.kv.getTagsLength();
676     }
677 
678     /**
679      * 
680      * @return Timestamp
681      */
682     @Override
683     public long getTimestamp() {
684       return this.kv.getTimestamp();
685     }
686 
687     /**
688      * @return KeyValue.TYPE byte representation
689      */
690     @Override
691     public byte getTypeByte() {
692       return this.kv.getTypeByte();
693     }
694 
695     /**
696      * @return the backing array of the entire KeyValue (all KeyValue fields are
697      *         in a single array)
698      */
699     @Override
700     public byte[] getValueArray() {
701       return this.kv.getValueArray();
702     }
703 
704     /**
705      * @return the value offset
706      */
707     @Override
708     public int getValueOffset() {
709       return this.kv.getValueOffset();
710     }
711 
712     /**
713      * @return Value length
714      */
715     @Override
716     public int getValueLength() {
717       return this.kv.getValueLength();
718     }
719 
720     /**
721      * @return the backing array of the entire KeyValue (all KeyValue fields are
722      *         in a single array)
723      */
724     @Override
725     public byte[] getRowArray() {
726       return this.kv.getRowArray();
727     }
728 
729     /**
730      * @return Row offset
731      */
732     @Override
733     public int getRowOffset() {
734       return this.kv.getRowOffset();
735     }
736 
737     /**
738      * @return Row length
739      */
740     @Override
741     public short getRowLength() {
742       return this.kv.getRowLength();
743     }
744 
745     /**
746      * @return the backing array of the entire KeyValue (all KeyValue fields are
747      *         in a single array)
748      */
749     @Override
750     public byte[] getFamilyArray() {
751       return this.kv.getFamilyArray();
752     }
753 
754     /**
755      * @return Family offset
756      */
757     @Override
758     public int getFamilyOffset() {
759       return this.kv.getFamilyOffset();
760     }
761 
762     /**
763      * @return Family length
764      */
765     @Override
766     public byte getFamilyLength() {
767       return this.kv.getFamilyLength();
768     }
769 
770     /**
771      * @return the backing array of the entire KeyValue (all KeyValue fields are
772      *         in a single array)
773      */
774     @Override
775     public byte[] getQualifierArray() {
776       return this.kv.getQualifierArray();
777     }
778 
779     /**
780      * @return Qualifier offset
781      */
782     @Override
783     public int getQualifierOffset() {
784       return this.kv.getQualifierOffset();
785     }
786 
787     /**
788      * @return Qualifier length
789      */
790     @Override
791     public int getQualifierLength() {
792       return this.kv.getQualifierLength();
793     }
794 
795     @Override
796     @Deprecated
797     public byte[] getValue() {
798       // TODO Auto-generated method stub
799       return null;
800     }
801 
802     @Override
803     @Deprecated
804     public byte[] getFamily() {
805       // TODO Auto-generated method stub
806       return null;
807     }
808 
809     @Override
810     @Deprecated
811     public byte[] getQualifier() {
812       // TODO Auto-generated method stub
813       return null;
814     }
815 
816     @Override
817     @Deprecated
818     public byte[] getRow() {
819       // TODO Auto-generated method stub
820       return null;
821     }
822 
823     /**
824      * @return the backing array of the entire KeyValue (all KeyValue fields are
825      *         in a single array)
826      */
827     @Override
828     public byte[] getTagsArray() {
829       return this.kv.getTagsArray();
830     }
831   }
832 }