1   /**
2    * Copyright 2009 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertFalse;
24  import static org.junit.Assert.assertSame;
25  import static org.junit.Assert.assertTrue;
26  import static org.junit.Assert.fail;
27  
28  import java.io.DataInputStream;
29  import java.io.DataOutputStream;
30  import java.io.File;
31  import java.io.FileInputStream;
32  import java.io.FileOutputStream;
33  import java.io.IOException;
34  import java.util.ArrayList;
35  import java.util.HashSet;
36  import java.util.Iterator;
37  import java.util.List;
38  import java.util.Map;
39  import java.util.NavigableMap;
40  import java.util.UUID;
41  import java.util.concurrent.SynchronousQueue;
42  import java.util.concurrent.ThreadPoolExecutor;
43  
44  import org.apache.commons.logging.Log;
45  import org.apache.commons.logging.LogFactory;
46  import org.apache.hadoop.conf.Configuration;
47  import org.apache.hadoop.fs.Path;
48  import org.apache.hadoop.hbase.HBaseTestingUtility;
49  import org.apache.hadoop.hbase.HColumnDescriptor;
50  import org.apache.hadoop.hbase.HConstants;
51  import org.apache.hadoop.hbase.HRegionInfo;
52  import org.apache.hadoop.hbase.HServerAddress;
53  import org.apache.hadoop.hbase.HTableDescriptor;
54  import org.apache.hadoop.hbase.KeyValue;
55  import org.apache.hadoop.hbase.filter.BinaryComparator;
56  import org.apache.hadoop.hbase.filter.CompareFilter;
57  import org.apache.hadoop.hbase.filter.Filter;
58  import org.apache.hadoop.hbase.filter.FilterList;
59  import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
60  import org.apache.hadoop.hbase.filter.PrefixFilter;
61  import org.apache.hadoop.hbase.filter.QualifierFilter;
62  import org.apache.hadoop.hbase.filter.RegexStringComparator;
63  import org.apache.hadoop.hbase.filter.RowFilter;
64  import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
65  import org.apache.hadoop.hbase.filter.WhileMatchFilter;
66  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
67  import org.apache.hadoop.hbase.util.Bytes;
68  import org.junit.After;
69  import org.junit.AfterClass;
70  import org.junit.Before;
71  import org.junit.BeforeClass;
72  import org.junit.Ignore;
73  import org.junit.Test;
74  
75  /**
76   * Run tests that use the HBase clients; {@link HTable} and {@link HTablePool}.
77   * Sets up the HBase mini cluster once at start and runs through all client tests.
78   * Each creates a table named for the method and does its stuff against that.
79   */
80  public class TestFromClientSide {
81    final Log LOG = LogFactory.getLog(getClass());
82    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
83    private static byte [] ROW = Bytes.toBytes("testRow");
84    private static byte [] FAMILY = Bytes.toBytes("testFamily");
85    private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
86    private static byte [] VALUE = Bytes.toBytes("testValue");
87  
88    /**
89     * @throws java.lang.Exception
90     */
91    @BeforeClass
92    public static void setUpBeforeClass() throws Exception {
93      TEST_UTIL.startMiniCluster(3);
94    }
95  
96    /**
97     * @throws java.lang.Exception
98     */
99    @AfterClass
100   public static void tearDownAfterClass() throws Exception {
101     TEST_UTIL.shutdownMiniCluster();
102   }
103 
104   /**
105    * @throws java.lang.Exception
106    */
107   @Before
108   public void setUp() throws Exception {
109     // Nothing to do.
110   }
111 
112   /**
113    * @throws java.lang.Exception
114    */
115   @After
116   public void tearDown() throws Exception {
117     // Nothing to do.
118   }
119 
120   /**
121    * Verifies that getConfiguration returns the same Configuration object used
122    * to create the HTable instance.
123    */
124   @Test
125   public void testGetConfiguration() throws Exception {
126     byte[] TABLE = Bytes.toBytes("testGetConfiguration");
127     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") };
128     Configuration conf = TEST_UTIL.getConfiguration();
129     HTable table = TEST_UTIL.createTable(TABLE, FAMILIES, conf);
130     assertSame(conf, table.getConfiguration());
131   }
132 
133   /**
134    * Test from client side of an involved filter against a multi family that
135    * involves deletes.
136    *
137    * @throws Exception
138    */
139   @Test
140   public void testWeirdCacheBehaviour() throws Exception {
141     byte [] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
142     byte [][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
143         Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
144         Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
145     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
146     String value = "this is the value";
147     String value2 = "this is some other value";
148     String keyPrefix1 = UUID.randomUUID().toString();
149     String keyPrefix2 = UUID.randomUUID().toString();
150     String keyPrefix3 = UUID.randomUUID().toString();
151     putRows(ht, 3, value, keyPrefix1);
152     putRows(ht, 3, value, keyPrefix2);
153     putRows(ht, 3, value, keyPrefix3);
154     ht.flushCommits();
155     putRows(ht, 3, value2, keyPrefix1);
156     putRows(ht, 3, value2, keyPrefix2);
157     putRows(ht, 3, value2, keyPrefix3);
158     HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLE);
159     System.out.println("Checking values for key: " + keyPrefix1);
160     assertEquals("Got back incorrect number of rows from scan", 3,
161         getNumberOfRows(keyPrefix1, value2, table));
162     System.out.println("Checking values for key: " + keyPrefix2);
163     assertEquals("Got back incorrect number of rows from scan", 3,
164         getNumberOfRows(keyPrefix2, value2, table));
165     System.out.println("Checking values for key: " + keyPrefix3);
166     assertEquals("Got back incorrect number of rows from scan", 3,
167         getNumberOfRows(keyPrefix3, value2, table));
168     deleteColumns(ht, value2, keyPrefix1);
169     deleteColumns(ht, value2, keyPrefix2);
170     deleteColumns(ht, value2, keyPrefix3);
171     System.out.println("Starting important checks.....");
172     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,
173       0, getNumberOfRows(keyPrefix1, value2, table));
174     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,
175       0, getNumberOfRows(keyPrefix2, value2, table));
176     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,
177       0, getNumberOfRows(keyPrefix3, value2, table));
178     ht.setScannerCaching(0);
179     assertEquals("Got back incorrect number of rows from scan", 0,
180       getNumberOfRows(keyPrefix1, value2, table)); ht.setScannerCaching(100);
181     assertEquals("Got back incorrect number of rows from scan", 0,
182       getNumberOfRows(keyPrefix2, value2, table));
183   }
184 
185   private void deleteColumns(HTable ht, String value, String keyPrefix)
186   throws IOException {
187     ResultScanner scanner = buildScanner(keyPrefix, value, ht);
188     Iterator<Result> it = scanner.iterator();
189     int count = 0;
190     while (it.hasNext()) {
191       Result result = it.next();
192       Delete delete = new Delete(result.getRow());
193       delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
194       ht.delete(delete);
195       count++;
196     }
197     assertEquals("Did not perform correct number of deletes", 3, count);
198   }
199 
200   private int getNumberOfRows(String keyPrefix, String value, HTable ht)
201       throws Exception {
202     ResultScanner resultScanner = buildScanner(keyPrefix, value, ht);
203     Iterator<Result> scanner = resultScanner.iterator();
204     int numberOfResults = 0;
205     while (scanner.hasNext()) {
206       Result result = scanner.next();
207       System.out.println("Got back key: " + Bytes.toString(result.getRow()));
208       for (KeyValue kv : result.raw()) {
209         System.out.println("kv=" + kv.toString() + ", "
210             + Bytes.toString(kv.getValue()));
211       }
212       numberOfResults++;
213     }
214     return numberOfResults;
215   }
216 
217   private ResultScanner buildScanner(String keyPrefix, String value, HTable ht)
218       throws IOException {
219     // OurFilterList allFilters = new OurFilterList();
220     FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */);
221     allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
222     SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes
223         .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes
224         .toBytes(value));
225     filter.setFilterIfMissing(true);
226     allFilters.addFilter(filter);
227 
228     // allFilters.addFilter(new
229     // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"),
230     // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)));
231 
232     Scan scan = new Scan();
233     scan.addFamily(Bytes.toBytes("trans-blob"));
234     scan.addFamily(Bytes.toBytes("trans-type"));
235     scan.addFamily(Bytes.toBytes("trans-date"));
236     scan.addFamily(Bytes.toBytes("trans-tags"));
237     scan.addFamily(Bytes.toBytes("trans-group"));
238     scan.setFilter(allFilters);
239 
240     return ht.getScanner(scan);
241   }
242 
243   private void putRows(HTable ht, int numRows, String value, String key)
244       throws IOException {
245     for (int i = 0; i < numRows; i++) {
246       String row = key + "_" + UUID.randomUUID().toString();
247       System.out.println(String.format("Saving row: %s, with value %s", row,
248           value));
249       Put put = new Put(Bytes.toBytes(row));
250       put.add(Bytes.toBytes("trans-blob"), null, Bytes
251           .toBytes("value for blob"));
252       put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
253       put.add(Bytes.toBytes("trans-date"), null, Bytes
254           .toBytes("20090921010101999"));
255       put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes
256           .toBytes(value));
257       put.add(Bytes.toBytes("trans-group"), null, Bytes
258           .toBytes("adhocTransactionGroupId"));
259       ht.put(put);
260     }
261   }
262 
263   /**
264    * Test filters when multiple regions.  It does counts.  Needs eye-balling of
265    * logs to ensure that we're not scanning more regions that we're supposed to.
266    * Related to the TestFilterAcrossRegions over in the o.a.h.h.filter package.
267    * @throws IOException
268    * @throws InterruptedException
269    */
270   @Test
271   public void testFilterAcrossMultipleRegions()
272   throws IOException, InterruptedException {
273     byte [] name = Bytes.toBytes("testFilterAcrossMutlipleRegions");
274     HTable t = TEST_UTIL.createTable(name, FAMILY);
275     int rowCount = TEST_UTIL.loadTable(t, FAMILY);
276     assertRowCount(t, rowCount);
277     // Split the table.  Should split on a reasonable key; 'lqj'
278     Map<HRegionInfo, HServerAddress> regions  = splitTable(t);
279     assertRowCount(t, rowCount);
280     // Get end key of first region.
281     byte [] endKey = regions.keySet().iterator().next().getEndKey();
282     // Count rows with a filter that stops us before passed 'endKey'.
283     // Should be count of rows in first region.
284     int endKeyCount = countRows(t, createScanWithRowFilter(endKey));
285     assertTrue(endKeyCount < rowCount);
286 
287     // How do I know I did not got to second region?  Thats tough.  Can't really
288     // do that in client-side region test.  I verified by tracing in debugger.
289     // I changed the messages that come out when set to DEBUG so should see
290     // when scanner is done. Says "Finished with scanning..." with region name.
291     // Check that its finished in right region.
292 
293     // New test.  Make it so scan goes into next region by one and then two.
294     // Make sure count comes out right.
295     byte [] key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 1)};
296     int plusOneCount = countRows(t, createScanWithRowFilter(key));
297     assertEquals(endKeyCount + 1, plusOneCount);
298     key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 2)};
299     int plusTwoCount = countRows(t, createScanWithRowFilter(key));
300     assertEquals(endKeyCount + 2, plusTwoCount);
301 
302     // New test.  Make it so I scan one less than endkey.
303     key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] - 1)};
304     int minusOneCount = countRows(t, createScanWithRowFilter(key));
305     assertEquals(endKeyCount - 1, minusOneCount);
306     // For above test... study logs.  Make sure we do "Finished with scanning.."
307     // in first region and that we do not fall into the next region.
308 
309     key = new byte [] {'a', 'a', 'a'};
310     int countBBB = countRows(t,
311       createScanWithRowFilter(key, null, CompareFilter.CompareOp.EQUAL));
312     assertEquals(1, countBBB);
313 
314     int countGreater = countRows(t, createScanWithRowFilter(endKey, null,
315       CompareFilter.CompareOp.GREATER_OR_EQUAL));
316     // Because started at start of table.
317     assertEquals(0, countGreater);
318     countGreater = countRows(t, createScanWithRowFilter(endKey, endKey,
319       CompareFilter.CompareOp.GREATER_OR_EQUAL));
320     assertEquals(rowCount - endKeyCount, countGreater);
321   }
322 
323   /*
324    * @param key
325    * @return Scan with RowFilter that does LESS than passed key.
326    */
327   private Scan createScanWithRowFilter(final byte [] key) {
328     return createScanWithRowFilter(key, null, CompareFilter.CompareOp.LESS);
329   }
330 
331   /*
332    * @param key
333    * @param op
334    * @param startRow
335    * @return Scan with RowFilter that does CompareOp op on passed key.
336    */
337   private Scan createScanWithRowFilter(final byte [] key,
338       final byte [] startRow, CompareFilter.CompareOp op) {
339     // Make sure key is of some substance... non-null and > than first key.
340     assertTrue(key != null && key.length > 0 &&
341       Bytes.BYTES_COMPARATOR.compare(key, new byte [] {'a', 'a', 'a'}) >= 0);
342     LOG.info("Key=" + Bytes.toString(key));
343     Scan s = startRow == null? new Scan(): new Scan(startRow);
344     Filter f = new RowFilter(op, new BinaryComparator(key));
345     f = new WhileMatchFilter(f);
346     s.setFilter(f);
347     return s;
348   }
349 
350   /*
351    * @param t
352    * @param s
353    * @return Count of rows in table.
354    * @throws IOException
355    */
356   private int countRows(final HTable t, final Scan s)
357   throws IOException {
358     // Assert all rows in table.
359     ResultScanner scanner = t.getScanner(s);
360     int count = 0;
361     for (Result result: scanner) {
362       count++;
363       assertTrue(result.size() > 0);
364       // LOG.info("Count=" + count + ", row=" + Bytes.toString(result.getRow()));
365     }
366     return count;
367   }
368 
369   private void assertRowCount(final HTable t, final int expected)
370   throws IOException {
371     assertEquals(expected, countRows(t, new Scan()));
372   }
373 
374   /*
375    * Split table into multiple regions.
376    * @param t Table to split.
377    * @return Map of regions to servers.
378    * @throws IOException
379    */
380   private Map<HRegionInfo, HServerAddress> splitTable(final HTable t)
381   throws IOException, InterruptedException {
382     // Split this table in two.
383     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
384     admin.split(t.getTableName());
385     Map<HRegionInfo, HServerAddress> regions = waitOnSplit(t);
386     assertTrue(regions.size() > 1);
387     return regions;
388   }
389 
390   /*
391    * Wait on table split.  May return because we waited long enough on the split
392    * and it didn't happen.  Caller should check.
393    * @param t
394    * @return Map of table regions; caller needs to check table actually split.
395    */
396   private Map<HRegionInfo, HServerAddress> waitOnSplit(final HTable t)
397   throws IOException {
398     Map<HRegionInfo, HServerAddress> regions = t.getRegionsInfo();
399     int originalCount = regions.size();
400     for (int i = 0; i < TEST_UTIL.getConfiguration().getInt("hbase.test.retries", 30); i++) {
401       Thread.currentThread();
402       try {
403         Thread.sleep(1000);
404       } catch (InterruptedException e) {
405         e.printStackTrace();
406       }
407       regions = t.getRegionsInfo();
408       if (regions.size() > originalCount) break;
409     }
410     return regions;
411   }
412 
413   @Test
414   public void testSuperSimple() throws Exception {
415     byte [] TABLE = Bytes.toBytes("testSuperSimple");
416     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
417     Put put = new Put(ROW);
418     put.add(FAMILY, QUALIFIER, VALUE);
419     ht.put(put);
420     Scan scan = new Scan();
421     scan.addColumn(FAMILY, TABLE);
422     ResultScanner scanner = ht.getScanner(scan);
423     Result result = scanner.next();
424     assertTrue("Expected null result", result == null);
425     scanner.close();
426     System.out.println("Done.");
427   }
428 
429   @Test
430   public void testMaxKeyValueSize() throws Exception {
431     byte [] TABLE = Bytes.toBytes("testMaxKeyValueSize");
432     Configuration conf = TEST_UTIL.getConfiguration();
433     String oldMaxSize = conf.get("hbase.client.keyvalue.maxsize");
434     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
435     byte[] value = new byte[4 * 1024 * 1024];
436     Put put = new Put(ROW);
437     put.add(FAMILY, QUALIFIER, value);
438     ht.put(put);
439     try {
440       conf.setInt("hbase.client.keyvalue.maxsize", 2 * 1024 * 1024);
441       TABLE = Bytes.toBytes("testMaxKeyValueSize2");
442       ht = TEST_UTIL.createTable(TABLE, FAMILY);
443       put = new Put(ROW);
444       put.add(FAMILY, QUALIFIER, value);
445       ht.put(put);
446       fail("Inserting a too large KeyValue worked, should throw exception");
447     } catch(Exception e) {}
448     conf.set("hbase.client.keyvalue.maxsize", oldMaxSize);
449   }
450 
451   @Test
452   public void testFilters() throws Exception {
453     byte [] TABLE = Bytes.toBytes("testFilters");
454     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
455     byte [][] ROWS = makeN(ROW, 10);
456     byte [][] QUALIFIERS = {
457         Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
458         Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
459         Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
460         Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
461         Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>")
462     };
463     for(int i=0;i<10;i++) {
464       Put put = new Put(ROWS[i]);
465       put.add(FAMILY, QUALIFIERS[i], VALUE);
466       ht.put(put);
467     }
468     Scan scan = new Scan();
469     scan.addFamily(FAMILY);
470     Filter filter = new QualifierFilter(CompareOp.EQUAL,
471       new RegexStringComparator("col[1-5]"));
472     scan.setFilter(filter);
473     ResultScanner scanner = ht.getScanner(scan);
474     int expectedIndex = 1;
475     for(Result result : ht.getScanner(scan)) {
476       assertEquals(result.size(), 1);
477       assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[expectedIndex]));
478       assertTrue(Bytes.equals(result.raw()[0].getQualifier(),
479           QUALIFIERS[expectedIndex]));
480       expectedIndex++;
481     }
482     assertEquals(expectedIndex, 6);
483     scanner.close();
484   }
485 
486   @Test
487   public void testKeyOnlyFilter() throws Exception {
488     byte [] TABLE = Bytes.toBytes("testKeyOnlyFilter");
489     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
490     byte [][] ROWS = makeN(ROW, 10);
491     byte [][] QUALIFIERS = {
492         Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
493         Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
494         Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
495         Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
496         Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>")
497     };
498     for(int i=0;i<10;i++) {
499       Put put = new Put(ROWS[i]);
500       put.add(FAMILY, QUALIFIERS[i], VALUE);
501       ht.put(put);
502     }
503     Scan scan = new Scan();
504     scan.addFamily(FAMILY);
505     Filter filter = new KeyOnlyFilter(true);
506     scan.setFilter(filter);
507     ResultScanner scanner = ht.getScanner(scan);
508     int count = 0;
509     for(Result result : ht.getScanner(scan)) {
510       assertEquals(result.size(), 1);
511       assertEquals(result.raw()[0].getValueLength(), Bytes.SIZEOF_INT);
512       assertEquals(Bytes.toInt(result.raw()[0].getValue()), VALUE.length);
513       count++;
514     }
515     assertEquals(count, 10);
516     scanner.close();
517   }
518 
519   /**
520    * Test simple table and non-existent row cases.
521    */
522   @Test
523   public void testSimpleMissing() throws Exception {
524     byte [] TABLE = Bytes.toBytes("testSimpleMissing");
525     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
526     byte [][] ROWS = makeN(ROW, 4);
527 
528     // Try to get a row on an empty table
529     Get get = new Get(ROWS[0]);
530     Result result = ht.get(get);
531     assertEmptyResult(result);
532 
533     get = new Get(ROWS[0]);
534     get.addFamily(FAMILY);
535     result = ht.get(get);
536     assertEmptyResult(result);
537 
538     get = new Get(ROWS[0]);
539     get.addColumn(FAMILY, QUALIFIER);
540     result = ht.get(get);
541     assertEmptyResult(result);
542 
543     Scan scan = new Scan();
544     result = getSingleScanResult(ht, scan);
545     assertNullResult(result);
546 
547 
548     scan = new Scan(ROWS[0]);
549     result = getSingleScanResult(ht, scan);
550     assertNullResult(result);
551 
552     scan = new Scan(ROWS[0],ROWS[1]);
553     result = getSingleScanResult(ht, scan);
554     assertNullResult(result);
555 
556     scan = new Scan();
557     scan.addFamily(FAMILY);
558     result = getSingleScanResult(ht, scan);
559     assertNullResult(result);
560 
561     scan = new Scan();
562     scan.addColumn(FAMILY, QUALIFIER);
563     result = getSingleScanResult(ht, scan);
564     assertNullResult(result);
565 
566     // Insert a row
567 
568     Put put = new Put(ROWS[2]);
569     put.add(FAMILY, QUALIFIER, VALUE);
570     ht.put(put);
571 
572     // Try to get empty rows around it
573 
574     get = new Get(ROWS[1]);
575     result = ht.get(get);
576     assertEmptyResult(result);
577 
578     get = new Get(ROWS[0]);
579     get.addFamily(FAMILY);
580     result = ht.get(get);
581     assertEmptyResult(result);
582 
583     get = new Get(ROWS[3]);
584     get.addColumn(FAMILY, QUALIFIER);
585     result = ht.get(get);
586     assertEmptyResult(result);
587 
588     // Try to scan empty rows around it
589 
590     scan = new Scan(ROWS[3]);
591     result = getSingleScanResult(ht, scan);
592     assertNullResult(result);
593 
594     scan = new Scan(ROWS[0],ROWS[2]);
595     result = getSingleScanResult(ht, scan);
596     assertNullResult(result);
597 
598     // Make sure we can actually get the row
599 
600     get = new Get(ROWS[2]);
601     result = ht.get(get);
602     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
603 
604     get = new Get(ROWS[2]);
605     get.addFamily(FAMILY);
606     result = ht.get(get);
607     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
608 
609     get = new Get(ROWS[2]);
610     get.addColumn(FAMILY, QUALIFIER);
611     result = ht.get(get);
612     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
613 
614     // Make sure we can scan the row
615 
616     scan = new Scan();
617     result = getSingleScanResult(ht, scan);
618     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
619 
620     scan = new Scan(ROWS[0],ROWS[3]);
621     result = getSingleScanResult(ht, scan);
622     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
623 
624     scan = new Scan(ROWS[2],ROWS[3]);
625     result = getSingleScanResult(ht, scan);
626     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
627   }
628 
629   /**
630    * Test basic puts, gets, scans, and deletes for a single row
631    * in a multiple family table.
632    */
633   @Test
634   public void testSingleRowMultipleFamily() throws Exception {
635     byte [] TABLE = Bytes.toBytes("testSingleRowMultipleFamily");
636     byte [][] ROWS = makeN(ROW, 3);
637     byte [][] FAMILIES = makeNAscii(FAMILY, 10);
638     byte [][] QUALIFIERS = makeN(QUALIFIER, 10);
639     byte [][] VALUES = makeN(VALUE, 10);
640 
641     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
642 
643     Get get;
644     Scan scan;
645     Delete delete;
646     Put put;
647     Result result;
648 
649     ////////////////////////////////////////////////////////////////////////////
650     // Insert one column to one family
651     ////////////////////////////////////////////////////////////////////////////
652 
653     put = new Put(ROWS[0]);
654     put.add(FAMILIES[4], QUALIFIERS[0], VALUES[0]);
655     ht.put(put);
656 
657     // Get the single column
658     getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
659 
660     // Scan the single column
661     scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
662 
663     // Get empty results around inserted column
664     getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
665 
666     // Scan empty results around inserted column
667     scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
668 
669     ////////////////////////////////////////////////////////////////////////////
670     // Flush memstore and run same tests from storefiles
671     ////////////////////////////////////////////////////////////////////////////
672 
673     TEST_UTIL.flush();
674 
675     // Redo get and scan tests from storefile
676     getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
677     scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
678     getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
679     scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
680 
681     ////////////////////////////////////////////////////////////////////////////
682     // Now, Test reading from memstore and storefiles at once
683     ////////////////////////////////////////////////////////////////////////////
684 
685     // Insert multiple columns to two other families
686     put = new Put(ROWS[0]);
687     put.add(FAMILIES[2], QUALIFIERS[2], VALUES[2]);
688     put.add(FAMILIES[2], QUALIFIERS[4], VALUES[4]);
689     put.add(FAMILIES[4], QUALIFIERS[4], VALUES[4]);
690     put.add(FAMILIES[6], QUALIFIERS[6], VALUES[6]);
691     put.add(FAMILIES[6], QUALIFIERS[7], VALUES[7]);
692     put.add(FAMILIES[7], QUALIFIERS[7], VALUES[7]);
693     put.add(FAMILIES[9], QUALIFIERS[0], VALUES[0]);
694     ht.put(put);
695 
696     // Get multiple columns across multiple families and get empties around it
697     singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
698 
699     // Scan multiple columns across multiple families and scan empties around it
700     singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
701 
702     ////////////////////////////////////////////////////////////////////////////
703     // Flush the table again
704     ////////////////////////////////////////////////////////////////////////////
705 
706     TEST_UTIL.flush();
707 
708     // Redo tests again
709     singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
710     singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
711 
712     // Insert more data to memstore
713     put = new Put(ROWS[0]);
714     put.add(FAMILIES[6], QUALIFIERS[5], VALUES[5]);
715     put.add(FAMILIES[6], QUALIFIERS[8], VALUES[8]);
716     put.add(FAMILIES[6], QUALIFIERS[9], VALUES[9]);
717     put.add(FAMILIES[4], QUALIFIERS[3], VALUES[3]);
718     ht.put(put);
719 
720     ////////////////////////////////////////////////////////////////////////////
721     // Delete a storefile column
722     ////////////////////////////////////////////////////////////////////////////
723     delete = new Delete(ROWS[0]);
724     delete.deleteColumns(FAMILIES[6], QUALIFIERS[7]);
725     ht.delete(delete);
726 
727     // Try to get deleted column
728     get = new Get(ROWS[0]);
729     get.addColumn(FAMILIES[6], QUALIFIERS[7]);
730     result = ht.get(get);
731     assertEmptyResult(result);
732 
733     // Try to scan deleted column
734     scan = new Scan();
735     scan.addColumn(FAMILIES[6], QUALIFIERS[7]);
736     result = getSingleScanResult(ht, scan);
737     assertNullResult(result);
738 
739     // Make sure we can still get a column before it and after it
740     get = new Get(ROWS[0]);
741     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
742     result = ht.get(get);
743     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
744 
745     get = new Get(ROWS[0]);
746     get.addColumn(FAMILIES[6], QUALIFIERS[8]);
747     result = ht.get(get);
748     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]);
749 
750     // Make sure we can still scan a column before it and after it
751     scan = new Scan();
752     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
753     result = getSingleScanResult(ht, scan);
754     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
755 
756     scan = new Scan();
757     scan.addColumn(FAMILIES[6], QUALIFIERS[8]);
758     result = getSingleScanResult(ht, scan);
759     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]);
760 
761     ////////////////////////////////////////////////////////////////////////////
762     // Delete a memstore column
763     ////////////////////////////////////////////////////////////////////////////
764     delete = new Delete(ROWS[0]);
765     delete.deleteColumns(FAMILIES[6], QUALIFIERS[8]);
766     ht.delete(delete);
767 
768     // Try to get deleted column
769     get = new Get(ROWS[0]);
770     get.addColumn(FAMILIES[6], QUALIFIERS[8]);
771     result = ht.get(get);
772     assertEmptyResult(result);
773 
774     // Try to scan deleted column
775     scan = new Scan();
776     scan.addColumn(FAMILIES[6], QUALIFIERS[8]);
777     result = getSingleScanResult(ht, scan);
778     assertNullResult(result);
779 
780     // Make sure we can still get a column before it and after it
781     get = new Get(ROWS[0]);
782     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
783     result = ht.get(get);
784     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
785 
786     get = new Get(ROWS[0]);
787     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
788     result = ht.get(get);
789     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
790 
791     // Make sure we can still scan a column before it and after it
792     scan = new Scan();
793     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
794     result = getSingleScanResult(ht, scan);
795     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
796 
797     scan = new Scan();
798     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
799     result = getSingleScanResult(ht, scan);
800     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
801 
802     ////////////////////////////////////////////////////////////////////////////
803     // Delete joint storefile/memstore family
804     ////////////////////////////////////////////////////////////////////////////
805 
806     delete = new Delete(ROWS[0]);
807     delete.deleteFamily(FAMILIES[4]);
808     ht.delete(delete);
809 
810     // Try to get storefile column in deleted family
811     get = new Get(ROWS[0]);
812     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
813     result = ht.get(get);
814     assertEmptyResult(result);
815 
816     // Try to get memstore column in deleted family
817     get = new Get(ROWS[0]);
818     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
819     result = ht.get(get);
820     assertEmptyResult(result);
821 
822     // Try to get deleted family
823     get = new Get(ROWS[0]);
824     get.addFamily(FAMILIES[4]);
825     result = ht.get(get);
826     assertEmptyResult(result);
827 
828     // Try to scan storefile column in deleted family
829     scan = new Scan();
830     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
831     result = getSingleScanResult(ht, scan);
832     assertNullResult(result);
833 
834     // Try to scan memstore column in deleted family
835     scan = new Scan();
836     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
837     result = getSingleScanResult(ht, scan);
838     assertNullResult(result);
839 
840     // Try to scan deleted family
841     scan = new Scan();
842     scan.addFamily(FAMILIES[4]);
843     result = getSingleScanResult(ht, scan);
844     assertNullResult(result);
845 
846     // Make sure we can still get another family
847     get = new Get(ROWS[0]);
848     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
849     result = ht.get(get);
850     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
851 
852     get = new Get(ROWS[0]);
853     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
854     result = ht.get(get);
855     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
856 
857     // Make sure we can still scan another family
858     scan = new Scan();
859     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
860     result = getSingleScanResult(ht, scan);
861     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
862 
863     scan = new Scan();
864     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
865     result = getSingleScanResult(ht, scan);
866     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
867 
868     ////////////////////////////////////////////////////////////////////////////
869     // Flush everything and rerun delete tests
870     ////////////////////////////////////////////////////////////////////////////
871 
872     TEST_UTIL.flush();
873 
874     // Try to get storefile column in deleted family
875     get = new Get(ROWS[0]);
876     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
877     result = ht.get(get);
878     assertEmptyResult(result);
879 
880     // Try to get memstore column in deleted family
881     get = new Get(ROWS[0]);
882     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
883     result = ht.get(get);
884     assertEmptyResult(result);
885 
886     // Try to get deleted family
887     get = new Get(ROWS[0]);
888     get.addFamily(FAMILIES[4]);
889     result = ht.get(get);
890     assertEmptyResult(result);
891 
892     // Try to scan storefile column in deleted family
893     scan = new Scan();
894     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
895     result = getSingleScanResult(ht, scan);
896     assertNullResult(result);
897 
898     // Try to scan memstore column in deleted family
899     scan = new Scan();
900     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
901     result = getSingleScanResult(ht, scan);
902     assertNullResult(result);
903 
904     // Try to scan deleted family
905     scan = new Scan();
906     scan.addFamily(FAMILIES[4]);
907     result = getSingleScanResult(ht, scan);
908     assertNullResult(result);
909 
910     // Make sure we can still get another family
911     get = new Get(ROWS[0]);
912     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
913     result = ht.get(get);
914     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
915 
916     get = new Get(ROWS[0]);
917     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
918     result = ht.get(get);
919     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
920 
921     // Make sure we can still scan another family
922     scan = new Scan();
923     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
924     result = getSingleScanResult(ht, scan);
925     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
926 
927     scan = new Scan();
928     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
929     result = getSingleScanResult(ht, scan);
930     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
931 
932   }
933 
934   @Test
935   public void testNull() throws Exception {
936     byte [] TABLE = Bytes.toBytes("testNull");
937 
938     // Null table name (should NOT work)
939     try {
940       TEST_UTIL.createTable(null, FAMILY);
941       fail("Creating a table with null name passed, should have failed");
942     } catch(Exception e) {}
943 
944     // Null family (should NOT work)
945     try {
946       TEST_UTIL.createTable(TABLE, (byte[])null);
947       fail("Creating a table with a null family passed, should fail");
948     } catch(Exception e) {}
949 
950     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
951 
952     // Null row (should NOT work)
953     try {
954       Put put = new Put((byte[])null);
955       put.add(FAMILY, QUALIFIER, VALUE);
956       ht.put(put);
957       fail("Inserting a null row worked, should throw exception");
958     } catch(Exception e) {}
959 
960     // Null qualifier (should work)
961     {
962       Put put = new Put(ROW);
963       put.add(FAMILY, null, VALUE);
964       ht.put(put);
965 
966       getTestNull(ht, ROW, FAMILY, VALUE);
967 
968       scanTestNull(ht, ROW, FAMILY, VALUE);
969 
970       Delete delete = new Delete(ROW);
971       delete.deleteColumns(FAMILY, null);
972       ht.delete(delete);
973 
974       Get get = new Get(ROW);
975       Result result = ht.get(get);
976       assertEmptyResult(result);
977     }
978 
979     // Use a new table
980     byte [] TABLE2 = Bytes.toBytes("testNull2");
981     ht = TEST_UTIL.createTable(TABLE2, FAMILY);
982 
983     // Empty qualifier, byte[0] instead of null (should work)
984     try {
985       Put put = new Put(ROW);
986       put.add(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE);
987       ht.put(put);
988 
989       getTestNull(ht, ROW, FAMILY, VALUE);
990 
991       scanTestNull(ht, ROW, FAMILY, VALUE);
992 
993       // Flush and try again
994 
995       TEST_UTIL.flush();
996 
997       getTestNull(ht, ROW, FAMILY, VALUE);
998 
999       scanTestNull(ht, ROW, FAMILY, VALUE);
1000 
1001       Delete delete = new Delete(ROW);
1002       delete.deleteColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY);
1003       ht.delete(delete);
1004 
1005       Get get = new Get(ROW);
1006       Result result = ht.get(get);
1007       assertEmptyResult(result);
1008 
1009     } catch(Exception e) {
1010       throw new IOException("Using a row with null qualifier threw exception, should ");
1011     }
1012 
1013     // Null value
1014     try {
1015       Put put = new Put(ROW);
1016       put.add(FAMILY, QUALIFIER, null);
1017       ht.put(put);
1018 
1019       Get get = new Get(ROW);
1020       get.addColumn(FAMILY, QUALIFIER);
1021       Result result = ht.get(get);
1022       assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
1023 
1024       Scan scan = new Scan();
1025       scan.addColumn(FAMILY, QUALIFIER);
1026       result = getSingleScanResult(ht, scan);
1027       assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
1028 
1029       Delete delete = new Delete(ROW);
1030       delete.deleteColumns(FAMILY, QUALIFIER);
1031       ht.delete(delete);
1032 
1033       get = new Get(ROW);
1034       result = ht.get(get);
1035       assertEmptyResult(result);
1036 
1037     } catch(Exception e) {
1038       throw new IOException("Null values should be allowed, but threw exception");
1039     }
1040   }
1041 
1042   @Test
1043   public void testVersions() throws Exception {
1044     byte [] TABLE = Bytes.toBytes("testVersions");
1045 
1046     long [] STAMPS = makeStamps(20);
1047     byte [][] VALUES = makeNAscii(VALUE, 20);
1048 
1049     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
1050 
1051     // Insert 4 versions of same column
1052     Put put = new Put(ROW);
1053     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1054     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1055     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1056     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1057     ht.put(put);
1058 
1059     // Verify we can get each one properly
1060     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1061     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1062     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1063     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1064     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1065     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1066     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1067     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1068 
1069     // Verify we don't accidentally get others
1070     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1071     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1072     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1073     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1074     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1075     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1076 
1077     // Ensure maxVersions in query is respected
1078     Get get = new Get(ROW);
1079     get.addColumn(FAMILY, QUALIFIER);
1080     get.setMaxVersions(2);
1081     Result result = ht.get(get);
1082     assertNResult(result, ROW, FAMILY, QUALIFIER,
1083         new long [] {STAMPS[4], STAMPS[5]},
1084         new byte[][] {VALUES[4], VALUES[5]},
1085         0, 1);
1086 
1087     Scan scan = new Scan(ROW);
1088     scan.addColumn(FAMILY, QUALIFIER);
1089     scan.setMaxVersions(2);
1090     result = getSingleScanResult(ht, scan);
1091     assertNResult(result, ROW, FAMILY, QUALIFIER,
1092         new long [] {STAMPS[4], STAMPS[5]},
1093         new byte[][] {VALUES[4], VALUES[5]},
1094         0, 1);
1095 
1096     // Flush and redo
1097 
1098     TEST_UTIL.flush();
1099 
1100     // Verify we can get each one properly
1101     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1102     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1103     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1104     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1105     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1106     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1107     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1108     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1109 
1110     // Verify we don't accidentally get others
1111     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1112     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1113     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1114     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1115     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1116     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1117 
1118     // Ensure maxVersions in query is respected
1119     get = new Get(ROW);
1120     get.addColumn(FAMILY, QUALIFIER);
1121     get.setMaxVersions(2);
1122     result = ht.get(get);
1123     assertNResult(result, ROW, FAMILY, QUALIFIER,
1124         new long [] {STAMPS[4], STAMPS[5]},
1125         new byte[][] {VALUES[4], VALUES[5]},
1126         0, 1);
1127 
1128     scan = new Scan(ROW);
1129     scan.addColumn(FAMILY, QUALIFIER);
1130     scan.setMaxVersions(2);
1131     result = getSingleScanResult(ht, scan);
1132     assertNResult(result, ROW, FAMILY, QUALIFIER,
1133         new long [] {STAMPS[4], STAMPS[5]},
1134         new byte[][] {VALUES[4], VALUES[5]},
1135         0, 1);
1136 
1137 
1138     // Add some memstore and retest
1139 
1140     // Insert 4 more versions of same column and a dupe
1141     put = new Put(ROW);
1142     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
1143     put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
1144     put.add(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1145     put.add(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
1146     ht.put(put);
1147 
1148     // Ensure maxVersions in query is respected
1149     get = new Get(ROW);
1150     get.addColumn(FAMILY, QUALIFIER);
1151     get.setMaxVersions();
1152     result = ht.get(get);
1153     assertNResult(result, ROW, FAMILY, QUALIFIER,
1154         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1155         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1156         0, 7);
1157 
1158     scan = new Scan(ROW);
1159     scan.addColumn(FAMILY, QUALIFIER);
1160     scan.setMaxVersions();
1161     result = getSingleScanResult(ht, scan);
1162     assertNResult(result, ROW, FAMILY, QUALIFIER,
1163         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1164         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1165         0, 7);
1166 
1167     get = new Get(ROW);
1168     get.setMaxVersions();
1169     result = ht.get(get);
1170     assertNResult(result, ROW, FAMILY, QUALIFIER,
1171         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1172         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1173         0, 7);
1174 
1175     scan = new Scan(ROW);
1176     scan.setMaxVersions();
1177     result = getSingleScanResult(ht, scan);
1178     assertNResult(result, ROW, FAMILY, QUALIFIER,
1179         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1180         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1181         0, 7);
1182 
1183     // Verify we can get each one properly
1184     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1185     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1186     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1187     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1188     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1189     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1190     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1191     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1192 
1193     // Verify we don't accidentally get others
1194     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1195     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
1196     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1197     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
1198 
1199     // Ensure maxVersions of table is respected
1200 
1201     TEST_UTIL.flush();
1202 
1203     // Insert 4 more versions of same column and a dupe
1204     put = new Put(ROW);
1205     put.add(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
1206     put.add(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
1207     put.add(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
1208     put.add(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
1209     ht.put(put);
1210 
1211     get = new Get(ROW);
1212     get.addColumn(FAMILY, QUALIFIER);
1213     get.setMaxVersions(Integer.MAX_VALUE);
1214     result = ht.get(get);
1215     assertNResult(result, ROW, FAMILY, QUALIFIER,
1216         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
1217         new byte[][] {VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
1218         0, 9);
1219 
1220     scan = new Scan(ROW);
1221     scan.addColumn(FAMILY, QUALIFIER);
1222     scan.setMaxVersions(Integer.MAX_VALUE);
1223     result = getSingleScanResult(ht, scan);
1224     assertNResult(result, ROW, FAMILY, QUALIFIER,
1225         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
1226         new byte[][] {VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
1227         0, 9);
1228 
1229     // Delete a version in the memstore and a version in a storefile
1230     Delete delete = new Delete(ROW);
1231     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[11]);
1232     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[7]);
1233     ht.delete(delete);
1234 
1235     // Test that it's gone
1236     get = new Get(ROW);
1237     get.addColumn(FAMILY, QUALIFIER);
1238     get.setMaxVersions(Integer.MAX_VALUE);
1239     result = ht.get(get);
1240     assertNResult(result, ROW, FAMILY, QUALIFIER,
1241         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
1242         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
1243         0, 9);
1244 
1245     scan = new Scan(ROW);
1246     scan.addColumn(FAMILY, QUALIFIER);
1247     scan.setMaxVersions(Integer.MAX_VALUE);
1248     result = getSingleScanResult(ht, scan);
1249     assertNResult(result, ROW, FAMILY, QUALIFIER,
1250         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
1251         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
1252         0, 9);
1253 
1254   }
1255 
1256   @Test
1257   public void testVersionLimits() throws Exception {
1258     byte [] TABLE = Bytes.toBytes("testVersionLimits");
1259     byte [][] FAMILIES = makeNAscii(FAMILY, 3);
1260     int [] LIMITS = {1,3,5};
1261     long [] STAMPS = makeStamps(10);
1262     byte [][] VALUES = makeNAscii(VALUE, 10);
1263     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, LIMITS);
1264 
1265     // Insert limit + 1 on each family
1266     Put put = new Put(ROW);
1267     put.add(FAMILIES[0], QUALIFIER, STAMPS[0], VALUES[0]);
1268     put.add(FAMILIES[0], QUALIFIER, STAMPS[1], VALUES[1]);
1269     put.add(FAMILIES[1], QUALIFIER, STAMPS[0], VALUES[0]);
1270     put.add(FAMILIES[1], QUALIFIER, STAMPS[1], VALUES[1]);
1271     put.add(FAMILIES[1], QUALIFIER, STAMPS[2], VALUES[2]);
1272     put.add(FAMILIES[1], QUALIFIER, STAMPS[3], VALUES[3]);
1273     put.add(FAMILIES[2], QUALIFIER, STAMPS[0], VALUES[0]);
1274     put.add(FAMILIES[2], QUALIFIER, STAMPS[1], VALUES[1]);
1275     put.add(FAMILIES[2], QUALIFIER, STAMPS[2], VALUES[2]);
1276     put.add(FAMILIES[2], QUALIFIER, STAMPS[3], VALUES[3]);
1277     put.add(FAMILIES[2], QUALIFIER, STAMPS[4], VALUES[4]);
1278     put.add(FAMILIES[2], QUALIFIER, STAMPS[5], VALUES[5]);
1279     put.add(FAMILIES[2], QUALIFIER, STAMPS[6], VALUES[6]);
1280     ht.put(put);
1281 
1282     // Verify we only get the right number out of each
1283 
1284     // Family0
1285 
1286     Get get = new Get(ROW);
1287     get.addColumn(FAMILIES[0], QUALIFIER);
1288     get.setMaxVersions(Integer.MAX_VALUE);
1289     Result result = ht.get(get);
1290     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1291         new long [] {STAMPS[1]},
1292         new byte[][] {VALUES[1]},
1293         0, 0);
1294 
1295     get = new Get(ROW);
1296     get.addFamily(FAMILIES[0]);
1297     get.setMaxVersions(Integer.MAX_VALUE);
1298     result = ht.get(get);
1299     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1300         new long [] {STAMPS[1]},
1301         new byte[][] {VALUES[1]},
1302         0, 0);
1303 
1304     Scan scan = new Scan(ROW);
1305     scan.addColumn(FAMILIES[0], QUALIFIER);
1306     scan.setMaxVersions(Integer.MAX_VALUE);
1307     result = getSingleScanResult(ht, scan);
1308     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1309         new long [] {STAMPS[1]},
1310         new byte[][] {VALUES[1]},
1311         0, 0);
1312 
1313     scan = new Scan(ROW);
1314     scan.addFamily(FAMILIES[0]);
1315     scan.setMaxVersions(Integer.MAX_VALUE);
1316     result = getSingleScanResult(ht, scan);
1317     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1318         new long [] {STAMPS[1]},
1319         new byte[][] {VALUES[1]},
1320         0, 0);
1321 
1322     // Family1
1323 
1324     get = new Get(ROW);
1325     get.addColumn(FAMILIES[1], QUALIFIER);
1326     get.setMaxVersions(Integer.MAX_VALUE);
1327     result = ht.get(get);
1328     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1329         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1330         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1331         0, 2);
1332 
1333     get = new Get(ROW);
1334     get.addFamily(FAMILIES[1]);
1335     get.setMaxVersions(Integer.MAX_VALUE);
1336     result = ht.get(get);
1337     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1338         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1339         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1340         0, 2);
1341 
1342     scan = new Scan(ROW);
1343     scan.addColumn(FAMILIES[1], QUALIFIER);
1344     scan.setMaxVersions(Integer.MAX_VALUE);
1345     result = getSingleScanResult(ht, scan);
1346     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1347         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1348         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1349         0, 2);
1350 
1351     scan = new Scan(ROW);
1352     scan.addFamily(FAMILIES[1]);
1353     scan.setMaxVersions(Integer.MAX_VALUE);
1354     result = getSingleScanResult(ht, scan);
1355     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1356         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1357         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1358         0, 2);
1359 
1360     // Family2
1361 
1362     get = new Get(ROW);
1363     get.addColumn(FAMILIES[2], QUALIFIER);
1364     get.setMaxVersions(Integer.MAX_VALUE);
1365     result = ht.get(get);
1366     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1367         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1368         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1369         0, 4);
1370 
1371     get = new Get(ROW);
1372     get.addFamily(FAMILIES[2]);
1373     get.setMaxVersions(Integer.MAX_VALUE);
1374     result = ht.get(get);
1375     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1376         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1377         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1378         0, 4);
1379 
1380     scan = new Scan(ROW);
1381     scan.addColumn(FAMILIES[2], QUALIFIER);
1382     scan.setMaxVersions(Integer.MAX_VALUE);
1383     result = getSingleScanResult(ht, scan);
1384     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1385         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1386         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1387         0, 4);
1388 
1389     scan = new Scan(ROW);
1390     scan.addFamily(FAMILIES[2]);
1391     scan.setMaxVersions(Integer.MAX_VALUE);
1392     result = getSingleScanResult(ht, scan);
1393     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1394         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1395         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1396         0, 4);
1397 
1398     // Try all families
1399 
1400     get = new Get(ROW);
1401     get.setMaxVersions(Integer.MAX_VALUE);
1402     result = ht.get(get);
1403     assertTrue("Expected 9 keys but received " + result.size(),
1404         result.size() == 9);
1405 
1406     get = new Get(ROW);
1407     get.addFamily(FAMILIES[0]);
1408     get.addFamily(FAMILIES[1]);
1409     get.addFamily(FAMILIES[2]);
1410     get.setMaxVersions(Integer.MAX_VALUE);
1411     result = ht.get(get);
1412     assertTrue("Expected 9 keys but received " + result.size(),
1413         result.size() == 9);
1414 
1415     get = new Get(ROW);
1416     get.addColumn(FAMILIES[0], QUALIFIER);
1417     get.addColumn(FAMILIES[1], QUALIFIER);
1418     get.addColumn(FAMILIES[2], QUALIFIER);
1419     get.setMaxVersions(Integer.MAX_VALUE);
1420     result = ht.get(get);
1421     assertTrue("Expected 9 keys but received " + result.size(),
1422         result.size() == 9);
1423 
1424     scan = new Scan(ROW);
1425     scan.setMaxVersions(Integer.MAX_VALUE);
1426     result = getSingleScanResult(ht, scan);
1427     assertTrue("Expected 9 keys but received " + result.size(),
1428         result.size() == 9);
1429 
1430     scan = new Scan(ROW);
1431     scan.setMaxVersions(Integer.MAX_VALUE);
1432     scan.addFamily(FAMILIES[0]);
1433     scan.addFamily(FAMILIES[1]);
1434     scan.addFamily(FAMILIES[2]);
1435     result = getSingleScanResult(ht, scan);
1436     assertTrue("Expected 9 keys but received " + result.size(),
1437         result.size() == 9);
1438 
1439     scan = new Scan(ROW);
1440     scan.setMaxVersions(Integer.MAX_VALUE);
1441     scan.addColumn(FAMILIES[0], QUALIFIER);
1442     scan.addColumn(FAMILIES[1], QUALIFIER);
1443     scan.addColumn(FAMILIES[2], QUALIFIER);
1444     result = getSingleScanResult(ht, scan);
1445     assertTrue("Expected 9 keys but received " + result.size(),
1446         result.size() == 9);
1447 
1448   }
1449 
1450   @Test
1451   public void testDeletes() throws Exception {
1452     byte [] TABLE = Bytes.toBytes("testDeletes");
1453 
1454     byte [][] ROWS = makeNAscii(ROW, 6);
1455     byte [][] FAMILIES = makeNAscii(FAMILY, 3);
1456     byte [][] VALUES = makeN(VALUE, 5);
1457     long [] ts = {1000, 2000, 3000, 4000, 5000};
1458 
1459     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
1460 
1461     Put put = new Put(ROW);
1462     put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
1463     put.add(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
1464     ht.put(put);
1465 
1466     Delete delete = new Delete(ROW);
1467     delete.deleteFamily(FAMILIES[0], ts[0]);
1468     ht.delete(delete);
1469 
1470     Get get = new Get(ROW);
1471     get.addFamily(FAMILIES[0]);
1472     get.setMaxVersions(Integer.MAX_VALUE);
1473     Result result = ht.get(get);
1474     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1475         new long [] {ts[1]},
1476         new byte[][] {VALUES[1]},
1477         0, 0);
1478 
1479     Scan scan = new Scan(ROW);
1480     scan.addFamily(FAMILIES[0]);
1481     scan.setMaxVersions(Integer.MAX_VALUE);
1482     result = getSingleScanResult(ht, scan);
1483     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1484         new long [] {ts[1]},
1485         new byte[][] {VALUES[1]},
1486         0, 0);
1487 
1488     // Test delete latest version
1489     put = new Put(ROW);
1490     put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
1491     put.add(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
1492     put.add(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
1493     put.add(FAMILIES[0], null, ts[4], VALUES[4]);
1494     put.add(FAMILIES[0], null, ts[2], VALUES[2]);
1495     put.add(FAMILIES[0], null, ts[3], VALUES[3]);
1496     ht.put(put);
1497 
1498     delete = new Delete(ROW);
1499     delete.deleteColumn(FAMILIES[0], QUALIFIER); // ts[4]
1500     ht.delete(delete);
1501 
1502     get = new Get(ROW);
1503     get.addColumn(FAMILIES[0], QUALIFIER);
1504     get.setMaxVersions(Integer.MAX_VALUE);
1505     result = ht.get(get);
1506     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1507         new long [] {ts[1], ts[2], ts[3]},
1508         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1509         0, 2);
1510 
1511     scan = new Scan(ROW);
1512     scan.addColumn(FAMILIES[0], QUALIFIER);
1513     scan.setMaxVersions(Integer.MAX_VALUE);
1514     result = getSingleScanResult(ht, scan);
1515     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1516         new long [] {ts[1], ts[2], ts[3]},
1517         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1518         0, 2);
1519 
1520     // Test for HBASE-1847
1521     delete = new Delete(ROW);
1522     delete.deleteColumn(FAMILIES[0], null);
1523     ht.delete(delete);
1524 
1525     // Cleanup null qualifier
1526     delete = new Delete(ROW);
1527     delete.deleteColumns(FAMILIES[0], null);
1528     ht.delete(delete);
1529 
1530     // Expected client behavior might be that you can re-put deleted values
1531     // But alas, this is not to be.  We can't put them back in either case.
1532 
1533     put = new Put(ROW);
1534     put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); // 1000
1535     put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); // 5000
1536     ht.put(put);
1537 
1538 
1539     // It used to be due to the internal implementation of Get, that
1540     // the Get() call would return ts[4] UNLIKE the Scan below. With
1541     // the switch to using Scan for Get this is no longer the case.
1542     get = new Get(ROW);
1543     get.addFamily(FAMILIES[0]);
1544     get.setMaxVersions(Integer.MAX_VALUE);
1545     result = ht.get(get);
1546     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1547         new long [] {ts[1], ts[2], ts[3]},
1548         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1549         0, 2);
1550 
1551     // The Scanner returns the previous values, the expected-naive-unexpected behavior
1552 
1553     scan = new Scan(ROW);
1554     scan.addFamily(FAMILIES[0]);
1555     scan.setMaxVersions(Integer.MAX_VALUE);
1556     result = getSingleScanResult(ht, scan);
1557     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1558         new long [] {ts[1], ts[2], ts[3]},
1559         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1560         0, 2);
1561 
1562     // Test deleting an entire family from one row but not the other various ways
1563 
1564     put = new Put(ROWS[0]);
1565     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
1566     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
1567     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
1568     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
1569     ht.put(put);
1570 
1571     put = new Put(ROWS[1]);
1572     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
1573     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
1574     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
1575     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
1576     ht.put(put);
1577 
1578     put = new Put(ROWS[2]);
1579     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
1580     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
1581     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
1582     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
1583     ht.put(put);
1584 
1585     // Assert that above went in.
1586     get = new Get(ROWS[2]);
1587     get.addFamily(FAMILIES[1]);
1588     get.addFamily(FAMILIES[2]);
1589     get.setMaxVersions(Integer.MAX_VALUE);
1590     result = ht.get(get);
1591     assertTrue("Expected 4 key but received " + result.size() + ": " + result,
1592         result.size() == 4);
1593 
1594     delete = new Delete(ROWS[0]);
1595     delete.deleteFamily(FAMILIES[2]);
1596     ht.delete(delete);
1597 
1598     delete = new Delete(ROWS[1]);
1599     delete.deleteColumns(FAMILIES[1], QUALIFIER);
1600     ht.delete(delete);
1601 
1602     delete = new Delete(ROWS[2]);
1603     delete.deleteColumn(FAMILIES[1], QUALIFIER);
1604     delete.deleteColumn(FAMILIES[1], QUALIFIER);
1605     delete.deleteColumn(FAMILIES[2], QUALIFIER);
1606     ht.delete(delete);
1607 
1608     get = new Get(ROWS[0]);
1609     get.addFamily(FAMILIES[1]);
1610     get.addFamily(FAMILIES[2]);
1611     get.setMaxVersions(Integer.MAX_VALUE);
1612     result = ht.get(get);
1613     assertTrue("Expected 2 keys but received " + result.size(),
1614         result.size() == 2);
1615     assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER,
1616         new long [] {ts[0], ts[1]},
1617         new byte[][] {VALUES[0], VALUES[1]},
1618         0, 1);
1619 
1620     scan = new Scan(ROWS[0]);
1621     scan.addFamily(FAMILIES[1]);
1622     scan.addFamily(FAMILIES[2]);
1623     scan.setMaxVersions(Integer.MAX_VALUE);
1624     result = getSingleScanResult(ht, scan);
1625     assertTrue("Expected 2 keys but received " + result.size(),
1626         result.size() == 2);
1627     assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER,
1628         new long [] {ts[0], ts[1]},
1629         new byte[][] {VALUES[0], VALUES[1]},
1630         0, 1);
1631 
1632     get = new Get(ROWS[1]);
1633     get.addFamily(FAMILIES[1]);
1634     get.addFamily(FAMILIES[2]);
1635     get.setMaxVersions(Integer.MAX_VALUE);
1636     result = ht.get(get);
1637     assertTrue("Expected 2 keys but received " + result.size(),
1638         result.size() == 2);
1639 
1640     scan = new Scan(ROWS[1]);
1641     scan.addFamily(FAMILIES[1]);
1642     scan.addFamily(FAMILIES[2]);
1643     scan.setMaxVersions(Integer.MAX_VALUE);
1644     result = getSingleScanResult(ht, scan);
1645     assertTrue("Expected 2 keys but received " + result.size(),
1646         result.size() == 2);
1647 
1648     get = new Get(ROWS[2]);
1649     get.addFamily(FAMILIES[1]);
1650     get.addFamily(FAMILIES[2]);
1651     get.setMaxVersions(Integer.MAX_VALUE);
1652     result = ht.get(get);
1653     assertEquals(1, result.size());
1654     assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
1655         new long [] {ts[2]},
1656         new byte[][] {VALUES[2]},
1657         0, 0);
1658 
1659     scan = new Scan(ROWS[2]);
1660     scan.addFamily(FAMILIES[1]);
1661     scan.addFamily(FAMILIES[2]);
1662     scan.setMaxVersions(Integer.MAX_VALUE);
1663     result = getSingleScanResult(ht, scan);
1664     assertEquals(1, result.size());
1665     assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
1666         new long [] {ts[2]},
1667         new byte[][] {VALUES[2]},
1668         0, 0);
1669 
1670     // Test if we delete the family first in one row (HBASE-1541)
1671 
1672     delete = new Delete(ROWS[3]);
1673     delete.deleteFamily(FAMILIES[1]);
1674     ht.delete(delete);
1675 
1676     put = new Put(ROWS[3]);
1677     put.add(FAMILIES[2], QUALIFIER, VALUES[0]);
1678     ht.put(put);
1679 
1680     put = new Put(ROWS[4]);
1681     put.add(FAMILIES[1], QUALIFIER, VALUES[1]);
1682     put.add(FAMILIES[2], QUALIFIER, VALUES[2]);
1683     ht.put(put);
1684 
1685     get = new Get(ROWS[3]);
1686     get.addFamily(FAMILIES[1]);
1687     get.addFamily(FAMILIES[2]);
1688     get.setMaxVersions(Integer.MAX_VALUE);
1689     result = ht.get(get);
1690     assertTrue("Expected 1 key but received " + result.size(),
1691         result.size() == 1);
1692 
1693     get = new Get(ROWS[4]);
1694     get.addFamily(FAMILIES[1]);
1695     get.addFamily(FAMILIES[2]);
1696     get.setMaxVersions(Integer.MAX_VALUE);
1697     result = ht.get(get);
1698     assertTrue("Expected 2 keys but received " + result.size(),
1699         result.size() == 2);
1700 
1701     scan = new Scan(ROWS[3]);
1702     scan.addFamily(FAMILIES[1]);
1703     scan.addFamily(FAMILIES[2]);
1704     scan.setMaxVersions(Integer.MAX_VALUE);
1705     ResultScanner scanner = ht.getScanner(scan);
1706     result = scanner.next();
1707     assertTrue("Expected 1 key but received " + result.size(),
1708         result.size() == 1);
1709     assertTrue(Bytes.equals(result.sorted()[0].getRow(), ROWS[3]));
1710     assertTrue(Bytes.equals(result.sorted()[0].getValue(), VALUES[0]));
1711     result = scanner.next();
1712     assertTrue("Expected 2 keys but received " + result.size(),
1713         result.size() == 2);
1714     assertTrue(Bytes.equals(result.sorted()[0].getRow(), ROWS[4]));
1715     assertTrue(Bytes.equals(result.sorted()[1].getRow(), ROWS[4]));
1716     assertTrue(Bytes.equals(result.sorted()[0].getValue(), VALUES[1]));
1717     assertTrue(Bytes.equals(result.sorted()[1].getValue(), VALUES[2]));
1718     scanner.close();
1719 
1720     // Add test of bulk deleting.
1721     for (int i = 0; i < 10; i++) {
1722       byte [] bytes = Bytes.toBytes(i);
1723       put = new Put(bytes);
1724       put.add(FAMILIES[0], QUALIFIER, bytes);
1725       ht.put(put);
1726     }
1727     for (int i = 0; i < 10; i++) {
1728       byte [] bytes = Bytes.toBytes(i);
1729       get = new Get(bytes);
1730       get.addFamily(FAMILIES[0]);
1731       result = ht.get(get);
1732       assertTrue(result.size() == 1);
1733     }
1734     ArrayList<Delete> deletes = new ArrayList<Delete>();
1735     for (int i = 0; i < 10; i++) {
1736       byte [] bytes = Bytes.toBytes(i);
1737       delete = new Delete(bytes);
1738       delete.deleteFamily(FAMILIES[0]);
1739       deletes.add(delete);
1740     }
1741     ht.delete(deletes);
1742     for (int i = 0; i < 10; i++) {
1743       byte [] bytes = Bytes.toBytes(i);
1744       get = new Get(bytes);
1745       get.addFamily(FAMILIES[0]);
1746       result = ht.get(get);
1747       assertTrue(result.size() == 0);
1748     }
1749   }
1750 
1751   /*
1752    * Baseline "scalability" test.
1753    *
1754    * Tests one hundred families, one million columns, one million versions
1755    */
1756   @Ignore @Test
1757   public void testMillions() throws Exception {
1758 
1759     // 100 families
1760 
1761     // millions of columns
1762 
1763     // millions of versions
1764 
1765   }
1766 
1767   @Ignore @Test
1768   public void testMultipleRegionsAndBatchPuts() throws Exception {
1769     // Two family table
1770 
1771     // Insert lots of rows
1772 
1773     // Insert to the same row with batched puts
1774 
1775     // Insert to multiple rows with batched puts
1776 
1777     // Split the table
1778 
1779     // Get row from first region
1780 
1781     // Get row from second region
1782 
1783     // Scan all rows
1784 
1785     // Insert to multiple regions with batched puts
1786 
1787     // Get row from first region
1788 
1789     // Get row from second region
1790 
1791     // Scan all rows
1792 
1793 
1794   }
1795 
1796   @Ignore @Test
1797   public void testMultipleRowMultipleFamily() throws Exception {
1798 
1799   }
1800 
1801   //
1802   // JIRA Testers
1803   //
1804 
1805   /**
1806    * HBASE-867
1807    *    If millions of columns in a column family, hbase scanner won't come up
1808    *
1809    *    Test will create numRows rows, each with numColsPerRow columns
1810    *    (1 version each), and attempt to scan them all.
1811    *
1812    *    To test at scale, up numColsPerRow to the millions
1813    *    (have not gotten that to work running as junit though)
1814    */
1815   @Test
1816   public void testJiraTest867() throws Exception {
1817     int numRows = 10;
1818     int numColsPerRow = 2000;
1819 
1820     byte [] TABLE = Bytes.toBytes("testJiraTest867");
1821 
1822     byte [][] ROWS = makeN(ROW, numRows);
1823     byte [][] QUALIFIERS = makeN(QUALIFIER, numColsPerRow);
1824 
1825     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
1826 
1827     // Insert rows
1828 
1829     for(int i=0;i<numRows;i++) {
1830       Put put = new Put(ROWS[i]);
1831       for(int j=0;j<numColsPerRow;j++) {
1832         put.add(FAMILY, QUALIFIERS[j], QUALIFIERS[j]);
1833       }
1834       assertTrue("Put expected to contain " + numColsPerRow + " columns but " +
1835           "only contains " + put.size(), put.size() == numColsPerRow);
1836       ht.put(put);
1837     }
1838 
1839     // Get a row
1840     Get get = new Get(ROWS[numRows-1]);
1841     Result result = ht.get(get);
1842     assertNumKeys(result, numColsPerRow);
1843     KeyValue [] keys = result.sorted();
1844     for(int i=0;i<result.size();i++) {
1845       assertKey(keys[i], ROWS[numRows-1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
1846     }
1847 
1848     // Scan the rows
1849     Scan scan = new Scan();
1850     ResultScanner scanner = ht.getScanner(scan);
1851     int rowCount = 0;
1852     while((result = scanner.next()) != null) {
1853       assertNumKeys(result, numColsPerRow);
1854       KeyValue [] kvs = result.sorted();
1855       for(int i=0;i<numColsPerRow;i++) {
1856         assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
1857       }
1858       rowCount++;
1859     }
1860     scanner.close();
1861     assertTrue("Expected to scan " + numRows + " rows but actually scanned "
1862         + rowCount + " rows", rowCount == numRows);
1863 
1864     // flush and try again
1865 
1866     TEST_UTIL.flush();
1867 
1868     // Get a row
1869     get = new Get(ROWS[numRows-1]);
1870     result = ht.get(get);
1871     assertNumKeys(result, numColsPerRow);
1872     keys = result.sorted();
1873     for(int i=0;i<result.size();i++) {
1874       assertKey(keys[i], ROWS[numRows-1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
1875     }
1876 
1877     // Scan the rows
1878     scan = new Scan();
1879     scanner = ht.getScanner(scan);
1880     rowCount = 0;
1881     while((result = scanner.next()) != null) {
1882       assertNumKeys(result, numColsPerRow);
1883       KeyValue [] kvs = result.sorted();
1884       for(int i=0;i<numColsPerRow;i++) {
1885         assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
1886       }
1887       rowCount++;
1888     }
1889     scanner.close();
1890     assertTrue("Expected to scan " + numRows + " rows but actually scanned "
1891         + rowCount + " rows", rowCount == numRows);
1892 
1893   }
1894 
1895   /**
1896    * HBASE-861
1897    *    get with timestamp will return a value if there is a version with an
1898    *    earlier timestamp
1899    */
1900   @Test
1901   public void testJiraTest861() throws Exception {
1902 
1903     byte [] TABLE = Bytes.toBytes("testJiraTest861");
1904     byte [][] VALUES = makeNAscii(VALUE, 7);
1905     long [] STAMPS = makeStamps(7);
1906 
1907     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
1908 
1909     // Insert three versions
1910 
1911     Put put = new Put(ROW);
1912     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
1913     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1914     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1915     ht.put(put);
1916 
1917     // Get the middle value
1918     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1919 
1920     // Try to get one version before (expect fail)
1921     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
1922 
1923     // Try to get one version after (expect fail)
1924     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
1925 
1926     // Try same from storefile
1927     TEST_UTIL.flush();
1928     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1929     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
1930     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
1931 
1932     // Insert two more versions surrounding others, into memstore
1933     put = new Put(ROW);
1934     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
1935     put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
1936     ht.put(put);
1937 
1938     // Check we can get everything we should and can't get what we shouldn't
1939     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
1940     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
1941     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1942     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
1943     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1944     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
1945     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
1946 
1947     // Try same from two storefiles
1948     TEST_UTIL.flush();
1949     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
1950     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
1951     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1952     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
1953     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1954     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
1955     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
1956 
1957   }
1958 
1959   /**
1960    * HBASE-33
1961    *    Add a HTable get/obtainScanner method that retrieves all versions of a
1962    *    particular column and row between two timestamps
1963    */
1964   @Test
1965   public void testJiraTest33() throws Exception {
1966 
1967     byte [] TABLE = Bytes.toBytes("testJiraTest33");
1968     byte [][] VALUES = makeNAscii(VALUE, 7);
1969     long [] STAMPS = makeStamps(7);
1970 
1971     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
1972 
1973     // Insert lots versions
1974 
1975     Put put = new Put(ROW);
1976     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
1977     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1978     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1979     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
1980     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1981     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1982     ht.put(put);
1983 
1984     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
1985     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
1986     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
1987     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
1988 
1989     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
1990     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
1991     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
1992     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
1993 
1994     // Try same from storefile
1995     TEST_UTIL.flush();
1996 
1997     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
1998     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
1999     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2000     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2001 
2002     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2003     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2004     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2005     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2006 
2007   }
2008 
2009   /**
2010    * HBASE-1014
2011    *    commit(BatchUpdate) method should return timestamp
2012    */
2013   @Test
2014   public void testJiraTest1014() throws Exception {
2015 
2016     byte [] TABLE = Bytes.toBytes("testJiraTest1014");
2017 
2018     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2019 
2020     long manualStamp = 12345;
2021 
2022     // Insert lots versions
2023 
2024     Put put = new Put(ROW);
2025     put.add(FAMILY, QUALIFIER, manualStamp, VALUE);
2026     ht.put(put);
2027 
2028     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, manualStamp, VALUE);
2029     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp-1);
2030     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp+1);
2031 
2032   }
2033 
2034   /**
2035    * HBASE-1182
2036    *    Scan for columns > some timestamp
2037    */
2038   @Test
2039   public void testJiraTest1182() throws Exception {
2040 
2041     byte [] TABLE = Bytes.toBytes("testJiraTest1182");
2042     byte [][] VALUES = makeNAscii(VALUE, 7);
2043     long [] STAMPS = makeStamps(7);
2044 
2045     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2046 
2047     // Insert lots versions
2048 
2049     Put put = new Put(ROW);
2050     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2051     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2052     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2053     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2054     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2055     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2056     ht.put(put);
2057 
2058     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2059     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2060     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2061 
2062     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2063     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2064     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2065 
2066     // Try same from storefile
2067     TEST_UTIL.flush();
2068 
2069     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2070     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2071     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2072 
2073     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2074     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2075     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2076   }
2077 
2078   /**
2079    * HBASE-52
2080    *    Add a means of scanning over all versions
2081    */
2082   @Test
2083   public void testJiraTest52() throws Exception {
2084     byte [] TABLE = Bytes.toBytes("testJiraTest52");
2085     byte [][] VALUES = makeNAscii(VALUE, 7);
2086     long [] STAMPS = makeStamps(7);
2087 
2088     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2089 
2090     // Insert lots versions
2091 
2092     Put put = new Put(ROW);
2093     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2094     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2095     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2096     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2097     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2098     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2099     ht.put(put);
2100 
2101     getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2102 
2103     scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2104 
2105     // Try same from storefile
2106     TEST_UTIL.flush();
2107 
2108     getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2109 
2110     scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2111   }
2112 
2113   //
2114   // Bulk Testers
2115   //
2116 
2117   private void getVersionRangeAndVerifyGreaterThan(HTable ht, byte [] row,
2118       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
2119       int start, int end)
2120   throws IOException {
2121     Get get = new Get(row);
2122     get.addColumn(family, qualifier);
2123     get.setMaxVersions(Integer.MAX_VALUE);
2124     get.setTimeRange(stamps[start+1], Long.MAX_VALUE);
2125     Result result = ht.get(get);
2126     assertNResult(result, row, family, qualifier, stamps, values, start+1, end);
2127   }
2128 
2129   private void getVersionRangeAndVerify(HTable ht, byte [] row, byte [] family,
2130       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2131   throws IOException {
2132     Get get = new Get(row);
2133     get.addColumn(family, qualifier);
2134     get.setMaxVersions(Integer.MAX_VALUE);
2135     get.setTimeRange(stamps[start], stamps[end]+1);
2136     Result result = ht.get(get);
2137     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2138   }
2139 
2140   private void getAllVersionsAndVerify(HTable ht, byte [] row, byte [] family,
2141       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2142   throws IOException {
2143     Get get = new Get(row);
2144     get.addColumn(family, qualifier);
2145     get.setMaxVersions(Integer.MAX_VALUE);
2146     Result result = ht.get(get);
2147     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2148   }
2149 
2150   private void scanVersionRangeAndVerifyGreaterThan(HTable ht, byte [] row,
2151       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
2152       int start, int end)
2153   throws IOException {
2154     Scan scan = new Scan(row);
2155     scan.addColumn(family, qualifier);
2156     scan.setMaxVersions(Integer.MAX_VALUE);
2157     scan.setTimeRange(stamps[start+1], Long.MAX_VALUE);
2158     Result result = getSingleScanResult(ht, scan);
2159     assertNResult(result, row, family, qualifier, stamps, values, start+1, end);
2160   }
2161 
2162   private void scanVersionRangeAndVerify(HTable ht, byte [] row, byte [] family,
2163       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2164   throws IOException {
2165     Scan scan = new Scan(row);
2166     scan.addColumn(family, qualifier);
2167     scan.setMaxVersions(Integer.MAX_VALUE);
2168     scan.setTimeRange(stamps[start], stamps[end]+1);
2169     Result result = getSingleScanResult(ht, scan);
2170     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2171   }
2172 
2173   private void scanAllVersionsAndVerify(HTable ht, byte [] row, byte [] family,
2174       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2175   throws IOException {
2176     Scan scan = new Scan(row);
2177     scan.addColumn(family, qualifier);
2178     scan.setMaxVersions(Integer.MAX_VALUE);
2179     Result result = getSingleScanResult(ht, scan);
2180     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2181   }
2182 
2183   private void getVersionAndVerify(HTable ht, byte [] row, byte [] family,
2184       byte [] qualifier, long stamp, byte [] value)
2185   throws Exception {
2186     Get get = new Get(row);
2187     get.addColumn(family, qualifier);
2188     get.setTimeStamp(stamp);
2189     get.setMaxVersions(Integer.MAX_VALUE);
2190     Result result = ht.get(get);
2191     assertSingleResult(result, row, family, qualifier, stamp, value);
2192   }
2193 
2194   private void getVersionAndVerifyMissing(HTable ht, byte [] row, byte [] family,
2195       byte [] qualifier, long stamp)
2196   throws Exception {
2197     Get get = new Get(row);
2198     get.addColumn(family, qualifier);
2199     get.setTimeStamp(stamp);
2200     get.setMaxVersions(Integer.MAX_VALUE);
2201     Result result = ht.get(get);
2202     assertEmptyResult(result);
2203   }
2204 
2205   private void scanVersionAndVerify(HTable ht, byte [] row, byte [] family,
2206       byte [] qualifier, long stamp, byte [] value)
2207   throws Exception {
2208     Scan scan = new Scan(row);
2209     scan.addColumn(family, qualifier);
2210     scan.setTimeStamp(stamp);
2211     scan.setMaxVersions(Integer.MAX_VALUE);
2212     Result result = getSingleScanResult(ht, scan);
2213     assertSingleResult(result, row, family, qualifier, stamp, value);
2214   }
2215 
2216   private void scanVersionAndVerifyMissing(HTable ht, byte [] row,
2217       byte [] family, byte [] qualifier, long stamp)
2218   throws Exception {
2219     Scan scan = new Scan(row);
2220     scan.addColumn(family, qualifier);
2221     scan.setTimeStamp(stamp);
2222     scan.setMaxVersions(Integer.MAX_VALUE);
2223     Result result = getSingleScanResult(ht, scan);
2224     assertNullResult(result);
2225   }
2226 
2227   private void getTestNull(HTable ht, byte [] row, byte [] family,
2228       byte [] value)
2229   throws Exception {
2230 
2231     Get get = new Get(row);
2232     get.addColumn(family, null);
2233     Result result = ht.get(get);
2234     assertSingleResult(result, row, family, null, value);
2235 
2236     get = new Get(row);
2237     get.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
2238     result = ht.get(get);
2239     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2240 
2241     get = new Get(row);
2242     get.addFamily(family);
2243     result = ht.get(get);
2244     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2245 
2246     get = new Get(row);
2247     result = ht.get(get);
2248     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2249 
2250   }
2251 
2252   private void scanTestNull(HTable ht, byte [] row, byte [] family,
2253       byte [] value)
2254   throws Exception {
2255 
2256     Scan scan = new Scan();
2257     scan.addColumn(family, null);
2258     Result result = getSingleScanResult(ht, scan);
2259     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2260 
2261     scan = new Scan();
2262     scan.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
2263     result = getSingleScanResult(ht, scan);
2264     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2265 
2266     scan = new Scan();
2267     scan.addFamily(family);
2268     result = getSingleScanResult(ht, scan);
2269     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2270 
2271     scan = new Scan();
2272     result = getSingleScanResult(ht, scan);
2273     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2274 
2275   }
2276 
2277   private void singleRowGetTest(HTable ht, byte [][] ROWS, byte [][] FAMILIES,
2278       byte [][] QUALIFIERS, byte [][] VALUES)
2279   throws Exception {
2280 
2281     // Single column from memstore
2282     Get get = new Get(ROWS[0]);
2283     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2284     Result result = ht.get(get);
2285     assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]);
2286 
2287     // Single column from storefile
2288     get = new Get(ROWS[0]);
2289     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
2290     result = ht.get(get);
2291     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
2292 
2293     // Single column from storefile, family match
2294     get = new Get(ROWS[0]);
2295     get.addFamily(FAMILIES[7]);
2296     result = ht.get(get);
2297     assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]);
2298 
2299     // Two columns, one from memstore one from storefile, same family,
2300     // wildcard match
2301     get = new Get(ROWS[0]);
2302     get.addFamily(FAMILIES[4]);
2303     result = ht.get(get);
2304     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2305         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2306 
2307     // Two columns, one from memstore one from storefile, same family,
2308     // explicit match
2309     get = new Get(ROWS[0]);
2310     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2311     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
2312     result = ht.get(get);
2313     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2314         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2315 
2316     // Three column, one from memstore two from storefile, different families,
2317     // wildcard match
2318     get = new Get(ROWS[0]);
2319     get.addFamily(FAMILIES[4]);
2320     get.addFamily(FAMILIES[7]);
2321     result = ht.get(get);
2322     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2323         new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} });
2324 
2325     // Multiple columns from everywhere storefile, many family, wildcard
2326     get = new Get(ROWS[0]);
2327     get.addFamily(FAMILIES[2]);
2328     get.addFamily(FAMILIES[4]);
2329     get.addFamily(FAMILIES[6]);
2330     get.addFamily(FAMILIES[7]);
2331     result = ht.get(get);
2332     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2333         new int [][] {
2334           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2335     });
2336 
2337     // Multiple columns from everywhere storefile, many family, wildcard
2338     get = new Get(ROWS[0]);
2339     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
2340     get.addColumn(FAMILIES[2], QUALIFIERS[4]);
2341     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2342     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
2343     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
2344     get.addColumn(FAMILIES[6], QUALIFIERS[7]);
2345     get.addColumn(FAMILIES[7], QUALIFIERS[7]);
2346     get.addColumn(FAMILIES[7], QUALIFIERS[8]);
2347     result = ht.get(get);
2348     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2349         new int [][] {
2350           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2351     });
2352 
2353     // Everything
2354     get = new Get(ROWS[0]);
2355     result = ht.get(get);
2356     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2357         new int [][] {
2358           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0}
2359     });
2360 
2361     // Get around inserted columns
2362 
2363     get = new Get(ROWS[1]);
2364     result = ht.get(get);
2365     assertEmptyResult(result);
2366 
2367     get = new Get(ROWS[0]);
2368     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
2369     get.addColumn(FAMILIES[2], QUALIFIERS[3]);
2370     result = ht.get(get);
2371     assertEmptyResult(result);
2372 
2373   }
2374 
2375   private void singleRowScanTest(HTable ht, byte [][] ROWS, byte [][] FAMILIES,
2376       byte [][] QUALIFIERS, byte [][] VALUES)
2377   throws Exception {
2378 
2379     // Single column from memstore
2380     Scan scan = new Scan();
2381     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
2382     Result result = getSingleScanResult(ht, scan);
2383     assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]);
2384 
2385     // Single column from storefile
2386     scan = new Scan();
2387     scan.addColumn(FAMILIES[2], QUALIFIERS[2]);
2388     result = getSingleScanResult(ht, scan);
2389     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
2390 
2391     // Single column from storefile, family match
2392     scan = new Scan();
2393     scan.addFamily(FAMILIES[7]);
2394     result = getSingleScanResult(ht, scan);
2395     assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]);
2396 
2397     // Two columns, one from memstore one from storefile, same family,
2398     // wildcard match
2399     scan = new Scan();
2400     scan.addFamily(FAMILIES[4]);
2401     result = getSingleScanResult(ht, scan);
2402     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2403         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2404 
2405     // Two columns, one from memstore one from storefile, same family,
2406     // explicit match
2407     scan = new Scan();
2408     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
2409     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
2410     result = getSingleScanResult(ht, scan);
2411     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2412         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2413 
2414     // Three column, one from memstore two from storefile, different families,
2415     // wildcard match
2416     scan = new Scan();
2417     scan.addFamily(FAMILIES[4]);
2418     scan.addFamily(FAMILIES[7]);
2419     result = getSingleScanResult(ht, scan);
2420     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2421         new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} });
2422 
2423     // Multiple columns from everywhere storefile, many family, wildcard
2424     scan = new Scan();
2425     scan.addFamily(FAMILIES[2]);
2426     scan.addFamily(FAMILIES[4]);
2427     scan.addFamily(FAMILIES[6]);
2428     scan.addFamily(FAMILIES[7]);
2429     result = getSingleScanResult(ht, scan);
2430     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2431         new int [][] {
2432           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2433     });
2434 
2435     // Multiple columns from everywhere storefile, many family, wildcard
2436     scan = new Scan();
2437     scan.addColumn(FAMILIES[2], QUALIFIERS[2]);
2438     scan.addColumn(FAMILIES[2], QUALIFIERS[4]);
2439     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
2440     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
2441     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
2442     scan.addColumn(FAMILIES[6], QUALIFIERS[7]);
2443     scan.addColumn(FAMILIES[7], QUALIFIERS[7]);
2444     scan.addColumn(FAMILIES[7], QUALIFIERS[8]);
2445     result = getSingleScanResult(ht, scan);
2446     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2447         new int [][] {
2448           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2449     });
2450 
2451     // Everything
2452     scan = new Scan();
2453     result = getSingleScanResult(ht, scan);
2454     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2455         new int [][] {
2456           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0}
2457     });
2458 
2459     // Scan around inserted columns
2460 
2461     scan = new Scan(ROWS[1]);
2462     result = getSingleScanResult(ht, scan);
2463     assertNullResult(result);
2464 
2465     scan = new Scan();
2466     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
2467     scan.addColumn(FAMILIES[2], QUALIFIERS[3]);
2468     result = getSingleScanResult(ht, scan);
2469     assertNullResult(result);
2470   }
2471 
2472   /**
2473    * Verify a single column using gets.
2474    * Expects family and qualifier arrays to be valid for at least
2475    * the range:  idx-2 < idx < idx+2
2476    */
2477   private void getVerifySingleColumn(HTable ht,
2478       byte [][] ROWS, int ROWIDX,
2479       byte [][] FAMILIES, int FAMILYIDX,
2480       byte [][] QUALIFIERS, int QUALIFIERIDX,
2481       byte [][] VALUES, int VALUEIDX)
2482   throws Exception {
2483 
2484     Get get = new Get(ROWS[ROWIDX]);
2485     Result result = ht.get(get);
2486     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2487         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2488 
2489     get = new Get(ROWS[ROWIDX]);
2490     get.addFamily(FAMILIES[FAMILYIDX]);
2491     result = ht.get(get);
2492     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2493         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2494 
2495     get = new Get(ROWS[ROWIDX]);
2496     get.addFamily(FAMILIES[FAMILYIDX-2]);
2497     get.addFamily(FAMILIES[FAMILYIDX]);
2498     get.addFamily(FAMILIES[FAMILYIDX+2]);
2499     result = ht.get(get);
2500     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2501         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2502 
2503     get = new Get(ROWS[ROWIDX]);
2504     get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[0]);
2505     result = ht.get(get);
2506     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2507         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2508 
2509     get = new Get(ROWS[ROWIDX]);
2510     get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[1]);
2511     get.addFamily(FAMILIES[FAMILYIDX]);
2512     result = ht.get(get);
2513     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2514         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2515 
2516     get = new Get(ROWS[ROWIDX]);
2517     get.addFamily(FAMILIES[FAMILYIDX]);
2518     get.addColumn(FAMILIES[FAMILYIDX+1], QUALIFIERS[1]);
2519     get.addColumn(FAMILIES[FAMILYIDX-2], QUALIFIERS[1]);
2520     get.addFamily(FAMILIES[FAMILYIDX-1]);
2521     get.addFamily(FAMILIES[FAMILYIDX+2]);
2522     result = ht.get(get);
2523     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2524         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2525 
2526   }
2527 
2528 
2529   /**
2530    * Verify a single column using scanners.
2531    * Expects family and qualifier arrays to be valid for at least
2532    * the range:  idx-2 to idx+2
2533    * Expects row array to be valid for at least idx to idx+2
2534    */
2535   private void scanVerifySingleColumn(HTable ht,
2536       byte [][] ROWS, int ROWIDX,
2537       byte [][] FAMILIES, int FAMILYIDX,
2538       byte [][] QUALIFIERS, int QUALIFIERIDX,
2539       byte [][] VALUES, int VALUEIDX)
2540   throws Exception {
2541 
2542     Scan scan = new Scan();
2543     Result result = getSingleScanResult(ht, scan);
2544     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2545         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2546 
2547     scan = new Scan(ROWS[ROWIDX]);
2548     result = getSingleScanResult(ht, scan);
2549     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2550         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2551 
2552     scan = new Scan(ROWS[ROWIDX], ROWS[ROWIDX+1]);
2553     result = getSingleScanResult(ht, scan);
2554     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2555         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2556 
2557     scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX+1]);
2558     result = getSingleScanResult(ht, scan);
2559     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2560         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2561 
2562     scan = new Scan();
2563     scan.addFamily(FAMILIES[FAMILYIDX]);
2564     result = getSingleScanResult(ht, scan);
2565     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2566         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2567 
2568     scan = new Scan();
2569     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]);
2570     result = getSingleScanResult(ht, scan);
2571     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2572         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2573 
2574     scan = new Scan();
2575     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]);
2576     scan.addFamily(FAMILIES[FAMILYIDX]);
2577     result = getSingleScanResult(ht, scan);
2578     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2579         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2580 
2581     scan = new Scan();
2582     scan.addColumn(FAMILIES[FAMILYIDX-1], QUALIFIERS[QUALIFIERIDX+1]);
2583     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]);
2584     scan.addFamily(FAMILIES[FAMILYIDX+1]);
2585     result = getSingleScanResult(ht, scan);
2586     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2587         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2588 
2589   }
2590 
2591   /**
2592    * Verify we do not read any values by accident around a single column
2593    * Same requirements as getVerifySingleColumn
2594    */
2595   private void getVerifySingleEmpty(HTable ht,
2596       byte [][] ROWS, int ROWIDX,
2597       byte [][] FAMILIES, int FAMILYIDX,
2598       byte [][] QUALIFIERS, int QUALIFIERIDX)
2599   throws Exception {
2600 
2601     Get get = new Get(ROWS[ROWIDX]);
2602     get.addFamily(FAMILIES[4]);
2603     get.addColumn(FAMILIES[4], QUALIFIERS[1]);
2604     Result result = ht.get(get);
2605     assertEmptyResult(result);
2606 
2607     get = new Get(ROWS[ROWIDX]);
2608     get.addFamily(FAMILIES[4]);
2609     get.addColumn(FAMILIES[4], QUALIFIERS[2]);
2610     result = ht.get(get);
2611     assertEmptyResult(result);
2612 
2613     get = new Get(ROWS[ROWIDX]);
2614     get.addFamily(FAMILIES[3]);
2615     get.addColumn(FAMILIES[4], QUALIFIERS[2]);
2616     get.addFamily(FAMILIES[5]);
2617     result = ht.get(get);
2618     assertEmptyResult(result);
2619 
2620     get = new Get(ROWS[ROWIDX+1]);
2621     result = ht.get(get);
2622     assertEmptyResult(result);
2623 
2624   }
2625 
2626   private void scanVerifySingleEmpty(HTable ht,
2627       byte [][] ROWS, int ROWIDX,
2628       byte [][] FAMILIES, int FAMILYIDX,
2629       byte [][] QUALIFIERS, int QUALIFIERIDX)
2630   throws Exception {
2631 
2632     Scan scan = new Scan(ROWS[ROWIDX+1]);
2633     Result result = getSingleScanResult(ht, scan);
2634     assertNullResult(result);
2635 
2636     scan = new Scan(ROWS[ROWIDX+1],ROWS[ROWIDX+2]);
2637     result = getSingleScanResult(ht, scan);
2638     assertNullResult(result);
2639 
2640     scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX]);
2641     result = getSingleScanResult(ht, scan);
2642     assertNullResult(result);
2643 
2644     scan = new Scan();
2645     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]);
2646     scan.addFamily(FAMILIES[FAMILYIDX-1]);
2647     result = getSingleScanResult(ht, scan);
2648     assertNullResult(result);
2649 
2650   }
2651 
2652   //
2653   // Verifiers
2654   //
2655 
2656   private void assertKey(KeyValue key, byte [] row, byte [] family,
2657       byte [] qualifier, byte [] value)
2658   throws Exception {
2659     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
2660         "Got row [" + Bytes.toString(key.getRow()) +"]",
2661         equals(row, key.getRow()));
2662     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
2663         "Got family [" + Bytes.toString(key.getFamily()) + "]",
2664         equals(family, key.getFamily()));
2665     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
2666         "Got qualifier [" + Bytes.toString(key.getQualifier()) + "]",
2667         equals(qualifier, key.getQualifier()));
2668     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
2669         "Got value [" + Bytes.toString(key.getValue()) + "]",
2670         equals(value, key.getValue()));
2671   }
2672 
2673   private void assertIncrementKey(KeyValue key, byte [] row, byte [] family,
2674       byte [] qualifier, long value)
2675   throws Exception {
2676     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
2677         "Got row [" + Bytes.toString(key.getRow()) +"]",
2678         equals(row, key.getRow()));
2679     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
2680         "Got family [" + Bytes.toString(key.getFamily()) + "]",
2681         equals(family, key.getFamily()));
2682     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
2683         "Got qualifier [" + Bytes.toString(key.getQualifier()) + "]",
2684         equals(qualifier, key.getQualifier()));
2685     assertTrue("Expected value [" + value + "] " +
2686         "Got value [" + Bytes.toLong(key.getValue()) + "]",
2687         Bytes.toLong(key.getValue()) == value);
2688   }
2689 
2690   private void assertNumKeys(Result result, int n) throws Exception {
2691     assertTrue("Expected " + n + " keys but got " + result.size(),
2692         result.size() == n);
2693   }
2694 
2695   private void assertNResult(Result result, byte [] row,
2696       byte [][] families, byte [][] qualifiers, byte [][] values,
2697       int [][] idxs)
2698   throws Exception {
2699     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
2700         "Got row [" + Bytes.toString(result.getRow()) +"]",
2701         equals(row, result.getRow()));
2702     assertTrue("Expected " + idxs.length + " keys but result contains "
2703         + result.size(), result.size() == idxs.length);
2704 
2705     KeyValue [] keys = result.sorted();
2706 
2707     for(int i=0;i<keys.length;i++) {
2708       byte [] family = families[idxs[i][0]];
2709       byte [] qualifier = qualifiers[idxs[i][1]];
2710       byte [] value = values[idxs[i][2]];
2711       KeyValue key = keys[i];
2712 
2713       assertTrue("(" + i + ") Expected family [" + Bytes.toString(family)
2714           + "] " + "Got family [" + Bytes.toString(key.getFamily()) + "]",
2715           equals(family, key.getFamily()));
2716       assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier)
2717           + "] " + "Got qualifier [" + Bytes.toString(key.getQualifier()) + "]",
2718           equals(qualifier, key.getQualifier()));
2719       assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] "
2720           + "Got value [" + Bytes.toString(key.getValue()) + "]",
2721           equals(value, key.getValue()));
2722     }
2723   }
2724 
2725   private void assertNResult(Result result, byte [] row,
2726       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
2727       int start, int end)
2728   throws IOException {
2729     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
2730         "Got row [" + Bytes.toString(result.getRow()) +"]",
2731         equals(row, result.getRow()));
2732     int expectedResults = end - start + 1;
2733     assertEquals(expectedResults, result.size());
2734 
2735     KeyValue [] keys = result.sorted();
2736 
2737     for (int i=0; i<keys.length; i++) {
2738       byte [] value = values[end-i];
2739       long ts = stamps[end-i];
2740       KeyValue key = keys[i];
2741 
2742       assertTrue("(" + i + ") Expected family [" + Bytes.toString(family)
2743           + "] " + "Got family [" + Bytes.toString(key.getFamily()) + "]",
2744           equals(family, key.getFamily()));
2745       assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier)
2746           + "] " + "Got qualifier [" + Bytes.toString(key.getQualifier()) + "]",
2747           equals(qualifier, key.getQualifier()));
2748       assertTrue("Expected ts [" + ts + "] " +
2749           "Got ts [" + key.getTimestamp() + "]", ts == key.getTimestamp());
2750       assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] "
2751           + "Got value [" + Bytes.toString(key.getValue()) + "]",
2752           equals(value, key.getValue()));
2753     }
2754   }
2755 
2756   /**
2757    * Validate that result contains two specified keys, exactly.
2758    * It is assumed key A sorts before key B.
2759    */
2760   private void assertDoubleResult(Result result, byte [] row,
2761       byte [] familyA, byte [] qualifierA, byte [] valueA,
2762       byte [] familyB, byte [] qualifierB, byte [] valueB)
2763   throws Exception {
2764     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
2765         "Got row [" + Bytes.toString(result.getRow()) +"]",
2766         equals(row, result.getRow()));
2767     assertTrue("Expected two keys but result contains " + result.size(),
2768         result.size() == 2);
2769     KeyValue [] kv = result.sorted();
2770     KeyValue kvA = kv[0];
2771     assertTrue("(A) Expected family [" + Bytes.toString(familyA) + "] " +
2772         "Got family [" + Bytes.toString(kvA.getFamily()) + "]",
2773         equals(familyA, kvA.getFamily()));
2774     assertTrue("(A) Expected qualifier [" + Bytes.toString(qualifierA) + "] " +
2775         "Got qualifier [" + Bytes.toString(kvA.getQualifier()) + "]",
2776         equals(qualifierA, kvA.getQualifier()));
2777     assertTrue("(A) Expected value [" + Bytes.toString(valueA) + "] " +
2778         "Got value [" + Bytes.toString(kvA.getValue()) + "]",
2779         equals(valueA, kvA.getValue()));
2780     KeyValue kvB = kv[1];
2781     assertTrue("(B) Expected family [" + Bytes.toString(familyB) + "] " +
2782         "Got family [" + Bytes.toString(kvB.getFamily()) + "]",
2783         equals(familyB, kvB.getFamily()));
2784     assertTrue("(B) Expected qualifier [" + Bytes.toString(qualifierB) + "] " +
2785         "Got qualifier [" + Bytes.toString(kvB.getQualifier()) + "]",
2786         equals(qualifierB, kvB.getQualifier()));
2787     assertTrue("(B) Expected value [" + Bytes.toString(valueB) + "] " +
2788         "Got value [" + Bytes.toString(kvB.getValue()) + "]",
2789         equals(valueB, kvB.getValue()));
2790   }
2791 
2792   private void assertSingleResult(Result result, byte [] row, byte [] family,
2793       byte [] qualifier, byte [] value)
2794   throws Exception {
2795     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
2796         "Got row [" + Bytes.toString(result.getRow()) +"]",
2797         equals(row, result.getRow()));
2798     assertTrue("Expected a single key but result contains " + result.size(),
2799         result.size() == 1);
2800     KeyValue kv = result.sorted()[0];
2801     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
2802         "Got family [" + Bytes.toString(kv.getFamily()) + "]",
2803         equals(family, kv.getFamily()));
2804     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
2805         "Got qualifier [" + Bytes.toString(kv.getQualifier()) + "]",
2806         equals(qualifier, kv.getQualifier()));
2807     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
2808         "Got value [" + Bytes.toString(kv.getValue()) + "]",
2809         equals(value, kv.getValue()));
2810   }
2811 
2812   private void assertSingleResult(Result result, byte [] row, byte [] family,
2813       byte [] qualifier, long ts, byte [] value)
2814   throws Exception {
2815     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
2816         "Got row [" + Bytes.toString(result.getRow()) +"]",
2817         equals(row, result.getRow()));
2818     assertTrue("Expected a single key but result contains " + result.size(),
2819         result.size() == 1);
2820     KeyValue kv = result.sorted()[0];
2821     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
2822         "Got family [" + Bytes.toString(kv.getFamily()) + "]",
2823         equals(family, kv.getFamily()));
2824     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
2825         "Got qualifier [" + Bytes.toString(kv.getQualifier()) + "]",
2826         equals(qualifier, kv.getQualifier()));
2827     assertTrue("Expected ts [" + ts + "] " +
2828         "Got ts [" + kv.getTimestamp() + "]", ts == kv.getTimestamp());
2829     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
2830         "Got value [" + Bytes.toString(kv.getValue()) + "]",
2831         equals(value, kv.getValue()));
2832   }
2833 
2834   private void assertEmptyResult(Result result) throws Exception {
2835     assertTrue("expected an empty result but result contains " +
2836         result.size() + " keys", result.isEmpty());
2837   }
2838 
2839   private void assertNullResult(Result result) throws Exception {
2840     assertTrue("expected null result but received a non-null result",
2841         result == null);
2842   }
2843 
2844   //
2845   // Helpers
2846   //
2847 
2848   private Result getSingleScanResult(HTable ht, Scan scan) throws IOException {
2849     ResultScanner scanner = ht.getScanner(scan);
2850     Result result = scanner.next();
2851     scanner.close();
2852     return result;
2853   }
2854 
2855   private byte [][] makeNAscii(byte [] base, int n) {
2856     if(n > 256) {
2857       return makeNBig(base, n);
2858     }
2859     byte [][] ret = new byte[n][];
2860     for(int i=0;i<n;i++) {
2861       byte [] tail = Bytes.toBytes(Integer.toString(i));
2862       ret[i] = Bytes.add(base, tail);
2863     }
2864     return ret;
2865   }
2866 
2867   private byte [][] makeN(byte [] base, int n) {
2868     if (n > 256) {
2869       return makeNBig(base, n);
2870     }
2871     byte [][] ret = new byte[n][];
2872     for(int i=0;i<n;i++) {
2873       ret[i] = Bytes.add(base, new byte[]{(byte)i});
2874     }
2875     return ret;
2876   }
2877 
2878   private byte [][] makeNBig(byte [] base, int n) {
2879     byte [][] ret = new byte[n][];
2880     for(int i=0;i<n;i++) {
2881       int byteA = (i % 256);
2882       int byteB = (i >> 8);
2883       ret[i] = Bytes.add(base, new byte[]{(byte)byteB,(byte)byteA});
2884     }
2885     return ret;
2886   }
2887 
2888   private long [] makeStamps(int n) {
2889     long [] stamps = new long[n];
2890     for(int i=0;i<n;i++) stamps[i] = i+1;
2891     return stamps;
2892   }
2893 
2894   private boolean equals(byte [] left, byte [] right) {
2895     if (left == null && right == null) return true;
2896     if (left == null && right.length == 0) return true;
2897     if (right == null && left.length == 0) return true;
2898     return Bytes.equals(left, right);
2899   }
2900 
2901   @Test
2902   public void testDuplicateVersions() throws Exception {
2903     byte [] TABLE = Bytes.toBytes("testDuplicateVersions");
2904 
2905     long [] STAMPS = makeStamps(20);
2906     byte [][] VALUES = makeNAscii(VALUE, 20);
2907 
2908     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2909 
2910     // Insert 4 versions of same column
2911     Put put = new Put(ROW);
2912     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2913     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2914     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2915     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2916     ht.put(put);
2917 
2918     // Verify we can get each one properly
2919     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2920     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2921     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2922     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2923     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2924     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2925     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2926     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2927 
2928     // Verify we don't accidentally get others
2929     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
2930     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
2931     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
2932     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
2933     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
2934     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
2935 
2936     // Ensure maxVersions in query is respected
2937     Get get = new Get(ROW);
2938     get.addColumn(FAMILY, QUALIFIER);
2939     get.setMaxVersions(2);
2940     Result result = ht.get(get);
2941     assertNResult(result, ROW, FAMILY, QUALIFIER,
2942         new long [] {STAMPS[4], STAMPS[5]},
2943         new byte[][] {VALUES[4], VALUES[5]},
2944         0, 1);
2945 
2946     Scan scan = new Scan(ROW);
2947     scan.addColumn(FAMILY, QUALIFIER);
2948     scan.setMaxVersions(2);
2949     result = getSingleScanResult(ht, scan);
2950     assertNResult(result, ROW, FAMILY, QUALIFIER,
2951         new long [] {STAMPS[4], STAMPS[5]},
2952         new byte[][] {VALUES[4], VALUES[5]},
2953         0, 1);
2954 
2955     // Flush and redo
2956 
2957     TEST_UTIL.flush();
2958 
2959     // Verify we can get each one properly
2960     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2961     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2962     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2963     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2964     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2965     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2966     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2967     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2968 
2969     // Verify we don't accidentally get others
2970     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
2971     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
2972     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
2973     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
2974     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
2975     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
2976 
2977     // Ensure maxVersions in query is respected
2978     get = new Get(ROW);
2979     get.addColumn(FAMILY, QUALIFIER);
2980     get.setMaxVersions(2);
2981     result = ht.get(get);
2982     assertNResult(result, ROW, FAMILY, QUALIFIER,
2983         new long [] {STAMPS[4], STAMPS[5]},
2984         new byte[][] {VALUES[4], VALUES[5]},
2985         0, 1);
2986 
2987     scan = new Scan(ROW);
2988     scan.addColumn(FAMILY, QUALIFIER);
2989     scan.setMaxVersions(2);
2990     result = getSingleScanResult(ht, scan);
2991     assertNResult(result, ROW, FAMILY, QUALIFIER,
2992         new long [] {STAMPS[4], STAMPS[5]},
2993         new byte[][] {VALUES[4], VALUES[5]},
2994         0, 1);
2995 
2996 
2997     // Add some memstore and retest
2998 
2999     // Insert 4 more versions of same column and a dupe
3000     put = new Put(ROW);
3001     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
3002     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3003     put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
3004     put.add(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3005     put.add(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
3006     ht.put(put);
3007 
3008     // Ensure maxVersions in query is respected
3009     get = new Get(ROW);
3010     get.addColumn(FAMILY, QUALIFIER);
3011     get.setMaxVersions(7);
3012     result = ht.get(get);
3013     assertNResult(result, ROW, FAMILY, QUALIFIER,
3014         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3015         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3016         0, 6);
3017 
3018     scan = new Scan(ROW);
3019     scan.addColumn(FAMILY, QUALIFIER);
3020     scan.setMaxVersions(7);
3021     result = getSingleScanResult(ht, scan);
3022     assertNResult(result, ROW, FAMILY, QUALIFIER,
3023         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3024         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3025         0, 6);
3026 
3027     get = new Get(ROW);
3028     get.setMaxVersions(7);
3029     result = ht.get(get);
3030     assertNResult(result, ROW, FAMILY, QUALIFIER,
3031         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3032         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3033         0, 6);
3034 
3035     scan = new Scan(ROW);
3036     scan.setMaxVersions(7);
3037     result = getSingleScanResult(ht, scan);
3038     assertNResult(result, ROW, FAMILY, QUALIFIER,
3039         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3040         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3041         0, 6);
3042 
3043     // Verify we can get each one properly
3044     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3045     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3046     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3047     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3048     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3049     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3050     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3051     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3052 
3053     // Verify we don't accidentally get others
3054     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3055     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
3056     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3057     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
3058 
3059     // Ensure maxVersions of table is respected
3060 
3061     TEST_UTIL.flush();
3062 
3063     // Insert 4 more versions of same column and a dupe
3064     put = new Put(ROW);
3065     put.add(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
3066     put.add(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
3067     put.add(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
3068     put.add(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
3069     ht.put(put);
3070 
3071     get = new Get(ROW);
3072     get.addColumn(FAMILY, QUALIFIER);
3073     get.setMaxVersions(Integer.MAX_VALUE);
3074     result = ht.get(get);
3075     assertNResult(result, ROW, FAMILY, QUALIFIER,
3076         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
3077         new byte[][] {VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
3078         0, 9);
3079 
3080     scan = new Scan(ROW);
3081     scan.addColumn(FAMILY, QUALIFIER);
3082     scan.setMaxVersions(Integer.MAX_VALUE);
3083     result = getSingleScanResult(ht, scan);
3084     assertNResult(result, ROW, FAMILY, QUALIFIER,
3085         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
3086         new byte[][] {VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
3087         0, 9);
3088 
3089     // Delete a version in the memstore and a version in a storefile
3090     Delete delete = new Delete(ROW);
3091     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[11]);
3092     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[7]);
3093     ht.delete(delete);
3094 
3095     // Test that it's gone
3096     get = new Get(ROW);
3097     get.addColumn(FAMILY, QUALIFIER);
3098     get.setMaxVersions(Integer.MAX_VALUE);
3099     result = ht.get(get);
3100     assertNResult(result, ROW, FAMILY, QUALIFIER,
3101         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
3102         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
3103         0, 9);
3104 
3105     scan = new Scan(ROW);
3106     scan.addColumn(FAMILY, QUALIFIER);
3107     scan.setMaxVersions(Integer.MAX_VALUE);
3108     result = getSingleScanResult(ht, scan);
3109     assertNResult(result, ROW, FAMILY, QUALIFIER,
3110         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
3111         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
3112         0, 9);
3113   }
3114 
3115   @Test
3116   public void testUpdates() throws Exception {
3117 
3118     byte [] TABLE = Bytes.toBytes("testUpdates");
3119     HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3120 
3121     // Write a column with values at timestamp 1, 2 and 3
3122     byte[] row = Bytes.toBytes("row1");
3123     byte[] qualifier = Bytes.toBytes("myCol");
3124     Put put = new Put(row);
3125     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3126     hTable.put(put);
3127 
3128     put = new Put(row);
3129     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3130     hTable.put(put);
3131 
3132     put = new Put(row);
3133     put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3134     hTable.put(put);
3135 
3136     Get get = new Get(row);
3137     get.addColumn(FAMILY, qualifier);
3138     get.setMaxVersions();
3139 
3140     // Check that the column indeed has the right values at timestamps 1 and
3141     // 2
3142     Result result = hTable.get(get);
3143     NavigableMap<Long, byte[]> navigableMap =
3144         result.getMap().get(FAMILY).get(qualifier);
3145     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3146     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3147 
3148     // Update the value at timestamp 1
3149     put = new Put(row);
3150     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3151     hTable.put(put);
3152 
3153     // Update the value at timestamp 2
3154     put = new Put(row);
3155     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3156     hTable.put(put);
3157 
3158     // Check that the values at timestamp 2 and 1 got updated
3159     result = hTable.get(get);
3160     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3161     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3162     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3163   }
3164 
3165   @Test
3166   public void testUpdatesWithMajorCompaction() throws Exception {
3167 
3168     String tableName = "testUpdatesWithMajorCompaction";
3169     byte [] TABLE = Bytes.toBytes(tableName);
3170     HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3171     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
3172 
3173     // Write a column with values at timestamp 1, 2 and 3
3174     byte[] row = Bytes.toBytes("row2");
3175     byte[] qualifier = Bytes.toBytes("myCol");
3176     Put put = new Put(row);
3177     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3178     hTable.put(put);
3179 
3180     put = new Put(row);
3181     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3182     hTable.put(put);
3183 
3184     put = new Put(row);
3185     put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3186     hTable.put(put);
3187 
3188     Get get = new Get(row);
3189     get.addColumn(FAMILY, qualifier);
3190     get.setMaxVersions();
3191 
3192     // Check that the column indeed has the right values at timestamps 1 and
3193     // 2
3194     Result result = hTable.get(get);
3195     NavigableMap<Long, byte[]> navigableMap =
3196         result.getMap().get(FAMILY).get(qualifier);
3197     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3198     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3199 
3200     // Trigger a major compaction
3201     admin.flush(tableName);
3202     admin.majorCompact(tableName);
3203     Thread.sleep(6000);
3204 
3205     // Update the value at timestamp 1
3206     put = new Put(row);
3207     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3208     hTable.put(put);
3209 
3210     // Update the value at timestamp 2
3211     put = new Put(row);
3212     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3213     hTable.put(put);
3214 
3215     // Trigger a major compaction
3216     admin.flush(tableName);
3217     admin.majorCompact(tableName);
3218     Thread.sleep(6000);
3219 
3220     // Check that the values at timestamp 2 and 1 got updated
3221     result = hTable.get(get);
3222     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3223     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3224     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3225   }
3226 
3227   @Test
3228   public void testMajorCompactionBetweenTwoUpdates() throws Exception {
3229 
3230     String tableName = "testMajorCompactionBetweenTwoUpdates";
3231     byte [] TABLE = Bytes.toBytes(tableName);
3232     HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3233     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
3234 
3235     // Write a column with values at timestamp 1, 2 and 3
3236     byte[] row = Bytes.toBytes("row3");
3237     byte[] qualifier = Bytes.toBytes("myCol");
3238     Put put = new Put(row);
3239     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3240     hTable.put(put);
3241 
3242     put = new Put(row);
3243     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3244     hTable.put(put);
3245 
3246     put = new Put(row);
3247     put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3248     hTable.put(put);
3249 
3250     Get get = new Get(row);
3251     get.addColumn(FAMILY, qualifier);
3252     get.setMaxVersions();
3253 
3254     // Check that the column indeed has the right values at timestamps 1 and
3255     // 2
3256     Result result = hTable.get(get);
3257     NavigableMap<Long, byte[]> navigableMap =
3258         result.getMap().get(FAMILY).get(qualifier);
3259     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3260     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3261 
3262     // Trigger a major compaction
3263     admin.flush(tableName);
3264     admin.majorCompact(tableName);
3265     Thread.sleep(6000);
3266 
3267     // Update the value at timestamp 1
3268     put = new Put(row);
3269     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3270     hTable.put(put);
3271 
3272     // Trigger a major compaction
3273     admin.flush(tableName);
3274     admin.majorCompact(tableName);
3275     Thread.sleep(6000);
3276 
3277     // Update the value at timestamp 2
3278     put = new Put(row);
3279     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3280     hTable.put(put);
3281 
3282     // Trigger a major compaction
3283     admin.flush(tableName);
3284     admin.majorCompact(tableName);
3285     Thread.sleep(6000);
3286 
3287     // Check that the values at timestamp 2 and 1 got updated
3288     result = hTable.get(get);
3289     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3290 
3291     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3292     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3293   }
3294 
3295   @Test
3296   public void testGet_EmptyTable() throws IOException {
3297     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testGet_EmptyTable"), FAMILY);
3298     Get get = new Get(ROW);
3299     get.addFamily(FAMILY);
3300     Result r = table.get(get);
3301     assertTrue(r.isEmpty());
3302   }
3303 
3304   @Test
3305   public void testGet_NonExistentRow() throws IOException {
3306     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testGet_NonExistentRow"), FAMILY);
3307     Put put = new Put(ROW);
3308     put.add(FAMILY, QUALIFIER, VALUE);
3309     table.put(put);
3310     LOG.info("Row put");
3311 
3312     Get get = new Get(ROW);
3313     get.addFamily(FAMILY);
3314     Result r = table.get(get);
3315     assertFalse(r.isEmpty());
3316     System.out.println("Row retrieved successfully");
3317 
3318     byte [] missingrow = Bytes.toBytes("missingrow");
3319     get = new Get(missingrow);
3320     get.addFamily(FAMILY);
3321     r = table.get(get);
3322     assertTrue(r.isEmpty());
3323     LOG.info("Row missing as it should be");
3324   }
3325 
3326   @Test
3327   public void testPut() throws IOException {
3328     final byte [] CONTENTS_FAMILY = Bytes.toBytes("contents");
3329     final byte [] SMALL_FAMILY = Bytes.toBytes("smallfam");
3330     final byte [] row1 = Bytes.toBytes("row1");
3331     final byte [] row2 = Bytes.toBytes("row2");
3332     final byte [] value = Bytes.toBytes("abcd");
3333     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testPut"),
3334       new byte [][] {CONTENTS_FAMILY, SMALL_FAMILY});
3335     Put put = new Put(row1);
3336     put.add(CONTENTS_FAMILY, null, value);
3337     table.put(put);
3338 
3339     put = new Put(row2);
3340     put.add(CONTENTS_FAMILY, null, value);
3341 
3342     assertEquals(put.size(), 1);
3343     assertEquals(put.getFamilyMap().get(CONTENTS_FAMILY).size(), 1);
3344 
3345     KeyValue kv = put.getFamilyMap().get(CONTENTS_FAMILY).get(0);
3346 
3347     assertTrue(Bytes.equals(kv.getFamily(), CONTENTS_FAMILY));
3348     // will it return null or an empty byte array?
3349     assertTrue(Bytes.equals(kv.getQualifier(), new byte[0]));
3350 
3351     assertTrue(Bytes.equals(kv.getValue(), value));
3352 
3353     table.put(put);
3354 
3355     Scan scan = new Scan();
3356     scan.addColumn(CONTENTS_FAMILY, null);
3357     ResultScanner scanner = table.getScanner(scan);
3358     for (Result r : scanner) {
3359       for(KeyValue key : r.sorted()) {
3360         System.out.println(Bytes.toString(r.getRow()) + ": " + key.toString());
3361       }
3362     }
3363   }
3364 
3365   @Test
3366   public void testRowsPut() throws IOException {
3367     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
3368     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
3369     final int NB_BATCH_ROWS = 10;
3370     final byte[] value = Bytes.toBytes("abcd");
3371     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPut"),
3372       new byte[][] {CONTENTS_FAMILY, SMALL_FAMILY });
3373     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
3374     for (int i = 0; i < NB_BATCH_ROWS; i++) {
3375       byte[] row = Bytes.toBytes("row" + i);
3376       Put put = new Put(row);
3377       put.add(CONTENTS_FAMILY, null, value);
3378       rowsUpdate.add(put);
3379     }
3380     table.put(rowsUpdate);
3381     Scan scan = new Scan();
3382     scan.addFamily(CONTENTS_FAMILY);
3383     ResultScanner scanner = table.getScanner(scan);
3384     int nbRows = 0;
3385     for (@SuppressWarnings("unused")
3386     Result row : scanner)
3387       nbRows++;
3388     assertEquals(NB_BATCH_ROWS, nbRows);
3389   }
3390 
3391   @Test
3392   public void testRowsPutBufferedOneFlush() throws IOException {
3393     final byte [] CONTENTS_FAMILY = Bytes.toBytes("contents");
3394     final byte [] SMALL_FAMILY = Bytes.toBytes("smallfam");
3395     final byte [] value = Bytes.toBytes("abcd");
3396     final int NB_BATCH_ROWS = 10;
3397     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPutBufferedOneFlush"),
3398       new byte [][] {CONTENTS_FAMILY, SMALL_FAMILY});
3399     table.setAutoFlush(false);
3400     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
3401     for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
3402       byte[] row = Bytes.toBytes("row" + i);
3403       Put put = new Put(row);
3404       put.add(CONTENTS_FAMILY, null, value);
3405       rowsUpdate.add(put);
3406     }
3407     table.put(rowsUpdate);
3408 
3409     Scan scan = new Scan();
3410     scan.addFamily(CONTENTS_FAMILY);
3411     ResultScanner scanner = table.getScanner(scan);
3412     int nbRows = 0;
3413     for (@SuppressWarnings("unused")
3414     Result row : scanner)
3415       nbRows++;
3416     assertEquals(0, nbRows);
3417     scanner.close();
3418 
3419     table.flushCommits();
3420 
3421     scan = new Scan();
3422     scan.addFamily(CONTENTS_FAMILY);
3423     scanner = table.getScanner(scan);
3424     nbRows = 0;
3425     for (@SuppressWarnings("unused")
3426     Result row : scanner)
3427       nbRows++;
3428     assertEquals(NB_BATCH_ROWS * 10, nbRows);
3429   }
3430 
3431   @Test
3432   public void testRowsPutBufferedManyManyFlushes() throws IOException {
3433     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
3434     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
3435     final byte[] value = Bytes.toBytes("abcd");
3436     final int NB_BATCH_ROWS = 10;
3437     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPutBufferedManyManyFlushes"),
3438       new byte[][] {CONTENTS_FAMILY, SMALL_FAMILY });
3439     table.setAutoFlush(false);
3440     table.setWriteBufferSize(10);
3441     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
3442     for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
3443       byte[] row = Bytes.toBytes("row" + i);
3444       Put put = new Put(row);
3445       put.add(CONTENTS_FAMILY, null, value);
3446       rowsUpdate.add(put);
3447     }
3448     table.put(rowsUpdate);
3449 
3450     table.flushCommits();
3451 
3452     Scan scan = new Scan();
3453     scan.addFamily(CONTENTS_FAMILY);
3454     ResultScanner scanner = table.getScanner(scan);
3455     int nbRows = 0;
3456     for (@SuppressWarnings("unused")
3457     Result row : scanner)
3458       nbRows++;
3459     assertEquals(NB_BATCH_ROWS * 10, nbRows);
3460   }
3461 
3462   @Test
3463   public void testAddKeyValue() throws IOException {
3464     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
3465     final byte[] value = Bytes.toBytes("abcd");
3466     final byte[] row1 = Bytes.toBytes("row1");
3467     final byte[] row2 = Bytes.toBytes("row2");
3468     byte[] qualifier = Bytes.toBytes("qf1");
3469     Put put = new Put(row1);
3470 
3471     // Adding KeyValue with the same row
3472     KeyValue kv = new KeyValue(row1, CONTENTS_FAMILY, qualifier, value);
3473     boolean ok = true;
3474     try {
3475       put.add(kv);
3476     } catch (IOException e) {
3477       ok = false;
3478     }
3479     assertEquals(true, ok);
3480 
3481     // Adding KeyValue with the different row
3482     kv = new KeyValue(row2, CONTENTS_FAMILY, qualifier, value);
3483     ok = false;
3484     try {
3485       put.add(kv);
3486     } catch (IOException e) {
3487       ok = true;
3488     }
3489     assertEquals(true, ok);
3490   }
3491 
3492   /**
3493    * test for HBASE-737
3494    * @throws IOException
3495    */
3496   @Test
3497   public void testHBase737 () throws IOException {
3498     final byte [] FAM1 = Bytes.toBytes("fam1");
3499     final byte [] FAM2 = Bytes.toBytes("fam2");
3500     // Open table
3501     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testHBase737"),
3502       new byte [][] {FAM1, FAM2});
3503     // Insert some values
3504     Put put = new Put(ROW);
3505     put.add(FAM1, Bytes.toBytes("letters"), Bytes.toBytes("abcdefg"));
3506     table.put(put);
3507     try {
3508       Thread.sleep(1000);
3509     } catch (InterruptedException i) {
3510       //ignore
3511     }
3512 
3513     put = new Put(ROW);
3514     put.add(FAM1, Bytes.toBytes("numbers"), Bytes.toBytes("123456"));
3515     table.put(put);
3516 
3517     try {
3518       Thread.sleep(1000);
3519     } catch (InterruptedException i) {
3520       //ignore
3521     }
3522 
3523     put = new Put(ROW);
3524     put.add(FAM2, Bytes.toBytes("letters"), Bytes.toBytes("hijklmnop"));
3525     table.put(put);
3526 
3527     long times[] = new long[3];
3528 
3529     // First scan the memstore
3530 
3531     Scan scan = new Scan();
3532     scan.addFamily(FAM1);
3533     scan.addFamily(FAM2);
3534     ResultScanner s = table.getScanner(scan);
3535     try {
3536       int index = 0;
3537       Result r = null;
3538       while ((r = s.next()) != null) {
3539         for(KeyValue key : r.sorted()) {
3540           times[index++] = key.getTimestamp();
3541         }
3542       }
3543     } finally {
3544       s.close();
3545     }
3546     for (int i = 0; i < times.length - 1; i++) {
3547       for (int j = i + 1; j < times.length; j++) {
3548         assertTrue(times[j] > times[i]);
3549       }
3550     }
3551 
3552     // Flush data to disk and try again
3553     TEST_UTIL.flush();
3554 
3555     // Reset times
3556     for(int i=0;i<times.length;i++) {
3557       times[i] = 0;
3558     }
3559 
3560     try {
3561       Thread.sleep(1000);
3562     } catch (InterruptedException i) {
3563       //ignore
3564     }
3565     scan = new Scan();
3566     scan.addFamily(FAM1);
3567     scan.addFamily(FAM2);
3568     s = table.getScanner(scan);
3569     try {
3570       int index = 0;
3571       Result r = null;
3572       while ((r = s.next()) != null) {
3573         for(KeyValue key : r.sorted()) {
3574           times[index++] = key.getTimestamp();
3575         }
3576       }
3577     } finally {
3578       s.close();
3579     }
3580     for (int i = 0; i < times.length - 1; i++) {
3581       for (int j = i + 1; j < times.length; j++) {
3582         assertTrue(times[j] > times[i]);
3583       }
3584     }
3585   }
3586 
3587   @Test
3588   public void testListTables() throws IOException, InterruptedException {
3589     byte [] t1 = Bytes.toBytes("testListTables1");
3590     byte [] t2 = Bytes.toBytes("testListTables2");
3591     byte [] t3 = Bytes.toBytes("testListTables3");
3592     byte [][] tables = new byte[][] { t1, t2, t3 };
3593     for (int i = 0; i < tables.length; i++) {
3594       TEST_UTIL.createTable(tables[i], FAMILY);
3595     }
3596     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
3597     HTableDescriptor[] ts = admin.listTables();
3598     HashSet<HTableDescriptor> result = new HashSet<HTableDescriptor>(ts.length);
3599     for (int i = 0; i < ts.length; i++) {
3600       result.add(ts[i]);
3601     }
3602     int size = result.size();
3603     assertTrue(size >= tables.length);
3604     for (int i = 0; i < tables.length && i < size; i++) {
3605       boolean found = false;
3606       for (int j = 0; j < ts.length; j++) {
3607         if (Bytes.equals(ts[j].getName(), tables[i])) {
3608           found = true;
3609           break;
3610         }
3611       }
3612       assertTrue("Not found: " + Bytes.toString(tables[i]), found);
3613     }
3614   }
3615 
3616   @Test
3617   public void testMiscHTableStuff() throws IOException {
3618     final byte[] tableAname = Bytes.toBytes("testMiscHTableStuffA");
3619     final byte[] tableBname = Bytes.toBytes("testMiscHTableStuffB");
3620     final byte[] attrName = Bytes.toBytes("TESTATTR");
3621     final byte[] attrValue = Bytes.toBytes("somevalue");
3622     byte[] value = Bytes.toBytes("value");
3623 
3624     HTable a = TEST_UTIL.createTable(tableAname, HConstants.CATALOG_FAMILY);
3625     HTable b = TEST_UTIL.createTable(tableBname, HConstants.CATALOG_FAMILY);
3626     Put put = new Put(ROW);
3627     put.add(HConstants.CATALOG_FAMILY, null, value);
3628     a.put(put);
3629 
3630     // open a new connection to A and a connection to b
3631     HTable newA = new HTable(TEST_UTIL.getConfiguration(), tableAname);
3632 
3633     // copy data from A to B
3634     Scan scan = new Scan();
3635     scan.addFamily(HConstants.CATALOG_FAMILY);
3636     ResultScanner s = newA.getScanner(scan);
3637     try {
3638       for (Result r : s) {
3639         put = new Put(r.getRow());
3640         for (KeyValue kv : r.sorted()) {
3641           put.add(kv);
3642         }
3643         b.put(put);
3644       }
3645     } finally {
3646       s.close();
3647     }
3648 
3649     // Opening a new connection to A will cause the tables to be reloaded
3650     HTable anotherA = new HTable(TEST_UTIL.getConfiguration(), tableAname);
3651     Get get = new Get(ROW);
3652     get.addFamily(HConstants.CATALOG_FAMILY);
3653     anotherA.get(get);
3654 
3655     // We can still access A through newA because it has the table information
3656     // cached. And if it needs to recalibrate, that will cause the information
3657     // to be reloaded.
3658 
3659     // Test user metadata
3660     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
3661     // make a modifiable descriptor
3662     HTableDescriptor desc = new HTableDescriptor(a.getTableDescriptor());
3663     // offline the table
3664     admin.disableTable(tableAname);
3665     // add a user attribute to HTD
3666     desc.setValue(attrName, attrValue);
3667     // add a user attribute to HCD
3668     for (HColumnDescriptor c : desc.getFamilies())
3669       c.setValue(attrName, attrValue);
3670     // update metadata for all regions of this table
3671     admin.modifyTable(tableAname, desc);
3672     // enable the table
3673     admin.enableTable(tableAname);
3674 
3675     // Test that attribute changes were applied
3676     desc = a.getTableDescriptor();
3677     assertTrue("wrong table descriptor returned",
3678       Bytes.compareTo(desc.getName(), tableAname) == 0);
3679     // check HTD attribute
3680     value = desc.getValue(attrName);
3681     assertFalse("missing HTD attribute value", value == null);
3682     assertFalse("HTD attribute value is incorrect",
3683       Bytes.compareTo(value, attrValue) != 0);
3684     // check HCD attribute
3685     for (HColumnDescriptor c : desc.getFamilies()) {
3686       value = c.getValue(attrName);
3687       assertFalse("missing HCD attribute value", value == null);
3688       assertFalse("HCD attribute value is incorrect",
3689         Bytes.compareTo(value, attrValue) != 0);
3690     }
3691   }
3692 
3693   @Test
3694   public void testGetClosestRowBefore() throws IOException {
3695     final byte [] tableAname = Bytes.toBytes("testGetClosestRowBefore");
3696     final byte [] row = Bytes.toBytes("row");
3697 
3698 
3699     byte[] firstRow = Bytes.toBytes("ro");
3700     byte[] beforeFirstRow = Bytes.toBytes("rn");
3701     byte[] beforeSecondRow = Bytes.toBytes("rov");
3702 
3703     HTable table = TEST_UTIL.createTable(tableAname,
3704       new byte [][] {HConstants.CATALOG_FAMILY, Bytes.toBytes("info2")});
3705     Put put = new Put(firstRow);
3706     Put put2 = new Put(row);
3707     byte[] zero = new byte[]{0};
3708     byte[] one = new byte[]{1};
3709 
3710     put.add(HConstants.CATALOG_FAMILY, null, zero);
3711     put2.add(HConstants.CATALOG_FAMILY, null, one);
3712 
3713     table.put(put);
3714     table.put(put2);
3715 
3716     Result result = null;
3717 
3718     // Test before first that null is returned
3719     result = table.getRowOrBefore(beforeFirstRow, HConstants.CATALOG_FAMILY);
3720     assertTrue(result == null);
3721 
3722     // Test at first that first is returned
3723     result = table.getRowOrBefore(firstRow, HConstants.CATALOG_FAMILY);
3724     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
3725     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), zero));
3726 
3727     // Test in between first and second that first is returned
3728     result = table.getRowOrBefore(beforeSecondRow, HConstants.CATALOG_FAMILY);
3729     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
3730     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), zero));
3731 
3732     // Test at second make sure second is returned
3733     result = table.getRowOrBefore(row, HConstants.CATALOG_FAMILY);
3734     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
3735     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
3736 
3737     // Test after second, make sure second is returned
3738     result = table.getRowOrBefore(Bytes.add(row,one), HConstants.CATALOG_FAMILY);
3739     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
3740     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
3741   }
3742 
3743   /**
3744    * For HBASE-2156
3745    * @throws Exception
3746    */
3747   @Test
3748   public void testScanVariableReuse() throws Exception {
3749     Scan scan = new Scan();
3750     scan.addFamily(FAMILY);
3751     scan.addColumn(FAMILY, ROW);
3752 
3753     assertTrue(scan.getFamilyMap().get(FAMILY).size() == 1);
3754 
3755     scan = new Scan();
3756     scan.addFamily(FAMILY);
3757 
3758     assertTrue(scan.getFamilyMap().get(FAMILY) == null);
3759     assertTrue(scan.getFamilyMap().containsKey(FAMILY));
3760   }
3761 
3762   /**
3763    * HBASE-2468 use case 1 and 2: region info de/serialization
3764    */
3765    @Test
3766    public void testRegionCacheDeSerialization() throws Exception {
3767      // 1. test serialization.
3768      LOG.info("Starting testRegionCacheDeSerialization");
3769      final byte[] TABLENAME = Bytes.toBytes("testCachePrewarm2");
3770      final byte[] FAMILY = Bytes.toBytes("family");
3771      Configuration conf = TEST_UTIL.getConfiguration();
3772      TEST_UTIL.createTable(TABLENAME, FAMILY);
3773 
3774      // Set up test table:
3775      // Create table:
3776      HTable table = new HTable(conf, TABLENAME);
3777 
3778      // Create multiple regions for this table
3779      TEST_UTIL.createMultiRegions(table, FAMILY);
3780 
3781      Path tempPath = new Path(HBaseTestingUtility.getTestDir(), "regions.dat");
3782 
3783      final String tempFileName = tempPath.toString();
3784 
3785      FileOutputStream fos = new FileOutputStream(tempFileName);
3786      DataOutputStream dos = new DataOutputStream(fos);
3787 
3788      // serialize the region info and output to a local file.
3789      table.serializeRegionInfo(dos);
3790      dos.flush();
3791      dos.close();
3792 
3793      // read a local file and deserialize the region info from it.
3794      FileInputStream fis = new FileInputStream(tempFileName);
3795      DataInputStream dis = new DataInputStream(fis);
3796 
3797      Map<HRegionInfo, HServerAddress> deserRegions =
3798        table.deserializeRegionInfo(dis);
3799      dis.close();
3800 
3801      // regions obtained from meta scanner.
3802      Map<HRegionInfo, HServerAddress> loadedRegions =
3803        table.getRegionsInfo();
3804 
3805      // set the deserialized regions to the global cache.
3806      table.getConnection().clearRegionCache();
3807 
3808      table.getConnection().prewarmRegionCache(table.getTableName(),
3809          deserRegions);
3810 
3811      // verify whether the 2 maps are identical or not.
3812      assertEquals("Number of cached region is incorrect",
3813          HConnectionManager.getCachedRegionCount(conf, TABLENAME),
3814          loadedRegions.size());
3815 
3816      // verify each region is prefetched or not.
3817      for (Map.Entry<HRegionInfo, HServerAddress> e: loadedRegions.entrySet()) {
3818        HRegionInfo hri = e.getKey();
3819        assertTrue(HConnectionManager.isRegionCached(conf,
3820            hri.getTableDesc().getName(), hri.getStartKey()));
3821      }
3822 
3823      // delete the temp file
3824      File f = new java.io.File(tempFileName);
3825      f.delete();
3826      LOG.info("Finishing testRegionCacheDeSerialization");
3827    }
3828 
3829   /**
3830    * HBASE-2468 use case 3:
3831    */
3832   @Test
3833   public void testRegionCachePreWarm() throws Exception {
3834     LOG.info("Starting testRegionCachePreWarm");
3835     final byte [] TABLENAME = Bytes.toBytes("testCachePrewarm");
3836     Configuration conf = TEST_UTIL.getConfiguration();
3837 
3838     // Set up test table:
3839     // Create table:
3840     TEST_UTIL.createTable(TABLENAME, FAMILY);
3841 
3842     // disable region cache for the table.
3843     HTable.setRegionCachePrefetch(conf, TABLENAME, false);
3844     assertFalse("The table is disabled for region cache prefetch",
3845         HTable.getRegionCachePrefetch(conf, TABLENAME));
3846 
3847     HTable table = new HTable(conf, TABLENAME);
3848 
3849     // create many regions for the table.
3850     TEST_UTIL.createMultiRegions(table, FAMILY);
3851     // This count effectively waits until the regions have been
3852     // fully assigned
3853     TEST_UTIL.countRows(table);
3854     table.getConnection().clearRegionCache();
3855     assertEquals("Clearing cache should have 0 cached ", 0,
3856         HConnectionManager.getCachedRegionCount(conf, TABLENAME));
3857 
3858     // A Get is suppose to do a region lookup request
3859     Get g = new Get(Bytes.toBytes("aaa"));
3860     table.get(g);
3861 
3862     // only one region should be cached if the cache prefetch is disabled.
3863     assertEquals("Number of cached region is incorrect ", 1,
3864         HConnectionManager.getCachedRegionCount(conf, TABLENAME));
3865 
3866     // now we enable cached prefetch.
3867     HTable.setRegionCachePrefetch(conf, TABLENAME, true);
3868     assertTrue("The table is enabled for region cache prefetch",
3869         HTable.getRegionCachePrefetch(conf, TABLENAME));
3870 
3871     HTable.setRegionCachePrefetch(conf, TABLENAME, false);
3872     assertFalse("The table is disabled for region cache prefetch",
3873         HTable.getRegionCachePrefetch(conf, TABLENAME));
3874 
3875     HTable.setRegionCachePrefetch(conf, TABLENAME, true);
3876     assertTrue("The table is enabled for region cache prefetch",
3877         HTable.getRegionCachePrefetch(conf, TABLENAME));
3878 
3879     table.getConnection().clearRegionCache();
3880 
3881     assertEquals("Number of cached region is incorrect ", 0,
3882         HConnectionManager.getCachedRegionCount(conf, TABLENAME));
3883 
3884     // if there is a cache miss, some additional regions should be prefetched.
3885     Get g2 = new Get(Bytes.toBytes("bbb"));
3886     table.get(g2);
3887 
3888     // Get the configured number of cache read-ahead regions.
3889     int prefetchRegionNumber = conf.getInt("hbase.client.prefetch.limit", 10);
3890 
3891     // the total number of cached regions == region('aaa") + prefeched regions.
3892     LOG.info("Testing how many regions cached");
3893     assertEquals("Number of cached region is incorrect ", prefetchRegionNumber,
3894         HConnectionManager.getCachedRegionCount(conf, TABLENAME));
3895 
3896     table.getConnection().clearRegionCache();
3897 
3898     Get g3 = new Get(Bytes.toBytes("abc"));
3899     table.get(g3);
3900     assertEquals("Number of cached region is incorrect ", prefetchRegionNumber,
3901         HConnectionManager.getCachedRegionCount(conf, TABLENAME));
3902 
3903     LOG.info("Finishing testRegionCachePreWarm");
3904   }
3905 
3906   @Test
3907   public void testIncrement() throws Exception {
3908     LOG.info("Starting testIncrement");
3909     final byte [] TABLENAME = Bytes.toBytes("testIncrement");
3910     HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
3911 
3912     byte [][] ROWS = new byte [][] {
3913         Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"),
3914         Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
3915         Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i")
3916     };
3917     byte [][] QUALIFIERS = new byte [][] {
3918         Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"),
3919         Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
3920         Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i")
3921     };
3922 
3923     // Do some simple single-column increments
3924 
3925     // First with old API
3926     ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[0], 1);
3927     ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[1], 2);
3928     ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[2], 3);
3929     ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[3], 4);
3930 
3931     // Now increment things incremented with old and do some new
3932     Increment inc = new Increment(ROW);
3933     inc.addColumn(FAMILY, QUALIFIERS[1], 1);
3934     inc.addColumn(FAMILY, QUALIFIERS[3], 1);
3935     inc.addColumn(FAMILY, QUALIFIERS[4], 1);
3936     ht.increment(inc);
3937 
3938     // Verify expected results
3939     Result r = ht.get(new Get(ROW));
3940     KeyValue [] kvs = r.raw();
3941     assertEquals(5, kvs.length);
3942     assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 1);
3943     assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 3);
3944     assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 3);
3945     assertIncrementKey(kvs[3], ROW, FAMILY, QUALIFIERS[3], 5);
3946     assertIncrementKey(kvs[4], ROW, FAMILY, QUALIFIERS[4], 1);
3947 
3948     // Now try multiple columns by different amounts
3949     inc = new Increment(ROWS[0]);
3950     for (int i=0;i<QUALIFIERS.length;i++) {
3951       inc.addColumn(FAMILY, QUALIFIERS[i], i+1);
3952     }
3953     ht.increment(inc);
3954     // Verify
3955     r = ht.get(new Get(ROWS[0]));
3956     kvs = r.raw();
3957     assertEquals(QUALIFIERS.length, kvs.length);
3958     for (int i=0;i<QUALIFIERS.length;i++) {
3959       assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], i+1);
3960     }
3961 
3962     // Re-increment them
3963     inc = new Increment(ROWS[0]);
3964     for (int i=0;i<QUALIFIERS.length;i++) {
3965       inc.addColumn(FAMILY, QUALIFIERS[i], i+1);
3966     }
3967     ht.increment(inc);
3968     // Verify
3969     r = ht.get(new Get(ROWS[0]));
3970     kvs = r.raw();
3971     assertEquals(QUALIFIERS.length, kvs.length);
3972     for (int i=0;i<QUALIFIERS.length;i++) {
3973       assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], 2*(i+1));
3974     }
3975   }
3976 
3977   /**
3978    * This test demonstrates how we use ThreadPoolExecutor.
3979    * It needs to show that we only use as many threads in the pool as we have
3980    * region servers. To do this, instead of doing real requests, we use a
3981    * SynchronousQueue where each put must wait for a take (and vice versa)
3982    * so that way we have full control of the number of active threads.
3983    * @throws IOException
3984    * @throws InterruptedException
3985    */
3986   @Test
3987   public void testPoolBehavior() throws IOException, InterruptedException {
3988     byte[] someBytes = Bytes.toBytes("pool");
3989     HTable table = TEST_UTIL.createTable(someBytes, someBytes);
3990     ThreadPoolExecutor pool = (ThreadPoolExecutor)table.getPool();
3991 
3992     // Make sure that the TPE stars with a core pool size of one and 0
3993     // initialized worker threads
3994     assertEquals(1, pool.getCorePoolSize());
3995     assertEquals(0, pool.getPoolSize());
3996 
3997     // Build a SynchronousQueue that we use for thread coordination
3998     final SynchronousQueue<Object> queue = new SynchronousQueue<Object>();
3999     List<Thread> threads = new ArrayList<Thread>(5);
4000     for (int i = 0; i < 5; i++) {
4001       threads.add(new Thread() {
4002         public void run() {
4003           try {
4004             // The thread blocks here until we decide to let it go
4005             queue.take();
4006           } catch (InterruptedException ie) { }
4007         }
4008       });
4009     }
4010     // First, add two threads and make sure the pool size follows
4011     pool.submit(threads.get(0));
4012     assertEquals(1, pool.getPoolSize());
4013     pool.submit(threads.get(1));
4014     assertEquals(2, pool.getPoolSize());
4015 
4016     // Next, terminate those threads and then make sure the pool is still the
4017     // same size
4018     queue.put(new Object());
4019     threads.get(0).join();
4020     queue.put(new Object());
4021     threads.get(1).join();
4022     assertEquals(2, pool.getPoolSize());
4023 
4024     // Now let's simulate adding a RS meaning that we'll go up to three
4025     // concurrent threads. The pool should not grow larger than three.
4026     pool.submit(threads.get(2));
4027     pool.submit(threads.get(3));
4028     pool.submit(threads.get(4));
4029     assertEquals(3, pool.getPoolSize());
4030     queue.put(new Object());
4031     queue.put(new Object());
4032     queue.put(new Object());
4033   }
4034 
4035 
4036 }
4037