1   /**
2    * Copyright 2009 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import static org.junit.Assert.assertArrayEquals;
23  import static org.junit.Assert.assertEquals;
24  import static org.junit.Assert.assertFalse;
25  import static org.junit.Assert.assertNotNull;
26  import static org.junit.Assert.assertNull;
27  import static org.junit.Assert.assertSame;
28  import static org.junit.Assert.assertTrue;
29  import static org.junit.Assert.fail;
30  
31  import java.io.DataInputStream;
32  import java.io.DataOutputStream;
33  import java.io.File;
34  import java.io.FileInputStream;
35  import java.io.FileOutputStream;
36  import java.io.IOException;
37  import java.util.ArrayList;
38  import java.util.Arrays;
39  import java.util.HashSet;
40  import java.util.Iterator;
41  import java.util.List;
42  import java.util.Map;
43  import java.util.NavigableMap;
44  import java.util.UUID;
45  import java.util.concurrent.Callable;
46  import java.util.concurrent.ExecutorService;
47  import java.util.concurrent.Executors;
48  import java.util.concurrent.SynchronousQueue;
49  import java.util.concurrent.ThreadPoolExecutor;
50  import java.util.concurrent.atomic.AtomicReference;
51  
52  import org.apache.commons.lang.ArrayUtils;
53  import org.apache.commons.logging.Log;
54  import org.apache.commons.logging.LogFactory;
55  import org.apache.hadoop.conf.Configuration;
56  import org.apache.hadoop.fs.Path;
57  import org.apache.hadoop.hbase.DoNotRetryIOException;
58  import org.apache.hadoop.hbase.HBaseTestingUtility;
59  import org.apache.hadoop.hbase.HColumnDescriptor;
60  import org.apache.hadoop.hbase.HConstants;
61  import org.apache.hadoop.hbase.HRegionInfo;
62  import org.apache.hadoop.hbase.HRegionLocation;
63  import org.apache.hadoop.hbase.HServerAddress;
64  import org.apache.hadoop.hbase.HTableDescriptor;
65  import org.apache.hadoop.hbase.KeyValue;
66  import org.apache.hadoop.hbase.LargeTests;
67  import org.apache.hadoop.hbase.MiniHBaseCluster;
68  import org.apache.hadoop.hbase.ServerName;
69  import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
70  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
71  import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
72  import org.apache.hadoop.hbase.coprocessor.MultiRowMutationProtocol;
73  import org.apache.hadoop.hbase.filter.BinaryComparator;
74  import org.apache.hadoop.hbase.filter.CompareFilter;
75  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
76  import org.apache.hadoop.hbase.filter.Filter;
77  import org.apache.hadoop.hbase.filter.FilterList;
78  import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
79  import org.apache.hadoop.hbase.filter.PrefixFilter;
80  import org.apache.hadoop.hbase.filter.QualifierFilter;
81  import org.apache.hadoop.hbase.filter.RegexStringComparator;
82  import org.apache.hadoop.hbase.filter.RowFilter;
83  import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
84  import org.apache.hadoop.hbase.filter.WhileMatchFilter;
85  import org.apache.hadoop.hbase.io.hfile.BlockCache;
86  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
87  import org.apache.hadoop.hbase.ipc.SecureRpcEngine;
88  import org.apache.hadoop.hbase.regionserver.HRegion;
89  import org.apache.hadoop.hbase.regionserver.HRegionServer;
90  import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
91  import org.apache.hadoop.hbase.regionserver.Store;
92  import org.apache.hadoop.hbase.util.Bytes;
93  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
94  import org.apache.hadoop.hbase.util.Threads;
95  import org.apache.hadoop.io.DataInputBuffer;
96  import org.junit.After;
97  import org.junit.AfterClass;
98  import org.junit.Before;
99  import org.junit.BeforeClass;
100 import org.junit.Ignore;
101 import org.junit.Test;
102 import org.junit.experimental.categories.Category;
103 
104 /**
105  * Run tests that use the HBase clients; {@link HTable} and {@link HTablePool}.
106  * Sets up the HBase mini cluster once at start and runs through all client tests.
107  * Each creates a table named for the method and does its stuff against that.
108  */
109 @Category(LargeTests.class)
110 @SuppressWarnings ("deprecation")
111 public class TestFromClientSide {
112   final Log LOG = LogFactory.getLog(getClass());
113   protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
114   private static byte [] ROW = Bytes.toBytes("testRow");
115   private static byte [] FAMILY = Bytes.toBytes("testFamily");
116   private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
117   private static byte [] VALUE = Bytes.toBytes("testValue");
118   protected static int SLAVES = 3;
119 
120   /**
121    * @throws java.lang.Exception
122    */
123   @BeforeClass
124   public static void setUpBeforeClass() throws Exception {
125     Configuration conf = TEST_UTIL.getConfiguration();
126     // force the rpc engine to the non-secure one in order to get coverage
127     conf.set("hbase.rpc.engine", "org.apache.hadoop.hbase.ipc.WritableRpcEngine");
128     conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
129         MultiRowMutationEndpoint.class.getName());
130     // We need more than one region server in this test
131     TEST_UTIL.startMiniCluster(SLAVES);
132   }
133 
134   /**
135    * @throws java.lang.Exception
136    */
137   @AfterClass
138   public static void tearDownAfterClass() throws Exception {
139     TEST_UTIL.shutdownMiniCluster();
140   }
141 
142   /**
143    * @throws java.lang.Exception
144    */
145   @Before
146   public void setUp() throws Exception {
147     // Nothing to do.
148   }
149 
150   /**
151    * @throws java.lang.Exception
152    */
153   @After
154   public void tearDown() throws Exception {
155     // Nothing to do.
156   }
157 
158   /**
159    * Basic client side validation of HBASE-4536
160    */
161    @Test
162    public void testKeepDeletedCells() throws Exception {
163      final byte[] TABLENAME = Bytes.toBytes("testKeepDeletesCells");
164      final byte[] FAMILY = Bytes.toBytes("family");
165      final byte[] C0 = Bytes.toBytes("c0");
166 
167      final byte[] T1 = Bytes.toBytes("T1");
168      final byte[] T2 = Bytes.toBytes("T2");
169      final byte[] T3 = Bytes.toBytes("T3");
170      HColumnDescriptor hcd = new HColumnDescriptor(FAMILY)
171          .setKeepDeletedCells(true);
172 
173      HTableDescriptor desc = new HTableDescriptor(TABLENAME);
174      desc.addFamily(hcd);
175      TEST_UTIL.getHBaseAdmin().createTable(desc);
176      Configuration c = TEST_UTIL.getConfiguration();
177      HTable h = new HTable(c, TABLENAME);
178 
179      long ts = System.currentTimeMillis();
180      Put p = new Put(T1, ts);
181      p.add(FAMILY, C0, T1);
182      h.put(p);
183      p = new Put(T1, ts+2);
184      p.add(FAMILY, C0, T2);
185      h.put(p);
186      p = new Put(T1, ts+4);
187      p.add(FAMILY, C0, T3);
188      h.put(p);
189 
190      Delete d = new Delete(T1, ts+3, null);
191      h.delete(d);
192 
193      d = new Delete(T1, ts+3, null);
194      d.deleteColumns(FAMILY, C0, ts+3);
195      h.delete(d);
196 
197      Get g = new Get(T1);
198      // does *not* include the delete
199      g.setTimeRange(0, ts+3);
200      Result r = h.get(g);
201      assertArrayEquals(T2, r.getValue(FAMILY, C0));
202 
203      Scan s = new Scan(T1);
204      s.setTimeRange(0, ts+3);
205      s.setMaxVersions();
206      ResultScanner scanner = h.getScanner(s);
207      KeyValue[] kvs = scanner.next().raw();
208      assertArrayEquals(T2, kvs[0].getValue());
209      assertArrayEquals(T1, kvs[1].getValue());
210      scanner.close();
211 
212      s = new Scan(T1);
213      s.setRaw(true);
214      s.setMaxVersions();
215      scanner = h.getScanner(s);
216      kvs = scanner.next().raw();
217      assertTrue(kvs[0].isDeleteFamily());
218      assertArrayEquals(T3, kvs[1].getValue());
219      assertTrue(kvs[2].isDelete());
220      assertArrayEquals(T2, kvs[3].getValue());
221      assertArrayEquals(T1, kvs[4].getValue());
222      scanner.close();
223      h.close();
224    }
225 
226   /**
227    * Basic client side validation of HBASE-10118
228    */
229   @Test
230   public void testPurgeFutureDeletes() throws Exception {
231     final byte[] TABLENAME = Bytes.toBytes("testPurgeFutureDeletes");
232     final byte[] ROW = Bytes.toBytes("row");
233     final byte[] FAMILY = Bytes.toBytes("family");
234     final byte[] COLUMN = Bytes.toBytes("column");
235     final byte[] VALUE = Bytes.toBytes("value");
236 
237     HTable table = TEST_UTIL.createTable(TABLENAME, FAMILY);
238 
239     // future timestamp
240     long ts = System.currentTimeMillis() * 2;
241     Put put = new Put(ROW, ts);
242     put.add(FAMILY, COLUMN, VALUE);
243     table.put(put);
244 
245     Get get = new Get(ROW);
246     Result result = table.get(get);
247     assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
248 
249     Delete del = new Delete(ROW);
250     del.deleteColumn(FAMILY, COLUMN, ts);
251     table.delete(del);
252 
253     get = new Get(ROW);
254     result = table.get(get);
255     assertNull(result.getValue(FAMILY, COLUMN));
256 
257     // major compaction, purged future deletes
258     TEST_UTIL.getHBaseAdmin().flush(TABLENAME);
259     TEST_UTIL.getHBaseAdmin().majorCompact(TABLENAME);
260 
261     // waiting for the major compaction to complete
262     int sleepTime = 0;
263     while (true) {
264       Thread.sleep(200);
265       sleepTime += 200;
266 
267       Scan s = new Scan();
268       s.setRaw(true);
269       ResultScanner scanner = table.getScanner(s);
270       if (scanner.next() == null) {
271         scanner.close();
272         break;
273       }
274       scanner.close();
275 
276       if (sleepTime > 6000) {
277         throw new IOException("Major compaction time is larger than 6000s");
278       }
279     }
280 
281     put = new Put(ROW, ts);
282     put.add(FAMILY, COLUMN, VALUE);
283     table.put(put);
284 
285     get = new Get(ROW);
286     result = table.get(get);
287     assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
288 
289     table.close();
290   }
291 
292    /**
293    * HBASE-2468 use case 1 and 2: region info de/serialization
294    */
295    @Test
296    public void testRegionCacheDeSerialization() throws Exception {
297      // 1. test serialization.
298      LOG.info("Starting testRegionCacheDeSerialization");
299      final byte[] TABLENAME = Bytes.toBytes("testCachePrewarm2");
300      final byte[] FAMILY = Bytes.toBytes("family");
301      Configuration conf = TEST_UTIL.getConfiguration();
302      TEST_UTIL.createTable(TABLENAME, FAMILY);
303 
304      // Set up test table:
305      // Create table:
306      HTable table = new HTable(conf, TABLENAME);
307 
308      // Create multiple regions for this table
309      TEST_UTIL.createMultiRegions(table, FAMILY);
310      Scan s = new Scan();
311      ResultScanner scanner = table.getScanner(s);
312      while (scanner.next() != null) continue;
313 
314      Path tempPath = new Path(TEST_UTIL.getDataTestDir(), "regions.dat");
315 
316      final String tempFileName = tempPath.toString();
317 
318      FileOutputStream fos = new FileOutputStream(tempFileName);
319      DataOutputStream dos = new DataOutputStream(fos);
320 
321      // serialize the region info and output to a local file.
322      table.serializeRegionInfo(dos);
323      dos.flush();
324      dos.close();
325 
326      // read a local file and deserialize the region info from it.
327      FileInputStream fis = new FileInputStream(tempFileName);
328      DataInputStream dis = new DataInputStream(fis);
329 
330      Map<HRegionInfo, HServerAddress> deserRegions =
331        table.deserializeRegionInfo(dis);
332      dis.close();
333 
334      // regions obtained from meta scanner.
335      Map<HRegionInfo, HServerAddress> loadedRegions =
336        table.getRegionsInfo();
337 
338      // set the deserialized regions to the global cache.
339      table.getConnection().clearRegionCache();
340 
341      table.getConnection().prewarmRegionCache(table.getTableName(),
342          deserRegions);
343 
344      // verify whether the 2 maps are identical or not.
345      assertEquals("Number of cached region is incorrect",
346          HConnectionManager.getCachedRegionCount(conf, TABLENAME),
347          loadedRegions.size());
348 
349      // verify each region is prefetched or not.
350      for (Map.Entry<HRegionInfo, HServerAddress> e: loadedRegions.entrySet()) {
351        HRegionInfo hri = e.getKey();
352        assertTrue(HConnectionManager.isRegionCached(conf,
353            hri.getTableName(), hri.getStartKey()));
354      }
355 
356      // delete the temp file
357      File f = new java.io.File(tempFileName);
358      f.delete();
359      LOG.info("Finishing testRegionCacheDeSerialization");
360    }
361 
362   /**
363    * HBASE-2468 use case 3:
364    */
365   @Test
366   public void testRegionCachePreWarm() throws Exception {
367     LOG.info("Starting testRegionCachePreWarm");
368     final byte [] TABLENAME = Bytes.toBytes("testCachePrewarm");
369     Configuration conf = TEST_UTIL.getConfiguration();
370 
371     // Set up test table:
372     // Create table:
373     TEST_UTIL.createTable(TABLENAME, FAMILY);
374 
375     // disable region cache for the table.
376     HTable.setRegionCachePrefetch(conf, TABLENAME, false);
377     assertFalse("The table is disabled for region cache prefetch",
378         HTable.getRegionCachePrefetch(conf, TABLENAME));
379 
380     HTable table = new HTable(conf, TABLENAME);
381 
382     // create many regions for the table.
383     TEST_UTIL.createMultiRegions(table, FAMILY);
384     // This count effectively waits until the regions have been
385     // fully assigned
386     TEST_UTIL.countRows(table);
387     table.getConnection().clearRegionCache();
388     assertEquals("Clearing cache should have 0 cached ", 0,
389         HConnectionManager.getCachedRegionCount(conf, TABLENAME));
390 
391     // A Get is suppose to do a region lookup request
392     Get g = new Get(Bytes.toBytes("aaa"));
393     table.get(g);
394 
395     // only one region should be cached if the cache prefetch is disabled.
396     assertEquals("Number of cached region is incorrect ", 1,
397         HConnectionManager.getCachedRegionCount(conf, TABLENAME));
398 
399     // now we enable cached prefetch.
400     HTable.setRegionCachePrefetch(conf, TABLENAME, true);
401     assertTrue("The table is enabled for region cache prefetch",
402         HTable.getRegionCachePrefetch(conf, TABLENAME));
403 
404     HTable.setRegionCachePrefetch(conf, TABLENAME, false);
405     assertFalse("The table is disabled for region cache prefetch",
406         HTable.getRegionCachePrefetch(conf, TABLENAME));
407 
408     HTable.setRegionCachePrefetch(conf, TABLENAME, true);
409     assertTrue("The table is enabled for region cache prefetch",
410         HTable.getRegionCachePrefetch(conf, TABLENAME));
411 
412     table.getConnection().clearRegionCache();
413 
414     assertEquals("Number of cached region is incorrect ", 0,
415         HConnectionManager.getCachedRegionCount(conf, TABLENAME));
416 
417     // if there is a cache miss, some additional regions should be prefetched.
418     Get g2 = new Get(Bytes.toBytes("bbb"));
419     table.get(g2);
420 
421     // Get the configured number of cache read-ahead regions.
422     int prefetchRegionNumber = conf.getInt("hbase.client.prefetch.limit", 10);
423 
424     // the total number of cached regions == region('aaa") + prefeched regions.
425     LOG.info("Testing how many regions cached");
426     assertEquals("Number of cached region is incorrect ", prefetchRegionNumber,
427         HConnectionManager.getCachedRegionCount(conf, TABLENAME));
428 
429     table.getConnection().clearRegionCache();
430 
431     Get g3 = new Get(Bytes.toBytes("abc"));
432     table.get(g3);
433     assertEquals("Number of cached region is incorrect ", prefetchRegionNumber,
434         HConnectionManager.getCachedRegionCount(conf, TABLENAME));
435 
436     LOG.info("Finishing testRegionCachePreWarm");
437   }
438 
439 
440   /**
441    * Verifies that getConfiguration returns the same Configuration object used
442    * to create the HTable instance.
443    */
444   @Test
445   public void testGetConfiguration() throws Exception {
446     byte[] TABLE = Bytes.toBytes("testGetConfiguration");
447     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") };
448     Configuration conf = TEST_UTIL.getConfiguration();
449     HTable table = TEST_UTIL.createTable(TABLE, FAMILIES, conf);
450     assertSame(conf, table.getConfiguration());
451   }
452 
453   /**
454    * Test from client side of an involved filter against a multi family that
455    * involves deletes.
456    *
457    * @throws Exception
458    */
459   @Test
460   public void testWeirdCacheBehaviour() throws Exception {
461     byte [] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
462     byte [][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
463         Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
464         Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
465     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
466     String value = "this is the value";
467     String value2 = "this is some other value";
468     String keyPrefix1 = UUID.randomUUID().toString();
469     String keyPrefix2 = UUID.randomUUID().toString();
470     String keyPrefix3 = UUID.randomUUID().toString();
471     putRows(ht, 3, value, keyPrefix1);
472     putRows(ht, 3, value, keyPrefix2);
473     putRows(ht, 3, value, keyPrefix3);
474     ht.flushCommits();
475     putRows(ht, 3, value2, keyPrefix1);
476     putRows(ht, 3, value2, keyPrefix2);
477     putRows(ht, 3, value2, keyPrefix3);
478     HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLE);
479     System.out.println("Checking values for key: " + keyPrefix1);
480     assertEquals("Got back incorrect number of rows from scan", 3,
481         getNumberOfRows(keyPrefix1, value2, table));
482     System.out.println("Checking values for key: " + keyPrefix2);
483     assertEquals("Got back incorrect number of rows from scan", 3,
484         getNumberOfRows(keyPrefix2, value2, table));
485     System.out.println("Checking values for key: " + keyPrefix3);
486     assertEquals("Got back incorrect number of rows from scan", 3,
487         getNumberOfRows(keyPrefix3, value2, table));
488     deleteColumns(ht, value2, keyPrefix1);
489     deleteColumns(ht, value2, keyPrefix2);
490     deleteColumns(ht, value2, keyPrefix3);
491     System.out.println("Starting important checks.....");
492     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,
493       0, getNumberOfRows(keyPrefix1, value2, table));
494     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,
495       0, getNumberOfRows(keyPrefix2, value2, table));
496     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,
497       0, getNumberOfRows(keyPrefix3, value2, table));
498     ht.setScannerCaching(0);
499     assertEquals("Got back incorrect number of rows from scan", 0,
500       getNumberOfRows(keyPrefix1, value2, table)); ht.setScannerCaching(100);
501     assertEquals("Got back incorrect number of rows from scan", 0,
502       getNumberOfRows(keyPrefix2, value2, table));
503   }
504 
505   private void deleteColumns(HTable ht, String value, String keyPrefix)
506   throws IOException {
507     ResultScanner scanner = buildScanner(keyPrefix, value, ht);
508     Iterator<Result> it = scanner.iterator();
509     int count = 0;
510     while (it.hasNext()) {
511       Result result = it.next();
512       Delete delete = new Delete(result.getRow());
513       delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
514       ht.delete(delete);
515       count++;
516     }
517     assertEquals("Did not perform correct number of deletes", 3, count);
518   }
519 
520   private int getNumberOfRows(String keyPrefix, String value, HTable ht)
521       throws Exception {
522     ResultScanner resultScanner = buildScanner(keyPrefix, value, ht);
523     Iterator<Result> scanner = resultScanner.iterator();
524     int numberOfResults = 0;
525     while (scanner.hasNext()) {
526       Result result = scanner.next();
527       System.out.println("Got back key: " + Bytes.toString(result.getRow()));
528       for (KeyValue kv : result.raw()) {
529         System.out.println("kv=" + kv.toString() + ", "
530             + Bytes.toString(kv.getValue()));
531       }
532       numberOfResults++;
533     }
534     return numberOfResults;
535   }
536 
537   private ResultScanner buildScanner(String keyPrefix, String value, HTable ht)
538       throws IOException {
539     // OurFilterList allFilters = new OurFilterList();
540     FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */);
541     allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
542     SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes
543         .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes
544         .toBytes(value));
545     filter.setFilterIfMissing(true);
546     allFilters.addFilter(filter);
547 
548     // allFilters.addFilter(new
549     // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"),
550     // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)));
551 
552     Scan scan = new Scan();
553     scan.addFamily(Bytes.toBytes("trans-blob"));
554     scan.addFamily(Bytes.toBytes("trans-type"));
555     scan.addFamily(Bytes.toBytes("trans-date"));
556     scan.addFamily(Bytes.toBytes("trans-tags"));
557     scan.addFamily(Bytes.toBytes("trans-group"));
558     scan.setFilter(allFilters);
559 
560     return ht.getScanner(scan);
561   }
562 
563   private void putRows(HTable ht, int numRows, String value, String key)
564       throws IOException {
565     for (int i = 0; i < numRows; i++) {
566       String row = key + "_" + UUID.randomUUID().toString();
567       System.out.println(String.format("Saving row: %s, with value %s", row,
568           value));
569       Put put = new Put(Bytes.toBytes(row));
570       put.setWriteToWAL(false);
571       put.add(Bytes.toBytes("trans-blob"), null, Bytes
572           .toBytes("value for blob"));
573       put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
574       put.add(Bytes.toBytes("trans-date"), null, Bytes
575           .toBytes("20090921010101999"));
576       put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes
577           .toBytes(value));
578       put.add(Bytes.toBytes("trans-group"), null, Bytes
579           .toBytes("adhocTransactionGroupId"));
580       ht.put(put);
581     }
582   }
583 
584   /**
585    * Test filters when multiple regions.  It does counts.  Needs eye-balling of
586    * logs to ensure that we're not scanning more regions that we're supposed to.
587    * Related to the TestFilterAcrossRegions over in the o.a.h.h.filter package.
588    * @throws IOException
589    * @throws InterruptedException
590    */
591   @Test
592   public void testFilterAcrossMultipleRegions()
593   throws IOException, InterruptedException {
594     byte [] name = Bytes.toBytes("testFilterAcrossMutlipleRegions");
595     HTable t = TEST_UTIL.createTable(name, FAMILY);
596     int rowCount = TEST_UTIL.loadTable(t, FAMILY);
597     assertRowCount(t, rowCount);
598     // Split the table.  Should split on a reasonable key; 'lqj'
599     Map<HRegionInfo, HServerAddress> regions  = splitTable(t);
600     assertRowCount(t, rowCount);
601     // Get end key of first region.
602     byte [] endKey = regions.keySet().iterator().next().getEndKey();
603     // Count rows with a filter that stops us before passed 'endKey'.
604     // Should be count of rows in first region.
605     int endKeyCount = countRows(t, createScanWithRowFilter(endKey));
606     assertTrue(endKeyCount < rowCount);
607 
608     // How do I know I did not got to second region?  Thats tough.  Can't really
609     // do that in client-side region test.  I verified by tracing in debugger.
610     // I changed the messages that come out when set to DEBUG so should see
611     // when scanner is done. Says "Finished with scanning..." with region name.
612     // Check that its finished in right region.
613 
614     // New test.  Make it so scan goes into next region by one and then two.
615     // Make sure count comes out right.
616     byte [] key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 1)};
617     int plusOneCount = countRows(t, createScanWithRowFilter(key));
618     assertEquals(endKeyCount + 1, plusOneCount);
619     key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 2)};
620     int plusTwoCount = countRows(t, createScanWithRowFilter(key));
621     assertEquals(endKeyCount + 2, plusTwoCount);
622 
623     // New test.  Make it so I scan one less than endkey.
624     key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] - 1)};
625     int minusOneCount = countRows(t, createScanWithRowFilter(key));
626     assertEquals(endKeyCount - 1, minusOneCount);
627     // For above test... study logs.  Make sure we do "Finished with scanning.."
628     // in first region and that we do not fall into the next region.
629 
630     key = new byte [] {'a', 'a', 'a'};
631     int countBBB = countRows(t,
632       createScanWithRowFilter(key, null, CompareFilter.CompareOp.EQUAL));
633     assertEquals(1, countBBB);
634 
635     int countGreater = countRows(t, createScanWithRowFilter(endKey, null,
636       CompareFilter.CompareOp.GREATER_OR_EQUAL));
637     // Because started at start of table.
638     assertEquals(0, countGreater);
639     countGreater = countRows(t, createScanWithRowFilter(endKey, endKey,
640       CompareFilter.CompareOp.GREATER_OR_EQUAL));
641     assertEquals(rowCount - endKeyCount, countGreater);
642   }
643 
644   /*
645    * @param key
646    * @return Scan with RowFilter that does LESS than passed key.
647    */
648   private Scan createScanWithRowFilter(final byte [] key) {
649     return createScanWithRowFilter(key, null, CompareFilter.CompareOp.LESS);
650   }
651 
652   /*
653    * @param key
654    * @param op
655    * @param startRow
656    * @return Scan with RowFilter that does CompareOp op on passed key.
657    */
658   private Scan createScanWithRowFilter(final byte [] key,
659       final byte [] startRow, CompareFilter.CompareOp op) {
660     // Make sure key is of some substance... non-null and > than first key.
661     assertTrue(key != null && key.length > 0 &&
662       Bytes.BYTES_COMPARATOR.compare(key, new byte [] {'a', 'a', 'a'}) >= 0);
663     LOG.info("Key=" + Bytes.toString(key));
664     Scan s = startRow == null? new Scan(): new Scan(startRow);
665     Filter f = new RowFilter(op, new BinaryComparator(key));
666     f = new WhileMatchFilter(f);
667     s.setFilter(f);
668     return s;
669   }
670 
671   /*
672    * @param t
673    * @param s
674    * @return Count of rows in table.
675    * @throws IOException
676    */
677   private int countRows(final HTable t, final Scan s)
678   throws IOException {
679     // Assert all rows in table.
680     ResultScanner scanner = t.getScanner(s);
681     int count = 0;
682     for (Result result: scanner) {
683       count++;
684       assertTrue(result.size() > 0);
685       // LOG.info("Count=" + count + ", row=" + Bytes.toString(result.getRow()));
686     }
687     return count;
688   }
689 
690   private void assertRowCount(final HTable t, final int expected)
691   throws IOException {
692     assertEquals(expected, countRows(t, new Scan()));
693   }
694 
695   /*
696    * Split table into multiple regions.
697    * @param t Table to split.
698    * @return Map of regions to servers.
699    * @throws IOException
700    */
701   private Map<HRegionInfo, HServerAddress> splitTable(final HTable t)
702   throws IOException, InterruptedException {
703     // Split this table in two.
704     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
705     admin.split(t.getTableName());
706     Map<HRegionInfo, HServerAddress> regions = waitOnSplit(t);
707     assertTrue(regions.size() > 1);
708     return regions;
709   }
710 
711   /*
712    * Wait on table split.  May return because we waited long enough on the split
713    * and it didn't happen.  Caller should check.
714    * @param t
715    * @return Map of table regions; caller needs to check table actually split.
716    */
717   private Map<HRegionInfo, HServerAddress> waitOnSplit(final HTable t)
718   throws IOException {
719     Map<HRegionInfo, HServerAddress> regions = t.getRegionsInfo();
720     int originalCount = regions.size();
721     for (int i = 0; i < TEST_UTIL.getConfiguration().getInt("hbase.test.retries", 30); i++) {
722       Thread.currentThread();
723       try {
724         Thread.sleep(1000);
725       } catch (InterruptedException e) {
726         e.printStackTrace();
727       }
728       regions = t.getRegionsInfo();
729       if (regions.size() > originalCount) break;
730     }
731     return regions;
732   }
733 
734   @Test
735   public void testSuperSimple() throws Exception {
736     byte [] TABLE = Bytes.toBytes("testSuperSimple");
737     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
738     Put put = new Put(ROW);
739     put.add(FAMILY, QUALIFIER, VALUE);
740     ht.put(put);
741     Scan scan = new Scan();
742     scan.addColumn(FAMILY, TABLE);
743     ResultScanner scanner = ht.getScanner(scan);
744     Result result = scanner.next();
745     assertTrue("Expected null result", result == null);
746     scanner.close();
747   }
748 
749   @Test
750   public void testMaxKeyValueSize() throws Exception {
751     byte [] TABLE = Bytes.toBytes("testMaxKeyValueSize");
752     Configuration conf = TEST_UTIL.getConfiguration();
753     String oldMaxSize = conf.get("hbase.client.keyvalue.maxsize");
754     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
755     byte[] value = new byte[4 * 1024 * 1024];
756     Put put = new Put(ROW);
757     put.add(FAMILY, QUALIFIER, value);
758     ht.put(put);
759     try {
760       conf.setInt("hbase.client.keyvalue.maxsize", 2 * 1024 * 1024);
761       TABLE = Bytes.toBytes("testMaxKeyValueSize2");
762       ht = TEST_UTIL.createTable(TABLE, FAMILY);
763       put = new Put(ROW);
764       put.add(FAMILY, QUALIFIER, value);
765       ht.put(put);
766       fail("Inserting a too large KeyValue worked, should throw exception");
767     } catch(Exception e) {}
768     conf.set("hbase.client.keyvalue.maxsize", oldMaxSize);
769   }
770 
771   @Test
772   public void testFilters() throws Exception {
773     byte [] TABLE = Bytes.toBytes("testFilters");
774     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
775     byte [][] ROWS = makeN(ROW, 10);
776     byte [][] QUALIFIERS = {
777         Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
778         Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
779         Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
780         Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
781         Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>")
782     };
783     for(int i=0;i<10;i++) {
784       Put put = new Put(ROWS[i]);
785       put.setWriteToWAL(false);
786       put.add(FAMILY, QUALIFIERS[i], VALUE);
787       ht.put(put);
788     }
789     Scan scan = new Scan();
790     scan.addFamily(FAMILY);
791     Filter filter = new QualifierFilter(CompareOp.EQUAL,
792       new RegexStringComparator("col[1-5]"));
793     scan.setFilter(filter);
794     ResultScanner scanner = ht.getScanner(scan);
795     int expectedIndex = 1;
796     for(Result result : ht.getScanner(scan)) {
797       assertEquals(result.size(), 1);
798       assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[expectedIndex]));
799       assertTrue(Bytes.equals(result.raw()[0].getQualifier(),
800           QUALIFIERS[expectedIndex]));
801       expectedIndex++;
802     }
803     assertEquals(expectedIndex, 6);
804     scanner.close();
805   }
806 
807   @Test
808   public void testKeyOnlyFilter() throws Exception {
809     byte [] TABLE = Bytes.toBytes("testKeyOnlyFilter");
810     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
811     byte [][] ROWS = makeN(ROW, 10);
812     byte [][] QUALIFIERS = {
813         Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
814         Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
815         Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
816         Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
817         Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>")
818     };
819     for(int i=0;i<10;i++) {
820       Put put = new Put(ROWS[i]);
821       put.setWriteToWAL(false);
822       put.add(FAMILY, QUALIFIERS[i], VALUE);
823       ht.put(put);
824     }
825     Scan scan = new Scan();
826     scan.addFamily(FAMILY);
827     Filter filter = new KeyOnlyFilter(true);
828     scan.setFilter(filter);
829     ResultScanner scanner = ht.getScanner(scan);
830     int count = 0;
831     for(Result result : ht.getScanner(scan)) {
832       assertEquals(result.size(), 1);
833       assertEquals(result.raw()[0].getValueLength(), Bytes.SIZEOF_INT);
834       assertEquals(Bytes.toInt(result.raw()[0].getValue()), VALUE.length);
835       count++;
836     }
837     assertEquals(count, 10);
838     scanner.close();
839   }
840 
841   /**
842    * Test simple table and non-existent row cases.
843    */
844   @Test
845   public void testSimpleMissing() throws Exception {
846     byte [] TABLE = Bytes.toBytes("testSimpleMissing");
847     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
848     byte [][] ROWS = makeN(ROW, 4);
849 
850     // Try to get a row on an empty table
851     Get get = new Get(ROWS[0]);
852     Result result = ht.get(get);
853     assertEmptyResult(result);
854 
855     get = new Get(ROWS[0]);
856     get.addFamily(FAMILY);
857     result = ht.get(get);
858     assertEmptyResult(result);
859 
860     get = new Get(ROWS[0]);
861     get.addColumn(FAMILY, QUALIFIER);
862     result = ht.get(get);
863     assertEmptyResult(result);
864 
865     Scan scan = new Scan();
866     result = getSingleScanResult(ht, scan);
867     assertNullResult(result);
868 
869 
870     scan = new Scan(ROWS[0]);
871     result = getSingleScanResult(ht, scan);
872     assertNullResult(result);
873 
874     scan = new Scan(ROWS[0],ROWS[1]);
875     result = getSingleScanResult(ht, scan);
876     assertNullResult(result);
877 
878     scan = new Scan();
879     scan.addFamily(FAMILY);
880     result = getSingleScanResult(ht, scan);
881     assertNullResult(result);
882 
883     scan = new Scan();
884     scan.addColumn(FAMILY, QUALIFIER);
885     result = getSingleScanResult(ht, scan);
886     assertNullResult(result);
887 
888     // Insert a row
889 
890     Put put = new Put(ROWS[2]);
891     put.add(FAMILY, QUALIFIER, VALUE);
892     ht.put(put);
893 
894     // Try to get empty rows around it
895 
896     get = new Get(ROWS[1]);
897     result = ht.get(get);
898     assertEmptyResult(result);
899 
900     get = new Get(ROWS[0]);
901     get.addFamily(FAMILY);
902     result = ht.get(get);
903     assertEmptyResult(result);
904 
905     get = new Get(ROWS[3]);
906     get.addColumn(FAMILY, QUALIFIER);
907     result = ht.get(get);
908     assertEmptyResult(result);
909 
910     // Try to scan empty rows around it
911 
912     scan = new Scan(ROWS[3]);
913     result = getSingleScanResult(ht, scan);
914     assertNullResult(result);
915 
916     scan = new Scan(ROWS[0],ROWS[2]);
917     result = getSingleScanResult(ht, scan);
918     assertNullResult(result);
919 
920     // Make sure we can actually get the row
921 
922     get = new Get(ROWS[2]);
923     result = ht.get(get);
924     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
925 
926     get = new Get(ROWS[2]);
927     get.addFamily(FAMILY);
928     result = ht.get(get);
929     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
930 
931     get = new Get(ROWS[2]);
932     get.addColumn(FAMILY, QUALIFIER);
933     result = ht.get(get);
934     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
935 
936     // Make sure we can scan the row
937 
938     scan = new Scan();
939     result = getSingleScanResult(ht, scan);
940     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
941 
942     scan = new Scan(ROWS[0],ROWS[3]);
943     result = getSingleScanResult(ht, scan);
944     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
945 
946     scan = new Scan(ROWS[2],ROWS[3]);
947     result = getSingleScanResult(ht, scan);
948     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
949   }
950 
951   /**
952    * Test basic puts, gets, scans, and deletes for a single row
953    * in a multiple family table.
954    */
955   @Test
956   public void testSingleRowMultipleFamily() throws Exception {
957     byte [] TABLE = Bytes.toBytes("testSingleRowMultipleFamily");
958     byte [][] ROWS = makeN(ROW, 3);
959     byte [][] FAMILIES = makeNAscii(FAMILY, 10);
960     byte [][] QUALIFIERS = makeN(QUALIFIER, 10);
961     byte [][] VALUES = makeN(VALUE, 10);
962 
963     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
964 
965     Get get;
966     Scan scan;
967     Delete delete;
968     Put put;
969     Result result;
970 
971     ////////////////////////////////////////////////////////////////////////////
972     // Insert one column to one family
973     ////////////////////////////////////////////////////////////////////////////
974 
975     put = new Put(ROWS[0]);
976     put.add(FAMILIES[4], QUALIFIERS[0], VALUES[0]);
977     ht.put(put);
978 
979     // Get the single column
980     getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
981 
982     // Scan the single column
983     scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
984 
985     // Get empty results around inserted column
986     getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
987 
988     // Scan empty results around inserted column
989     scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
990 
991     ////////////////////////////////////////////////////////////////////////////
992     // Flush memstore and run same tests from storefiles
993     ////////////////////////////////////////////////////////////////////////////
994 
995     TEST_UTIL.flush();
996 
997     // Redo get and scan tests from storefile
998     getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
999     scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
1000     getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
1001     scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
1002 
1003     ////////////////////////////////////////////////////////////////////////////
1004     // Now, Test reading from memstore and storefiles at once
1005     ////////////////////////////////////////////////////////////////////////////
1006 
1007     // Insert multiple columns to two other families
1008     put = new Put(ROWS[0]);
1009     put.add(FAMILIES[2], QUALIFIERS[2], VALUES[2]);
1010     put.add(FAMILIES[2], QUALIFIERS[4], VALUES[4]);
1011     put.add(FAMILIES[4], QUALIFIERS[4], VALUES[4]);
1012     put.add(FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1013     put.add(FAMILIES[6], QUALIFIERS[7], VALUES[7]);
1014     put.add(FAMILIES[7], QUALIFIERS[7], VALUES[7]);
1015     put.add(FAMILIES[9], QUALIFIERS[0], VALUES[0]);
1016     ht.put(put);
1017 
1018     // Get multiple columns across multiple families and get empties around it
1019     singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1020 
1021     // Scan multiple columns across multiple families and scan empties around it
1022     singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1023 
1024     ////////////////////////////////////////////////////////////////////////////
1025     // Flush the table again
1026     ////////////////////////////////////////////////////////////////////////////
1027 
1028     TEST_UTIL.flush();
1029 
1030     // Redo tests again
1031     singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1032     singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1033 
1034     // Insert more data to memstore
1035     put = new Put(ROWS[0]);
1036     put.add(FAMILIES[6], QUALIFIERS[5], VALUES[5]);
1037     put.add(FAMILIES[6], QUALIFIERS[8], VALUES[8]);
1038     put.add(FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1039     put.add(FAMILIES[4], QUALIFIERS[3], VALUES[3]);
1040     ht.put(put);
1041 
1042     ////////////////////////////////////////////////////////////////////////////
1043     // Delete a storefile column
1044     ////////////////////////////////////////////////////////////////////////////
1045     delete = new Delete(ROWS[0]);
1046     delete.deleteColumns(FAMILIES[6], QUALIFIERS[7]);
1047     ht.delete(delete);
1048 
1049     // Try to get deleted column
1050     get = new Get(ROWS[0]);
1051     get.addColumn(FAMILIES[6], QUALIFIERS[7]);
1052     result = ht.get(get);
1053     assertEmptyResult(result);
1054 
1055     // Try to scan deleted column
1056     scan = new Scan();
1057     scan.addColumn(FAMILIES[6], QUALIFIERS[7]);
1058     result = getSingleScanResult(ht, scan);
1059     assertNullResult(result);
1060 
1061     // Make sure we can still get a column before it and after it
1062     get = new Get(ROWS[0]);
1063     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
1064     result = ht.get(get);
1065     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1066 
1067     get = new Get(ROWS[0]);
1068     get.addColumn(FAMILIES[6], QUALIFIERS[8]);
1069     result = ht.get(get);
1070     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]);
1071 
1072     // Make sure we can still scan a column before it and after it
1073     scan = new Scan();
1074     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1075     result = getSingleScanResult(ht, scan);
1076     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1077 
1078     scan = new Scan();
1079     scan.addColumn(FAMILIES[6], QUALIFIERS[8]);
1080     result = getSingleScanResult(ht, scan);
1081     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]);
1082 
1083     ////////////////////////////////////////////////////////////////////////////
1084     // Delete a memstore column
1085     ////////////////////////////////////////////////////////////////////////////
1086     delete = new Delete(ROWS[0]);
1087     delete.deleteColumns(FAMILIES[6], QUALIFIERS[8]);
1088     ht.delete(delete);
1089 
1090     // Try to get deleted column
1091     get = new Get(ROWS[0]);
1092     get.addColumn(FAMILIES[6], QUALIFIERS[8]);
1093     result = ht.get(get);
1094     assertEmptyResult(result);
1095 
1096     // Try to scan deleted column
1097     scan = new Scan();
1098     scan.addColumn(FAMILIES[6], QUALIFIERS[8]);
1099     result = getSingleScanResult(ht, scan);
1100     assertNullResult(result);
1101 
1102     // Make sure we can still get a column before it and after it
1103     get = new Get(ROWS[0]);
1104     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
1105     result = ht.get(get);
1106     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1107 
1108     get = new Get(ROWS[0]);
1109     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1110     result = ht.get(get);
1111     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1112 
1113     // Make sure we can still scan a column before it and after it
1114     scan = new Scan();
1115     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1116     result = getSingleScanResult(ht, scan);
1117     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1118 
1119     scan = new Scan();
1120     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1121     result = getSingleScanResult(ht, scan);
1122     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1123 
1124     ////////////////////////////////////////////////////////////////////////////
1125     // Delete joint storefile/memstore family
1126     ////////////////////////////////////////////////////////////////////////////
1127 
1128     delete = new Delete(ROWS[0]);
1129     delete.deleteFamily(FAMILIES[4]);
1130     ht.delete(delete);
1131 
1132     // Try to get storefile column in deleted family
1133     get = new Get(ROWS[0]);
1134     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
1135     result = ht.get(get);
1136     assertEmptyResult(result);
1137 
1138     // Try to get memstore column in deleted family
1139     get = new Get(ROWS[0]);
1140     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
1141     result = ht.get(get);
1142     assertEmptyResult(result);
1143 
1144     // Try to get deleted family
1145     get = new Get(ROWS[0]);
1146     get.addFamily(FAMILIES[4]);
1147     result = ht.get(get);
1148     assertEmptyResult(result);
1149 
1150     // Try to scan storefile column in deleted family
1151     scan = new Scan();
1152     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
1153     result = getSingleScanResult(ht, scan);
1154     assertNullResult(result);
1155 
1156     // Try to scan memstore column in deleted family
1157     scan = new Scan();
1158     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
1159     result = getSingleScanResult(ht, scan);
1160     assertNullResult(result);
1161 
1162     // Try to scan deleted family
1163     scan = new Scan();
1164     scan.addFamily(FAMILIES[4]);
1165     result = getSingleScanResult(ht, scan);
1166     assertNullResult(result);
1167 
1168     // Make sure we can still get another family
1169     get = new Get(ROWS[0]);
1170     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
1171     result = ht.get(get);
1172     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
1173 
1174     get = new Get(ROWS[0]);
1175     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1176     result = ht.get(get);
1177     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1178 
1179     // Make sure we can still scan another family
1180     scan = new Scan();
1181     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1182     result = getSingleScanResult(ht, scan);
1183     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1184 
1185     scan = new Scan();
1186     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1187     result = getSingleScanResult(ht, scan);
1188     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1189 
1190     ////////////////////////////////////////////////////////////////////////////
1191     // Flush everything and rerun delete tests
1192     ////////////////////////////////////////////////////////////////////////////
1193 
1194     TEST_UTIL.flush();
1195 
1196     // Try to get storefile column in deleted family
1197     get = new Get(ROWS[0]);
1198     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
1199     result = ht.get(get);
1200     assertEmptyResult(result);
1201 
1202     // Try to get memstore column in deleted family
1203     get = new Get(ROWS[0]);
1204     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
1205     result = ht.get(get);
1206     assertEmptyResult(result);
1207 
1208     // Try to get deleted family
1209     get = new Get(ROWS[0]);
1210     get.addFamily(FAMILIES[4]);
1211     result = ht.get(get);
1212     assertEmptyResult(result);
1213 
1214     // Try to scan storefile column in deleted family
1215     scan = new Scan();
1216     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
1217     result = getSingleScanResult(ht, scan);
1218     assertNullResult(result);
1219 
1220     // Try to scan memstore column in deleted family
1221     scan = new Scan();
1222     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
1223     result = getSingleScanResult(ht, scan);
1224     assertNullResult(result);
1225 
1226     // Try to scan deleted family
1227     scan = new Scan();
1228     scan.addFamily(FAMILIES[4]);
1229     result = getSingleScanResult(ht, scan);
1230     assertNullResult(result);
1231 
1232     // Make sure we can still get another family
1233     get = new Get(ROWS[0]);
1234     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
1235     result = ht.get(get);
1236     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
1237 
1238     get = new Get(ROWS[0]);
1239     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1240     result = ht.get(get);
1241     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1242 
1243     // Make sure we can still scan another family
1244     scan = new Scan();
1245     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1246     result = getSingleScanResult(ht, scan);
1247     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1248 
1249     scan = new Scan();
1250     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1251     result = getSingleScanResult(ht, scan);
1252     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1253 
1254   }
1255 
1256   @Test
1257   public void testNull() throws Exception {
1258     byte [] TABLE = Bytes.toBytes("testNull");
1259 
1260     // Null table name (should NOT work)
1261     try {
1262       TEST_UTIL.createTable(null, FAMILY);
1263       fail("Creating a table with null name passed, should have failed");
1264     } catch(Exception e) {}
1265 
1266     // Null family (should NOT work)
1267     try {
1268       TEST_UTIL.createTable(TABLE, (byte[])null);
1269       fail("Creating a table with a null family passed, should fail");
1270     } catch(Exception e) {}
1271 
1272     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
1273 
1274     // Null row (should NOT work)
1275     try {
1276       Put put = new Put((byte[])null);
1277       put.add(FAMILY, QUALIFIER, VALUE);
1278       ht.put(put);
1279       fail("Inserting a null row worked, should throw exception");
1280     } catch(Exception e) {}
1281 
1282     // Null qualifier (should work)
1283     {
1284       Put put = new Put(ROW);
1285       put.add(FAMILY, null, VALUE);
1286       ht.put(put);
1287 
1288       getTestNull(ht, ROW, FAMILY, VALUE);
1289 
1290       scanTestNull(ht, ROW, FAMILY, VALUE);
1291 
1292       Delete delete = new Delete(ROW);
1293       delete.deleteColumns(FAMILY, null);
1294       ht.delete(delete);
1295 
1296       Get get = new Get(ROW);
1297       Result result = ht.get(get);
1298       assertEmptyResult(result);
1299     }
1300 
1301     // Use a new table
1302     byte [] TABLE2 = Bytes.toBytes("testNull2");
1303     ht = TEST_UTIL.createTable(TABLE2, FAMILY);
1304 
1305     // Empty qualifier, byte[0] instead of null (should work)
1306     try {
1307       Put put = new Put(ROW);
1308       put.add(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE);
1309       ht.put(put);
1310 
1311       getTestNull(ht, ROW, FAMILY, VALUE);
1312 
1313       scanTestNull(ht, ROW, FAMILY, VALUE);
1314 
1315       // Flush and try again
1316 
1317       TEST_UTIL.flush();
1318 
1319       getTestNull(ht, ROW, FAMILY, VALUE);
1320 
1321       scanTestNull(ht, ROW, FAMILY, VALUE);
1322 
1323       Delete delete = new Delete(ROW);
1324       delete.deleteColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY);
1325       ht.delete(delete);
1326 
1327       Get get = new Get(ROW);
1328       Result result = ht.get(get);
1329       assertEmptyResult(result);
1330 
1331     } catch(Exception e) {
1332       throw new IOException("Using a row with null qualifier threw exception, should ");
1333     }
1334 
1335     // Null value
1336     try {
1337       Put put = new Put(ROW);
1338       put.add(FAMILY, QUALIFIER, null);
1339       ht.put(put);
1340 
1341       Get get = new Get(ROW);
1342       get.addColumn(FAMILY, QUALIFIER);
1343       Result result = ht.get(get);
1344       assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
1345 
1346       Scan scan = new Scan();
1347       scan.addColumn(FAMILY, QUALIFIER);
1348       result = getSingleScanResult(ht, scan);
1349       assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
1350 
1351       Delete delete = new Delete(ROW);
1352       delete.deleteColumns(FAMILY, QUALIFIER);
1353       ht.delete(delete);
1354 
1355       get = new Get(ROW);
1356       result = ht.get(get);
1357       assertEmptyResult(result);
1358 
1359     } catch(Exception e) {
1360       throw new IOException("Null values should be allowed, but threw exception");
1361     }
1362   }
1363 
1364   @Test
1365   public void testVersions() throws Exception {
1366     byte [] TABLE = Bytes.toBytes("testVersions");
1367 
1368     long [] STAMPS = makeStamps(20);
1369     byte [][] VALUES = makeNAscii(VALUE, 20);
1370 
1371     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
1372 
1373     // Insert 4 versions of same column
1374     Put put = new Put(ROW);
1375     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1376     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1377     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1378     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1379     ht.put(put);
1380 
1381     // Verify we can get each one properly
1382     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1383     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1384     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1385     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1386     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1387     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1388     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1389     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1390 
1391     // Verify we don't accidentally get others
1392     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1393     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1394     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1395     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1396     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1397     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1398 
1399     // Ensure maxVersions in query is respected
1400     Get get = new Get(ROW);
1401     get.addColumn(FAMILY, QUALIFIER);
1402     get.setMaxVersions(2);
1403     Result result = ht.get(get);
1404     assertNResult(result, ROW, FAMILY, QUALIFIER,
1405         new long [] {STAMPS[4], STAMPS[5]},
1406         new byte[][] {VALUES[4], VALUES[5]},
1407         0, 1);
1408 
1409     Scan scan = new Scan(ROW);
1410     scan.addColumn(FAMILY, QUALIFIER);
1411     scan.setMaxVersions(2);
1412     result = getSingleScanResult(ht, scan);
1413     assertNResult(result, ROW, FAMILY, QUALIFIER,
1414         new long [] {STAMPS[4], STAMPS[5]},
1415         new byte[][] {VALUES[4], VALUES[5]},
1416         0, 1);
1417 
1418     // Flush and redo
1419 
1420     TEST_UTIL.flush();
1421 
1422     // Verify we can get each one properly
1423     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1424     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1425     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1426     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1427     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1428     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1429     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1430     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1431 
1432     // Verify we don't accidentally get others
1433     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1434     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1435     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1436     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1437     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1438     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1439 
1440     // Ensure maxVersions in query is respected
1441     get = new Get(ROW);
1442     get.addColumn(FAMILY, QUALIFIER);
1443     get.setMaxVersions(2);
1444     result = ht.get(get);
1445     assertNResult(result, ROW, FAMILY, QUALIFIER,
1446         new long [] {STAMPS[4], STAMPS[5]},
1447         new byte[][] {VALUES[4], VALUES[5]},
1448         0, 1);
1449 
1450     scan = new Scan(ROW);
1451     scan.addColumn(FAMILY, QUALIFIER);
1452     scan.setMaxVersions(2);
1453     result = getSingleScanResult(ht, scan);
1454     assertNResult(result, ROW, FAMILY, QUALIFIER,
1455         new long [] {STAMPS[4], STAMPS[5]},
1456         new byte[][] {VALUES[4], VALUES[5]},
1457         0, 1);
1458 
1459 
1460     // Add some memstore and retest
1461 
1462     // Insert 4 more versions of same column and a dupe
1463     put = new Put(ROW);
1464     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
1465     put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
1466     put.add(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1467     put.add(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
1468     ht.put(put);
1469 
1470     // Ensure maxVersions in query is respected
1471     get = new Get(ROW);
1472     get.addColumn(FAMILY, QUALIFIER);
1473     get.setMaxVersions();
1474     result = ht.get(get);
1475     assertNResult(result, ROW, FAMILY, QUALIFIER,
1476         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1477         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1478         0, 7);
1479 
1480     scan = new Scan(ROW);
1481     scan.addColumn(FAMILY, QUALIFIER);
1482     scan.setMaxVersions();
1483     result = getSingleScanResult(ht, scan);
1484     assertNResult(result, ROW, FAMILY, QUALIFIER,
1485         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1486         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1487         0, 7);
1488 
1489     get = new Get(ROW);
1490     get.setMaxVersions();
1491     result = ht.get(get);
1492     assertNResult(result, ROW, FAMILY, QUALIFIER,
1493         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1494         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1495         0, 7);
1496 
1497     scan = new Scan(ROW);
1498     scan.setMaxVersions();
1499     result = getSingleScanResult(ht, scan);
1500     assertNResult(result, ROW, FAMILY, QUALIFIER,
1501         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1502         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1503         0, 7);
1504 
1505     // Verify we can get each one properly
1506     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1507     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1508     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1509     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1510     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1511     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1512     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1513     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1514 
1515     // Verify we don't accidentally get others
1516     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1517     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
1518     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1519     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
1520 
1521     // Ensure maxVersions of table is respected
1522 
1523     TEST_UTIL.flush();
1524 
1525     // Insert 4 more versions of same column and a dupe
1526     put = new Put(ROW);
1527     put.add(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
1528     put.add(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
1529     put.add(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
1530     put.add(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
1531     ht.put(put);
1532 
1533     get = new Get(ROW);
1534     get.addColumn(FAMILY, QUALIFIER);
1535     get.setMaxVersions(Integer.MAX_VALUE);
1536     result = ht.get(get);
1537     assertNResult(result, ROW, FAMILY, QUALIFIER,
1538         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
1539         new byte[][] {VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
1540         0, 9);
1541 
1542     scan = new Scan(ROW);
1543     scan.addColumn(FAMILY, QUALIFIER);
1544     scan.setMaxVersions(Integer.MAX_VALUE);
1545     result = getSingleScanResult(ht, scan);
1546     assertNResult(result, ROW, FAMILY, QUALIFIER,
1547         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
1548         new byte[][] {VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
1549         0, 9);
1550 
1551     // Delete a version in the memstore and a version in a storefile
1552     Delete delete = new Delete(ROW);
1553     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[11]);
1554     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[7]);
1555     ht.delete(delete);
1556 
1557     // Test that it's gone
1558     get = new Get(ROW);
1559     get.addColumn(FAMILY, QUALIFIER);
1560     get.setMaxVersions(Integer.MAX_VALUE);
1561     result = ht.get(get);
1562     assertNResult(result, ROW, FAMILY, QUALIFIER,
1563         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
1564         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
1565         0, 9);
1566 
1567     scan = new Scan(ROW);
1568     scan.addColumn(FAMILY, QUALIFIER);
1569     scan.setMaxVersions(Integer.MAX_VALUE);
1570     result = getSingleScanResult(ht, scan);
1571     assertNResult(result, ROW, FAMILY, QUALIFIER,
1572         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
1573         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
1574         0, 9);
1575 
1576   }
1577 
1578   @Test
1579   public void testVersionLimits() throws Exception {
1580     byte [] TABLE = Bytes.toBytes("testVersionLimits");
1581     byte [][] FAMILIES = makeNAscii(FAMILY, 3);
1582     int [] LIMITS = {1,3,5};
1583     long [] STAMPS = makeStamps(10);
1584     byte [][] VALUES = makeNAscii(VALUE, 10);
1585     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, LIMITS);
1586 
1587     // Insert limit + 1 on each family
1588     Put put = new Put(ROW);
1589     put.add(FAMILIES[0], QUALIFIER, STAMPS[0], VALUES[0]);
1590     put.add(FAMILIES[0], QUALIFIER, STAMPS[1], VALUES[1]);
1591     put.add(FAMILIES[1], QUALIFIER, STAMPS[0], VALUES[0]);
1592     put.add(FAMILIES[1], QUALIFIER, STAMPS[1], VALUES[1]);
1593     put.add(FAMILIES[1], QUALIFIER, STAMPS[2], VALUES[2]);
1594     put.add(FAMILIES[1], QUALIFIER, STAMPS[3], VALUES[3]);
1595     put.add(FAMILIES[2], QUALIFIER, STAMPS[0], VALUES[0]);
1596     put.add(FAMILIES[2], QUALIFIER, STAMPS[1], VALUES[1]);
1597     put.add(FAMILIES[2], QUALIFIER, STAMPS[2], VALUES[2]);
1598     put.add(FAMILIES[2], QUALIFIER, STAMPS[3], VALUES[3]);
1599     put.add(FAMILIES[2], QUALIFIER, STAMPS[4], VALUES[4]);
1600     put.add(FAMILIES[2], QUALIFIER, STAMPS[5], VALUES[5]);
1601     put.add(FAMILIES[2], QUALIFIER, STAMPS[6], VALUES[6]);
1602     ht.put(put);
1603 
1604     // Verify we only get the right number out of each
1605 
1606     // Family0
1607 
1608     Get get = new Get(ROW);
1609     get.addColumn(FAMILIES[0], QUALIFIER);
1610     get.setMaxVersions(Integer.MAX_VALUE);
1611     Result result = ht.get(get);
1612     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1613         new long [] {STAMPS[1]},
1614         new byte[][] {VALUES[1]},
1615         0, 0);
1616 
1617     get = new Get(ROW);
1618     get.addFamily(FAMILIES[0]);
1619     get.setMaxVersions(Integer.MAX_VALUE);
1620     result = ht.get(get);
1621     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1622         new long [] {STAMPS[1]},
1623         new byte[][] {VALUES[1]},
1624         0, 0);
1625 
1626     Scan scan = new Scan(ROW);
1627     scan.addColumn(FAMILIES[0], QUALIFIER);
1628     scan.setMaxVersions(Integer.MAX_VALUE);
1629     result = getSingleScanResult(ht, scan);
1630     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1631         new long [] {STAMPS[1]},
1632         new byte[][] {VALUES[1]},
1633         0, 0);
1634 
1635     scan = new Scan(ROW);
1636     scan.addFamily(FAMILIES[0]);
1637     scan.setMaxVersions(Integer.MAX_VALUE);
1638     result = getSingleScanResult(ht, scan);
1639     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1640         new long [] {STAMPS[1]},
1641         new byte[][] {VALUES[1]},
1642         0, 0);
1643 
1644     // Family1
1645 
1646     get = new Get(ROW);
1647     get.addColumn(FAMILIES[1], QUALIFIER);
1648     get.setMaxVersions(Integer.MAX_VALUE);
1649     result = ht.get(get);
1650     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1651         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1652         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1653         0, 2);
1654 
1655     get = new Get(ROW);
1656     get.addFamily(FAMILIES[1]);
1657     get.setMaxVersions(Integer.MAX_VALUE);
1658     result = ht.get(get);
1659     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1660         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1661         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1662         0, 2);
1663 
1664     scan = new Scan(ROW);
1665     scan.addColumn(FAMILIES[1], QUALIFIER);
1666     scan.setMaxVersions(Integer.MAX_VALUE);
1667     result = getSingleScanResult(ht, scan);
1668     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1669         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1670         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1671         0, 2);
1672 
1673     scan = new Scan(ROW);
1674     scan.addFamily(FAMILIES[1]);
1675     scan.setMaxVersions(Integer.MAX_VALUE);
1676     result = getSingleScanResult(ht, scan);
1677     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1678         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1679         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1680         0, 2);
1681 
1682     // Family2
1683 
1684     get = new Get(ROW);
1685     get.addColumn(FAMILIES[2], QUALIFIER);
1686     get.setMaxVersions(Integer.MAX_VALUE);
1687     result = ht.get(get);
1688     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1689         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1690         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1691         0, 4);
1692 
1693     get = new Get(ROW);
1694     get.addFamily(FAMILIES[2]);
1695     get.setMaxVersions(Integer.MAX_VALUE);
1696     result = ht.get(get);
1697     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1698         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1699         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1700         0, 4);
1701 
1702     scan = new Scan(ROW);
1703     scan.addColumn(FAMILIES[2], QUALIFIER);
1704     scan.setMaxVersions(Integer.MAX_VALUE);
1705     result = getSingleScanResult(ht, scan);
1706     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1707         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1708         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1709         0, 4);
1710 
1711     scan = new Scan(ROW);
1712     scan.addFamily(FAMILIES[2]);
1713     scan.setMaxVersions(Integer.MAX_VALUE);
1714     result = getSingleScanResult(ht, scan);
1715     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1716         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1717         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1718         0, 4);
1719 
1720     // Try all families
1721 
1722     get = new Get(ROW);
1723     get.setMaxVersions(Integer.MAX_VALUE);
1724     result = ht.get(get);
1725     assertTrue("Expected 9 keys but received " + result.size(),
1726         result.size() == 9);
1727 
1728     get = new Get(ROW);
1729     get.addFamily(FAMILIES[0]);
1730     get.addFamily(FAMILIES[1]);
1731     get.addFamily(FAMILIES[2]);
1732     get.setMaxVersions(Integer.MAX_VALUE);
1733     result = ht.get(get);
1734     assertTrue("Expected 9 keys but received " + result.size(),
1735         result.size() == 9);
1736 
1737     get = new Get(ROW);
1738     get.addColumn(FAMILIES[0], QUALIFIER);
1739     get.addColumn(FAMILIES[1], QUALIFIER);
1740     get.addColumn(FAMILIES[2], QUALIFIER);
1741     get.setMaxVersions(Integer.MAX_VALUE);
1742     result = ht.get(get);
1743     assertTrue("Expected 9 keys but received " + result.size(),
1744         result.size() == 9);
1745 
1746     scan = new Scan(ROW);
1747     scan.setMaxVersions(Integer.MAX_VALUE);
1748     result = getSingleScanResult(ht, scan);
1749     assertTrue("Expected 9 keys but received " + result.size(),
1750         result.size() == 9);
1751 
1752     scan = new Scan(ROW);
1753     scan.setMaxVersions(Integer.MAX_VALUE);
1754     scan.addFamily(FAMILIES[0]);
1755     scan.addFamily(FAMILIES[1]);
1756     scan.addFamily(FAMILIES[2]);
1757     result = getSingleScanResult(ht, scan);
1758     assertTrue("Expected 9 keys but received " + result.size(),
1759         result.size() == 9);
1760 
1761     scan = new Scan(ROW);
1762     scan.setMaxVersions(Integer.MAX_VALUE);
1763     scan.addColumn(FAMILIES[0], QUALIFIER);
1764     scan.addColumn(FAMILIES[1], QUALIFIER);
1765     scan.addColumn(FAMILIES[2], QUALIFIER);
1766     result = getSingleScanResult(ht, scan);
1767     assertTrue("Expected 9 keys but received " + result.size(),
1768         result.size() == 9);
1769 
1770   }
1771 
1772   @Test
1773   public void testDeletes() throws Exception {
1774     byte [] TABLE = Bytes.toBytes("testDeletes");
1775 
1776     byte [][] ROWS = makeNAscii(ROW, 6);
1777     byte [][] FAMILIES = makeNAscii(FAMILY, 3);
1778     byte [][] VALUES = makeN(VALUE, 5);
1779     long [] ts = {1000, 2000, 3000, 4000, 5000};
1780 
1781     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
1782 
1783     Put put = new Put(ROW);
1784     put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
1785     put.add(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
1786     ht.put(put);
1787 
1788     Delete delete = new Delete(ROW);
1789     delete.deleteFamily(FAMILIES[0], ts[0]);
1790     ht.delete(delete);
1791 
1792     Get get = new Get(ROW);
1793     get.addFamily(FAMILIES[0]);
1794     get.setMaxVersions(Integer.MAX_VALUE);
1795     Result result = ht.get(get);
1796     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1797         new long [] {ts[1]},
1798         new byte[][] {VALUES[1]},
1799         0, 0);
1800 
1801     Scan scan = new Scan(ROW);
1802     scan.addFamily(FAMILIES[0]);
1803     scan.setMaxVersions(Integer.MAX_VALUE);
1804     result = getSingleScanResult(ht, scan);
1805     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1806         new long [] {ts[1]},
1807         new byte[][] {VALUES[1]},
1808         0, 0);
1809 
1810     // Test delete latest version
1811     put = new Put(ROW);
1812     put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
1813     put.add(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
1814     put.add(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
1815     put.add(FAMILIES[0], null, ts[4], VALUES[4]);
1816     put.add(FAMILIES[0], null, ts[2], VALUES[2]);
1817     put.add(FAMILIES[0], null, ts[3], VALUES[3]);
1818     ht.put(put);
1819 
1820     delete = new Delete(ROW);
1821     delete.deleteColumn(FAMILIES[0], QUALIFIER); // ts[4]
1822     ht.delete(delete);
1823 
1824     get = new Get(ROW);
1825     get.addColumn(FAMILIES[0], QUALIFIER);
1826     get.setMaxVersions(Integer.MAX_VALUE);
1827     result = ht.get(get);
1828     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1829         new long [] {ts[1], ts[2], ts[3]},
1830         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1831         0, 2);
1832 
1833     scan = new Scan(ROW);
1834     scan.addColumn(FAMILIES[0], QUALIFIER);
1835     scan.setMaxVersions(Integer.MAX_VALUE);
1836     result = getSingleScanResult(ht, scan);
1837     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1838         new long [] {ts[1], ts[2], ts[3]},
1839         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1840         0, 2);
1841 
1842     // Test for HBASE-1847
1843     delete = new Delete(ROW);
1844     delete.deleteColumn(FAMILIES[0], null);
1845     ht.delete(delete);
1846 
1847     // Cleanup null qualifier
1848     delete = new Delete(ROW);
1849     delete.deleteColumns(FAMILIES[0], null);
1850     ht.delete(delete);
1851 
1852     // Expected client behavior might be that you can re-put deleted values
1853     // But alas, this is not to be.  We can't put them back in either case.
1854 
1855     put = new Put(ROW);
1856     put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); // 1000
1857     put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); // 5000
1858     ht.put(put);
1859 
1860 
1861     // It used to be due to the internal implementation of Get, that
1862     // the Get() call would return ts[4] UNLIKE the Scan below. With
1863     // the switch to using Scan for Get this is no longer the case.
1864     get = new Get(ROW);
1865     get.addFamily(FAMILIES[0]);
1866     get.setMaxVersions(Integer.MAX_VALUE);
1867     result = ht.get(get);
1868     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1869         new long [] {ts[1], ts[2], ts[3]},
1870         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1871         0, 2);
1872 
1873     // The Scanner returns the previous values, the expected-naive-unexpected behavior
1874 
1875     scan = new Scan(ROW);
1876     scan.addFamily(FAMILIES[0]);
1877     scan.setMaxVersions(Integer.MAX_VALUE);
1878     result = getSingleScanResult(ht, scan);
1879     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1880         new long [] {ts[1], ts[2], ts[3]},
1881         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1882         0, 2);
1883 
1884     // Test deleting an entire family from one row but not the other various ways
1885 
1886     put = new Put(ROWS[0]);
1887     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
1888     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
1889     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
1890     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
1891     ht.put(put);
1892 
1893     put = new Put(ROWS[1]);
1894     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
1895     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
1896     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
1897     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
1898     ht.put(put);
1899 
1900     put = new Put(ROWS[2]);
1901     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
1902     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
1903     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
1904     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
1905     ht.put(put);
1906 
1907     // Assert that above went in.
1908     get = new Get(ROWS[2]);
1909     get.addFamily(FAMILIES[1]);
1910     get.addFamily(FAMILIES[2]);
1911     get.setMaxVersions(Integer.MAX_VALUE);
1912     result = ht.get(get);
1913     assertTrue("Expected 4 key but received " + result.size() + ": " + result,
1914         result.size() == 4);
1915 
1916     delete = new Delete(ROWS[0]);
1917     delete.deleteFamily(FAMILIES[2]);
1918     ht.delete(delete);
1919 
1920     delete = new Delete(ROWS[1]);
1921     delete.deleteColumns(FAMILIES[1], QUALIFIER);
1922     ht.delete(delete);
1923 
1924     delete = new Delete(ROWS[2]);
1925     delete.deleteColumn(FAMILIES[1], QUALIFIER);
1926     delete.deleteColumn(FAMILIES[1], QUALIFIER);
1927     delete.deleteColumn(FAMILIES[2], QUALIFIER);
1928     ht.delete(delete);
1929 
1930     get = new Get(ROWS[0]);
1931     get.addFamily(FAMILIES[1]);
1932     get.addFamily(FAMILIES[2]);
1933     get.setMaxVersions(Integer.MAX_VALUE);
1934     result = ht.get(get);
1935     assertTrue("Expected 2 keys but received " + result.size(),
1936         result.size() == 2);
1937     assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER,
1938         new long [] {ts[0], ts[1]},
1939         new byte[][] {VALUES[0], VALUES[1]},
1940         0, 1);
1941 
1942     scan = new Scan(ROWS[0]);
1943     scan.addFamily(FAMILIES[1]);
1944     scan.addFamily(FAMILIES[2]);
1945     scan.setMaxVersions(Integer.MAX_VALUE);
1946     result = getSingleScanResult(ht, scan);
1947     assertTrue("Expected 2 keys but received " + result.size(),
1948         result.size() == 2);
1949     assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER,
1950         new long [] {ts[0], ts[1]},
1951         new byte[][] {VALUES[0], VALUES[1]},
1952         0, 1);
1953 
1954     get = new Get(ROWS[1]);
1955     get.addFamily(FAMILIES[1]);
1956     get.addFamily(FAMILIES[2]);
1957     get.setMaxVersions(Integer.MAX_VALUE);
1958     result = ht.get(get);
1959     assertTrue("Expected 2 keys but received " + result.size(),
1960         result.size() == 2);
1961 
1962     scan = new Scan(ROWS[1]);
1963     scan.addFamily(FAMILIES[1]);
1964     scan.addFamily(FAMILIES[2]);
1965     scan.setMaxVersions(Integer.MAX_VALUE);
1966     result = getSingleScanResult(ht, scan);
1967     assertTrue("Expected 2 keys but received " + result.size(),
1968         result.size() == 2);
1969 
1970     get = new Get(ROWS[2]);
1971     get.addFamily(FAMILIES[1]);
1972     get.addFamily(FAMILIES[2]);
1973     get.setMaxVersions(Integer.MAX_VALUE);
1974     result = ht.get(get);
1975     assertEquals(1, result.size());
1976     assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
1977         new long [] {ts[2]},
1978         new byte[][] {VALUES[2]},
1979         0, 0);
1980 
1981     scan = new Scan(ROWS[2]);
1982     scan.addFamily(FAMILIES[1]);
1983     scan.addFamily(FAMILIES[2]);
1984     scan.setMaxVersions(Integer.MAX_VALUE);
1985     result = getSingleScanResult(ht, scan);
1986     assertEquals(1, result.size());
1987     assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
1988         new long [] {ts[2]},
1989         new byte[][] {VALUES[2]},
1990         0, 0);
1991 
1992     // Test if we delete the family first in one row (HBASE-1541)
1993 
1994     delete = new Delete(ROWS[3]);
1995     delete.deleteFamily(FAMILIES[1]);
1996     ht.delete(delete);
1997 
1998     put = new Put(ROWS[3]);
1999     put.add(FAMILIES[2], QUALIFIER, VALUES[0]);
2000     ht.put(put);
2001 
2002     put = new Put(ROWS[4]);
2003     put.add(FAMILIES[1], QUALIFIER, VALUES[1]);
2004     put.add(FAMILIES[2], QUALIFIER, VALUES[2]);
2005     ht.put(put);
2006 
2007     get = new Get(ROWS[3]);
2008     get.addFamily(FAMILIES[1]);
2009     get.addFamily(FAMILIES[2]);
2010     get.setMaxVersions(Integer.MAX_VALUE);
2011     result = ht.get(get);
2012     assertTrue("Expected 1 key but received " + result.size(),
2013         result.size() == 1);
2014 
2015     get = new Get(ROWS[4]);
2016     get.addFamily(FAMILIES[1]);
2017     get.addFamily(FAMILIES[2]);
2018     get.setMaxVersions(Integer.MAX_VALUE);
2019     result = ht.get(get);
2020     assertTrue("Expected 2 keys but received " + result.size(),
2021         result.size() == 2);
2022 
2023     scan = new Scan(ROWS[3]);
2024     scan.addFamily(FAMILIES[1]);
2025     scan.addFamily(FAMILIES[2]);
2026     scan.setMaxVersions(Integer.MAX_VALUE);
2027     ResultScanner scanner = ht.getScanner(scan);
2028     result = scanner.next();
2029     assertTrue("Expected 1 key but received " + result.size(),
2030         result.size() == 1);
2031     assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[3]));
2032     assertTrue(Bytes.equals(result.raw()[0].getValue(), VALUES[0]));
2033     result = scanner.next();
2034     assertTrue("Expected 2 keys but received " + result.size(),
2035         result.size() == 2);
2036     assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[4]));
2037     assertTrue(Bytes.equals(result.raw()[1].getRow(), ROWS[4]));
2038     assertTrue(Bytes.equals(result.raw()[0].getValue(), VALUES[1]));
2039     assertTrue(Bytes.equals(result.raw()[1].getValue(), VALUES[2]));
2040     scanner.close();
2041 
2042     // Add test of bulk deleting.
2043     for (int i = 0; i < 10; i++) {
2044       byte [] bytes = Bytes.toBytes(i);
2045       put = new Put(bytes);
2046       put.setWriteToWAL(false);
2047       put.add(FAMILIES[0], QUALIFIER, bytes);
2048       ht.put(put);
2049     }
2050     for (int i = 0; i < 10; i++) {
2051       byte [] bytes = Bytes.toBytes(i);
2052       get = new Get(bytes);
2053       get.addFamily(FAMILIES[0]);
2054       result = ht.get(get);
2055       assertTrue(result.size() == 1);
2056     }
2057     ArrayList<Delete> deletes = new ArrayList<Delete>();
2058     for (int i = 0; i < 10; i++) {
2059       byte [] bytes = Bytes.toBytes(i);
2060       delete = new Delete(bytes);
2061       delete.deleteFamily(FAMILIES[0]);
2062       deletes.add(delete);
2063     }
2064     ht.delete(deletes);
2065     for (int i = 0; i < 10; i++) {
2066       byte [] bytes = Bytes.toBytes(i);
2067       get = new Get(bytes);
2068       get.addFamily(FAMILIES[0]);
2069       result = ht.get(get);
2070       assertTrue(result.size() == 0);
2071     }
2072   }
2073 
2074   /*
2075    * Baseline "scalability" test.
2076    *
2077    * Tests one hundred families, one million columns, one million versions
2078    */
2079   @Ignore @Test
2080   public void testMillions() throws Exception {
2081 
2082     // 100 families
2083 
2084     // millions of columns
2085 
2086     // millions of versions
2087 
2088   }
2089 
2090   @Ignore @Test
2091   public void testMultipleRegionsAndBatchPuts() throws Exception {
2092     // Two family table
2093 
2094     // Insert lots of rows
2095 
2096     // Insert to the same row with batched puts
2097 
2098     // Insert to multiple rows with batched puts
2099 
2100     // Split the table
2101 
2102     // Get row from first region
2103 
2104     // Get row from second region
2105 
2106     // Scan all rows
2107 
2108     // Insert to multiple regions with batched puts
2109 
2110     // Get row from first region
2111 
2112     // Get row from second region
2113 
2114     // Scan all rows
2115 
2116 
2117   }
2118 
2119   @Ignore @Test
2120   public void testMultipleRowMultipleFamily() throws Exception {
2121 
2122   }
2123 
2124   //
2125   // JIRA Testers
2126   //
2127 
2128   /**
2129    * HBASE-867
2130    *    If millions of columns in a column family, hbase scanner won't come up
2131    *
2132    *    Test will create numRows rows, each with numColsPerRow columns
2133    *    (1 version each), and attempt to scan them all.
2134    *
2135    *    To test at scale, up numColsPerRow to the millions
2136    *    (have not gotten that to work running as junit though)
2137    */
2138   @Test
2139   public void testJiraTest867() throws Exception {
2140     int numRows = 10;
2141     int numColsPerRow = 2000;
2142 
2143     byte [] TABLE = Bytes.toBytes("testJiraTest867");
2144 
2145     byte [][] ROWS = makeN(ROW, numRows);
2146     byte [][] QUALIFIERS = makeN(QUALIFIER, numColsPerRow);
2147 
2148     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
2149 
2150     // Insert rows
2151 
2152     for(int i=0;i<numRows;i++) {
2153       Put put = new Put(ROWS[i]);
2154       put.setWriteToWAL(false);
2155       for(int j=0;j<numColsPerRow;j++) {
2156         put.add(FAMILY, QUALIFIERS[j], QUALIFIERS[j]);
2157       }
2158       assertTrue("Put expected to contain " + numColsPerRow + " columns but " +
2159           "only contains " + put.size(), put.size() == numColsPerRow);
2160       ht.put(put);
2161     }
2162 
2163     // Get a row
2164     Get get = new Get(ROWS[numRows-1]);
2165     Result result = ht.get(get);
2166     assertNumKeys(result, numColsPerRow);
2167     KeyValue [] keys = result.raw();
2168     for(int i=0;i<result.size();i++) {
2169       assertKey(keys[i], ROWS[numRows-1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2170     }
2171 
2172     // Scan the rows
2173     Scan scan = new Scan();
2174     ResultScanner scanner = ht.getScanner(scan);
2175     int rowCount = 0;
2176     while((result = scanner.next()) != null) {
2177       assertNumKeys(result, numColsPerRow);
2178       KeyValue [] kvs = result.raw();
2179       for(int i=0;i<numColsPerRow;i++) {
2180         assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2181       }
2182       rowCount++;
2183     }
2184     scanner.close();
2185     assertTrue("Expected to scan " + numRows + " rows but actually scanned "
2186         + rowCount + " rows", rowCount == numRows);
2187 
2188     // flush and try again
2189 
2190     TEST_UTIL.flush();
2191 
2192     // Get a row
2193     get = new Get(ROWS[numRows-1]);
2194     result = ht.get(get);
2195     assertNumKeys(result, numColsPerRow);
2196     keys = result.raw();
2197     for(int i=0;i<result.size();i++) {
2198       assertKey(keys[i], ROWS[numRows-1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2199     }
2200 
2201     // Scan the rows
2202     scan = new Scan();
2203     scanner = ht.getScanner(scan);
2204     rowCount = 0;
2205     while((result = scanner.next()) != null) {
2206       assertNumKeys(result, numColsPerRow);
2207       KeyValue [] kvs = result.raw();
2208       for(int i=0;i<numColsPerRow;i++) {
2209         assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2210       }
2211       rowCount++;
2212     }
2213     scanner.close();
2214     assertTrue("Expected to scan " + numRows + " rows but actually scanned "
2215         + rowCount + " rows", rowCount == numRows);
2216 
2217   }
2218 
2219   /**
2220    * HBASE-861
2221    *    get with timestamp will return a value if there is a version with an
2222    *    earlier timestamp
2223    */
2224   @Test
2225   public void testJiraTest861() throws Exception {
2226 
2227     byte [] TABLE = Bytes.toBytes("testJiraTest861");
2228     byte [][] VALUES = makeNAscii(VALUE, 7);
2229     long [] STAMPS = makeStamps(7);
2230 
2231     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2232 
2233     // Insert three versions
2234 
2235     Put put = new Put(ROW);
2236     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2237     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2238     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2239     ht.put(put);
2240 
2241     // Get the middle value
2242     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2243 
2244     // Try to get one version before (expect fail)
2245     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2246 
2247     // Try to get one version after (expect fail)
2248     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2249 
2250     // Try same from storefile
2251     TEST_UTIL.flush();
2252     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2253     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2254     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2255 
2256     // Insert two more versions surrounding others, into memstore
2257     put = new Put(ROW);
2258     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2259     put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2260     ht.put(put);
2261 
2262     // Check we can get everything we should and can't get what we shouldn't
2263     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2264     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2265     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2266     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2267     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2268     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2269     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2270 
2271     // Try same from two storefiles
2272     TEST_UTIL.flush();
2273     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2274     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2275     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2276     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2277     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2278     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2279     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2280 
2281   }
2282 
2283   /**
2284    * HBASE-33
2285    *    Add a HTable get/obtainScanner method that retrieves all versions of a
2286    *    particular column and row between two timestamps
2287    */
2288   @Test
2289   public void testJiraTest33() throws Exception {
2290 
2291     byte [] TABLE = Bytes.toBytes("testJiraTest33");
2292     byte [][] VALUES = makeNAscii(VALUE, 7);
2293     long [] STAMPS = makeStamps(7);
2294 
2295     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2296 
2297     // Insert lots versions
2298 
2299     Put put = new Put(ROW);
2300     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2301     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2302     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2303     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2304     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2305     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2306     ht.put(put);
2307 
2308     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2309     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2310     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2311     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2312 
2313     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2314     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2315     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2316     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2317 
2318     // Try same from storefile
2319     TEST_UTIL.flush();
2320 
2321     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2322     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2323     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2324     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2325 
2326     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2327     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2328     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2329     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2330 
2331   }
2332 
2333   /**
2334    * HBASE-1014
2335    *    commit(BatchUpdate) method should return timestamp
2336    */
2337   @Test
2338   public void testJiraTest1014() throws Exception {
2339 
2340     byte [] TABLE = Bytes.toBytes("testJiraTest1014");
2341 
2342     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2343 
2344     long manualStamp = 12345;
2345 
2346     // Insert lots versions
2347 
2348     Put put = new Put(ROW);
2349     put.add(FAMILY, QUALIFIER, manualStamp, VALUE);
2350     ht.put(put);
2351 
2352     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, manualStamp, VALUE);
2353     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp-1);
2354     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp+1);
2355 
2356   }
2357 
2358   /**
2359    * HBASE-1182
2360    *    Scan for columns > some timestamp
2361    */
2362   @Test
2363   public void testJiraTest1182() throws Exception {
2364 
2365     byte [] TABLE = Bytes.toBytes("testJiraTest1182");
2366     byte [][] VALUES = makeNAscii(VALUE, 7);
2367     long [] STAMPS = makeStamps(7);
2368 
2369     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2370 
2371     // Insert lots versions
2372 
2373     Put put = new Put(ROW);
2374     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2375     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2376     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2377     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2378     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2379     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2380     ht.put(put);
2381 
2382     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2383     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2384     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2385 
2386     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2387     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2388     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2389 
2390     // Try same from storefile
2391     TEST_UTIL.flush();
2392 
2393     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2394     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2395     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2396 
2397     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2398     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2399     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2400   }
2401 
2402   /**
2403    * HBASE-52
2404    *    Add a means of scanning over all versions
2405    */
2406   @Test
2407   public void testJiraTest52() throws Exception {
2408     byte [] TABLE = Bytes.toBytes("testJiraTest52");
2409     byte [][] VALUES = makeNAscii(VALUE, 7);
2410     long [] STAMPS = makeStamps(7);
2411 
2412     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2413 
2414     // Insert lots versions
2415 
2416     Put put = new Put(ROW);
2417     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2418     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2419     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2420     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2421     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2422     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2423     ht.put(put);
2424 
2425     getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2426 
2427     scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2428 
2429     // Try same from storefile
2430     TEST_UTIL.flush();
2431 
2432     getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2433 
2434     scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2435   }
2436 
2437   //
2438   // Bulk Testers
2439   //
2440 
2441   private void getVersionRangeAndVerifyGreaterThan(HTable ht, byte [] row,
2442       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
2443       int start, int end)
2444   throws IOException {
2445     Get get = new Get(row);
2446     get.addColumn(family, qualifier);
2447     get.setMaxVersions(Integer.MAX_VALUE);
2448     get.setTimeRange(stamps[start+1], Long.MAX_VALUE);
2449     Result result = ht.get(get);
2450     assertNResult(result, row, family, qualifier, stamps, values, start+1, end);
2451   }
2452 
2453   private void getVersionRangeAndVerify(HTable ht, byte [] row, byte [] family,
2454       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2455   throws IOException {
2456     Get get = new Get(row);
2457     get.addColumn(family, qualifier);
2458     get.setMaxVersions(Integer.MAX_VALUE);
2459     get.setTimeRange(stamps[start], stamps[end]+1);
2460     Result result = ht.get(get);
2461     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2462   }
2463 
2464   private void getAllVersionsAndVerify(HTable ht, byte [] row, byte [] family,
2465       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2466   throws IOException {
2467     Get get = new Get(row);
2468     get.addColumn(family, qualifier);
2469     get.setMaxVersions(Integer.MAX_VALUE);
2470     Result result = ht.get(get);
2471     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2472   }
2473 
2474   private void scanVersionRangeAndVerifyGreaterThan(HTable ht, byte [] row,
2475       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
2476       int start, int end)
2477   throws IOException {
2478     Scan scan = new Scan(row);
2479     scan.addColumn(family, qualifier);
2480     scan.setMaxVersions(Integer.MAX_VALUE);
2481     scan.setTimeRange(stamps[start+1], Long.MAX_VALUE);
2482     Result result = getSingleScanResult(ht, scan);
2483     assertNResult(result, row, family, qualifier, stamps, values, start+1, end);
2484   }
2485 
2486   private void scanVersionRangeAndVerify(HTable ht, byte [] row, byte [] family,
2487       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2488   throws IOException {
2489     Scan scan = new Scan(row);
2490     scan.addColumn(family, qualifier);
2491     scan.setMaxVersions(Integer.MAX_VALUE);
2492     scan.setTimeRange(stamps[start], stamps[end]+1);
2493     Result result = getSingleScanResult(ht, scan);
2494     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2495   }
2496 
2497   private void scanAllVersionsAndVerify(HTable ht, byte [] row, byte [] family,
2498       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2499   throws IOException {
2500     Scan scan = new Scan(row);
2501     scan.addColumn(family, qualifier);
2502     scan.setMaxVersions(Integer.MAX_VALUE);
2503     Result result = getSingleScanResult(ht, scan);
2504     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2505   }
2506 
2507   private void getVersionAndVerify(HTable ht, byte [] row, byte [] family,
2508       byte [] qualifier, long stamp, byte [] value)
2509   throws Exception {
2510     Get get = new Get(row);
2511     get.addColumn(family, qualifier);
2512     get.setTimeStamp(stamp);
2513     get.setMaxVersions(Integer.MAX_VALUE);
2514     Result result = ht.get(get);
2515     assertSingleResult(result, row, family, qualifier, stamp, value);
2516   }
2517 
2518   private void getVersionAndVerifyMissing(HTable ht, byte [] row, byte [] family,
2519       byte [] qualifier, long stamp)
2520   throws Exception {
2521     Get get = new Get(row);
2522     get.addColumn(family, qualifier);
2523     get.setTimeStamp(stamp);
2524     get.setMaxVersions(Integer.MAX_VALUE);
2525     Result result = ht.get(get);
2526     assertEmptyResult(result);
2527   }
2528 
2529   private void scanVersionAndVerify(HTable ht, byte [] row, byte [] family,
2530       byte [] qualifier, long stamp, byte [] value)
2531   throws Exception {
2532     Scan scan = new Scan(row);
2533     scan.addColumn(family, qualifier);
2534     scan.setTimeStamp(stamp);
2535     scan.setMaxVersions(Integer.MAX_VALUE);
2536     Result result = getSingleScanResult(ht, scan);
2537     assertSingleResult(result, row, family, qualifier, stamp, value);
2538   }
2539 
2540   private void scanVersionAndVerifyMissing(HTable ht, byte [] row,
2541       byte [] family, byte [] qualifier, long stamp)
2542   throws Exception {
2543     Scan scan = new Scan(row);
2544     scan.addColumn(family, qualifier);
2545     scan.setTimeStamp(stamp);
2546     scan.setMaxVersions(Integer.MAX_VALUE);
2547     Result result = getSingleScanResult(ht, scan);
2548     assertNullResult(result);
2549   }
2550 
2551   private void getTestNull(HTable ht, byte [] row, byte [] family,
2552       byte [] value)
2553   throws Exception {
2554 
2555     Get get = new Get(row);
2556     get.addColumn(family, null);
2557     Result result = ht.get(get);
2558     assertSingleResult(result, row, family, null, value);
2559 
2560     get = new Get(row);
2561     get.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
2562     result = ht.get(get);
2563     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2564 
2565     get = new Get(row);
2566     get.addFamily(family);
2567     result = ht.get(get);
2568     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2569 
2570     get = new Get(row);
2571     result = ht.get(get);
2572     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2573 
2574   }
2575 
2576   private void scanTestNull(HTable ht, byte [] row, byte [] family,
2577       byte [] value)
2578   throws Exception {
2579 
2580     Scan scan = new Scan();
2581     scan.addColumn(family, null);
2582     Result result = getSingleScanResult(ht, scan);
2583     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2584 
2585     scan = new Scan();
2586     scan.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
2587     result = getSingleScanResult(ht, scan);
2588     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2589 
2590     scan = new Scan();
2591     scan.addFamily(family);
2592     result = getSingleScanResult(ht, scan);
2593     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2594 
2595     scan = new Scan();
2596     result = getSingleScanResult(ht, scan);
2597     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2598 
2599   }
2600 
2601   private void singleRowGetTest(HTable ht, byte [][] ROWS, byte [][] FAMILIES,
2602       byte [][] QUALIFIERS, byte [][] VALUES)
2603   throws Exception {
2604 
2605     // Single column from memstore
2606     Get get = new Get(ROWS[0]);
2607     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2608     Result result = ht.get(get);
2609     assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]);
2610 
2611     // Single column from storefile
2612     get = new Get(ROWS[0]);
2613     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
2614     result = ht.get(get);
2615     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
2616 
2617     // Single column from storefile, family match
2618     get = new Get(ROWS[0]);
2619     get.addFamily(FAMILIES[7]);
2620     result = ht.get(get);
2621     assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]);
2622 
2623     // Two columns, one from memstore one from storefile, same family,
2624     // wildcard match
2625     get = new Get(ROWS[0]);
2626     get.addFamily(FAMILIES[4]);
2627     result = ht.get(get);
2628     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2629         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2630 
2631     // Two columns, one from memstore one from storefile, same family,
2632     // explicit match
2633     get = new Get(ROWS[0]);
2634     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2635     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
2636     result = ht.get(get);
2637     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2638         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2639 
2640     // Three column, one from memstore two from storefile, different families,
2641     // wildcard match
2642     get = new Get(ROWS[0]);
2643     get.addFamily(FAMILIES[4]);
2644     get.addFamily(FAMILIES[7]);
2645     result = ht.get(get);
2646     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2647         new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} });
2648 
2649     // Multiple columns from everywhere storefile, many family, wildcard
2650     get = new Get(ROWS[0]);
2651     get.addFamily(FAMILIES[2]);
2652     get.addFamily(FAMILIES[4]);
2653     get.addFamily(FAMILIES[6]);
2654     get.addFamily(FAMILIES[7]);
2655     result = ht.get(get);
2656     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2657         new int [][] {
2658           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2659     });
2660 
2661     // Multiple columns from everywhere storefile, many family, wildcard
2662     get = new Get(ROWS[0]);
2663     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
2664     get.addColumn(FAMILIES[2], QUALIFIERS[4]);
2665     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2666     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
2667     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
2668     get.addColumn(FAMILIES[6], QUALIFIERS[7]);
2669     get.addColumn(FAMILIES[7], QUALIFIERS[7]);
2670     get.addColumn(FAMILIES[7], QUALIFIERS[8]);
2671     result = ht.get(get);
2672     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2673         new int [][] {
2674           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2675     });
2676 
2677     // Everything
2678     get = new Get(ROWS[0]);
2679     result = ht.get(get);
2680     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2681         new int [][] {
2682           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0}
2683     });
2684 
2685     // Get around inserted columns
2686 
2687     get = new Get(ROWS[1]);
2688     result = ht.get(get);
2689     assertEmptyResult(result);
2690 
2691     get = new Get(ROWS[0]);
2692     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
2693     get.addColumn(FAMILIES[2], QUALIFIERS[3]);
2694     result = ht.get(get);
2695     assertEmptyResult(result);
2696 
2697   }
2698 
2699   private void singleRowScanTest(HTable ht, byte [][] ROWS, byte [][] FAMILIES,
2700       byte [][] QUALIFIERS, byte [][] VALUES)
2701   throws Exception {
2702 
2703     // Single column from memstore
2704     Scan scan = new Scan();
2705     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
2706     Result result = getSingleScanResult(ht, scan);
2707     assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]);
2708 
2709     // Single column from storefile
2710     scan = new Scan();
2711     scan.addColumn(FAMILIES[2], QUALIFIERS[2]);
2712     result = getSingleScanResult(ht, scan);
2713     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
2714 
2715     // Single column from storefile, family match
2716     scan = new Scan();
2717     scan.addFamily(FAMILIES[7]);
2718     result = getSingleScanResult(ht, scan);
2719     assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]);
2720 
2721     // Two columns, one from memstore one from storefile, same family,
2722     // wildcard match
2723     scan = new Scan();
2724     scan.addFamily(FAMILIES[4]);
2725     result = getSingleScanResult(ht, scan);
2726     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2727         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2728 
2729     // Two columns, one from memstore one from storefile, same family,
2730     // explicit match
2731     scan = new Scan();
2732     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
2733     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
2734     result = getSingleScanResult(ht, scan);
2735     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2736         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2737 
2738     // Three column, one from memstore two from storefile, different families,
2739     // wildcard match
2740     scan = new Scan();
2741     scan.addFamily(FAMILIES[4]);
2742     scan.addFamily(FAMILIES[7]);
2743     result = getSingleScanResult(ht, scan);
2744     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2745         new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} });
2746 
2747     // Multiple columns from everywhere storefile, many family, wildcard
2748     scan = new Scan();
2749     scan.addFamily(FAMILIES[2]);
2750     scan.addFamily(FAMILIES[4]);
2751     scan.addFamily(FAMILIES[6]);
2752     scan.addFamily(FAMILIES[7]);
2753     result = getSingleScanResult(ht, scan);
2754     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2755         new int [][] {
2756           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2757     });
2758 
2759     // Multiple columns from everywhere storefile, many family, wildcard
2760     scan = new Scan();
2761     scan.addColumn(FAMILIES[2], QUALIFIERS[2]);
2762     scan.addColumn(FAMILIES[2], QUALIFIERS[4]);
2763     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
2764     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
2765     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
2766     scan.addColumn(FAMILIES[6], QUALIFIERS[7]);
2767     scan.addColumn(FAMILIES[7], QUALIFIERS[7]);
2768     scan.addColumn(FAMILIES[7], QUALIFIERS[8]);
2769     result = getSingleScanResult(ht, scan);
2770     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2771         new int [][] {
2772           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2773     });
2774 
2775     // Everything
2776     scan = new Scan();
2777     result = getSingleScanResult(ht, scan);
2778     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2779         new int [][] {
2780           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0}
2781     });
2782 
2783     // Scan around inserted columns
2784 
2785     scan = new Scan(ROWS[1]);
2786     result = getSingleScanResult(ht, scan);
2787     assertNullResult(result);
2788 
2789     scan = new Scan();
2790     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
2791     scan.addColumn(FAMILIES[2], QUALIFIERS[3]);
2792     result = getSingleScanResult(ht, scan);
2793     assertNullResult(result);
2794   }
2795 
2796   /**
2797    * Verify a single column using gets.
2798    * Expects family and qualifier arrays to be valid for at least
2799    * the range:  idx-2 < idx < idx+2
2800    */
2801   private void getVerifySingleColumn(HTable ht,
2802       byte [][] ROWS, int ROWIDX,
2803       byte [][] FAMILIES, int FAMILYIDX,
2804       byte [][] QUALIFIERS, int QUALIFIERIDX,
2805       byte [][] VALUES, int VALUEIDX)
2806   throws Exception {
2807 
2808     Get get = new Get(ROWS[ROWIDX]);
2809     Result result = ht.get(get);
2810     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2811         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2812 
2813     get = new Get(ROWS[ROWIDX]);
2814     get.addFamily(FAMILIES[FAMILYIDX]);
2815     result = ht.get(get);
2816     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2817         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2818 
2819     get = new Get(ROWS[ROWIDX]);
2820     get.addFamily(FAMILIES[FAMILYIDX-2]);
2821     get.addFamily(FAMILIES[FAMILYIDX]);
2822     get.addFamily(FAMILIES[FAMILYIDX+2]);
2823     result = ht.get(get);
2824     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2825         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2826 
2827     get = new Get(ROWS[ROWIDX]);
2828     get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[0]);
2829     result = ht.get(get);
2830     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2831         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2832 
2833     get = new Get(ROWS[ROWIDX]);
2834     get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[1]);
2835     get.addFamily(FAMILIES[FAMILYIDX]);
2836     result = ht.get(get);
2837     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2838         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2839 
2840     get = new Get(ROWS[ROWIDX]);
2841     get.addFamily(FAMILIES[FAMILYIDX]);
2842     get.addColumn(FAMILIES[FAMILYIDX+1], QUALIFIERS[1]);
2843     get.addColumn(FAMILIES[FAMILYIDX-2], QUALIFIERS[1]);
2844     get.addFamily(FAMILIES[FAMILYIDX-1]);
2845     get.addFamily(FAMILIES[FAMILYIDX+2]);
2846     result = ht.get(get);
2847     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2848         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2849 
2850   }
2851 
2852 
2853   /**
2854    * Verify a single column using scanners.
2855    * Expects family and qualifier arrays to be valid for at least
2856    * the range:  idx-2 to idx+2
2857    * Expects row array to be valid for at least idx to idx+2
2858    */
2859   private void scanVerifySingleColumn(HTable ht,
2860       byte [][] ROWS, int ROWIDX,
2861       byte [][] FAMILIES, int FAMILYIDX,
2862       byte [][] QUALIFIERS, int QUALIFIERIDX,
2863       byte [][] VALUES, int VALUEIDX)
2864   throws Exception {
2865 
2866     Scan scan = new Scan();
2867     Result result = getSingleScanResult(ht, scan);
2868     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2869         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2870 
2871     scan = new Scan(ROWS[ROWIDX]);
2872     result = getSingleScanResult(ht, scan);
2873     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2874         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2875 
2876     scan = new Scan(ROWS[ROWIDX], ROWS[ROWIDX+1]);
2877     result = getSingleScanResult(ht, scan);
2878     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2879         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2880 
2881     scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX+1]);
2882     result = getSingleScanResult(ht, scan);
2883     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2884         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2885 
2886     scan = new Scan();
2887     scan.addFamily(FAMILIES[FAMILYIDX]);
2888     result = getSingleScanResult(ht, scan);
2889     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2890         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2891 
2892     scan = new Scan();
2893     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]);
2894     result = getSingleScanResult(ht, scan);
2895     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2896         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2897 
2898     scan = new Scan();
2899     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]);
2900     scan.addFamily(FAMILIES[FAMILYIDX]);
2901     result = getSingleScanResult(ht, scan);
2902     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2903         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2904 
2905     scan = new Scan();
2906     scan.addColumn(FAMILIES[FAMILYIDX-1], QUALIFIERS[QUALIFIERIDX+1]);
2907     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]);
2908     scan.addFamily(FAMILIES[FAMILYIDX+1]);
2909     result = getSingleScanResult(ht, scan);
2910     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2911         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2912 
2913   }
2914 
2915   /**
2916    * Verify we do not read any values by accident around a single column
2917    * Same requirements as getVerifySingleColumn
2918    */
2919   private void getVerifySingleEmpty(HTable ht,
2920       byte [][] ROWS, int ROWIDX,
2921       byte [][] FAMILIES, int FAMILYIDX,
2922       byte [][] QUALIFIERS, int QUALIFIERIDX)
2923   throws Exception {
2924 
2925     Get get = new Get(ROWS[ROWIDX]);
2926     get.addFamily(FAMILIES[4]);
2927     get.addColumn(FAMILIES[4], QUALIFIERS[1]);
2928     Result result = ht.get(get);
2929     assertEmptyResult(result);
2930 
2931     get = new Get(ROWS[ROWIDX]);
2932     get.addFamily(FAMILIES[4]);
2933     get.addColumn(FAMILIES[4], QUALIFIERS[2]);
2934     result = ht.get(get);
2935     assertEmptyResult(result);
2936 
2937     get = new Get(ROWS[ROWIDX]);
2938     get.addFamily(FAMILIES[3]);
2939     get.addColumn(FAMILIES[4], QUALIFIERS[2]);
2940     get.addFamily(FAMILIES[5]);
2941     result = ht.get(get);
2942     assertEmptyResult(result);
2943 
2944     get = new Get(ROWS[ROWIDX+1]);
2945     result = ht.get(get);
2946     assertEmptyResult(result);
2947 
2948   }
2949 
2950   private void scanVerifySingleEmpty(HTable ht,
2951       byte [][] ROWS, int ROWIDX,
2952       byte [][] FAMILIES, int FAMILYIDX,
2953       byte [][] QUALIFIERS, int QUALIFIERIDX)
2954   throws Exception {
2955 
2956     Scan scan = new Scan(ROWS[ROWIDX+1]);
2957     Result result = getSingleScanResult(ht, scan);
2958     assertNullResult(result);
2959 
2960     scan = new Scan(ROWS[ROWIDX+1],ROWS[ROWIDX+2]);
2961     result = getSingleScanResult(ht, scan);
2962     assertNullResult(result);
2963 
2964     scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX]);
2965     result = getSingleScanResult(ht, scan);
2966     assertNullResult(result);
2967 
2968     scan = new Scan();
2969     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]);
2970     scan.addFamily(FAMILIES[FAMILYIDX-1]);
2971     result = getSingleScanResult(ht, scan);
2972     assertNullResult(result);
2973 
2974   }
2975 
2976   //
2977   // Verifiers
2978   //
2979 
2980   private void assertKey(KeyValue key, byte [] row, byte [] family,
2981       byte [] qualifier, byte [] value)
2982   throws Exception {
2983     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
2984         "Got row [" + Bytes.toString(key.getRow()) +"]",
2985         equals(row, key.getRow()));
2986     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
2987         "Got family [" + Bytes.toString(key.getFamily()) + "]",
2988         equals(family, key.getFamily()));
2989     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
2990         "Got qualifier [" + Bytes.toString(key.getQualifier()) + "]",
2991         equals(qualifier, key.getQualifier()));
2992     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
2993         "Got value [" + Bytes.toString(key.getValue()) + "]",
2994         equals(value, key.getValue()));
2995   }
2996 
2997   private void assertIncrementKey(KeyValue key, byte [] row, byte [] family,
2998       byte [] qualifier, long value)
2999   throws Exception {
3000     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3001         "Got row [" + Bytes.toString(key.getRow()) +"]",
3002         equals(row, key.getRow()));
3003     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3004         "Got family [" + Bytes.toString(key.getFamily()) + "]",
3005         equals(family, key.getFamily()));
3006     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3007         "Got qualifier [" + Bytes.toString(key.getQualifier()) + "]",
3008         equals(qualifier, key.getQualifier()));
3009     assertTrue("Expected value [" + value + "] " +
3010         "Got value [" + Bytes.toLong(key.getValue()) + "]",
3011         Bytes.toLong(key.getValue()) == value);
3012   }
3013 
3014   private void assertNumKeys(Result result, int n) throws Exception {
3015     assertTrue("Expected " + n + " keys but got " + result.size(),
3016         result.size() == n);
3017   }
3018 
3019   private void assertNResult(Result result, byte [] row,
3020       byte [][] families, byte [][] qualifiers, byte [][] values,
3021       int [][] idxs)
3022   throws Exception {
3023     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3024         "Got row [" + Bytes.toString(result.getRow()) +"]",
3025         equals(row, result.getRow()));
3026     assertTrue("Expected " + idxs.length + " keys but result contains "
3027         + result.size(), result.size() == idxs.length);
3028 
3029     KeyValue [] keys = result.raw();
3030 
3031     for(int i=0;i<keys.length;i++) {
3032       byte [] family = families[idxs[i][0]];
3033       byte [] qualifier = qualifiers[idxs[i][1]];
3034       byte [] value = values[idxs[i][2]];
3035       KeyValue key = keys[i];
3036 
3037       assertTrue("(" + i + ") Expected family [" + Bytes.toString(family)
3038           + "] " + "Got family [" + Bytes.toString(key.getFamily()) + "]",
3039           equals(family, key.getFamily()));
3040       assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier)
3041           + "] " + "Got qualifier [" + Bytes.toString(key.getQualifier()) + "]",
3042           equals(qualifier, key.getQualifier()));
3043       assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] "
3044           + "Got value [" + Bytes.toString(key.getValue()) + "]",
3045           equals(value, key.getValue()));
3046     }
3047   }
3048 
3049   private void assertNResult(Result result, byte [] row,
3050       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
3051       int start, int end)
3052   throws IOException {
3053     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3054         "Got row [" + Bytes.toString(result.getRow()) +"]",
3055         equals(row, result.getRow()));
3056     int expectedResults = end - start + 1;
3057     assertEquals(expectedResults, result.size());
3058 
3059     KeyValue [] keys = result.raw();
3060 
3061     for (int i=0; i<keys.length; i++) {
3062       byte [] value = values[end-i];
3063       long ts = stamps[end-i];
3064       KeyValue key = keys[i];
3065 
3066       assertTrue("(" + i + ") Expected family [" + Bytes.toString(family)
3067           + "] " + "Got family [" + Bytes.toString(key.getFamily()) + "]",
3068           equals(family, key.getFamily()));
3069       assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier)
3070           + "] " + "Got qualifier [" + Bytes.toString(key.getQualifier()) + "]",
3071           equals(qualifier, key.getQualifier()));
3072       assertTrue("Expected ts [" + ts + "] " +
3073           "Got ts [" + key.getTimestamp() + "]", ts == key.getTimestamp());
3074       assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] "
3075           + "Got value [" + Bytes.toString(key.getValue()) + "]",
3076           equals(value, key.getValue()));
3077     }
3078   }
3079 
3080   /**
3081    * Validate that result contains two specified keys, exactly.
3082    * It is assumed key A sorts before key B.
3083    */
3084   private void assertDoubleResult(Result result, byte [] row,
3085       byte [] familyA, byte [] qualifierA, byte [] valueA,
3086       byte [] familyB, byte [] qualifierB, byte [] valueB)
3087   throws Exception {
3088     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3089         "Got row [" + Bytes.toString(result.getRow()) +"]",
3090         equals(row, result.getRow()));
3091     assertTrue("Expected two keys but result contains " + result.size(),
3092         result.size() == 2);
3093     KeyValue [] kv = result.raw();
3094     KeyValue kvA = kv[0];
3095     assertTrue("(A) Expected family [" + Bytes.toString(familyA) + "] " +
3096         "Got family [" + Bytes.toString(kvA.getFamily()) + "]",
3097         equals(familyA, kvA.getFamily()));
3098     assertTrue("(A) Expected qualifier [" + Bytes.toString(qualifierA) + "] " +
3099         "Got qualifier [" + Bytes.toString(kvA.getQualifier()) + "]",
3100         equals(qualifierA, kvA.getQualifier()));
3101     assertTrue("(A) Expected value [" + Bytes.toString(valueA) + "] " +
3102         "Got value [" + Bytes.toString(kvA.getValue()) + "]",
3103         equals(valueA, kvA.getValue()));
3104     KeyValue kvB = kv[1];
3105     assertTrue("(B) Expected family [" + Bytes.toString(familyB) + "] " +
3106         "Got family [" + Bytes.toString(kvB.getFamily()) + "]",
3107         equals(familyB, kvB.getFamily()));
3108     assertTrue("(B) Expected qualifier [" + Bytes.toString(qualifierB) + "] " +
3109         "Got qualifier [" + Bytes.toString(kvB.getQualifier()) + "]",
3110         equals(qualifierB, kvB.getQualifier()));
3111     assertTrue("(B) Expected value [" + Bytes.toString(valueB) + "] " +
3112         "Got value [" + Bytes.toString(kvB.getValue()) + "]",
3113         equals(valueB, kvB.getValue()));
3114   }
3115 
3116   private void assertSingleResult(Result result, byte [] row, byte [] family,
3117       byte [] qualifier, byte [] value)
3118   throws Exception {
3119     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3120         "Got row [" + Bytes.toString(result.getRow()) +"]",
3121         equals(row, result.getRow()));
3122     assertTrue("Expected a single key but result contains " + result.size(),
3123         result.size() == 1);
3124     KeyValue kv = result.raw()[0];
3125     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3126         "Got family [" + Bytes.toString(kv.getFamily()) + "]",
3127         equals(family, kv.getFamily()));
3128     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3129         "Got qualifier [" + Bytes.toString(kv.getQualifier()) + "]",
3130         equals(qualifier, kv.getQualifier()));
3131     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
3132         "Got value [" + Bytes.toString(kv.getValue()) + "]",
3133         equals(value, kv.getValue()));
3134   }
3135 
3136   private void assertSingleResult(Result result, byte [] row, byte [] family,
3137       byte [] qualifier, long ts, byte [] value)
3138   throws Exception {
3139     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3140         "Got row [" + Bytes.toString(result.getRow()) +"]",
3141         equals(row, result.getRow()));
3142     assertTrue("Expected a single key but result contains " + result.size(),
3143         result.size() == 1);
3144     KeyValue kv = result.raw()[0];
3145     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3146         "Got family [" + Bytes.toString(kv.getFamily()) + "]",
3147         equals(family, kv.getFamily()));
3148     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3149         "Got qualifier [" + Bytes.toString(kv.getQualifier()) + "]",
3150         equals(qualifier, kv.getQualifier()));
3151     assertTrue("Expected ts [" + ts + "] " +
3152         "Got ts [" + kv.getTimestamp() + "]", ts == kv.getTimestamp());
3153     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
3154         "Got value [" + Bytes.toString(kv.getValue()) + "]",
3155         equals(value, kv.getValue()));
3156   }
3157 
3158   private void assertEmptyResult(Result result) throws Exception {
3159     assertTrue("expected an empty result but result contains " +
3160         result.size() + " keys", result.isEmpty());
3161   }
3162 
3163   private void assertNullResult(Result result) throws Exception {
3164     assertTrue("expected null result but received a non-null result",
3165         result == null);
3166   }
3167 
3168   //
3169   // Helpers
3170   //
3171 
3172   private Result getSingleScanResult(HTable ht, Scan scan) throws IOException {
3173     ResultScanner scanner = ht.getScanner(scan);
3174     Result result = scanner.next();
3175     scanner.close();
3176     return result;
3177   }
3178 
3179   private byte [][] makeNAscii(byte [] base, int n) {
3180     if(n > 256) {
3181       return makeNBig(base, n);
3182     }
3183     byte [][] ret = new byte[n][];
3184     for(int i=0;i<n;i++) {
3185       byte [] tail = Bytes.toBytes(Integer.toString(i));
3186       ret[i] = Bytes.add(base, tail);
3187     }
3188     return ret;
3189   }
3190 
3191   private byte [][] makeN(byte [] base, int n) {
3192     if (n > 256) {
3193       return makeNBig(base, n);
3194     }
3195     byte [][] ret = new byte[n][];
3196     for(int i=0;i<n;i++) {
3197       ret[i] = Bytes.add(base, new byte[]{(byte)i});
3198     }
3199     return ret;
3200   }
3201 
3202   private byte [][] makeNBig(byte [] base, int n) {
3203     byte [][] ret = new byte[n][];
3204     for(int i=0;i<n;i++) {
3205       int byteA = (i % 256);
3206       int byteB = (i >> 8);
3207       ret[i] = Bytes.add(base, new byte[]{(byte)byteB,(byte)byteA});
3208     }
3209     return ret;
3210   }
3211 
3212   private long [] makeStamps(int n) {
3213     long [] stamps = new long[n];
3214     for(int i=0;i<n;i++) stamps[i] = i+1;
3215     return stamps;
3216   }
3217 
3218   private boolean equals(byte [] left, byte [] right) {
3219     if (left == null && right == null) return true;
3220     if (left == null && right.length == 0) return true;
3221     if (right == null && left.length == 0) return true;
3222     return Bytes.equals(left, right);
3223   }
3224 
3225   @Test
3226   public void testDuplicateVersions() throws Exception {
3227     byte [] TABLE = Bytes.toBytes("testDuplicateVersions");
3228 
3229     long [] STAMPS = makeStamps(20);
3230     byte [][] VALUES = makeNAscii(VALUE, 20);
3231 
3232     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3233 
3234     // Insert 4 versions of same column
3235     Put put = new Put(ROW);
3236     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3237     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3238     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3239     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3240     ht.put(put);
3241 
3242     // Verify we can get each one properly
3243     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3244     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3245     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3246     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3247     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3248     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3249     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3250     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3251 
3252     // Verify we don't accidentally get others
3253     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3254     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3255     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3256     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3257     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3258     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3259 
3260     // Ensure maxVersions in query is respected
3261     Get get = new Get(ROW);
3262     get.addColumn(FAMILY, QUALIFIER);
3263     get.setMaxVersions(2);
3264     Result result = ht.get(get);
3265     assertNResult(result, ROW, FAMILY, QUALIFIER,
3266         new long [] {STAMPS[4], STAMPS[5]},
3267         new byte[][] {VALUES[4], VALUES[5]},
3268         0, 1);
3269 
3270     Scan scan = new Scan(ROW);
3271     scan.addColumn(FAMILY, QUALIFIER);
3272     scan.setMaxVersions(2);
3273     result = getSingleScanResult(ht, scan);
3274     assertNResult(result, ROW, FAMILY, QUALIFIER,
3275         new long [] {STAMPS[4], STAMPS[5]},
3276         new byte[][] {VALUES[4], VALUES[5]},
3277         0, 1);
3278 
3279     // Flush and redo
3280 
3281     TEST_UTIL.flush();
3282 
3283     // Verify we can get each one properly
3284     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3285     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3286     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3287     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3288     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3289     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3290     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3291     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3292 
3293     // Verify we don't accidentally get others
3294     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3295     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3296     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3297     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3298     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3299     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3300 
3301     // Ensure maxVersions in query is respected
3302     get = new Get(ROW);
3303     get.addColumn(FAMILY, QUALIFIER);
3304     get.setMaxVersions(2);
3305     result = ht.get(get);
3306     assertNResult(result, ROW, FAMILY, QUALIFIER,
3307         new long [] {STAMPS[4], STAMPS[5]},
3308         new byte[][] {VALUES[4], VALUES[5]},
3309         0, 1);
3310 
3311     scan = new Scan(ROW);
3312     scan.addColumn(FAMILY, QUALIFIER);
3313     scan.setMaxVersions(2);
3314     result = getSingleScanResult(ht, scan);
3315     assertNResult(result, ROW, FAMILY, QUALIFIER,
3316         new long [] {STAMPS[4], STAMPS[5]},
3317         new byte[][] {VALUES[4], VALUES[5]},
3318         0, 1);
3319 
3320 
3321     // Add some memstore and retest
3322 
3323     // Insert 4 more versions of same column and a dupe
3324     put = new Put(ROW);
3325     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
3326     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3327     put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
3328     put.add(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3329     put.add(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
3330     ht.put(put);
3331 
3332     // Ensure maxVersions in query is respected
3333     get = new Get(ROW);
3334     get.addColumn(FAMILY, QUALIFIER);
3335     get.setMaxVersions(7);
3336     result = ht.get(get);
3337     assertNResult(result, ROW, FAMILY, QUALIFIER,
3338         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3339         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3340         0, 6);
3341 
3342     scan = new Scan(ROW);
3343     scan.addColumn(FAMILY, QUALIFIER);
3344     scan.setMaxVersions(7);
3345     result = getSingleScanResult(ht, scan);
3346     assertNResult(result, ROW, FAMILY, QUALIFIER,
3347         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3348         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3349         0, 6);
3350 
3351     get = new Get(ROW);
3352     get.setMaxVersions(7);
3353     result = ht.get(get);
3354     assertNResult(result, ROW, FAMILY, QUALIFIER,
3355         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3356         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3357         0, 6);
3358 
3359     scan = new Scan(ROW);
3360     scan.setMaxVersions(7);
3361     result = getSingleScanResult(ht, scan);
3362     assertNResult(result, ROW, FAMILY, QUALIFIER,
3363         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3364         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3365         0, 6);
3366 
3367     // Verify we can get each one properly
3368     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3369     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3370     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3371     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3372     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3373     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3374     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3375     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3376 
3377     // Verify we don't accidentally get others
3378     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3379     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
3380     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3381     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
3382 
3383     // Ensure maxVersions of table is respected
3384 
3385     TEST_UTIL.flush();
3386 
3387     // Insert 4 more versions of same column and a dupe
3388     put = new Put(ROW);
3389     put.add(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
3390     put.add(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
3391     put.add(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
3392     put.add(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
3393     ht.put(put);
3394 
3395     get = new Get(ROW);
3396     get.addColumn(FAMILY, QUALIFIER);
3397     get.setMaxVersions(Integer.MAX_VALUE);
3398     result = ht.get(get);
3399     assertNResult(result, ROW, FAMILY, QUALIFIER,
3400         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
3401         new byte[][] {VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
3402         0, 9);
3403 
3404     scan = new Scan(ROW);
3405     scan.addColumn(FAMILY, QUALIFIER);
3406     scan.setMaxVersions(Integer.MAX_VALUE);
3407     result = getSingleScanResult(ht, scan);
3408     assertNResult(result, ROW, FAMILY, QUALIFIER,
3409         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
3410         new byte[][] {VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
3411         0, 9);
3412 
3413     // Delete a version in the memstore and a version in a storefile
3414     Delete delete = new Delete(ROW);
3415     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[11]);
3416     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[7]);
3417     ht.delete(delete);
3418 
3419     // Test that it's gone
3420     get = new Get(ROW);
3421     get.addColumn(FAMILY, QUALIFIER);
3422     get.setMaxVersions(Integer.MAX_VALUE);
3423     result = ht.get(get);
3424     assertNResult(result, ROW, FAMILY, QUALIFIER,
3425         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
3426         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
3427         0, 9);
3428 
3429     scan = new Scan(ROW);
3430     scan.addColumn(FAMILY, QUALIFIER);
3431     scan.setMaxVersions(Integer.MAX_VALUE);
3432     result = getSingleScanResult(ht, scan);
3433     assertNResult(result, ROW, FAMILY, QUALIFIER,
3434         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
3435         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
3436         0, 9);
3437   }
3438 
3439   @Test
3440   public void testUpdates() throws Exception {
3441 
3442     byte [] TABLE = Bytes.toBytes("testUpdates");
3443     HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3444 
3445     // Write a column with values at timestamp 1, 2 and 3
3446     byte[] row = Bytes.toBytes("row1");
3447     byte[] qualifier = Bytes.toBytes("myCol");
3448     Put put = new Put(row);
3449     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3450     hTable.put(put);
3451 
3452     put = new Put(row);
3453     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3454     hTable.put(put);
3455 
3456     put = new Put(row);
3457     put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3458     hTable.put(put);
3459 
3460     Get get = new Get(row);
3461     get.addColumn(FAMILY, qualifier);
3462     get.setMaxVersions();
3463 
3464     // Check that the column indeed has the right values at timestamps 1 and
3465     // 2
3466     Result result = hTable.get(get);
3467     NavigableMap<Long, byte[]> navigableMap =
3468         result.getMap().get(FAMILY).get(qualifier);
3469     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3470     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3471 
3472     // Update the value at timestamp 1
3473     put = new Put(row);
3474     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3475     hTable.put(put);
3476 
3477     // Update the value at timestamp 2
3478     put = new Put(row);
3479     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3480     hTable.put(put);
3481 
3482     // Check that the values at timestamp 2 and 1 got updated
3483     result = hTable.get(get);
3484     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3485     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3486     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3487   }
3488 
3489   @Test
3490   public void testUpdatesWithMajorCompaction() throws Exception {
3491 
3492     String tableName = "testUpdatesWithMajorCompaction";
3493     byte [] TABLE = Bytes.toBytes(tableName);
3494     HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3495     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
3496 
3497     // Write a column with values at timestamp 1, 2 and 3
3498     byte[] row = Bytes.toBytes("row2");
3499     byte[] qualifier = Bytes.toBytes("myCol");
3500     Put put = new Put(row);
3501     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3502     hTable.put(put);
3503 
3504     put = new Put(row);
3505     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3506     hTable.put(put);
3507 
3508     put = new Put(row);
3509     put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3510     hTable.put(put);
3511 
3512     Get get = new Get(row);
3513     get.addColumn(FAMILY, qualifier);
3514     get.setMaxVersions();
3515 
3516     // Check that the column indeed has the right values at timestamps 1 and
3517     // 2
3518     Result result = hTable.get(get);
3519     NavigableMap<Long, byte[]> navigableMap =
3520         result.getMap().get(FAMILY).get(qualifier);
3521     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3522     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3523 
3524     // Trigger a major compaction
3525     admin.flush(tableName);
3526     admin.majorCompact(tableName);
3527     Thread.sleep(6000);
3528 
3529     // Update the value at timestamp 1
3530     put = new Put(row);
3531     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3532     hTable.put(put);
3533 
3534     // Update the value at timestamp 2
3535     put = new Put(row);
3536     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3537     hTable.put(put);
3538 
3539     // Trigger a major compaction
3540     admin.flush(tableName);
3541     admin.majorCompact(tableName);
3542     Thread.sleep(6000);
3543 
3544     // Check that the values at timestamp 2 and 1 got updated
3545     result = hTable.get(get);
3546     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3547     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3548     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3549   }
3550 
3551   @Test
3552   public void testMajorCompactionBetweenTwoUpdates() throws Exception {
3553 
3554     String tableName = "testMajorCompactionBetweenTwoUpdates";
3555     byte [] TABLE = Bytes.toBytes(tableName);
3556     HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3557     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
3558 
3559     // Write a column with values at timestamp 1, 2 and 3
3560     byte[] row = Bytes.toBytes("row3");
3561     byte[] qualifier = Bytes.toBytes("myCol");
3562     Put put = new Put(row);
3563     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3564     hTable.put(put);
3565 
3566     put = new Put(row);
3567     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3568     hTable.put(put);
3569 
3570     put = new Put(row);
3571     put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3572     hTable.put(put);
3573 
3574     Get get = new Get(row);
3575     get.addColumn(FAMILY, qualifier);
3576     get.setMaxVersions();
3577 
3578     // Check that the column indeed has the right values at timestamps 1 and
3579     // 2
3580     Result result = hTable.get(get);
3581     NavigableMap<Long, byte[]> navigableMap =
3582         result.getMap().get(FAMILY).get(qualifier);
3583     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3584     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3585 
3586     // Trigger a major compaction
3587     admin.flush(tableName);
3588     admin.majorCompact(tableName);
3589     Thread.sleep(6000);
3590 
3591     // Update the value at timestamp 1
3592     put = new Put(row);
3593     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3594     hTable.put(put);
3595 
3596     // Trigger a major compaction
3597     admin.flush(tableName);
3598     admin.majorCompact(tableName);
3599     Thread.sleep(6000);
3600 
3601     // Update the value at timestamp 2
3602     put = new Put(row);
3603     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3604     hTable.put(put);
3605 
3606     // Trigger a major compaction
3607     admin.flush(tableName);
3608     admin.majorCompact(tableName);
3609     Thread.sleep(6000);
3610 
3611     // Check that the values at timestamp 2 and 1 got updated
3612     result = hTable.get(get);
3613     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3614 
3615     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3616     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3617   }
3618 
3619   @Test
3620   public void testGet_EmptyTable() throws IOException {
3621     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testGet_EmptyTable"), FAMILY);
3622     Get get = new Get(ROW);
3623     get.addFamily(FAMILY);
3624     Result r = table.get(get);
3625     assertTrue(r.isEmpty());
3626   }
3627 
3628   @Test
3629   public void testGet_NonExistentRow() throws IOException {
3630     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testGet_NonExistentRow"), FAMILY);
3631     Put put = new Put(ROW);
3632     put.add(FAMILY, QUALIFIER, VALUE);
3633     table.put(put);
3634     LOG.info("Row put");
3635 
3636     Get get = new Get(ROW);
3637     get.addFamily(FAMILY);
3638     Result r = table.get(get);
3639     assertFalse(r.isEmpty());
3640     System.out.println("Row retrieved successfully");
3641 
3642     byte [] missingrow = Bytes.toBytes("missingrow");
3643     get = new Get(missingrow);
3644     get.addFamily(FAMILY);
3645     r = table.get(get);
3646     assertTrue(r.isEmpty());
3647     LOG.info("Row missing as it should be");
3648   }
3649 
3650   @Test
3651   public void testPut() throws IOException {
3652     final byte [] CONTENTS_FAMILY = Bytes.toBytes("contents");
3653     final byte [] SMALL_FAMILY = Bytes.toBytes("smallfam");
3654     final byte [] row1 = Bytes.toBytes("row1");
3655     final byte [] row2 = Bytes.toBytes("row2");
3656     final byte [] value = Bytes.toBytes("abcd");
3657     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testPut"),
3658       new byte [][] {CONTENTS_FAMILY, SMALL_FAMILY});
3659     Put put = new Put(row1);
3660     put.add(CONTENTS_FAMILY, null, value);
3661     table.put(put);
3662 
3663     put = new Put(row2);
3664     put.add(CONTENTS_FAMILY, null, value);
3665 
3666     assertEquals(put.size(), 1);
3667     assertEquals(put.getFamilyMap().get(CONTENTS_FAMILY).size(), 1);
3668 
3669     KeyValue kv = put.getFamilyMap().get(CONTENTS_FAMILY).get(0);
3670 
3671     assertTrue(Bytes.equals(kv.getFamily(), CONTENTS_FAMILY));
3672     // will it return null or an empty byte array?
3673     assertTrue(Bytes.equals(kv.getQualifier(), new byte[0]));
3674 
3675     assertTrue(Bytes.equals(kv.getValue(), value));
3676 
3677     table.put(put);
3678 
3679     Scan scan = new Scan();
3680     scan.addColumn(CONTENTS_FAMILY, null);
3681     ResultScanner scanner = table.getScanner(scan);
3682     for (Result r : scanner) {
3683       for(KeyValue key : r.raw()) {
3684         System.out.println(Bytes.toString(r.getRow()) + ": " + key.toString());
3685       }
3686     }
3687   }
3688 
3689   @Test
3690   public void testPutNoCF() throws IOException {
3691     final byte[] BAD_FAM = Bytes.toBytes("BAD_CF");
3692     final byte[] VAL = Bytes.toBytes(100);
3693     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testPutNoCF"), new byte[][]{FAMILY});
3694 
3695     boolean caughtNSCFE = false;
3696 
3697     try {
3698       Put p = new Put(ROW);
3699       p.add(BAD_FAM, QUALIFIER, VAL);
3700       table.put(p);
3701     } catch (RetriesExhaustedWithDetailsException e) {
3702       caughtNSCFE = e.getCause(0) instanceof NoSuchColumnFamilyException;
3703     }
3704     assertTrue("Should throw NoSuchColumnFamilyException", caughtNSCFE);
3705 
3706   }
3707 
3708   @Test
3709   public void testRowsPut() throws IOException {
3710     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
3711     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
3712     final int NB_BATCH_ROWS = 10;
3713     final byte[] value = Bytes.toBytes("abcd");
3714     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPut"),
3715       new byte[][] {CONTENTS_FAMILY, SMALL_FAMILY });
3716     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
3717     for (int i = 0; i < NB_BATCH_ROWS; i++) {
3718       byte[] row = Bytes.toBytes("row" + i);
3719       Put put = new Put(row);
3720       put.setWriteToWAL(false);
3721       put.add(CONTENTS_FAMILY, null, value);
3722       rowsUpdate.add(put);
3723     }
3724     table.put(rowsUpdate);
3725     Scan scan = new Scan();
3726     scan.addFamily(CONTENTS_FAMILY);
3727     ResultScanner scanner = table.getScanner(scan);
3728     int nbRows = 0;
3729     for (@SuppressWarnings("unused")
3730     Result row : scanner)
3731       nbRows++;
3732     assertEquals(NB_BATCH_ROWS, nbRows);
3733   }
3734 
3735   @Test
3736   public void testRowsPutBufferedOneFlush() throws IOException {
3737     final byte [] CONTENTS_FAMILY = Bytes.toBytes("contents");
3738     final byte [] SMALL_FAMILY = Bytes.toBytes("smallfam");
3739     final byte [] value = Bytes.toBytes("abcd");
3740     final int NB_BATCH_ROWS = 10;
3741     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPutBufferedOneFlush"),
3742       new byte [][] {CONTENTS_FAMILY, SMALL_FAMILY});
3743     table.setAutoFlush(false);
3744     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
3745     for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
3746       byte[] row = Bytes.toBytes("row" + i);
3747       Put put = new Put(row);
3748       put.setWriteToWAL(false);
3749       put.add(CONTENTS_FAMILY, null, value);
3750       rowsUpdate.add(put);
3751     }
3752     table.put(rowsUpdate);
3753 
3754     Scan scan = new Scan();
3755     scan.addFamily(CONTENTS_FAMILY);
3756     ResultScanner scanner = table.getScanner(scan);
3757     int nbRows = 0;
3758     for (@SuppressWarnings("unused")
3759     Result row : scanner)
3760       nbRows++;
3761     assertEquals(0, nbRows);
3762     scanner.close();
3763 
3764     table.flushCommits();
3765 
3766     scan = new Scan();
3767     scan.addFamily(CONTENTS_FAMILY);
3768     scanner = table.getScanner(scan);
3769     nbRows = 0;
3770     for (@SuppressWarnings("unused")
3771     Result row : scanner)
3772       nbRows++;
3773     assertEquals(NB_BATCH_ROWS * 10, nbRows);
3774   }
3775 
3776   @Test
3777   public void testRowsPutBufferedManyManyFlushes() throws IOException {
3778     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
3779     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
3780     final byte[] value = Bytes.toBytes("abcd");
3781     final int NB_BATCH_ROWS = 10;
3782     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPutBufferedManyManyFlushes"),
3783       new byte[][] {CONTENTS_FAMILY, SMALL_FAMILY });
3784     table.setAutoFlush(false);
3785     table.setWriteBufferSize(10);
3786     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
3787     for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
3788       byte[] row = Bytes.toBytes("row" + i);
3789       Put put = new Put(row);
3790       put.setWriteToWAL(false);
3791       put.add(CONTENTS_FAMILY, null, value);
3792       rowsUpdate.add(put);
3793     }
3794     table.put(rowsUpdate);
3795 
3796     table.flushCommits();
3797 
3798     Scan scan = new Scan();
3799     scan.addFamily(CONTENTS_FAMILY);
3800     ResultScanner scanner = table.getScanner(scan);
3801     int nbRows = 0;
3802     for (@SuppressWarnings("unused")
3803     Result row : scanner)
3804       nbRows++;
3805     assertEquals(NB_BATCH_ROWS * 10, nbRows);
3806   }
3807 
3808   @Test
3809   public void testAddKeyValue() throws IOException {
3810     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
3811     final byte[] value = Bytes.toBytes("abcd");
3812     final byte[] row1 = Bytes.toBytes("row1");
3813     final byte[] row2 = Bytes.toBytes("row2");
3814     byte[] qualifier = Bytes.toBytes("qf1");
3815     Put put = new Put(row1);
3816 
3817     // Adding KeyValue with the same row
3818     KeyValue kv = new KeyValue(row1, CONTENTS_FAMILY, qualifier, value);
3819     boolean ok = true;
3820     try {
3821       put.add(kv);
3822     } catch (IOException e) {
3823       ok = false;
3824     }
3825     assertEquals(true, ok);
3826 
3827     // Adding KeyValue with the different row
3828     kv = new KeyValue(row2, CONTENTS_FAMILY, qualifier, value);
3829     ok = false;
3830     try {
3831       put.add(kv);
3832     } catch (IOException e) {
3833       ok = true;
3834     }
3835     assertEquals(true, ok);
3836   }
3837 
3838   /**
3839    * test for HBASE-737
3840    * @throws IOException
3841    */
3842   @Test
3843   public void testHBase737 () throws IOException {
3844     final byte [] FAM1 = Bytes.toBytes("fam1");
3845     final byte [] FAM2 = Bytes.toBytes("fam2");
3846     // Open table
3847     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testHBase737"),
3848       new byte [][] {FAM1, FAM2});
3849     // Insert some values
3850     Put put = new Put(ROW);
3851     put.add(FAM1, Bytes.toBytes("letters"), Bytes.toBytes("abcdefg"));
3852     table.put(put);
3853     try {
3854       Thread.sleep(1000);
3855     } catch (InterruptedException i) {
3856       //ignore
3857     }
3858 
3859     put = new Put(ROW);
3860     put.add(FAM1, Bytes.toBytes("numbers"), Bytes.toBytes("123456"));
3861     table.put(put);
3862 
3863     try {
3864       Thread.sleep(1000);
3865     } catch (InterruptedException i) {
3866       //ignore
3867     }
3868 
3869     put = new Put(ROW);
3870     put.add(FAM2, Bytes.toBytes("letters"), Bytes.toBytes("hijklmnop"));
3871     table.put(put);
3872 
3873     long times[] = new long[3];
3874 
3875     // First scan the memstore
3876 
3877     Scan scan = new Scan();
3878     scan.addFamily(FAM1);
3879     scan.addFamily(FAM2);
3880     ResultScanner s = table.getScanner(scan);
3881     try {
3882       int index = 0;
3883       Result r = null;
3884       while ((r = s.next()) != null) {
3885         for(KeyValue key : r.raw()) {
3886           times[index++] = key.getTimestamp();
3887         }
3888       }
3889     } finally {
3890       s.close();
3891     }
3892     for (int i = 0; i < times.length - 1; i++) {
3893       for (int j = i + 1; j < times.length; j++) {
3894         assertTrue(times[j] > times[i]);
3895       }
3896     }
3897 
3898     // Flush data to disk and try again
3899     TEST_UTIL.flush();
3900 
3901     // Reset times
3902     for(int i=0;i<times.length;i++) {
3903       times[i] = 0;
3904     }
3905 
3906     try {
3907       Thread.sleep(1000);
3908     } catch (InterruptedException i) {
3909       //ignore
3910     }
3911     scan = new Scan();
3912     scan.addFamily(FAM1);
3913     scan.addFamily(FAM2);
3914     s = table.getScanner(scan);
3915     try {
3916       int index = 0;
3917       Result r = null;
3918       while ((r = s.next()) != null) {
3919         for(KeyValue key : r.raw()) {
3920           times[index++] = key.getTimestamp();
3921         }
3922       }
3923     } finally {
3924       s.close();
3925     }
3926     for (int i = 0; i < times.length - 1; i++) {
3927       for (int j = i + 1; j < times.length; j++) {
3928         assertTrue(times[j] > times[i]);
3929       }
3930     }
3931   }
3932 
3933   @Test
3934   public void testListTables() throws IOException, InterruptedException {
3935     byte [] t1 = Bytes.toBytes("testListTables1");
3936     byte [] t2 = Bytes.toBytes("testListTables2");
3937     byte [] t3 = Bytes.toBytes("testListTables3");
3938     byte [][] tables = new byte[][] { t1, t2, t3 };
3939     for (int i = 0; i < tables.length; i++) {
3940       TEST_UTIL.createTable(tables[i], FAMILY);
3941     }
3942     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
3943     HTableDescriptor[] ts = admin.listTables();
3944     HashSet<HTableDescriptor> result = new HashSet<HTableDescriptor>(ts.length);
3945     for (int i = 0; i < ts.length; i++) {
3946       result.add(ts[i]);
3947     }
3948     int size = result.size();
3949     assertTrue(size >= tables.length);
3950     for (int i = 0; i < tables.length && i < size; i++) {
3951       boolean found = false;
3952       for (int j = 0; j < ts.length; j++) {
3953         if (Bytes.equals(ts[j].getName(), tables[i])) {
3954           found = true;
3955           break;
3956         }
3957       }
3958       assertTrue("Not found: " + Bytes.toString(tables[i]), found);
3959     }
3960   }
3961 
3962   /**
3963    * creates an HTable for tableName using an unmanaged HConnection.
3964    *
3965    * @param tableName - table to create
3966    * @return the created HTable object
3967    * @throws IOException
3968    */
3969   HTable createUnmangedHConnectionHTable(final byte [] tableName) throws IOException {
3970     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
3971     HConnection conn = HConnectionManager.createConnection(TEST_UTIL.getConfiguration());
3972     return (HTable)conn.getTable(tableName);
3973   }
3974 
3975   /**
3976    * simple test that just executes parts of the client
3977    * API that accept a pre-created HConnction instance
3978    *
3979    * @throws IOException
3980    */
3981   @Test
3982   public void testUnmanagedHConnection() throws IOException {
3983     final byte[] tableName = Bytes.toBytes("testUnmanagedHConnection");
3984     HTable t = createUnmangedHConnectionHTable(tableName);
3985     HBaseAdmin ha = new HBaseAdmin(t.getConnection());
3986     assertTrue(ha.tableExists(tableName));
3987     assertTrue(t.get(new Get(ROW)).isEmpty());
3988   }
3989 
3990   /**
3991    * test of that unmanaged HConnections are able to reconnect
3992    * properly (see HBASE-5058)
3993    *
3994    * @throws Exception
3995    */
3996   @Test
3997   public void testUnmanagedHConnectionReconnect() throws Exception {
3998     final byte[] tableName = Bytes.toBytes("testUnmanagedHConnectionReconnect");
3999     HTable t = createUnmangedHConnectionHTable(tableName);
4000     HConnection conn = t.getConnection();
4001     HBaseAdmin ha = new HBaseAdmin(conn);
4002     assertTrue(ha.tableExists(tableName));
4003     assertTrue(t.get(new Get(ROW)).isEmpty());
4004 
4005     // stop the master
4006     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
4007     cluster.stopMaster(0, false);
4008     cluster.waitOnMaster(0);
4009 
4010     // start up a new master
4011     cluster.startMaster();
4012     assertTrue(cluster.waitForActiveAndReadyMaster());
4013 
4014     // test that the same unmanaged connection works with a new
4015     // HBaseAdmin and can connect to the new master;
4016     HBaseAdmin newAdmin = new HBaseAdmin(conn);
4017     assertTrue(newAdmin.tableExists(tableName));
4018     assert(newAdmin.getClusterStatus().getServersSize() == SLAVES);
4019   }
4020 
4021   @Test
4022   public void testMiscHTableStuff() throws IOException {
4023     final byte[] tableAname = Bytes.toBytes("testMiscHTableStuffA");
4024     final byte[] tableBname = Bytes.toBytes("testMiscHTableStuffB");
4025     final byte[] attrName = Bytes.toBytes("TESTATTR");
4026     final byte[] attrValue = Bytes.toBytes("somevalue");
4027     byte[] value = Bytes.toBytes("value");
4028 
4029     HTable a = TEST_UTIL.createTable(tableAname, HConstants.CATALOG_FAMILY);
4030     HTable b = TEST_UTIL.createTable(tableBname, HConstants.CATALOG_FAMILY);
4031     Put put = new Put(ROW);
4032     put.add(HConstants.CATALOG_FAMILY, null, value);
4033     a.put(put);
4034 
4035     // open a new connection to A and a connection to b
4036     HTable newA = new HTable(TEST_UTIL.getConfiguration(), tableAname);
4037 
4038     // copy data from A to B
4039     Scan scan = new Scan();
4040     scan.addFamily(HConstants.CATALOG_FAMILY);
4041     ResultScanner s = newA.getScanner(scan);
4042     try {
4043       for (Result r : s) {
4044         put = new Put(r.getRow());
4045         put.setWriteToWAL(false);
4046         for (KeyValue kv : r.raw()) {
4047           put.add(kv);
4048         }
4049         b.put(put);
4050       }
4051     } finally {
4052       s.close();
4053     }
4054 
4055     // Opening a new connection to A will cause the tables to be reloaded
4056     HTable anotherA = new HTable(TEST_UTIL.getConfiguration(), tableAname);
4057     Get get = new Get(ROW);
4058     get.addFamily(HConstants.CATALOG_FAMILY);
4059     anotherA.get(get);
4060 
4061     // We can still access A through newA because it has the table information
4062     // cached. And if it needs to recalibrate, that will cause the information
4063     // to be reloaded.
4064 
4065     // Test user metadata
4066     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
4067     // make a modifiable descriptor
4068     HTableDescriptor desc = new HTableDescriptor(a.getTableDescriptor());
4069     // offline the table
4070     admin.disableTable(tableAname);
4071     // add a user attribute to HTD
4072     desc.setValue(attrName, attrValue);
4073     // add a user attribute to HCD
4074     for (HColumnDescriptor c : desc.getFamilies())
4075       c.setValue(attrName, attrValue);
4076     // update metadata for all regions of this table
4077     admin.modifyTable(tableAname, desc);
4078     // enable the table
4079     admin.enableTable(tableAname);
4080 
4081     // Test that attribute changes were applied
4082     desc = a.getTableDescriptor();
4083     assertTrue("wrong table descriptor returned",
4084       Bytes.compareTo(desc.getName(), tableAname) == 0);
4085     // check HTD attribute
4086     value = desc.getValue(attrName);
4087     assertFalse("missing HTD attribute value", value == null);
4088     assertFalse("HTD attribute value is incorrect",
4089       Bytes.compareTo(value, attrValue) != 0);
4090     // check HCD attribute
4091     for (HColumnDescriptor c : desc.getFamilies()) {
4092       value = c.getValue(attrName);
4093       assertFalse("missing HCD attribute value", value == null);
4094       assertFalse("HCD attribute value is incorrect",
4095         Bytes.compareTo(value, attrValue) != 0);
4096     }
4097   }
4098 
4099   @Test
4100   public void testGetClosestRowBefore() throws IOException {
4101     final byte [] tableAname = Bytes.toBytes("testGetClosestRowBefore");
4102     final byte [] row = Bytes.toBytes("row");
4103 
4104 
4105     byte[] firstRow = Bytes.toBytes("ro");
4106     byte[] beforeFirstRow = Bytes.toBytes("rn");
4107     byte[] beforeSecondRow = Bytes.toBytes("rov");
4108 
4109     HTable table = TEST_UTIL.createTable(tableAname,
4110       new byte [][] {HConstants.CATALOG_FAMILY, Bytes.toBytes("info2")});
4111     Put put = new Put(firstRow);
4112     Put put2 = new Put(row);
4113     byte[] zero = new byte[]{0};
4114     byte[] one = new byte[]{1};
4115 
4116     put.add(HConstants.CATALOG_FAMILY, null, zero);
4117     put2.add(HConstants.CATALOG_FAMILY, null, one);
4118 
4119     table.put(put);
4120     table.put(put2);
4121 
4122     Result result = null;
4123 
4124     // Test before first that null is returned
4125     result = table.getRowOrBefore(beforeFirstRow, HConstants.CATALOG_FAMILY);
4126     assertTrue(result == null);
4127 
4128     // Test at first that first is returned
4129     result = table.getRowOrBefore(firstRow, HConstants.CATALOG_FAMILY);
4130     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4131     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), zero));
4132 
4133     // Test in between first and second that first is returned
4134     result = table.getRowOrBefore(beforeSecondRow, HConstants.CATALOG_FAMILY);
4135     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4136     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), zero));
4137 
4138     // Test at second make sure second is returned
4139     result = table.getRowOrBefore(row, HConstants.CATALOG_FAMILY);
4140     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4141     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
4142 
4143     // Test after second, make sure second is returned
4144     result = table.getRowOrBefore(Bytes.add(row,one), HConstants.CATALOG_FAMILY);
4145     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4146     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
4147   }
4148 
4149   /**
4150    * For HBASE-2156
4151    * @throws Exception
4152    */
4153   @Test
4154   public void testScanVariableReuse() throws Exception {
4155     Scan scan = new Scan();
4156     scan.addFamily(FAMILY);
4157     scan.addColumn(FAMILY, ROW);
4158 
4159     assertTrue(scan.getFamilyMap().get(FAMILY).size() == 1);
4160 
4161     scan = new Scan();
4162     scan.addFamily(FAMILY);
4163 
4164     assertTrue(scan.getFamilyMap().get(FAMILY) == null);
4165     assertTrue(scan.getFamilyMap().containsKey(FAMILY));
4166   }
4167 
4168   @Test
4169   public void testMultiRowMutation() throws Exception {
4170     LOG.info("Starting testMultiRowMutation");
4171     final byte [] TABLENAME = Bytes.toBytes("testMultiRowMutation");
4172     final byte [] ROW1 = Bytes.toBytes("testRow1");
4173 
4174     HTable t = TEST_UTIL.createTable(TABLENAME, FAMILY);
4175     List<Mutation> mrm = new ArrayList<Mutation>();
4176     Put p = new Put(ROW);
4177     p.add(FAMILY, QUALIFIER, VALUE);
4178     mrm.add(p);
4179     p = new Put(ROW1);
4180     p.add(FAMILY, QUALIFIER, VALUE);
4181     mrm.add(p);
4182     MultiRowMutationProtocol mr = t.coprocessorProxy(
4183         MultiRowMutationProtocol.class, ROW);
4184     mr.mutateRows(mrm);
4185     Get g = new Get(ROW);
4186     Result r = t.get(g);
4187     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)));
4188     g = new Get(ROW1);
4189     r = t.get(g);
4190     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)));
4191   }
4192 
4193   @Test
4194   public void testRowMutation() throws Exception {
4195     LOG.info("Starting testRowMutation");
4196     final byte [] TABLENAME = Bytes.toBytes("testRowMutation");
4197     HTable t = TEST_UTIL.createTable(TABLENAME, FAMILY);
4198     byte [][] QUALIFIERS = new byte [][] {
4199         Bytes.toBytes("a"), Bytes.toBytes("b")
4200     };
4201     RowMutations arm = new RowMutations(ROW);
4202     Put p = new Put(ROW);
4203     p.add(FAMILY, QUALIFIERS[0], VALUE);
4204     arm.add(p);
4205     t.mutateRow(arm);
4206 
4207     Get g = new Get(ROW);
4208     Result r = t.get(g);
4209     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[0])));
4210 
4211     arm = new RowMutations(ROW);
4212     p = new Put(ROW);
4213     p.add(FAMILY, QUALIFIERS[1], VALUE);
4214     arm.add(p);
4215     Delete d = new Delete(ROW);
4216     d.deleteColumns(FAMILY, QUALIFIERS[0]);
4217     arm.add(d);
4218     t.batch(Arrays.asList((Row)arm));
4219     r = t.get(g);
4220     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[1])));
4221     assertNull(r.getValue(FAMILY, QUALIFIERS[0]));
4222   }
4223 
4224   @Test
4225   public void testAppend() throws Exception {
4226     LOG.info("Starting testAppend");
4227     final byte [] TABLENAME = Bytes.toBytes("testAppend");
4228     HTable t = TEST_UTIL.createTable(TABLENAME, FAMILY);
4229     byte[] v1 = Bytes.toBytes("42");
4230     byte[] v2 = Bytes.toBytes("23");
4231     byte [][] QUALIFIERS = new byte [][] {
4232         Bytes.toBytes("b"), Bytes.toBytes("a")
4233     };
4234     Append a = new Append(ROW);
4235     a.add(FAMILY, QUALIFIERS[0], v1);
4236     a.add(FAMILY, QUALIFIERS[1], v2);
4237     a.setReturnResults(false);
4238     assertNullResult(t.append(a));
4239 
4240     a = new Append(ROW);
4241     a.add(FAMILY, QUALIFIERS[0], v2);
4242     a.add(FAMILY, QUALIFIERS[1], v1);
4243     Result r = t.append(a);
4244     assertEquals(0, Bytes.compareTo(Bytes.add(v1,v2), r.getValue(FAMILY, QUALIFIERS[0])));
4245     assertEquals(0, Bytes.compareTo(Bytes.add(v2,v1), r.getValue(FAMILY, QUALIFIERS[1])));
4246   }
4247 
4248   @Test
4249   public void testIncrementWithDeletes() throws Exception {
4250     LOG.info("Starting testIncrementWithDeletes");
4251     final byte [] TABLENAME = Bytes.toBytes("testIncrementWithDeletes");
4252     HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
4253     final byte[] COLUMN = Bytes.toBytes("column");
4254 
4255     ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
4256     TEST_UTIL.flush(TABLENAME);
4257 
4258     Delete del = new Delete(ROW);
4259     ht.delete(del);
4260 
4261     ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
4262 
4263     Get get = new Get(ROW);
4264     Result r = ht.get(get);
4265     assertEquals(1, r.size());
4266     assertEquals(5, Bytes.toLong(r.getValue(FAMILY, COLUMN)));
4267   }
4268 
4269   @Test
4270   public void testIncrementingInvalidValue() throws Exception {
4271     LOG.info("Starting testIncrementingInvalidValue");
4272     final byte [] TABLENAME = Bytes.toBytes("testIncrementingInvalidValue");
4273     HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
4274     final byte[] COLUMN = Bytes.toBytes("column");
4275     Put p = new Put(ROW);
4276     // write an integer here (not a Long)
4277     p.add(FAMILY, COLUMN, Bytes.toBytes(5));
4278     ht.put(p);
4279     try {
4280       ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
4281       fail("Should have thrown DoNotRetryIOException");
4282     } catch (DoNotRetryIOException iox) {
4283       // success
4284     }
4285     Increment inc = new Increment(ROW);
4286     inc.addColumn(FAMILY, COLUMN, 5);
4287     try {
4288       ht.increment(inc);
4289       fail("Should have thrown DoNotRetryIOException");
4290     } catch (DoNotRetryIOException iox) {
4291       // success
4292     }
4293   }
4294 
4295 
4296 
4297   @Test
4298   public void testIncrement() throws Exception {
4299     LOG.info("Starting testIncrement");
4300     final byte [] TABLENAME = Bytes.toBytes("testIncrement");
4301     HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
4302 
4303     byte [][] ROWS = new byte [][] {
4304         Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"),
4305         Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
4306         Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i")
4307     };
4308     byte [][] QUALIFIERS = new byte [][] {
4309         Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"),
4310         Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
4311         Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i")
4312     };
4313 
4314     // Do some simple single-column increments
4315 
4316     // First with old API
4317     ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[0], 1);
4318     ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[1], 2);
4319     ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[2], 3);
4320     ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[3], 4);
4321 
4322     // Now increment things incremented with old and do some new
4323     Increment inc = new Increment(ROW);
4324     inc.addColumn(FAMILY, QUALIFIERS[1], 1);
4325     inc.addColumn(FAMILY, QUALIFIERS[3], 1);
4326     inc.addColumn(FAMILY, QUALIFIERS[4], 1);
4327     ht.increment(inc);
4328 
4329     // Verify expected results
4330     Result r = ht.get(new Get(ROW));
4331     KeyValue [] kvs = r.raw();
4332     assertEquals(5, kvs.length);
4333     assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 1);
4334     assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 3);
4335     assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 3);
4336     assertIncrementKey(kvs[3], ROW, FAMILY, QUALIFIERS[3], 5);
4337     assertIncrementKey(kvs[4], ROW, FAMILY, QUALIFIERS[4], 1);
4338 
4339     // Now try multiple columns by different amounts
4340     inc = new Increment(ROWS[0]);
4341     for (int i=0;i<QUALIFIERS.length;i++) {
4342       inc.addColumn(FAMILY, QUALIFIERS[i], i+1);
4343     }
4344     ht.increment(inc);
4345     // Verify
4346     r = ht.get(new Get(ROWS[0]));
4347     kvs = r.raw();
4348     assertEquals(QUALIFIERS.length, kvs.length);
4349     for (int i=0;i<QUALIFIERS.length;i++) {
4350       assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], i+1);
4351     }
4352 
4353     // Re-increment them
4354     inc = new Increment(ROWS[0]);
4355     for (int i=0;i<QUALIFIERS.length;i++) {
4356       inc.addColumn(FAMILY, QUALIFIERS[i], i+1);
4357     }
4358     ht.increment(inc);
4359     // Verify
4360     r = ht.get(new Get(ROWS[0]));
4361     kvs = r.raw();
4362     assertEquals(QUALIFIERS.length, kvs.length);
4363     for (int i=0;i<QUALIFIERS.length;i++) {
4364       assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], 2*(i+1));
4365     }
4366   }
4367 
4368   /**
4369    * This test demonstrates how we use ThreadPoolExecutor.
4370    * It needs to show that we only use as many threads in the pool as we have
4371    * region servers. To do this, instead of doing real requests, we use a
4372    * SynchronousQueue where each put must wait for a take (and vice versa)
4373    * so that way we have full control of the number of active threads.
4374    * @throws IOException
4375    * @throws InterruptedException
4376    */
4377   @Test
4378   public void testPoolBehavior() throws IOException, InterruptedException {
4379     byte[] someBytes = Bytes.toBytes("pool");
4380     HTable table = TEST_UTIL.createTable(someBytes, someBytes);
4381     ThreadPoolExecutor pool = (ThreadPoolExecutor)table.getPool();
4382 
4383     // Make sure that the TPE stars with a core pool size of one and 0
4384     // initialized worker threads
4385     assertEquals(1, pool.getCorePoolSize());
4386     assertEquals(0, pool.getPoolSize());
4387 
4388     // Build a SynchronousQueue that we use for thread coordination
4389     final SynchronousQueue<Object> queue = new SynchronousQueue<Object>();
4390     List<Runnable> tasks = new ArrayList<Runnable>(5);
4391     for (int i = 0; i < 5; i++) {
4392       tasks.add(new Runnable() {
4393         public void run() {
4394           try {
4395             // The thread blocks here until we decide to let it go
4396             queue.take();
4397           } catch (InterruptedException ie) { }
4398         }
4399       });
4400     }
4401     // First, add two tasks and make sure the pool size follows
4402     pool.submit(tasks.get(0));
4403     assertEquals(1, pool.getPoolSize());
4404     pool.submit(tasks.get(1));
4405     assertEquals(2, pool.getPoolSize());
4406 
4407     // Next, terminate those tasks and then make sure the pool is still the
4408     // same size
4409     queue.put(new Object());
4410     queue.put(new Object());
4411     assertEquals(2, pool.getPoolSize());
4412 
4413     //ensure that ThreadPoolExecutor knows that tasks are finished.
4414     while (pool.getCompletedTaskCount() < 2) {
4415       Threads.sleep(1);
4416     }
4417 
4418     //ensure that ThreadPoolExecutor knows that threads are finished.
4419     while (pool.getCompletedTaskCount() < 2) {
4420       Threads.sleep(1);
4421     }
4422     // Now let's simulate adding a RS meaning that we'll go up to three
4423     // concurrent threads. The pool should not grow larger than three.
4424     pool.submit(tasks.get(2));
4425     pool.submit(tasks.get(3));
4426     pool.submit(tasks.get(4));
4427     assertEquals(3, pool.getPoolSize());
4428     queue.put(new Object());
4429     queue.put(new Object());
4430     queue.put(new Object());
4431   }
4432 
4433   @Test
4434   public void testClientPoolRoundRobin() throws IOException {
4435     final byte[] tableName = Bytes.toBytes("testClientPoolRoundRobin");
4436 
4437     int poolSize = 3;
4438     int numVersions = poolSize * 2;
4439     Configuration conf = TEST_UTIL.getConfiguration();
4440     conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "round-robin");
4441     conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
4442 
4443     HTable table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY },
4444         conf, Integer.MAX_VALUE);
4445     table.setAutoFlush(true);
4446 
4447     final long ts = EnvironmentEdgeManager.currentTimeMillis();
4448     Get get = new Get(ROW);
4449     get.addColumn(FAMILY, QUALIFIER);
4450     get.setMaxVersions();
4451 
4452     for (int versions = 1; versions <= numVersions; versions++) {
4453       Put put = new Put(ROW);
4454       put.add(FAMILY, QUALIFIER, ts + versions, VALUE);
4455       table.put(put);
4456 
4457       Result result = table.get(get);
4458       NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY)
4459           .get(QUALIFIER);
4460 
4461       assertEquals("The number of versions of '" + FAMILY + ":" + QUALIFIER
4462           + " did not match " + versions, versions, navigableMap.size());
4463       for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4464         assertTrue("The value at time " + entry.getKey()
4465             + " did not match what was put",
4466             Bytes.equals(VALUE, entry.getValue()));
4467       }
4468     }
4469   }
4470 
4471   @Test
4472   public void testClientPoolThreadLocal() throws IOException {
4473     final byte[] tableName = Bytes.toBytes("testClientPoolThreadLocal");
4474 
4475     int poolSize = Integer.MAX_VALUE;
4476     int numVersions = 3;
4477     Configuration conf = TEST_UTIL.getConfiguration();
4478     conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "thread-local");
4479     conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
4480 
4481     final HTable table = TEST_UTIL.createTable(tableName,
4482         new byte[][] { FAMILY }, conf);
4483     table.setAutoFlush(true);
4484 
4485     final long ts = EnvironmentEdgeManager.currentTimeMillis();
4486     final Get get = new Get(ROW);
4487     get.addColumn(FAMILY, QUALIFIER);
4488     get.setMaxVersions();
4489 
4490     for (int versions = 1; versions <= numVersions; versions++) {
4491       Put put = new Put(ROW);
4492       put.add(FAMILY, QUALIFIER, ts + versions, VALUE);
4493       table.put(put);
4494 
4495       Result result = table.get(get);
4496       NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY)
4497           .get(QUALIFIER);
4498 
4499       assertEquals("The number of versions of '" + FAMILY + ":" + QUALIFIER
4500           + " did not match " + versions, versions, navigableMap.size());
4501       for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4502         assertTrue("The value at time " + entry.getKey()
4503             + " did not match what was put",
4504             Bytes.equals(VALUE, entry.getValue()));
4505       }
4506     }
4507 
4508     final Object waitLock = new Object();
4509     ExecutorService executorService = Executors.newFixedThreadPool(numVersions);
4510     final AtomicReference<AssertionError> error = new AtomicReference<AssertionError>(null);
4511     for (int versions = numVersions; versions < numVersions * 2; versions++) {
4512       final int versionsCopy = versions;
4513       executorService.submit(new Callable<Void>() {
4514         @Override
4515         public Void call() {
4516           try {
4517             Put put = new Put(ROW);
4518             put.add(FAMILY, QUALIFIER, ts + versionsCopy, VALUE);
4519             table.put(put);
4520 
4521             Result result = table.get(get);
4522             NavigableMap<Long, byte[]> navigableMap = result.getMap()
4523                 .get(FAMILY).get(QUALIFIER);
4524 
4525             assertEquals("The number of versions of '" + FAMILY + ":"
4526                 + QUALIFIER + " did not match " + versionsCopy, versionsCopy,
4527                 navigableMap.size());
4528             for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4529               assertTrue("The value at time " + entry.getKey()
4530                   + " did not match what was put",
4531                   Bytes.equals(VALUE, entry.getValue()));
4532             }
4533             synchronized (waitLock) {
4534               waitLock.wait();
4535             }
4536           } catch (Exception e) {
4537           } catch (AssertionError e) {
4538             // the error happens in a thread, it won't fail the test,
4539             // need to pass it to the caller for proper handling.
4540             error.set(e);
4541             LOG.error(e);
4542           }
4543 
4544           return null;
4545         }
4546       });
4547     }
4548     synchronized (waitLock) {
4549       waitLock.notifyAll();
4550     }
4551     executorService.shutdownNow();
4552     assertNull(error.get());
4553   }
4554 
4555   @Test
4556   public void testCheckAndPut() throws IOException {
4557     final byte [] anotherrow = Bytes.toBytes("anotherrow");
4558     final byte [] value2 = Bytes.toBytes("abcd");
4559 
4560     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testCheckAndPut"),
4561       new byte [][] {FAMILY});
4562     Put put1 = new Put(ROW);
4563     put1.add(FAMILY, QUALIFIER, VALUE);
4564 
4565     // row doesn't exist, so using non-null value should be considered "not match".
4566     boolean ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE, put1);
4567     assertEquals(ok, false);
4568 
4569     // row doesn't exist, so using "null" to check for existence should be considered "match".
4570     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put1);
4571     assertEquals(ok, true);
4572 
4573     // row now exists, so using "null" to check for existence should be considered "not match".
4574     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put1);
4575     assertEquals(ok, false);
4576 
4577     Put put2 = new Put(ROW);
4578     put2.add(FAMILY, QUALIFIER, value2);
4579 
4580     // row now exists, use the matching value to check
4581     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE, put2);
4582     assertEquals(ok, true);
4583 
4584     Put put3 = new Put(anotherrow);
4585     put3.add(FAMILY, QUALIFIER, VALUE);
4586 
4587     // try to do CheckAndPut on different rows
4588     try {
4589         ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, value2, put3);
4590         fail("trying to check and modify different rows should have failed.");
4591     } catch(Exception e) {}
4592 
4593   }
4594 
4595   /**
4596   * Test ScanMetrics
4597   * @throws Exception
4598   */
4599   @Test
4600   @SuppressWarnings ("unused")
4601   public void testScanMetrics() throws Exception {
4602     byte [] TABLENAME = Bytes.toBytes("testScanMetrics");
4603 
4604     Configuration conf = TEST_UTIL.getConfiguration();
4605     TEST_UTIL.createTable(TABLENAME, FAMILY);
4606 
4607     // Set up test table:
4608     // Create table:
4609     HTable ht = new HTable(conf, TABLENAME);
4610 
4611     // Create multiple regions for this table
4612     int numOfRegions = TEST_UTIL.createMultiRegions(ht, FAMILY);
4613     // Create 3 rows in the table, with rowkeys starting with "z*" so that
4614     // scan are forced to hit all the regions.
4615     Put put1 = new Put(Bytes.toBytes("z1"));
4616     put1.add(FAMILY, QUALIFIER, VALUE);
4617     Put put2 = new Put(Bytes.toBytes("z2"));
4618     put2.add(FAMILY, QUALIFIER, VALUE);
4619     Put put3 = new Put(Bytes.toBytes("z3"));
4620     put3.add(FAMILY, QUALIFIER, VALUE);
4621     ht.put(Arrays.asList(put1, put2, put3));
4622 
4623     Scan scan1 = new Scan();
4624     int numRecords = 0;
4625     for(Result result : ht.getScanner(scan1)) {
4626       numRecords++;
4627     }
4628     LOG.info("test data has " + numRecords + " records.");
4629 
4630     // by default, scan metrics collection is turned off
4631     assertEquals(null, scan1.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA));
4632 
4633     // turn on scan metrics
4634     Scan scan = new Scan();
4635     scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
4636     ResultScanner scanner = ht.getScanner(scan);
4637     // per HBASE-5717, this should still collect even if you don't run all the way to
4638     // the end of the scanner. So this is asking for 2 of the 3 rows we inserted.
4639     for (Result result : scanner.next(numRecords - 1)) {
4640     }
4641     scanner.close();
4642 
4643     ScanMetrics scanMetrics = getScanMetrics(scan);
4644     assertEquals("Did not access all the regions in the table", numOfRegions,
4645         scanMetrics.countOfRegions.getCurrentIntervalValue());
4646 
4647     // set caching to 100
4648     scan = new Scan();
4649     scan.setCaching(100);
4650     scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
4651     scanner = ht.getScanner(scan);
4652     for (Result result : scanner.next(numRecords - 1)) {
4653     }
4654     scanner.close();
4655     
4656     scanMetrics = getScanMetrics(scan);
4657     assertEquals("Did not access all the regions in the table", numOfRegions,
4658         scanMetrics.countOfRegions.getCurrentIntervalValue());
4659     
4660     // now, test that the metrics are still collected even if you don't call close, but do
4661     // run past the end of all the records
4662     Scan scanWithoutClose = new Scan();
4663     scanWithoutClose.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
4664     ResultScanner scannerWithoutClose = ht.getScanner(scanWithoutClose);
4665     for (Result result : scannerWithoutClose.next(numRecords + 1)) {
4666     }
4667     ScanMetrics scanMetricsWithoutClose = getScanMetrics(scanWithoutClose);
4668     assertEquals("Did not access all the regions in the table", numOfRegions,
4669         scanMetricsWithoutClose.countOfRegions.getCurrentIntervalValue());
4670 
4671     // finally, test that the metrics are collected correctly if you both run past all the records,
4672     // AND close the scanner
4673     Scan scanWithClose = new Scan();
4674     scanWithClose.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
4675     ResultScanner scannerWithClose = ht.getScanner(scanWithClose);
4676     for (Result result : scannerWithClose.next(numRecords + 1)) {
4677     }
4678     scannerWithClose.close();
4679     ScanMetrics scanMetricsWithClose = getScanMetrics(scanWithClose);
4680     assertEquals("Did not access all the regions in the table", numOfRegions,
4681         scanMetricsWithClose.countOfRegions.getCurrentIntervalValue());
4682   }
4683 
4684   private ScanMetrics getScanMetrics(Scan scan) throws Exception {
4685     byte[] serializedMetrics = scan.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA);
4686     assertTrue("Serialized metrics were not found.", serializedMetrics != null);
4687 
4688     DataInputBuffer in = new DataInputBuffer();
4689     in.reset(serializedMetrics, 0, serializedMetrics.length);
4690     ScanMetrics scanMetrics = new ScanMetrics();
4691     scanMetrics.readFields(in);
4692     return scanMetrics;
4693   }
4694 
4695   /**
4696    * Tests that cache on write works all the way up from the client-side.
4697    *
4698    * Performs inserts, flushes, and compactions, verifying changes in the block
4699    * cache along the way.
4700    *
4701    * @throws Exception
4702    */
4703   @Test
4704   public void testCacheOnWriteEvictOnClose() throws Exception {
4705     byte [] tableName = Bytes.toBytes("testCOWEOCfromClient");
4706     byte [] data = Bytes.toBytes("data");
4707     HTable table = TEST_UTIL.createTable(tableName, new byte [][] {FAMILY});
4708     // get the block cache and region
4709     String regionName = table.getRegionLocations().firstKey().getEncodedName();
4710     HRegion region = TEST_UTIL.getRSForFirstRegionInTable(
4711         tableName).getFromOnlineRegions(regionName);
4712     Store store = region.getStores().values().iterator().next();
4713     CacheConfig cacheConf = store.getCacheConfig();
4714     cacheConf.setCacheDataOnWrite(true);
4715     cacheConf.setEvictOnClose(true);
4716     BlockCache cache = cacheConf.getBlockCache();
4717 
4718     // establish baseline stats
4719     long startBlockCount = cache.getBlockCount();
4720     long startBlockHits = cache.getStats().getHitCount();
4721     long startBlockMiss = cache.getStats().getMissCount();
4722 
4723     // wait till baseline is stable, (minimal 500 ms)
4724     for (int i = 0; i < 5; i++) {
4725       Thread.sleep(100);
4726       if (startBlockCount != cache.getBlockCount()
4727           || startBlockHits != cache.getStats().getHitCount()
4728           || startBlockMiss != cache.getStats().getMissCount()) {
4729         startBlockCount = cache.getBlockCount();
4730         startBlockHits = cache.getStats().getHitCount();
4731         startBlockMiss = cache.getStats().getMissCount();
4732         i = -1;
4733       }
4734     }
4735 
4736     // insert data
4737     Put put = new Put(ROW);
4738     put.add(FAMILY, QUALIFIER, data);
4739     table.put(put);
4740     assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
4741     // data was in memstore so don't expect any changes
4742     assertEquals(startBlockCount, cache.getBlockCount());
4743     assertEquals(startBlockHits, cache.getStats().getHitCount());
4744     assertEquals(startBlockMiss, cache.getStats().getMissCount());
4745     // flush the data
4746     System.out.println("Flushing cache");
4747     region.flushcache();
4748     // expect one more block in cache, no change in hits/misses
4749     long expectedBlockCount = startBlockCount + 1;
4750     long expectedBlockHits = startBlockHits;
4751     long expectedBlockMiss = startBlockMiss;
4752     assertEquals(expectedBlockCount, cache.getBlockCount());
4753     assertEquals(expectedBlockHits, cache.getStats().getHitCount());
4754     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
4755     // read the data and expect same blocks, one new hit, no misses
4756     assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
4757     assertEquals(expectedBlockCount, cache.getBlockCount());
4758     assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
4759     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
4760     // insert a second column, read the row, no new blocks, one new hit
4761     byte [] QUALIFIER2 = Bytes.add(QUALIFIER, QUALIFIER);
4762     byte [] data2 = Bytes.add(data, data);
4763     put = new Put(ROW);
4764     put.add(FAMILY, QUALIFIER2, data2);
4765     table.put(put);
4766     Result r = table.get(new Get(ROW));
4767     assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
4768     assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
4769     assertEquals(expectedBlockCount, cache.getBlockCount());
4770     assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
4771     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
4772     // flush, one new block
4773     System.out.println("Flushing cache");
4774     region.flushcache();
4775     assertEquals(++expectedBlockCount, cache.getBlockCount());
4776     assertEquals(expectedBlockHits, cache.getStats().getHitCount());
4777     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
4778     // compact, net minus two blocks, two hits, no misses
4779     System.out.println("Compacting");
4780     assertEquals(2, store.getNumberOfStoreFiles());
4781     store.triggerMajorCompaction();
4782     region.compactStores();
4783     waitForStoreFileCount(store, 1, 10000); // wait 10 seconds max
4784     assertEquals(1, store.getNumberOfStoreFiles());
4785     expectedBlockCount -= 2; // evicted two blocks, cached none
4786     assertEquals(expectedBlockCount, cache.getBlockCount());
4787     expectedBlockHits += 2;
4788     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
4789     assertEquals(expectedBlockHits, cache.getStats().getHitCount());
4790     // read the row, this should be a cache miss because we don't cache data
4791     // blocks on compaction
4792     r = table.get(new Get(ROW));
4793     assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
4794     assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
4795     expectedBlockCount += 1; // cached one data block
4796     assertEquals(expectedBlockCount, cache.getBlockCount());
4797     assertEquals(expectedBlockHits, cache.getStats().getHitCount());
4798     assertEquals(++expectedBlockMiss, cache.getStats().getMissCount());
4799   }
4800 
4801   private void waitForStoreFileCount(Store store, int count, int timeout)
4802   throws InterruptedException {
4803     long start = System.currentTimeMillis();
4804     while (start + timeout > System.currentTimeMillis() &&
4805         store.getNumberOfStoreFiles() != count) {
4806       Thread.sleep(100);
4807     }
4808     System.out.println("start=" + start + ", now=" +
4809         System.currentTimeMillis() + ", cur=" + store.getNumberOfStoreFiles());
4810     assertEquals(count, store.getNumberOfStoreFiles());
4811   }
4812 
4813   @Test
4814   /**
4815    * Tests the non cached version of getRegionLocation by moving a region.
4816    */
4817   public void testNonCachedGetRegionLocation() throws Exception {
4818     // Test Initialization.
4819     String tableName = "testNonCachedGetRegionLocation";
4820     byte [] TABLE = Bytes.toBytes(tableName);
4821     byte [] family1 = Bytes.toBytes("f1");
4822     byte [] family2 = Bytes.toBytes("f2");
4823     HTable table = TEST_UTIL.createTable(TABLE, new byte[][] {family1, family2}, 10);
4824     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
4825     Map <HRegionInfo, ServerName> regionsMap = table.getRegionLocations();
4826     assertEquals(1, regionsMap.size());
4827     HRegionInfo regionInfo = regionsMap.keySet().iterator().next();
4828     ServerName addrBefore = regionsMap.get(regionInfo);
4829     // Verify region location before move.
4830     HServerAddress addrCache =
4831       table.getRegionLocation(regionInfo.getStartKey(), false).getServerAddress();
4832     HServerAddress addrNoCache =
4833       table.getRegionLocation(regionInfo.getStartKey(),
4834           true).getServerAddress();
4835 
4836     assertEquals(addrBefore.getPort(), addrCache.getPort());
4837     assertEquals(addrBefore.getPort(), addrNoCache.getPort());
4838 
4839     ServerName addrAfter = null;
4840     // Now move the region to a different server.
4841     for (int i = 0; i < SLAVES; i++) {
4842       HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(i);
4843       ServerName addr = regionServer.getServerName();
4844       if (addr.getPort() != addrBefore.getPort()) {
4845         admin.move(regionInfo.getEncodedNameAsBytes(),
4846             Bytes.toBytes(addr.toString()));
4847         // Wait for the region to move.
4848         Thread.sleep(5000);
4849         addrAfter = addr;
4850         break;
4851       }
4852     }
4853 
4854     // Verify the region was moved.
4855     addrCache =
4856       table.getRegionLocation(regionInfo.getStartKey(), false).getServerAddress();
4857     addrNoCache =
4858       table.getRegionLocation(regionInfo.getStartKey(),
4859           true).getServerAddress();
4860     assertNotNull(addrAfter);
4861     assertTrue(addrAfter.getPort() != addrCache.getPort());
4862     assertEquals(addrAfter.getPort(), addrNoCache.getPort());
4863   }
4864 
4865   @Test
4866   /**
4867    * Tests getRegionsInRange by creating some regions over which a range of
4868    * keys spans; then changing the key range.
4869    */
4870   public void testGetRegionsInRange() throws Exception {
4871     // Test Initialization.
4872     byte [] startKey = Bytes.toBytes("ddc");
4873     byte [] endKey = Bytes.toBytes("mmm");
4874     byte [] TABLE = Bytes.toBytes("testGetRegionsInRange");
4875     HTable table = TEST_UTIL.createTable(TABLE, new byte[][] {FAMILY}, 10);
4876     int numOfRegions = TEST_UTIL.createMultiRegions(table, FAMILY);
4877     assertEquals(25, numOfRegions);
4878 
4879     // Get the regions in this range
4880     List<HRegionLocation> regionsList = table.getRegionsInRange(startKey,
4881       endKey);
4882     assertEquals(10, regionsList.size());
4883 
4884     // Change the start key
4885     startKey = Bytes.toBytes("fff");
4886     regionsList = table.getRegionsInRange(startKey, endKey);
4887     assertEquals(7, regionsList.size());
4888 
4889     // Change the end key
4890     endKey = Bytes.toBytes("nnn");
4891     regionsList = table.getRegionsInRange(startKey, endKey);
4892     assertEquals(8, regionsList.size());
4893 
4894     // Empty start key
4895     regionsList = table.getRegionsInRange(HConstants.EMPTY_START_ROW, endKey);
4896     assertEquals(13, regionsList.size());
4897 
4898     // Empty end key
4899     regionsList = table.getRegionsInRange(startKey, HConstants.EMPTY_END_ROW);
4900     assertEquals(20, regionsList.size());
4901 
4902     // Both start and end keys empty
4903     regionsList = table.getRegionsInRange(HConstants.EMPTY_START_ROW,
4904       HConstants.EMPTY_END_ROW);
4905     assertEquals(25, regionsList.size());
4906 
4907     // Change the end key to somewhere in the last block
4908     endKey = Bytes.toBytes("yyz");
4909     regionsList = table.getRegionsInRange(startKey, endKey);
4910     assertEquals(20, regionsList.size());
4911 
4912     // Change the start key to somewhere in the first block
4913     startKey = Bytes.toBytes("aac");
4914     regionsList = table.getRegionsInRange(startKey, endKey);
4915     assertEquals(25, regionsList.size());
4916 
4917     // Make start and end key the same
4918     startKey = endKey = Bytes.toBytes("ccc");
4919     regionsList = table.getRegionsInRange(startKey, endKey);
4920     assertEquals(1, regionsList.size());
4921   }
4922 
4923   @Test
4924   public void testJira6912() throws Exception {
4925     byte [] TABLE = Bytes.toBytes("testJira6912");
4926     HTable foo = TEST_UTIL.createTable(TABLE, new byte[][] {FAMILY}, 10);
4927 
4928     List<Put> puts = new ArrayList<Put>();
4929     for (int i=0;i !=100; i++){
4930       Put put = new Put(Bytes.toBytes(i));
4931       put.add(FAMILY, FAMILY, Bytes.toBytes(i));
4932       puts.add(put);
4933     }
4934     foo.put(puts);
4935     // If i comment this out it works
4936     TEST_UTIL.flush();
4937 
4938     Scan scan = new Scan();
4939     scan.setStartRow(Bytes.toBytes(1));
4940     scan.setStopRow(Bytes.toBytes(3));
4941     scan.addColumn(FAMILY, FAMILY);
4942     scan.setFilter(new RowFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator(Bytes.toBytes(1))));
4943 
4944     ResultScanner scanner = foo.getScanner(scan);
4945     Result[] bar = scanner.next(100);
4946     assertEquals(1, bar.length);
4947   }
4948 
4949   @Test
4950   public void testRawScanRespectsVersions() throws Exception {
4951     byte[] TABLE = Bytes.toBytes("testRawScan");
4952     HTable table = TEST_UTIL.createTable(TABLE, new byte[][] { FAMILY });
4953     byte[] row = Bytes.toBytes("row");
4954 
4955     // put the same row 4 times, with different values
4956     Put p = new Put(row);
4957     p.add(FAMILY, QUALIFIER, 10, VALUE);
4958     table.put(p);
4959     table.flushCommits();
4960 
4961     p = new Put(row);
4962     p.add(FAMILY, QUALIFIER, 11, ArrayUtils.add(VALUE, (byte) 2));
4963     table.put(p);
4964     table.flushCommits();
4965 
4966     p = new Put(row);
4967     p.add(FAMILY, QUALIFIER, 12, ArrayUtils.add(VALUE, (byte) 3));
4968     table.put(p);
4969     table.flushCommits();
4970 
4971     p = new Put(row);
4972     p.add(FAMILY, QUALIFIER, 13, ArrayUtils.add(VALUE, (byte) 4));
4973     table.put(p);
4974     table.flushCommits();
4975 
4976     int versions = 4;
4977     Scan s = new Scan(row);
4978     // get all the possible versions
4979     s.setMaxVersions();
4980     s.setRaw(true);
4981 
4982     ResultScanner scanner = table.getScanner(s);
4983     int count = 0;
4984     for (Result r : scanner) {
4985       assertEquals("Found an unexpected number of results for the row!", versions, r.list().size());
4986       count++;
4987     }
4988     assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
4989       count);
4990     scanner.close();
4991 
4992     // then if we decrease the number of versions, but keep the scan raw, we should see exactly that
4993     // number of versions
4994     versions = 2;
4995     s.setMaxVersions(versions);
4996     scanner = table.getScanner(s);
4997     count = 0;
4998     for (Result r : scanner) {
4999       assertEquals("Found an unexpected number of results for the row!", versions, r.list().size());
5000       count++;
5001     }
5002     assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
5003       count);
5004     scanner.close();
5005 
5006     // finally, if we turn off raw scanning, but max out the number of versions, we should go back
5007     // to seeing just three
5008     versions = 3;
5009     s.setMaxVersions(versions);
5010     scanner = table.getScanner(s);
5011     count = 0;
5012     for (Result r : scanner) {
5013       assertEquals("Found an unexpected number of results for the row!", versions, r.list().size());
5014       count++;
5015     }
5016     assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
5017       count);
5018     scanner.close();
5019 
5020     table.close();
5021     TEST_UTIL.deleteTable(TABLE);
5022   }
5023 
5024   @Test
5025   public void testSmallScan() throws Exception {
5026     // Test Initialization.
5027     byte[] TABLE = Bytes.toBytes("testSmallScan");
5028     HTable table = TEST_UTIL.createTable(TABLE, FAMILY);
5029 
5030     // Insert one row each region
5031     int insertNum = 10;
5032     for (int i = 0; i < 10; i++) {
5033       Put put = new Put(Bytes.toBytes("row" + String.format("%03d", i)));
5034       put.add(FAMILY, QUALIFIER, VALUE);
5035       table.put(put);
5036     }
5037 
5038     // nomal scan
5039     ResultScanner scanner = table.getScanner(new Scan());
5040     int count = 0;
5041     for (Result r : scanner) {
5042       assertTrue(!r.isEmpty());
5043       count++;
5044     }
5045     assertEquals(insertNum, count);
5046 
5047     // small scan
5048     Scan scan = new Scan(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
5049     scan.setSmall(true);
5050     scan.setCaching(2);
5051     scanner = table.getScanner(scan);
5052     count = 0;
5053     for (Result r : scanner) {
5054       assertTrue(!r.isEmpty());
5055       count++;
5056     }
5057     assertEquals(insertNum, count);
5058 
5059   }
5060 
5061   @org.junit.Rule
5062   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
5063     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
5064 }
5065