View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  import static org.junit.Assert.assertArrayEquals;
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertFalse;
24  import static org.junit.Assert.assertNotNull;
25  import static org.junit.Assert.assertNull;
26  import static org.junit.Assert.assertSame;
27  import static org.junit.Assert.assertTrue;
28  import static org.junit.Assert.fail;
29  
30  import java.io.IOException;
31  import java.lang.reflect.Method;
32  import java.util.ArrayList;
33  import java.util.Arrays;
34  import java.util.Collections;
35  import java.util.HashSet;
36  import java.util.Iterator;
37  import java.util.List;
38  import java.util.Map;
39  import java.util.NavigableMap;
40  import java.util.UUID;
41  import java.util.concurrent.Callable;
42  import java.util.concurrent.ExecutorService;
43  import java.util.concurrent.Executors;
44  import java.util.concurrent.atomic.AtomicReference;
45  
46  import org.apache.log4j.Level;
47  import org.apache.commons.lang.ArrayUtils;
48  import org.apache.commons.logging.Log;
49  import org.apache.commons.logging.LogFactory;
50  import org.apache.hadoop.conf.Configuration;
51  import org.apache.hadoop.hbase.Abortable;
52  import org.apache.hadoop.hbase.Cell;
53  import org.apache.hadoop.hbase.CellUtil;
54  import org.apache.hadoop.hbase.DoNotRetryIOException;
55  import org.apache.hadoop.hbase.HBaseTestingUtility;
56  import org.apache.hadoop.hbase.HColumnDescriptor;
57  import org.apache.hadoop.hbase.HConstants;
58  import org.apache.hadoop.hbase.HRegionInfo;
59  import org.apache.hadoop.hbase.HRegionLocation;
60  import org.apache.hadoop.hbase.HTableDescriptor;
61  import org.apache.hadoop.hbase.KeyValue;
62  import org.apache.hadoop.hbase.KeyValueUtil;
63  import org.apache.hadoop.hbase.testclassification.LargeTests;
64  import org.apache.hadoop.hbase.MiniHBaseCluster;
65  import org.apache.hadoop.hbase.ServerName;
66  import org.apache.hadoop.hbase.TableName;
67  import org.apache.hadoop.hbase.Waiter;
68  import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionImplementation;
69  import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
70  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
71  import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
72  import org.apache.hadoop.hbase.filter.BinaryComparator;
73  import org.apache.hadoop.hbase.filter.CompareFilter;
74  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
75  import org.apache.hadoop.hbase.filter.Filter;
76  import org.apache.hadoop.hbase.filter.FilterList;
77  import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
78  import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
79  import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
80  import org.apache.hadoop.hbase.filter.LongComparator;
81  import org.apache.hadoop.hbase.filter.PrefixFilter;
82  import org.apache.hadoop.hbase.filter.QualifierFilter;
83  import org.apache.hadoop.hbase.filter.RegexStringComparator;
84  import org.apache.hadoop.hbase.filter.RowFilter;
85  import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
86  import org.apache.hadoop.hbase.filter.WhileMatchFilter;
87  import org.apache.hadoop.hbase.io.hfile.BlockCache;
88  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
89  import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
90  import org.apache.hadoop.hbase.master.HMaster;
91  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
92  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
93  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
94  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
95  import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
96  import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService;
97  import org.apache.hadoop.hbase.regionserver.HRegion;
98  import org.apache.hadoop.hbase.regionserver.HRegionServer;
99  import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
100 import org.apache.hadoop.hbase.regionserver.Store;
101 import org.apache.hadoop.hbase.util.Bytes;
102 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
103 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
104 import org.apache.log4j.AppenderSkeleton;
105 import org.apache.log4j.Logger;
106 import org.apache.log4j.spi.LoggingEvent;
107 import org.junit.After;
108 import org.junit.AfterClass;
109 import org.junit.Before;
110 import org.junit.BeforeClass;
111 import org.junit.Ignore;
112 import org.junit.Test;
113 import org.junit.experimental.categories.Category;
114 
115 /**
116  * Run tests that use the HBase clients; {@link HTable} and {@link HTablePool}.
117  * Sets up the HBase mini cluster once at start and runs through all client tests.
118  * Each creates a table named for the method and does its stuff against that.
119  */
120 @Category(LargeTests.class)
121 @SuppressWarnings ("deprecation")
122 public class TestFromClientSide {
123   final Log LOG = LogFactory.getLog(getClass());
124   protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
125   private static byte [] ROW = Bytes.toBytes("testRow");
126   private static byte [] FAMILY = Bytes.toBytes("testFamily");
127   private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
128   private static byte [] VALUE = Bytes.toBytes("testValue");
129   protected static int SLAVES = 3;
130 
131   /**
132    * @throws java.lang.Exception
133    */
134   @BeforeClass
135   public static void setUpBeforeClass() throws Exception {
136     // Uncomment the following lines if more verbosity is needed for
137     // debugging (see HBASE-12285 for details).
138     //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
139     //((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
140     //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
141     Configuration conf = TEST_UTIL.getConfiguration();
142     conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
143         MultiRowMutationEndpoint.class.getName());
144     conf.setBoolean("hbase.table.sanity.checks", true); // enable for below tests
145     // We need more than one region server in this test
146     TEST_UTIL.startMiniCluster(SLAVES);
147   }
148 
149   /**
150    * @throws java.lang.Exception
151    */
152   @AfterClass
153   public static void tearDownAfterClass() throws Exception {
154     TEST_UTIL.shutdownMiniCluster();
155   }
156 
157   /**
158    * @throws java.lang.Exception
159    */
160   @Before
161   public void setUp() throws Exception {
162     // Nothing to do.
163   }
164 
165   /**
166    * @throws java.lang.Exception
167    */
168   @After
169   public void tearDown() throws Exception {
170     // Nothing to do.
171   }
172 
173   /**
174    * Basic client side validation of HBASE-4536
175    */
176    @Test
177    public void testKeepDeletedCells() throws Exception {
178      final byte[] TABLENAME = Bytes.toBytes("testKeepDeletesCells");
179      final byte[] FAMILY = Bytes.toBytes("family");
180      final byte[] C0 = Bytes.toBytes("c0");
181 
182      final byte[] T1 = Bytes.toBytes("T1");
183      final byte[] T2 = Bytes.toBytes("T2");
184      final byte[] T3 = Bytes.toBytes("T3");
185      HColumnDescriptor hcd = new HColumnDescriptor(FAMILY)
186          .setKeepDeletedCells(true).setMaxVersions(3);
187 
188      HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(TABLENAME));
189      desc.addFamily(hcd);
190      TEST_UTIL.getHBaseAdmin().createTable(desc);
191      Configuration c = TEST_UTIL.getConfiguration();
192      HTable h = new HTable(c, TABLENAME);
193 
194      long ts = System.currentTimeMillis();
195      Put p = new Put(T1, ts);
196      p.add(FAMILY, C0, T1);
197      h.put(p);
198      p = new Put(T1, ts+2);
199      p.add(FAMILY, C0, T2);
200      h.put(p);
201      p = new Put(T1, ts+4);
202      p.add(FAMILY, C0, T3);
203      h.put(p);
204 
205      Delete d = new Delete(T1, ts+3);
206      h.delete(d);
207 
208      d = new Delete(T1, ts+3);
209      d.deleteColumns(FAMILY, C0, ts+3);
210      h.delete(d);
211 
212      Get g = new Get(T1);
213      // does *not* include the delete
214      g.setTimeRange(0, ts+3);
215      Result r = h.get(g);
216      assertArrayEquals(T2, r.getValue(FAMILY, C0));
217 
218      Scan s = new Scan(T1);
219      s.setTimeRange(0, ts+3);
220      s.setMaxVersions();
221      ResultScanner scanner = h.getScanner(s);
222      Cell[] kvs = scanner.next().rawCells();
223      assertArrayEquals(T2, CellUtil.cloneValue(kvs[0]));
224      assertArrayEquals(T1, CellUtil.cloneValue(kvs[1]));
225      scanner.close();
226 
227      s = new Scan(T1);
228      s.setRaw(true);
229      s.setMaxVersions();
230      scanner = h.getScanner(s);
231      kvs = scanner.next().rawCells();
232      assertTrue(CellUtil.isDeleteFamily(kvs[0]));
233      assertArrayEquals(T3, CellUtil.cloneValue(kvs[1]));
234      assertTrue(CellUtil.isDelete(kvs[2]));
235      assertArrayEquals(T2, CellUtil.cloneValue(kvs[3]));
236      assertArrayEquals(T1, CellUtil.cloneValue(kvs[4]));
237      scanner.close();
238      h.close();
239    }
240 
241     /**
242     * Basic client side validation of HBASE-10118
243     */
244    @Test
245    public void testPurgeFutureDeletes() throws Exception {
246      final byte[] TABLENAME = Bytes.toBytes("testPurgeFutureDeletes");
247      final byte[] ROW = Bytes.toBytes("row");
248      final byte[] FAMILY = Bytes.toBytes("family");
249      final byte[] COLUMN = Bytes.toBytes("column");
250      final byte[] VALUE = Bytes.toBytes("value");
251 
252      HTable table = TEST_UTIL.createTable(TABLENAME, FAMILY);
253 
254      // future timestamp
255      long ts = System.currentTimeMillis() * 2;
256      Put put = new Put(ROW, ts);
257      put.add(FAMILY, COLUMN, VALUE);
258      table.put(put);
259 
260      Get get = new Get(ROW);
261      Result result = table.get(get);
262      assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
263 
264      Delete del = new Delete(ROW);
265      del.deleteColumn(FAMILY, COLUMN, ts);
266      table.delete(del);
267 
268      get = new Get(ROW);
269      result = table.get(get);
270      assertNull(result.getValue(FAMILY, COLUMN));
271 
272      // major compaction, purged future deletes
273      TEST_UTIL.getHBaseAdmin().flush(TABLENAME);
274      TEST_UTIL.getHBaseAdmin().majorCompact(TABLENAME);
275 
276      // waiting for the major compaction to complete
277      TEST_UTIL.waitFor(6000, new Waiter.Predicate<IOException>() {
278        @Override
279        public boolean evaluate() throws IOException {
280          try {
281            return TEST_UTIL.getHBaseAdmin().getCompactionState(TABLENAME) ==
282                AdminProtos.GetRegionInfoResponse.CompactionState.NONE;
283          } catch (InterruptedException e) {
284            throw new IOException(e);
285          }
286        }
287      });
288 
289      put = new Put(ROW, ts);
290      put.add(FAMILY, COLUMN, VALUE);
291      table.put(put);
292 
293      get = new Get(ROW);
294      result = table.get(get);
295      assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
296 
297      table.close();
298    }
299 
300    @Test
301    public void testSharedZooKeeper() throws Exception {
302      Configuration newConfig = new Configuration(TEST_UTIL.getConfiguration());
303      newConfig.set(HConstants.HBASE_CLIENT_INSTANCE_ID, "12345");
304 
305      // First with a simple ZKW
306      ZooKeeperWatcher z0 = new ZooKeeperWatcher(
307        newConfig, "hconnection", new Abortable() {
308        @Override public void abort(String why, Throwable e) {}
309        @Override public boolean isAborted() {return false;}
310      });
311      z0.getRecoverableZooKeeper().getZooKeeper().exists("/oldZooKeeperWatcher", false);
312      z0.close();
313 
314      // Then a ZooKeeperKeepAliveConnection
315      HConnectionManager.HConnectionImplementation connection1 =
316        (HConnectionManager.HConnectionImplementation)
317          HConnectionManager.getConnection(newConfig);
318 
319      ZooKeeperKeepAliveConnection z1 = connection1.getKeepAliveZooKeeperWatcher();
320      z1.getRecoverableZooKeeper().getZooKeeper().exists("/z1", false);
321 
322      z1.close();
323 
324      // will still work, because the real connection is not closed yet
325      // Not do be done in real code
326      z1.getRecoverableZooKeeper().getZooKeeper().exists("/z1afterclose", false);
327 
328 
329      ZooKeeperKeepAliveConnection z2 = connection1.getKeepAliveZooKeeperWatcher();
330      assertTrue(
331        "ZooKeeperKeepAliveConnection equals on same connection", z1 == z2);
332 
333 
334 
335      Configuration newConfig2 = new Configuration(TEST_UTIL.getConfiguration());
336      newConfig2.set(HConstants.HBASE_CLIENT_INSTANCE_ID, "6789");
337      HConnectionManager.HConnectionImplementation connection2 =
338        (HConnectionManager.HConnectionImplementation)
339          HConnectionManager.getConnection(newConfig2);
340 
341      assertTrue("connections should be different ", connection1 != connection2);
342 
343      ZooKeeperKeepAliveConnection z3 = connection2.getKeepAliveZooKeeperWatcher();
344      assertTrue(
345        "ZooKeeperKeepAliveConnection should be different" +
346          " on different connections", z1 != z3);
347 
348      // Bypass the private access
349      Method m = HConnectionManager.HConnectionImplementation.class.
350        getDeclaredMethod("closeZooKeeperWatcher");
351      m.setAccessible(true);
352      m.invoke(connection2);
353 
354      ZooKeeperKeepAliveConnection z4 = connection2.getKeepAliveZooKeeperWatcher();
355      assertTrue(
356        "ZooKeeperKeepAliveConnection should be recreated" +
357          " when previous connections was closed"
358        , z3 != z4);
359 
360 
361      z2.getRecoverableZooKeeper().getZooKeeper().exists("/z2", false);
362      z4.getRecoverableZooKeeper().getZooKeeper().exists("/z4", false);
363 
364 
365      HConnectionManager.deleteConnection(newConfig);
366      try {
367        z2.getRecoverableZooKeeper().getZooKeeper().exists("/z2", false);
368        assertTrue("We should not have a valid connection for z2", false);
369      } catch (Exception e){
370      }
371 
372      z4.getRecoverableZooKeeper().getZooKeeper().exists("/z4", false);
373      // We expect success here.
374 
375 
376      HConnectionManager.deleteConnection(newConfig2);
377      try {
378        z4.getRecoverableZooKeeper().getZooKeeper().exists("/z4", false);
379        assertTrue("We should not have a valid connection for z4", false);
380      } catch (Exception e){
381      }
382    }
383 
384   /**
385    * HBASE-2468 use case 3:
386    */
387   @Test
388   public void testRegionCachePreWarm() throws Exception {
389     LOG.info("Starting testRegionCachePreWarm");
390     final TableName TABLENAME =
391         TableName.valueOf("testCachePrewarm");
392     Configuration conf = TEST_UTIL.getConfiguration();
393 
394     // Set up test table:
395     // Create table:
396     TEST_UTIL.createTable(TABLENAME, FAMILY);
397 
398     // disable region cache for the table.
399     HTable.setRegionCachePrefetch(conf, TABLENAME, false);
400     assertFalse("The table is disabled for region cache prefetch",
401         HTable.getRegionCachePrefetch(conf, TABLENAME));
402 
403     HTable table = new HTable(conf, TABLENAME);
404 
405     // create many regions for the table.
406     TEST_UTIL.createMultiRegions(table, FAMILY);
407     // This count effectively waits until the regions have been
408     // fully assigned
409     TEST_UTIL.countRows(table);
410     table.getConnection().clearRegionCache();
411     assertEquals("Clearing cache should have 0 cached ", 0,
412         HConnectionManager.getCachedRegionCount(conf, TABLENAME));
413 
414     // A Get is suppose to do a region lookup request
415     Get g = new Get(Bytes.toBytes("aaa"));
416     table.get(g);
417 
418     // only one region should be cached if the cache prefetch is disabled.
419     assertEquals("Number of cached region is incorrect ", 1,
420         HConnectionManager.getCachedRegionCount(conf, TABLENAME));
421 
422     // now we enable cached prefetch.
423     HTable.setRegionCachePrefetch(conf, TABLENAME, true);
424     assertTrue("The table is enabled for region cache prefetch",
425         HTable.getRegionCachePrefetch(conf, TABLENAME));
426 
427     HTable.setRegionCachePrefetch(conf, TABLENAME, false);
428     assertFalse("The table is disabled for region cache prefetch",
429         HTable.getRegionCachePrefetch(conf, TABLENAME));
430 
431     HTable.setRegionCachePrefetch(conf, TABLENAME, true);
432     assertTrue("The table is enabled for region cache prefetch",
433         HTable.getRegionCachePrefetch(conf, TABLENAME));
434 
435     table.getConnection().clearRegionCache();
436 
437     assertEquals("Number of cached region is incorrect ", 0,
438         HConnectionManager.getCachedRegionCount(conf, TABLENAME));
439 
440     // if there is a cache miss, some additional regions should be prefetched.
441     Get g2 = new Get(Bytes.toBytes("bbb"));
442     table.get(g2);
443 
444     // Get the configured number of cache read-ahead regions.
445     int prefetchRegionNumber = conf.getInt("hbase.client.prefetch.limit", 10);
446 
447     // the total number of cached regions == region('aaa") + prefeched regions.
448     LOG.info("Testing how many regions cached");
449     assertEquals("Number of cached region is incorrect ", prefetchRegionNumber,
450         HConnectionManager.getCachedRegionCount(conf, TABLENAME));
451 
452     table.getConnection().clearRegionCache();
453 
454     Get g3 = new Get(Bytes.toBytes("abc"));
455     table.get(g3);
456     assertEquals("Number of cached region is incorrect ", prefetchRegionNumber,
457         HConnectionManager.getCachedRegionCount(conf, TABLENAME));
458 
459     LOG.info("Finishing testRegionCachePreWarm");
460   }
461 
462   @Test
463   public void testPrefetchDisableToggle() throws Exception {
464     final TableName TABLENAME = TableName.valueOf("testPrefetchDisableToggle");
465     Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
466     conf.setBoolean(HConstants.HBASE_CLIENT_PREFETCH, false);
467     // We have to create a connection for the test because we are changing a
468     // setting normally picked up from the site configuration
469     HConnection connection = HConnectionManager.createConnection(conf);
470     assertFalse("The table is not disabled for region cache prefetch",
471       ((HConnectionImplementation)connection).getRegionCachePrefetch(TABLENAME));
472   }
473 
474   /**
475    * Verifies that getConfiguration returns the same Configuration object used
476    * to create the HTable instance.
477    */
478   @Test
479   public void testGetConfiguration() throws Exception {
480     byte[] TABLE = Bytes.toBytes("testGetConfiguration");
481     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") };
482     Configuration conf = TEST_UTIL.getConfiguration();
483     HTable table = TEST_UTIL.createTable(TABLE, FAMILIES, conf);
484     assertSame(conf, table.getConfiguration());
485   }
486 
487   /**
488    * Test from client side of an involved filter against a multi family that
489    * involves deletes.
490    *
491    * @throws Exception
492    */
493   @Test
494   public void testWeirdCacheBehaviour() throws Exception {
495     byte [] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
496     byte [][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
497         Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
498         Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
499     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
500     String value = "this is the value";
501     String value2 = "this is some other value";
502     String keyPrefix1 = UUID.randomUUID().toString();
503     String keyPrefix2 = UUID.randomUUID().toString();
504     String keyPrefix3 = UUID.randomUUID().toString();
505     putRows(ht, 3, value, keyPrefix1);
506     putRows(ht, 3, value, keyPrefix2);
507     putRows(ht, 3, value, keyPrefix3);
508     ht.flushCommits();
509     putRows(ht, 3, value2, keyPrefix1);
510     putRows(ht, 3, value2, keyPrefix2);
511     putRows(ht, 3, value2, keyPrefix3);
512     HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLE);
513     System.out.println("Checking values for key: " + keyPrefix1);
514     assertEquals("Got back incorrect number of rows from scan", 3,
515         getNumberOfRows(keyPrefix1, value2, table));
516     System.out.println("Checking values for key: " + keyPrefix2);
517     assertEquals("Got back incorrect number of rows from scan", 3,
518         getNumberOfRows(keyPrefix2, value2, table));
519     System.out.println("Checking values for key: " + keyPrefix3);
520     assertEquals("Got back incorrect number of rows from scan", 3,
521         getNumberOfRows(keyPrefix3, value2, table));
522     deleteColumns(ht, value2, keyPrefix1);
523     deleteColumns(ht, value2, keyPrefix2);
524     deleteColumns(ht, value2, keyPrefix3);
525     System.out.println("Starting important checks.....");
526     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,
527       0, getNumberOfRows(keyPrefix1, value2, table));
528     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,
529       0, getNumberOfRows(keyPrefix2, value2, table));
530     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,
531       0, getNumberOfRows(keyPrefix3, value2, table));
532     ht.setScannerCaching(0);
533     assertEquals("Got back incorrect number of rows from scan", 0,
534       getNumberOfRows(keyPrefix1, value2, table)); ht.setScannerCaching(100);
535     assertEquals("Got back incorrect number of rows from scan", 0,
536       getNumberOfRows(keyPrefix2, value2, table));
537   }
538 
539   private void deleteColumns(HTable ht, String value, String keyPrefix)
540   throws IOException {
541     ResultScanner scanner = buildScanner(keyPrefix, value, ht);
542     Iterator<Result> it = scanner.iterator();
543     int count = 0;
544     while (it.hasNext()) {
545       Result result = it.next();
546       Delete delete = new Delete(result.getRow());
547       delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
548       ht.delete(delete);
549       count++;
550     }
551     assertEquals("Did not perform correct number of deletes", 3, count);
552   }
553 
554   private int getNumberOfRows(String keyPrefix, String value, HTable ht)
555       throws Exception {
556     ResultScanner resultScanner = buildScanner(keyPrefix, value, ht);
557     Iterator<Result> scanner = resultScanner.iterator();
558     int numberOfResults = 0;
559     while (scanner.hasNext()) {
560       Result result = scanner.next();
561       System.out.println("Got back key: " + Bytes.toString(result.getRow()));
562       for (Cell kv : result.rawCells()) {
563         System.out.println("kv=" + kv.toString() + ", "
564             + Bytes.toString(CellUtil.cloneValue(kv)));
565       }
566       numberOfResults++;
567     }
568     return numberOfResults;
569   }
570 
571   private ResultScanner buildScanner(String keyPrefix, String value, HTable ht)
572       throws IOException {
573     // OurFilterList allFilters = new OurFilterList();
574     FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */);
575     allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
576     SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes
577         .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes
578         .toBytes(value));
579     filter.setFilterIfMissing(true);
580     allFilters.addFilter(filter);
581 
582     // allFilters.addFilter(new
583     // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"),
584     // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)));
585 
586     Scan scan = new Scan();
587     scan.addFamily(Bytes.toBytes("trans-blob"));
588     scan.addFamily(Bytes.toBytes("trans-type"));
589     scan.addFamily(Bytes.toBytes("trans-date"));
590     scan.addFamily(Bytes.toBytes("trans-tags"));
591     scan.addFamily(Bytes.toBytes("trans-group"));
592     scan.setFilter(allFilters);
593 
594     return ht.getScanner(scan);
595   }
596 
597   private void putRows(HTable ht, int numRows, String value, String key)
598       throws IOException {
599     for (int i = 0; i < numRows; i++) {
600       String row = key + "_" + UUID.randomUUID().toString();
601       System.out.println(String.format("Saving row: %s, with value %s", row,
602           value));
603       Put put = new Put(Bytes.toBytes(row));
604       put.setDurability(Durability.SKIP_WAL);
605       put.add(Bytes.toBytes("trans-blob"), null, Bytes
606           .toBytes("value for blob"));
607       put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
608       put.add(Bytes.toBytes("trans-date"), null, Bytes
609           .toBytes("20090921010101999"));
610       put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes
611           .toBytes(value));
612       put.add(Bytes.toBytes("trans-group"), null, Bytes
613           .toBytes("adhocTransactionGroupId"));
614       ht.put(put);
615     }
616   }
617 
618   /**
619    * Test filters when multiple regions.  It does counts.  Needs eye-balling of
620    * logs to ensure that we're not scanning more regions that we're supposed to.
621    * Related to the TestFilterAcrossRegions over in the o.a.h.h.filter package.
622    * @throws IOException
623    * @throws InterruptedException
624    */
625   @Test
626   public void testFilterAcrossMultipleRegions()
627   throws IOException, InterruptedException {
628     byte [] name = Bytes.toBytes("testFilterAcrossMutlipleRegions");
629     HTable t = TEST_UTIL.createTable(name, FAMILY);
630     int rowCount = TEST_UTIL.loadTable(t, FAMILY, false);
631     assertRowCount(t, rowCount);
632     // Split the table.  Should split on a reasonable key; 'lqj'
633     Map<HRegionInfo, ServerName> regions  = splitTable(t);
634     assertRowCount(t, rowCount);
635     // Get end key of first region.
636     byte [] endKey = regions.keySet().iterator().next().getEndKey();
637     // Count rows with a filter that stops us before passed 'endKey'.
638     // Should be count of rows in first region.
639     int endKeyCount = countRows(t, createScanWithRowFilter(endKey));
640     assertTrue(endKeyCount < rowCount);
641 
642     // How do I know I did not got to second region?  Thats tough.  Can't really
643     // do that in client-side region test.  I verified by tracing in debugger.
644     // I changed the messages that come out when set to DEBUG so should see
645     // when scanner is done. Says "Finished with scanning..." with region name.
646     // Check that its finished in right region.
647 
648     // New test.  Make it so scan goes into next region by one and then two.
649     // Make sure count comes out right.
650     byte [] key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 1)};
651     int plusOneCount = countRows(t, createScanWithRowFilter(key));
652     assertEquals(endKeyCount + 1, plusOneCount);
653     key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 2)};
654     int plusTwoCount = countRows(t, createScanWithRowFilter(key));
655     assertEquals(endKeyCount + 2, plusTwoCount);
656 
657     // New test.  Make it so I scan one less than endkey.
658     key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] - 1)};
659     int minusOneCount = countRows(t, createScanWithRowFilter(key));
660     assertEquals(endKeyCount - 1, minusOneCount);
661     // For above test... study logs.  Make sure we do "Finished with scanning.."
662     // in first region and that we do not fall into the next region.
663 
664     key = new byte [] {'a', 'a', 'a'};
665     int countBBB = countRows(t,
666       createScanWithRowFilter(key, null, CompareFilter.CompareOp.EQUAL));
667     assertEquals(1, countBBB);
668 
669     int countGreater = countRows(t, createScanWithRowFilter(endKey, null,
670       CompareFilter.CompareOp.GREATER_OR_EQUAL));
671     // Because started at start of table.
672     assertEquals(0, countGreater);
673     countGreater = countRows(t, createScanWithRowFilter(endKey, endKey,
674       CompareFilter.CompareOp.GREATER_OR_EQUAL));
675     assertEquals(rowCount - endKeyCount, countGreater);
676   }
677 
678   /*
679    * @param key
680    * @return Scan with RowFilter that does LESS than passed key.
681    */
682   private Scan createScanWithRowFilter(final byte [] key) {
683     return createScanWithRowFilter(key, null, CompareFilter.CompareOp.LESS);
684   }
685 
686   /*
687    * @param key
688    * @param op
689    * @param startRow
690    * @return Scan with RowFilter that does CompareOp op on passed key.
691    */
692   private Scan createScanWithRowFilter(final byte [] key,
693       final byte [] startRow, CompareFilter.CompareOp op) {
694     // Make sure key is of some substance... non-null and > than first key.
695     assertTrue(key != null && key.length > 0 &&
696       Bytes.BYTES_COMPARATOR.compare(key, new byte [] {'a', 'a', 'a'}) >= 0);
697     LOG.info("Key=" + Bytes.toString(key));
698     Scan s = startRow == null? new Scan(): new Scan(startRow);
699     Filter f = new RowFilter(op, new BinaryComparator(key));
700     f = new WhileMatchFilter(f);
701     s.setFilter(f);
702     return s;
703   }
704 
705   /*
706    * @param t
707    * @param s
708    * @return Count of rows in table.
709    * @throws IOException
710    */
711   private int countRows(final HTable t, final Scan s)
712   throws IOException {
713     // Assert all rows in table.
714     ResultScanner scanner = t.getScanner(s);
715     int count = 0;
716     for (Result result: scanner) {
717       count++;
718       assertTrue(result.size() > 0);
719       // LOG.info("Count=" + count + ", row=" + Bytes.toString(result.getRow()));
720     }
721     return count;
722   }
723 
724   private void assertRowCount(final HTable t, final int expected)
725   throws IOException {
726     assertEquals(expected, countRows(t, new Scan()));
727   }
728 
729   /*
730    * Split table into multiple regions.
731    * @param t Table to split.
732    * @return Map of regions to servers.
733    * @throws IOException
734    */
735   private Map<HRegionInfo, ServerName> splitTable(final HTable t)
736   throws IOException, InterruptedException {
737     // Split this table in two.
738     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
739     admin.split(t.getTableName());
740     admin.close();
741     Map<HRegionInfo, ServerName> regions = waitOnSplit(t);
742     assertTrue(regions.size() > 1);
743     return regions;
744   }
745 
746   /*
747    * Wait on table split.  May return because we waited long enough on the split
748    * and it didn't happen.  Caller should check.
749    * @param t
750    * @return Map of table regions; caller needs to check table actually split.
751    */
752   private Map<HRegionInfo, ServerName> waitOnSplit(final HTable t)
753   throws IOException {
754     Map<HRegionInfo, ServerName> regions = t.getRegionLocations();
755     int originalCount = regions.size();
756     for (int i = 0; i < TEST_UTIL.getConfiguration().getInt("hbase.test.retries", 30); i++) {
757       Thread.currentThread();
758       try {
759         Thread.sleep(1000);
760       } catch (InterruptedException e) {
761         e.printStackTrace();
762       }
763       regions = t.getRegionLocations();
764       if (regions.size() > originalCount) break;
765     }
766     return regions;
767   }
768 
769   @Test
770   public void testSuperSimple() throws Exception {
771     byte [] TABLE = Bytes.toBytes("testSuperSimple");
772     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
773     Put put = new Put(ROW);
774     put.add(FAMILY, QUALIFIER, VALUE);
775     ht.put(put);
776     Scan scan = new Scan();
777     scan.addColumn(FAMILY, TABLE);
778     ResultScanner scanner = ht.getScanner(scan);
779     Result result = scanner.next();
780     assertTrue("Expected null result", result == null);
781     scanner.close();
782   }
783 
784   @Test
785   public void testMaxKeyValueSize() throws Exception {
786     byte [] TABLE = Bytes.toBytes("testMaxKeyValueSize");
787     Configuration conf = TEST_UTIL.getConfiguration();
788     String oldMaxSize = conf.get("hbase.client.keyvalue.maxsize");
789     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
790     byte[] value = new byte[4 * 1024 * 1024];
791     Put put = new Put(ROW);
792     put.add(FAMILY, QUALIFIER, value);
793     ht.put(put);
794     try {
795       conf.setInt("hbase.client.keyvalue.maxsize", 2 * 1024 * 1024);
796       TABLE = Bytes.toBytes("testMaxKeyValueSize2");
797       ht = TEST_UTIL.createTable(TABLE, FAMILY);
798       put = new Put(ROW);
799       put.add(FAMILY, QUALIFIER, value);
800       ht.put(put);
801       fail("Inserting a too large KeyValue worked, should throw exception");
802     } catch(Exception e) {}
803     conf.set("hbase.client.keyvalue.maxsize", oldMaxSize);
804   }
805 
806   @Test
807   public void testFilters() throws Exception {
808     byte [] TABLE = Bytes.toBytes("testFilters");
809     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
810     byte [][] ROWS = makeN(ROW, 10);
811     byte [][] QUALIFIERS = {
812         Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
813         Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
814         Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
815         Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
816         Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>")
817     };
818     for(int i=0;i<10;i++) {
819       Put put = new Put(ROWS[i]);
820       put.setDurability(Durability.SKIP_WAL);
821       put.add(FAMILY, QUALIFIERS[i], VALUE);
822       ht.put(put);
823     }
824     Scan scan = new Scan();
825     scan.addFamily(FAMILY);
826     Filter filter = new QualifierFilter(CompareOp.EQUAL,
827       new RegexStringComparator("col[1-5]"));
828     scan.setFilter(filter);
829     ResultScanner scanner = ht.getScanner(scan);
830     int expectedIndex = 1;
831     for(Result result : ht.getScanner(scan)) {
832       assertEquals(result.size(), 1);
833       assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[expectedIndex]));
834       assertTrue(Bytes.equals(CellUtil.cloneQualifier(result.rawCells()[0]),
835           QUALIFIERS[expectedIndex]));
836       expectedIndex++;
837     }
838     assertEquals(expectedIndex, 6);
839     scanner.close();
840   }
841 
842   @Test
843   public void testFilterWithLongCompartor() throws Exception {
844     byte [] TABLE = Bytes.toBytes("testFilterWithLongCompartor");
845     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
846     byte [][] ROWS = makeN(ROW, 10);
847     byte [][] values = new byte[10][];
848     for (int i = 0; i < 10; i ++) {
849         values[i] = Bytes.toBytes(100L * i);
850     }
851     for(int i = 0; i < 10; i ++) {
852       Put put = new Put(ROWS[i]);
853       put.setDurability(Durability.SKIP_WAL);
854       put.add(FAMILY, QUALIFIER, values[i]);
855       ht.put(put);
856     }
857     Scan scan = new Scan();
858     scan.addFamily(FAMILY);
859     Filter filter = new SingleColumnValueFilter(FAMILY, QUALIFIER, CompareOp.GREATER,
860       new LongComparator(500));
861     scan.setFilter(filter);
862     ResultScanner scanner = ht.getScanner(scan);
863     int expectedIndex = 0;
864     for(Result result : ht.getScanner(scan)) {
865       assertEquals(result.size(), 1);
866       assertTrue(Bytes.toLong(result.getValue(FAMILY, QUALIFIER)) > 500);
867       expectedIndex++;
868     }
869     assertEquals(expectedIndex, 4);
870     scanner.close();
871 }
872 
873   @Test
874   public void testKeyOnlyFilter() throws Exception {
875     byte [] TABLE = Bytes.toBytes("testKeyOnlyFilter");
876     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
877     byte [][] ROWS = makeN(ROW, 10);
878     byte [][] QUALIFIERS = {
879         Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
880         Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
881         Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
882         Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
883         Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>")
884     };
885     for(int i=0;i<10;i++) {
886       Put put = new Put(ROWS[i]);
887       put.setDurability(Durability.SKIP_WAL);
888       put.add(FAMILY, QUALIFIERS[i], VALUE);
889       ht.put(put);
890     }
891     Scan scan = new Scan();
892     scan.addFamily(FAMILY);
893     Filter filter = new KeyOnlyFilter(true);
894     scan.setFilter(filter);
895     ResultScanner scanner = ht.getScanner(scan);
896     int count = 0;
897     for(Result result : ht.getScanner(scan)) {
898       assertEquals(result.size(), 1);
899       assertEquals(result.rawCells()[0].getValueLength(), Bytes.SIZEOF_INT);
900       assertEquals(Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])), VALUE.length);
901       count++;
902     }
903     assertEquals(count, 10);
904     scanner.close();
905   }
906 
907   /**
908    * Test simple table and non-existent row cases.
909    */
910   @Test
911   public void testSimpleMissing() throws Exception {
912     byte [] TABLE = Bytes.toBytes("testSimpleMissing");
913     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
914     byte [][] ROWS = makeN(ROW, 4);
915 
916     // Try to get a row on an empty table
917     Get get = new Get(ROWS[0]);
918     Result result = ht.get(get);
919     assertEmptyResult(result);
920 
921     get = new Get(ROWS[0]);
922     get.addFamily(FAMILY);
923     result = ht.get(get);
924     assertEmptyResult(result);
925 
926     get = new Get(ROWS[0]);
927     get.addColumn(FAMILY, QUALIFIER);
928     result = ht.get(get);
929     assertEmptyResult(result);
930 
931     Scan scan = new Scan();
932     result = getSingleScanResult(ht, scan);
933     assertNullResult(result);
934 
935 
936     scan = new Scan(ROWS[0]);
937     result = getSingleScanResult(ht, scan);
938     assertNullResult(result);
939 
940     scan = new Scan(ROWS[0],ROWS[1]);
941     result = getSingleScanResult(ht, scan);
942     assertNullResult(result);
943 
944     scan = new Scan();
945     scan.addFamily(FAMILY);
946     result = getSingleScanResult(ht, scan);
947     assertNullResult(result);
948 
949     scan = new Scan();
950     scan.addColumn(FAMILY, QUALIFIER);
951     result = getSingleScanResult(ht, scan);
952     assertNullResult(result);
953 
954     // Insert a row
955 
956     Put put = new Put(ROWS[2]);
957     put.add(FAMILY, QUALIFIER, VALUE);
958     ht.put(put);
959 
960     // Try to get empty rows around it
961 
962     get = new Get(ROWS[1]);
963     result = ht.get(get);
964     assertEmptyResult(result);
965 
966     get = new Get(ROWS[0]);
967     get.addFamily(FAMILY);
968     result = ht.get(get);
969     assertEmptyResult(result);
970 
971     get = new Get(ROWS[3]);
972     get.addColumn(FAMILY, QUALIFIER);
973     result = ht.get(get);
974     assertEmptyResult(result);
975 
976     // Try to scan empty rows around it
977 
978     scan = new Scan(ROWS[3]);
979     result = getSingleScanResult(ht, scan);
980     assertNullResult(result);
981 
982     scan = new Scan(ROWS[0],ROWS[2]);
983     result = getSingleScanResult(ht, scan);
984     assertNullResult(result);
985 
986     // Make sure we can actually get the row
987 
988     get = new Get(ROWS[2]);
989     result = ht.get(get);
990     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
991 
992     get = new Get(ROWS[2]);
993     get.addFamily(FAMILY);
994     result = ht.get(get);
995     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
996 
997     get = new Get(ROWS[2]);
998     get.addColumn(FAMILY, QUALIFIER);
999     result = ht.get(get);
1000     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
1001 
1002     // Make sure we can scan the row
1003 
1004     scan = new Scan();
1005     result = getSingleScanResult(ht, scan);
1006     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
1007 
1008     scan = new Scan(ROWS[0],ROWS[3]);
1009     result = getSingleScanResult(ht, scan);
1010     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
1011 
1012     scan = new Scan(ROWS[2],ROWS[3]);
1013     result = getSingleScanResult(ht, scan);
1014     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
1015   }
1016 
1017   /**
1018    * Test basic puts, gets, scans, and deletes for a single row
1019    * in a multiple family table.
1020    */
1021   @Test
1022   public void testSingleRowMultipleFamily() throws Exception {
1023     byte [] TABLE = Bytes.toBytes("testSingleRowMultipleFamily");
1024     byte [][] ROWS = makeN(ROW, 3);
1025     byte [][] FAMILIES = makeNAscii(FAMILY, 10);
1026     byte [][] QUALIFIERS = makeN(QUALIFIER, 10);
1027     byte [][] VALUES = makeN(VALUE, 10);
1028 
1029     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
1030 
1031     Get get;
1032     Scan scan;
1033     Delete delete;
1034     Put put;
1035     Result result;
1036 
1037     ////////////////////////////////////////////////////////////////////////////
1038     // Insert one column to one family
1039     ////////////////////////////////////////////////////////////////////////////
1040 
1041     put = new Put(ROWS[0]);
1042     put.add(FAMILIES[4], QUALIFIERS[0], VALUES[0]);
1043     ht.put(put);
1044 
1045     // Get the single column
1046     getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
1047 
1048     // Scan the single column
1049     scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
1050 
1051     // Get empty results around inserted column
1052     getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
1053 
1054     // Scan empty results around inserted column
1055     scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
1056 
1057     ////////////////////////////////////////////////////////////////////////////
1058     // Flush memstore and run same tests from storefiles
1059     ////////////////////////////////////////////////////////////////////////////
1060 
1061     TEST_UTIL.flush();
1062 
1063     // Redo get and scan tests from storefile
1064     getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
1065     scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
1066     getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
1067     scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
1068 
1069     ////////////////////////////////////////////////////////////////////////////
1070     // Now, Test reading from memstore and storefiles at once
1071     ////////////////////////////////////////////////////////////////////////////
1072 
1073     // Insert multiple columns to two other families
1074     put = new Put(ROWS[0]);
1075     put.add(FAMILIES[2], QUALIFIERS[2], VALUES[2]);
1076     put.add(FAMILIES[2], QUALIFIERS[4], VALUES[4]);
1077     put.add(FAMILIES[4], QUALIFIERS[4], VALUES[4]);
1078     put.add(FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1079     put.add(FAMILIES[6], QUALIFIERS[7], VALUES[7]);
1080     put.add(FAMILIES[7], QUALIFIERS[7], VALUES[7]);
1081     put.add(FAMILIES[9], QUALIFIERS[0], VALUES[0]);
1082     ht.put(put);
1083 
1084     // Get multiple columns across multiple families and get empties around it
1085     singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1086 
1087     // Scan multiple columns across multiple families and scan empties around it
1088     singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1089 
1090     ////////////////////////////////////////////////////////////////////////////
1091     // Flush the table again
1092     ////////////////////////////////////////////////////////////////////////////
1093 
1094     TEST_UTIL.flush();
1095 
1096     // Redo tests again
1097     singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1098     singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1099 
1100     // Insert more data to memstore
1101     put = new Put(ROWS[0]);
1102     put.add(FAMILIES[6], QUALIFIERS[5], VALUES[5]);
1103     put.add(FAMILIES[6], QUALIFIERS[8], VALUES[8]);
1104     put.add(FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1105     put.add(FAMILIES[4], QUALIFIERS[3], VALUES[3]);
1106     ht.put(put);
1107 
1108     ////////////////////////////////////////////////////////////////////////////
1109     // Delete a storefile column
1110     ////////////////////////////////////////////////////////////////////////////
1111     delete = new Delete(ROWS[0]);
1112     delete.deleteColumns(FAMILIES[6], QUALIFIERS[7]);
1113     ht.delete(delete);
1114 
1115     // Try to get deleted column
1116     get = new Get(ROWS[0]);
1117     get.addColumn(FAMILIES[6], QUALIFIERS[7]);
1118     result = ht.get(get);
1119     assertEmptyResult(result);
1120 
1121     // Try to scan deleted column
1122     scan = new Scan();
1123     scan.addColumn(FAMILIES[6], QUALIFIERS[7]);
1124     result = getSingleScanResult(ht, scan);
1125     assertNullResult(result);
1126 
1127     // Make sure we can still get a column before it and after it
1128     get = new Get(ROWS[0]);
1129     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
1130     result = ht.get(get);
1131     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1132 
1133     get = new Get(ROWS[0]);
1134     get.addColumn(FAMILIES[6], QUALIFIERS[8]);
1135     result = ht.get(get);
1136     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]);
1137 
1138     // Make sure we can still scan a column before it and after it
1139     scan = new Scan();
1140     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1141     result = getSingleScanResult(ht, scan);
1142     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1143 
1144     scan = new Scan();
1145     scan.addColumn(FAMILIES[6], QUALIFIERS[8]);
1146     result = getSingleScanResult(ht, scan);
1147     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]);
1148 
1149     ////////////////////////////////////////////////////////////////////////////
1150     // Delete a memstore column
1151     ////////////////////////////////////////////////////////////////////////////
1152     delete = new Delete(ROWS[0]);
1153     delete.deleteColumns(FAMILIES[6], QUALIFIERS[8]);
1154     ht.delete(delete);
1155 
1156     // Try to get deleted column
1157     get = new Get(ROWS[0]);
1158     get.addColumn(FAMILIES[6], QUALIFIERS[8]);
1159     result = ht.get(get);
1160     assertEmptyResult(result);
1161 
1162     // Try to scan deleted column
1163     scan = new Scan();
1164     scan.addColumn(FAMILIES[6], QUALIFIERS[8]);
1165     result = getSingleScanResult(ht, scan);
1166     assertNullResult(result);
1167 
1168     // Make sure we can still get a column before it and after it
1169     get = new Get(ROWS[0]);
1170     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
1171     result = ht.get(get);
1172     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1173 
1174     get = new Get(ROWS[0]);
1175     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1176     result = ht.get(get);
1177     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1178 
1179     // Make sure we can still scan a column before it and after it
1180     scan = new Scan();
1181     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1182     result = getSingleScanResult(ht, scan);
1183     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1184 
1185     scan = new Scan();
1186     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1187     result = getSingleScanResult(ht, scan);
1188     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1189 
1190     ////////////////////////////////////////////////////////////////////////////
1191     // Delete joint storefile/memstore family
1192     ////////////////////////////////////////////////////////////////////////////
1193 
1194     delete = new Delete(ROWS[0]);
1195     delete.deleteFamily(FAMILIES[4]);
1196     ht.delete(delete);
1197 
1198     // Try to get storefile column in deleted family
1199     get = new Get(ROWS[0]);
1200     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
1201     result = ht.get(get);
1202     assertEmptyResult(result);
1203 
1204     // Try to get memstore column in deleted family
1205     get = new Get(ROWS[0]);
1206     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
1207     result = ht.get(get);
1208     assertEmptyResult(result);
1209 
1210     // Try to get deleted family
1211     get = new Get(ROWS[0]);
1212     get.addFamily(FAMILIES[4]);
1213     result = ht.get(get);
1214     assertEmptyResult(result);
1215 
1216     // Try to scan storefile column in deleted family
1217     scan = new Scan();
1218     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
1219     result = getSingleScanResult(ht, scan);
1220     assertNullResult(result);
1221 
1222     // Try to scan memstore column in deleted family
1223     scan = new Scan();
1224     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
1225     result = getSingleScanResult(ht, scan);
1226     assertNullResult(result);
1227 
1228     // Try to scan deleted family
1229     scan = new Scan();
1230     scan.addFamily(FAMILIES[4]);
1231     result = getSingleScanResult(ht, scan);
1232     assertNullResult(result);
1233 
1234     // Make sure we can still get another family
1235     get = new Get(ROWS[0]);
1236     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
1237     result = ht.get(get);
1238     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
1239 
1240     get = new Get(ROWS[0]);
1241     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1242     result = ht.get(get);
1243     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1244 
1245     // Make sure we can still scan another family
1246     scan = new Scan();
1247     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1248     result = getSingleScanResult(ht, scan);
1249     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1250 
1251     scan = new Scan();
1252     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1253     result = getSingleScanResult(ht, scan);
1254     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1255 
1256     ////////////////////////////////////////////////////////////////////////////
1257     // Flush everything and rerun delete tests
1258     ////////////////////////////////////////////////////////////////////////////
1259 
1260     TEST_UTIL.flush();
1261 
1262     // Try to get storefile column in deleted family
1263     get = new Get(ROWS[0]);
1264     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
1265     result = ht.get(get);
1266     assertEmptyResult(result);
1267 
1268     // Try to get memstore column in deleted family
1269     get = new Get(ROWS[0]);
1270     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
1271     result = ht.get(get);
1272     assertEmptyResult(result);
1273 
1274     // Try to get deleted family
1275     get = new Get(ROWS[0]);
1276     get.addFamily(FAMILIES[4]);
1277     result = ht.get(get);
1278     assertEmptyResult(result);
1279 
1280     // Try to scan storefile column in deleted family
1281     scan = new Scan();
1282     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
1283     result = getSingleScanResult(ht, scan);
1284     assertNullResult(result);
1285 
1286     // Try to scan memstore column in deleted family
1287     scan = new Scan();
1288     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
1289     result = getSingleScanResult(ht, scan);
1290     assertNullResult(result);
1291 
1292     // Try to scan deleted family
1293     scan = new Scan();
1294     scan.addFamily(FAMILIES[4]);
1295     result = getSingleScanResult(ht, scan);
1296     assertNullResult(result);
1297 
1298     // Make sure we can still get another family
1299     get = new Get(ROWS[0]);
1300     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
1301     result = ht.get(get);
1302     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
1303 
1304     get = new Get(ROWS[0]);
1305     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1306     result = ht.get(get);
1307     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1308 
1309     // Make sure we can still scan another family
1310     scan = new Scan();
1311     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1312     result = getSingleScanResult(ht, scan);
1313     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1314 
1315     scan = new Scan();
1316     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1317     result = getSingleScanResult(ht, scan);
1318     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1319 
1320   }
1321 
1322   @Test
1323   public void testNull() throws Exception {
1324     byte [] TABLE = Bytes.toBytes("testNull");
1325 
1326     // Null table name (should NOT work)
1327     try {
1328       TEST_UTIL.createTable((TableName)null, FAMILY);
1329       fail("Creating a table with null name passed, should have failed");
1330     } catch(Exception e) {}
1331 
1332     // Null family (should NOT work)
1333     try {
1334       TEST_UTIL.createTable(TABLE, (byte[])null);
1335       fail("Creating a table with a null family passed, should fail");
1336     } catch(Exception e) {}
1337 
1338     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
1339 
1340     // Null row (should NOT work)
1341     try {
1342       Put put = new Put((byte[])null);
1343       put.add(FAMILY, QUALIFIER, VALUE);
1344       ht.put(put);
1345       fail("Inserting a null row worked, should throw exception");
1346     } catch(Exception e) {}
1347 
1348     // Null qualifier (should work)
1349     {
1350       Put put = new Put(ROW);
1351       put.add(FAMILY, null, VALUE);
1352       ht.put(put);
1353 
1354       getTestNull(ht, ROW, FAMILY, VALUE);
1355 
1356       scanTestNull(ht, ROW, FAMILY, VALUE);
1357 
1358       Delete delete = new Delete(ROW);
1359       delete.deleteColumns(FAMILY, null);
1360       ht.delete(delete);
1361 
1362       Get get = new Get(ROW);
1363       Result result = ht.get(get);
1364       assertEmptyResult(result);
1365     }
1366 
1367     // Use a new table
1368     byte [] TABLE2 = Bytes.toBytes("testNull2");
1369     ht = TEST_UTIL.createTable(TABLE2, FAMILY);
1370 
1371     // Empty qualifier, byte[0] instead of null (should work)
1372     try {
1373       Put put = new Put(ROW);
1374       put.add(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE);
1375       ht.put(put);
1376 
1377       getTestNull(ht, ROW, FAMILY, VALUE);
1378 
1379       scanTestNull(ht, ROW, FAMILY, VALUE);
1380 
1381       // Flush and try again
1382 
1383       TEST_UTIL.flush();
1384 
1385       getTestNull(ht, ROW, FAMILY, VALUE);
1386 
1387       scanTestNull(ht, ROW, FAMILY, VALUE);
1388 
1389       Delete delete = new Delete(ROW);
1390       delete.deleteColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY);
1391       ht.delete(delete);
1392 
1393       Get get = new Get(ROW);
1394       Result result = ht.get(get);
1395       assertEmptyResult(result);
1396 
1397     } catch(Exception e) {
1398       throw new IOException("Using a row with null qualifier threw exception, should ");
1399     }
1400 
1401     // Null value
1402     try {
1403       Put put = new Put(ROW);
1404       put.add(FAMILY, QUALIFIER, null);
1405       ht.put(put);
1406 
1407       Get get = new Get(ROW);
1408       get.addColumn(FAMILY, QUALIFIER);
1409       Result result = ht.get(get);
1410       assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
1411 
1412       Scan scan = new Scan();
1413       scan.addColumn(FAMILY, QUALIFIER);
1414       result = getSingleScanResult(ht, scan);
1415       assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
1416 
1417       Delete delete = new Delete(ROW);
1418       delete.deleteColumns(FAMILY, QUALIFIER);
1419       ht.delete(delete);
1420 
1421       get = new Get(ROW);
1422       result = ht.get(get);
1423       assertEmptyResult(result);
1424 
1425     } catch(Exception e) {
1426       throw new IOException("Null values should be allowed, but threw exception");
1427     }
1428   }
1429 
1430   @Test
1431   public void testVersions() throws Exception {
1432     byte [] TABLE = Bytes.toBytes("testVersions");
1433 
1434     long [] STAMPS = makeStamps(20);
1435     byte [][] VALUES = makeNAscii(VALUE, 20);
1436 
1437     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
1438 
1439     // Insert 4 versions of same column
1440     Put put = new Put(ROW);
1441     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1442     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1443     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1444     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1445     ht.put(put);
1446 
1447     // Verify we can get each one properly
1448     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1449     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1450     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1451     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1452     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1453     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1454     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1455     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1456 
1457     // Verify we don't accidentally get others
1458     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1459     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1460     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1461     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1462     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1463     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1464 
1465     // Ensure maxVersions in query is respected
1466     Get get = new Get(ROW);
1467     get.addColumn(FAMILY, QUALIFIER);
1468     get.setMaxVersions(2);
1469     Result result = ht.get(get);
1470     assertNResult(result, ROW, FAMILY, QUALIFIER,
1471         new long [] {STAMPS[4], STAMPS[5]},
1472         new byte[][] {VALUES[4], VALUES[5]},
1473         0, 1);
1474 
1475     Scan scan = new Scan(ROW);
1476     scan.addColumn(FAMILY, QUALIFIER);
1477     scan.setMaxVersions(2);
1478     result = getSingleScanResult(ht, scan);
1479     assertNResult(result, ROW, FAMILY, QUALIFIER,
1480         new long [] {STAMPS[4], STAMPS[5]},
1481         new byte[][] {VALUES[4], VALUES[5]},
1482         0, 1);
1483 
1484     // Flush and redo
1485 
1486     TEST_UTIL.flush();
1487 
1488     // Verify we can get each one properly
1489     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1490     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1491     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1492     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1493     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1494     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1495     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1496     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1497 
1498     // Verify we don't accidentally get others
1499     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1500     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1501     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1502     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1503     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1504     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1505 
1506     // Ensure maxVersions in query is respected
1507     get = new Get(ROW);
1508     get.addColumn(FAMILY, QUALIFIER);
1509     get.setMaxVersions(2);
1510     result = ht.get(get);
1511     assertNResult(result, ROW, FAMILY, QUALIFIER,
1512         new long [] {STAMPS[4], STAMPS[5]},
1513         new byte[][] {VALUES[4], VALUES[5]},
1514         0, 1);
1515 
1516     scan = new Scan(ROW);
1517     scan.addColumn(FAMILY, QUALIFIER);
1518     scan.setMaxVersions(2);
1519     result = getSingleScanResult(ht, scan);
1520     assertNResult(result, ROW, FAMILY, QUALIFIER,
1521         new long [] {STAMPS[4], STAMPS[5]},
1522         new byte[][] {VALUES[4], VALUES[5]},
1523         0, 1);
1524 
1525 
1526     // Add some memstore and retest
1527 
1528     // Insert 4 more versions of same column and a dupe
1529     put = new Put(ROW);
1530     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
1531     put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
1532     put.add(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1533     put.add(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
1534     ht.put(put);
1535 
1536     // Ensure maxVersions in query is respected
1537     get = new Get(ROW);
1538     get.addColumn(FAMILY, QUALIFIER);
1539     get.setMaxVersions();
1540     result = ht.get(get);
1541     assertNResult(result, ROW, FAMILY, QUALIFIER,
1542         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1543         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1544         0, 7);
1545 
1546     scan = new Scan(ROW);
1547     scan.addColumn(FAMILY, QUALIFIER);
1548     scan.setMaxVersions();
1549     result = getSingleScanResult(ht, scan);
1550     assertNResult(result, ROW, FAMILY, QUALIFIER,
1551         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1552         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1553         0, 7);
1554 
1555     get = new Get(ROW);
1556     get.setMaxVersions();
1557     result = ht.get(get);
1558     assertNResult(result, ROW, FAMILY, QUALIFIER,
1559         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1560         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1561         0, 7);
1562 
1563     scan = new Scan(ROW);
1564     scan.setMaxVersions();
1565     result = getSingleScanResult(ht, scan);
1566     assertNResult(result, ROW, FAMILY, QUALIFIER,
1567         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1568         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1569         0, 7);
1570 
1571     // Verify we can get each one properly
1572     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1573     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1574     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1575     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1576     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1577     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1578     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1579     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1580 
1581     // Verify we don't accidentally get others
1582     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1583     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
1584     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1585     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
1586 
1587     // Ensure maxVersions of table is respected
1588 
1589     TEST_UTIL.flush();
1590 
1591     // Insert 4 more versions of same column and a dupe
1592     put = new Put(ROW);
1593     put.add(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
1594     put.add(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
1595     put.add(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
1596     put.add(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
1597     ht.put(put);
1598 
1599     get = new Get(ROW);
1600     get.addColumn(FAMILY, QUALIFIER);
1601     get.setMaxVersions(Integer.MAX_VALUE);
1602     result = ht.get(get);
1603     assertNResult(result, ROW, FAMILY, QUALIFIER,
1604         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
1605         new byte[][] {VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
1606         0, 9);
1607 
1608     scan = new Scan(ROW);
1609     scan.addColumn(FAMILY, QUALIFIER);
1610     scan.setMaxVersions(Integer.MAX_VALUE);
1611     result = getSingleScanResult(ht, scan);
1612     assertNResult(result, ROW, FAMILY, QUALIFIER,
1613         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
1614         new byte[][] {VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
1615         0, 9);
1616 
1617     // Delete a version in the memstore and a version in a storefile
1618     Delete delete = new Delete(ROW);
1619     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[11]);
1620     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[7]);
1621     ht.delete(delete);
1622 
1623     // Test that it's gone
1624     get = new Get(ROW);
1625     get.addColumn(FAMILY, QUALIFIER);
1626     get.setMaxVersions(Integer.MAX_VALUE);
1627     result = ht.get(get);
1628     assertNResult(result, ROW, FAMILY, QUALIFIER,
1629         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
1630         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
1631         0, 9);
1632 
1633     scan = new Scan(ROW);
1634     scan.addColumn(FAMILY, QUALIFIER);
1635     scan.setMaxVersions(Integer.MAX_VALUE);
1636     result = getSingleScanResult(ht, scan);
1637     assertNResult(result, ROW, FAMILY, QUALIFIER,
1638         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
1639         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
1640         0, 9);
1641 
1642   }
1643 
1644   @Test
1645   public void testVersionLimits() throws Exception {
1646     byte [] TABLE = Bytes.toBytes("testVersionLimits");
1647     byte [][] FAMILIES = makeNAscii(FAMILY, 3);
1648     int [] LIMITS = {1,3,5};
1649     long [] STAMPS = makeStamps(10);
1650     byte [][] VALUES = makeNAscii(VALUE, 10);
1651     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, LIMITS);
1652 
1653     // Insert limit + 1 on each family
1654     Put put = new Put(ROW);
1655     put.add(FAMILIES[0], QUALIFIER, STAMPS[0], VALUES[0]);
1656     put.add(FAMILIES[0], QUALIFIER, STAMPS[1], VALUES[1]);
1657     put.add(FAMILIES[1], QUALIFIER, STAMPS[0], VALUES[0]);
1658     put.add(FAMILIES[1], QUALIFIER, STAMPS[1], VALUES[1]);
1659     put.add(FAMILIES[1], QUALIFIER, STAMPS[2], VALUES[2]);
1660     put.add(FAMILIES[1], QUALIFIER, STAMPS[3], VALUES[3]);
1661     put.add(FAMILIES[2], QUALIFIER, STAMPS[0], VALUES[0]);
1662     put.add(FAMILIES[2], QUALIFIER, STAMPS[1], VALUES[1]);
1663     put.add(FAMILIES[2], QUALIFIER, STAMPS[2], VALUES[2]);
1664     put.add(FAMILIES[2], QUALIFIER, STAMPS[3], VALUES[3]);
1665     put.add(FAMILIES[2], QUALIFIER, STAMPS[4], VALUES[4]);
1666     put.add(FAMILIES[2], QUALIFIER, STAMPS[5], VALUES[5]);
1667     put.add(FAMILIES[2], QUALIFIER, STAMPS[6], VALUES[6]);
1668     ht.put(put);
1669 
1670     // Verify we only get the right number out of each
1671 
1672     // Family0
1673 
1674     Get get = new Get(ROW);
1675     get.addColumn(FAMILIES[0], QUALIFIER);
1676     get.setMaxVersions(Integer.MAX_VALUE);
1677     Result result = ht.get(get);
1678     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1679         new long [] {STAMPS[1]},
1680         new byte[][] {VALUES[1]},
1681         0, 0);
1682 
1683     get = new Get(ROW);
1684     get.addFamily(FAMILIES[0]);
1685     get.setMaxVersions(Integer.MAX_VALUE);
1686     result = ht.get(get);
1687     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1688         new long [] {STAMPS[1]},
1689         new byte[][] {VALUES[1]},
1690         0, 0);
1691 
1692     Scan scan = new Scan(ROW);
1693     scan.addColumn(FAMILIES[0], QUALIFIER);
1694     scan.setMaxVersions(Integer.MAX_VALUE);
1695     result = getSingleScanResult(ht, scan);
1696     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1697         new long [] {STAMPS[1]},
1698         new byte[][] {VALUES[1]},
1699         0, 0);
1700 
1701     scan = new Scan(ROW);
1702     scan.addFamily(FAMILIES[0]);
1703     scan.setMaxVersions(Integer.MAX_VALUE);
1704     result = getSingleScanResult(ht, scan);
1705     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1706         new long [] {STAMPS[1]},
1707         new byte[][] {VALUES[1]},
1708         0, 0);
1709 
1710     // Family1
1711 
1712     get = new Get(ROW);
1713     get.addColumn(FAMILIES[1], QUALIFIER);
1714     get.setMaxVersions(Integer.MAX_VALUE);
1715     result = ht.get(get);
1716     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1717         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1718         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1719         0, 2);
1720 
1721     get = new Get(ROW);
1722     get.addFamily(FAMILIES[1]);
1723     get.setMaxVersions(Integer.MAX_VALUE);
1724     result = ht.get(get);
1725     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1726         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1727         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1728         0, 2);
1729 
1730     scan = new Scan(ROW);
1731     scan.addColumn(FAMILIES[1], QUALIFIER);
1732     scan.setMaxVersions(Integer.MAX_VALUE);
1733     result = getSingleScanResult(ht, scan);
1734     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1735         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1736         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1737         0, 2);
1738 
1739     scan = new Scan(ROW);
1740     scan.addFamily(FAMILIES[1]);
1741     scan.setMaxVersions(Integer.MAX_VALUE);
1742     result = getSingleScanResult(ht, scan);
1743     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1744         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1745         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1746         0, 2);
1747 
1748     // Family2
1749 
1750     get = new Get(ROW);
1751     get.addColumn(FAMILIES[2], QUALIFIER);
1752     get.setMaxVersions(Integer.MAX_VALUE);
1753     result = ht.get(get);
1754     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1755         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1756         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1757         0, 4);
1758 
1759     get = new Get(ROW);
1760     get.addFamily(FAMILIES[2]);
1761     get.setMaxVersions(Integer.MAX_VALUE);
1762     result = ht.get(get);
1763     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1764         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1765         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1766         0, 4);
1767 
1768     scan = new Scan(ROW);
1769     scan.addColumn(FAMILIES[2], QUALIFIER);
1770     scan.setMaxVersions(Integer.MAX_VALUE);
1771     result = getSingleScanResult(ht, scan);
1772     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1773         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1774         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1775         0, 4);
1776 
1777     scan = new Scan(ROW);
1778     scan.addFamily(FAMILIES[2]);
1779     scan.setMaxVersions(Integer.MAX_VALUE);
1780     result = getSingleScanResult(ht, scan);
1781     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1782         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1783         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1784         0, 4);
1785 
1786     // Try all families
1787 
1788     get = new Get(ROW);
1789     get.setMaxVersions(Integer.MAX_VALUE);
1790     result = ht.get(get);
1791     assertTrue("Expected 9 keys but received " + result.size(),
1792         result.size() == 9);
1793 
1794     get = new Get(ROW);
1795     get.addFamily(FAMILIES[0]);
1796     get.addFamily(FAMILIES[1]);
1797     get.addFamily(FAMILIES[2]);
1798     get.setMaxVersions(Integer.MAX_VALUE);
1799     result = ht.get(get);
1800     assertTrue("Expected 9 keys but received " + result.size(),
1801         result.size() == 9);
1802 
1803     get = new Get(ROW);
1804     get.addColumn(FAMILIES[0], QUALIFIER);
1805     get.addColumn(FAMILIES[1], QUALIFIER);
1806     get.addColumn(FAMILIES[2], QUALIFIER);
1807     get.setMaxVersions(Integer.MAX_VALUE);
1808     result = ht.get(get);
1809     assertTrue("Expected 9 keys but received " + result.size(),
1810         result.size() == 9);
1811 
1812     scan = new Scan(ROW);
1813     scan.setMaxVersions(Integer.MAX_VALUE);
1814     result = getSingleScanResult(ht, scan);
1815     assertTrue("Expected 9 keys but received " + result.size(),
1816         result.size() == 9);
1817 
1818     scan = new Scan(ROW);
1819     scan.setMaxVersions(Integer.MAX_VALUE);
1820     scan.addFamily(FAMILIES[0]);
1821     scan.addFamily(FAMILIES[1]);
1822     scan.addFamily(FAMILIES[2]);
1823     result = getSingleScanResult(ht, scan);
1824     assertTrue("Expected 9 keys but received " + result.size(),
1825         result.size() == 9);
1826 
1827     scan = new Scan(ROW);
1828     scan.setMaxVersions(Integer.MAX_VALUE);
1829     scan.addColumn(FAMILIES[0], QUALIFIER);
1830     scan.addColumn(FAMILIES[1], QUALIFIER);
1831     scan.addColumn(FAMILIES[2], QUALIFIER);
1832     result = getSingleScanResult(ht, scan);
1833     assertTrue("Expected 9 keys but received " + result.size(),
1834         result.size() == 9);
1835 
1836   }
1837 
1838   @Test
1839   public void testDeleteFamilyVersion() throws Exception {
1840     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
1841     byte [] TABLE = Bytes.toBytes("testDeleteFamilyVersion");
1842 
1843     byte [][] QUALIFIERS = makeNAscii(QUALIFIER, 1);
1844     byte [][] VALUES = makeN(VALUE, 5);
1845     long [] ts = {1000, 2000, 3000, 4000, 5000};
1846 
1847     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 5);
1848 
1849     Put put = new Put(ROW);
1850     for (int q = 0; q < 1; q++)
1851       for (int t = 0; t < 5; t++)
1852         put.add(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
1853     ht.put(put);
1854     admin.flush(TABLE);
1855 
1856     Delete delete = new Delete(ROW);
1857     delete.deleteFamilyVersion(FAMILY, ts[1]);  // delete version '2000'
1858     delete.deleteFamilyVersion(FAMILY, ts[3]);  // delete version '4000'
1859     ht.delete(delete);
1860     admin.flush(TABLE);
1861 
1862     for (int i = 0; i < 1; i++) {
1863       Get get = new Get(ROW);
1864       get.addColumn(FAMILY, QUALIFIERS[i]);
1865       get.setMaxVersions(Integer.MAX_VALUE);
1866       Result result = ht.get(get);
1867       // verify version '1000'/'3000'/'5000' remains for all columns
1868       assertNResult(result, ROW, FAMILY, QUALIFIERS[i],
1869           new long [] {ts[0], ts[2], ts[4]},
1870           new byte[][] {VALUES[0], VALUES[2], VALUES[4]},
1871           0, 2);
1872     }
1873     ht.close();
1874     admin.close();
1875   }
1876 
1877   @Test
1878   public void testDeleteFamilyVersionWithOtherDeletes() throws Exception {
1879     byte [] TABLE = Bytes.toBytes("testDeleteFamilyVersionWithOtherDeletes");
1880 
1881     byte [][] QUALIFIERS = makeNAscii(QUALIFIER, 5);
1882     byte [][] VALUES = makeN(VALUE, 5);
1883     long [] ts = {1000, 2000, 3000, 4000, 5000};
1884 
1885     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
1886     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 5);
1887     Put put = null;
1888     Result result = null;
1889     Get get = null;
1890     Delete delete = null;
1891 
1892     // 1. put on ROW
1893     put = new Put(ROW);
1894     for (int q = 0; q < 5; q++)
1895       for (int t = 0; t < 5; t++)
1896         put.add(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
1897     ht.put(put);
1898     admin.flush(TABLE);
1899 
1900     // 2. put on ROWS[0]
1901     byte [] ROW2 = Bytes.toBytes("myRowForTest");
1902     put = new Put(ROW2);
1903     for (int q = 0; q < 5; q++)
1904       for (int t = 0; t < 5; t++)
1905         put.add(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
1906     ht.put(put);
1907     admin.flush(TABLE);
1908 
1909     // 3. delete on ROW
1910     delete = new Delete(ROW);
1911     // delete version <= 2000 of all columns
1912     // note: deleteFamily must be the first since it will mask
1913     // the subsequent other type deletes!
1914     delete.deleteFamily(FAMILY, ts[1]);
1915     // delete version '4000' of all columns
1916     delete.deleteFamilyVersion(FAMILY, ts[3]);
1917    // delete version <= 3000 of column 0
1918     delete.deleteColumns(FAMILY, QUALIFIERS[0], ts[2]);
1919     // delete version <= 5000 of column 2
1920     delete.deleteColumns(FAMILY, QUALIFIERS[2], ts[4]);
1921     // delete version 5000 of column 4
1922     delete.deleteColumn(FAMILY, QUALIFIERS[4], ts[4]);
1923     ht.delete(delete);
1924     admin.flush(TABLE);
1925 
1926      // 4. delete on ROWS[0]
1927     delete = new Delete(ROW2);
1928     delete.deleteFamilyVersion(FAMILY, ts[1]);  // delete version '2000'
1929     delete.deleteFamilyVersion(FAMILY, ts[3]);  // delete version '4000'
1930     ht.delete(delete);
1931     admin.flush(TABLE);
1932 
1933     // 5. check ROW
1934     get = new Get(ROW);
1935     get.addColumn(FAMILY, QUALIFIERS[0]);
1936     get.setMaxVersions(Integer.MAX_VALUE);
1937     result = ht.get(get);
1938     assertNResult(result, ROW, FAMILY, QUALIFIERS[0],
1939         new long [] {ts[4]},
1940         new byte[][] {VALUES[4]},
1941         0, 0);
1942 
1943     get = new Get(ROW);
1944     get.addColumn(FAMILY, QUALIFIERS[1]);
1945     get.setMaxVersions(Integer.MAX_VALUE);
1946     result = ht.get(get);
1947     assertNResult(result, ROW, FAMILY, QUALIFIERS[1],
1948         new long [] {ts[2], ts[4]},
1949         new byte[][] {VALUES[2], VALUES[4]},
1950         0, 1);
1951 
1952     get = new Get(ROW);
1953     get.addColumn(FAMILY, QUALIFIERS[2]);
1954     get.setMaxVersions(Integer.MAX_VALUE);
1955     result = ht.get(get);
1956     assertEquals(0, result.size());
1957 
1958     get = new Get(ROW);
1959     get.addColumn(FAMILY, QUALIFIERS[3]);
1960     get.setMaxVersions(Integer.MAX_VALUE);
1961     result = ht.get(get);
1962     assertNResult(result, ROW, FAMILY, QUALIFIERS[3],
1963         new long [] {ts[2], ts[4]},
1964         new byte[][] {VALUES[2], VALUES[4]},
1965         0, 1);
1966 
1967     get = new Get(ROW);
1968     get.addColumn(FAMILY, QUALIFIERS[4]);
1969     get.setMaxVersions(Integer.MAX_VALUE);
1970     result = ht.get(get);
1971     assertNResult(result, ROW, FAMILY, QUALIFIERS[4],
1972         new long [] {ts[2]},
1973         new byte[][] {VALUES[2]},
1974         0, 0);
1975 
1976     // 6. check ROWS[0]
1977     for (int i = 0; i < 5; i++) {
1978       get = new Get(ROW2);
1979       get.addColumn(FAMILY, QUALIFIERS[i]);
1980       get.setMaxVersions(Integer.MAX_VALUE);
1981       result = ht.get(get);
1982       // verify version '1000'/'3000'/'5000' remains for all columns
1983       assertNResult(result, ROW2, FAMILY, QUALIFIERS[i],
1984           new long [] {ts[0], ts[2], ts[4]},
1985           new byte[][] {VALUES[0], VALUES[2], VALUES[4]},
1986           0, 2);
1987     }
1988     ht.close();
1989     admin.close();
1990   }
1991 
1992   @Test
1993   public void testDeletes() throws Exception {
1994     byte [] TABLE = Bytes.toBytes("testDeletes");
1995 
1996     byte [][] ROWS = makeNAscii(ROW, 6);
1997     byte [][] FAMILIES = makeNAscii(FAMILY, 3);
1998     byte [][] VALUES = makeN(VALUE, 5);
1999     long [] ts = {1000, 2000, 3000, 4000, 5000};
2000 
2001     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, 3);
2002 
2003     Put put = new Put(ROW);
2004     put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
2005     put.add(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
2006     ht.put(put);
2007 
2008     Delete delete = new Delete(ROW);
2009     delete.deleteFamily(FAMILIES[0], ts[0]);
2010     ht.delete(delete);
2011 
2012     Get get = new Get(ROW);
2013     get.addFamily(FAMILIES[0]);
2014     get.setMaxVersions(Integer.MAX_VALUE);
2015     Result result = ht.get(get);
2016     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2017         new long [] {ts[1]},
2018         new byte[][] {VALUES[1]},
2019         0, 0);
2020 
2021     Scan scan = new Scan(ROW);
2022     scan.addFamily(FAMILIES[0]);
2023     scan.setMaxVersions(Integer.MAX_VALUE);
2024     result = getSingleScanResult(ht, scan);
2025     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2026         new long [] {ts[1]},
2027         new byte[][] {VALUES[1]},
2028         0, 0);
2029 
2030     // Test delete latest version
2031     put = new Put(ROW);
2032     put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
2033     put.add(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
2034     put.add(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
2035     put.add(FAMILIES[0], null, ts[4], VALUES[4]);
2036     put.add(FAMILIES[0], null, ts[2], VALUES[2]);
2037     put.add(FAMILIES[0], null, ts[3], VALUES[3]);
2038     ht.put(put);
2039 
2040     delete = new Delete(ROW);
2041     delete.deleteColumn(FAMILIES[0], QUALIFIER); // ts[4]
2042     ht.delete(delete);
2043 
2044     get = new Get(ROW);
2045     get.addColumn(FAMILIES[0], QUALIFIER);
2046     get.setMaxVersions(Integer.MAX_VALUE);
2047     result = ht.get(get);
2048     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2049         new long [] {ts[1], ts[2], ts[3]},
2050         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
2051         0, 2);
2052 
2053     scan = new Scan(ROW);
2054     scan.addColumn(FAMILIES[0], QUALIFIER);
2055     scan.setMaxVersions(Integer.MAX_VALUE);
2056     result = getSingleScanResult(ht, scan);
2057     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2058         new long [] {ts[1], ts[2], ts[3]},
2059         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
2060         0, 2);
2061 
2062     // Test for HBASE-1847
2063     delete = new Delete(ROW);
2064     delete.deleteColumn(FAMILIES[0], null);
2065     ht.delete(delete);
2066 
2067     // Cleanup null qualifier
2068     delete = new Delete(ROW);
2069     delete.deleteColumns(FAMILIES[0], null);
2070     ht.delete(delete);
2071 
2072     // Expected client behavior might be that you can re-put deleted values
2073     // But alas, this is not to be.  We can't put them back in either case.
2074 
2075     put = new Put(ROW);
2076     put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); // 1000
2077     put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); // 5000
2078     ht.put(put);
2079 
2080 
2081     // It used to be due to the internal implementation of Get, that
2082     // the Get() call would return ts[4] UNLIKE the Scan below. With
2083     // the switch to using Scan for Get this is no longer the case.
2084     get = new Get(ROW);
2085     get.addFamily(FAMILIES[0]);
2086     get.setMaxVersions(Integer.MAX_VALUE);
2087     result = ht.get(get);
2088     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2089         new long [] {ts[1], ts[2], ts[3]},
2090         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
2091         0, 2);
2092 
2093     // The Scanner returns the previous values, the expected-naive-unexpected behavior
2094 
2095     scan = new Scan(ROW);
2096     scan.addFamily(FAMILIES[0]);
2097     scan.setMaxVersions(Integer.MAX_VALUE);
2098     result = getSingleScanResult(ht, scan);
2099     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2100         new long [] {ts[1], ts[2], ts[3]},
2101         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
2102         0, 2);
2103 
2104     // Test deleting an entire family from one row but not the other various ways
2105 
2106     put = new Put(ROWS[0]);
2107     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2108     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2109     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2110     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2111     ht.put(put);
2112 
2113     put = new Put(ROWS[1]);
2114     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2115     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2116     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2117     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2118     ht.put(put);
2119 
2120     put = new Put(ROWS[2]);
2121     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2122     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2123     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2124     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2125     ht.put(put);
2126 
2127     // Assert that above went in.
2128     get = new Get(ROWS[2]);
2129     get.addFamily(FAMILIES[1]);
2130     get.addFamily(FAMILIES[2]);
2131     get.setMaxVersions(Integer.MAX_VALUE);
2132     result = ht.get(get);
2133     assertTrue("Expected 4 key but received " + result.size() + ": " + result,
2134         result.size() == 4);
2135 
2136     delete = new Delete(ROWS[0]);
2137     delete.deleteFamily(FAMILIES[2]);
2138     ht.delete(delete);
2139 
2140     delete = new Delete(ROWS[1]);
2141     delete.deleteColumns(FAMILIES[1], QUALIFIER);
2142     ht.delete(delete);
2143 
2144     delete = new Delete(ROWS[2]);
2145     delete.deleteColumn(FAMILIES[1], QUALIFIER);
2146     delete.deleteColumn(FAMILIES[1], QUALIFIER);
2147     delete.deleteColumn(FAMILIES[2], QUALIFIER);
2148     ht.delete(delete);
2149 
2150     get = new Get(ROWS[0]);
2151     get.addFamily(FAMILIES[1]);
2152     get.addFamily(FAMILIES[2]);
2153     get.setMaxVersions(Integer.MAX_VALUE);
2154     result = ht.get(get);
2155     assertTrue("Expected 2 keys but received " + result.size(),
2156         result.size() == 2);
2157     assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER,
2158         new long [] {ts[0], ts[1]},
2159         new byte[][] {VALUES[0], VALUES[1]},
2160         0, 1);
2161 
2162     scan = new Scan(ROWS[0]);
2163     scan.addFamily(FAMILIES[1]);
2164     scan.addFamily(FAMILIES[2]);
2165     scan.setMaxVersions(Integer.MAX_VALUE);
2166     result = getSingleScanResult(ht, scan);
2167     assertTrue("Expected 2 keys but received " + result.size(),
2168         result.size() == 2);
2169     assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER,
2170         new long [] {ts[0], ts[1]},
2171         new byte[][] {VALUES[0], VALUES[1]},
2172         0, 1);
2173 
2174     get = new Get(ROWS[1]);
2175     get.addFamily(FAMILIES[1]);
2176     get.addFamily(FAMILIES[2]);
2177     get.setMaxVersions(Integer.MAX_VALUE);
2178     result = ht.get(get);
2179     assertTrue("Expected 2 keys but received " + result.size(),
2180         result.size() == 2);
2181 
2182     scan = new Scan(ROWS[1]);
2183     scan.addFamily(FAMILIES[1]);
2184     scan.addFamily(FAMILIES[2]);
2185     scan.setMaxVersions(Integer.MAX_VALUE);
2186     result = getSingleScanResult(ht, scan);
2187     assertTrue("Expected 2 keys but received " + result.size(),
2188         result.size() == 2);
2189 
2190     get = new Get(ROWS[2]);
2191     get.addFamily(FAMILIES[1]);
2192     get.addFamily(FAMILIES[2]);
2193     get.setMaxVersions(Integer.MAX_VALUE);
2194     result = ht.get(get);
2195     assertEquals(1, result.size());
2196     assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
2197         new long [] {ts[2]},
2198         new byte[][] {VALUES[2]},
2199         0, 0);
2200 
2201     scan = new Scan(ROWS[2]);
2202     scan.addFamily(FAMILIES[1]);
2203     scan.addFamily(FAMILIES[2]);
2204     scan.setMaxVersions(Integer.MAX_VALUE);
2205     result = getSingleScanResult(ht, scan);
2206     assertEquals(1, result.size());
2207     assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
2208         new long [] {ts[2]},
2209         new byte[][] {VALUES[2]},
2210         0, 0);
2211 
2212     // Test if we delete the family first in one row (HBASE-1541)
2213 
2214     delete = new Delete(ROWS[3]);
2215     delete.deleteFamily(FAMILIES[1]);
2216     ht.delete(delete);
2217 
2218     put = new Put(ROWS[3]);
2219     put.add(FAMILIES[2], QUALIFIER, VALUES[0]);
2220     ht.put(put);
2221 
2222     put = new Put(ROWS[4]);
2223     put.add(FAMILIES[1], QUALIFIER, VALUES[1]);
2224     put.add(FAMILIES[2], QUALIFIER, VALUES[2]);
2225     ht.put(put);
2226 
2227     get = new Get(ROWS[3]);
2228     get.addFamily(FAMILIES[1]);
2229     get.addFamily(FAMILIES[2]);
2230     get.setMaxVersions(Integer.MAX_VALUE);
2231     result = ht.get(get);
2232     assertTrue("Expected 1 key but received " + result.size(),
2233         result.size() == 1);
2234 
2235     get = new Get(ROWS[4]);
2236     get.addFamily(FAMILIES[1]);
2237     get.addFamily(FAMILIES[2]);
2238     get.setMaxVersions(Integer.MAX_VALUE);
2239     result = ht.get(get);
2240     assertTrue("Expected 2 keys but received " + result.size(),
2241         result.size() == 2);
2242 
2243     scan = new Scan(ROWS[3]);
2244     scan.addFamily(FAMILIES[1]);
2245     scan.addFamily(FAMILIES[2]);
2246     scan.setMaxVersions(Integer.MAX_VALUE);
2247     ResultScanner scanner = ht.getScanner(scan);
2248     result = scanner.next();
2249     assertTrue("Expected 1 key but received " + result.size(),
2250         result.size() == 1);
2251     assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[3]));
2252     assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[0]));
2253     result = scanner.next();
2254     assertTrue("Expected 2 keys but received " + result.size(),
2255         result.size() == 2);
2256     assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[4]));
2257     assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[1]), ROWS[4]));
2258     assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[1]));
2259     assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[1]), VALUES[2]));
2260     scanner.close();
2261 
2262     // Add test of bulk deleting.
2263     for (int i = 0; i < 10; i++) {
2264       byte [] bytes = Bytes.toBytes(i);
2265       put = new Put(bytes);
2266       put.setDurability(Durability.SKIP_WAL);
2267       put.add(FAMILIES[0], QUALIFIER, bytes);
2268       ht.put(put);
2269     }
2270     for (int i = 0; i < 10; i++) {
2271       byte [] bytes = Bytes.toBytes(i);
2272       get = new Get(bytes);
2273       get.addFamily(FAMILIES[0]);
2274       result = ht.get(get);
2275       assertTrue(result.size() == 1);
2276     }
2277     ArrayList<Delete> deletes = new ArrayList<Delete>();
2278     for (int i = 0; i < 10; i++) {
2279       byte [] bytes = Bytes.toBytes(i);
2280       delete = new Delete(bytes);
2281       delete.deleteFamily(FAMILIES[0]);
2282       deletes.add(delete);
2283     }
2284     ht.delete(deletes);
2285     for (int i = 0; i < 10; i++) {
2286       byte [] bytes = Bytes.toBytes(i);
2287       get = new Get(bytes);
2288       get.addFamily(FAMILIES[0]);
2289       result = ht.get(get);
2290       assertTrue(result.size() == 0);
2291     }
2292   }
2293 
2294   /*
2295    * Baseline "scalability" test.
2296    *
2297    * Tests one hundred families, one million columns, one million versions
2298    */
2299   @Ignore @Test
2300   public void testMillions() throws Exception {
2301 
2302     // 100 families
2303 
2304     // millions of columns
2305 
2306     // millions of versions
2307 
2308   }
2309 
2310   @Ignore @Test
2311   public void testMultipleRegionsAndBatchPuts() throws Exception {
2312     // Two family table
2313 
2314     // Insert lots of rows
2315 
2316     // Insert to the same row with batched puts
2317 
2318     // Insert to multiple rows with batched puts
2319 
2320     // Split the table
2321 
2322     // Get row from first region
2323 
2324     // Get row from second region
2325 
2326     // Scan all rows
2327 
2328     // Insert to multiple regions with batched puts
2329 
2330     // Get row from first region
2331 
2332     // Get row from second region
2333 
2334     // Scan all rows
2335 
2336 
2337   }
2338 
2339   @Ignore @Test
2340   public void testMultipleRowMultipleFamily() throws Exception {
2341 
2342   }
2343 
2344   //
2345   // JIRA Testers
2346   //
2347 
2348   /**
2349    * HBASE-867
2350    *    If millions of columns in a column family, hbase scanner won't come up
2351    *
2352    *    Test will create numRows rows, each with numColsPerRow columns
2353    *    (1 version each), and attempt to scan them all.
2354    *
2355    *    To test at scale, up numColsPerRow to the millions
2356    *    (have not gotten that to work running as junit though)
2357    */
2358   @Test
2359   public void testJiraTest867() throws Exception {
2360     int numRows = 10;
2361     int numColsPerRow = 2000;
2362 
2363     byte [] TABLE = Bytes.toBytes("testJiraTest867");
2364 
2365     byte [][] ROWS = makeN(ROW, numRows);
2366     byte [][] QUALIFIERS = makeN(QUALIFIER, numColsPerRow);
2367 
2368     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
2369 
2370     // Insert rows
2371 
2372     for(int i=0;i<numRows;i++) {
2373       Put put = new Put(ROWS[i]);
2374       put.setDurability(Durability.SKIP_WAL);
2375       for(int j=0;j<numColsPerRow;j++) {
2376         put.add(FAMILY, QUALIFIERS[j], QUALIFIERS[j]);
2377       }
2378       assertTrue("Put expected to contain " + numColsPerRow + " columns but " +
2379           "only contains " + put.size(), put.size() == numColsPerRow);
2380       ht.put(put);
2381     }
2382 
2383     // Get a row
2384     Get get = new Get(ROWS[numRows-1]);
2385     Result result = ht.get(get);
2386     assertNumKeys(result, numColsPerRow);
2387     Cell [] keys = result.rawCells();
2388     for(int i=0;i<result.size();i++) {
2389       assertKey(keys[i], ROWS[numRows-1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2390     }
2391 
2392     // Scan the rows
2393     Scan scan = new Scan();
2394     ResultScanner scanner = ht.getScanner(scan);
2395     int rowCount = 0;
2396     while((result = scanner.next()) != null) {
2397       assertNumKeys(result, numColsPerRow);
2398       Cell [] kvs = result.rawCells();
2399       for(int i=0;i<numColsPerRow;i++) {
2400         assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2401       }
2402       rowCount++;
2403     }
2404     scanner.close();
2405     assertTrue("Expected to scan " + numRows + " rows but actually scanned "
2406         + rowCount + " rows", rowCount == numRows);
2407 
2408     // flush and try again
2409 
2410     TEST_UTIL.flush();
2411 
2412     // Get a row
2413     get = new Get(ROWS[numRows-1]);
2414     result = ht.get(get);
2415     assertNumKeys(result, numColsPerRow);
2416     keys = result.rawCells();
2417     for(int i=0;i<result.size();i++) {
2418       assertKey(keys[i], ROWS[numRows-1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2419     }
2420 
2421     // Scan the rows
2422     scan = new Scan();
2423     scanner = ht.getScanner(scan);
2424     rowCount = 0;
2425     while((result = scanner.next()) != null) {
2426       assertNumKeys(result, numColsPerRow);
2427       Cell [] kvs = result.rawCells();
2428       for(int i=0;i<numColsPerRow;i++) {
2429         assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2430       }
2431       rowCount++;
2432     }
2433     scanner.close();
2434     assertTrue("Expected to scan " + numRows + " rows but actually scanned "
2435         + rowCount + " rows", rowCount == numRows);
2436 
2437   }
2438 
2439   /**
2440    * HBASE-861
2441    *    get with timestamp will return a value if there is a version with an
2442    *    earlier timestamp
2443    */
2444   @Test
2445   public void testJiraTest861() throws Exception {
2446 
2447     byte [] TABLE = Bytes.toBytes("testJiraTest861");
2448     byte [][] VALUES = makeNAscii(VALUE, 7);
2449     long [] STAMPS = makeStamps(7);
2450 
2451     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2452 
2453     // Insert three versions
2454 
2455     Put put = new Put(ROW);
2456     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2457     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2458     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2459     ht.put(put);
2460 
2461     // Get the middle value
2462     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2463 
2464     // Try to get one version before (expect fail)
2465     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2466 
2467     // Try to get one version after (expect fail)
2468     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2469 
2470     // Try same from storefile
2471     TEST_UTIL.flush();
2472     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2473     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2474     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2475 
2476     // Insert two more versions surrounding others, into memstore
2477     put = new Put(ROW);
2478     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2479     put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2480     ht.put(put);
2481 
2482     // Check we can get everything we should and can't get what we shouldn't
2483     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2484     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2485     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2486     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2487     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2488     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2489     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2490 
2491     // Try same from two storefiles
2492     TEST_UTIL.flush();
2493     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2494     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2495     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2496     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2497     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2498     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2499     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2500 
2501   }
2502 
2503   /**
2504    * HBASE-33
2505    *    Add a HTable get/obtainScanner method that retrieves all versions of a
2506    *    particular column and row between two timestamps
2507    */
2508   @Test
2509   public void testJiraTest33() throws Exception {
2510 
2511     byte [] TABLE = Bytes.toBytes("testJiraTest33");
2512     byte [][] VALUES = makeNAscii(VALUE, 7);
2513     long [] STAMPS = makeStamps(7);
2514 
2515     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2516 
2517     // Insert lots versions
2518 
2519     Put put = new Put(ROW);
2520     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2521     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2522     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2523     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2524     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2525     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2526     ht.put(put);
2527 
2528     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2529     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2530     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2531     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2532 
2533     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2534     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2535     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2536     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2537 
2538     // Try same from storefile
2539     TEST_UTIL.flush();
2540 
2541     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2542     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2543     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2544     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2545 
2546     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2547     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2548     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2549     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2550 
2551   }
2552 
2553   /**
2554    * HBASE-1014
2555    *    commit(BatchUpdate) method should return timestamp
2556    */
2557   @Test
2558   public void testJiraTest1014() throws Exception {
2559 
2560     byte [] TABLE = Bytes.toBytes("testJiraTest1014");
2561 
2562     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2563 
2564     long manualStamp = 12345;
2565 
2566     // Insert lots versions
2567 
2568     Put put = new Put(ROW);
2569     put.add(FAMILY, QUALIFIER, manualStamp, VALUE);
2570     ht.put(put);
2571 
2572     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, manualStamp, VALUE);
2573     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp-1);
2574     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp+1);
2575 
2576   }
2577 
2578   /**
2579    * HBASE-1182
2580    *    Scan for columns > some timestamp
2581    */
2582   @Test
2583   public void testJiraTest1182() throws Exception {
2584 
2585     byte [] TABLE = Bytes.toBytes("testJiraTest1182");
2586     byte [][] VALUES = makeNAscii(VALUE, 7);
2587     long [] STAMPS = makeStamps(7);
2588 
2589     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2590 
2591     // Insert lots versions
2592 
2593     Put put = new Put(ROW);
2594     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2595     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2596     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2597     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2598     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2599     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2600     ht.put(put);
2601 
2602     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2603     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2604     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2605 
2606     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2607     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2608     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2609 
2610     // Try same from storefile
2611     TEST_UTIL.flush();
2612 
2613     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2614     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2615     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2616 
2617     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2618     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2619     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2620   }
2621 
2622   /**
2623    * HBASE-52
2624    *    Add a means of scanning over all versions
2625    */
2626   @Test
2627   public void testJiraTest52() throws Exception {
2628     byte [] TABLE = Bytes.toBytes("testJiraTest52");
2629     byte [][] VALUES = makeNAscii(VALUE, 7);
2630     long [] STAMPS = makeStamps(7);
2631 
2632     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2633 
2634     // Insert lots versions
2635 
2636     Put put = new Put(ROW);
2637     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2638     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2639     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2640     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2641     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2642     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2643     ht.put(put);
2644 
2645     getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2646 
2647     scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2648 
2649     // Try same from storefile
2650     TEST_UTIL.flush();
2651 
2652     getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2653 
2654     scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2655   }
2656 
2657   //
2658   // Bulk Testers
2659   //
2660 
2661   private void getVersionRangeAndVerifyGreaterThan(HTable ht, byte [] row,
2662       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
2663       int start, int end)
2664   throws IOException {
2665     Get get = new Get(row);
2666     get.addColumn(family, qualifier);
2667     get.setMaxVersions(Integer.MAX_VALUE);
2668     get.setTimeRange(stamps[start+1], Long.MAX_VALUE);
2669     Result result = ht.get(get);
2670     assertNResult(result, row, family, qualifier, stamps, values, start+1, end);
2671   }
2672 
2673   private void getVersionRangeAndVerify(HTable ht, byte [] row, byte [] family,
2674       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2675   throws IOException {
2676     Get get = new Get(row);
2677     get.addColumn(family, qualifier);
2678     get.setMaxVersions(Integer.MAX_VALUE);
2679     get.setTimeRange(stamps[start], stamps[end]+1);
2680     Result result = ht.get(get);
2681     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2682   }
2683 
2684   private void getAllVersionsAndVerify(HTable ht, byte [] row, byte [] family,
2685       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2686   throws IOException {
2687     Get get = new Get(row);
2688     get.addColumn(family, qualifier);
2689     get.setMaxVersions(Integer.MAX_VALUE);
2690     Result result = ht.get(get);
2691     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2692   }
2693 
2694   private void scanVersionRangeAndVerifyGreaterThan(HTable ht, byte [] row,
2695       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
2696       int start, int end)
2697   throws IOException {
2698     Scan scan = new Scan(row);
2699     scan.addColumn(family, qualifier);
2700     scan.setMaxVersions(Integer.MAX_VALUE);
2701     scan.setTimeRange(stamps[start+1], Long.MAX_VALUE);
2702     Result result = getSingleScanResult(ht, scan);
2703     assertNResult(result, row, family, qualifier, stamps, values, start+1, end);
2704   }
2705 
2706   private void scanVersionRangeAndVerify(HTable ht, byte [] row, byte [] family,
2707       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2708   throws IOException {
2709     Scan scan = new Scan(row);
2710     scan.addColumn(family, qualifier);
2711     scan.setMaxVersions(Integer.MAX_VALUE);
2712     scan.setTimeRange(stamps[start], stamps[end]+1);
2713     Result result = getSingleScanResult(ht, scan);
2714     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2715   }
2716 
2717   private void scanAllVersionsAndVerify(HTable ht, byte [] row, byte [] family,
2718       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2719   throws IOException {
2720     Scan scan = new Scan(row);
2721     scan.addColumn(family, qualifier);
2722     scan.setMaxVersions(Integer.MAX_VALUE);
2723     Result result = getSingleScanResult(ht, scan);
2724     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2725   }
2726 
2727   private void getVersionAndVerify(HTable ht, byte [] row, byte [] family,
2728       byte [] qualifier, long stamp, byte [] value)
2729   throws Exception {
2730     Get get = new Get(row);
2731     get.addColumn(family, qualifier);
2732     get.setTimeStamp(stamp);
2733     get.setMaxVersions(Integer.MAX_VALUE);
2734     Result result = ht.get(get);
2735     assertSingleResult(result, row, family, qualifier, stamp, value);
2736   }
2737 
2738   private void getVersionAndVerifyMissing(HTable ht, byte [] row, byte [] family,
2739       byte [] qualifier, long stamp)
2740   throws Exception {
2741     Get get = new Get(row);
2742     get.addColumn(family, qualifier);
2743     get.setTimeStamp(stamp);
2744     get.setMaxVersions(Integer.MAX_VALUE);
2745     Result result = ht.get(get);
2746     assertEmptyResult(result);
2747   }
2748 
2749   private void scanVersionAndVerify(HTable ht, byte [] row, byte [] family,
2750       byte [] qualifier, long stamp, byte [] value)
2751   throws Exception {
2752     Scan scan = new Scan(row);
2753     scan.addColumn(family, qualifier);
2754     scan.setTimeStamp(stamp);
2755     scan.setMaxVersions(Integer.MAX_VALUE);
2756     Result result = getSingleScanResult(ht, scan);
2757     assertSingleResult(result, row, family, qualifier, stamp, value);
2758   }
2759 
2760   private void scanVersionAndVerifyMissing(HTable ht, byte [] row,
2761       byte [] family, byte [] qualifier, long stamp)
2762   throws Exception {
2763     Scan scan = new Scan(row);
2764     scan.addColumn(family, qualifier);
2765     scan.setTimeStamp(stamp);
2766     scan.setMaxVersions(Integer.MAX_VALUE);
2767     Result result = getSingleScanResult(ht, scan);
2768     assertNullResult(result);
2769   }
2770 
2771   private void getTestNull(HTable ht, byte [] row, byte [] family,
2772       byte [] value)
2773   throws Exception {
2774 
2775     Get get = new Get(row);
2776     get.addColumn(family, null);
2777     Result result = ht.get(get);
2778     assertSingleResult(result, row, family, null, value);
2779 
2780     get = new Get(row);
2781     get.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
2782     result = ht.get(get);
2783     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2784 
2785     get = new Get(row);
2786     get.addFamily(family);
2787     result = ht.get(get);
2788     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2789 
2790     get = new Get(row);
2791     result = ht.get(get);
2792     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2793 
2794   }
2795 
2796   private void scanTestNull(HTable ht, byte[] row, byte[] family, byte[] value)
2797       throws Exception {
2798     scanTestNull(ht, row, family, value, false);
2799   }
2800 
2801   private void scanTestNull(HTable ht, byte[] row, byte[] family, byte[] value,
2802       boolean isReversedScan) throws Exception {
2803 
2804     Scan scan = new Scan();
2805     scan.setReversed(isReversedScan);
2806     scan.addColumn(family, null);
2807     Result result = getSingleScanResult(ht, scan);
2808     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2809 
2810     scan = new Scan();
2811     scan.setReversed(isReversedScan);
2812     scan.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
2813     result = getSingleScanResult(ht, scan);
2814     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2815 
2816     scan = new Scan();
2817     scan.setReversed(isReversedScan);
2818     scan.addFamily(family);
2819     result = getSingleScanResult(ht, scan);
2820     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2821 
2822     scan = new Scan();
2823     scan.setReversed(isReversedScan);
2824     result = getSingleScanResult(ht, scan);
2825     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2826 
2827   }
2828 
2829   private void singleRowGetTest(HTable ht, byte [][] ROWS, byte [][] FAMILIES,
2830       byte [][] QUALIFIERS, byte [][] VALUES)
2831   throws Exception {
2832 
2833     // Single column from memstore
2834     Get get = new Get(ROWS[0]);
2835     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2836     Result result = ht.get(get);
2837     assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]);
2838 
2839     // Single column from storefile
2840     get = new Get(ROWS[0]);
2841     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
2842     result = ht.get(get);
2843     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
2844 
2845     // Single column from storefile, family match
2846     get = new Get(ROWS[0]);
2847     get.addFamily(FAMILIES[7]);
2848     result = ht.get(get);
2849     assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]);
2850 
2851     // Two columns, one from memstore one from storefile, same family,
2852     // wildcard match
2853     get = new Get(ROWS[0]);
2854     get.addFamily(FAMILIES[4]);
2855     result = ht.get(get);
2856     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2857         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2858 
2859     // Two columns, one from memstore one from storefile, same family,
2860     // explicit match
2861     get = new Get(ROWS[0]);
2862     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2863     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
2864     result = ht.get(get);
2865     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2866         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2867 
2868     // Three column, one from memstore two from storefile, different families,
2869     // wildcard match
2870     get = new Get(ROWS[0]);
2871     get.addFamily(FAMILIES[4]);
2872     get.addFamily(FAMILIES[7]);
2873     result = ht.get(get);
2874     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2875         new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} });
2876 
2877     // Multiple columns from everywhere storefile, many family, wildcard
2878     get = new Get(ROWS[0]);
2879     get.addFamily(FAMILIES[2]);
2880     get.addFamily(FAMILIES[4]);
2881     get.addFamily(FAMILIES[6]);
2882     get.addFamily(FAMILIES[7]);
2883     result = ht.get(get);
2884     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2885         new int [][] {
2886           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2887     });
2888 
2889     // Multiple columns from everywhere storefile, many family, wildcard
2890     get = new Get(ROWS[0]);
2891     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
2892     get.addColumn(FAMILIES[2], QUALIFIERS[4]);
2893     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2894     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
2895     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
2896     get.addColumn(FAMILIES[6], QUALIFIERS[7]);
2897     get.addColumn(FAMILIES[7], QUALIFIERS[7]);
2898     get.addColumn(FAMILIES[7], QUALIFIERS[8]);
2899     result = ht.get(get);
2900     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2901         new int [][] {
2902           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2903     });
2904 
2905     // Everything
2906     get = new Get(ROWS[0]);
2907     result = ht.get(get);
2908     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2909         new int [][] {
2910           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0}
2911     });
2912 
2913     // Get around inserted columns
2914 
2915     get = new Get(ROWS[1]);
2916     result = ht.get(get);
2917     assertEmptyResult(result);
2918 
2919     get = new Get(ROWS[0]);
2920     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
2921     get.addColumn(FAMILIES[2], QUALIFIERS[3]);
2922     result = ht.get(get);
2923     assertEmptyResult(result);
2924 
2925   }
2926 
2927   private void singleRowScanTest(HTable ht, byte [][] ROWS, byte [][] FAMILIES,
2928       byte [][] QUALIFIERS, byte [][] VALUES)
2929   throws Exception {
2930 
2931     // Single column from memstore
2932     Scan scan = new Scan();
2933     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
2934     Result result = getSingleScanResult(ht, scan);
2935     assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]);
2936 
2937     // Single column from storefile
2938     scan = new Scan();
2939     scan.addColumn(FAMILIES[2], QUALIFIERS[2]);
2940     result = getSingleScanResult(ht, scan);
2941     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
2942 
2943     // Single column from storefile, family match
2944     scan = new Scan();
2945     scan.addFamily(FAMILIES[7]);
2946     result = getSingleScanResult(ht, scan);
2947     assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]);
2948 
2949     // Two columns, one from memstore one from storefile, same family,
2950     // wildcard match
2951     scan = new Scan();
2952     scan.addFamily(FAMILIES[4]);
2953     result = getSingleScanResult(ht, scan);
2954     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2955         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2956 
2957     // Two columns, one from memstore one from storefile, same family,
2958     // explicit match
2959     scan = new Scan();
2960     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
2961     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
2962     result = getSingleScanResult(ht, scan);
2963     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2964         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2965 
2966     // Three column, one from memstore two from storefile, different families,
2967     // wildcard match
2968     scan = new Scan();
2969     scan.addFamily(FAMILIES[4]);
2970     scan.addFamily(FAMILIES[7]);
2971     result = getSingleScanResult(ht, scan);
2972     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2973         new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} });
2974 
2975     // Multiple columns from everywhere storefile, many family, wildcard
2976     scan = new Scan();
2977     scan.addFamily(FAMILIES[2]);
2978     scan.addFamily(FAMILIES[4]);
2979     scan.addFamily(FAMILIES[6]);
2980     scan.addFamily(FAMILIES[7]);
2981     result = getSingleScanResult(ht, scan);
2982     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2983         new int [][] {
2984           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2985     });
2986 
2987     // Multiple columns from everywhere storefile, many family, wildcard
2988     scan = new Scan();
2989     scan.addColumn(FAMILIES[2], QUALIFIERS[2]);
2990     scan.addColumn(FAMILIES[2], QUALIFIERS[4]);
2991     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
2992     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
2993     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
2994     scan.addColumn(FAMILIES[6], QUALIFIERS[7]);
2995     scan.addColumn(FAMILIES[7], QUALIFIERS[7]);
2996     scan.addColumn(FAMILIES[7], QUALIFIERS[8]);
2997     result = getSingleScanResult(ht, scan);
2998     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2999         new int [][] {
3000           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
3001     });
3002 
3003     // Everything
3004     scan = new Scan();
3005     result = getSingleScanResult(ht, scan);
3006     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
3007         new int [][] {
3008           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0}
3009     });
3010 
3011     // Scan around inserted columns
3012 
3013     scan = new Scan(ROWS[1]);
3014     result = getSingleScanResult(ht, scan);
3015     assertNullResult(result);
3016 
3017     scan = new Scan();
3018     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
3019     scan.addColumn(FAMILIES[2], QUALIFIERS[3]);
3020     result = getSingleScanResult(ht, scan);
3021     assertNullResult(result);
3022   }
3023 
3024   /**
3025    * Verify a single column using gets.
3026    * Expects family and qualifier arrays to be valid for at least
3027    * the range:  idx-2 < idx < idx+2
3028    */
3029   private void getVerifySingleColumn(HTable ht,
3030       byte [][] ROWS, int ROWIDX,
3031       byte [][] FAMILIES, int FAMILYIDX,
3032       byte [][] QUALIFIERS, int QUALIFIERIDX,
3033       byte [][] VALUES, int VALUEIDX)
3034   throws Exception {
3035 
3036     Get get = new Get(ROWS[ROWIDX]);
3037     Result result = ht.get(get);
3038     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3039         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3040 
3041     get = new Get(ROWS[ROWIDX]);
3042     get.addFamily(FAMILIES[FAMILYIDX]);
3043     result = ht.get(get);
3044     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3045         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3046 
3047     get = new Get(ROWS[ROWIDX]);
3048     get.addFamily(FAMILIES[FAMILYIDX-2]);
3049     get.addFamily(FAMILIES[FAMILYIDX]);
3050     get.addFamily(FAMILIES[FAMILYIDX+2]);
3051     result = ht.get(get);
3052     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3053         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3054 
3055     get = new Get(ROWS[ROWIDX]);
3056     get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[0]);
3057     result = ht.get(get);
3058     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3059         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3060 
3061     get = new Get(ROWS[ROWIDX]);
3062     get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[1]);
3063     get.addFamily(FAMILIES[FAMILYIDX]);
3064     result = ht.get(get);
3065     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3066         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3067 
3068     get = new Get(ROWS[ROWIDX]);
3069     get.addFamily(FAMILIES[FAMILYIDX]);
3070     get.addColumn(FAMILIES[FAMILYIDX+1], QUALIFIERS[1]);
3071     get.addColumn(FAMILIES[FAMILYIDX-2], QUALIFIERS[1]);
3072     get.addFamily(FAMILIES[FAMILYIDX-1]);
3073     get.addFamily(FAMILIES[FAMILYIDX+2]);
3074     result = ht.get(get);
3075     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3076         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3077 
3078   }
3079 
3080 
3081   /**
3082    * Verify a single column using scanners.
3083    * Expects family and qualifier arrays to be valid for at least
3084    * the range:  idx-2 to idx+2
3085    * Expects row array to be valid for at least idx to idx+2
3086    */
3087   private void scanVerifySingleColumn(HTable ht,
3088       byte [][] ROWS, int ROWIDX,
3089       byte [][] FAMILIES, int FAMILYIDX,
3090       byte [][] QUALIFIERS, int QUALIFIERIDX,
3091       byte [][] VALUES, int VALUEIDX)
3092   throws Exception {
3093 
3094     Scan scan = new Scan();
3095     Result result = getSingleScanResult(ht, scan);
3096     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3097         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3098 
3099     scan = new Scan(ROWS[ROWIDX]);
3100     result = getSingleScanResult(ht, scan);
3101     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3102         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3103 
3104     scan = new Scan(ROWS[ROWIDX], ROWS[ROWIDX+1]);
3105     result = getSingleScanResult(ht, scan);
3106     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3107         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3108 
3109     scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX+1]);
3110     result = getSingleScanResult(ht, scan);
3111     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3112         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3113 
3114     scan = new Scan();
3115     scan.addFamily(FAMILIES[FAMILYIDX]);
3116     result = getSingleScanResult(ht, scan);
3117     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3118         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3119 
3120     scan = new Scan();
3121     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]);
3122     result = getSingleScanResult(ht, scan);
3123     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3124         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3125 
3126     scan = new Scan();
3127     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]);
3128     scan.addFamily(FAMILIES[FAMILYIDX]);
3129     result = getSingleScanResult(ht, scan);
3130     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3131         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3132 
3133     scan = new Scan();
3134     scan.addColumn(FAMILIES[FAMILYIDX-1], QUALIFIERS[QUALIFIERIDX+1]);
3135     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]);
3136     scan.addFamily(FAMILIES[FAMILYIDX+1]);
3137     result = getSingleScanResult(ht, scan);
3138     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3139         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3140 
3141   }
3142 
3143   /**
3144    * Verify we do not read any values by accident around a single column
3145    * Same requirements as getVerifySingleColumn
3146    */
3147   private void getVerifySingleEmpty(HTable ht,
3148       byte [][] ROWS, int ROWIDX,
3149       byte [][] FAMILIES, int FAMILYIDX,
3150       byte [][] QUALIFIERS, int QUALIFIERIDX)
3151   throws Exception {
3152 
3153     Get get = new Get(ROWS[ROWIDX]);
3154     get.addFamily(FAMILIES[4]);
3155     get.addColumn(FAMILIES[4], QUALIFIERS[1]);
3156     Result result = ht.get(get);
3157     assertEmptyResult(result);
3158 
3159     get = new Get(ROWS[ROWIDX]);
3160     get.addFamily(FAMILIES[4]);
3161     get.addColumn(FAMILIES[4], QUALIFIERS[2]);
3162     result = ht.get(get);
3163     assertEmptyResult(result);
3164 
3165     get = new Get(ROWS[ROWIDX]);
3166     get.addFamily(FAMILIES[3]);
3167     get.addColumn(FAMILIES[4], QUALIFIERS[2]);
3168     get.addFamily(FAMILIES[5]);
3169     result = ht.get(get);
3170     assertEmptyResult(result);
3171 
3172     get = new Get(ROWS[ROWIDX+1]);
3173     result = ht.get(get);
3174     assertEmptyResult(result);
3175 
3176   }
3177 
3178   private void scanVerifySingleEmpty(HTable ht,
3179       byte [][] ROWS, int ROWIDX,
3180       byte [][] FAMILIES, int FAMILYIDX,
3181       byte [][] QUALIFIERS, int QUALIFIERIDX)
3182   throws Exception {
3183 
3184     Scan scan = new Scan(ROWS[ROWIDX+1]);
3185     Result result = getSingleScanResult(ht, scan);
3186     assertNullResult(result);
3187 
3188     scan = new Scan(ROWS[ROWIDX+1],ROWS[ROWIDX+2]);
3189     result = getSingleScanResult(ht, scan);
3190     assertNullResult(result);
3191 
3192     scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX]);
3193     result = getSingleScanResult(ht, scan);
3194     assertNullResult(result);
3195 
3196     scan = new Scan();
3197     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]);
3198     scan.addFamily(FAMILIES[FAMILYIDX-1]);
3199     result = getSingleScanResult(ht, scan);
3200     assertNullResult(result);
3201 
3202   }
3203 
3204   //
3205   // Verifiers
3206   //
3207 
3208   private void assertKey(Cell key, byte [] row, byte [] family,
3209       byte [] qualifier, byte [] value)
3210   throws Exception {
3211     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3212         "Got row [" + Bytes.toString(CellUtil.cloneRow(key)) +"]",
3213         equals(row, CellUtil.cloneRow(key)));
3214     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3215         "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
3216         equals(family, CellUtil.cloneFamily(key)));
3217     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3218         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key)) + "]",
3219         equals(qualifier, CellUtil.cloneQualifier(key)));
3220     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
3221         "Got value [" + Bytes.toString(CellUtil.cloneValue(key)) + "]",
3222         equals(value, CellUtil.cloneValue(key)));
3223   }
3224 
3225   private void assertIncrementKey(Cell key, byte [] row, byte [] family,
3226       byte [] qualifier, long value)
3227   throws Exception {
3228     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3229         "Got row [" + Bytes.toString(CellUtil.cloneRow(key)) +"]",
3230         equals(row, CellUtil.cloneRow(key)));
3231     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3232         "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
3233         equals(family, CellUtil.cloneFamily(key)));
3234     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3235         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key)) + "]",
3236         equals(qualifier, CellUtil.cloneQualifier(key)));
3237     assertTrue("Expected value [" + value + "] " +
3238         "Got value [" + Bytes.toLong(CellUtil.cloneValue(key)) + "]",
3239         Bytes.toLong(CellUtil.cloneValue(key)) == value);
3240   }
3241 
3242   private void assertNumKeys(Result result, int n) throws Exception {
3243     assertTrue("Expected " + n + " keys but got " + result.size(),
3244         result.size() == n);
3245   }
3246 
3247   private void assertNResult(Result result, byte [] row,
3248       byte [][] families, byte [][] qualifiers, byte [][] values,
3249       int [][] idxs)
3250   throws Exception {
3251     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3252         "Got row [" + Bytes.toString(result.getRow()) +"]",
3253         equals(row, result.getRow()));
3254     assertTrue("Expected " + idxs.length + " keys but result contains "
3255         + result.size(), result.size() == idxs.length);
3256 
3257     Cell [] keys = result.rawCells();
3258 
3259     for(int i=0;i<keys.length;i++) {
3260       byte [] family = families[idxs[i][0]];
3261       byte [] qualifier = qualifiers[idxs[i][1]];
3262       byte [] value = values[idxs[i][2]];
3263       Cell key = keys[i];
3264 
3265       byte[] famb = CellUtil.cloneFamily(key);
3266       byte[] qualb = CellUtil.cloneQualifier(key);
3267       byte[] valb = CellUtil.cloneValue(key);
3268       assertTrue("(" + i + ") Expected family [" + Bytes.toString(family)
3269           + "] " + "Got family [" + Bytes.toString(famb) + "]",
3270           equals(family, famb));
3271       assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier)
3272           + "] " + "Got qualifier [" + Bytes.toString(qualb) + "]",
3273           equals(qualifier, qualb));
3274       assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] "
3275           + "Got value [" + Bytes.toString(valb) + "]",
3276           equals(value, valb));
3277     }
3278   }
3279 
3280   private void assertNResult(Result result, byte [] row,
3281       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
3282       int start, int end)
3283   throws IOException {
3284     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3285         "Got row [" + Bytes.toString(result.getRow()) +"]",
3286         equals(row, result.getRow()));
3287     int expectedResults = end - start + 1;
3288     assertEquals(expectedResults, result.size());
3289 
3290     Cell[] keys = result.rawCells();
3291 
3292     for (int i=0; i<keys.length; i++) {
3293       byte [] value = values[end-i];
3294       long ts = stamps[end-i];
3295       Cell key = keys[i];
3296 
3297       assertTrue("(" + i + ") Expected family [" + Bytes.toString(family)
3298           + "] " + "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
3299           CellUtil.matchingFamily(key, family));
3300       assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier)
3301           + "] " + "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key))+ "]",
3302           CellUtil.matchingQualifier(key, qualifier));
3303       assertTrue("Expected ts [" + ts + "] " +
3304           "Got ts [" + key.getTimestamp() + "]", ts == key.getTimestamp());
3305       assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] "
3306           + "Got value [" + Bytes.toString(CellUtil.cloneValue(key)) + "]",
3307           CellUtil.matchingValue(key,  value));
3308     }
3309   }
3310 
3311   /**
3312    * Validate that result contains two specified keys, exactly.
3313    * It is assumed key A sorts before key B.
3314    */
3315   private void assertDoubleResult(Result result, byte [] row,
3316       byte [] familyA, byte [] qualifierA, byte [] valueA,
3317       byte [] familyB, byte [] qualifierB, byte [] valueB)
3318   throws Exception {
3319     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3320         "Got row [" + Bytes.toString(result.getRow()) +"]",
3321         equals(row, result.getRow()));
3322     assertTrue("Expected two keys but result contains " + result.size(),
3323         result.size() == 2);
3324     Cell [] kv = result.rawCells();
3325     Cell kvA = kv[0];
3326     assertTrue("(A) Expected family [" + Bytes.toString(familyA) + "] " +
3327         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kvA)) + "]",
3328         equals(familyA, CellUtil.cloneFamily(kvA)));
3329     assertTrue("(A) Expected qualifier [" + Bytes.toString(qualifierA) + "] " +
3330         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kvA)) + "]",
3331         equals(qualifierA, CellUtil.cloneQualifier(kvA)));
3332     assertTrue("(A) Expected value [" + Bytes.toString(valueA) + "] " +
3333         "Got value [" + Bytes.toString(CellUtil.cloneValue(kvA)) + "]",
3334         equals(valueA, CellUtil.cloneValue(kvA)));
3335     Cell kvB = kv[1];
3336     assertTrue("(B) Expected family [" + Bytes.toString(familyB) + "] " +
3337         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kvB)) + "]",
3338         equals(familyB, CellUtil.cloneFamily(kvB)));
3339     assertTrue("(B) Expected qualifier [" + Bytes.toString(qualifierB) + "] " +
3340         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kvB)) + "]",
3341         equals(qualifierB, CellUtil.cloneQualifier(kvB)));
3342     assertTrue("(B) Expected value [" + Bytes.toString(valueB) + "] " +
3343         "Got value [" + Bytes.toString(CellUtil.cloneValue(kvB)) + "]",
3344         equals(valueB, CellUtil.cloneValue(kvB)));
3345   }
3346 
3347   private void assertSingleResult(Result result, byte [] row, byte [] family,
3348       byte [] qualifier, byte [] value)
3349   throws Exception {
3350     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3351         "Got row [" + Bytes.toString(result.getRow()) +"]",
3352         equals(row, result.getRow()));
3353     assertTrue("Expected a single key but result contains " + result.size(),
3354         result.size() == 1);
3355     Cell kv = result.rawCells()[0];
3356     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3357         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kv)) + "]",
3358         equals(family, CellUtil.cloneFamily(kv)));
3359     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3360         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]",
3361         equals(qualifier, CellUtil.cloneQualifier(kv)));
3362     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
3363         "Got value [" + Bytes.toString(CellUtil.cloneValue(kv)) + "]",
3364         equals(value, CellUtil.cloneValue(kv)));
3365   }
3366 
3367   private void assertSingleResult(Result result, byte [] row, byte [] family,
3368       byte [] qualifier, long ts, byte [] value)
3369   throws Exception {
3370     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3371         "Got row [" + Bytes.toString(result.getRow()) +"]",
3372         equals(row, result.getRow()));
3373     assertTrue("Expected a single key but result contains " + result.size(),
3374         result.size() == 1);
3375     Cell kv = result.rawCells()[0];
3376     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3377         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kv)) + "]",
3378         equals(family, CellUtil.cloneFamily(kv)));
3379     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3380         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]",
3381         equals(qualifier, CellUtil.cloneQualifier(kv)));
3382     assertTrue("Expected ts [" + ts + "] " +
3383         "Got ts [" + kv.getTimestamp() + "]", ts == kv.getTimestamp());
3384     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
3385         "Got value [" + Bytes.toString(CellUtil.cloneValue(kv)) + "]",
3386         equals(value, CellUtil.cloneValue(kv)));
3387   }
3388 
3389   private void assertEmptyResult(Result result) throws Exception {
3390     assertTrue("expected an empty result but result contains " +
3391         result.size() + " keys", result.isEmpty());
3392   }
3393 
3394   private void assertNullResult(Result result) throws Exception {
3395     assertTrue("expected null result but received a non-null result",
3396         result == null);
3397   }
3398 
3399   //
3400   // Helpers
3401   //
3402 
3403   private Result getSingleScanResult(HTable ht, Scan scan) throws IOException {
3404     ResultScanner scanner = ht.getScanner(scan);
3405     Result result = scanner.next();
3406     scanner.close();
3407     return result;
3408   }
3409 
3410   private byte [][] makeNAscii(byte [] base, int n) {
3411     if(n > 256) {
3412       return makeNBig(base, n);
3413     }
3414     byte [][] ret = new byte[n][];
3415     for(int i=0;i<n;i++) {
3416       byte [] tail = Bytes.toBytes(Integer.toString(i));
3417       ret[i] = Bytes.add(base, tail);
3418     }
3419     return ret;
3420   }
3421 
3422   private byte [][] makeN(byte [] base, int n) {
3423     if (n > 256) {
3424       return makeNBig(base, n);
3425     }
3426     byte [][] ret = new byte[n][];
3427     for(int i=0;i<n;i++) {
3428       ret[i] = Bytes.add(base, new byte[]{(byte)i});
3429     }
3430     return ret;
3431   }
3432 
3433   private byte [][] makeNBig(byte [] base, int n) {
3434     byte [][] ret = new byte[n][];
3435     for(int i=0;i<n;i++) {
3436       int byteA = (i % 256);
3437       int byteB = (i >> 8);
3438       ret[i] = Bytes.add(base, new byte[]{(byte)byteB,(byte)byteA});
3439     }
3440     return ret;
3441   }
3442 
3443   private long [] makeStamps(int n) {
3444     long [] stamps = new long[n];
3445     for(int i=0;i<n;i++) stamps[i] = i+1;
3446     return stamps;
3447   }
3448 
3449   private boolean equals(byte [] left, byte [] right) {
3450     if (left == null && right == null) return true;
3451     if (left == null && right.length == 0) return true;
3452     if (right == null && left.length == 0) return true;
3453     return Bytes.equals(left, right);
3454   }
3455 
3456   @Test
3457   public void testDuplicateVersions() throws Exception {
3458     byte [] TABLE = Bytes.toBytes("testDuplicateVersions");
3459 
3460     long [] STAMPS = makeStamps(20);
3461     byte [][] VALUES = makeNAscii(VALUE, 20);
3462 
3463     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3464 
3465     // Insert 4 versions of same column
3466     Put put = new Put(ROW);
3467     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3468     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3469     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3470     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3471     ht.put(put);
3472 
3473     // Verify we can get each one properly
3474     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3475     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3476     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3477     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3478     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3479     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3480     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3481     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3482 
3483     // Verify we don't accidentally get others
3484     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3485     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3486     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3487     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3488     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3489     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3490 
3491     // Ensure maxVersions in query is respected
3492     Get get = new Get(ROW);
3493     get.addColumn(FAMILY, QUALIFIER);
3494     get.setMaxVersions(2);
3495     Result result = ht.get(get);
3496     assertNResult(result, ROW, FAMILY, QUALIFIER,
3497         new long [] {STAMPS[4], STAMPS[5]},
3498         new byte[][] {VALUES[4], VALUES[5]},
3499         0, 1);
3500 
3501     Scan scan = new Scan(ROW);
3502     scan.addColumn(FAMILY, QUALIFIER);
3503     scan.setMaxVersions(2);
3504     result = getSingleScanResult(ht, scan);
3505     assertNResult(result, ROW, FAMILY, QUALIFIER,
3506         new long [] {STAMPS[4], STAMPS[5]},
3507         new byte[][] {VALUES[4], VALUES[5]},
3508         0, 1);
3509 
3510     // Flush and redo
3511 
3512     TEST_UTIL.flush();
3513 
3514     // Verify we can get each one properly
3515     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3516     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3517     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3518     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3519     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3520     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3521     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3522     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3523 
3524     // Verify we don't accidentally get others
3525     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3526     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3527     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3528     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3529     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3530     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3531 
3532     // Ensure maxVersions in query is respected
3533     get = new Get(ROW);
3534     get.addColumn(FAMILY, QUALIFIER);
3535     get.setMaxVersions(2);
3536     result = ht.get(get);
3537     assertNResult(result, ROW, FAMILY, QUALIFIER,
3538         new long [] {STAMPS[4], STAMPS[5]},
3539         new byte[][] {VALUES[4], VALUES[5]},
3540         0, 1);
3541 
3542     scan = new Scan(ROW);
3543     scan.addColumn(FAMILY, QUALIFIER);
3544     scan.setMaxVersions(2);
3545     result = getSingleScanResult(ht, scan);
3546     assertNResult(result, ROW, FAMILY, QUALIFIER,
3547         new long [] {STAMPS[4], STAMPS[5]},
3548         new byte[][] {VALUES[4], VALUES[5]},
3549         0, 1);
3550 
3551 
3552     // Add some memstore and retest
3553 
3554     // Insert 4 more versions of same column and a dupe
3555     put = new Put(ROW);
3556     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
3557     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3558     put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
3559     put.add(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3560     put.add(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
3561     ht.put(put);
3562 
3563     // Ensure maxVersions in query is respected
3564     get = new Get(ROW);
3565     get.addColumn(FAMILY, QUALIFIER);
3566     get.setMaxVersions(7);
3567     result = ht.get(get);
3568     assertNResult(result, ROW, FAMILY, QUALIFIER,
3569         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3570         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3571         0, 6);
3572 
3573     scan = new Scan(ROW);
3574     scan.addColumn(FAMILY, QUALIFIER);
3575     scan.setMaxVersions(7);
3576     result = getSingleScanResult(ht, scan);
3577     assertNResult(result, ROW, FAMILY, QUALIFIER,
3578         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3579         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3580         0, 6);
3581 
3582     get = new Get(ROW);
3583     get.setMaxVersions(7);
3584     result = ht.get(get);
3585     assertNResult(result, ROW, FAMILY, QUALIFIER,
3586         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3587         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3588         0, 6);
3589 
3590     scan = new Scan(ROW);
3591     scan.setMaxVersions(7);
3592     result = getSingleScanResult(ht, scan);
3593     assertNResult(result, ROW, FAMILY, QUALIFIER,
3594         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3595         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3596         0, 6);
3597 
3598     // Verify we can get each one properly
3599     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3600     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3601     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3602     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3603     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3604     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3605     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3606     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3607 
3608     // Verify we don't accidentally get others
3609     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3610     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
3611     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3612     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
3613 
3614     // Ensure maxVersions of table is respected
3615 
3616     TEST_UTIL.flush();
3617 
3618     // Insert 4 more versions of same column and a dupe
3619     put = new Put(ROW);
3620     put.add(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
3621     put.add(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
3622     put.add(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
3623     put.add(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
3624     ht.put(put);
3625 
3626     get = new Get(ROW);
3627     get.addColumn(FAMILY, QUALIFIER);
3628     get.setMaxVersions(Integer.MAX_VALUE);
3629     result = ht.get(get);
3630     assertNResult(result, ROW, FAMILY, QUALIFIER,
3631         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
3632         new byte[][] {VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
3633         0, 9);
3634 
3635     scan = new Scan(ROW);
3636     scan.addColumn(FAMILY, QUALIFIER);
3637     scan.setMaxVersions(Integer.MAX_VALUE);
3638     result = getSingleScanResult(ht, scan);
3639     assertNResult(result, ROW, FAMILY, QUALIFIER,
3640         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
3641         new byte[][] {VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
3642         0, 9);
3643 
3644     // Delete a version in the memstore and a version in a storefile
3645     Delete delete = new Delete(ROW);
3646     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[11]);
3647     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[7]);
3648     ht.delete(delete);
3649 
3650     // Test that it's gone
3651     get = new Get(ROW);
3652     get.addColumn(FAMILY, QUALIFIER);
3653     get.setMaxVersions(Integer.MAX_VALUE);
3654     result = ht.get(get);
3655     assertNResult(result, ROW, FAMILY, QUALIFIER,
3656         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
3657         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
3658         0, 9);
3659 
3660     scan = new Scan(ROW);
3661     scan.addColumn(FAMILY, QUALIFIER);
3662     scan.setMaxVersions(Integer.MAX_VALUE);
3663     result = getSingleScanResult(ht, scan);
3664     assertNResult(result, ROW, FAMILY, QUALIFIER,
3665         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
3666         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
3667         0, 9);
3668   }
3669 
3670   @Test
3671   public void testUpdates() throws Exception {
3672 
3673     byte [] TABLE = Bytes.toBytes("testUpdates");
3674     HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3675 
3676     // Write a column with values at timestamp 1, 2 and 3
3677     byte[] row = Bytes.toBytes("row1");
3678     byte[] qualifier = Bytes.toBytes("myCol");
3679     Put put = new Put(row);
3680     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3681     hTable.put(put);
3682 
3683     put = new Put(row);
3684     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3685     hTable.put(put);
3686 
3687     put = new Put(row);
3688     put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3689     hTable.put(put);
3690 
3691     Get get = new Get(row);
3692     get.addColumn(FAMILY, qualifier);
3693     get.setMaxVersions();
3694 
3695     // Check that the column indeed has the right values at timestamps 1 and
3696     // 2
3697     Result result = hTable.get(get);
3698     NavigableMap<Long, byte[]> navigableMap =
3699         result.getMap().get(FAMILY).get(qualifier);
3700     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3701     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3702 
3703     // Update the value at timestamp 1
3704     put = new Put(row);
3705     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3706     hTable.put(put);
3707 
3708     // Update the value at timestamp 2
3709     put = new Put(row);
3710     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3711     hTable.put(put);
3712 
3713     // Check that the values at timestamp 2 and 1 got updated
3714     result = hTable.get(get);
3715     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3716     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3717     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3718   }
3719 
3720   @Test
3721   public void testUpdatesWithMajorCompaction() throws Exception {
3722 
3723     String tableName = "testUpdatesWithMajorCompaction";
3724     byte [] TABLE = Bytes.toBytes(tableName);
3725     HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3726     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
3727 
3728     // Write a column with values at timestamp 1, 2 and 3
3729     byte[] row = Bytes.toBytes("row2");
3730     byte[] qualifier = Bytes.toBytes("myCol");
3731     Put put = new Put(row);
3732     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3733     hTable.put(put);
3734 
3735     put = new Put(row);
3736     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3737     hTable.put(put);
3738 
3739     put = new Put(row);
3740     put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3741     hTable.put(put);
3742 
3743     Get get = new Get(row);
3744     get.addColumn(FAMILY, qualifier);
3745     get.setMaxVersions();
3746 
3747     // Check that the column indeed has the right values at timestamps 1 and
3748     // 2
3749     Result result = hTable.get(get);
3750     NavigableMap<Long, byte[]> navigableMap =
3751         result.getMap().get(FAMILY).get(qualifier);
3752     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3753     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3754 
3755     // Trigger a major compaction
3756     admin.flush(tableName);
3757     admin.majorCompact(tableName);
3758     Thread.sleep(6000);
3759 
3760     // Update the value at timestamp 1
3761     put = new Put(row);
3762     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3763     hTable.put(put);
3764 
3765     // Update the value at timestamp 2
3766     put = new Put(row);
3767     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3768     hTable.put(put);
3769 
3770     // Trigger a major compaction
3771     admin.flush(tableName);
3772     admin.majorCompact(tableName);
3773     Thread.sleep(6000);
3774 
3775     // Check that the values at timestamp 2 and 1 got updated
3776     result = hTable.get(get);
3777     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3778     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3779     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3780   }
3781 
3782   @Test
3783   public void testMajorCompactionBetweenTwoUpdates() throws Exception {
3784 
3785     String tableName = "testMajorCompactionBetweenTwoUpdates";
3786     byte [] TABLE = Bytes.toBytes(tableName);
3787     HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3788     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
3789 
3790     // Write a column with values at timestamp 1, 2 and 3
3791     byte[] row = Bytes.toBytes("row3");
3792     byte[] qualifier = Bytes.toBytes("myCol");
3793     Put put = new Put(row);
3794     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3795     hTable.put(put);
3796 
3797     put = new Put(row);
3798     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3799     hTable.put(put);
3800 
3801     put = new Put(row);
3802     put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3803     hTable.put(put);
3804 
3805     Get get = new Get(row);
3806     get.addColumn(FAMILY, qualifier);
3807     get.setMaxVersions();
3808 
3809     // Check that the column indeed has the right values at timestamps 1 and
3810     // 2
3811     Result result = hTable.get(get);
3812     NavigableMap<Long, byte[]> navigableMap =
3813         result.getMap().get(FAMILY).get(qualifier);
3814     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3815     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3816 
3817     // Trigger a major compaction
3818     admin.flush(tableName);
3819     admin.majorCompact(tableName);
3820     Thread.sleep(6000);
3821 
3822     // Update the value at timestamp 1
3823     put = new Put(row);
3824     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3825     hTable.put(put);
3826 
3827     // Trigger a major compaction
3828     admin.flush(tableName);
3829     admin.majorCompact(tableName);
3830     Thread.sleep(6000);
3831 
3832     // Update the value at timestamp 2
3833     put = new Put(row);
3834     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3835     hTable.put(put);
3836 
3837     // Trigger a major compaction
3838     admin.flush(tableName);
3839     admin.majorCompact(tableName);
3840     Thread.sleep(6000);
3841 
3842     // Check that the values at timestamp 2 and 1 got updated
3843     result = hTable.get(get);
3844     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3845 
3846     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3847     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3848   }
3849 
3850   @Test
3851   public void testGet_EmptyTable() throws IOException {
3852     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testGet_EmptyTable"), FAMILY);
3853     Get get = new Get(ROW);
3854     get.addFamily(FAMILY);
3855     Result r = table.get(get);
3856     assertTrue(r.isEmpty());
3857   }
3858 
3859   @Test
3860   public void testGet_NullQualifier() throws IOException {
3861     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testGet_NullQualifier"), FAMILY);
3862     Put put = new Put(ROW);
3863     put.add(FAMILY, QUALIFIER, VALUE);
3864     table.put(put);
3865 
3866     put = new Put(ROW);
3867     put.add(FAMILY, null, VALUE);
3868     table.put(put);
3869     LOG.info("Row put");
3870 
3871     Get get = new Get(ROW);
3872     get.addColumn(FAMILY, null);
3873     Result r = table.get(get);
3874     assertEquals(1, r.size());
3875 
3876     get = new Get(ROW);
3877     get.addFamily(FAMILY);
3878     r = table.get(get);
3879     assertEquals(2, r.size());
3880   }
3881 
3882   @Test
3883   public void testGet_NonExistentRow() throws IOException {
3884     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testGet_NonExistentRow"), FAMILY);
3885     Put put = new Put(ROW);
3886     put.add(FAMILY, QUALIFIER, VALUE);
3887     table.put(put);
3888     LOG.info("Row put");
3889 
3890     Get get = new Get(ROW);
3891     get.addFamily(FAMILY);
3892     Result r = table.get(get);
3893     assertFalse(r.isEmpty());
3894     System.out.println("Row retrieved successfully");
3895 
3896     byte [] missingrow = Bytes.toBytes("missingrow");
3897     get = new Get(missingrow);
3898     get.addFamily(FAMILY);
3899     r = table.get(get);
3900     assertTrue(r.isEmpty());
3901     LOG.info("Row missing as it should be");
3902   }
3903 
3904   @Test
3905   public void testPut() throws IOException {
3906     final byte [] CONTENTS_FAMILY = Bytes.toBytes("contents");
3907     final byte [] SMALL_FAMILY = Bytes.toBytes("smallfam");
3908     final byte [] row1 = Bytes.toBytes("row1");
3909     final byte [] row2 = Bytes.toBytes("row2");
3910     final byte [] value = Bytes.toBytes("abcd");
3911     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testPut"),
3912       new byte [][] {CONTENTS_FAMILY, SMALL_FAMILY});
3913     Put put = new Put(row1);
3914     put.add(CONTENTS_FAMILY, null, value);
3915     table.put(put);
3916 
3917     put = new Put(row2);
3918     put.add(CONTENTS_FAMILY, null, value);
3919 
3920     assertEquals(put.size(), 1);
3921     assertEquals(put.getFamilyCellMap().get(CONTENTS_FAMILY).size(), 1);
3922 
3923     // KeyValue v1 expectation.  Cast for now until we go all Cell all the time. TODO
3924     KeyValue kv = (KeyValue)put.getFamilyCellMap().get(CONTENTS_FAMILY).get(0);
3925 
3926     assertTrue(Bytes.equals(kv.getFamily(), CONTENTS_FAMILY));
3927     // will it return null or an empty byte array?
3928     assertTrue(Bytes.equals(kv.getQualifier(), new byte[0]));
3929 
3930     assertTrue(Bytes.equals(kv.getValue(), value));
3931 
3932     table.put(put);
3933 
3934     Scan scan = new Scan();
3935     scan.addColumn(CONTENTS_FAMILY, null);
3936     ResultScanner scanner = table.getScanner(scan);
3937     for (Result r : scanner) {
3938       for(Cell key : r.rawCells()) {
3939         System.out.println(Bytes.toString(r.getRow()) + ": " + key.toString());
3940       }
3941     }
3942   }
3943 
3944   @Test
3945   public void testPutNoCF() throws IOException {
3946     final byte[] BAD_FAM = Bytes.toBytes("BAD_CF");
3947     final byte[] VAL = Bytes.toBytes(100);
3948     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testPutNoCF"), new byte[][]{FAMILY});
3949 
3950     boolean caughtNSCFE = false;
3951 
3952     try {
3953       Put p = new Put(ROW);
3954       p.add(BAD_FAM, QUALIFIER, VAL);
3955       table.put(p);
3956     } catch (RetriesExhaustedWithDetailsException e) {
3957       caughtNSCFE = e.getCause(0) instanceof NoSuchColumnFamilyException;
3958     }
3959     assertTrue("Should throw NoSuchColumnFamilyException", caughtNSCFE);
3960 
3961   }
3962 
3963   @Test
3964   public void testRowsPut() throws IOException {
3965     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
3966     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
3967     final int NB_BATCH_ROWS = 10;
3968     final byte[] value = Bytes.toBytes("abcd");
3969     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPut"),
3970       new byte[][] {CONTENTS_FAMILY, SMALL_FAMILY });
3971     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
3972     for (int i = 0; i < NB_BATCH_ROWS; i++) {
3973       byte[] row = Bytes.toBytes("row" + i);
3974       Put put = new Put(row);
3975       put.setDurability(Durability.SKIP_WAL);
3976       put.add(CONTENTS_FAMILY, null, value);
3977       rowsUpdate.add(put);
3978     }
3979     table.put(rowsUpdate);
3980     Scan scan = new Scan();
3981     scan.addFamily(CONTENTS_FAMILY);
3982     ResultScanner scanner = table.getScanner(scan);
3983     int nbRows = 0;
3984     for (@SuppressWarnings("unused")
3985     Result row : scanner)
3986       nbRows++;
3987     assertEquals(NB_BATCH_ROWS, nbRows);
3988   }
3989 
3990   @Test
3991   public void testRowsPutBufferedOneFlush() throws IOException {
3992     final byte [] CONTENTS_FAMILY = Bytes.toBytes("contents");
3993     final byte [] SMALL_FAMILY = Bytes.toBytes("smallfam");
3994     final byte [] value = Bytes.toBytes("abcd");
3995     final int NB_BATCH_ROWS = 10;
3996     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPutBufferedOneFlush"),
3997       new byte [][] {CONTENTS_FAMILY, SMALL_FAMILY});
3998     table.setAutoFlush(false, true);
3999     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
4000     for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
4001       byte[] row = Bytes.toBytes("row" + i);
4002       Put put = new Put(row);
4003       put.setDurability(Durability.SKIP_WAL);
4004       put.add(CONTENTS_FAMILY, null, value);
4005       rowsUpdate.add(put);
4006     }
4007     table.put(rowsUpdate);
4008 
4009     Scan scan = new Scan();
4010     scan.addFamily(CONTENTS_FAMILY);
4011     ResultScanner scanner = table.getScanner(scan);
4012     int nbRows = 0;
4013     for (@SuppressWarnings("unused")
4014     Result row : scanner)
4015       nbRows++;
4016     assertEquals(0, nbRows);
4017     scanner.close();
4018 
4019     table.flushCommits();
4020 
4021     scan = new Scan();
4022     scan.addFamily(CONTENTS_FAMILY);
4023     scanner = table.getScanner(scan);
4024     nbRows = 0;
4025     for (@SuppressWarnings("unused")
4026     Result row : scanner)
4027       nbRows++;
4028     assertEquals(NB_BATCH_ROWS * 10, nbRows);
4029   }
4030 
4031   @Test
4032   public void testRowsPutBufferedManyManyFlushes() throws IOException {
4033     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
4034     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
4035     final byte[] value = Bytes.toBytes("abcd");
4036     final int NB_BATCH_ROWS = 10;
4037     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPutBufferedManyManyFlushes"),
4038       new byte[][] {CONTENTS_FAMILY, SMALL_FAMILY });
4039     table.setAutoFlush(false, true);
4040     table.setWriteBufferSize(10);
4041     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
4042     for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
4043       byte[] row = Bytes.toBytes("row" + i);
4044       Put put = new Put(row);
4045       put.setDurability(Durability.SKIP_WAL);
4046       put.add(CONTENTS_FAMILY, null, value);
4047       rowsUpdate.add(put);
4048     }
4049     table.put(rowsUpdate);
4050 
4051     table.flushCommits();
4052 
4053     Scan scan = new Scan();
4054     scan.addFamily(CONTENTS_FAMILY);
4055     ResultScanner scanner = table.getScanner(scan);
4056     int nbRows = 0;
4057     for (@SuppressWarnings("unused")
4058     Result row : scanner)
4059       nbRows++;
4060     assertEquals(NB_BATCH_ROWS * 10, nbRows);
4061   }
4062 
4063   @Test
4064   public void testAddKeyValue() throws IOException {
4065     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
4066     final byte[] value = Bytes.toBytes("abcd");
4067     final byte[] row1 = Bytes.toBytes("row1");
4068     final byte[] row2 = Bytes.toBytes("row2");
4069     byte[] qualifier = Bytes.toBytes("qf1");
4070     Put put = new Put(row1);
4071 
4072     // Adding KeyValue with the same row
4073     KeyValue kv = new KeyValue(row1, CONTENTS_FAMILY, qualifier, value);
4074     boolean ok = true;
4075     try {
4076       put.add(kv);
4077     } catch (IOException e) {
4078       ok = false;
4079     }
4080     assertEquals(true, ok);
4081 
4082     // Adding KeyValue with the different row
4083     kv = new KeyValue(row2, CONTENTS_FAMILY, qualifier, value);
4084     ok = false;
4085     try {
4086       put.add(kv);
4087     } catch (IOException e) {
4088       ok = true;
4089     }
4090     assertEquals(true, ok);
4091   }
4092 
4093   /**
4094    * test for HBASE-737
4095    * @throws IOException
4096    */
4097   @Test
4098   public void testHBase737 () throws IOException {
4099     final byte [] FAM1 = Bytes.toBytes("fam1");
4100     final byte [] FAM2 = Bytes.toBytes("fam2");
4101     // Open table
4102     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testHBase737"),
4103       new byte [][] {FAM1, FAM2});
4104     // Insert some values
4105     Put put = new Put(ROW);
4106     put.add(FAM1, Bytes.toBytes("letters"), Bytes.toBytes("abcdefg"));
4107     table.put(put);
4108     try {
4109       Thread.sleep(1000);
4110     } catch (InterruptedException i) {
4111       //ignore
4112     }
4113 
4114     put = new Put(ROW);
4115     put.add(FAM1, Bytes.toBytes("numbers"), Bytes.toBytes("123456"));
4116     table.put(put);
4117 
4118     try {
4119       Thread.sleep(1000);
4120     } catch (InterruptedException i) {
4121       //ignore
4122     }
4123 
4124     put = new Put(ROW);
4125     put.add(FAM2, Bytes.toBytes("letters"), Bytes.toBytes("hijklmnop"));
4126     table.put(put);
4127 
4128     long times[] = new long[3];
4129 
4130     // First scan the memstore
4131 
4132     Scan scan = new Scan();
4133     scan.addFamily(FAM1);
4134     scan.addFamily(FAM2);
4135     ResultScanner s = table.getScanner(scan);
4136     try {
4137       int index = 0;
4138       Result r = null;
4139       while ((r = s.next()) != null) {
4140         for(Cell key : r.rawCells()) {
4141           times[index++] = key.getTimestamp();
4142         }
4143       }
4144     } finally {
4145       s.close();
4146     }
4147     for (int i = 0; i < times.length - 1; i++) {
4148       for (int j = i + 1; j < times.length; j++) {
4149         assertTrue(times[j] > times[i]);
4150       }
4151     }
4152 
4153     // Flush data to disk and try again
4154     TEST_UTIL.flush();
4155 
4156     // Reset times
4157     for(int i=0;i<times.length;i++) {
4158       times[i] = 0;
4159     }
4160 
4161     try {
4162       Thread.sleep(1000);
4163     } catch (InterruptedException i) {
4164       //ignore
4165     }
4166     scan = new Scan();
4167     scan.addFamily(FAM1);
4168     scan.addFamily(FAM2);
4169     s = table.getScanner(scan);
4170     try {
4171       int index = 0;
4172       Result r = null;
4173       while ((r = s.next()) != null) {
4174         for(Cell key : r.rawCells()) {
4175           times[index++] = key.getTimestamp();
4176         }
4177       }
4178     } finally {
4179       s.close();
4180     }
4181     for (int i = 0; i < times.length - 1; i++) {
4182       for (int j = i + 1; j < times.length; j++) {
4183         assertTrue(times[j] > times[i]);
4184       }
4185     }
4186   }
4187 
4188   @Test
4189   public void testListTables() throws IOException, InterruptedException {
4190     byte [] t1 = Bytes.toBytes("testListTables1");
4191     byte [] t2 = Bytes.toBytes("testListTables2");
4192     byte [] t3 = Bytes.toBytes("testListTables3");
4193     byte [][] tables = new byte[][] { t1, t2, t3 };
4194     for (int i = 0; i < tables.length; i++) {
4195       TEST_UTIL.createTable(tables[i], FAMILY);
4196     }
4197     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
4198     HTableDescriptor[] ts = admin.listTables();
4199     HashSet<HTableDescriptor> result = new HashSet<HTableDescriptor>(ts.length);
4200     Collections.addAll(result, ts);
4201     int size = result.size();
4202     assertTrue(size >= tables.length);
4203     for (int i = 0; i < tables.length && i < size; i++) {
4204       boolean found = false;
4205       for (int j = 0; j < ts.length; j++) {
4206         if (Bytes.equals(ts[j].getTableName().getName(), tables[i])) {
4207           found = true;
4208           break;
4209         }
4210       }
4211       assertTrue("Not found: " + Bytes.toString(tables[i]), found);
4212     }
4213   }
4214 
4215   /**
4216    * creates an HTable for tableName using an unmanaged HConnection.
4217    *
4218    * @param tableName - table to create
4219    * @return the created HTable object
4220    * @throws IOException
4221    */
4222   HTable createUnmangedHConnectionHTable(final byte [] tableName) throws IOException {
4223     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
4224     HConnection conn = HConnectionManager.createConnection(TEST_UTIL.getConfiguration());
4225     return (HTable)conn.getTable(tableName);
4226   }
4227 
4228   /**
4229    * simple test that just executes parts of the client
4230    * API that accept a pre-created HConnection instance
4231    *
4232    * @throws IOException
4233    */
4234   @Test
4235   public void testUnmanagedHConnection() throws IOException {
4236     final byte[] tableName = Bytes.toBytes("testUnmanagedHConnection");
4237     HTable t = createUnmangedHConnectionHTable(tableName);
4238     HBaseAdmin ha = new HBaseAdmin(t.getConnection());
4239     assertTrue(ha.tableExists(tableName));
4240     assertTrue(t.get(new Get(ROW)).isEmpty());
4241   }
4242 
4243   /**
4244    * test of that unmanaged HConnections are able to reconnect
4245    * properly (see HBASE-5058)
4246    *
4247    * @throws Exception
4248    */
4249   @Test
4250   public void testUnmanagedHConnectionReconnect() throws Exception {
4251     final byte[] tableName = Bytes.toBytes("testUnmanagedHConnectionReconnect");
4252     HTable t = createUnmangedHConnectionHTable(tableName);
4253     HConnection conn = t.getConnection();
4254     HBaseAdmin ha = new HBaseAdmin(conn);
4255     assertTrue(ha.tableExists(tableName));
4256     assertTrue(t.get(new Get(ROW)).isEmpty());
4257 
4258     // stop the master
4259     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
4260     cluster.stopMaster(0, false);
4261     cluster.waitOnMaster(0);
4262 
4263     // start up a new master
4264     cluster.startMaster();
4265     assertTrue(cluster.waitForActiveAndReadyMaster());
4266 
4267     // test that the same unmanaged connection works with a new
4268     // HBaseAdmin and can connect to the new master;
4269     HBaseAdmin newAdmin = new HBaseAdmin(conn);
4270     assertTrue(newAdmin.tableExists(tableName));
4271     assert(newAdmin.getClusterStatus().getServersSize() == SLAVES);
4272   }
4273 
4274   @Test
4275   public void testMiscHTableStuff() throws IOException {
4276     final byte[] tableAname = Bytes.toBytes("testMiscHTableStuffA");
4277     final byte[] tableBname = Bytes.toBytes("testMiscHTableStuffB");
4278     final byte[] attrName = Bytes.toBytes("TESTATTR");
4279     final byte[] attrValue = Bytes.toBytes("somevalue");
4280     byte[] value = Bytes.toBytes("value");
4281 
4282     HTable a = TEST_UTIL.createTable(tableAname, HConstants.CATALOG_FAMILY);
4283     HTable b = TEST_UTIL.createTable(tableBname, HConstants.CATALOG_FAMILY);
4284     Put put = new Put(ROW);
4285     put.add(HConstants.CATALOG_FAMILY, null, value);
4286     a.put(put);
4287 
4288     // open a new connection to A and a connection to b
4289     HTable newA = new HTable(TEST_UTIL.getConfiguration(), tableAname);
4290 
4291     // copy data from A to B
4292     Scan scan = new Scan();
4293     scan.addFamily(HConstants.CATALOG_FAMILY);
4294     ResultScanner s = newA.getScanner(scan);
4295     try {
4296       for (Result r : s) {
4297         put = new Put(r.getRow());
4298         put.setDurability(Durability.SKIP_WAL);
4299         for (Cell kv : r.rawCells()) {
4300           put.add(kv);
4301         }
4302         b.put(put);
4303       }
4304     } finally {
4305       s.close();
4306     }
4307 
4308     // Opening a new connection to A will cause the tables to be reloaded
4309     HTable anotherA = new HTable(TEST_UTIL.getConfiguration(), tableAname);
4310     Get get = new Get(ROW);
4311     get.addFamily(HConstants.CATALOG_FAMILY);
4312     anotherA.get(get);
4313 
4314     // We can still access A through newA because it has the table information
4315     // cached. And if it needs to recalibrate, that will cause the information
4316     // to be reloaded.
4317 
4318     // Test user metadata
4319     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
4320     // make a modifiable descriptor
4321     HTableDescriptor desc = new HTableDescriptor(a.getTableDescriptor());
4322     // offline the table
4323     admin.disableTable(tableAname);
4324     // add a user attribute to HTD
4325     desc.setValue(attrName, attrValue);
4326     // add a user attribute to HCD
4327     for (HColumnDescriptor c : desc.getFamilies())
4328       c.setValue(attrName, attrValue);
4329     // update metadata for all regions of this table
4330     admin.modifyTable(tableAname, desc);
4331     // enable the table
4332     admin.enableTable(tableAname);
4333 
4334     // Test that attribute changes were applied
4335     desc = a.getTableDescriptor();
4336     assertTrue("wrong table descriptor returned",
4337       Bytes.compareTo(desc.getTableName().getName(), tableAname) == 0);
4338     // check HTD attribute
4339     value = desc.getValue(attrName);
4340     assertFalse("missing HTD attribute value", value == null);
4341     assertFalse("HTD attribute value is incorrect",
4342       Bytes.compareTo(value, attrValue) != 0);
4343     // check HCD attribute
4344     for (HColumnDescriptor c : desc.getFamilies()) {
4345       value = c.getValue(attrName);
4346       assertFalse("missing HCD attribute value", value == null);
4347       assertFalse("HCD attribute value is incorrect",
4348         Bytes.compareTo(value, attrValue) != 0);
4349     }
4350   }
4351 
4352   @Test
4353   public void testGetClosestRowBefore() throws IOException, InterruptedException {
4354     final byte[] tableAname = Bytes.toBytes("testGetClosestRowBefore");
4355     final byte[] firstRow = Bytes.toBytes("row111");
4356     final byte[] secondRow = Bytes.toBytes("row222");
4357     final byte[] thirdRow = Bytes.toBytes("row333");
4358     final byte[] forthRow = Bytes.toBytes("row444");
4359     final byte[] beforeFirstRow = Bytes.toBytes("row");
4360     final byte[] beforeSecondRow = Bytes.toBytes("row22");
4361     final byte[] beforeThirdRow = Bytes.toBytes("row33");
4362     final byte[] beforeForthRow = Bytes.toBytes("row44");
4363 
4364     HTable table =
4365         TEST_UTIL.createTable(tableAname,
4366           new byte[][] { HConstants.CATALOG_FAMILY, Bytes.toBytes("info2") }, 1, 1024);
4367     // set block size to 64 to making 2 kvs into one block, bypassing the walkForwardInSingleRow
4368     // in Store.rowAtOrBeforeFromStoreFile
4369     table.setAutoFlush(true);
4370     String regionName = table.getRegionLocations().firstKey().getEncodedName();
4371     HRegion region =
4372         TEST_UTIL.getRSForFirstRegionInTable(tableAname).getFromOnlineRegions(regionName);
4373     Put put1 = new Put(firstRow);
4374     Put put2 = new Put(secondRow);
4375     Put put3 = new Put(thirdRow);
4376     Put put4 = new Put(forthRow);
4377     byte[] one = new byte[] { 1 };
4378     byte[] two = new byte[] { 2 };
4379     byte[] three = new byte[] { 3 };
4380     byte[] four = new byte[] { 4 };
4381 
4382     put1.add(HConstants.CATALOG_FAMILY, null, one);
4383     put2.add(HConstants.CATALOG_FAMILY, null, two);
4384     put3.add(HConstants.CATALOG_FAMILY, null, three);
4385     put4.add(HConstants.CATALOG_FAMILY, null, four);
4386     table.put(put1);
4387     table.put(put2);
4388     table.put(put3);
4389     table.put(put4);
4390     region.flushcache();
4391     Result result = null;
4392 
4393     // Test before first that null is returned
4394     result = table.getRowOrBefore(beforeFirstRow, HConstants.CATALOG_FAMILY);
4395     assertTrue(result == null);
4396 
4397     // Test at first that first is returned
4398     result = table.getRowOrBefore(firstRow, HConstants.CATALOG_FAMILY);
4399     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4400     assertTrue(Bytes.equals(result.getRow(), firstRow));
4401     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
4402 
4403     // Test in between first and second that first is returned
4404     result = table.getRowOrBefore(beforeSecondRow, HConstants.CATALOG_FAMILY);
4405     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4406     assertTrue(Bytes.equals(result.getRow(), firstRow));
4407     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
4408 
4409     // Test at second make sure second is returned
4410     result = table.getRowOrBefore(secondRow, HConstants.CATALOG_FAMILY);
4411     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4412     assertTrue(Bytes.equals(result.getRow(), secondRow));
4413     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two));
4414 
4415     // Test in second and third, make sure second is returned
4416     result = table.getRowOrBefore(beforeThirdRow, HConstants.CATALOG_FAMILY);
4417     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4418     assertTrue(Bytes.equals(result.getRow(), secondRow));
4419     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two));
4420 
4421     // Test at third make sure third is returned
4422     result = table.getRowOrBefore(thirdRow, HConstants.CATALOG_FAMILY);
4423     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4424     assertTrue(Bytes.equals(result.getRow(), thirdRow));
4425     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three));
4426 
4427     // Test in third and forth, make sure third is returned
4428     result = table.getRowOrBefore(beforeForthRow, HConstants.CATALOG_FAMILY);
4429     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4430     assertTrue(Bytes.equals(result.getRow(), thirdRow));
4431     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three));
4432 
4433     // Test at forth make sure forth is returned
4434     result = table.getRowOrBefore(forthRow, HConstants.CATALOG_FAMILY);
4435     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4436     assertTrue(Bytes.equals(result.getRow(), forthRow));
4437     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four));
4438 
4439     // Test after forth make sure forth is returned
4440     result = table.getRowOrBefore(Bytes.add(forthRow, one), HConstants.CATALOG_FAMILY);
4441     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4442     assertTrue(Bytes.equals(result.getRow(), forthRow));
4443     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four));
4444   }
4445 
4446   /**
4447    * For HBASE-2156
4448    * @throws Exception
4449    */
4450   @Test
4451   public void testScanVariableReuse() throws Exception {
4452     Scan scan = new Scan();
4453     scan.addFamily(FAMILY);
4454     scan.addColumn(FAMILY, ROW);
4455 
4456     assertTrue(scan.getFamilyMap().get(FAMILY).size() == 1);
4457 
4458     scan = new Scan();
4459     scan.addFamily(FAMILY);
4460 
4461     assertTrue(scan.getFamilyMap().get(FAMILY) == null);
4462     assertTrue(scan.getFamilyMap().containsKey(FAMILY));
4463   }
4464 
4465   @Test
4466   public void testMultiRowMutation() throws Exception {
4467     LOG.info("Starting testMultiRowMutation");
4468     final byte [] TABLENAME = Bytes.toBytes("testMultiRowMutation");
4469     final byte [] ROW1 = Bytes.toBytes("testRow1");
4470 
4471     HTable t = TEST_UTIL.createTable(TABLENAME, FAMILY);
4472     Put p = new Put(ROW);
4473     p.add(FAMILY, QUALIFIER, VALUE);
4474     MutationProto m1 = ProtobufUtil.toMutation(MutationType.PUT, p);
4475 
4476     p = new Put(ROW1);
4477     p.add(FAMILY, QUALIFIER, VALUE);
4478     MutationProto m2 = ProtobufUtil.toMutation(MutationType.PUT, p);
4479 
4480     MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
4481     mrmBuilder.addMutationRequest(m1);
4482     mrmBuilder.addMutationRequest(m2);
4483     MutateRowsRequest mrm = mrmBuilder.build();
4484     CoprocessorRpcChannel channel = t.coprocessorService(ROW);
4485     MultiRowMutationService.BlockingInterface service =
4486        MultiRowMutationService.newBlockingStub(channel);
4487     service.mutateRows(null, mrm);
4488     Get g = new Get(ROW);
4489     Result r = t.get(g);
4490     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)));
4491     g = new Get(ROW1);
4492     r = t.get(g);
4493     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)));
4494   }
4495 
4496   @Test
4497   public void testRowMutation() throws Exception {
4498     LOG.info("Starting testRowMutation");
4499     final byte [] TABLENAME = Bytes.toBytes("testRowMutation");
4500     HTable t = TEST_UTIL.createTable(TABLENAME, FAMILY);
4501     byte [][] QUALIFIERS = new byte [][] {
4502         Bytes.toBytes("a"), Bytes.toBytes("b")
4503     };
4504     RowMutations arm = new RowMutations(ROW);
4505     Put p = new Put(ROW);
4506     p.add(FAMILY, QUALIFIERS[0], VALUE);
4507     arm.add(p);
4508     t.mutateRow(arm);
4509 
4510     Get g = new Get(ROW);
4511     Result r = t.get(g);
4512     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[0])));
4513 
4514     arm = new RowMutations(ROW);
4515     p = new Put(ROW);
4516     p.add(FAMILY, QUALIFIERS[1], VALUE);
4517     arm.add(p);
4518     Delete d = new Delete(ROW);
4519     d.deleteColumns(FAMILY, QUALIFIERS[0]);
4520     arm.add(d);
4521     // TODO: Trying mutateRow again.  The batch was failing with a one try only.
4522     t.mutateRow(arm);
4523     r = t.get(g);
4524     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[1])));
4525     assertNull(r.getValue(FAMILY, QUALIFIERS[0]));
4526 
4527     //Test that we get a region level exception
4528     try {
4529       arm = new RowMutations(ROW);
4530       p = new Put(ROW);
4531       p.add(new byte[]{'b', 'o', 'g', 'u', 's'}, QUALIFIERS[0], VALUE);
4532       arm.add(p);
4533       t.mutateRow(arm);
4534       fail("Expected NoSuchColumnFamilyException");
4535     } catch(NoSuchColumnFamilyException e) {
4536     }
4537   }
4538 
4539   @Test
4540   public void testAppend() throws Exception {
4541     LOG.info("Starting testAppend");
4542     final byte [] TABLENAME = Bytes.toBytes("testAppend");
4543     HTable t = TEST_UTIL.createTable(TABLENAME, FAMILY);
4544     byte[] v1 = Bytes.toBytes("42");
4545     byte[] v2 = Bytes.toBytes("23");
4546     byte [][] QUALIFIERS = new byte [][] {
4547         Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("c")
4548     };
4549     Append a = new Append(ROW);
4550     a.add(FAMILY, QUALIFIERS[0], v1);
4551     a.add(FAMILY, QUALIFIERS[1], v2);
4552     a.setReturnResults(false);
4553     assertNullResult(t.append(a));
4554 
4555     a = new Append(ROW);
4556     a.add(FAMILY, QUALIFIERS[0], v2);
4557     a.add(FAMILY, QUALIFIERS[1], v1);
4558     a.add(FAMILY, QUALIFIERS[2], v2);
4559     Result r = t.append(a);
4560     assertEquals(0, Bytes.compareTo(Bytes.add(v1,v2), r.getValue(FAMILY, QUALIFIERS[0])));
4561     assertEquals(0, Bytes.compareTo(Bytes.add(v2,v1), r.getValue(FAMILY, QUALIFIERS[1])));
4562     // QUALIFIERS[2] previously not exist, verify both value and timestamp are correct
4563     assertEquals(0, Bytes.compareTo(v2, r.getValue(FAMILY, QUALIFIERS[2])));
4564     assertEquals(r.getColumnLatest(FAMILY, QUALIFIERS[0]).getTimestamp(),
4565         r.getColumnLatest(FAMILY, QUALIFIERS[2]).getTimestamp());
4566   }
4567 
4568   @Test
4569   public void testIncrementWithDeletes() throws Exception {
4570     LOG.info("Starting testIncrementWithDeletes");
4571     final TableName TABLENAME =
4572         TableName.valueOf("testIncrementWithDeletes");
4573     HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
4574     final byte[] COLUMN = Bytes.toBytes("column");
4575 
4576     ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
4577     TEST_UTIL.flush(TABLENAME);
4578 
4579     Delete del = new Delete(ROW);
4580     ht.delete(del);
4581 
4582     ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
4583 
4584     Get get = new Get(ROW);
4585     Result r = ht.get(get);
4586     assertEquals(1, r.size());
4587     assertEquals(5, Bytes.toLong(r.getValue(FAMILY, COLUMN)));
4588   }
4589 
4590   @Test
4591   public void testIncrementingInvalidValue() throws Exception {
4592     LOG.info("Starting testIncrementingInvalidValue");
4593     final byte [] TABLENAME = Bytes.toBytes("testIncrementingInvalidValue");
4594     HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
4595     final byte[] COLUMN = Bytes.toBytes("column");
4596     Put p = new Put(ROW);
4597     // write an integer here (not a Long)
4598     p.add(FAMILY, COLUMN, Bytes.toBytes(5));
4599     ht.put(p);
4600     try {
4601       ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
4602       fail("Should have thrown DoNotRetryIOException");
4603     } catch (DoNotRetryIOException iox) {
4604       // success
4605     }
4606     Increment inc = new Increment(ROW);
4607     inc.addColumn(FAMILY, COLUMN, 5);
4608     try {
4609       ht.increment(inc);
4610       fail("Should have thrown DoNotRetryIOException");
4611     } catch (DoNotRetryIOException iox) {
4612       // success
4613     }
4614   }
4615 
4616   @Test
4617   public void testIncrementInvalidArguments() throws Exception {
4618     LOG.info("Starting testIncrementInvalidArguments");
4619     final byte[] TABLENAME = Bytes.toBytes("testIncrementInvalidArguments");
4620     HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
4621     final byte[] COLUMN = Bytes.toBytes("column");
4622     try {
4623       // try null row
4624       ht.incrementColumnValue(null, FAMILY, COLUMN, 5);
4625       fail("Should have thrown IOException");
4626     } catch (IOException iox) {
4627       // success
4628     }
4629     try {
4630       // try null family
4631       ht.incrementColumnValue(ROW, null, COLUMN, 5);
4632       fail("Should have thrown IOException");
4633     } catch (IOException iox) {
4634       // success
4635     }
4636     try {
4637       // try null qualifier
4638       ht.incrementColumnValue(ROW, FAMILY, null, 5);
4639       fail("Should have thrown IOException");
4640     } catch (IOException iox) {
4641       // success
4642     }
4643     // try null row
4644     try {
4645       Increment incNoRow = new Increment((byte [])null);
4646       incNoRow.addColumn(FAMILY, COLUMN, 5);
4647       fail("Should have thrown IllegalArgumentException");
4648     } catch (IllegalArgumentException iax) {
4649       // success
4650     } catch (NullPointerException npe) {
4651       // success
4652     }
4653     // try null family
4654     try {
4655       Increment incNoFamily = new Increment(ROW);
4656       incNoFamily.addColumn(null, COLUMN, 5);
4657       fail("Should have thrown IllegalArgumentException");
4658     } catch (IllegalArgumentException iax) {
4659       // success
4660     }
4661     // try null qualifier
4662     try {
4663       Increment incNoQualifier = new Increment(ROW);
4664       incNoQualifier.addColumn(FAMILY, null, 5);
4665       fail("Should have thrown IllegalArgumentException");
4666     } catch (IllegalArgumentException iax) {
4667       // success
4668     }
4669   }
4670 
4671   @Test
4672   public void testIncrementOutOfOrder() throws Exception {
4673     LOG.info("Starting testIncrementOutOfOrder");
4674     final byte [] TABLENAME = Bytes.toBytes("testIncrementOutOfOrder");
4675     HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
4676 
4677     byte [][] QUALIFIERS = new byte [][] {
4678       Bytes.toBytes("B"), Bytes.toBytes("A"), Bytes.toBytes("C")
4679     };
4680 
4681     Increment inc = new Increment(ROW);
4682     for (int i=0; i<QUALIFIERS.length; i++) {
4683       inc.addColumn(FAMILY, QUALIFIERS[i], 1);
4684     }
4685     ht.increment(inc);
4686 
4687     // Verify expected results
4688     Result r = ht.get(new Get(ROW));
4689     Cell [] kvs = r.rawCells();
4690     assertEquals(3, kvs.length);
4691     assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[1], 1);
4692     assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[0], 1);
4693     assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 1);
4694 
4695     // Now try multiple columns again
4696     inc = new Increment(ROW);
4697     for (int i=0; i<QUALIFIERS.length; i++) {
4698       inc.addColumn(FAMILY, QUALIFIERS[i], 1);
4699     }
4700     ht.increment(inc);
4701 
4702     // Verify
4703     r = ht.get(new Get(ROW));
4704     kvs = r.rawCells();
4705     assertEquals(3, kvs.length);
4706     assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[1], 2);
4707     assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[0], 2);
4708     assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 2);
4709   }
4710 
4711   @Test
4712   public void testIncrementOnSameColumn() throws Exception {
4713     LOG.info("Starting testIncrementOnSameColumn");
4714     final byte[] TABLENAME = Bytes.toBytes("testIncrementOnSameColumn");
4715     HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
4716 
4717     byte[][] QUALIFIERS =
4718         new byte[][] { Bytes.toBytes("A"), Bytes.toBytes("B"), Bytes.toBytes("C") };
4719 
4720     Increment inc = new Increment(ROW);
4721     for (int i = 0; i < QUALIFIERS.length; i++) {
4722       inc.addColumn(FAMILY, QUALIFIERS[i], 1);
4723       inc.addColumn(FAMILY, QUALIFIERS[i], 1);
4724     }
4725     ht.increment(inc);
4726 
4727     // Verify expected results
4728     Result r = ht.get(new Get(ROW));
4729     Cell[] kvs = r.rawCells();
4730     assertEquals(3, kvs.length);
4731     assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 1);
4732     assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 1);
4733     assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 1);
4734 
4735     // Now try multiple columns again
4736     inc = new Increment(ROW);
4737     for (int i = 0; i < QUALIFIERS.length; i++) {
4738       inc.addColumn(FAMILY, QUALIFIERS[i], 1);
4739       inc.addColumn(FAMILY, QUALIFIERS[i], 1);
4740     }
4741     ht.increment(inc);
4742 
4743     // Verify
4744     r = ht.get(new Get(ROW));
4745     kvs = r.rawCells();
4746     assertEquals(3, kvs.length);
4747     assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 2);
4748     assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 2);
4749     assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 2);
4750     
4751     ht.close();
4752   }
4753 
4754   @Test
4755   public void testIncrement() throws Exception {
4756     LOG.info("Starting testIncrement");
4757     final byte [] TABLENAME = Bytes.toBytes("testIncrement");
4758     HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
4759 
4760     byte [][] ROWS = new byte [][] {
4761         Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"),
4762         Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
4763         Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i")
4764     };
4765     byte [][] QUALIFIERS = new byte [][] {
4766         Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"),
4767         Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
4768         Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i")
4769     };
4770 
4771     // Do some simple single-column increments
4772 
4773     // First with old API
4774     ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[0], 1);
4775     ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[1], 2);
4776     ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[2], 3);
4777     ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[3], 4);
4778 
4779     // Now increment things incremented with old and do some new
4780     Increment inc = new Increment(ROW);
4781     inc.addColumn(FAMILY, QUALIFIERS[1], 1);
4782     inc.addColumn(FAMILY, QUALIFIERS[3], 1);
4783     inc.addColumn(FAMILY, QUALIFIERS[4], 1);
4784     ht.increment(inc);
4785 
4786     // Verify expected results
4787     Result r = ht.get(new Get(ROW));
4788     Cell [] kvs = r.rawCells();
4789     assertEquals(5, kvs.length);
4790     assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 1);
4791     assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 3);
4792     assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 3);
4793     assertIncrementKey(kvs[3], ROW, FAMILY, QUALIFIERS[3], 5);
4794     assertIncrementKey(kvs[4], ROW, FAMILY, QUALIFIERS[4], 1);
4795 
4796     // Now try multiple columns by different amounts
4797     inc = new Increment(ROWS[0]);
4798     for (int i=0;i<QUALIFIERS.length;i++) {
4799       inc.addColumn(FAMILY, QUALIFIERS[i], i+1);
4800     }
4801     ht.increment(inc);
4802     // Verify
4803     r = ht.get(new Get(ROWS[0]));
4804     kvs = r.rawCells();
4805     assertEquals(QUALIFIERS.length, kvs.length);
4806     for (int i=0;i<QUALIFIERS.length;i++) {
4807       assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], i+1);
4808     }
4809 
4810     // Re-increment them
4811     inc = new Increment(ROWS[0]);
4812     for (int i=0;i<QUALIFIERS.length;i++) {
4813       inc.addColumn(FAMILY, QUALIFIERS[i], i+1);
4814     }
4815     ht.increment(inc);
4816     // Verify
4817     r = ht.get(new Get(ROWS[0]));
4818     kvs = r.rawCells();
4819     assertEquals(QUALIFIERS.length, kvs.length);
4820     for (int i=0;i<QUALIFIERS.length;i++) {
4821       assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], 2*(i+1));
4822     }
4823   }
4824 
4825 
4826   @Test
4827   public void testClientPoolRoundRobin() throws IOException {
4828     final byte[] tableName = Bytes.toBytes("testClientPoolRoundRobin");
4829 
4830     int poolSize = 3;
4831     int numVersions = poolSize * 2;
4832     Configuration conf = TEST_UTIL.getConfiguration();
4833     conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "round-robin");
4834     conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
4835 
4836     HTable table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY },
4837         conf, Integer.MAX_VALUE);
4838 
4839     final long ts = EnvironmentEdgeManager.currentTimeMillis();
4840     Get get = new Get(ROW);
4841     get.addColumn(FAMILY, QUALIFIER);
4842     get.setMaxVersions();
4843 
4844     for (int versions = 1; versions <= numVersions; versions++) {
4845       Put put = new Put(ROW);
4846       put.add(FAMILY, QUALIFIER, ts + versions, VALUE);
4847       table.put(put);
4848 
4849       Result result = table.get(get);
4850       NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY)
4851           .get(QUALIFIER);
4852 
4853       assertEquals("The number of versions of '" + FAMILY + ":" + QUALIFIER
4854           + " did not match " + versions, versions, navigableMap.size());
4855       for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4856         assertTrue("The value at time " + entry.getKey()
4857             + " did not match what was put",
4858             Bytes.equals(VALUE, entry.getValue()));
4859       }
4860     }
4861   }
4862 
4863   @Ignore ("Flakey: HBASE-8989") @Test
4864   public void testClientPoolThreadLocal() throws IOException {
4865     final byte[] tableName = Bytes.toBytes("testClientPoolThreadLocal");
4866 
4867     int poolSize = Integer.MAX_VALUE;
4868     int numVersions = 3;
4869     Configuration conf = TEST_UTIL.getConfiguration();
4870     conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "thread-local");
4871     conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
4872 
4873     final HTable table = TEST_UTIL.createTable(tableName,
4874         new byte[][] { FAMILY }, conf, 3);
4875 
4876     final long ts = EnvironmentEdgeManager.currentTimeMillis();
4877     final Get get = new Get(ROW);
4878     get.addColumn(FAMILY, QUALIFIER);
4879     get.setMaxVersions();
4880 
4881     for (int versions = 1; versions <= numVersions; versions++) {
4882       Put put = new Put(ROW);
4883       put.add(FAMILY, QUALIFIER, ts + versions, VALUE);
4884       table.put(put);
4885 
4886       Result result = table.get(get);
4887       NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY)
4888           .get(QUALIFIER);
4889 
4890       assertEquals("The number of versions of '" + FAMILY + ":" + QUALIFIER + " did not match " +
4891         versions + "; " + put.toString() + ", " + get.toString(), versions, navigableMap.size());
4892       for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4893         assertTrue("The value at time " + entry.getKey()
4894             + " did not match what was put",
4895             Bytes.equals(VALUE, entry.getValue()));
4896       }
4897     }
4898 
4899     final Object waitLock = new Object();
4900     ExecutorService executorService = Executors.newFixedThreadPool(numVersions);
4901     final AtomicReference<AssertionError> error = new AtomicReference<AssertionError>(null);
4902     for (int versions = numVersions; versions < numVersions * 2; versions++) {
4903       final int versionsCopy = versions;
4904       executorService.submit(new Callable<Void>() {
4905         @Override
4906         public Void call() {
4907           try {
4908             Put put = new Put(ROW);
4909             put.add(FAMILY, QUALIFIER, ts + versionsCopy, VALUE);
4910             table.put(put);
4911 
4912             Result result = table.get(get);
4913             NavigableMap<Long, byte[]> navigableMap = result.getMap()
4914                 .get(FAMILY).get(QUALIFIER);
4915 
4916             assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":"
4917                 + Bytes.toString(QUALIFIER) + " did not match " + versionsCopy, versionsCopy,
4918                 navigableMap.size());
4919             for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4920               assertTrue("The value at time " + entry.getKey()
4921                   + " did not match what was put",
4922                   Bytes.equals(VALUE, entry.getValue()));
4923             }
4924             synchronized (waitLock) {
4925               waitLock.wait();
4926             }
4927           } catch (Exception e) {
4928           } catch (AssertionError e) {
4929             // the error happens in a thread, it won't fail the test,
4930             // need to pass it to the caller for proper handling.
4931             error.set(e);
4932             LOG.error(e);
4933           }
4934 
4935           return null;
4936         }
4937       });
4938     }
4939     synchronized (waitLock) {
4940       waitLock.notifyAll();
4941     }
4942     executorService.shutdownNow();
4943     assertNull(error.get());
4944   }
4945 
4946   @Test
4947   public void testCheckAndPut() throws IOException {
4948     final byte [] anotherrow = Bytes.toBytes("anotherrow");
4949     final byte [] value2 = Bytes.toBytes("abcd");
4950 
4951     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testCheckAndPut"),
4952       new byte [][] {FAMILY});
4953     Put put1 = new Put(ROW);
4954     put1.add(FAMILY, QUALIFIER, VALUE);
4955 
4956     // row doesn't exist, so using non-null value should be considered "not match".
4957     boolean ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE, put1);
4958     assertEquals(ok, false);
4959 
4960     // row doesn't exist, so using "null" to check for existence should be considered "match".
4961     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put1);
4962     assertEquals(ok, true);
4963 
4964     // row now exists, so using "null" to check for existence should be considered "not match".
4965     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put1);
4966     assertEquals(ok, false);
4967 
4968     Put put2 = new Put(ROW);
4969     put2.add(FAMILY, QUALIFIER, value2);
4970 
4971     // row now exists, use the matching value to check
4972     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE, put2);
4973     assertEquals(ok, true);
4974 
4975     Put put3 = new Put(anotherrow);
4976     put3.add(FAMILY, QUALIFIER, VALUE);
4977 
4978     // try to do CheckAndPut on different rows
4979     try {
4980         ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, value2, put3);
4981         fail("trying to check and modify different rows should have failed.");
4982     } catch(Exception e) {}
4983 
4984   }
4985 
4986   /**
4987   * Test ScanMetrics
4988   * @throws Exception
4989   */
4990   @Test
4991   @SuppressWarnings ("unused")
4992   public void testScanMetrics() throws Exception {
4993     byte [] TABLENAME = Bytes.toBytes("testScanMetrics");
4994 
4995     Configuration conf = TEST_UTIL.getConfiguration();
4996     TEST_UTIL.createTable(TABLENAME, FAMILY);
4997 
4998     // Set up test table:
4999     // Create table:
5000     HTable ht = new HTable(conf, TABLENAME);
5001 
5002     // Create multiple regions for this table
5003     int numOfRegions = TEST_UTIL.createMultiRegions(ht, FAMILY);
5004     // Create 3 rows in the table, with rowkeys starting with "z*" so that
5005     // scan are forced to hit all the regions.
5006     Put put1 = new Put(Bytes.toBytes("z1"));
5007     put1.add(FAMILY, QUALIFIER, VALUE);
5008     Put put2 = new Put(Bytes.toBytes("z2"));
5009     put2.add(FAMILY, QUALIFIER, VALUE);
5010     Put put3 = new Put(Bytes.toBytes("z3"));
5011     put3.add(FAMILY, QUALIFIER, VALUE);
5012     ht.put(Arrays.asList(put1, put2, put3));
5013 
5014     Scan scan1 = new Scan();
5015     int numRecords = 0;
5016     for(Result result : ht.getScanner(scan1)) {
5017       numRecords++;
5018     }
5019     LOG.info("test data has " + numRecords + " records.");
5020 
5021     // by default, scan metrics collection is turned off
5022     assertEquals(null, scan1.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA));
5023 
5024     // turn on scan metrics
5025     Scan scan = new Scan();
5026     scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
5027     scan.setCaching(numRecords+1);
5028     ResultScanner scanner = ht.getScanner(scan);
5029     for (Result result : scanner.next(numRecords - 1)) {
5030     }
5031     scanner.close();
5032     // closing the scanner will set the metrics.
5033     assertNotNull(scan.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA));
5034 
5035     // set caching to 1, becasue metrics are collected in each roundtrip only
5036     scan = new Scan();
5037     scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
5038     scan.setCaching(1);
5039     scanner = ht.getScanner(scan);
5040     // per HBASE-5717, this should still collect even if you don't run all the way to
5041     // the end of the scanner. So this is asking for 2 of the 3 rows we inserted.
5042     for (Result result : scanner.next(numRecords - 1)) {
5043     }
5044     scanner.close();
5045 
5046     ScanMetrics scanMetrics = getScanMetrics(scan);
5047     assertEquals("Did not access all the regions in the table", numOfRegions,
5048         scanMetrics.countOfRegions.get());
5049 
5050     // check byte counters
5051     scan = new Scan();
5052     scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
5053     scan.setCaching(1);
5054     scanner = ht.getScanner(scan);
5055     int numBytes = 0;
5056     for (Result result : scanner.next(1)) {
5057       for (Cell cell: result.listCells()) {
5058         numBytes += KeyValueUtil.ensureKeyValue(cell).getLength();
5059       }
5060     }
5061     scanner.close();
5062     scanMetrics = getScanMetrics(scan);
5063     assertEquals("Did not count the result bytes", numBytes,
5064       scanMetrics.countOfBytesInResults.get());
5065 
5066     // check byte counters on a small scan
5067     scan = new Scan();
5068     scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
5069     scan.setCaching(1);
5070     scan.setSmall(true);
5071     scanner = ht.getScanner(scan);
5072     numBytes = 0;
5073     for (Result result : scanner.next(1)) {
5074       for (Cell cell: result.listCells()) {
5075         numBytes += KeyValueUtil.ensureKeyValue(cell).getLength();
5076       }
5077     }
5078     scanner.close();
5079     scanMetrics = getScanMetrics(scan);
5080     assertEquals("Did not count the result bytes", numBytes,
5081       scanMetrics.countOfBytesInResults.get());
5082 
5083     // now, test that the metrics are still collected even if you don't call close, but do
5084     // run past the end of all the records
5085     Scan scanWithoutClose = new Scan();
5086     scanWithoutClose.setCaching(1);
5087     scanWithoutClose.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
5088     ResultScanner scannerWithoutClose = ht.getScanner(scanWithoutClose);
5089     for (Result result : scannerWithoutClose.next(numRecords + 1)) {
5090     }
5091     ScanMetrics scanMetricsWithoutClose = getScanMetrics(scanWithoutClose);
5092     assertEquals("Did not access all the regions in the table", numOfRegions,
5093         scanMetricsWithoutClose.countOfRegions.get());
5094 
5095     // finally, test that the metrics are collected correctly if you both run past all the records,
5096     // AND close the scanner
5097     Scan scanWithClose = new Scan();
5098     // make sure we can set caching up to the number of a scanned values
5099     scanWithClose.setCaching(numRecords);
5100     scanWithClose.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
5101     ResultScanner scannerWithClose = ht.getScanner(scanWithClose);
5102     for (Result result : scannerWithClose.next(numRecords + 1)) {
5103     }
5104     scannerWithClose.close();
5105     ScanMetrics scanMetricsWithClose = getScanMetrics(scanWithClose);
5106     assertEquals("Did not access all the regions in the table", numOfRegions,
5107         scanMetricsWithClose.countOfRegions.get());
5108   }
5109 
5110   private ScanMetrics getScanMetrics(Scan scan) throws Exception {
5111     byte[] serializedMetrics = scan.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA);
5112     assertTrue("Serialized metrics were not found.", serializedMetrics != null);
5113 
5114 
5115     ScanMetrics scanMetrics = ProtobufUtil.toScanMetrics(serializedMetrics);
5116 
5117     return scanMetrics;
5118   }
5119 
5120   /**
5121    * Tests that cache on write works all the way up from the client-side.
5122    *
5123    * Performs inserts, flushes, and compactions, verifying changes in the block
5124    * cache along the way.
5125    *
5126    * @throws Exception
5127    */
5128   @Test
5129   public void testCacheOnWriteEvictOnClose() throws Exception {
5130     byte [] tableName = Bytes.toBytes("testCOWEOCfromClient");
5131     byte [] data = Bytes.toBytes("data");
5132     HTable table = TEST_UTIL.createTable(tableName, new byte [][] {FAMILY});
5133     // get the block cache and region
5134     String regionName = table.getRegionLocations().firstKey().getEncodedName();
5135     HRegion region = TEST_UTIL.getRSForFirstRegionInTable(
5136         tableName).getFromOnlineRegions(regionName);
5137     Store store = region.getStores().values().iterator().next();
5138     CacheConfig cacheConf = store.getCacheConfig();
5139     cacheConf.setCacheDataOnWrite(true);
5140     cacheConf.setEvictOnClose(true);
5141     BlockCache cache = cacheConf.getBlockCache();
5142 
5143     // establish baseline stats
5144     long startBlockCount = cache.getBlockCount();
5145     long startBlockHits = cache.getStats().getHitCount();
5146     long startBlockMiss = cache.getStats().getMissCount();
5147 
5148     // wait till baseline is stable, (minimal 500 ms)
5149     for (int i = 0; i < 5; i++) {
5150       Thread.sleep(100);
5151       if (startBlockCount != cache.getBlockCount()
5152           || startBlockHits != cache.getStats().getHitCount()
5153           || startBlockMiss != cache.getStats().getMissCount()) {
5154         startBlockCount = cache.getBlockCount();
5155         startBlockHits = cache.getStats().getHitCount();
5156         startBlockMiss = cache.getStats().getMissCount();
5157         i = -1;
5158       }
5159     }
5160 
5161     // insert data
5162     Put put = new Put(ROW);
5163     put.add(FAMILY, QUALIFIER, data);
5164     table.put(put);
5165     assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
5166     // data was in memstore so don't expect any changes
5167     assertEquals(startBlockCount, cache.getBlockCount());
5168     assertEquals(startBlockHits, cache.getStats().getHitCount());
5169     assertEquals(startBlockMiss, cache.getStats().getMissCount());
5170     // flush the data
5171     System.out.println("Flushing cache");
5172     region.flushcache();
5173     // expect one more block in cache, no change in hits/misses
5174     long expectedBlockCount = startBlockCount + 1;
5175     long expectedBlockHits = startBlockHits;
5176     long expectedBlockMiss = startBlockMiss;
5177     assertEquals(expectedBlockCount, cache.getBlockCount());
5178     assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5179     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5180     // read the data and expect same blocks, one new hit, no misses
5181     assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
5182     assertEquals(expectedBlockCount, cache.getBlockCount());
5183     assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
5184     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5185     // insert a second column, read the row, no new blocks, one new hit
5186     byte [] QUALIFIER2 = Bytes.add(QUALIFIER, QUALIFIER);
5187     byte [] data2 = Bytes.add(data, data);
5188     put = new Put(ROW);
5189     put.add(FAMILY, QUALIFIER2, data2);
5190     table.put(put);
5191     Result r = table.get(new Get(ROW));
5192     assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
5193     assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
5194     assertEquals(expectedBlockCount, cache.getBlockCount());
5195     assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
5196     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5197     // flush, one new block
5198     System.out.println("Flushing cache");
5199     region.flushcache();
5200     assertEquals(++expectedBlockCount, cache.getBlockCount());
5201     assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5202     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5203     // compact, net minus two blocks, two hits, no misses
5204     System.out.println("Compacting");
5205     assertEquals(2, store.getStorefilesCount());
5206     store.triggerMajorCompaction();
5207     region.compactStores();
5208     waitForStoreFileCount(store, 1, 10000); // wait 10 seconds max
5209     assertEquals(1, store.getStorefilesCount());
5210     expectedBlockCount -= 2; // evicted two blocks, cached none
5211     assertEquals(expectedBlockCount, cache.getBlockCount());
5212     expectedBlockHits += 2;
5213     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5214     assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5215     // read the row, this should be a cache miss because we don't cache data
5216     // blocks on compaction
5217     r = table.get(new Get(ROW));
5218     assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
5219     assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
5220     expectedBlockCount += 1; // cached one data block
5221     assertEquals(expectedBlockCount, cache.getBlockCount());
5222     assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5223     assertEquals(++expectedBlockMiss, cache.getStats().getMissCount());
5224   }
5225 
5226   private void waitForStoreFileCount(Store store, int count, int timeout)
5227   throws InterruptedException {
5228     long start = System.currentTimeMillis();
5229     while (start + timeout > System.currentTimeMillis() &&
5230         store.getStorefilesCount() != count) {
5231       Thread.sleep(100);
5232     }
5233     System.out.println("start=" + start + ", now=" +
5234         System.currentTimeMillis() + ", cur=" + store.getStorefilesCount());
5235     assertEquals(count, store.getStorefilesCount());
5236   }
5237 
5238   @Test
5239   /**
5240    * Tests the non cached version of getRegionLocation by moving a region.
5241    */
5242   public void testNonCachedGetRegionLocation() throws Exception {
5243     // Test Initialization.
5244     String tableName = "testNonCachedGetRegionLocation";
5245     byte [] TABLE = Bytes.toBytes(tableName);
5246     byte [] family1 = Bytes.toBytes("f1");
5247     byte [] family2 = Bytes.toBytes("f2");
5248     HTable table = TEST_UTIL.createTable(TABLE, new byte[][] {family1, family2}, 10);
5249     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
5250     Map <HRegionInfo, ServerName> regionsMap = table.getRegionLocations();
5251     assertEquals(1, regionsMap.size());
5252     HRegionInfo regionInfo = regionsMap.keySet().iterator().next();
5253     ServerName addrBefore = regionsMap.get(regionInfo);
5254     // Verify region location before move.
5255     HRegionLocation addrCache = table.getRegionLocation(regionInfo.getStartKey(), false);
5256     HRegionLocation addrNoCache = table.getRegionLocation(regionInfo.getStartKey(),  true);
5257 
5258     assertEquals(addrBefore.getPort(), addrCache.getPort());
5259     assertEquals(addrBefore.getPort(), addrNoCache.getPort());
5260 
5261     ServerName addrAfter = null;
5262     // Now move the region to a different server.
5263     for (int i = 0; i < SLAVES; i++) {
5264       HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(i);
5265       ServerName addr = regionServer.getServerName();
5266       if (addr.getPort() != addrBefore.getPort()) {
5267         admin.move(regionInfo.getEncodedNameAsBytes(),
5268             Bytes.toBytes(addr.toString()));
5269         // Wait for the region to move.
5270         Thread.sleep(5000);
5271         addrAfter = addr;
5272         break;
5273       }
5274     }
5275 
5276     // Verify the region was moved.
5277     addrCache = table.getRegionLocation(regionInfo.getStartKey(), false);
5278     addrNoCache = table.getRegionLocation(regionInfo.getStartKey(), true);
5279     assertNotNull(addrAfter);
5280     assertTrue(addrAfter.getPort() != addrCache.getPort());
5281     assertEquals(addrAfter.getPort(), addrNoCache.getPort());
5282   }
5283 
5284   @Test
5285   /**
5286    * Tests getRegionsInRange by creating some regions over which a range of
5287    * keys spans; then changing the key range.
5288    */
5289   public void testGetRegionsInRange() throws Exception {
5290     // Test Initialization.
5291     byte [] startKey = Bytes.toBytes("ddc");
5292     byte [] endKey = Bytes.toBytes("mmm");
5293     byte [] TABLE = Bytes.toBytes("testGetRegionsInRange");
5294     HTable table = TEST_UTIL.createTable(TABLE, new byte[][] {FAMILY}, 10);
5295     int numOfRegions = TEST_UTIL.createMultiRegions(table, FAMILY);
5296     assertEquals(25, numOfRegions);
5297 
5298     // Get the regions in this range
5299     List<HRegionLocation> regionsList = table.getRegionsInRange(startKey,
5300       endKey);
5301     assertEquals(10, regionsList.size());
5302 
5303     // Change the start key
5304     startKey = Bytes.toBytes("fff");
5305     regionsList = table.getRegionsInRange(startKey, endKey);
5306     assertEquals(7, regionsList.size());
5307 
5308     // Change the end key
5309     endKey = Bytes.toBytes("nnn");
5310     regionsList = table.getRegionsInRange(startKey, endKey);
5311     assertEquals(8, regionsList.size());
5312 
5313     // Empty start key
5314     regionsList = table.getRegionsInRange(HConstants.EMPTY_START_ROW, endKey);
5315     assertEquals(13, regionsList.size());
5316 
5317     // Empty end key
5318     regionsList = table.getRegionsInRange(startKey, HConstants.EMPTY_END_ROW);
5319     assertEquals(20, regionsList.size());
5320 
5321     // Both start and end keys empty
5322     regionsList = table.getRegionsInRange(HConstants.EMPTY_START_ROW,
5323       HConstants.EMPTY_END_ROW);
5324     assertEquals(25, regionsList.size());
5325 
5326     // Change the end key to somewhere in the last block
5327     endKey = Bytes.toBytes("yyz");
5328     regionsList = table.getRegionsInRange(startKey, endKey);
5329     assertEquals(20, regionsList.size());
5330 
5331     // Change the start key to somewhere in the first block
5332     startKey = Bytes.toBytes("aac");
5333     regionsList = table.getRegionsInRange(startKey, endKey);
5334     assertEquals(25, regionsList.size());
5335 
5336     // Make start and end key the same
5337     startKey = endKey = Bytes.toBytes("ccc");
5338     regionsList = table.getRegionsInRange(startKey, endKey);
5339     assertEquals(1, regionsList.size());
5340   }
5341 
5342   @Test
5343   public void testJira6912() throws Exception {
5344     byte [] TABLE = Bytes.toBytes("testJira6912");
5345     HTable foo = TEST_UTIL.createTable(TABLE, new byte[][] {FAMILY}, 10);
5346 
5347     List<Put> puts = new ArrayList<Put>();
5348     for (int i=0;i !=100; i++){
5349       Put put = new Put(Bytes.toBytes(i));
5350       put.add(FAMILY, FAMILY, Bytes.toBytes(i));
5351       puts.add(put);
5352     }
5353     foo.put(puts);
5354     // If i comment this out it works
5355     TEST_UTIL.flush();
5356 
5357     Scan scan = new Scan();
5358     scan.setStartRow(Bytes.toBytes(1));
5359     scan.setStopRow(Bytes.toBytes(3));
5360     scan.addColumn(FAMILY, FAMILY);
5361     scan.setFilter(new RowFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator(Bytes.toBytes(1))));
5362 
5363     ResultScanner scanner = foo.getScanner(scan);
5364     Result[] bar = scanner.next(100);
5365     assertEquals(1, bar.length);
5366   }
5367 
5368   @Test
5369   public void testScan_NullQualifier() throws IOException {
5370     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testScan_NullQualifier"), FAMILY);
5371     Put put = new Put(ROW);
5372     put.add(FAMILY, QUALIFIER, VALUE);
5373     table.put(put);
5374 
5375     put = new Put(ROW);
5376     put.add(FAMILY, null, VALUE);
5377     table.put(put);
5378     LOG.info("Row put");
5379 
5380     Scan scan = new Scan();
5381     scan.addColumn(FAMILY, null);
5382 
5383     ResultScanner scanner = table.getScanner(scan);
5384     Result[] bar = scanner.next(100);
5385     assertEquals(1, bar.length);
5386     assertEquals(1, bar[0].size());
5387 
5388     scan = new Scan();
5389     scan.addFamily(FAMILY);
5390 
5391     scanner = table.getScanner(scan);
5392     bar = scanner.next(100);
5393     assertEquals(1, bar.length);
5394     assertEquals(2, bar[0].size());
5395   }
5396 
5397   @Test
5398   public void testNegativeTimestamp() throws IOException {
5399     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testNegativeTimestamp"), FAMILY);
5400 
5401     try {
5402       Put put = new Put(ROW, -1);
5403       put.add(FAMILY, QUALIFIER, VALUE);
5404       table.put(put);
5405       fail("Negative timestamps should not have been allowed");
5406     } catch (IllegalArgumentException ex) {
5407       assertTrue(ex.getMessage().contains("negative"));
5408     }
5409 
5410     try {
5411       Put put = new Put(ROW);
5412       put.add(FAMILY, QUALIFIER, -1, VALUE);
5413       table.put(put);
5414       fail("Negative timestamps should not have been allowed");
5415     } catch (IllegalArgumentException ex) {
5416       assertTrue(ex.getMessage().contains("negative"));
5417     }
5418 
5419     try {
5420       Delete delete = new Delete(ROW, -1);
5421       table.delete(delete);
5422       fail("Negative timestamps should not have been allowed");
5423     } catch (IllegalArgumentException ex) {
5424       assertTrue(ex.getMessage().contains("negative"));
5425     }
5426 
5427     try {
5428       Delete delete = new Delete(ROW);
5429       delete.deleteFamily(FAMILY, -1);
5430       table.delete(delete);
5431       fail("Negative timestamps should not have been allowed");
5432     } catch (IllegalArgumentException ex) {
5433       assertTrue(ex.getMessage().contains("negative"));
5434     }
5435 
5436     try {
5437       Scan scan = new Scan();
5438       scan.setTimeRange(-1, 1);
5439       table.getScanner(scan);
5440       fail("Negative timestamps should not have been allowed");
5441     } catch (IllegalArgumentException ex) {
5442       assertTrue(ex.getMessage().contains("negative"));
5443     }
5444 
5445     // KeyValue should allow negative timestamps for backwards compat. Otherwise, if the user
5446     // already has negative timestamps in cluster data, HBase won't be able to handle that
5447     try {
5448       new KeyValue(Bytes.toBytes(42), Bytes.toBytes(42), Bytes.toBytes(42), -1, Bytes.toBytes(42));
5449     } catch (IllegalArgumentException ex) {
5450       fail("KeyValue SHOULD allow negative timestamps");
5451     }
5452 
5453     table.close();
5454   }
5455 
5456   @Test
5457   public void testIllegalTableDescriptor() throws Exception {
5458     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testIllegalTableDescriptor"));
5459     HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
5460 
5461     // create table with 0 families
5462     checkTableIsIllegal(htd);
5463     htd.addFamily(hcd);
5464     checkTableIsLegal(htd);
5465 
5466     htd.setMaxFileSize(1024); // 1K
5467     checkTableIsIllegal(htd);
5468     htd.setMaxFileSize(0);
5469     checkTableIsIllegal(htd);
5470     htd.setMaxFileSize(1024 * 1024 * 1024); // 1G
5471     checkTableIsLegal(htd);
5472 
5473     htd.setMemStoreFlushSize(1024);
5474     checkTableIsIllegal(htd);
5475     htd.setMemStoreFlushSize(0);
5476     checkTableIsIllegal(htd);
5477     htd.setMemStoreFlushSize(128 * 1024 * 1024); // 128M
5478     checkTableIsLegal(htd);
5479 
5480     htd.setRegionSplitPolicyClassName("nonexisting.foo.class");
5481     checkTableIsIllegal(htd);
5482     htd.setRegionSplitPolicyClassName(null);
5483     checkTableIsLegal(htd);
5484 
5485     hcd.setBlocksize(0);
5486     checkTableIsIllegal(htd);
5487     hcd.setBlocksize(1024 * 1024 * 128); // 128M
5488     checkTableIsIllegal(htd);
5489     hcd.setBlocksize(1024);
5490     checkTableIsLegal(htd);
5491 
5492     hcd.setTimeToLive(0);
5493     checkTableIsIllegal(htd);
5494     hcd.setTimeToLive(-1);
5495     checkTableIsIllegal(htd);
5496     hcd.setTimeToLive(1);
5497     checkTableIsLegal(htd);
5498 
5499     hcd.setMinVersions(-1);
5500     checkTableIsIllegal(htd);
5501     hcd.setMinVersions(3);
5502     try {
5503       hcd.setMaxVersions(2);
5504       fail();
5505     } catch (IllegalArgumentException ex) {
5506       // expected
5507       hcd.setMaxVersions(10);
5508     }
5509     checkTableIsLegal(htd);
5510 
5511     // HBASE-13776 Setting illegal versions for HColumnDescriptor
5512     //  does not throw IllegalArgumentException
5513     // finally, minVersions must be less than or equal to maxVersions
5514     hcd.setMaxVersions(4);
5515     hcd.setMinVersions(5);
5516     checkTableIsIllegal(htd);
5517     hcd.setMinVersions(3);
5518 
5519     hcd.setScope(-1);
5520     checkTableIsIllegal(htd);
5521     hcd.setScope(0);
5522     checkTableIsLegal(htd);
5523 
5524     try {
5525       hcd.setDFSReplication((short) -1);
5526       fail("Illegal value for setDFSReplication did not throw");
5527     } catch (IllegalArgumentException e) {
5528       // pass
5529     }
5530     // set an illegal DFS replication value by hand
5531     hcd.setValue(HColumnDescriptor.DFS_REPLICATION, "-1");
5532     checkTableIsIllegal(htd);
5533     try {
5534       hcd.setDFSReplication((short) -1);
5535       fail("Should throw exception if an illegal value is explicitly being set");
5536     } catch (IllegalArgumentException e) {
5537       // pass
5538     }
5539 
5540     // check the conf settings to disable sanity checks
5541     htd.setMemStoreFlushSize(0);
5542 
5543     // Check that logs warn on invalid table but allow it.
5544     ListAppender listAppender = new ListAppender();
5545     Logger log = Logger.getLogger(HMaster.class);
5546     log.addAppender(listAppender);
5547     log.setLevel(Level.WARN);
5548 
5549     htd.setConfiguration("hbase.table.sanity.checks", Boolean.FALSE.toString());
5550     checkTableIsLegal(htd);
5551 
5552     assertFalse(listAppender.getMessages().isEmpty());
5553     assertTrue(listAppender.getMessages().get(0).startsWith("MEMSTORE_FLUSHSIZE for table "
5554         + "descriptor or \"hbase.hregion.memstore.flush.size\" (0) is too small, which might "
5555         + "cause very frequent flushing."));
5556 
5557     log.removeAppender(listAppender);
5558   }
5559 
5560   private static class ListAppender extends AppenderSkeleton {
5561     private final List<String> messages = new ArrayList<String>();
5562 
5563     @Override
5564     protected void append(LoggingEvent event) {
5565       messages.add(event.getMessage().toString());
5566     }
5567 
5568     @Override
5569     public void close() {
5570     }
5571 
5572     @Override
5573     public boolean requiresLayout() {
5574       return false;
5575     }
5576 
5577     public List<String> getMessages() {
5578       return messages;
5579     }
5580   }
5581 
5582   private void checkTableIsLegal(HTableDescriptor htd) throws IOException {
5583     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
5584     admin.createTable(htd);
5585     assertTrue(admin.tableExists(htd.getTableName()));
5586     admin.disableTable(htd.getTableName());
5587     admin.deleteTable(htd.getTableName());
5588   }
5589 
5590   private void checkTableIsIllegal(HTableDescriptor htd) throws IOException {
5591     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
5592     try {
5593       admin.createTable(htd);
5594       fail();
5595     } catch(Exception ex) {
5596       // should throw ex
5597     }
5598     assertFalse(admin.tableExists(htd.getTableName()));
5599   }
5600 
5601   @Test
5602   public void testRawScanRespectsVersions() throws Exception {
5603     byte[] TABLE = Bytes.toBytes("testRawScan");
5604     HTable table = TEST_UTIL.createTable(TABLE, new byte[][] { FAMILY });
5605     byte[] row = Bytes.toBytes("row");
5606 
5607     // put the same row 4 times, with different values
5608     Put p = new Put(row);
5609     p.add(FAMILY, QUALIFIER, 10, VALUE);
5610     table.put(p);
5611     table.flushCommits();
5612 
5613     p = new Put(row);
5614     p.add(FAMILY, QUALIFIER, 11, ArrayUtils.add(VALUE, (byte) 2));
5615     table.put(p);
5616     table.flushCommits();
5617 
5618     p = new Put(row);
5619     p.add(FAMILY, QUALIFIER, 12, ArrayUtils.add(VALUE, (byte) 3));
5620     table.put(p);
5621     table.flushCommits();
5622 
5623     p = new Put(row);
5624     p.add(FAMILY, QUALIFIER, 13, ArrayUtils.add(VALUE, (byte) 4));
5625     table.put(p);
5626     table.flushCommits();
5627 
5628     int versions = 4;
5629     Scan s = new Scan(row);
5630     // get all the possible versions
5631     s.setMaxVersions();
5632     s.setRaw(true);
5633 
5634     ResultScanner scanner = table.getScanner(s);
5635     int count = 0;
5636     for (Result r : scanner) {
5637       assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
5638       count++;
5639     }
5640     assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
5641       count);
5642     scanner.close();
5643 
5644     // then if we decrease the number of versions, but keep the scan raw, we should see exactly that
5645     // number of versions
5646     versions = 2;
5647     s.setMaxVersions(versions);
5648     scanner = table.getScanner(s);
5649     count = 0;
5650     for (Result r : scanner) {
5651       assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
5652       count++;
5653     }
5654     assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
5655       count);
5656     scanner.close();
5657 
5658     // finally, if we turn off raw scanning, but max out the number of versions, we should go back
5659     // to seeing just three
5660     versions = 3;
5661     s.setMaxVersions(versions);
5662     scanner = table.getScanner(s);
5663     count = 0;
5664     for (Result r : scanner) {
5665       assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
5666       count++;
5667     }
5668     assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
5669       count);
5670     scanner.close();
5671 
5672     table.close();
5673     TEST_UTIL.deleteTable(TABLE);
5674   }
5675 
5676   @Test
5677   public void testSmallScan() throws Exception {
5678     // Test Initialization.
5679     byte[] TABLE = Bytes.toBytes("testSmallScan");
5680     HTable table = TEST_UTIL.createTable(TABLE, FAMILY);
5681 
5682     // Insert one row each region
5683     int insertNum = 10;
5684     for (int i = 0; i < 10; i++) {
5685       Put put = new Put(Bytes.toBytes("row" + String.format("%03d", i)));
5686       put.add(FAMILY, QUALIFIER, VALUE);
5687       table.put(put);
5688     }
5689 
5690     // nomal scan
5691     ResultScanner scanner = table.getScanner(new Scan());
5692     int count = 0;
5693     for (Result r : scanner) {
5694       assertTrue(!r.isEmpty());
5695       count++;
5696     }
5697     assertEquals(insertNum, count);
5698 
5699     // small scan
5700     Scan scan = new Scan(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
5701     scan.setSmall(true);
5702     scan.setCaching(2);
5703     scanner = table.getScanner(scan);
5704     count = 0;
5705     for (Result r : scanner) {
5706       assertTrue(!r.isEmpty());
5707       count++;
5708     }
5709     assertEquals(insertNum, count);
5710 
5711   }
5712 
5713   @Test
5714   public void testSuperSimpleWithReverseScan() throws Exception {
5715     byte[] TABLE = Bytes.toBytes("testSuperSimpleWithReverseScan");
5716     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
5717     Put put = new Put(Bytes.toBytes("0-b11111-0000000000000000000"));
5718     put.add(FAMILY, QUALIFIER, VALUE);
5719     ht.put(put);
5720     put = new Put(Bytes.toBytes("0-b11111-0000000000000000002"));
5721     put.add(FAMILY, QUALIFIER, VALUE);
5722     ht.put(put);
5723     put = new Put(Bytes.toBytes("0-b11111-0000000000000000004"));
5724     put.add(FAMILY, QUALIFIER, VALUE);
5725     ht.put(put);
5726     put = new Put(Bytes.toBytes("0-b11111-0000000000000000006"));
5727     put.add(FAMILY, QUALIFIER, VALUE);
5728     ht.put(put);
5729     put = new Put(Bytes.toBytes("0-b11111-0000000000000000008"));
5730     put.add(FAMILY, QUALIFIER, VALUE);
5731     ht.put(put);
5732     put = new Put(Bytes.toBytes("0-b22222-0000000000000000001"));
5733     put.add(FAMILY, QUALIFIER, VALUE);
5734     ht.put(put);
5735     put = new Put(Bytes.toBytes("0-b22222-0000000000000000003"));
5736     put.add(FAMILY, QUALIFIER, VALUE);
5737     ht.put(put);
5738     put = new Put(Bytes.toBytes("0-b22222-0000000000000000005"));
5739     put.add(FAMILY, QUALIFIER, VALUE);
5740     ht.put(put);
5741     put = new Put(Bytes.toBytes("0-b22222-0000000000000000007"));
5742     put.add(FAMILY, QUALIFIER, VALUE);
5743     ht.put(put);
5744     put = new Put(Bytes.toBytes("0-b22222-0000000000000000009"));
5745     put.add(FAMILY, QUALIFIER, VALUE);
5746     ht.put(put);
5747     ht.flushCommits();
5748     Scan scan = new Scan(Bytes.toBytes("0-b11111-9223372036854775807"),
5749         Bytes.toBytes("0-b11111-0000000000000000000"));
5750     scan.setReversed(true);
5751     ResultScanner scanner = ht.getScanner(scan);
5752     Result result = scanner.next();
5753     assertTrue(Bytes.equals(result.getRow(),
5754         Bytes.toBytes("0-b11111-0000000000000000008")));
5755     scanner.close();
5756     ht.close();
5757   }
5758 
5759   @Test
5760   public void testFiltersWithReverseScan() throws Exception {
5761     byte[] TABLE = Bytes.toBytes("testFiltersWithReverseScan");
5762     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
5763     byte[][] ROWS = makeN(ROW, 10);
5764     byte[][] QUALIFIERS = { Bytes.toBytes("col0-<d2v1>-<d3v2>"),
5765         Bytes.toBytes("col1-<d2v1>-<d3v2>"),
5766         Bytes.toBytes("col2-<d2v1>-<d3v2>"),
5767         Bytes.toBytes("col3-<d2v1>-<d3v2>"),
5768         Bytes.toBytes("col4-<d2v1>-<d3v2>"),
5769         Bytes.toBytes("col5-<d2v1>-<d3v2>"),
5770         Bytes.toBytes("col6-<d2v1>-<d3v2>"),
5771         Bytes.toBytes("col7-<d2v1>-<d3v2>"),
5772         Bytes.toBytes("col8-<d2v1>-<d3v2>"),
5773         Bytes.toBytes("col9-<d2v1>-<d3v2>") };
5774     for (int i = 0; i < 10; i++) {
5775       Put put = new Put(ROWS[i]);
5776       put.add(FAMILY, QUALIFIERS[i], VALUE);
5777       ht.put(put);
5778     }
5779     Scan scan = new Scan();
5780     scan.setReversed(true);
5781     scan.addFamily(FAMILY);
5782     Filter filter = new QualifierFilter(CompareOp.EQUAL,
5783         new RegexStringComparator("col[1-5]"));
5784     scan.setFilter(filter);
5785     ResultScanner scanner = ht.getScanner(scan);
5786     int expectedIndex = 5;
5787     for (Result result : scanner) {
5788       assertEquals(result.size(), 1);
5789       assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[expectedIndex]));
5790       assertTrue(Bytes.equals(result.raw()[0].getQualifier(),
5791           QUALIFIERS[expectedIndex]));
5792       expectedIndex--;
5793     }
5794     assertEquals(expectedIndex, 0);
5795     scanner.close();
5796     ht.close();
5797   }
5798 
5799   @Test
5800   public void testKeyOnlyFilterWithReverseScan() throws Exception {
5801     byte[] TABLE = Bytes.toBytes("testKeyOnlyFilterWithReverseScan");
5802     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
5803     byte[][] ROWS = makeN(ROW, 10);
5804     byte[][] QUALIFIERS = { Bytes.toBytes("col0-<d2v1>-<d3v2>"),
5805         Bytes.toBytes("col1-<d2v1>-<d3v2>"),
5806         Bytes.toBytes("col2-<d2v1>-<d3v2>"),
5807         Bytes.toBytes("col3-<d2v1>-<d3v2>"),
5808         Bytes.toBytes("col4-<d2v1>-<d3v2>"),
5809         Bytes.toBytes("col5-<d2v1>-<d3v2>"),
5810         Bytes.toBytes("col6-<d2v1>-<d3v2>"),
5811         Bytes.toBytes("col7-<d2v1>-<d3v2>"),
5812         Bytes.toBytes("col8-<d2v1>-<d3v2>"),
5813         Bytes.toBytes("col9-<d2v1>-<d3v2>") };
5814     for (int i = 0; i < 10; i++) {
5815       Put put = new Put(ROWS[i]);
5816       put.add(FAMILY, QUALIFIERS[i], VALUE);
5817       ht.put(put);
5818     }
5819     Scan scan = new Scan();
5820     scan.setReversed(true);
5821     scan.addFamily(FAMILY);
5822     Filter filter = new KeyOnlyFilter(true);
5823     scan.setFilter(filter);
5824     ResultScanner scanner = ht.getScanner(scan);
5825     int count = 0;
5826     for (Result result : ht.getScanner(scan)) {
5827       assertEquals(result.size(), 1);
5828       assertEquals(result.raw()[0].getValueLength(), Bytes.SIZEOF_INT);
5829       assertEquals(Bytes.toInt(result.raw()[0].getValue()), VALUE.length);
5830       count++;
5831     }
5832     assertEquals(count, 10);
5833     scanner.close();
5834     ht.close();
5835   }
5836 
5837   /**
5838    * Test simple table and non-existent row cases.
5839    */
5840   @Test
5841   public void testSimpleMissingWithReverseScan() throws Exception {
5842     byte[] TABLE = Bytes.toBytes("testSimpleMissingWithReverseScan");
5843     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
5844     byte[][] ROWS = makeN(ROW, 4);
5845 
5846     // Try to get a row on an empty table
5847     Scan scan = new Scan();
5848     scan.setReversed(true);
5849     Result result = getSingleScanResult(ht, scan);
5850     assertNullResult(result);
5851 
5852     scan = new Scan(ROWS[0]);
5853     scan.setReversed(true);
5854     result = getSingleScanResult(ht, scan);
5855     assertNullResult(result);
5856 
5857     scan = new Scan(ROWS[0], ROWS[1]);
5858     scan.setReversed(true);
5859     result = getSingleScanResult(ht, scan);
5860     assertNullResult(result);
5861 
5862     scan = new Scan();
5863     scan.setReversed(true);
5864     scan.addFamily(FAMILY);
5865     result = getSingleScanResult(ht, scan);
5866     assertNullResult(result);
5867 
5868     scan = new Scan();
5869     scan.setReversed(true);
5870     scan.addColumn(FAMILY, QUALIFIER);
5871     result = getSingleScanResult(ht, scan);
5872     assertNullResult(result);
5873 
5874     // Insert a row
5875 
5876     Put put = new Put(ROWS[2]);
5877     put.add(FAMILY, QUALIFIER, VALUE);
5878     ht.put(put);
5879 
5880     // Make sure we can scan the row
5881     scan = new Scan();
5882     scan.setReversed(true);
5883     result = getSingleScanResult(ht, scan);
5884     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
5885 
5886     scan = new Scan(ROWS[3], ROWS[0]);
5887     scan.setReversed(true);
5888     result = getSingleScanResult(ht, scan);
5889     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
5890 
5891     scan = new Scan(ROWS[2], ROWS[1]);
5892     scan.setReversed(true);
5893     result = getSingleScanResult(ht, scan);
5894     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
5895 
5896     // Try to scan empty rows around it
5897     // Introduced MemStore#shouldSeekForReverseScan to fix the following
5898     scan = new Scan(ROWS[1]);
5899     scan.setReversed(true);
5900     result = getSingleScanResult(ht, scan);
5901     assertNullResult(result);
5902     ht.close();
5903   }
5904 
5905   @Test
5906   public void testNullWithReverseScan() throws Exception {
5907     byte[] TABLE = Bytes.toBytes("testNullWithReverseScan");
5908     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
5909     // Null qualifier (should work)
5910     Put put = new Put(ROW);
5911     put.add(FAMILY, null, VALUE);
5912     ht.put(put);
5913     scanTestNull(ht, ROW, FAMILY, VALUE, true);
5914     Delete delete = new Delete(ROW);
5915     delete.deleteColumns(FAMILY, null);
5916     ht.delete(delete);
5917     // Use a new table
5918     byte[] TABLE2 = Bytes.toBytes("testNull2WithReverseScan");
5919     ht = TEST_UTIL.createTable(TABLE2, FAMILY);
5920     // Empty qualifier, byte[0] instead of null (should work)
5921     put = new Put(ROW);
5922     put.add(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE);
5923     ht.put(put);
5924     scanTestNull(ht, ROW, FAMILY, VALUE, true);
5925     TEST_UTIL.flush();
5926     scanTestNull(ht, ROW, FAMILY, VALUE, true);
5927     delete = new Delete(ROW);
5928     delete.deleteColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY);
5929     ht.delete(delete);
5930     // Null value
5931     put = new Put(ROW);
5932     put.add(FAMILY, QUALIFIER, null);
5933     ht.put(put);
5934     Scan scan = new Scan();
5935     scan.setReversed(true);
5936     scan.addColumn(FAMILY, QUALIFIER);
5937     Result result = getSingleScanResult(ht, scan);
5938     assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
5939     ht.close();
5940   }
5941 
5942   @Test
5943   public void testDeletesWithReverseScan() throws Exception {
5944     byte[] TABLE = Bytes.toBytes("testDeletesWithReverseScan");
5945     byte[][] ROWS = makeNAscii(ROW, 6);
5946     byte[][] FAMILIES = makeNAscii(FAMILY, 3);
5947     byte[][] VALUES = makeN(VALUE, 5);
5948     long[] ts = { 1000, 2000, 3000, 4000, 5000 };
5949     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES,
5950         TEST_UTIL.getConfiguration(), 3);
5951 
5952     Put put = new Put(ROW);
5953     put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
5954     put.add(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
5955     ht.put(put);
5956 
5957     Delete delete = new Delete(ROW);
5958     delete.deleteFamily(FAMILIES[0], ts[0]);
5959     ht.delete(delete);
5960 
5961     Scan scan = new Scan(ROW);
5962     scan.setReversed(true);
5963     scan.addFamily(FAMILIES[0]);
5964     scan.setMaxVersions(Integer.MAX_VALUE);
5965     Result result = getSingleScanResult(ht, scan);
5966     assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1] },
5967         new byte[][] { VALUES[1] }, 0, 0);
5968 
5969     // Test delete latest version
5970     put = new Put(ROW);
5971     put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
5972     put.add(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
5973     put.add(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
5974     put.add(FAMILIES[0], null, ts[4], VALUES[4]);
5975     put.add(FAMILIES[0], null, ts[2], VALUES[2]);
5976     put.add(FAMILIES[0], null, ts[3], VALUES[3]);
5977     ht.put(put);
5978 
5979     delete = new Delete(ROW);
5980     delete.deleteColumn(FAMILIES[0], QUALIFIER); // ts[4]
5981     ht.delete(delete);
5982 
5983     scan = new Scan(ROW);
5984     scan.setReversed(true);
5985     scan.addColumn(FAMILIES[0], QUALIFIER);
5986     scan.setMaxVersions(Integer.MAX_VALUE);
5987     result = getSingleScanResult(ht, scan);
5988     assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1],
5989         ts[2], ts[3] }, new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);
5990 
5991     // Test for HBASE-1847
5992     delete = new Delete(ROW);
5993     delete.deleteColumn(FAMILIES[0], null);
5994     ht.delete(delete);
5995 
5996     // Cleanup null qualifier
5997     delete = new Delete(ROW);
5998     delete.deleteColumns(FAMILIES[0], null);
5999     ht.delete(delete);
6000 
6001     // Expected client behavior might be that you can re-put deleted values
6002     // But alas, this is not to be. We can't put them back in either case.
6003 
6004     put = new Put(ROW);
6005     put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); // 1000
6006     put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); // 5000
6007     ht.put(put);
6008 
6009     // The Scanner returns the previous values, the expected-naive-unexpected
6010     // behavior
6011 
6012     scan = new Scan(ROW);
6013     scan.setReversed(true);
6014     scan.addFamily(FAMILIES[0]);
6015     scan.setMaxVersions(Integer.MAX_VALUE);
6016     result = getSingleScanResult(ht, scan);
6017     assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1],
6018         ts[2], ts[3] }, new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);
6019 
6020     // Test deleting an entire family from one row but not the other various
6021     // ways
6022 
6023     put = new Put(ROWS[0]);
6024     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
6025     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
6026     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
6027     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
6028     ht.put(put);
6029 
6030     put = new Put(ROWS[1]);
6031     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
6032     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
6033     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
6034     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
6035     ht.put(put);
6036 
6037     put = new Put(ROWS[2]);
6038     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
6039     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
6040     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
6041     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
6042     ht.put(put);
6043 
6044     delete = new Delete(ROWS[0]);
6045     delete.deleteFamily(FAMILIES[2]);
6046     ht.delete(delete);
6047 
6048     delete = new Delete(ROWS[1]);
6049     delete.deleteColumns(FAMILIES[1], QUALIFIER);
6050     ht.delete(delete);
6051 
6052     delete = new Delete(ROWS[2]);
6053     delete.deleteColumn(FAMILIES[1], QUALIFIER);
6054     delete.deleteColumn(FAMILIES[1], QUALIFIER);
6055     delete.deleteColumn(FAMILIES[2], QUALIFIER);
6056     ht.delete(delete);
6057 
6058     scan = new Scan(ROWS[0]);
6059     scan.setReversed(true);
6060     scan.addFamily(FAMILIES[1]);
6061     scan.addFamily(FAMILIES[2]);
6062     scan.setMaxVersions(Integer.MAX_VALUE);
6063     result = getSingleScanResult(ht, scan);
6064     assertTrue("Expected 2 keys but received " + result.size(),
6065         result.size() == 2);
6066     assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER, new long[] { ts[0],
6067         ts[1] }, new byte[][] { VALUES[0], VALUES[1] }, 0, 1);
6068 
6069     scan = new Scan(ROWS[1]);
6070     scan.setReversed(true);
6071     scan.addFamily(FAMILIES[1]);
6072     scan.addFamily(FAMILIES[2]);
6073     scan.setMaxVersions(Integer.MAX_VALUE);
6074     result = getSingleScanResult(ht, scan);
6075     assertTrue("Expected 2 keys but received " + result.size(),
6076         result.size() == 2);
6077 
6078     scan = new Scan(ROWS[2]);
6079     scan.setReversed(true);
6080     scan.addFamily(FAMILIES[1]);
6081     scan.addFamily(FAMILIES[2]);
6082     scan.setMaxVersions(Integer.MAX_VALUE);
6083     result = getSingleScanResult(ht, scan);
6084     assertEquals(1, result.size());
6085     assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
6086         new long[] { ts[2] }, new byte[][] { VALUES[2] }, 0, 0);
6087 
6088     // Test if we delete the family first in one row (HBASE-1541)
6089 
6090     delete = new Delete(ROWS[3]);
6091     delete.deleteFamily(FAMILIES[1]);
6092     ht.delete(delete);
6093 
6094     put = new Put(ROWS[3]);
6095     put.add(FAMILIES[2], QUALIFIER, VALUES[0]);
6096     ht.put(put);
6097 
6098     put = new Put(ROWS[4]);
6099     put.add(FAMILIES[1], QUALIFIER, VALUES[1]);
6100     put.add(FAMILIES[2], QUALIFIER, VALUES[2]);
6101     ht.put(put);
6102 
6103     scan = new Scan(ROWS[4]);
6104     scan.setReversed(true);
6105     scan.addFamily(FAMILIES[1]);
6106     scan.addFamily(FAMILIES[2]);
6107     scan.setMaxVersions(Integer.MAX_VALUE);
6108     ResultScanner scanner = ht.getScanner(scan);
6109     result = scanner.next();
6110     assertTrue("Expected 2 keys but received " + result.size(),
6111         result.size() == 2);
6112     assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[4]));
6113     assertTrue(Bytes.equals(result.raw()[1].getRow(), ROWS[4]));
6114     assertTrue(Bytes.equals(result.raw()[0].getValue(), VALUES[1]));
6115     assertTrue(Bytes.equals(result.raw()[1].getValue(), VALUES[2]));
6116     result = scanner.next();
6117     assertTrue("Expected 1 key but received " + result.size(),
6118         result.size() == 1);
6119     assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[3]));
6120     assertTrue(Bytes.equals(result.raw()[0].getValue(), VALUES[0]));
6121     scanner.close();
6122     ht.close();
6123   }
6124 
6125   /**
6126    * Tests reversed scan under multi regions
6127    */
6128   @Test
6129   public void testReversedScanUnderMultiRegions() throws Exception {
6130     // Test Initialization.
6131     byte[] TABLE = Bytes.toBytes("testReversedScanUnderMultiRegions");
6132     byte[] maxByteArray = ReversedClientScanner.MAX_BYTE_ARRAY;
6133     byte[][] splitRows = new byte[][] { Bytes.toBytes("005"),
6134         Bytes.add(Bytes.toBytes("005"), Bytes.multiple(maxByteArray, 16)),
6135         Bytes.toBytes("006"),
6136         Bytes.add(Bytes.toBytes("006"), Bytes.multiple(maxByteArray, 8)),
6137         Bytes.toBytes("007"),
6138         Bytes.add(Bytes.toBytes("007"), Bytes.multiple(maxByteArray, 4)),
6139         Bytes.toBytes("008"), Bytes.multiple(maxByteArray, 2) };
6140     HTable table = TEST_UTIL.createTable(TABLE, FAMILY, splitRows);
6141     TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
6142 
6143     assertEquals(splitRows.length + 1, table.getRegionLocations().size());
6144     // Insert one row each region
6145     int insertNum = splitRows.length;
6146     for (int i = 0; i < insertNum; i++) {
6147       Put put = new Put(splitRows[i]);
6148       put.add(FAMILY, QUALIFIER, VALUE);
6149       table.put(put);
6150     }
6151 
6152     // scan forward
6153     ResultScanner scanner = table.getScanner(new Scan());
6154     int count = 0;
6155     for (Result r : scanner) {
6156       assertTrue(!r.isEmpty());
6157       count++;
6158     }
6159     assertEquals(insertNum, count);
6160 
6161     // scan backward
6162     Scan scan = new Scan();
6163     scan.setReversed(true);
6164     scanner = table.getScanner(scan);
6165     count = 0;
6166     byte[] lastRow = null;
6167     for (Result r : scanner) {
6168       assertTrue(!r.isEmpty());
6169       count++;
6170       byte[] thisRow = r.getRow();
6171       if (lastRow != null) {
6172         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6173             + ",this row=" + Bytes.toString(thisRow),
6174             Bytes.compareTo(thisRow, lastRow) < 0);
6175       }
6176       lastRow = thisRow;
6177     }
6178     assertEquals(insertNum, count);
6179     table.close();
6180   }
6181 
6182 
6183   /**
6184    * Tests reversed scan under multi regions
6185    */
6186   @Test
6187   public void testSmallReversedScanUnderMultiRegions() throws Exception {
6188     // Test Initialization.
6189     byte[] TABLE = Bytes.toBytes("testSmallReversedScanUnderMultiRegions");
6190     byte[][] splitRows = new byte[][]{
6191         Bytes.toBytes("000"), Bytes.toBytes("002"), Bytes.toBytes("004"),
6192         Bytes.toBytes("006"), Bytes.toBytes("008"), Bytes.toBytes("010")};
6193     HTable table = TEST_UTIL.createTable(TABLE, FAMILY, splitRows);
6194     TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
6195 
6196     assertEquals(splitRows.length + 1, table.getRegionLocations().size());
6197     for (byte[] splitRow : splitRows) {
6198       Put put = new Put(splitRow);
6199       put.add(FAMILY, QUALIFIER, VALUE);
6200       table.put(put);
6201 
6202       byte[] nextRow = Bytes.copy(splitRow);
6203       nextRow[nextRow.length - 1]++;
6204 
6205       put = new Put(nextRow);
6206       put.add(FAMILY, QUALIFIER, VALUE);
6207       table.put(put);
6208     }
6209 
6210     // scan forward
6211     ResultScanner scanner = table.getScanner(new Scan());
6212     int count = 0;
6213     for (Result r : scanner) {
6214       assertTrue(!r.isEmpty());
6215       count++;
6216     }
6217     assertEquals(12, count);
6218 
6219     reverseScanTest(table, false);
6220     reverseScanTest(table, true);
6221 
6222     table.close();
6223   }
6224 
6225   private void reverseScanTest(HTable table, boolean small) throws IOException {
6226     // scan backward
6227     Scan scan = new Scan();
6228     scan.setReversed(true);
6229     ResultScanner scanner = table.getScanner(scan);
6230     int count = 0;
6231     byte[] lastRow = null;
6232     for (Result r : scanner) {
6233       assertTrue(!r.isEmpty());
6234       count++;
6235       byte[] thisRow = r.getRow();
6236       if (lastRow != null) {
6237         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6238             + ",this row=" + Bytes.toString(thisRow),
6239             Bytes.compareTo(thisRow, lastRow) < 0);
6240       }
6241       lastRow = thisRow;
6242     }
6243     assertEquals(12, count);
6244 
6245     scan = new Scan();
6246     scan.setSmall(small);
6247     scan.setReversed(true);
6248     scan.setStartRow(Bytes.toBytes("002"));
6249     scanner = table.getScanner(scan);
6250     count = 0;
6251     lastRow = null;
6252     for (Result r : scanner) {
6253       assertTrue(!r.isEmpty());
6254       count++;
6255       byte[] thisRow = r.getRow();
6256       if (lastRow != null) {
6257         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6258             + ",this row=" + Bytes.toString(thisRow),
6259             Bytes.compareTo(thisRow, lastRow) < 0);
6260       }
6261       lastRow = thisRow;
6262     }
6263     assertEquals(3, count); // 000 001 002
6264 
6265     scan = new Scan();
6266     scan.setSmall(small);
6267     scan.setReversed(true);
6268     scan.setStartRow(Bytes.toBytes("002"));
6269     scan.setStopRow(Bytes.toBytes("000"));
6270     scanner = table.getScanner(scan);
6271     count = 0;
6272     lastRow = null;
6273     for (Result r : scanner) {
6274       assertTrue(!r.isEmpty());
6275       count++;
6276       byte[] thisRow = r.getRow();
6277       if (lastRow != null) {
6278         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6279             + ",this row=" + Bytes.toString(thisRow),
6280             Bytes.compareTo(thisRow, lastRow) < 0);
6281       }
6282       lastRow = thisRow;
6283     }
6284     assertEquals(2, count); // 001 002
6285 
6286     scan = new Scan();
6287     scan.setSmall(small);
6288     scan.setReversed(true);
6289     scan.setStartRow(Bytes.toBytes("001"));
6290     scanner = table.getScanner(scan);
6291     count = 0;
6292     lastRow = null;
6293     for (Result r : scanner) {
6294       assertTrue(!r.isEmpty());
6295       count++;
6296       byte[] thisRow = r.getRow();
6297       if (lastRow != null) {
6298         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6299             + ",this row=" + Bytes.toString(thisRow),
6300             Bytes.compareTo(thisRow, lastRow) < 0);
6301       }
6302       lastRow = thisRow;
6303     }
6304     assertEquals(2, count); // 000 001
6305 
6306     scan = new Scan();
6307     scan.setSmall(small);
6308     scan.setReversed(true);
6309     scan.setStartRow(Bytes.toBytes("000"));
6310     scanner = table.getScanner(scan);
6311     count = 0;
6312     lastRow = null;
6313     for (Result r : scanner) {
6314       assertTrue(!r.isEmpty());
6315       count++;
6316       byte[] thisRow = r.getRow();
6317       if (lastRow != null) {
6318         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6319             + ",this row=" + Bytes.toString(thisRow),
6320             Bytes.compareTo(thisRow, lastRow) < 0);
6321       }
6322       lastRow = thisRow;
6323     }
6324     assertEquals(1, count); // 000
6325 
6326     scan = new Scan();
6327     scan.setSmall(small);
6328     scan.setReversed(true);
6329     scan.setStartRow(Bytes.toBytes("006"));
6330     scan.setStopRow(Bytes.toBytes("002"));
6331     scanner = table.getScanner(scan);
6332     count = 0;
6333     lastRow = null;
6334     for (Result r : scanner) {
6335       assertTrue(!r.isEmpty());
6336       count++;
6337       byte[] thisRow = r.getRow();
6338       if (lastRow != null) {
6339         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6340             + ",this row=" + Bytes.toString(thisRow),
6341             Bytes.compareTo(thisRow, lastRow) < 0);
6342       }
6343       lastRow = thisRow;
6344     }
6345     assertEquals(4, count); // 003 004 005 006
6346   }
6347 
6348   @Test
6349   public void testFilterAllRecords() throws IOException {
6350     Scan scan = new Scan();
6351     scan.setBatch(1);
6352     scan.setCaching(1);
6353     // Filter out any records
6354     scan.setFilter(new FilterList(new FirstKeyOnlyFilter(), new InclusiveStopFilter(new byte[0])));
6355     HTable table = new HTable(TEST_UTIL.getConfiguration(), TableName.NAMESPACE_TABLE_NAME);
6356     try {
6357       ResultScanner s = table.getScanner(scan);
6358       try {
6359         assertNull(s.next());
6360       } finally {
6361         s.close();
6362       }
6363     } finally {
6364       table.close();
6365     }
6366   }
6367 }