1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.fs.Path;
25  import org.apache.hadoop.hbase.HBaseConfiguration;
26  import org.apache.hadoop.hbase.HBaseTestCase;
27  import org.apache.hadoop.hbase.HBaseTestingUtility;
28  import org.apache.hadoop.hbase.HColumnDescriptor;
29  import org.apache.hadoop.hbase.HConstants;
30  import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
31  import org.apache.hadoop.hbase.HRegionInfo;
32  import org.apache.hadoop.hbase.HTableDescriptor;
33  import org.apache.hadoop.hbase.KeyValue;
34  import org.apache.hadoop.hbase.MultithreadedTestUtil;
35  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
36  import org.apache.hadoop.hbase.client.Delete;
37  import org.apache.hadoop.hbase.client.Get;
38  import org.apache.hadoop.hbase.client.Put;
39  import org.apache.hadoop.hbase.client.Result;
40  import org.apache.hadoop.hbase.client.Scan;
41  import org.apache.hadoop.hbase.filter.BinaryComparator;
42  import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
43  import org.apache.hadoop.hbase.filter.CompareFilter;
44  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
45  import org.apache.hadoop.hbase.filter.Filter;
46  import org.apache.hadoop.hbase.filter.FilterList;
47  import org.apache.hadoop.hbase.filter.PrefixFilter;
48  import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
49  import org.apache.hadoop.hbase.regionserver.HRegion.RegionScanner;
50  import org.apache.hadoop.hbase.regionserver.wal.HLog;
51  import org.apache.hadoop.hbase.util.Bytes;
52  import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
53  import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
54  import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
55  import org.apache.hadoop.hbase.util.Pair;
56  import org.apache.hadoop.hbase.util.Threads;
57  
58  import com.google.common.base.Joiner;
59  import com.google.common.collect.Lists;
60  
61  import java.io.IOException;
62  import java.util.ArrayList;
63  import java.util.Arrays;
64  import java.util.HashMap;
65  import java.util.List;
66  import java.util.Map;
67  import java.util.TreeMap;
68  import java.util.concurrent.atomic.AtomicBoolean;
69  import java.util.concurrent.atomic.AtomicInteger;
70  import java.util.concurrent.atomic.AtomicReference;
71  
72  
73  /**
74   * Basic stand-alone testing of HRegion.
75   *
76   * A lot of the meta information for an HRegion now lives inside other
77   * HRegions or in the HBaseMaster, so only basic testing is possible.
78   */
79  public class TestHRegion extends HBaseTestCase {
80    static final Log LOG = LogFactory.getLog(TestHRegion.class);
81  
82    HRegion region = null;
83    private final String DIR = HBaseTestingUtility.getTestDir() +
84      "/TestHRegion/";
85  
86    private final int MAX_VERSIONS = 2;
87  
88    // Test names
89    protected final byte[] tableName = Bytes.toBytes("testtable");;
90    protected final byte[] qual1 = Bytes.toBytes("qual1");
91    protected final byte[] qual2 = Bytes.toBytes("qual2");
92    protected final byte[] qual3 = Bytes.toBytes("qual3");
93    protected final byte[] value1 = Bytes.toBytes("value1");
94    protected final byte[] value2 = Bytes.toBytes("value2");
95    protected final byte [] row = Bytes.toBytes("rowA");
96  
97    /**
98     * @see org.apache.hadoop.hbase.HBaseTestCase#setUp()
99     */
100   @Override
101   protected void setUp() throws Exception {
102     super.setUp();
103   }
104 
105   @Override
106   protected void tearDown() throws Exception {
107     super.tearDown();
108     EnvironmentEdgeManagerTestHelper.reset();
109   }
110 
111   //////////////////////////////////////////////////////////////////////////////
112   // New tests that doesn't spin up a mini cluster but rather just test the
113   // individual code pieces in the HRegion. Putting files locally in
114   // /tmp/testtable
115   //////////////////////////////////////////////////////////////////////////////
116 
117   public void testGetWhileRegionClose() throws IOException {
118     HBaseConfiguration hc = initSplit();
119     int numRows = 100;
120     byte [][] families = {fam1, fam2, fam3};
121 
122     //Setting up region
123     String method = this.getName();
124     initHRegion(tableName, method, hc, families);
125 
126     // Put data in region
127     final int startRow = 100;
128     putData(startRow, numRows, qual1, families);
129     putData(startRow, numRows, qual2, families);
130     putData(startRow, numRows, qual3, families);
131     // this.region.flushcache();
132     final AtomicBoolean done = new AtomicBoolean(false);
133     final AtomicInteger gets = new AtomicInteger(0);
134     GetTillDoneOrException [] threads = new GetTillDoneOrException[10];
135     try {
136       // Set ten threads running concurrently getting from the region.
137       for (int i = 0; i < threads.length / 2; i++) {
138         threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow),
139           done, gets);
140         threads[i].setDaemon(true);
141         threads[i].start();
142       }
143       // Artificially make the condition by setting closing flag explicitly.
144       // I can't make the issue happen with a call to region.close().
145       this.region.closing.set(true);
146       for (int i = threads.length / 2; i < threads.length; i++) {
147         threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow),
148           done, gets);
149         threads[i].setDaemon(true);
150         threads[i].start();
151       }
152     } finally {
153       if (this.region != null) {
154         this.region.close();
155         this.region.getLog().closeAndDelete();
156       }
157     }
158     done.set(true);
159     for (GetTillDoneOrException t: threads) {
160       try {
161         t.join();
162       } catch (InterruptedException e) {
163         e.printStackTrace();
164       }
165       if (t.e != null) {
166         LOG.info("Exception=" + t.e);
167         assertFalse("Found a NPE in " + t.getName(),
168           t.e instanceof NullPointerException);
169       }
170     }
171   }
172 
173   /*
174    * Thread that does get on single row until 'done' flag is flipped.  If an
175    * exception causes us to fail, it records it.
176    */
177   class GetTillDoneOrException extends Thread {
178     private final Get g;
179     private final AtomicBoolean done;
180     private final AtomicInteger count;
181     private Exception e;
182 
183     GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d,
184         final AtomicInteger c) {
185       super("getter." + i);
186       this.g = new Get(r);
187       this.done = d;
188       this.count = c;
189     }
190 
191     @Override
192     public void run() {
193       while (!this.done.get()) {
194         try {
195           assertTrue(region.get(g, null).size() > 0);
196           this.count.incrementAndGet();
197         } catch (Exception e) {
198           this.e = e;
199           break;
200         }
201       }
202     }
203   }
204 
205   /*
206    * An involved filter test.  Has multiple column families and deletes in mix.
207    */
208   public void testWeirdCacheBehaviour() throws Exception {
209     byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
210     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
211         Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
212         Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
213     initHRegion(TABLE, getName(), FAMILIES);
214     String value = "this is the value";
215     String value2 = "this is some other value";
216     String keyPrefix1 = "prefix1"; // UUID.randomUUID().toString();
217     String keyPrefix2 = "prefix2"; // UUID.randomUUID().toString();
218     String keyPrefix3 = "prefix3"; // UUID.randomUUID().toString();
219     putRows(this.region, 3, value, keyPrefix1);
220     putRows(this.region, 3, value, keyPrefix2);
221     putRows(this.region, 3, value, keyPrefix3);
222     // this.region.flushCommits();
223     putRows(this.region, 3, value2, keyPrefix1);
224     putRows(this.region, 3, value2, keyPrefix2);
225     putRows(this.region, 3, value2, keyPrefix3);
226     System.out.println("Checking values for key: " + keyPrefix1);
227     assertEquals("Got back incorrect number of rows from scan", 3,
228       getNumberOfRows(keyPrefix1, value2, this.region));
229     System.out.println("Checking values for key: " + keyPrefix2);
230     assertEquals("Got back incorrect number of rows from scan", 3,
231       getNumberOfRows(keyPrefix2, value2, this.region));
232     System.out.println("Checking values for key: " + keyPrefix3);
233     assertEquals("Got back incorrect number of rows from scan", 3,
234       getNumberOfRows(keyPrefix3, value2, this.region));
235     deleteColumns(this.region, value2, keyPrefix1);
236     deleteColumns(this.region, value2, keyPrefix2);
237     deleteColumns(this.region, value2, keyPrefix3);
238     System.out.println("Starting important checks.....");
239     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,
240       0, getNumberOfRows(keyPrefix1, value2, this.region));
241     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,
242       0, getNumberOfRows(keyPrefix2, value2, this.region));
243     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,
244       0, getNumberOfRows(keyPrefix3, value2, this.region));
245   }
246 
247   private void deleteColumns(HRegion r, String value, String keyPrefix)
248   throws IOException {
249     InternalScanner scanner = buildScanner(keyPrefix, value, r);
250     int count = 0;
251     boolean more = false;
252     List<KeyValue> results = new ArrayList<KeyValue>();
253     do {
254       more = scanner.next(results);
255       if (results != null && !results.isEmpty())
256         count++;
257       else
258         break;
259       Delete delete = new Delete(results.get(0).getRow());
260       delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
261       r.delete(delete, null, false);
262       results.clear();
263     } while (more);
264     assertEquals("Did not perform correct number of deletes", 3, count);
265   }
266 
267   private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
268     InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
269     int numberOfResults = 0;
270     List<KeyValue> results = new ArrayList<KeyValue>();
271     boolean more = false;
272     do {
273       more = resultScanner.next(results);
274       if (results != null && !results.isEmpty()) numberOfResults++;
275       else break;
276       for (KeyValue kv: results) {
277         System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(kv.getValue()));
278       }
279       results.clear();
280     } while(more);
281     return numberOfResults;
282   }
283 
284   private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
285   throws IOException {
286     // Defaults FilterList.Operator.MUST_PASS_ALL.
287     FilterList allFilters = new FilterList();
288     allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
289     // Only return rows where this column value exists in the row.
290     SingleColumnValueFilter filter =
291       new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
292         Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
293     filter.setFilterIfMissing(true);
294     allFilters.addFilter(filter);
295     Scan scan = new Scan();
296     scan.addFamily(Bytes.toBytes("trans-blob"));
297     scan.addFamily(Bytes.toBytes("trans-type"));
298     scan.addFamily(Bytes.toBytes("trans-date"));
299     scan.addFamily(Bytes.toBytes("trans-tags"));
300     scan.addFamily(Bytes.toBytes("trans-group"));
301     scan.setFilter(allFilters);
302     return r.getScanner(scan);
303   }
304 
305   private void putRows(HRegion r, int numRows, String value, String key)
306   throws IOException {
307     for (int i = 0; i < numRows; i++) {
308       String row = key + "_" + i/* UUID.randomUUID().toString() */;
309       System.out.println(String.format("Saving row: %s, with value %s", row,
310         value));
311       Put put = new Put(Bytes.toBytes(row));
312       put.add(Bytes.toBytes("trans-blob"), null,
313         Bytes.toBytes("value for blob"));
314       put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
315       put.add(Bytes.toBytes("trans-date"), null,
316         Bytes.toBytes("20090921010101999"));
317       put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"),
318         Bytes.toBytes(value));
319       put.add(Bytes.toBytes("trans-group"), null,
320         Bytes.toBytes("adhocTransactionGroupId"));
321       r.put(put);
322     }
323   }
324 
325   public void testFamilyWithAndWithoutColon() throws Exception {
326     byte [] b = Bytes.toBytes(getName());
327     byte [] cf = Bytes.toBytes("cf");
328     initHRegion(b, getName(), cf);
329     Put p = new Put(b);
330     byte [] cfwithcolon = Bytes.toBytes("cf:");
331     p.add(cfwithcolon, cfwithcolon, cfwithcolon);
332     boolean exception = false;
333     try {
334       this.region.put(p);
335     } catch (NoSuchColumnFamilyException e) {
336       exception = true;
337     }
338     assertTrue(exception);
339   }
340 
341   @SuppressWarnings("unchecked")
342   public void testBatchPut() throws Exception {
343     byte[] b = Bytes.toBytes(getName());
344     byte[] cf = Bytes.toBytes("cf");
345     byte[] qual = Bytes.toBytes("qual");
346     byte[] val = Bytes.toBytes("val");
347     initHRegion(b, getName(), cf);
348 
349     HLog.getSyncOps(); // clear counter from prior tests
350     assertEquals(0, HLog.getSyncOps());
351 
352     LOG.info("First a batch put with all valid puts");
353     final Put[] puts = new Put[10];
354     for (int i = 0; i < 10; i++) {
355       puts[i] = new Put(Bytes.toBytes("row_" + i));
356       puts[i].add(cf, qual, val);
357     }
358 
359     OperationStatusCode[] codes = this.region.put(puts);
360     assertEquals(10, codes.length);
361     for (int i = 0; i < 10; i++) {
362       assertEquals(OperationStatusCode.SUCCESS, codes[i]);
363     }
364     assertEquals(1, HLog.getSyncOps());
365 
366     LOG.info("Next a batch put with one invalid family");
367     puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
368     codes = this.region.put(puts);
369     assertEquals(10, codes.length);
370     for (int i = 0; i < 10; i++) {
371       assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY :
372         OperationStatusCode.SUCCESS, codes[i]);
373     }
374     assertEquals(1, HLog.getSyncOps());
375 
376     LOG.info("Next a batch put that has to break into two batches to avoid a lock");
377     Integer lockedRow = region.obtainRowLock(Bytes.toBytes("row_2"));
378 
379     MultithreadedTestUtil.TestContext ctx =
380       new MultithreadedTestUtil.TestContext(HBaseConfiguration.create());
381     final AtomicReference<OperationStatusCode[]> retFromThread =
382       new AtomicReference<OperationStatusCode[]>();
383     TestThread putter = new TestThread(ctx) {
384       @Override
385       public void doWork() throws IOException {
386         retFromThread.set(region.put(puts));
387       }
388     };
389     LOG.info("...starting put thread while holding lock");
390     ctx.addThread(putter);
391     ctx.startThreads();
392 
393     LOG.info("...waiting for put thread to sync first time");
394     long startWait = System.currentTimeMillis();
395     while (HLog.getSyncOps() == 0) {
396       Thread.sleep(100);
397       if (System.currentTimeMillis() - startWait > 10000) {
398         fail("Timed out waiting for thread to sync first minibatch");
399       }
400     }
401     LOG.info("...releasing row lock, which should let put thread continue");
402     region.releaseRowLock(lockedRow);
403     LOG.info("...joining on thread");
404     ctx.stop();
405     LOG.info("...checking that next batch was synced");
406     assertEquals(1, HLog.getSyncOps());
407     codes = retFromThread.get();
408     for (int i = 0; i < 10; i++) {
409       assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY :
410         OperationStatusCode.SUCCESS, codes[i]);
411     }
412 
413     LOG.info("Nexta, a batch put which uses an already-held lock");
414     lockedRow = region.obtainRowLock(Bytes.toBytes("row_2"));
415     LOG.info("...obtained row lock");
416     List<Pair<Put, Integer>> putsAndLocks = Lists.newArrayList();
417     for (int i = 0; i < 10; i++) {
418       Pair<Put, Integer> pair = new Pair<Put, Integer>(puts[i], null);
419       if (i == 2) pair.setSecond(lockedRow);
420       putsAndLocks.add(pair);
421     }
422 
423     codes = region.put(putsAndLocks.toArray(new Pair[0]));
424     LOG.info("...performed put");
425     for (int i = 0; i < 10; i++) {
426       assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY :
427         OperationStatusCode.SUCCESS, codes[i]);
428     }
429     // Make sure we didn't do an extra batch
430     assertEquals(1, HLog.getSyncOps());
431 
432     // Make sure we still hold lock
433     assertTrue(region.isRowLocked(lockedRow));
434     LOG.info("...releasing lock");
435     region.releaseRowLock(lockedRow);
436   }
437 
438   //////////////////////////////////////////////////////////////////////////////
439   // checkAndMutate tests
440   //////////////////////////////////////////////////////////////////////////////
441   public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
442     byte [] tableName = Bytes.toBytes("testtable");
443     byte [] row1 = Bytes.toBytes("row1");
444     byte [] fam1 = Bytes.toBytes("fam1");
445     byte [] qf1  = Bytes.toBytes("qualifier");
446     byte [] emptyVal  = new byte[] {};
447     byte [] val1  = Bytes.toBytes("value1");
448     byte [] val2  = Bytes.toBytes("value2");
449     Integer lockId = null;
450 
451     //Setting up region
452     String method = this.getName();
453     initHRegion(tableName, method, fam1);
454     //Putting data in key
455     Put put = new Put(row1);
456     put.add(fam1, qf1, val1);
457 
458     //checkAndPut with correct value
459     boolean res = region.checkAndMutate(row1, fam1, qf1, emptyVal, put, lockId,
460         true);
461     assertTrue(res);
462 
463     // not empty anymore
464     res = region.checkAndMutate(row1, fam1, qf1, emptyVal, put, lockId, true);
465     assertFalse(res);
466 
467     Delete delete = new Delete(row1);
468     delete.deleteColumn(fam1, qf1);
469     res = region.checkAndMutate(row1, fam1, qf1, emptyVal, delete, lockId,
470         true);
471     assertFalse(res);
472 
473     put = new Put(row1);
474     put.add(fam1, qf1, val2);
475     //checkAndPut with correct value
476     res = region.checkAndMutate(row1, fam1, qf1, val1, put, lockId, true);
477     assertTrue(res);
478 
479     //checkAndDelete with correct value
480     delete = new Delete(row1);
481     delete.deleteColumn(fam1, qf1);
482     delete.deleteColumn(fam1, qf1);
483     res = region.checkAndMutate(row1, fam1, qf1, val2, delete, lockId, true);
484     assertTrue(res);
485 
486     delete = new Delete(row1);
487     res = region.checkAndMutate(row1, fam1, qf1, emptyVal, delete, lockId,
488         true);
489     assertTrue(res);
490   }
491 
492   public void testCheckAndMutate_WithWrongValue() throws IOException{
493     byte [] tableName = Bytes.toBytes("testtable");
494     byte [] row1 = Bytes.toBytes("row1");
495     byte [] fam1 = Bytes.toBytes("fam1");
496     byte [] qf1  = Bytes.toBytes("qualifier");
497     byte [] val1  = Bytes.toBytes("value1");
498     byte [] val2  = Bytes.toBytes("value2");
499     Integer lockId = null;
500 
501     //Setting up region
502     String method = this.getName();
503     initHRegion(tableName, method, fam1);
504 
505     //Putting data in key
506     Put put = new Put(row1);
507     put.add(fam1, qf1, val1);
508     region.put(put);
509 
510     //checkAndPut with wrong value
511     boolean res = region.checkAndMutate(row1, fam1, qf1, val2, put, lockId, true);
512     assertEquals(false, res);
513 
514     //checkAndDelete with wrong value
515     Delete delete = new Delete(row1);
516     delete.deleteFamily(fam1);
517     res = region.checkAndMutate(row1, fam1, qf1, val2, delete, lockId, true);
518     assertEquals(false, res);
519   }
520 
521   public void testCheckAndMutate_WithCorrectValue() throws IOException{
522     byte [] tableName = Bytes.toBytes("testtable");
523     byte [] row1 = Bytes.toBytes("row1");
524     byte [] fam1 = Bytes.toBytes("fam1");
525     byte [] qf1  = Bytes.toBytes("qualifier");
526     byte [] val1  = Bytes.toBytes("value1");
527     Integer lockId = null;
528 
529     //Setting up region
530     String method = this.getName();
531     initHRegion(tableName, method, fam1);
532 
533     //Putting data in key
534     Put put = new Put(row1);
535     put.add(fam1, qf1, val1);
536     region.put(put);
537 
538     //checkAndPut with correct value
539     boolean res = region.checkAndMutate(row1, fam1, qf1, val1, put, lockId, true);
540     assertEquals(true, res);
541 
542     //checkAndDelete with correct value
543     Delete delete = new Delete(row1);
544     delete.deleteColumn(fam1, qf1);
545     res = region.checkAndMutate(row1, fam1, qf1, val1, put, lockId, true);
546     assertEquals(true, res);
547   }
548 
549   public void testCheckAndPut_ThatPutWasWritten() throws IOException{
550     byte [] tableName = Bytes.toBytes("testtable");
551     byte [] row1 = Bytes.toBytes("row1");
552     byte [] fam1 = Bytes.toBytes("fam1");
553     byte [] fam2 = Bytes.toBytes("fam2");
554     byte [] qf1  = Bytes.toBytes("qualifier");
555     byte [] val1  = Bytes.toBytes("value1");
556     byte [] val2  = Bytes.toBytes("value2");
557     Integer lockId = null;
558 
559     byte [][] families = {fam1, fam2};
560 
561     //Setting up region
562     String method = this.getName();
563     initHRegion(tableName, method, families);
564 
565     //Putting data in the key to check
566     Put put = new Put(row1);
567     put.add(fam1, qf1, val1);
568     region.put(put);
569 
570     //Creating put to add
571     long ts = System.currentTimeMillis();
572     KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
573     put = new Put(row1);
574     put.add(kv);
575 
576     //checkAndPut with wrong value
577     Store store = region.getStore(fam1);
578     store.memstore.kvset.size();
579 
580     boolean res = region.checkAndMutate(row1, fam1, qf1, val1, put, lockId, true);
581     assertEquals(true, res);
582     store.memstore.kvset.size();
583 
584     Get get = new Get(row1);
585     get.addColumn(fam2, qf1);
586     KeyValue [] actual = region.get(get, null).raw();
587 
588     KeyValue [] expected = {kv};
589 
590     assertEquals(expected.length, actual.length);
591     for(int i=0; i<actual.length; i++) {
592       assertEquals(expected[i], actual[i]);
593     }
594 
595   }
596 
597   public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException{
598     byte [] tableName = Bytes.toBytes("testtable");
599     byte [] row1 = Bytes.toBytes("row1");
600     byte [] fam1 = Bytes.toBytes("fam1");
601     byte [] fam2 = Bytes.toBytes("fam2");
602     byte [] qf1  = Bytes.toBytes("qualifier1");
603     byte [] qf2  = Bytes.toBytes("qualifier2");
604     byte [] qf3  = Bytes.toBytes("qualifier3");
605     byte [] val1  = Bytes.toBytes("value1");
606     byte [] val2  = Bytes.toBytes("value2");
607     byte [] val3  = Bytes.toBytes("value3");
608     byte[] emptyVal = new byte[] { };
609     Integer lockId = null;
610 
611     byte [][] families = {fam1, fam2};
612 
613     //Setting up region
614     String method = this.getName();
615     initHRegion(tableName, method, families);
616 
617     //Put content
618     Put put = new Put(row1);
619     put.add(fam1, qf1, val1);
620     region.put(put);
621     Threads.sleep(2);
622 
623     put = new Put(row1);
624     put.add(fam1, qf1, val2);
625     put.add(fam2, qf1, val3);
626     put.add(fam2, qf2, val2);
627     put.add(fam2, qf3, val1);
628     put.add(fam1, qf3, val1);
629     region.put(put);
630 
631     //Multi-column delete
632     Delete delete = new Delete(row1);
633     delete.deleteColumn(fam1, qf1);
634     delete.deleteColumn(fam2, qf1);
635     delete.deleteColumn(fam1, qf3);
636     boolean res = region.checkAndMutate(row1, fam1, qf1, val2, delete, lockId,
637         true);
638     assertEquals(true, res);
639 
640     Get get = new Get(row1);
641     get.addColumn(fam1, qf1);
642     get.addColumn(fam1, qf3);
643     get.addColumn(fam2, qf2);
644     Result r = region.get(get, null);
645     assertEquals(2, r.size());
646     assertEquals(val1, r.getValue(fam1, qf1));
647     assertEquals(val2, r.getValue(fam2, qf2));
648 
649     //Family delete
650     delete = new Delete(row1);
651     delete.deleteFamily(fam2);
652     res = region.checkAndMutate(row1, fam2, qf1, emptyVal, delete, lockId,
653         true);
654     assertEquals(true, res);
655 
656     get = new Get(row1);
657     r = region.get(get, null);
658     assertEquals(1, r.size());
659     assertEquals(val1, r.getValue(fam1, qf1));
660 
661     //Row delete
662     delete = new Delete(row1);
663     res = region.checkAndMutate(row1, fam1, qf1, val1, delete, lockId,
664         true);
665     assertEquals(true, res);
666     get = new Get(row1);
667     r = region.get(get, null);
668     assertEquals(0, r.size());
669   }
670 
671   //////////////////////////////////////////////////////////////////////////////
672   // Delete tests
673   //////////////////////////////////////////////////////////////////////////////
674   public void testDelete_multiDeleteColumn() throws IOException {
675     byte [] tableName = Bytes.toBytes("testtable");
676     byte [] row1 = Bytes.toBytes("row1");
677     byte [] fam1 = Bytes.toBytes("fam1");
678     byte [] qual = Bytes.toBytes("qualifier");
679     byte [] value = Bytes.toBytes("value");
680 
681     Put put = new Put(row1);
682     put.add(fam1, qual, 1, value);
683     put.add(fam1, qual, 2, value);
684 
685     String method = this.getName();
686     initHRegion(tableName, method, fam1);
687 
688     region.put(put);
689 
690     // We do support deleting more than 1 'latest' version
691     Delete delete = new Delete(row1);
692     delete.deleteColumn(fam1, qual);
693     delete.deleteColumn(fam1, qual);
694     region.delete(delete, null, false);
695 
696     Get get = new Get(row1);
697     get.addFamily(fam1);
698     Result r = region.get(get, null);
699     assertEquals(0, r.size());
700   }
701 
702   public void testDelete_CheckFamily() throws IOException {
703     byte [] tableName = Bytes.toBytes("testtable");
704     byte [] row1 = Bytes.toBytes("row1");
705     byte [] fam1 = Bytes.toBytes("fam1");
706     byte [] fam2 = Bytes.toBytes("fam2");
707     byte [] fam3 = Bytes.toBytes("fam3");
708     byte [] fam4 = Bytes.toBytes("fam4");
709 
710     //Setting up region
711     String method = this.getName();
712     initHRegion(tableName, method, fam1, fam2, fam3);
713 
714     List<KeyValue> kvs  = new ArrayList<KeyValue>();
715     kvs.add(new KeyValue(row1, fam4, null, null));
716 
717 
718     //testing existing family
719     byte [] family = fam2;
720     try {
721       Map<byte[], List<KeyValue>> deleteMap = new HashMap<byte[], List<KeyValue>>();
722       deleteMap.put(family, kvs);
723       region.delete(deleteMap, true);
724     } catch (Exception e) {
725       assertTrue("Family " +new String(family)+ " does not exist", false);
726     }
727 
728     //testing non existing family
729     boolean ok = false;
730     family = fam4;
731     try {
732       Map<byte[], List<KeyValue>> deleteMap = new HashMap<byte[], List<KeyValue>>();
733       deleteMap.put(family, kvs);
734       region.delete(deleteMap, true);
735     } catch (Exception e) {
736       ok = true;
737     }
738     assertEquals("Family " +new String(family)+ " does exist", true, ok);
739   }
740 
741   public void testDelete_mixed() throws IOException, InterruptedException {
742     byte [] tableName = Bytes.toBytes("testtable");
743     byte [] fam = Bytes.toBytes("info");
744     byte [][] families = {fam};
745     String method = this.getName();
746     initHRegion(tableName, method, families);
747     EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
748 
749     byte [] row = Bytes.toBytes("table_name");
750     // column names
751     byte [] serverinfo = Bytes.toBytes("serverinfo");
752     byte [] splitA = Bytes.toBytes("splitA");
753     byte [] splitB = Bytes.toBytes("splitB");
754 
755     // add some data:
756     Put put = new Put(row);
757     put.add(fam, splitA, Bytes.toBytes("reference_A"));
758     region.put(put);
759 
760     put = new Put(row);
761     put.add(fam, splitB, Bytes.toBytes("reference_B"));
762     region.put(put);
763 
764     put = new Put(row);
765     put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
766     region.put(put);
767 
768     // ok now delete a split:
769     Delete delete = new Delete(row);
770     delete.deleteColumns(fam, splitA);
771     region.delete(delete, null, true);
772 
773     // assert some things:
774     Get get = new Get(row).addColumn(fam, serverinfo);
775     Result result = region.get(get, null);
776     assertEquals(1, result.size());
777 
778     get = new Get(row).addColumn(fam, splitA);
779     result = region.get(get, null);
780     assertEquals(0, result.size());
781 
782     get = new Get(row).addColumn(fam, splitB);
783     result = region.get(get, null);
784     assertEquals(1, result.size());
785 
786     // Assert that after a delete, I can put.
787     put = new Put(row);
788     put.add(fam, splitA, Bytes.toBytes("reference_A"));
789     region.put(put);
790     get = new Get(row);
791     result = region.get(get, null);
792     assertEquals(3, result.size());
793 
794     // Now delete all... then test I can add stuff back
795     delete = new Delete(row);
796     region.delete(delete, null, false);
797     assertEquals(0, region.get(get, null).size());
798 
799     region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
800     result = region.get(get, null);
801     assertEquals(1, result.size());
802   }
803 
804   public void testDeleteRowWithFutureTs() throws IOException {
805     byte [] tableName = Bytes.toBytes("testtable");
806     byte [] fam = Bytes.toBytes("info");
807     byte [][] families = {fam};
808     String method = this.getName();
809     initHRegion(tableName, method, families);
810 
811     byte [] row = Bytes.toBytes("table_name");
812     // column names
813     byte [] serverinfo = Bytes.toBytes("serverinfo");
814 
815     // add data in the far future
816     Put put = new Put(row);
817     put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP-5,Bytes.toBytes("value"));
818     region.put(put);
819 
820     // now delete something in the present
821     Delete delete = new Delete(row);
822     region.delete(delete, null, true);
823 
824     // make sure we still see our data
825     Get get = new Get(row).addColumn(fam, serverinfo);
826     Result result = region.get(get, null);
827     assertEquals(1, result.size());
828 
829     // delete the future row
830     delete = new Delete(row,HConstants.LATEST_TIMESTAMP-3,null);
831     region.delete(delete, null, true);
832 
833     // make sure it is gone
834     get = new Get(row).addColumn(fam, serverinfo);
835     result = region.get(get, null);
836     assertEquals(0, result.size());
837   }
838 
839   /**
840    * Tests that the special LATEST_TIMESTAMP option for puts gets
841    * replaced by the actual timestamp
842    */
843   public void testPutWithLatestTS() throws IOException {
844     byte [] tableName = Bytes.toBytes("testtable");
845     byte [] fam = Bytes.toBytes("info");
846     byte [][] families = {fam};
847     String method = this.getName();
848     initHRegion(tableName, method, families);
849 
850     byte [] row = Bytes.toBytes("row1");
851     // column names
852     byte [] qual = Bytes.toBytes("qual");
853 
854     // add data with LATEST_TIMESTAMP, put without WAL
855     Put put = new Put(row);
856     put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
857     region.put(put, false);
858 
859     // Make sure it shows up with an actual timestamp
860     Get get = new Get(row).addColumn(fam, qual);
861     Result result = region.get(get, null);
862     assertEquals(1, result.size());
863     KeyValue kv = result.raw()[0];
864     LOG.info("Got: " + kv);
865     assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
866         kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
867 
868     // Check same with WAL enabled (historically these took different
869     // code paths, so check both)
870     row = Bytes.toBytes("row2");
871     put = new Put(row);
872     put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
873     region.put(put, true);
874 
875     // Make sure it shows up with an actual timestamp
876     get = new Get(row).addColumn(fam, qual);
877     result = region.get(get, null);
878     assertEquals(1, result.size());
879     kv = result.raw()[0];
880     LOG.info("Got: " + kv);
881     assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
882         kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
883 
884   }
885 
886   public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
887     byte [] tableName = Bytes.toBytes("test_table");
888     byte [] fam1 = Bytes.toBytes("columnA");
889     byte [] fam2 = Bytes.toBytes("columnB");
890     initHRegion(tableName, getName(), fam1, fam2);
891 
892     byte [] rowA = Bytes.toBytes("rowA");
893     byte [] rowB = Bytes.toBytes("rowB");
894 
895     byte [] value = Bytes.toBytes("value");
896 
897     Delete delete = new Delete(rowA);
898     delete.deleteFamily(fam1);
899 
900     region.delete(delete, null, true);
901 
902     // now create data.
903     Put put = new Put(rowA);
904     put.add(fam2, null, value);
905     region.put(put);
906 
907     put = new Put(rowB);
908     put.add(fam1, null, value);
909     put.add(fam2, null, value);
910     region.put(put);
911 
912     Scan scan = new Scan();
913     scan.addFamily(fam1).addFamily(fam2);
914     InternalScanner s = region.getScanner(scan);
915     List<KeyValue> results = new ArrayList<KeyValue>();
916     s.next(results);
917     assertTrue(Bytes.equals(rowA, results.get(0).getRow()));
918 
919     results.clear();
920     s.next(results);
921     assertTrue(Bytes.equals(rowB, results.get(0).getRow()));
922 
923   }
924 
925   public void testDeleteColumns_PostInsert() throws IOException,
926       InterruptedException {
927     Delete delete = new Delete(row);
928     delete.deleteColumns(fam1, qual1);
929     doTestDelete_AndPostInsert(delete);
930   }
931 
932   public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
933     Delete delete = new Delete(row);
934     delete.deleteFamily(fam1);
935     doTestDelete_AndPostInsert(delete);
936   }
937 
938   public void doTestDelete_AndPostInsert(Delete delete)
939       throws IOException, InterruptedException {
940     initHRegion(tableName, getName(), fam1);
941     EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
942     Put put = new Put(row);
943     put.add(fam1, qual1, value1);
944     region.put(put);
945 
946     // now delete the value:
947     region.delete(delete, null, true);
948 
949 
950     // ok put data:
951     put = new Put(row);
952     put.add(fam1, qual1, value2);
953     region.put(put);
954 
955     // ok get:
956     Get get = new Get(row);
957     get.addColumn(fam1, qual1);
958 
959     Result r = region.get(get, null);
960     assertEquals(1, r.size());
961     assertByteEquals(value2, r.getValue(fam1, qual1));
962 
963     // next:
964     Scan scan = new Scan(row);
965     scan.addColumn(fam1, qual1);
966     InternalScanner s = region.getScanner(scan);
967 
968     List<KeyValue> results = new ArrayList<KeyValue>();
969     assertEquals(false, s.next(results));
970     assertEquals(1, results.size());
971     KeyValue kv = results.get(0);
972 
973     assertByteEquals(value2, kv.getValue());
974     assertByteEquals(fam1, kv.getFamily());
975     assertByteEquals(qual1, kv.getQualifier());
976     assertByteEquals(row, kv.getRow());
977   }
978 
979 
980 
981   public void testDelete_CheckTimestampUpdated()
982   throws IOException {
983     byte [] row1 = Bytes.toBytes("row1");
984     byte [] col1 = Bytes.toBytes("col1");
985     byte [] col2 = Bytes.toBytes("col2");
986     byte [] col3 = Bytes.toBytes("col3");
987 
988     //Setting up region
989     String method = this.getName();
990     initHRegion(tableName, method, fam1);
991 
992     //Building checkerList
993     List<KeyValue> kvs  = new ArrayList<KeyValue>();
994     kvs.add(new KeyValue(row1, fam1, col1, null));
995     kvs.add(new KeyValue(row1, fam1, col2, null));
996     kvs.add(new KeyValue(row1, fam1, col3, null));
997 
998     Map<byte[], List<KeyValue>> deleteMap = new HashMap<byte[], List<KeyValue>>();
999     deleteMap.put(fam1, kvs);
1000     region.delete(deleteMap, true);
1001 
1002     // extract the key values out the memstore:
1003     // This is kinda hacky, but better than nothing...
1004     long now = System.currentTimeMillis();
1005     KeyValue firstKv = region.getStore(fam1).memstore.kvset.first();
1006     assertTrue(firstKv.getTimestamp() <= now);
1007     now = firstKv.getTimestamp();
1008     for (KeyValue kv: region.getStore(fam1).memstore.kvset) {
1009       assertTrue(kv.getTimestamp() <= now);
1010       now = kv.getTimestamp();
1011     }
1012   }
1013 
1014   //////////////////////////////////////////////////////////////////////////////
1015   // Get tests
1016   //////////////////////////////////////////////////////////////////////////////
1017   public void testGet_FamilyChecker() throws IOException {
1018     byte [] tableName = Bytes.toBytes("testtable");
1019     byte [] row1 = Bytes.toBytes("row1");
1020     byte [] fam1 = Bytes.toBytes("fam1");
1021     byte [] fam2 = Bytes.toBytes("False");
1022     byte [] col1 = Bytes.toBytes("col1");
1023 
1024     //Setting up region
1025     String method = this.getName();
1026     initHRegion(tableName, method, fam1);
1027 
1028     Get get = new Get(row1);
1029     get.addColumn(fam2, col1);
1030 
1031     //Test
1032     try {
1033       region.get(get, null);
1034     } catch (NoSuchColumnFamilyException e){
1035       assertFalse(false);
1036       return;
1037     }
1038     assertFalse(true);
1039   }
1040 
1041   public void testGet_Basic() throws IOException {
1042     byte [] tableName = Bytes.toBytes("testtable");
1043     byte [] row1 = Bytes.toBytes("row1");
1044     byte [] fam1 = Bytes.toBytes("fam1");
1045     byte [] col1 = Bytes.toBytes("col1");
1046     byte [] col2 = Bytes.toBytes("col2");
1047     byte [] col3 = Bytes.toBytes("col3");
1048     byte [] col4 = Bytes.toBytes("col4");
1049     byte [] col5 = Bytes.toBytes("col5");
1050 
1051     //Setting up region
1052     String method = this.getName();
1053     initHRegion(tableName, method, fam1);
1054 
1055     //Add to memstore
1056     Put put = new Put(row1);
1057     put.add(fam1, col1, null);
1058     put.add(fam1, col2, null);
1059     put.add(fam1, col3, null);
1060     put.add(fam1, col4, null);
1061     put.add(fam1, col5, null);
1062     region.put(put);
1063 
1064     Get get = new Get(row1);
1065     get.addColumn(fam1, col2);
1066     get.addColumn(fam1, col4);
1067     //Expected result
1068     KeyValue kv1 = new KeyValue(row1, fam1, col2);
1069     KeyValue kv2 = new KeyValue(row1, fam1, col4);
1070     KeyValue [] expected = {kv1, kv2};
1071 
1072     //Test
1073     Result res = region.get(get, null);
1074     assertEquals(expected.length, res.size());
1075     for(int i=0; i<res.size(); i++){
1076       assertEquals(0,
1077           Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
1078       assertEquals(0,
1079           Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
1080       assertEquals(0,
1081           Bytes.compareTo(
1082               expected[i].getQualifier(), res.raw()[i].getQualifier()));
1083     }
1084 
1085     // Test using a filter on a Get
1086     Get g = new Get(row1);
1087     final int count = 2;
1088     g.setFilter(new ColumnCountGetFilter(count));
1089     res = region.get(g, null);
1090     assertEquals(count, res.size());
1091   }
1092 
1093   public void testGet_Empty() throws IOException {
1094     byte [] tableName = Bytes.toBytes("emptytable");
1095     byte [] row = Bytes.toBytes("row");
1096     byte [] fam = Bytes.toBytes("fam");
1097 
1098     String method = this.getName();
1099     initHRegion(tableName, method, fam);
1100 
1101     Get get = new Get(row);
1102     get.addFamily(fam);
1103     Result r = region.get(get, null);
1104 
1105     assertTrue(r.isEmpty());
1106   }
1107 
1108   //Test that checked if there was anything special when reading from the ROOT
1109   //table. To be able to use this test you need to comment the part in
1110   //HTableDescriptor that checks for '-' and '.'. You also need to remove the
1111   //s in the beginning of the name.
1112   public void stestGet_Root() throws IOException {
1113     //Setting up region
1114     String method = this.getName();
1115     initHRegion(HConstants.ROOT_TABLE_NAME, method, HConstants.CATALOG_FAMILY);
1116 
1117     //Add to memstore
1118     Put put = new Put(HConstants.EMPTY_START_ROW);
1119     put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, null);
1120     region.put(put);
1121 
1122     Get get = new Get(HConstants.EMPTY_START_ROW);
1123     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
1124 
1125     //Expected result
1126     KeyValue kv1 = new KeyValue(HConstants.EMPTY_START_ROW,
1127         HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
1128     KeyValue [] expected = {kv1};
1129 
1130     //Test from memstore
1131     Result res = region.get(get, null);
1132 
1133     assertEquals(expected.length, res.size());
1134     for(int i=0; i<res.size(); i++){
1135       assertEquals(0,
1136           Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
1137       assertEquals(0,
1138           Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
1139       assertEquals(0,
1140           Bytes.compareTo(
1141               expected[i].getQualifier(), res.raw()[i].getQualifier()));
1142     }
1143 
1144     //flush
1145     region.flushcache();
1146 
1147     //test2
1148     res = region.get(get, null);
1149 
1150     assertEquals(expected.length, res.size());
1151     for(int i=0; i<res.size(); i++){
1152       assertEquals(0,
1153           Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
1154       assertEquals(0,
1155           Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
1156       assertEquals(0,
1157           Bytes.compareTo(
1158               expected[i].getQualifier(), res.raw()[i].getQualifier()));
1159     }
1160 
1161     //Scan
1162     Scan scan = new Scan();
1163     scan.addFamily(HConstants.CATALOG_FAMILY);
1164     InternalScanner s = region.getScanner(scan);
1165     List<KeyValue> result = new ArrayList<KeyValue>();
1166     s.next(result);
1167 
1168     assertEquals(expected.length, result.size());
1169     for(int i=0; i<res.size(); i++){
1170       assertEquals(0,
1171           Bytes.compareTo(expected[i].getRow(), result.get(i).getRow()));
1172       assertEquals(0,
1173           Bytes.compareTo(expected[i].getFamily(), result.get(i).getFamily()));
1174       assertEquals(0,
1175           Bytes.compareTo(
1176               expected[i].getQualifier(), result.get(i).getQualifier()));
1177     }
1178   }
1179 
1180   //////////////////////////////////////////////////////////////////////////////
1181   // Lock test
1182   //////////////////////////////////////////////////////////////////////////////
1183   public void testLocks() throws IOException{
1184     byte [] tableName = Bytes.toBytes("testtable");
1185     byte [][] families = {fam1, fam2, fam3};
1186 
1187     HBaseConfiguration hc = initSplit();
1188     //Setting up region
1189     String method = this.getName();
1190     initHRegion(tableName, method, hc, families);
1191 
1192     final int threadCount = 10;
1193     final int lockCount = 10;
1194 
1195     List<Thread>threads = new ArrayList<Thread>(threadCount);
1196     for (int i = 0; i < threadCount; i++) {
1197       threads.add(new Thread(Integer.toString(i)) {
1198         @Override
1199         public void run() {
1200           Integer [] lockids = new Integer[lockCount];
1201           // Get locks.
1202           for (int i = 0; i < lockCount; i++) {
1203             try {
1204               byte [] rowid = Bytes.toBytes(Integer.toString(i));
1205               lockids[i] = region.obtainRowLock(rowid);
1206               assertEquals(rowid, region.getRowFromLock(lockids[i]));
1207               LOG.debug(getName() + " locked " + Bytes.toString(rowid));
1208             } catch (IOException e) {
1209               e.printStackTrace();
1210             }
1211           }
1212           LOG.debug(getName() + " set " +
1213               Integer.toString(lockCount) + " locks");
1214 
1215           // Abort outstanding locks.
1216           for (int i = lockCount - 1; i >= 0; i--) {
1217             region.releaseRowLock(lockids[i]);
1218             LOG.debug(getName() + " unlocked " + i);
1219           }
1220           LOG.debug(getName() + " released " +
1221               Integer.toString(lockCount) + " locks");
1222         }
1223       });
1224     }
1225 
1226     // Startup all our threads.
1227     for (Thread t : threads) {
1228       t.start();
1229     }
1230 
1231     // Now wait around till all are done.
1232     for (Thread t: threads) {
1233       while (t.isAlive()) {
1234         try {
1235           Thread.sleep(1);
1236         } catch (InterruptedException e) {
1237           // Go around again.
1238         }
1239       }
1240     }
1241     LOG.info("locks completed.");
1242   }
1243 
1244   //////////////////////////////////////////////////////////////////////////////
1245   // Merge test
1246   //////////////////////////////////////////////////////////////////////////////
1247   public void testMerge() throws IOException {
1248     byte [] tableName = Bytes.toBytes("testtable");
1249     byte [][] families = {fam1, fam2, fam3};
1250 
1251     HBaseConfiguration hc = initSplit();
1252     //Setting up region
1253     String method = this.getName();
1254     initHRegion(tableName, method, hc, families);
1255 
1256     try {
1257       LOG.info("" + addContent(region, fam3));
1258       region.flushcache();
1259       byte [] splitRow = region.compactStores();
1260       assertNotNull(splitRow);
1261       LOG.info("SplitRow: " + Bytes.toString(splitRow));
1262       HRegion [] regions = split(region, splitRow);
1263       try {
1264         // Need to open the regions.
1265         // TODO: Add an 'open' to HRegion... don't do open by constructing
1266         // instance.
1267         for (int i = 0; i < regions.length; i++) {
1268           regions[i] = openClosedRegion(regions[i]);
1269         }
1270         Path oldRegionPath = region.getRegionDir();
1271         long startTime = System.currentTimeMillis();
1272         HRegion subregions [] = region.splitRegion(splitRow);
1273         if (subregions != null) {
1274           LOG.info("Split region elapsed time: "
1275               + ((System.currentTimeMillis() - startTime) / 1000.0));
1276           assertEquals("Number of subregions", subregions.length, 2);
1277           for (int i = 0; i < subregions.length; i++) {
1278             subregions[i] = openClosedRegion(subregions[i]);
1279             subregions[i].compactStores();
1280           }
1281 
1282           // Now merge it back together
1283           Path oldRegion1 = subregions[0].getRegionDir();
1284           Path oldRegion2 = subregions[1].getRegionDir();
1285           startTime = System.currentTimeMillis();
1286           region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
1287           LOG.info("Merge regions elapsed time: " +
1288               ((System.currentTimeMillis() - startTime) / 1000.0));
1289           fs.delete(oldRegion1, true);
1290           fs.delete(oldRegion2, true);
1291           fs.delete(oldRegionPath, true);
1292         }
1293         LOG.info("splitAndMerge completed.");
1294       } finally {
1295         for (int i = 0; i < regions.length; i++) {
1296           try {
1297             regions[i].close();
1298           } catch (IOException e) {
1299             // Ignore.
1300           }
1301         }
1302       }
1303     } finally {
1304       if (region != null) {
1305         region.close();
1306         region.getLog().closeAndDelete();
1307       }
1308     }
1309   }
1310 
1311   //////////////////////////////////////////////////////////////////////////////
1312   // Scanner tests
1313   //////////////////////////////////////////////////////////////////////////////
1314   public void testGetScanner_WithOkFamilies() throws IOException {
1315     byte [] tableName = Bytes.toBytes("testtable");
1316     byte [] fam1 = Bytes.toBytes("fam1");
1317     byte [] fam2 = Bytes.toBytes("fam2");
1318 
1319     byte [][] families = {fam1, fam2};
1320 
1321     //Setting up region
1322     String method = this.getName();
1323     initHRegion(tableName, method, families);
1324 
1325     Scan scan = new Scan();
1326     scan.addFamily(fam1);
1327     scan.addFamily(fam2);
1328     try {
1329       region.getScanner(scan);
1330     } catch (Exception e) {
1331       assertTrue("Families could not be found in Region", false);
1332     }
1333   }
1334 
1335   public void testGetScanner_WithNotOkFamilies() throws IOException {
1336     byte [] tableName = Bytes.toBytes("testtable");
1337     byte [] fam1 = Bytes.toBytes("fam1");
1338     byte [] fam2 = Bytes.toBytes("fam2");
1339 
1340     byte [][] families = {fam1};
1341 
1342     //Setting up region
1343     String method = this.getName();
1344     initHRegion(tableName, method, families);
1345 
1346     Scan scan = new Scan();
1347     scan.addFamily(fam2);
1348     boolean ok = false;
1349     try {
1350       region.getScanner(scan);
1351     } catch (Exception e) {
1352       ok = true;
1353     }
1354     assertTrue("Families could not be found in Region", ok);
1355   }
1356 
1357   public void testGetScanner_WithNoFamilies() throws IOException {
1358     byte [] tableName = Bytes.toBytes("testtable");
1359     byte [] row1 = Bytes.toBytes("row1");
1360     byte [] fam1 = Bytes.toBytes("fam1");
1361     byte [] fam2 = Bytes.toBytes("fam2");
1362     byte [] fam3 = Bytes.toBytes("fam3");
1363     byte [] fam4 = Bytes.toBytes("fam4");
1364 
1365     byte [][] families = {fam1, fam2, fam3, fam4};
1366 
1367     //Setting up region
1368     String method = this.getName();
1369     initHRegion(tableName, method, families);
1370 
1371 
1372     //Putting data in Region
1373     Put put = new Put(row1);
1374     put.add(fam1, null, null);
1375     put.add(fam2, null, null);
1376     put.add(fam3, null, null);
1377     put.add(fam4, null, null);
1378     region.put(put);
1379 
1380     Scan scan = null;
1381     HRegion.RegionScanner is = null;
1382 
1383     //Testing to see how many scanners that is produced by getScanner, starting
1384     //with known number, 2 - current = 1
1385     scan = new Scan();
1386     scan.addFamily(fam2);
1387     scan.addFamily(fam4);
1388     is = (RegionScanner) region.getScanner(scan);
1389     ReadWriteConsistencyControl.resetThreadReadPoint(region.getRWCC());
1390     assertEquals(1, ((RegionScanner)is).storeHeap.getHeap().size());
1391 
1392     scan = new Scan();
1393     is = (RegionScanner) region.getScanner(scan);
1394     ReadWriteConsistencyControl.resetThreadReadPoint(region.getRWCC());
1395     assertEquals(families.length -1,
1396         ((RegionScanner)is).storeHeap.getHeap().size());
1397   }
1398 
1399   /**
1400    * This method tests https://issues.apache.org/jira/browse/HBASE-2516.
1401    */
1402   public void testGetScanner_WithRegionClosed() {
1403     byte[] tableName = Bytes.toBytes("testtable");
1404     byte[] fam1 = Bytes.toBytes("fam1");
1405     byte[] fam2 = Bytes.toBytes("fam2");
1406 
1407     byte[][] families = {fam1, fam2};
1408 
1409     //Setting up region
1410     String method = this.getName();
1411     try {
1412       initHRegion(tableName, method, families);
1413     } catch (IOException e) {
1414       e.printStackTrace();
1415       fail("Got IOException during initHRegion, " + e.getMessage());
1416     }
1417     region.closed.set(true);
1418     try {
1419       region.getScanner(null);
1420       fail("Expected to get an exception during getScanner on a region that is closed");
1421     } catch (org.apache.hadoop.hbase.NotServingRegionException e) {
1422       //this is the correct exception that is expected
1423     } catch (IOException e) {
1424       fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
1425               + e.getMessage());
1426     }
1427   }
1428 
1429   public void testRegionScanner_Next() throws IOException {
1430     byte [] tableName = Bytes.toBytes("testtable");
1431     byte [] row1 = Bytes.toBytes("row1");
1432     byte [] row2 = Bytes.toBytes("row2");
1433     byte [] fam1 = Bytes.toBytes("fam1");
1434     byte [] fam2 = Bytes.toBytes("fam2");
1435     byte [] fam3 = Bytes.toBytes("fam3");
1436     byte [] fam4 = Bytes.toBytes("fam4");
1437 
1438     byte [][] families = {fam1, fam2, fam3, fam4};
1439     long ts = System.currentTimeMillis();
1440 
1441     //Setting up region
1442     String method = this.getName();
1443     initHRegion(tableName, method, families);
1444 
1445     //Putting data in Region
1446     Put put = null;
1447     put = new Put(row1);
1448     put.add(fam1, null, ts, null);
1449     put.add(fam2, null, ts, null);
1450     put.add(fam3, null, ts, null);
1451     put.add(fam4, null, ts, null);
1452     region.put(put);
1453 
1454     put = new Put(row2);
1455     put.add(fam1, null, ts, null);
1456     put.add(fam2, null, ts, null);
1457     put.add(fam3, null, ts, null);
1458     put.add(fam4, null, ts, null);
1459     region.put(put);
1460 
1461     Scan scan = new Scan();
1462     scan.addFamily(fam2);
1463     scan.addFamily(fam4);
1464     InternalScanner is = region.getScanner(scan);
1465 
1466     List<KeyValue> res = null;
1467 
1468     //Result 1
1469     List<KeyValue> expected1 = new ArrayList<KeyValue>();
1470     expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
1471     expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
1472 
1473     res = new ArrayList<KeyValue>();
1474     is.next(res);
1475     for(int i=0; i<res.size(); i++) {
1476       assertEquals(expected1.get(i), res.get(i));
1477     }
1478 
1479     //Result 2
1480     List<KeyValue> expected2 = new ArrayList<KeyValue>();
1481     expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
1482     expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
1483 
1484     res = new ArrayList<KeyValue>();
1485     is.next(res);
1486     for(int i=0; i<res.size(); i++) {
1487       assertEquals(expected2.get(i), res.get(i));
1488     }
1489 
1490   }
1491 
1492   public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions()
1493   throws IOException {
1494     byte [] tableName = Bytes.toBytes("testtable");
1495     byte [] row1 = Bytes.toBytes("row1");
1496     byte [] qf1 = Bytes.toBytes("qualifier1");
1497     byte [] qf2 = Bytes.toBytes("qualifier2");
1498     byte [] fam1 = Bytes.toBytes("fam1");
1499     byte [][] families = {fam1};
1500 
1501     long ts1 = System.currentTimeMillis();
1502     long ts2 = ts1 + 1;
1503     long ts3 = ts1 + 2;
1504 
1505     //Setting up region
1506     String method = this.getName();
1507     initHRegion(tableName, method, families);
1508 
1509     //Putting data in Region
1510     Put put = null;
1511     KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
1512     KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
1513     KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
1514 
1515     KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
1516     KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
1517     KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
1518 
1519     put = new Put(row1);
1520     put.add(kv13);
1521     put.add(kv12);
1522     put.add(kv11);
1523     put.add(kv23);
1524     put.add(kv22);
1525     put.add(kv21);
1526     region.put(put);
1527 
1528     //Expected
1529     List<KeyValue> expected = new ArrayList<KeyValue>();
1530     expected.add(kv13);
1531     expected.add(kv12);
1532 
1533     Scan scan = new Scan(row1);
1534     scan.addColumn(fam1, qf1);
1535     scan.setMaxVersions(MAX_VERSIONS);
1536     List<KeyValue> actual = new ArrayList<KeyValue>();
1537     InternalScanner scanner = region.getScanner(scan);
1538 
1539     boolean hasNext = scanner.next(actual);
1540     assertEquals(false, hasNext);
1541 
1542     //Verify result
1543     for(int i=0; i<expected.size(); i++) {
1544       assertEquals(expected.get(i), actual.get(i));
1545     }
1546   }
1547 
1548   public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions()
1549   throws IOException{
1550     byte [] tableName = Bytes.toBytes("testtable");
1551     byte [] row1 = Bytes.toBytes("row1");
1552     byte [] qf1 = Bytes.toBytes("qualifier1");
1553     byte [] qf2 = Bytes.toBytes("qualifier2");
1554     byte [] fam1 = Bytes.toBytes("fam1");
1555     byte [][] families = {fam1};
1556 
1557     long ts1 = 1; //System.currentTimeMillis();
1558     long ts2 = ts1 + 1;
1559     long ts3 = ts1 + 2;
1560 
1561     //Setting up region
1562     String method = this.getName();
1563     initHRegion(tableName, method, families);
1564 
1565     //Putting data in Region
1566     Put put = null;
1567     KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
1568     KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
1569     KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
1570 
1571     KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
1572     KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
1573     KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
1574 
1575     put = new Put(row1);
1576     put.add(kv13);
1577     put.add(kv12);
1578     put.add(kv11);
1579     put.add(kv23);
1580     put.add(kv22);
1581     put.add(kv21);
1582     region.put(put);
1583     region.flushcache();
1584 
1585     //Expected
1586     List<KeyValue> expected = new ArrayList<KeyValue>();
1587     expected.add(kv13);
1588     expected.add(kv12);
1589     expected.add(kv23);
1590     expected.add(kv22);
1591 
1592     Scan scan = new Scan(row1);
1593     scan.addColumn(fam1, qf1);
1594     scan.addColumn(fam1, qf2);
1595     scan.setMaxVersions(MAX_VERSIONS);
1596     List<KeyValue> actual = new ArrayList<KeyValue>();
1597     InternalScanner scanner = region.getScanner(scan);
1598 
1599     boolean hasNext = scanner.next(actual);
1600     assertEquals(false, hasNext);
1601 
1602     //Verify result
1603     for(int i=0; i<expected.size(); i++) {
1604       assertEquals(expected.get(i), actual.get(i));
1605     }
1606   }
1607 
1608   public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions()
1609   throws IOException {
1610     byte [] tableName = Bytes.toBytes("testtable");
1611     byte [] row1 = Bytes.toBytes("row1");
1612     byte [] fam1 = Bytes.toBytes("fam1");
1613     byte [][] families = {fam1};
1614     byte [] qf1 = Bytes.toBytes("qualifier1");
1615     byte [] qf2 = Bytes.toBytes("qualifier2");
1616 
1617     long ts1 = 1;
1618     long ts2 = ts1 + 1;
1619     long ts3 = ts1 + 2;
1620     long ts4 = ts1 + 3;
1621 
1622     //Setting up region
1623     String method = this.getName();
1624     initHRegion(tableName, method, families);
1625 
1626     //Putting data in Region
1627     KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
1628     KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
1629     KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
1630     KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
1631 
1632     KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
1633     KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
1634     KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
1635     KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
1636 
1637     Put put = null;
1638     put = new Put(row1);
1639     put.add(kv14);
1640     put.add(kv24);
1641     region.put(put);
1642     region.flushcache();
1643 
1644     put = new Put(row1);
1645     put.add(kv23);
1646     put.add(kv13);
1647     region.put(put);
1648     region.flushcache();
1649 
1650     put = new Put(row1);
1651     put.add(kv22);
1652     put.add(kv12);
1653     region.put(put);
1654     region.flushcache();
1655 
1656     put = new Put(row1);
1657     put.add(kv21);
1658     put.add(kv11);
1659     region.put(put);
1660 
1661     //Expected
1662     List<KeyValue> expected = new ArrayList<KeyValue>();
1663     expected.add(kv14);
1664     expected.add(kv13);
1665     expected.add(kv12);
1666     expected.add(kv24);
1667     expected.add(kv23);
1668     expected.add(kv22);
1669 
1670     Scan scan = new Scan(row1);
1671     scan.addColumn(fam1, qf1);
1672     scan.addColumn(fam1, qf2);
1673     int versions = 3;
1674     scan.setMaxVersions(versions);
1675     List<KeyValue> actual = new ArrayList<KeyValue>();
1676     InternalScanner scanner = region.getScanner(scan);
1677 
1678     boolean hasNext = scanner.next(actual);
1679     assertEquals(false, hasNext);
1680 
1681     //Verify result
1682     for(int i=0; i<expected.size(); i++) {
1683       assertEquals(expected.get(i), actual.get(i));
1684     }
1685   }
1686 
1687   public void testScanner_Wildcard_FromMemStore_EnforceVersions()
1688   throws IOException {
1689     byte [] tableName = Bytes.toBytes("testtable");
1690     byte [] row1 = Bytes.toBytes("row1");
1691     byte [] qf1 = Bytes.toBytes("qualifier1");
1692     byte [] qf2 = Bytes.toBytes("qualifier2");
1693     byte [] fam1 = Bytes.toBytes("fam1");
1694     byte [][] families = {fam1};
1695 
1696     long ts1 = System.currentTimeMillis();
1697     long ts2 = ts1 + 1;
1698     long ts3 = ts1 + 2;
1699 
1700     //Setting up region
1701     String method = this.getName();
1702     initHRegion(tableName, method, families);
1703 
1704     //Putting data in Region
1705     Put put = null;
1706     KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
1707     KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
1708     KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
1709 
1710     KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
1711     KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
1712     KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
1713 
1714     put = new Put(row1);
1715     put.add(kv13);
1716     put.add(kv12);
1717     put.add(kv11);
1718     put.add(kv23);
1719     put.add(kv22);
1720     put.add(kv21);
1721     region.put(put);
1722 
1723     //Expected
1724     List<KeyValue> expected = new ArrayList<KeyValue>();
1725     expected.add(kv13);
1726     expected.add(kv12);
1727     expected.add(kv23);
1728     expected.add(kv22);
1729 
1730     Scan scan = new Scan(row1);
1731     scan.addFamily(fam1);
1732     scan.setMaxVersions(MAX_VERSIONS);
1733     List<KeyValue> actual = new ArrayList<KeyValue>();
1734     InternalScanner scanner = region.getScanner(scan);
1735 
1736     boolean hasNext = scanner.next(actual);
1737     assertEquals(false, hasNext);
1738 
1739     //Verify result
1740     for(int i=0; i<expected.size(); i++) {
1741       assertEquals(expected.get(i), actual.get(i));
1742     }
1743   }
1744 
1745   public void testScanner_Wildcard_FromFilesOnly_EnforceVersions()
1746   throws IOException{
1747     byte [] tableName = Bytes.toBytes("testtable");
1748     byte [] row1 = Bytes.toBytes("row1");
1749     byte [] qf1 = Bytes.toBytes("qualifier1");
1750     byte [] qf2 = Bytes.toBytes("qualifier2");
1751     byte [] fam1 = Bytes.toBytes("fam1");
1752 
1753     long ts1 = 1; //System.currentTimeMillis();
1754     long ts2 = ts1 + 1;
1755     long ts3 = ts1 + 2;
1756 
1757     //Setting up region
1758     String method = this.getName();
1759     initHRegion(tableName, method, fam1);
1760 
1761     //Putting data in Region
1762     Put put = null;
1763     KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
1764     KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
1765     KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
1766 
1767     KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
1768     KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
1769     KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
1770 
1771     put = new Put(row1);
1772     put.add(kv13);
1773     put.add(kv12);
1774     put.add(kv11);
1775     put.add(kv23);
1776     put.add(kv22);
1777     put.add(kv21);
1778     region.put(put);
1779     region.flushcache();
1780 
1781     //Expected
1782     List<KeyValue> expected = new ArrayList<KeyValue>();
1783     expected.add(kv13);
1784     expected.add(kv12);
1785     expected.add(kv23);
1786     expected.add(kv22);
1787 
1788     Scan scan = new Scan(row1);
1789     scan.addFamily(fam1);
1790     scan.setMaxVersions(MAX_VERSIONS);
1791     List<KeyValue> actual = new ArrayList<KeyValue>();
1792     InternalScanner scanner = region.getScanner(scan);
1793 
1794     boolean hasNext = scanner.next(actual);
1795     assertEquals(false, hasNext);
1796 
1797     //Verify result
1798     for(int i=0; i<expected.size(); i++) {
1799       assertEquals(expected.get(i), actual.get(i));
1800     }
1801   }
1802 
1803   public void testScanner_StopRow1542() throws IOException {
1804     byte [] tableName = Bytes.toBytes("test_table");
1805     byte [] family = Bytes.toBytes("testFamily");
1806     initHRegion(tableName, getName(), family);
1807 
1808     byte [] row1 = Bytes.toBytes("row111");
1809     byte [] row2 = Bytes.toBytes("row222");
1810     byte [] row3 = Bytes.toBytes("row333");
1811     byte [] row4 = Bytes.toBytes("row444");
1812     byte [] row5 = Bytes.toBytes("row555");
1813 
1814     byte [] col1 = Bytes.toBytes("Pub111");
1815     byte [] col2 = Bytes.toBytes("Pub222");
1816 
1817 
1818     Put put = new Put(row1);
1819     put.add(family, col1, Bytes.toBytes(10L));
1820     region.put(put);
1821 
1822     put = new Put(row2);
1823     put.add(family, col1, Bytes.toBytes(15L));
1824     region.put(put);
1825 
1826     put = new Put(row3);
1827     put.add(family, col2, Bytes.toBytes(20L));
1828     region.put(put);
1829 
1830     put = new Put(row4);
1831     put.add(family, col2, Bytes.toBytes(30L));
1832     region.put(put);
1833 
1834     put = new Put(row5);
1835     put.add(family, col1, Bytes.toBytes(40L));
1836     region.put(put);
1837 
1838     Scan scan = new Scan(row3, row4);
1839     scan.setMaxVersions();
1840     scan.addColumn(family, col1);
1841     InternalScanner s = region.getScanner(scan);
1842 
1843     List<KeyValue> results = new ArrayList<KeyValue>();
1844     assertEquals(false, s.next(results));
1845     assertEquals(0, results.size());
1846   }
1847 
1848   public void testIncrementColumnValue_UpdatingInPlace() throws IOException {
1849     initHRegion(tableName, getName(), fam1);
1850 
1851     long value = 1L;
1852     long amount = 3L;
1853 
1854     Put put = new Put(row);
1855     put.add(fam1, qual1, Bytes.toBytes(value));
1856     region.put(put);
1857 
1858     long result = region.incrementColumnValue(row, fam1, qual1, amount, true);
1859 
1860     assertEquals(value+amount, result);
1861 
1862     Store store = region.getStore(fam1);
1863     // ICV removes any extra values floating around in there.
1864     assertEquals(1, store.memstore.kvset.size());
1865     assertTrue(store.memstore.snapshot.isEmpty());
1866 
1867     assertICV(row, fam1, qual1, value+amount);
1868   }
1869 
1870   public void testIncrementColumnValue_BumpSnapshot() throws IOException {
1871     ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
1872     EnvironmentEdgeManagerTestHelper.injectEdge(mee);
1873     initHRegion(tableName, getName(), fam1);
1874 
1875     long value = 42L;
1876     long incr = 44L;
1877 
1878     // first put something in kvset, then snapshot it.
1879     Put put = new Put(row);
1880     put.add(fam1, qual1, Bytes.toBytes(value));
1881     region.put(put);
1882 
1883     // get the store in question:
1884     Store s = region.getStore(fam1);
1885     s.snapshot(); //bam
1886 
1887     // now increment:
1888     long newVal = region.incrementColumnValue(row, fam1, qual1,
1889         incr, false);
1890 
1891     assertEquals(value+incr, newVal);
1892 
1893     // get both versions:
1894     Get get = new Get(row);
1895     get.setMaxVersions();
1896     get.addColumn(fam1,qual1);
1897 
1898     Result r = region.get(get, null);
1899     assertEquals(2, r.size());
1900     KeyValue first = r.raw()[0];
1901     KeyValue second = r.raw()[1];
1902 
1903     assertTrue("ICV failed to upgrade timestamp",
1904         first.getTimestamp() != second.getTimestamp());
1905   }
1906 
1907   public void testIncrementColumnValue_ConcurrentFlush() throws IOException {
1908     initHRegion(tableName, getName(), fam1);
1909 
1910     long value = 1L;
1911     long amount = 3L;
1912 
1913     Put put = new Put(row);
1914     put.add(fam1, qual1, Bytes.toBytes(value));
1915     region.put(put);
1916 
1917     // now increment during a flush
1918     Thread t = new Thread() {
1919       public void run() {
1920         try {
1921           region.flushcache();
1922         } catch (IOException e) {
1923           LOG.info("test ICV, got IOE during flushcache()");
1924         }
1925       }
1926     };
1927     t.start();
1928     long r = region.incrementColumnValue(row, fam1, qual1, amount, true);
1929     assertEquals(value+amount, r);
1930 
1931     // this also asserts there is only 1 KeyValue in the set.
1932     assertICV(row, fam1, qual1, value+amount);
1933   }
1934 
1935   public void testIncrementColumnValue_UpdatingInPlace_Negative()
1936     throws IOException {
1937     initHRegion(tableName, getName(), fam1);
1938 
1939     long value = 3L;
1940     long amount = -1L;
1941 
1942     Put put = new Put(row);
1943     put.add(fam1, qual1, Bytes.toBytes(value));
1944     region.put(put);
1945 
1946     long result = region.incrementColumnValue(row, fam1, qual1, amount, true);
1947     assertEquals(value+amount, result);
1948 
1949     assertICV(row, fam1, qual1, value+amount);
1950   }
1951 
1952   public void testIncrementColumnValue_AddingNew()
1953     throws IOException {
1954     initHRegion(tableName, getName(), fam1);
1955 
1956     long value = 1L;
1957     long amount = 3L;
1958 
1959     Put put = new Put(row);
1960     put.add(fam1, qual1, Bytes.toBytes(value));
1961     put.add(fam1, qual2, Bytes.toBytes(value));
1962     region.put(put);
1963 
1964     long result = region.incrementColumnValue(row, fam1, qual3, amount, true);
1965     assertEquals(amount, result);
1966 
1967     Get get = new Get(row);
1968     get.addColumn(fam1, qual3);
1969     Result rr = region.get(get, null);
1970     assertEquals(1, rr.size());
1971 
1972     // ensure none of the other cols were incremented.
1973     assertICV(row, fam1, qual1, value);
1974     assertICV(row, fam1, qual2, value);
1975     assertICV(row, fam1, qual3, amount);
1976   }
1977 
1978   public void testIncrementColumnValue_UpdatingFromSF() throws IOException {
1979     initHRegion(tableName, getName(), fam1);
1980 
1981     long value = 1L;
1982     long amount = 3L;
1983 
1984     Put put = new Put(row);
1985     put.add(fam1, qual1, Bytes.toBytes(value));
1986     put.add(fam1, qual2, Bytes.toBytes(value));
1987     region.put(put);
1988 
1989     // flush to disk.
1990     region.flushcache();
1991 
1992     Store store = region.getStore(fam1);
1993     assertEquals(0, store.memstore.kvset.size());
1994 
1995     long r = region.incrementColumnValue(row, fam1, qual1, amount, true);
1996     assertEquals(value+amount, r);
1997 
1998     assertICV(row, fam1, qual1, value+amount);
1999   }
2000 
2001   public void testIncrementColumnValue_AddingNewAfterSFCheck()
2002     throws IOException {
2003     initHRegion(tableName, getName(), fam1);
2004 
2005     long value = 1L;
2006     long amount = 3L;
2007 
2008     Put put = new Put(row);
2009     put.add(fam1, qual1, Bytes.toBytes(value));
2010     put.add(fam1, qual2, Bytes.toBytes(value));
2011     region.put(put);
2012     region.flushcache();
2013 
2014     Store store = region.getStore(fam1);
2015     assertEquals(0, store.memstore.kvset.size());
2016 
2017     long r = region.incrementColumnValue(row, fam1, qual3, amount, true);
2018     assertEquals(amount, r);
2019 
2020     assertICV(row, fam1, qual3, amount);
2021 
2022     region.flushcache();
2023 
2024     // ensure that this gets to disk.
2025     assertICV(row, fam1, qual3, amount);
2026   }
2027 
2028   private void assertICV(byte [] row,
2029                          byte [] familiy,
2030                          byte[] qualifier,
2031                          long amount) throws IOException {
2032     // run a get and see?
2033     Get get = new Get(row);
2034     get.addColumn(familiy, qualifier);
2035     Result result = region.get(get, null);
2036     assertEquals(1, result.size());
2037 
2038     KeyValue kv = result.raw()[0];
2039     long r = Bytes.toLong(kv.getValue());
2040     assertEquals(amount, r);
2041   }
2042 
2043 
2044 
2045   public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions()
2046   throws IOException {
2047     byte [] tableName = Bytes.toBytes("testtable");
2048     byte [] row1 = Bytes.toBytes("row1");
2049     byte [] fam1 = Bytes.toBytes("fam1");
2050     byte [] qf1 = Bytes.toBytes("qualifier1");
2051     byte [] qf2 = Bytes.toBytes("quateslifier2");
2052 
2053     long ts1 = 1;
2054     long ts2 = ts1 + 1;
2055     long ts3 = ts1 + 2;
2056     long ts4 = ts1 + 3;
2057 
2058     //Setting up region
2059     String method = this.getName();
2060     initHRegion(tableName, method, fam1);
2061 
2062     //Putting data in Region
2063     KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2064     KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2065     KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2066     KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2067 
2068     KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2069     KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2070     KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2071     KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2072 
2073     Put put = null;
2074     put = new Put(row1);
2075     put.add(kv14);
2076     put.add(kv24);
2077     region.put(put);
2078     region.flushcache();
2079 
2080     put = new Put(row1);
2081     put.add(kv23);
2082     put.add(kv13);
2083     region.put(put);
2084     region.flushcache();
2085 
2086     put = new Put(row1);
2087     put.add(kv22);
2088     put.add(kv12);
2089     region.put(put);
2090     region.flushcache();
2091 
2092     put = new Put(row1);
2093     put.add(kv21);
2094     put.add(kv11);
2095     region.put(put);
2096 
2097     //Expected
2098     List<KeyValue> expected = new ArrayList<KeyValue>();
2099     expected.add(kv14);
2100     expected.add(kv13);
2101     expected.add(kv12);
2102     expected.add(kv24);
2103     expected.add(kv23);
2104     expected.add(kv22);
2105 
2106     Scan scan = new Scan(row1);
2107     int versions = 3;
2108     scan.setMaxVersions(versions);
2109     List<KeyValue> actual = new ArrayList<KeyValue>();
2110     InternalScanner scanner = region.getScanner(scan);
2111 
2112     boolean hasNext = scanner.next(actual);
2113     assertEquals(false, hasNext);
2114 
2115     //Verify result
2116     for(int i=0; i<expected.size(); i++) {
2117       assertEquals(expected.get(i), actual.get(i));
2118     }
2119   }
2120 
2121   //////////////////////////////////////////////////////////////////////////////
2122   // Split test
2123   //////////////////////////////////////////////////////////////////////////////
2124   /**
2125    * Splits twice and verifies getting from each of the split regions.
2126    * @throws Exception
2127    */
2128   public void testBasicSplit() throws Exception {
2129     byte [] tableName = Bytes.toBytes("testtable");
2130     byte [][] families = {fam1, fam2, fam3};
2131 
2132     HBaseConfiguration hc = initSplit();
2133     //Setting up region
2134     String method = this.getName();
2135     initHRegion(tableName, method, hc, families);
2136 
2137     try {
2138       LOG.info("" + addContent(region, fam3));
2139       region.flushcache();
2140       byte [] splitRow = region.compactStores();
2141       assertNotNull(splitRow);
2142       LOG.info("SplitRow: " + Bytes.toString(splitRow));
2143       HRegion [] regions = split(region, splitRow);
2144       try {
2145         // Need to open the regions.
2146         // TODO: Add an 'open' to HRegion... don't do open by constructing
2147         // instance.
2148         for (int i = 0; i < regions.length; i++) {
2149           regions[i] = openClosedRegion(regions[i]);
2150         }
2151         // Assert can get rows out of new regions. Should be able to get first
2152         // row from first region and the midkey from second region.
2153         assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
2154         assertGet(regions[1], fam3, splitRow);
2155         // Test I can get scanner and that it starts at right place.
2156         assertScan(regions[0], fam3,
2157             Bytes.toBytes(START_KEY));
2158         assertScan(regions[1], fam3, splitRow);
2159         // Now prove can't split regions that have references.
2160         for (int i = 0; i < regions.length; i++) {
2161           // Add so much data to this region, we create a store file that is >
2162           // than one of our unsplitable references. it will.
2163           for (int j = 0; j < 2; j++) {
2164             addContent(regions[i], fam3);
2165           }
2166           addContent(regions[i], fam2);
2167           addContent(regions[i], fam1);
2168           regions[i].flushcache();
2169         }
2170 
2171         byte [][] midkeys = new byte [regions.length][];
2172         // To make regions splitable force compaction.
2173         for (int i = 0; i < regions.length; i++) {
2174           midkeys[i] = regions[i].compactStores();
2175         }
2176 
2177         TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
2178         // Split these two daughter regions so then I'll have 4 regions. Will
2179         // split because added data above.
2180         for (int i = 0; i < regions.length; i++) {
2181           HRegion[] rs = null;
2182           if (midkeys[i] != null) {
2183             rs = split(regions[i], midkeys[i]);
2184             for (int j = 0; j < rs.length; j++) {
2185               sortedMap.put(Bytes.toString(rs[j].getRegionName()),
2186                 openClosedRegion(rs[j]));
2187             }
2188           }
2189         }
2190         LOG.info("Made 4 regions");
2191         // The splits should have been even. Test I can get some arbitrary row
2192         // out of each.
2193         int interval = (LAST_CHAR - FIRST_CHAR) / 3;
2194         byte[] b = Bytes.toBytes(START_KEY);
2195         for (HRegion r : sortedMap.values()) {
2196           assertGet(r, fam3, b);
2197           b[0] += interval;
2198         }
2199       } finally {
2200         for (int i = 0; i < regions.length; i++) {
2201           try {
2202             regions[i].close();
2203           } catch (IOException e) {
2204             // Ignore.
2205           }
2206         }
2207       }
2208     } finally {
2209       if (region != null) {
2210         region.close();
2211         region.getLog().closeAndDelete();
2212       }
2213     }
2214   }
2215 
2216   public void testSplitRegion() throws IOException {
2217     byte [] tableName = Bytes.toBytes("testtable");
2218     byte [] qualifier = Bytes.toBytes("qualifier");
2219     HBaseConfiguration hc = initSplit();
2220     int numRows = 10;
2221     byte [][] families = {fam1, fam3};
2222 
2223     //Setting up region
2224     String method = this.getName();
2225     initHRegion(tableName, method, hc, families);
2226 
2227     //Put data in region
2228     int startRow = 100;
2229     putData(startRow, numRows, qualifier, families);
2230     int splitRow = startRow + numRows;
2231     putData(splitRow, numRows, qualifier, families);
2232     region.flushcache();
2233 
2234     HRegion [] regions = null;
2235     try {
2236       regions = region.splitRegion(Bytes.toBytes("" + splitRow));
2237       //Opening the regions returned.
2238       for (int i = 0; i < regions.length; i++) {
2239         regions[i] = openClosedRegion(regions[i]);
2240       }
2241       //Verifying that the region has been split
2242       assertEquals(2, regions.length);
2243 
2244       //Verifying that all data is still there and that data is in the right
2245       //place
2246       verifyData(regions[0], startRow, numRows, qualifier, families);
2247       verifyData(regions[1], splitRow, numRows, qualifier, families);
2248 
2249     } finally {
2250       if (region != null) {
2251         region.close();
2252         region.getLog().closeAndDelete();
2253       }
2254     }
2255   }
2256 
2257 
2258   /**
2259    * Flushes the cache in a thread while scanning. The tests verify that the
2260    * scan is coherent - e.g. the returned results are always of the same or
2261    * later update as the previous results.
2262    * @throws IOException scan / compact
2263    * @throws InterruptedException thread join
2264    */
2265   public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
2266     byte[] tableName = Bytes.toBytes("testFlushCacheWhileScanning");
2267     byte[] family = Bytes.toBytes("family");
2268     int numRows = 1000;
2269     int flushAndScanInterval = 10;
2270     int compactInterval = 10 * flushAndScanInterval;
2271 
2272     String method = "testFlushCacheWhileScanning";
2273     initHRegion(tableName,method, family);
2274     FlushThread flushThread = new FlushThread();
2275     flushThread.start();
2276 
2277     Scan scan = new Scan();
2278     scan.addFamily(family);
2279     scan.setFilter(new SingleColumnValueFilter(family, qual1,
2280       CompareFilter.CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(5L))));
2281 
2282     int expectedCount = 0;
2283     List<KeyValue> res = new ArrayList<KeyValue>();
2284 
2285     boolean toggle=true;
2286     for (long i = 0; i < numRows; i++) {
2287       Put put = new Put(Bytes.toBytes(i));
2288       put.add(family, qual1, Bytes.toBytes(i % 10));
2289       region.put(put);
2290 
2291       if (i != 0 && i % compactInterval == 0) {
2292         //System.out.println("iteration = " + i);
2293         region.compactStores(true);
2294       }
2295 
2296       if (i % 10 == 5L) {
2297         expectedCount++;
2298       }
2299 
2300       if (i != 0 && i % flushAndScanInterval == 0) {
2301         res.clear();
2302         InternalScanner scanner = region.getScanner(scan);
2303         if (toggle) {
2304           flushThread.flush();
2305         }
2306         while (scanner.next(res)) ;
2307         if (!toggle) {
2308           flushThread.flush();
2309         }
2310         assertEquals("i=" + i, expectedCount, res.size());
2311         toggle = !toggle;
2312       }
2313     }
2314 
2315     flushThread.done();
2316     flushThread.join();
2317     flushThread.checkNoError();
2318   }
2319 
2320   protected class FlushThread extends Thread {
2321     private volatile boolean done;
2322     private Throwable error = null;
2323 
2324     public void done() {
2325       done = true;
2326       synchronized (this) {
2327         interrupt();
2328       }
2329     }
2330 
2331     public void checkNoError() {
2332       if (error != null) {
2333         assertNull(error);
2334       }
2335     }
2336 
2337     @Override
2338     public void run() {
2339       done = false;
2340       while (!done) {
2341         synchronized (this) {
2342           try {
2343             wait();
2344           } catch (InterruptedException ignored) {
2345             if (done) {
2346               break;
2347             }
2348           }
2349         }
2350         try {
2351           region.flushcache();
2352         } catch (IOException e) {
2353           if (!done) {
2354             LOG.error("Error while flusing cache", e);
2355             error = e;
2356           }
2357           break;
2358         }
2359       }
2360 
2361     }
2362 
2363     public void flush() {
2364       synchronized (this) {
2365         notify();
2366       }
2367 
2368     }
2369   }
2370 
2371   /**
2372    * Writes very wide records and scans for the latest every time..
2373    * Flushes and compacts the region every now and then to keep things
2374    * realistic.
2375    *
2376    * @throws IOException          by flush / scan / compaction
2377    * @throws InterruptedException when joining threads
2378    */
2379   public void testWritesWhileScanning()
2380     throws IOException, InterruptedException {
2381     byte[] tableName = Bytes.toBytes("testWritesWhileScanning");
2382     int testCount = 100;
2383     int numRows = 1;
2384     int numFamilies = 10;
2385     int numQualifiers = 100;
2386     int flushInterval = 7;
2387     int compactInterval = 5 * flushInterval;
2388     byte[][] families = new byte[numFamilies][];
2389     for (int i = 0; i < numFamilies; i++) {
2390       families[i] = Bytes.toBytes("family" + i);
2391     }
2392     byte[][] qualifiers = new byte[numQualifiers][];
2393     for (int i = 0; i < numQualifiers; i++) {
2394       qualifiers[i] = Bytes.toBytes("qual" + i);
2395     }
2396 
2397     String method = "testWritesWhileScanning";
2398     initHRegion(tableName, method, families);
2399     PutThread putThread = new PutThread(numRows, families, qualifiers);
2400     putThread.start();
2401     putThread.waitForFirstPut();
2402 
2403     FlushThread flushThread = new FlushThread();
2404     flushThread.start();
2405 
2406     Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
2407 //    scan.setFilter(new RowFilter(CompareFilter.CompareOp.EQUAL,
2408 //      new BinaryComparator(Bytes.toBytes("row0"))));
2409 
2410     int expectedCount = numFamilies * numQualifiers;
2411     List<KeyValue> res = new ArrayList<KeyValue>();
2412 
2413     long prevTimestamp = 0L;
2414     for (int i = 0; i < testCount; i++) {
2415 
2416       if (i != 0 && i % compactInterval == 0) {
2417         region.compactStores(true);
2418       }
2419 
2420       if (i != 0 && i % flushInterval == 0) {
2421         //System.out.println("flush scan iteration = " + i);
2422         flushThread.flush();
2423       }
2424 
2425       boolean previousEmpty = res.isEmpty();
2426       res.clear();
2427       InternalScanner scanner = region.getScanner(scan);
2428       while (scanner.next(res)) ;
2429       if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
2430         assertEquals("i=" + i, expectedCount, res.size());
2431         long timestamp = res.get(0).getTimestamp();
2432         assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
2433             timestamp >= prevTimestamp);
2434         prevTimestamp = timestamp;
2435       }
2436     }
2437 
2438     putThread.done();
2439 
2440     region.flushcache();
2441 
2442     putThread.join();
2443     putThread.checkNoError();
2444 
2445     flushThread.done();
2446     flushThread.join();
2447     flushThread.checkNoError();
2448   }
2449 
2450   protected class PutThread extends Thread {
2451     private volatile boolean done;
2452     private volatile int numPutsFinished = 0;
2453 
2454     private Throwable error = null;
2455     private int numRows;
2456     private byte[][] families;
2457     private byte[][] qualifiers;
2458 
2459     private PutThread(int numRows, byte[][] families,
2460       byte[][] qualifiers) {
2461       this.numRows = numRows;
2462       this.families = families;
2463       this.qualifiers = qualifiers;
2464     }
2465 
2466     /**
2467      * Block until this thread has put at least one row.
2468      */
2469     public void waitForFirstPut() throws InterruptedException {
2470       // wait until put thread actually puts some data
2471       while (numPutsFinished == 0) {
2472         checkNoError();
2473         Thread.sleep(50);
2474       }
2475     }
2476 
2477     public void done() {
2478       done = true;
2479       synchronized (this) {
2480         interrupt();
2481       }
2482     }
2483 
2484     public void checkNoError() {
2485       if (error != null) {
2486         assertNull(error);
2487       }
2488     }
2489 
2490     @Override
2491     public void run() {
2492       done = false;
2493       while (!done) {
2494         try {
2495           for (int r = 0; r < numRows; r++) {
2496             byte[] row = Bytes.toBytes("row" + r);
2497             Put put = new Put(row);
2498             for (byte[] family : families) {
2499               for (byte[] qualifier : qualifiers) {
2500                 put.add(family, qualifier, (long) numPutsFinished,
2501                     Bytes.toBytes(numPutsFinished));
2502               }
2503             }
2504 //            System.out.println("Putting of kvsetsize=" + put.size());
2505             region.put(put);
2506             numPutsFinished++;
2507             if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
2508               System.out.println("put iteration = " + numPutsFinished);
2509               Delete delete = new Delete(row, (long)numPutsFinished-30, null);
2510               region.delete(delete, null, true);
2511             }
2512             numPutsFinished++;
2513           }
2514         } catch (IOException e) {
2515           LOG.error("error while putting records", e);
2516           error = e;
2517           break;
2518         }
2519       }
2520 
2521     }
2522 
2523   }
2524 
2525 
2526   /**
2527    * Writes very wide records and gets the latest row every time..
2528    * Flushes and compacts the region every now and then to keep things
2529    * realistic.
2530    *
2531    * @throws IOException          by flush / scan / compaction
2532    * @throws InterruptedException when joining threads
2533    */
2534   public void testWritesWhileGetting()
2535     throws IOException, InterruptedException {
2536     byte[] tableName = Bytes.toBytes("testWritesWhileScanning");
2537     int testCount = 100;
2538     int numRows = 1;
2539     int numFamilies = 10;
2540     int numQualifiers = 100;
2541     int flushInterval = 10;
2542     int compactInterval = 10 * flushInterval;
2543     byte[][] families = new byte[numFamilies][];
2544     for (int i = 0; i < numFamilies; i++) {
2545       families[i] = Bytes.toBytes("family" + i);
2546     }
2547     byte[][] qualifiers = new byte[numQualifiers][];
2548     for (int i = 0; i < numQualifiers; i++) {
2549       qualifiers[i] = Bytes.toBytes("qual" + i);
2550     }
2551 
2552     String method = "testWritesWhileScanning";
2553     initHRegion(tableName, method, families);
2554     PutThread putThread = new PutThread(numRows, families, qualifiers);
2555     putThread.start();
2556     putThread.waitForFirstPut();
2557 
2558     FlushThread flushThread = new FlushThread();
2559     flushThread.start();
2560 
2561     Get get = new Get(Bytes.toBytes("row0"));
2562     Result result = null;
2563 
2564     int expectedCount = numFamilies * numQualifiers;
2565 
2566     long prevTimestamp = 0L;
2567     for (int i = 0; i < testCount; i++) {
2568 
2569       if (i != 0 && i % compactInterval == 0) {
2570         region.compactStores(true);
2571       }
2572 
2573       if (i != 0 && i % flushInterval == 0) {
2574         //System.out.println("iteration = " + i);
2575         flushThread.flush();
2576       }
2577 
2578       boolean previousEmpty = result == null || result.isEmpty();
2579       result = region.get(get, null);
2580       if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
2581         assertEquals("i=" + i, expectedCount, result.size());
2582         // TODO this was removed, now what dangit?!
2583         // search looking for the qualifier in question?
2584         long timestamp = 0;
2585         for (KeyValue kv : result.sorted()) {
2586           if (Bytes.equals(kv.getFamily(), families[0])
2587             && Bytes.equals(kv.getQualifier(), qualifiers[0])) {
2588             timestamp = kv.getTimestamp();
2589           }
2590         }
2591         assertTrue(timestamp >= prevTimestamp);
2592         prevTimestamp = timestamp;
2593 
2594         byte [] gotValue = null;
2595         for (KeyValue kv : result.raw()) {
2596           byte [] thisValue = kv.getValue();
2597           if (gotValue != null) {
2598             assertEquals(gotValue, thisValue);
2599           }
2600           gotValue = thisValue;
2601         }
2602       }
2603     }
2604 
2605     putThread.done();
2606 
2607     region.flushcache();
2608 
2609     putThread.join();
2610     putThread.checkNoError();
2611 
2612     flushThread.done();
2613     flushThread.join();
2614     flushThread.checkNoError();
2615   }
2616 
2617 
2618   public void testIndexesScanWithOneDeletedRow() throws IOException {
2619     byte[] tableName = Bytes.toBytes("testIndexesScanWithOneDeletedRow");
2620     byte[] family = Bytes.toBytes("family");
2621 
2622     //Setting up region
2623     String method = "testIndexesScanWithOneDeletedRow";
2624     initHRegion(tableName, method, new HBaseConfiguration(), family);
2625 
2626     Put put = new Put(Bytes.toBytes(1L));
2627     put.add(family, qual1, 1L, Bytes.toBytes(1L));
2628     region.put(put);
2629 
2630     region.flushcache();
2631 
2632     Delete delete = new Delete(Bytes.toBytes(1L), 1L, null);
2633     //delete.deleteColumn(family, qual1);
2634     region.delete(delete, null, true);
2635 
2636     put = new Put(Bytes.toBytes(2L));
2637     put.add(family, qual1, 2L, Bytes.toBytes(2L));
2638     region.put(put);
2639 
2640     Scan idxScan = new Scan();
2641     idxScan.addFamily(family);
2642     idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,
2643       Arrays.<Filter>asList(new SingleColumnValueFilter(family, qual1,
2644         CompareFilter.CompareOp.GREATER_OR_EQUAL,
2645         new BinaryComparator(Bytes.toBytes(0L))),
2646         new SingleColumnValueFilter(family, qual1,
2647           CompareFilter.CompareOp.LESS_OR_EQUAL,
2648           new BinaryComparator(Bytes.toBytes(3L)))
2649       )));
2650     InternalScanner scanner = region.getScanner(idxScan);
2651     List<KeyValue> res = new ArrayList<KeyValue>();
2652 
2653     //long start = System.nanoTime();
2654     while (scanner.next(res)) ;
2655     //long end = System.nanoTime();
2656     //System.out.println("memStoreEmpty=" + memStoreEmpty + ", time=" + (end - start)/1000000D);
2657     assertEquals(1L, res.size());
2658 
2659   }
2660 
2661   //////////////////////////////////////////////////////////////////////////////
2662   // Bloom filter test
2663   //////////////////////////////////////////////////////////////////////////////
2664 
2665   public void testAllColumnsWithBloomFilter() throws IOException {
2666     byte [] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
2667     byte [] FAMILY = Bytes.toBytes("family");
2668 
2669     //Create table
2670     HColumnDescriptor hcd = new HColumnDescriptor(FAMILY, Integer.MAX_VALUE,
2671         HColumnDescriptor.DEFAULT_COMPRESSION,
2672         HColumnDescriptor.DEFAULT_IN_MEMORY,
2673         HColumnDescriptor.DEFAULT_BLOCKCACHE,
2674         Integer.MAX_VALUE, HColumnDescriptor.DEFAULT_TTL,
2675         "rowcol",
2676         HColumnDescriptor.DEFAULT_REPLICATION_SCOPE);
2677     HTableDescriptor htd = new HTableDescriptor(TABLE);
2678     htd.addFamily(hcd);
2679     HRegionInfo info = new HRegionInfo(htd, null, null, false);
2680     Path path = new Path(DIR + "testAllColumnsWithBloomFilter");
2681     region = HRegion.createHRegion(info, path, conf);
2682 
2683     // For row:0, col:0: insert versions 1 through 5.
2684     byte row[] = Bytes.toBytes("row:" + 0);
2685     byte column[] = Bytes.toBytes("column:" + 0);
2686     Put put = new Put(row);
2687     for (long idx = 1; idx <= 4; idx++) {
2688       put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
2689     }
2690     region.put(put);
2691 
2692     //Flush
2693     region.flushcache();
2694 
2695     //Get rows
2696     Get get = new Get(row);
2697     get.setMaxVersions();
2698     KeyValue[] kvs = region.get(get, null).raw();
2699 
2700     //Check if rows are correct
2701     assertEquals(4, kvs.length);
2702     checkOneCell(kvs[0], FAMILY, 0, 0, 4);
2703     checkOneCell(kvs[1], FAMILY, 0, 0, 3);
2704     checkOneCell(kvs[2], FAMILY, 0, 0, 2);
2705     checkOneCell(kvs[3], FAMILY, 0, 0, 1);
2706   }
2707 
2708   private void putData(int startRow, int numRows, byte [] qf,
2709       byte [] ...families)
2710   throws IOException {
2711     for(int i=startRow; i<startRow+numRows; i++) {
2712       Put put = new Put(Bytes.toBytes("" + i));
2713       for(byte [] family : families) {
2714         put.add(family, qf, null);
2715       }
2716       region.put(put);
2717     }
2718   }
2719 
2720   private void verifyData(HRegion newReg, int startRow, int numRows, byte [] qf,
2721       byte [] ... families)
2722   throws IOException {
2723     for(int i=startRow; i<startRow + numRows; i++) {
2724       byte [] row = Bytes.toBytes("" + i);
2725       Get get = new Get(row);
2726       for(byte [] family : families) {
2727         get.addColumn(family, qf);
2728       }
2729       Result result = newReg.get(get, null);
2730       KeyValue [] raw = result.sorted();
2731       assertEquals(families.length, result.size());
2732       for(int j=0; j<families.length; j++) {
2733         assertEquals(0, Bytes.compareTo(row, raw[j].getRow()));
2734         assertEquals(0, Bytes.compareTo(families[j], raw[j].getFamily()));
2735         assertEquals(0, Bytes.compareTo(qf, raw[j].getQualifier()));
2736       }
2737     }
2738   }
2739 
2740   private void assertGet(final HRegion r, final byte [] family, final byte [] k)
2741   throws IOException {
2742     // Now I have k, get values out and assert they are as expected.
2743     Get get = new Get(k).addFamily(family).setMaxVersions();
2744     KeyValue [] results = r.get(get, null).raw();
2745     for (int j = 0; j < results.length; j++) {
2746       byte [] tmp = results[j].getValue();
2747       // Row should be equal to value every time.
2748       assertTrue(Bytes.equals(k, tmp));
2749     }
2750   }
2751 
2752   /*
2753    * Assert first value in the passed region is <code>firstValue</code>.
2754    * @param r
2755    * @param fs
2756    * @param firstValue
2757    * @throws IOException
2758    */
2759   private void assertScan(final HRegion r, final byte [] fs,
2760       final byte [] firstValue)
2761   throws IOException {
2762     byte [][] families = {fs};
2763     Scan scan = new Scan();
2764     for (int i = 0; i < families.length; i++) scan.addFamily(families[i]);
2765     InternalScanner s = r.getScanner(scan);
2766     try {
2767       List<KeyValue> curVals = new ArrayList<KeyValue>();
2768       boolean first = true;
2769       OUTER_LOOP: while(s.next(curVals)) {
2770         for (KeyValue kv: curVals) {
2771           byte [] val = kv.getValue();
2772           byte [] curval = val;
2773           if (first) {
2774             first = false;
2775             assertTrue(Bytes.compareTo(curval, firstValue) == 0);
2776           } else {
2777             // Not asserting anything.  Might as well break.
2778             break OUTER_LOOP;
2779           }
2780         }
2781       }
2782     } finally {
2783       s.close();
2784     }
2785   }
2786 
2787   protected HRegion [] split(final HRegion r, final byte [] splitRow)
2788   throws IOException {
2789     // Assert can get mid key from passed region.
2790     assertGet(r, fam3, splitRow);
2791     HRegion [] regions = r.splitRegion(splitRow);
2792     assertEquals(regions.length, 2);
2793     return regions;
2794   }
2795 
2796   private HBaseConfiguration initSplit() {
2797     HBaseConfiguration conf = new HBaseConfiguration();
2798     // Always compact if there is more than one store file.
2799     conf.setInt("hbase.hstore.compactionThreshold", 2);
2800 
2801     // Make lease timeout longer, lease checks less frequent
2802     conf.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
2803 
2804     conf.setInt(HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, 10 * 1000);
2805 
2806     // Increase the amount of time between client retries
2807     conf.setLong("hbase.client.pause", 15 * 1000);
2808 
2809     // This size should make it so we always split using the addContent
2810     // below.  After adding all data, the first region is 1.3M
2811     conf.setLong("hbase.hregion.max.filesize", 1024 * 128);
2812     return conf;
2813   }
2814 
2815   private void initHRegion (byte [] tableName, String callingMethod,
2816     byte[] ... families)
2817   throws IOException {
2818     initHRegion(tableName, callingMethod, new HBaseConfiguration(), families);
2819   }
2820 
2821   private void initHRegion (byte [] tableName, String callingMethod,
2822     HBaseConfiguration conf, byte [] ... families)
2823   throws IOException{
2824     HTableDescriptor htd = new HTableDescriptor(tableName);
2825     for(byte [] family : families) {
2826       htd.addFamily(new HColumnDescriptor(family));
2827     }
2828     HRegionInfo info = new HRegionInfo(htd, null, null, false);
2829     Path path = new Path(DIR + callingMethod);
2830     region = HRegion.createHRegion(info, path, conf);
2831   }
2832 
2833   /**
2834    * Assert that the passed in KeyValue has expected contents for the
2835    * specified row, column & timestamp.
2836    */
2837   private void checkOneCell(KeyValue kv, byte[] cf,
2838                              int rowIdx, int colIdx, long ts) {
2839     String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
2840     assertEquals("Row mismatch which checking: " + ctx,
2841                  "row:"+ rowIdx, Bytes.toString(kv.getRow()));
2842     assertEquals("ColumnFamily mismatch while checking: " + ctx,
2843                  Bytes.toString(cf), Bytes.toString(kv.getFamily()));
2844     assertEquals("Column qualifier mismatch while checking: " + ctx,
2845                  "column:" + colIdx, Bytes.toString(kv.getQualifier()));
2846     assertEquals("Timestamp mismatch while checking: " + ctx,
2847                  ts, kv.getTimestamp());
2848     assertEquals("Value mismatch while checking: " + ctx,
2849                  "value-version-" + ts, Bytes.toString(kv.getValue()));
2850   }
2851 
2852 }