View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  
22  import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23  import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24  import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25  import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29  import static org.junit.Assert.assertArrayEquals;
30  import static org.junit.Assert.assertEquals;
31  import static org.junit.Assert.assertFalse;
32  import static org.junit.Assert.assertNotNull;
33  import static org.junit.Assert.assertNull;
34  import static org.junit.Assert.assertTrue;
35  import static org.junit.Assert.fail;
36  import static org.mockito.Matchers.any;
37  import static org.mockito.Matchers.anyLong;
38  import static org.mockito.Matchers.eq;
39  import static org.mockito.Mockito.never;
40  import static org.mockito.Mockito.spy;
41  import static org.mockito.Mockito.times;
42  import static org.mockito.Mockito.verify;
43  
44  import java.io.IOException;
45  import java.io.InterruptedIOException;
46  import java.util.ArrayList;
47  import java.util.Arrays;
48  import java.util.Collection;
49  import java.util.List;
50  import java.util.Map;
51  import java.util.NavigableMap;
52  import java.util.TreeMap;
53  import java.util.UUID;
54  import java.util.concurrent.atomic.AtomicBoolean;
55  import java.util.concurrent.atomic.AtomicInteger;
56  import java.util.concurrent.atomic.AtomicReference;
57  
58  import org.apache.commons.logging.Log;
59  import org.apache.commons.logging.LogFactory;
60  import org.apache.hadoop.conf.Configuration;
61  import org.apache.hadoop.fs.FSDataOutputStream;
62  import org.apache.hadoop.fs.FileStatus;
63  import org.apache.hadoop.fs.FileSystem;
64  import org.apache.hadoop.fs.Path;
65  import org.apache.hadoop.hbase.Cell;
66  import org.apache.hadoop.hbase.CellComparator;
67  import org.apache.hadoop.hbase.CellUtil;
68  import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
69  import org.apache.hadoop.hbase.HBaseTestCase;
70  import org.apache.hadoop.hbase.HBaseTestingUtility;
71  import org.apache.hadoop.hbase.HColumnDescriptor;
72  import org.apache.hadoop.hbase.HConstants;
73  import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
74  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
75  import org.apache.hadoop.hbase.HRegionInfo;
76  import org.apache.hadoop.hbase.HTableDescriptor;
77  import org.apache.hadoop.hbase.KeyValue;
78  import org.apache.hadoop.hbase.MediumTests;
79  import org.apache.hadoop.hbase.MiniHBaseCluster;
80  import org.apache.hadoop.hbase.MultithreadedTestUtil;
81  import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
82  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
83  import org.apache.hadoop.hbase.NotServingRegionException;
84  import org.apache.hadoop.hbase.TableName;
85  import org.apache.hadoop.hbase.Waiter;
86  import org.apache.hadoop.hbase.client.Append;
87  import org.apache.hadoop.hbase.client.Delete;
88  import org.apache.hadoop.hbase.client.Durability;
89  import org.apache.hadoop.hbase.client.Get;
90  import org.apache.hadoop.hbase.client.HTable;
91  import org.apache.hadoop.hbase.client.Increment;
92  import org.apache.hadoop.hbase.client.Put;
93  import org.apache.hadoop.hbase.client.Result;
94  import org.apache.hadoop.hbase.client.Scan;
95  import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
96  import org.apache.hadoop.hbase.filter.BinaryComparator;
97  import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
98  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
99  import org.apache.hadoop.hbase.filter.Filter;
100 import org.apache.hadoop.hbase.filter.FilterBase;
101 import org.apache.hadoop.hbase.filter.FilterList;
102 import org.apache.hadoop.hbase.filter.NullComparator;
103 import org.apache.hadoop.hbase.filter.PrefixFilter;
104 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
105 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
106 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
107 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
108 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
109 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
110 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
111 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
112 import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
113 import org.apache.hadoop.hbase.regionserver.wal.HLog;
114 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
115 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
116 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
117 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
118 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
119 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
120 import org.apache.hadoop.hbase.util.Bytes;
121 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
122 import org.apache.hadoop.hbase.util.FSUtils;
123 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
124 import org.apache.hadoop.hbase.util.PairOfSameType;
125 import org.apache.hadoop.hbase.util.Threads;
126 import org.junit.After;
127 import org.junit.Assert;
128 import org.junit.Before;
129 import org.junit.Rule;
130 import org.junit.Test;
131 import org.junit.experimental.categories.Category;
132 import org.junit.rules.TestName;
133 import org.mockito.Mockito;
134 
135 import com.google.common.collect.Lists;
136 
137 /**
138  * Basic stand-alone testing of HRegion.
139  * 
140  * A lot of the meta information for an HRegion now lives inside other HRegions
141  * or in the HBaseMaster, so only basic testing is possible.
142  */
143 @Category(MediumTests.class)
144 @SuppressWarnings("deprecation")
145 public class TestHRegion {
146   // Do not spin up clusters in here. If you need to spin up a cluster, do it
147   // over in TestHRegionOnCluster.
148   static final Log LOG = LogFactory.getLog(TestHRegion.class);
149   @Rule public TestName name = new TestName();
150 
151   private static final String COLUMN_FAMILY = "MyCF";
152 
153   HRegion region = null;
154   private static HBaseTestingUtility TEST_UTIL; // do not run unit tests in parallel 
155   public static Configuration conf ;
156   private String DIR;
157   private static FileSystem fs;
158   private final int MAX_VERSIONS = 2;
159 
160   // Test names
161   protected byte[] tableName;
162   protected String method;
163   protected final byte[] qual1 = Bytes.toBytes("qual1");
164   protected final byte[] qual2 = Bytes.toBytes("qual2");
165   protected final byte[] qual3 = Bytes.toBytes("qual3");
166   protected final byte[] value1 = Bytes.toBytes("value1");
167   protected final byte[] value2 = Bytes.toBytes("value2");
168   protected final byte[] row = Bytes.toBytes("rowA");
169   protected final byte[] row2 = Bytes.toBytes("rowB");
170 
171   protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
172       .getInstance(MetricsAssertHelper.class);
173 
174   @Before
175   public void setup() throws IOException {
176     this.TEST_UTIL = HBaseTestingUtility.createLocalHTU();
177     this.fs = TEST_UTIL.getTestFileSystem();
178     this.conf = TEST_UTIL.getConfiguration();
179     this.DIR = TEST_UTIL.getDataTestDir("TestHRegion").toString();
180     method = name.getMethodName();
181     tableName = Bytes.toBytes(name.getMethodName());
182   }
183 
184   @After
185   public void tearDown() throws Exception {
186     EnvironmentEdgeManagerTestHelper.reset();
187   }
188 
189   String getName() {
190     return name.getMethodName();
191   }
192   
193   // ////////////////////////////////////////////////////////////////////////////
194   // New tests that doesn't spin up a mini cluster but rather just test the
195   // individual code pieces in the HRegion. Putting files locally in
196   // /tmp/testtable
197   // ////////////////////////////////////////////////////////////////////////////
198 
199   @Test
200   public void testCompactionAffectedByScanners() throws Exception {
201     byte[] family = Bytes.toBytes("family");
202     this.region = initHRegion(tableName, method, conf, family);
203 
204     Put put = new Put(Bytes.toBytes("r1"));
205     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
206     region.put(put);
207     region.flushcache();
208 
209     Scan scan = new Scan();
210     scan.setMaxVersions(3);
211     // open the first scanner
212     RegionScanner scanner1 = region.getScanner(scan);
213 
214     Delete delete = new Delete(Bytes.toBytes("r1"));
215     region.delete(delete);
216     region.flushcache();
217 
218     // open the second scanner
219     RegionScanner scanner2 = region.getScanner(scan);
220 
221     List<Cell> results = new ArrayList<Cell>();
222 
223     System.out.println("Smallest read point:" + region.getSmallestReadPoint());
224 
225     // make a major compaction
226     region.compactStores(true);
227 
228     // open the third scanner
229     RegionScanner scanner3 = region.getScanner(scan);
230 
231     // get data from scanner 1, 2, 3 after major compaction
232     scanner1.next(results);
233     System.out.println(results);
234     assertEquals(1, results.size());
235 
236     results.clear();
237     scanner2.next(results);
238     System.out.println(results);
239     assertEquals(0, results.size());
240 
241     results.clear();
242     scanner3.next(results);
243     System.out.println(results);
244     assertEquals(0, results.size());
245   }
246 
247   @Test
248   public void testToShowNPEOnRegionScannerReseek() throws Exception {
249     byte[] family = Bytes.toBytes("family");
250     this.region = initHRegion(tableName, method, conf, family);
251 
252     Put put = new Put(Bytes.toBytes("r1"));
253     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
254     region.put(put);
255     put = new Put(Bytes.toBytes("r2"));
256     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
257     region.put(put);
258     region.flushcache();
259 
260     Scan scan = new Scan();
261     scan.setMaxVersions(3);
262     // open the first scanner
263     RegionScanner scanner1 = region.getScanner(scan);
264 
265     System.out.println("Smallest read point:" + region.getSmallestReadPoint());
266 
267     region.compactStores(true);
268 
269     scanner1.reseek(Bytes.toBytes("r2"));
270     List<Cell> results = new ArrayList<Cell>();
271     scanner1.next(results);
272     Cell keyValue = results.get(0);
273     Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
274     scanner1.close();
275   }
276 
277   @Test
278   public void testSkipRecoveredEditsReplay() throws Exception {
279     String method = "testSkipRecoveredEditsReplay";
280     TableName tableName = TableName.valueOf(method);
281     byte[] family = Bytes.toBytes("family");
282     this.region = initHRegion(tableName, method, conf, family);
283     try {
284       Path regiondir = region.getRegionFileSystem().getRegionDir();
285       FileSystem fs = region.getRegionFileSystem().getFileSystem();
286       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
287 
288       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
289 
290       long maxSeqId = 1050;
291       long minSeqId = 1000;
292 
293       for (long i = minSeqId; i <= maxSeqId; i += 10) {
294         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
295         fs.create(recoveredEdits);
296         HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, conf);
297 
298         long time = System.nanoTime();
299         WALEdit edit = new WALEdit();
300         edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
301             .toBytes(i)));
302         writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
303             HConstants.DEFAULT_CLUSTER_ID), edit));
304 
305         writer.close();
306       }
307       MonitoredTask status = TaskMonitor.get().createStatus(method);
308       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
309       for (Store store : region.getStores().values()) {
310         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
311       }
312       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
313       assertEquals(maxSeqId, seqId);
314       Get get = new Get(row);
315       Result result = region.get(get);
316       for (long i = minSeqId; i <= maxSeqId; i += 10) {
317         List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
318         assertEquals(1, kvs.size());
319         assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
320       }
321     } finally {
322       HRegion.closeHRegion(this.region);
323       this.region = null;
324     }
325   }
326 
327   @Test
328   public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
329     String method = "testSkipRecoveredEditsReplaySomeIgnored";
330     TableName tableName = TableName.valueOf(method);
331     byte[] family = Bytes.toBytes("family");
332     this.region = initHRegion(tableName, method, conf, family);
333     try {
334       Path regiondir = region.getRegionFileSystem().getRegionDir();
335       FileSystem fs = region.getRegionFileSystem().getFileSystem();
336       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
337 
338       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
339 
340       long maxSeqId = 1050;
341       long minSeqId = 1000;
342 
343       for (long i = minSeqId; i <= maxSeqId; i += 10) {
344         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
345         fs.create(recoveredEdits);
346         HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, conf);
347 
348         long time = System.nanoTime();
349         WALEdit edit = new WALEdit();
350         edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
351             .toBytes(i)));
352         writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
353             HConstants.DEFAULT_CLUSTER_ID), edit));
354 
355         writer.close();
356       }
357       long recoverSeqId = 1030;
358       MonitoredTask status = TaskMonitor.get().createStatus(method);
359       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
360       for (Store store : region.getStores().values()) {
361         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
362       }
363       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
364       assertEquals(maxSeqId, seqId);
365       Get get = new Get(row);
366       Result result = region.get(get);
367       for (long i = minSeqId; i <= maxSeqId; i += 10) {
368         List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
369         if (i < recoverSeqId) {
370           assertEquals(0, kvs.size());
371         } else {
372           assertEquals(1, kvs.size());
373           assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
374         }
375       }
376     } finally {
377       HRegion.closeHRegion(this.region);
378       this.region = null;
379     }
380   }
381 
382   @Test
383   public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
384     byte[] family = Bytes.toBytes("family");
385     this.region = initHRegion(tableName, method, conf, family);
386     try {
387       Path regiondir = region.getRegionFileSystem().getRegionDir();
388       FileSystem fs = region.getRegionFileSystem().getFileSystem();
389 
390       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
391       for (int i = 1000; i < 1050; i += 10) {
392         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
393         FSDataOutputStream dos = fs.create(recoveredEdits);
394         dos.writeInt(i);
395         dos.close();
396       }
397       long minSeqId = 2000;
398       Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
399       FSDataOutputStream dos = fs.create(recoveredEdits);
400       dos.close();
401 
402       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
403       for (Store store : region.getStores().values()) {
404         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
405       }
406       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
407       assertEquals(minSeqId, seqId);
408     } finally {
409       HRegion.closeHRegion(this.region);
410       this.region = null;
411     }
412   }
413 
414   @Test
415   public void testRecoveredEditsReplayCompaction() throws Exception {
416     String method = name.getMethodName();
417     TableName tableName = TableName.valueOf(method);
418     byte[] family = Bytes.toBytes("family");
419     this.region = initHRegion(tableName, method, conf, family);
420     try {
421       Path regiondir = region.getRegionFileSystem().getRegionDir();
422       FileSystem fs = region.getRegionFileSystem().getFileSystem();
423       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
424 
425       long maxSeqId = 3;
426       long minSeqId = 0;
427 
428       for (long i = minSeqId; i < maxSeqId; i++) {
429         Put put = new Put(Bytes.toBytes(i));
430         put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
431         region.put(put);
432         region.flushcache();
433       }
434 
435       // this will create a region with 3 files
436       assertEquals(3, region.getStore(family).getStorefilesCount());
437       List<Path> storeFiles = new ArrayList<Path>(3);
438       for (StoreFile sf : region.getStore(family).getStorefiles()) {
439         storeFiles.add(sf.getPath());
440       }
441 
442       // disable compaction completion
443       conf.setBoolean("hbase.hstore.compaction.complete", false);
444       region.compactStores();
445 
446       // ensure that nothing changed
447       assertEquals(3, region.getStore(family).getStorefilesCount());
448 
449       // now find the compacted file, and manually add it to the recovered edits
450       Path tmpDir = region.getRegionFileSystem().getTempDir();
451       FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
452       String errorMsg = "Expected to find 1 file in the region temp directory "
453           + "from the compaction, could not find any";
454       assertNotNull(errorMsg, files);
455       assertEquals(errorMsg, 1, files.length);
456       // move the file inside region dir
457       Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
458           files[0].getPath());
459 
460       CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
461           .getRegionInfo(), family, storeFiles, Lists.newArrayList(newFile), region
462           .getRegionFileSystem().getStoreDir(Bytes.toString(family)));
463 
464       HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
465           this.region.getRegionInfo(), compactionDescriptor);
466 
467       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
468 
469       Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
470       fs.create(recoveredEdits);
471       HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, conf);
472 
473       long time = System.nanoTime();
474 
475       writer.append(new HLog.Entry(new HLogKey(regionName, tableName, 10, time,
476           HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(compactionDescriptor)));
477       writer.close();
478 
479       // close the region now, and reopen again
480       HTableDescriptor htd = region.getTableDesc();
481       HRegionInfo info = region.getRegionInfo();
482       region.close();
483       region = HRegion.openHRegion(region, null);
484 
485       // now check whether we have only one store file, the compacted one
486       Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
487       for (StoreFile sf : sfs) {
488         LOG.info(sf.getPath());
489       }
490       assertEquals(1, region.getStore(family).getStorefilesCount());
491       files = FSUtils.listStatus(fs, tmpDir);
492       assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
493 
494       for (long i = minSeqId; i < maxSeqId; i++) {
495         Get get = new Get(Bytes.toBytes(i));
496         Result result = region.get(get);
497         byte[] value = result.getValue(family, Bytes.toBytes(i));
498         assertArrayEquals(Bytes.toBytes(i), value);
499       }
500     } finally {
501       HRegion.closeHRegion(this.region);
502       this.region = null;
503     }
504   }
505 
506   @Test
507   public void testGetWhileRegionClose() throws IOException {
508     TableName tableName = TableName.valueOf(name.getMethodName());
509     Configuration hc = initSplit();
510     int numRows = 100;
511     byte[][] families = { fam1, fam2, fam3 };
512 
513     // Setting up region
514     String method = name.getMethodName();
515     this.region = initHRegion(tableName, method, hc, families);
516     try {
517       // Put data in region
518       final int startRow = 100;
519       putData(startRow, numRows, qual1, families);
520       putData(startRow, numRows, qual2, families);
521       putData(startRow, numRows, qual3, families);
522       final AtomicBoolean done = new AtomicBoolean(false);
523       final AtomicInteger gets = new AtomicInteger(0);
524       GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
525       try {
526         // Set ten threads running concurrently getting from the region.
527         for (int i = 0; i < threads.length / 2; i++) {
528           threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
529           threads[i].setDaemon(true);
530           threads[i].start();
531         }
532         // Artificially make the condition by setting closing flag explicitly.
533         // I can't make the issue happen with a call to region.close().
534         this.region.closing.set(true);
535         for (int i = threads.length / 2; i < threads.length; i++) {
536           threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
537           threads[i].setDaemon(true);
538           threads[i].start();
539         }
540       } finally {
541         if (this.region != null) {
542           HRegion.closeHRegion(this.region);
543         }
544       }
545       done.set(true);
546       for (GetTillDoneOrException t : threads) {
547         try {
548           t.join();
549         } catch (InterruptedException e) {
550           e.printStackTrace();
551         }
552         if (t.e != null) {
553           LOG.info("Exception=" + t.e);
554           assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
555         }
556       }
557     } finally {
558       HRegion.closeHRegion(this.region);
559       this.region = null;
560     }
561   }
562 
563   /*
564    * Thread that does get on single row until 'done' flag is flipped. If an
565    * exception causes us to fail, it records it.
566    */
567   class GetTillDoneOrException extends Thread {
568     private final Get g;
569     private final AtomicBoolean done;
570     private final AtomicInteger count;
571     private Exception e;
572 
573     GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
574       super("getter." + i);
575       this.g = new Get(r);
576       this.done = d;
577       this.count = c;
578     }
579 
580     @Override
581     public void run() {
582       while (!this.done.get()) {
583         try {
584           assertTrue(region.get(g).size() > 0);
585           this.count.incrementAndGet();
586         } catch (Exception e) {
587           this.e = e;
588           break;
589         }
590       }
591     }
592   }
593 
594   /*
595    * An involved filter test. Has multiple column families and deletes in mix.
596    */
597   @Test
598   public void testWeirdCacheBehaviour() throws Exception {
599     byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
600     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
601         Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
602     this.region = initHRegion(TABLE, getName(), conf, FAMILIES);
603     try {
604       String value = "this is the value";
605       String value2 = "this is some other value";
606       String keyPrefix1 = "prefix1";
607       String keyPrefix2 = "prefix2";
608       String keyPrefix3 = "prefix3";
609       putRows(this.region, 3, value, keyPrefix1);
610       putRows(this.region, 3, value, keyPrefix2);
611       putRows(this.region, 3, value, keyPrefix3);
612       putRows(this.region, 3, value2, keyPrefix1);
613       putRows(this.region, 3, value2, keyPrefix2);
614       putRows(this.region, 3, value2, keyPrefix3);
615       System.out.println("Checking values for key: " + keyPrefix1);
616       assertEquals("Got back incorrect number of rows from scan", 3,
617           getNumberOfRows(keyPrefix1, value2, this.region));
618       System.out.println("Checking values for key: " + keyPrefix2);
619       assertEquals("Got back incorrect number of rows from scan", 3,
620           getNumberOfRows(keyPrefix2, value2, this.region));
621       System.out.println("Checking values for key: " + keyPrefix3);
622       assertEquals("Got back incorrect number of rows from scan", 3,
623           getNumberOfRows(keyPrefix3, value2, this.region));
624       deleteColumns(this.region, value2, keyPrefix1);
625       deleteColumns(this.region, value2, keyPrefix2);
626       deleteColumns(this.region, value2, keyPrefix3);
627       System.out.println("Starting important checks.....");
628       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
629           getNumberOfRows(keyPrefix1, value2, this.region));
630       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
631           getNumberOfRows(keyPrefix2, value2, this.region));
632       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
633           getNumberOfRows(keyPrefix3, value2, this.region));
634     } finally {
635       HRegion.closeHRegion(this.region);
636       this.region = null;
637     }
638   }
639 
640   @Test
641   public void testAppendWithReadOnlyTable() throws Exception {
642     byte[] TABLE = Bytes.toBytes("readOnlyTable");
643     this.region = initHRegion(TABLE, getName(), conf, true, Bytes.toBytes("somefamily"));
644     boolean exceptionCaught = false;
645     Append append = new Append(Bytes.toBytes("somerow"));
646     append.setDurability(Durability.SKIP_WAL);
647     append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
648         Bytes.toBytes("somevalue"));
649     try {
650       region.append(append);
651     } catch (IOException e) {
652       exceptionCaught = true;
653     } finally {
654       HRegion.closeHRegion(this.region);
655       this.region = null;
656     }
657     assertTrue(exceptionCaught == true);
658   }
659 
660   @Test
661   public void testIncrWithReadOnlyTable() throws Exception {
662     byte[] TABLE = Bytes.toBytes("readOnlyTable");
663     this.region = initHRegion(TABLE, getName(), conf, true, Bytes.toBytes("somefamily"));
664     boolean exceptionCaught = false;
665     Increment inc = new Increment(Bytes.toBytes("somerow"));
666     inc.setDurability(Durability.SKIP_WAL);
667     inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
668     try {
669       region.increment(inc);
670     } catch (IOException e) {
671       exceptionCaught = true;
672     } finally {
673       HRegion.closeHRegion(this.region);
674       this.region = null;
675     }
676     assertTrue(exceptionCaught == true);
677   }
678 
679   private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
680     InternalScanner scanner = buildScanner(keyPrefix, value, r);
681     int count = 0;
682     boolean more = false;
683     List<Cell> results = new ArrayList<Cell>();
684     do {
685       more = scanner.next(results);
686       if (results != null && !results.isEmpty())
687         count++;
688       else
689         break;
690       Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
691       delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
692       r.delete(delete);
693       results.clear();
694     } while (more);
695     assertEquals("Did not perform correct number of deletes", 3, count);
696   }
697 
698   private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
699     InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
700     int numberOfResults = 0;
701     List<Cell> results = new ArrayList<Cell>();
702     boolean more = false;
703     do {
704       more = resultScanner.next(results);
705       if (results != null && !results.isEmpty())
706         numberOfResults++;
707       else
708         break;
709       for (Cell kv : results) {
710         System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
711       }
712       results.clear();
713     } while (more);
714     return numberOfResults;
715   }
716 
717   private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
718       throws IOException {
719     // Defaults FilterList.Operator.MUST_PASS_ALL.
720     FilterList allFilters = new FilterList();
721     allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
722     // Only return rows where this column value exists in the row.
723     SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
724         Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
725     filter.setFilterIfMissing(true);
726     allFilters.addFilter(filter);
727     Scan scan = new Scan();
728     scan.addFamily(Bytes.toBytes("trans-blob"));
729     scan.addFamily(Bytes.toBytes("trans-type"));
730     scan.addFamily(Bytes.toBytes("trans-date"));
731     scan.addFamily(Bytes.toBytes("trans-tags"));
732     scan.addFamily(Bytes.toBytes("trans-group"));
733     scan.setFilter(allFilters);
734     return r.getScanner(scan);
735   }
736 
737   private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
738     for (int i = 0; i < numRows; i++) {
739       String row = key + "_" + i/* UUID.randomUUID().toString() */;
740       System.out.println(String.format("Saving row: %s, with value %s", row, value));
741       Put put = new Put(Bytes.toBytes(row));
742       put.setDurability(Durability.SKIP_WAL);
743       put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
744       put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
745       put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
746       put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
747       put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
748       r.put(put);
749     }
750   }
751 
752   @Test
753   public void testFamilyWithAndWithoutColon() throws Exception {
754     byte[] b = Bytes.toBytes(getName());
755     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
756     this.region = initHRegion(b, getName(), conf, cf);
757     try {
758       Put p = new Put(b);
759       byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
760       p.add(cfwithcolon, cfwithcolon, cfwithcolon);
761       boolean exception = false;
762       try {
763         this.region.put(p);
764       } catch (NoSuchColumnFamilyException e) {
765         exception = true;
766       }
767       assertTrue(exception);
768     } finally {
769       HRegion.closeHRegion(this.region);
770       this.region = null;
771     }
772   }
773 
774   @Test
775   public void testBatchPut() throws Exception {
776     byte[] b = Bytes.toBytes(getName());
777     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
778     byte[] qual = Bytes.toBytes("qual");
779     byte[] val = Bytes.toBytes("val");
780     this.region = initHRegion(b, getName(), conf, cf);
781     MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
782     try {
783       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
784       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
785 
786       LOG.info("First a batch put with all valid puts");
787       final Put[] puts = new Put[10];
788       for (int i = 0; i < 10; i++) {
789         puts[i] = new Put(Bytes.toBytes("row_" + i));
790         puts[i].add(cf, qual, val);
791       }
792 
793       OperationStatus[] codes = this.region.batchMutate(puts);
794       assertEquals(10, codes.length);
795       for (int i = 0; i < 10; i++) {
796         assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
797       }
798       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
799 
800       LOG.info("Next a batch put with one invalid family");
801       puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
802       codes = this.region.batchMutate(puts);
803       assertEquals(10, codes.length);
804       for (int i = 0; i < 10; i++) {
805         assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
806             codes[i].getOperationStatusCode());
807       }
808 
809       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
810 
811       LOG.info("Next a batch put that has to break into two batches to avoid a lock");
812       RowLock rowLock = region.getRowLock(Bytes.toBytes("row_2"));
813 
814       MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
815       final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
816       TestThread putter = new TestThread(ctx) {
817         @Override
818         public void doWork() throws IOException {
819           retFromThread.set(region.batchMutate(puts));
820         }
821       };
822       LOG.info("...starting put thread while holding lock");
823       ctx.addThread(putter);
824       ctx.startThreads();
825 
826       LOG.info("...waiting for put thread to sync first time");
827       long startWait = System.currentTimeMillis();
828       while (metricsAssertHelper.getCounter("syncTimeNumOps", source) == syncs + 2) {
829         Thread.sleep(100);
830         if (System.currentTimeMillis() - startWait > 10000) {
831           fail("Timed out waiting for thread to sync first minibatch");
832         }
833       }
834       LOG.info("...releasing row lock, which should let put thread continue");
835       rowLock.release();
836       LOG.info("...joining on thread");
837       ctx.stop();
838       LOG.info("...checking that next batch was synced");
839       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 4, source);
840       codes = retFromThread.get();
841       for (int i = 0; i < 10; i++) {
842         assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
843             codes[i].getOperationStatusCode());
844       }
845 
846     } finally {
847       HRegion.closeHRegion(this.region);
848       this.region = null;
849     }
850   }
851 
852   @Test
853   public void testBatchPutWithTsSlop() throws Exception {
854     byte[] b = Bytes.toBytes(getName());
855     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
856     byte[] qual = Bytes.toBytes("qual");
857     byte[] val = Bytes.toBytes("val");
858 
859     // add data with a timestamp that is too recent for range. Ensure assert
860     conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
861     this.region = initHRegion(b, getName(), conf, cf);
862 
863     try {
864       MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
865       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
866       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
867 
868       final Put[] puts = new Put[10];
869       for (int i = 0; i < 10; i++) {
870         puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
871         puts[i].add(cf, qual, val);
872       }
873 
874       OperationStatus[] codes = this.region.batchMutate(puts);
875       assertEquals(10, codes.length);
876       for (int i = 0; i < 10; i++) {
877         assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
878       }
879       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
880 
881     } finally {
882       HRegion.closeHRegion(this.region);
883       this.region = null;
884     }
885 
886   }
887 
888   // ////////////////////////////////////////////////////////////////////////////
889   // checkAndMutate tests
890   // ////////////////////////////////////////////////////////////////////////////
891   @Test
892   public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
893     byte[] row1 = Bytes.toBytes("row1");
894     byte[] fam1 = Bytes.toBytes("fam1");
895     byte[] qf1 = Bytes.toBytes("qualifier");
896     byte[] emptyVal = new byte[] {};
897     byte[] val1 = Bytes.toBytes("value1");
898     byte[] val2 = Bytes.toBytes("value2");
899 
900     // Setting up region
901     String method = this.getName();
902     this.region = initHRegion(tableName, method, conf, fam1);
903     try {
904       // Putting empty data in key
905       Put put = new Put(row1);
906       put.add(fam1, qf1, emptyVal);
907 
908       // checkAndPut with empty value
909       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
910           emptyVal), put, true);
911       assertTrue(res);
912 
913       // Putting data in key
914       put = new Put(row1);
915       put.add(fam1, qf1, val1);
916 
917       // checkAndPut with correct value
918       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
919           put, true);
920       assertTrue(res);
921 
922       // not empty anymore
923       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
924           put, true);
925       assertFalse(res);
926 
927       Delete delete = new Delete(row1);
928       delete.deleteColumn(fam1, qf1);
929       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
930           delete, true);
931       assertFalse(res);
932 
933       put = new Put(row1);
934       put.add(fam1, qf1, val2);
935       // checkAndPut with correct value
936       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
937           put, true);
938       assertTrue(res);
939 
940       // checkAndDelete with correct value
941       delete = new Delete(row1);
942       delete.deleteColumn(fam1, qf1);
943       delete.deleteColumn(fam1, qf1);
944       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
945           delete, true);
946       assertTrue(res);
947 
948       delete = new Delete(row1);
949       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
950           delete, true);
951       assertTrue(res);
952 
953       // checkAndPut looking for a null value
954       put = new Put(row1);
955       put.add(fam1, qf1, val1);
956 
957       res = region
958           .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
959       assertTrue(res);
960     } finally {
961       HRegion.closeHRegion(this.region);
962       this.region = null;
963     }
964   }
965 
966   @Test
967   public void testCheckAndMutate_WithWrongValue() throws IOException {
968     byte[] row1 = Bytes.toBytes("row1");
969     byte[] fam1 = Bytes.toBytes("fam1");
970     byte[] qf1 = Bytes.toBytes("qualifier");
971     byte[] val1 = Bytes.toBytes("value1");
972     byte[] val2 = Bytes.toBytes("value2");
973 
974     // Setting up region
975     String method = this.getName();
976     this.region = initHRegion(tableName, method, conf, fam1);
977     try {
978       // Putting data in key
979       Put put = new Put(row1);
980       put.add(fam1, qf1, val1);
981       region.put(put);
982 
983       // checkAndPut with wrong value
984       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
985           val2), put, true);
986       assertEquals(false, res);
987 
988       // checkAndDelete with wrong value
989       Delete delete = new Delete(row1);
990       delete.deleteFamily(fam1);
991       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
992           delete, true);
993       assertEquals(false, res);
994     } finally {
995       HRegion.closeHRegion(this.region);
996       this.region = null;
997     }
998   }
999 
1000   @Test
1001   public void testCheckAndMutate_WithCorrectValue() throws IOException {
1002     byte[] row1 = Bytes.toBytes("row1");
1003     byte[] fam1 = Bytes.toBytes("fam1");
1004     byte[] qf1 = Bytes.toBytes("qualifier");
1005     byte[] val1 = Bytes.toBytes("value1");
1006 
1007     // Setting up region
1008     String method = this.getName();
1009     this.region = initHRegion(tableName, method, conf, fam1);
1010     try {
1011       // Putting data in key
1012       Put put = new Put(row1);
1013       put.add(fam1, qf1, val1);
1014       region.put(put);
1015 
1016       // checkAndPut with correct value
1017       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1018           val1), put, true);
1019       assertEquals(true, res);
1020 
1021       // checkAndDelete with correct value
1022       Delete delete = new Delete(row1);
1023       delete.deleteColumn(fam1, qf1);
1024       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1025           put, true);
1026       assertEquals(true, res);
1027     } finally {
1028       HRegion.closeHRegion(this.region);
1029       this.region = null;
1030     }
1031   }
1032 
1033   @Test
1034   public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1035     byte[] row1 = Bytes.toBytes("row1");
1036     byte[] fam1 = Bytes.toBytes("fam1");
1037     byte[] fam2 = Bytes.toBytes("fam2");
1038     byte[] qf1 = Bytes.toBytes("qualifier");
1039     byte[] val1 = Bytes.toBytes("value1");
1040     byte[] val2 = Bytes.toBytes("value2");
1041 
1042     byte[][] families = { fam1, fam2 };
1043 
1044     // Setting up region
1045     String method = this.getName();
1046     this.region = initHRegion(tableName, method, conf, families);
1047     try {
1048       // Putting data in the key to check
1049       Put put = new Put(row1);
1050       put.add(fam1, qf1, val1);
1051       region.put(put);
1052 
1053       // Creating put to add
1054       long ts = System.currentTimeMillis();
1055       KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1056       put = new Put(row1);
1057       put.add(kv);
1058 
1059       // checkAndPut with wrong value
1060       HStore store = (HStore) region.getStore(fam1);
1061       store.memstore.kvset.size();
1062 
1063       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1064           val1), put, true);
1065       assertEquals(true, res);
1066       store.memstore.kvset.size();
1067 
1068       Get get = new Get(row1);
1069       get.addColumn(fam2, qf1);
1070       Cell[] actual = region.get(get).rawCells();
1071 
1072       Cell[] expected = { kv };
1073 
1074       assertEquals(expected.length, actual.length);
1075       for (int i = 0; i < actual.length; i++) {
1076         assertEquals(expected[i], actual[i]);
1077       }
1078     } finally {
1079       HRegion.closeHRegion(this.region);
1080       this.region = null;
1081     }
1082   }
1083 
1084   @Test
1085   public void testCheckAndPut_wrongRowInPut() throws IOException {
1086     TableName tableName = TableName.valueOf(name.getMethodName());
1087     this.region = initHRegion(tableName, this.getName(), conf, COLUMNS);
1088     try {
1089       Put put = new Put(row2);
1090       put.add(fam1, qual1, value1);
1091       try {
1092         boolean res = region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1093             new BinaryComparator(value2), put, false);
1094         fail();
1095       } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1096         // expected exception.
1097       }
1098     } finally {
1099       HRegion.closeHRegion(this.region);
1100       this.region = null;
1101     }
1102   }
1103 
1104   @Test
1105   public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1106     byte[] row1 = Bytes.toBytes("row1");
1107     byte[] fam1 = Bytes.toBytes("fam1");
1108     byte[] fam2 = Bytes.toBytes("fam2");
1109     byte[] qf1 = Bytes.toBytes("qualifier1");
1110     byte[] qf2 = Bytes.toBytes("qualifier2");
1111     byte[] qf3 = Bytes.toBytes("qualifier3");
1112     byte[] val1 = Bytes.toBytes("value1");
1113     byte[] val2 = Bytes.toBytes("value2");
1114     byte[] val3 = Bytes.toBytes("value3");
1115     byte[] emptyVal = new byte[] {};
1116 
1117     byte[][] families = { fam1, fam2 };
1118 
1119     // Setting up region
1120     String method = this.getName();
1121     this.region = initHRegion(tableName, method, conf, families);
1122     try {
1123       // Put content
1124       Put put = new Put(row1);
1125       put.add(fam1, qf1, val1);
1126       region.put(put);
1127       Threads.sleep(2);
1128 
1129       put = new Put(row1);
1130       put.add(fam1, qf1, val2);
1131       put.add(fam2, qf1, val3);
1132       put.add(fam2, qf2, val2);
1133       put.add(fam2, qf3, val1);
1134       put.add(fam1, qf3, val1);
1135       region.put(put);
1136 
1137       // Multi-column delete
1138       Delete delete = new Delete(row1);
1139       delete.deleteColumn(fam1, qf1);
1140       delete.deleteColumn(fam2, qf1);
1141       delete.deleteColumn(fam1, qf3);
1142       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1143           val2), delete, true);
1144       assertEquals(true, res);
1145 
1146       Get get = new Get(row1);
1147       get.addColumn(fam1, qf1);
1148       get.addColumn(fam1, qf3);
1149       get.addColumn(fam2, qf2);
1150       Result r = region.get(get);
1151       assertEquals(2, r.size());
1152       assertArrayEquals(val1, r.getValue(fam1, qf1));
1153       assertArrayEquals(val2, r.getValue(fam2, qf2));
1154 
1155       // Family delete
1156       delete = new Delete(row1);
1157       delete.deleteFamily(fam2);
1158       res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1159           delete, true);
1160       assertEquals(true, res);
1161 
1162       get = new Get(row1);
1163       r = region.get(get);
1164       assertEquals(1, r.size());
1165       assertArrayEquals(val1, r.getValue(fam1, qf1));
1166 
1167       // Row delete
1168       delete = new Delete(row1);
1169       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1170           delete, true);
1171       assertEquals(true, res);
1172       get = new Get(row1);
1173       r = region.get(get);
1174       assertEquals(0, r.size());
1175     } finally {
1176       HRegion.closeHRegion(this.region);
1177       this.region = null;
1178     }
1179   }
1180 
1181   // ////////////////////////////////////////////////////////////////////////////
1182   // Delete tests
1183   // ////////////////////////////////////////////////////////////////////////////
1184   @Test
1185   public void testDelete_multiDeleteColumn() throws IOException {
1186     byte[] row1 = Bytes.toBytes("row1");
1187     byte[] fam1 = Bytes.toBytes("fam1");
1188     byte[] qual = Bytes.toBytes("qualifier");
1189     byte[] value = Bytes.toBytes("value");
1190 
1191     Put put = new Put(row1);
1192     put.add(fam1, qual, 1, value);
1193     put.add(fam1, qual, 2, value);
1194 
1195     String method = this.getName();
1196     this.region = initHRegion(tableName, method, conf, fam1);
1197     try {
1198       region.put(put);
1199 
1200       // We do support deleting more than 1 'latest' version
1201       Delete delete = new Delete(row1);
1202       delete.deleteColumn(fam1, qual);
1203       delete.deleteColumn(fam1, qual);
1204       region.delete(delete);
1205 
1206       Get get = new Get(row1);
1207       get.addFamily(fam1);
1208       Result r = region.get(get);
1209       assertEquals(0, r.size());
1210     } finally {
1211       HRegion.closeHRegion(this.region);
1212       this.region = null;
1213     }
1214   }
1215 
1216   @Test
1217   public void testDelete_CheckFamily() throws IOException {
1218     byte[] row1 = Bytes.toBytes("row1");
1219     byte[] fam1 = Bytes.toBytes("fam1");
1220     byte[] fam2 = Bytes.toBytes("fam2");
1221     byte[] fam3 = Bytes.toBytes("fam3");
1222     byte[] fam4 = Bytes.toBytes("fam4");
1223 
1224     // Setting up region
1225     String method = this.getName();
1226     this.region = initHRegion(tableName, method, conf, fam1, fam2, fam3);
1227     try {
1228       List<Cell> kvs = new ArrayList<Cell>();
1229       kvs.add(new KeyValue(row1, fam4, null, null));
1230 
1231       // testing existing family
1232       byte[] family = fam2;
1233       try {
1234         NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1235             Bytes.BYTES_COMPARATOR);
1236         deleteMap.put(family, kvs);
1237         region.delete(deleteMap, Durability.SYNC_WAL);
1238       } catch (Exception e) {
1239         assertTrue("Family " + new String(family) + " does not exist", false);
1240       }
1241 
1242       // testing non existing family
1243       boolean ok = false;
1244       family = fam4;
1245       try {
1246         NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1247             Bytes.BYTES_COMPARATOR);
1248         deleteMap.put(family, kvs);
1249         region.delete(deleteMap, Durability.SYNC_WAL);
1250       } catch (Exception e) {
1251         ok = true;
1252       }
1253       assertEquals("Family " + new String(family) + " does exist", true, ok);
1254     } finally {
1255       HRegion.closeHRegion(this.region);
1256       this.region = null;
1257     }
1258   }
1259 
1260   @Test
1261   public void testDelete_mixed() throws IOException, InterruptedException {
1262     byte[] fam = Bytes.toBytes("info");
1263     byte[][] families = { fam };
1264     String method = this.getName();
1265     this.region = initHRegion(tableName, method, conf, families);
1266     try {
1267       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1268 
1269       byte[] row = Bytes.toBytes("table_name");
1270       // column names
1271       byte[] serverinfo = Bytes.toBytes("serverinfo");
1272       byte[] splitA = Bytes.toBytes("splitA");
1273       byte[] splitB = Bytes.toBytes("splitB");
1274 
1275       // add some data:
1276       Put put = new Put(row);
1277       put.add(fam, splitA, Bytes.toBytes("reference_A"));
1278       region.put(put);
1279 
1280       put = new Put(row);
1281       put.add(fam, splitB, Bytes.toBytes("reference_B"));
1282       region.put(put);
1283 
1284       put = new Put(row);
1285       put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1286       region.put(put);
1287 
1288       // ok now delete a split:
1289       Delete delete = new Delete(row);
1290       delete.deleteColumns(fam, splitA);
1291       region.delete(delete);
1292 
1293       // assert some things:
1294       Get get = new Get(row).addColumn(fam, serverinfo);
1295       Result result = region.get(get);
1296       assertEquals(1, result.size());
1297 
1298       get = new Get(row).addColumn(fam, splitA);
1299       result = region.get(get);
1300       assertEquals(0, result.size());
1301 
1302       get = new Get(row).addColumn(fam, splitB);
1303       result = region.get(get);
1304       assertEquals(1, result.size());
1305 
1306       // Assert that after a delete, I can put.
1307       put = new Put(row);
1308       put.add(fam, splitA, Bytes.toBytes("reference_A"));
1309       region.put(put);
1310       get = new Get(row);
1311       result = region.get(get);
1312       assertEquals(3, result.size());
1313 
1314       // Now delete all... then test I can add stuff back
1315       delete = new Delete(row);
1316       region.delete(delete);
1317       assertEquals(0, region.get(get).size());
1318 
1319       region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
1320       result = region.get(get);
1321       assertEquals(1, result.size());
1322     } finally {
1323       HRegion.closeHRegion(this.region);
1324       this.region = null;
1325     }
1326   }
1327 
1328   @Test
1329   public void testDeleteRowWithFutureTs() throws IOException {
1330     byte[] fam = Bytes.toBytes("info");
1331     byte[][] families = { fam };
1332     String method = this.getName();
1333     this.region = initHRegion(tableName, method, conf, families);
1334     try {
1335       byte[] row = Bytes.toBytes("table_name");
1336       // column names
1337       byte[] serverinfo = Bytes.toBytes("serverinfo");
1338 
1339       // add data in the far future
1340       Put put = new Put(row);
1341       put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
1342       region.put(put);
1343 
1344       // now delete something in the present
1345       Delete delete = new Delete(row);
1346       region.delete(delete);
1347 
1348       // make sure we still see our data
1349       Get get = new Get(row).addColumn(fam, serverinfo);
1350       Result result = region.get(get);
1351       assertEquals(1, result.size());
1352 
1353       // delete the future row
1354       delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
1355       region.delete(delete);
1356 
1357       // make sure it is gone
1358       get = new Get(row).addColumn(fam, serverinfo);
1359       result = region.get(get);
1360       assertEquals(0, result.size());
1361     } finally {
1362       HRegion.closeHRegion(this.region);
1363       this.region = null;
1364     }
1365   }
1366 
1367   /**
1368    * Tests that the special LATEST_TIMESTAMP option for puts gets replaced by
1369    * the actual timestamp
1370    */
1371   @Test
1372   public void testPutWithLatestTS() throws IOException {
1373     byte[] fam = Bytes.toBytes("info");
1374     byte[][] families = { fam };
1375     String method = this.getName();
1376     this.region = initHRegion(tableName, method, conf, families);
1377     try {
1378       byte[] row = Bytes.toBytes("row1");
1379       // column names
1380       byte[] qual = Bytes.toBytes("qual");
1381 
1382       // add data with LATEST_TIMESTAMP, put without WAL
1383       Put put = new Put(row);
1384       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1385       region.put(put);
1386 
1387       // Make sure it shows up with an actual timestamp
1388       Get get = new Get(row).addColumn(fam, qual);
1389       Result result = region.get(get);
1390       assertEquals(1, result.size());
1391       Cell kv = result.rawCells()[0];
1392       LOG.info("Got: " + kv);
1393       assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1394           kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1395 
1396       // Check same with WAL enabled (historically these took different
1397       // code paths, so check both)
1398       row = Bytes.toBytes("row2");
1399       put = new Put(row);
1400       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1401       region.put(put);
1402 
1403       // Make sure it shows up with an actual timestamp
1404       get = new Get(row).addColumn(fam, qual);
1405       result = region.get(get);
1406       assertEquals(1, result.size());
1407       kv = result.rawCells()[0];
1408       LOG.info("Got: " + kv);
1409       assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1410           kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1411     } finally {
1412       HRegion.closeHRegion(this.region);
1413       this.region = null;
1414     }
1415 
1416   }
1417 
1418   /**
1419    * Tests that there is server-side filtering for invalid timestamp upper
1420    * bound. Note that the timestamp lower bound is automatically handled for us
1421    * by the TTL field.
1422    */
1423   @Test
1424   public void testPutWithTsSlop() throws IOException {
1425     byte[] fam = Bytes.toBytes("info");
1426     byte[][] families = { fam };
1427     String method = this.getName();
1428 
1429     // add data with a timestamp that is too recent for range. Ensure assert
1430     conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1431     this.region = initHRegion(tableName, method, conf, families);
1432     boolean caughtExcep = false;
1433     try {
1434       try {
1435         // no TS specified == use latest. should not error
1436         region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
1437         // TS out of range. should error
1438         region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
1439             Bytes.toBytes("value")));
1440         fail("Expected IOE for TS out of configured timerange");
1441       } catch (FailedSanityCheckException ioe) {
1442         LOG.debug("Received expected exception", ioe);
1443         caughtExcep = true;
1444       }
1445       assertTrue("Should catch FailedSanityCheckException", caughtExcep);
1446     } finally {
1447       HRegion.closeHRegion(this.region);
1448       this.region = null;
1449     }
1450   }
1451 
1452   @Test
1453   public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
1454     byte[] fam1 = Bytes.toBytes("columnA");
1455     byte[] fam2 = Bytes.toBytes("columnB");
1456     this.region = initHRegion(tableName, getName(), conf, fam1, fam2);
1457     try {
1458       byte[] rowA = Bytes.toBytes("rowA");
1459       byte[] rowB = Bytes.toBytes("rowB");
1460 
1461       byte[] value = Bytes.toBytes("value");
1462 
1463       Delete delete = new Delete(rowA);
1464       delete.deleteFamily(fam1);
1465 
1466       region.delete(delete);
1467 
1468       // now create data.
1469       Put put = new Put(rowA);
1470       put.add(fam2, null, value);
1471       region.put(put);
1472 
1473       put = new Put(rowB);
1474       put.add(fam1, null, value);
1475       put.add(fam2, null, value);
1476       region.put(put);
1477 
1478       Scan scan = new Scan();
1479       scan.addFamily(fam1).addFamily(fam2);
1480       InternalScanner s = region.getScanner(scan);
1481       List<Cell> results = new ArrayList<Cell>();
1482       s.next(results);
1483       assertTrue(CellUtil.matchingRow(results.get(0), rowA));
1484 
1485       results.clear();
1486       s.next(results);
1487       assertTrue(CellUtil.matchingRow(results.get(0), rowB));
1488     } finally {
1489       HRegion.closeHRegion(this.region);
1490       this.region = null;
1491     }
1492   }
1493 
1494   @Test
1495   public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
1496     Delete delete = new Delete(row);
1497     delete.deleteColumns(fam1, qual1);
1498     doTestDelete_AndPostInsert(delete);
1499   }
1500 
1501   @Test
1502   public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
1503     Delete delete = new Delete(row);
1504     delete.deleteFamily(fam1);
1505     doTestDelete_AndPostInsert(delete);
1506   }
1507 
1508   public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
1509     TableName tableName = TableName.valueOf(name.getMethodName());
1510     this.region = initHRegion(tableName, getName(), conf, fam1);
1511     try {
1512       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1513       Put put = new Put(row);
1514       put.add(fam1, qual1, value1);
1515       region.put(put);
1516 
1517       // now delete the value:
1518       region.delete(delete);
1519 
1520       // ok put data:
1521       put = new Put(row);
1522       put.add(fam1, qual1, value2);
1523       region.put(put);
1524 
1525       // ok get:
1526       Get get = new Get(row);
1527       get.addColumn(fam1, qual1);
1528 
1529       Result r = region.get(get);
1530       assertEquals(1, r.size());
1531       assertArrayEquals(value2, r.getValue(fam1, qual1));
1532 
1533       // next:
1534       Scan scan = new Scan(row);
1535       scan.addColumn(fam1, qual1);
1536       InternalScanner s = region.getScanner(scan);
1537 
1538       List<Cell> results = new ArrayList<Cell>();
1539       assertEquals(false, s.next(results));
1540       assertEquals(1, results.size());
1541       Cell kv = results.get(0);
1542 
1543       assertArrayEquals(value2, CellUtil.cloneValue(kv));
1544       assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
1545       assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
1546       assertArrayEquals(row, CellUtil.cloneRow(kv));
1547     } finally {
1548       HRegion.closeHRegion(this.region);
1549       this.region = null;
1550     }
1551   }
1552 
1553   @Test
1554   public void testDelete_CheckTimestampUpdated() throws IOException {
1555     TableName tableName = TableName.valueOf(name.getMethodName());
1556     byte[] row1 = Bytes.toBytes("row1");
1557     byte[] col1 = Bytes.toBytes("col1");
1558     byte[] col2 = Bytes.toBytes("col2");
1559     byte[] col3 = Bytes.toBytes("col3");
1560 
1561     // Setting up region
1562     String method = this.getName();
1563     this.region = initHRegion(tableName, method, conf, fam1);
1564     try {
1565       // Building checkerList
1566       List<Cell> kvs = new ArrayList<Cell>();
1567       kvs.add(new KeyValue(row1, fam1, col1, null));
1568       kvs.add(new KeyValue(row1, fam1, col2, null));
1569       kvs.add(new KeyValue(row1, fam1, col3, null));
1570 
1571       NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1572           Bytes.BYTES_COMPARATOR);
1573       deleteMap.put(fam1, kvs);
1574       region.delete(deleteMap, Durability.SYNC_WAL);
1575 
1576       // extract the key values out the memstore:
1577       // This is kinda hacky, but better than nothing...
1578       long now = System.currentTimeMillis();
1579       KeyValue firstKv = ((HStore) region.getStore(fam1)).memstore.kvset.first();
1580       assertTrue(firstKv.getTimestamp() <= now);
1581       now = firstKv.getTimestamp();
1582       for (KeyValue kv : ((HStore) region.getStore(fam1)).memstore.kvset) {
1583         assertTrue(kv.getTimestamp() <= now);
1584         now = kv.getTimestamp();
1585       }
1586     } finally {
1587       HRegion.closeHRegion(this.region);
1588       this.region = null;
1589     }
1590   }
1591 
1592   // ////////////////////////////////////////////////////////////////////////////
1593   // Get tests
1594   // ////////////////////////////////////////////////////////////////////////////
1595   @Test
1596   public void testGet_FamilyChecker() throws IOException {
1597     byte[] row1 = Bytes.toBytes("row1");
1598     byte[] fam1 = Bytes.toBytes("fam1");
1599     byte[] fam2 = Bytes.toBytes("False");
1600     byte[] col1 = Bytes.toBytes("col1");
1601 
1602     // Setting up region
1603     String method = this.getName();
1604     this.region = initHRegion(tableName, method, conf, fam1);
1605     try {
1606       Get get = new Get(row1);
1607       get.addColumn(fam2, col1);
1608 
1609       // Test
1610       try {
1611         region.get(get);
1612       } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
1613         assertFalse(false);
1614         return;
1615       }
1616       assertFalse(true);
1617     } finally {
1618       HRegion.closeHRegion(this.region);
1619       this.region = null;
1620     }
1621   }
1622 
1623   @Test
1624   public void testGet_Basic() throws IOException {
1625     byte[] row1 = Bytes.toBytes("row1");
1626     byte[] fam1 = Bytes.toBytes("fam1");
1627     byte[] col1 = Bytes.toBytes("col1");
1628     byte[] col2 = Bytes.toBytes("col2");
1629     byte[] col3 = Bytes.toBytes("col3");
1630     byte[] col4 = Bytes.toBytes("col4");
1631     byte[] col5 = Bytes.toBytes("col5");
1632 
1633     // Setting up region
1634     String method = this.getName();
1635     this.region = initHRegion(tableName, method, conf, fam1);
1636     try {
1637       // Add to memstore
1638       Put put = new Put(row1);
1639       put.add(fam1, col1, null);
1640       put.add(fam1, col2, null);
1641       put.add(fam1, col3, null);
1642       put.add(fam1, col4, null);
1643       put.add(fam1, col5, null);
1644       region.put(put);
1645 
1646       Get get = new Get(row1);
1647       get.addColumn(fam1, col2);
1648       get.addColumn(fam1, col4);
1649       // Expected result
1650       KeyValue kv1 = new KeyValue(row1, fam1, col2);
1651       KeyValue kv2 = new KeyValue(row1, fam1, col4);
1652       KeyValue[] expected = { kv1, kv2 };
1653 
1654       // Test
1655       Result res = region.get(get);
1656       assertEquals(expected.length, res.size());
1657       for (int i = 0; i < res.size(); i++) {
1658         assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
1659         assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
1660         assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
1661       }
1662 
1663       // Test using a filter on a Get
1664       Get g = new Get(row1);
1665       final int count = 2;
1666       g.setFilter(new ColumnCountGetFilter(count));
1667       res = region.get(g);
1668       assertEquals(count, res.size());
1669     } finally {
1670       HRegion.closeHRegion(this.region);
1671       this.region = null;
1672     }
1673   }
1674 
1675   @Test
1676   public void testGet_Empty() throws IOException {
1677     byte[] row = Bytes.toBytes("row");
1678     byte[] fam = Bytes.toBytes("fam");
1679 
1680     String method = this.getName();
1681     this.region = initHRegion(tableName, method, conf, fam);
1682     try {
1683       Get get = new Get(row);
1684       get.addFamily(fam);
1685       Result r = region.get(get);
1686 
1687       assertTrue(r.isEmpty());
1688     } finally {
1689       HRegion.closeHRegion(this.region);
1690       this.region = null;
1691     }
1692   }
1693 
1694   // ////////////////////////////////////////////////////////////////////////////
1695   // Merge test
1696   // ////////////////////////////////////////////////////////////////////////////
1697   @Test
1698   public void testMerge() throws IOException {
1699     byte[][] families = { fam1, fam2, fam3 };
1700     Configuration hc = initSplit();
1701     // Setting up region
1702     String method = this.getName();
1703     this.region = initHRegion(tableName, method, hc, families);
1704     try {
1705       LOG.info("" + HBaseTestCase.addContent(region, fam3));
1706       region.flushcache();
1707       region.compactStores();
1708       byte[] splitRow = region.checkSplit();
1709       assertNotNull(splitRow);
1710       LOG.info("SplitRow: " + Bytes.toString(splitRow));
1711       HRegion[] subregions = splitRegion(region, splitRow);
1712       try {
1713         // Need to open the regions.
1714         for (int i = 0; i < subregions.length; i++) {
1715           HRegion.openHRegion(subregions[i], null);
1716           subregions[i].compactStores();
1717         }
1718         Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
1719         Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
1720         Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
1721         long startTime = System.currentTimeMillis();
1722         region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
1723         LOG.info("Merge regions elapsed time: "
1724             + ((System.currentTimeMillis() - startTime) / 1000.0));
1725         fs.delete(oldRegion1, true);
1726         fs.delete(oldRegion2, true);
1727         fs.delete(oldRegionPath, true);
1728         LOG.info("splitAndMerge completed.");
1729       } finally {
1730         for (int i = 0; i < subregions.length; i++) {
1731           try {
1732             HRegion.closeHRegion(subregions[i]);
1733           } catch (IOException e) {
1734             // Ignore.
1735           }
1736         }
1737       }
1738     } finally {
1739       HRegion.closeHRegion(this.region);
1740       this.region = null;
1741     }
1742   }
1743 
1744   /**
1745    * @param parent
1746    *          Region to split.
1747    * @param midkey
1748    *          Key to split around.
1749    * @return The Regions we created.
1750    * @throws IOException
1751    */
1752   HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
1753     PairOfSameType<HRegion> result = null;
1754     SplitTransaction st = new SplitTransaction(parent, midkey);
1755     // If prepare does not return true, for some reason -- logged inside in
1756     // the prepare call -- we are not ready to split just now. Just return.
1757     if (!st.prepare())
1758       return null;
1759     try {
1760       result = st.execute(null, null);
1761     } catch (IOException ioe) {
1762       try {
1763         LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; "
1764             + ioe.getMessage());
1765         st.rollback(null, null);
1766         LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString());
1767         return null;
1768       } catch (RuntimeException e) {
1769         // If failed rollback, kill this server to avoid having a hole in table.
1770         LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString()
1771             + " -- aborting server", e);
1772       }
1773     }
1774     return new HRegion[] { result.getFirst(), result.getSecond() };
1775   }
1776 
1777   // ////////////////////////////////////////////////////////////////////////////
1778   // Scanner tests
1779   // ////////////////////////////////////////////////////////////////////////////
1780   @Test
1781   public void testGetScanner_WithOkFamilies() throws IOException {
1782     byte[] fam1 = Bytes.toBytes("fam1");
1783     byte[] fam2 = Bytes.toBytes("fam2");
1784 
1785     byte[][] families = { fam1, fam2 };
1786 
1787     // Setting up region
1788     String method = this.getName();
1789     this.region = initHRegion(tableName, method, conf, families);
1790     try {
1791       Scan scan = new Scan();
1792       scan.addFamily(fam1);
1793       scan.addFamily(fam2);
1794       try {
1795         region.getScanner(scan);
1796       } catch (Exception e) {
1797         assertTrue("Families could not be found in Region", false);
1798       }
1799     } finally {
1800       HRegion.closeHRegion(this.region);
1801       this.region = null;
1802     }
1803   }
1804 
1805   @Test
1806   public void testGetScanner_WithNotOkFamilies() throws IOException {
1807     byte[] fam1 = Bytes.toBytes("fam1");
1808     byte[] fam2 = Bytes.toBytes("fam2");
1809 
1810     byte[][] families = { fam1 };
1811 
1812     // Setting up region
1813     String method = this.getName();
1814     this.region = initHRegion(tableName, method, conf, families);
1815     try {
1816       Scan scan = new Scan();
1817       scan.addFamily(fam2);
1818       boolean ok = false;
1819       try {
1820         region.getScanner(scan);
1821       } catch (Exception e) {
1822         ok = true;
1823       }
1824       assertTrue("Families could not be found in Region", ok);
1825     } finally {
1826       HRegion.closeHRegion(this.region);
1827       this.region = null;
1828     }
1829   }
1830 
1831   @Test
1832   public void testGetScanner_WithNoFamilies() throws IOException {
1833     byte[] row1 = Bytes.toBytes("row1");
1834     byte[] fam1 = Bytes.toBytes("fam1");
1835     byte[] fam2 = Bytes.toBytes("fam2");
1836     byte[] fam3 = Bytes.toBytes("fam3");
1837     byte[] fam4 = Bytes.toBytes("fam4");
1838 
1839     byte[][] families = { fam1, fam2, fam3, fam4 };
1840 
1841     // Setting up region
1842     String method = this.getName();
1843     this.region = initHRegion(tableName, method, conf, families);
1844     try {
1845 
1846       // Putting data in Region
1847       Put put = new Put(row1);
1848       put.add(fam1, null, null);
1849       put.add(fam2, null, null);
1850       put.add(fam3, null, null);
1851       put.add(fam4, null, null);
1852       region.put(put);
1853 
1854       Scan scan = null;
1855       HRegion.RegionScannerImpl is = null;
1856 
1857       // Testing to see how many scanners that is produced by getScanner,
1858       // starting
1859       // with known number, 2 - current = 1
1860       scan = new Scan();
1861       scan.addFamily(fam2);
1862       scan.addFamily(fam4);
1863       is = (RegionScannerImpl) region.getScanner(scan);
1864       MultiVersionConsistencyControl.resetThreadReadPoint(region.getMVCC());
1865       assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
1866 
1867       scan = new Scan();
1868       is = (RegionScannerImpl) region.getScanner(scan);
1869       MultiVersionConsistencyControl.resetThreadReadPoint(region.getMVCC());
1870       assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
1871     } finally {
1872       HRegion.closeHRegion(this.region);
1873       this.region = null;
1874     }
1875   }
1876 
1877   /**
1878    * This method tests https://issues.apache.org/jira/browse/HBASE-2516.
1879    * 
1880    * @throws IOException
1881    */
1882   @Test
1883   public void testGetScanner_WithRegionClosed() throws IOException {
1884     byte[] fam1 = Bytes.toBytes("fam1");
1885     byte[] fam2 = Bytes.toBytes("fam2");
1886 
1887     byte[][] families = { fam1, fam2 };
1888 
1889     // Setting up region
1890     String method = this.getName();
1891     try {
1892       this.region = initHRegion(tableName, method, conf, families);
1893     } catch (IOException e) {
1894       e.printStackTrace();
1895       fail("Got IOException during initHRegion, " + e.getMessage());
1896     }
1897     try {
1898       region.closed.set(true);
1899       try {
1900         region.getScanner(null);
1901         fail("Expected to get an exception during getScanner on a region that is closed");
1902       } catch (NotServingRegionException e) {
1903         // this is the correct exception that is expected
1904       } catch (IOException e) {
1905         fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
1906             + e.getMessage());
1907       }
1908     } finally {
1909       HRegion.closeHRegion(this.region);
1910       this.region = null;
1911     }
1912   }
1913 
1914   @Test
1915   public void testRegionScanner_Next() throws IOException {
1916     byte[] row1 = Bytes.toBytes("row1");
1917     byte[] row2 = Bytes.toBytes("row2");
1918     byte[] fam1 = Bytes.toBytes("fam1");
1919     byte[] fam2 = Bytes.toBytes("fam2");
1920     byte[] fam3 = Bytes.toBytes("fam3");
1921     byte[] fam4 = Bytes.toBytes("fam4");
1922 
1923     byte[][] families = { fam1, fam2, fam3, fam4 };
1924     long ts = System.currentTimeMillis();
1925 
1926     // Setting up region
1927     String method = this.getName();
1928     this.region = initHRegion(tableName, method, conf, families);
1929     try {
1930       // Putting data in Region
1931       Put put = null;
1932       put = new Put(row1);
1933       put.add(fam1, (byte[]) null, ts, null);
1934       put.add(fam2, (byte[]) null, ts, null);
1935       put.add(fam3, (byte[]) null, ts, null);
1936       put.add(fam4, (byte[]) null, ts, null);
1937       region.put(put);
1938 
1939       put = new Put(row2);
1940       put.add(fam1, (byte[]) null, ts, null);
1941       put.add(fam2, (byte[]) null, ts, null);
1942       put.add(fam3, (byte[]) null, ts, null);
1943       put.add(fam4, (byte[]) null, ts, null);
1944       region.put(put);
1945 
1946       Scan scan = new Scan();
1947       scan.addFamily(fam2);
1948       scan.addFamily(fam4);
1949       InternalScanner is = region.getScanner(scan);
1950 
1951       List<Cell> res = null;
1952 
1953       // Result 1
1954       List<Cell> expected1 = new ArrayList<Cell>();
1955       expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
1956       expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
1957 
1958       res = new ArrayList<Cell>();
1959       is.next(res);
1960       for (int i = 0; i < res.size(); i++) {
1961         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
1962       }
1963 
1964       // Result 2
1965       List<Cell> expected2 = new ArrayList<Cell>();
1966       expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
1967       expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
1968 
1969       res = new ArrayList<Cell>();
1970       is.next(res);
1971       for (int i = 0; i < res.size(); i++) {
1972         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
1973       }
1974     } finally {
1975       HRegion.closeHRegion(this.region);
1976       this.region = null;
1977     }
1978   }
1979 
1980   @Test
1981   public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
1982     byte[] row1 = Bytes.toBytes("row1");
1983     byte[] qf1 = Bytes.toBytes("qualifier1");
1984     byte[] qf2 = Bytes.toBytes("qualifier2");
1985     byte[] fam1 = Bytes.toBytes("fam1");
1986     byte[][] families = { fam1 };
1987 
1988     long ts1 = System.currentTimeMillis();
1989     long ts2 = ts1 + 1;
1990     long ts3 = ts1 + 2;
1991 
1992     // Setting up region
1993     String method = this.getName();
1994     this.region = initHRegion(tableName, method, conf, families);
1995     try {
1996       // Putting data in Region
1997       Put put = null;
1998       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
1999       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2000       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2001 
2002       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2003       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2004       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2005 
2006       put = new Put(row1);
2007       put.add(kv13);
2008       put.add(kv12);
2009       put.add(kv11);
2010       put.add(kv23);
2011       put.add(kv22);
2012       put.add(kv21);
2013       region.put(put);
2014 
2015       // Expected
2016       List<Cell> expected = new ArrayList<Cell>();
2017       expected.add(kv13);
2018       expected.add(kv12);
2019 
2020       Scan scan = new Scan(row1);
2021       scan.addColumn(fam1, qf1);
2022       scan.setMaxVersions(MAX_VERSIONS);
2023       List<Cell> actual = new ArrayList<Cell>();
2024       InternalScanner scanner = region.getScanner(scan);
2025 
2026       boolean hasNext = scanner.next(actual);
2027       assertEquals(false, hasNext);
2028 
2029       // Verify result
2030       for (int i = 0; i < expected.size(); i++) {
2031         assertEquals(expected.get(i), actual.get(i));
2032       }
2033     } finally {
2034       HRegion.closeHRegion(this.region);
2035       this.region = null;
2036     }
2037   }
2038 
2039   @Test
2040   public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2041     byte[] row1 = Bytes.toBytes("row1");
2042     byte[] qf1 = Bytes.toBytes("qualifier1");
2043     byte[] qf2 = Bytes.toBytes("qualifier2");
2044     byte[] fam1 = Bytes.toBytes("fam1");
2045     byte[][] families = { fam1 };
2046 
2047     long ts1 = 1; // System.currentTimeMillis();
2048     long ts2 = ts1 + 1;
2049     long ts3 = ts1 + 2;
2050 
2051     // Setting up region
2052     String method = this.getName();
2053     this.region = initHRegion(tableName, method, conf, families);
2054     try {
2055       // Putting data in Region
2056       Put put = null;
2057       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2058       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2059       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2060 
2061       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2062       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2063       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2064 
2065       put = new Put(row1);
2066       put.add(kv13);
2067       put.add(kv12);
2068       put.add(kv11);
2069       put.add(kv23);
2070       put.add(kv22);
2071       put.add(kv21);
2072       region.put(put);
2073       region.flushcache();
2074 
2075       // Expected
2076       List<Cell> expected = new ArrayList<Cell>();
2077       expected.add(kv13);
2078       expected.add(kv12);
2079       expected.add(kv23);
2080       expected.add(kv22);
2081 
2082       Scan scan = new Scan(row1);
2083       scan.addColumn(fam1, qf1);
2084       scan.addColumn(fam1, qf2);
2085       scan.setMaxVersions(MAX_VERSIONS);
2086       List<Cell> actual = new ArrayList<Cell>();
2087       InternalScanner scanner = region.getScanner(scan);
2088 
2089       boolean hasNext = scanner.next(actual);
2090       assertEquals(false, hasNext);
2091 
2092       // Verify result
2093       for (int i = 0; i < expected.size(); i++) {
2094         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2095       }
2096     } finally {
2097       HRegion.closeHRegion(this.region);
2098       this.region = null;
2099     }
2100   }
2101 
2102   @Test
2103   public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2104     byte[] row1 = Bytes.toBytes("row1");
2105     byte[] fam1 = Bytes.toBytes("fam1");
2106     byte[][] families = { fam1 };
2107     byte[] qf1 = Bytes.toBytes("qualifier1");
2108     byte[] qf2 = Bytes.toBytes("qualifier2");
2109 
2110     long ts1 = 1;
2111     long ts2 = ts1 + 1;
2112     long ts3 = ts1 + 2;
2113     long ts4 = ts1 + 3;
2114 
2115     // Setting up region
2116     String method = this.getName();
2117     this.region = initHRegion(tableName, method, conf, families);
2118     try {
2119       // Putting data in Region
2120       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2121       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2122       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2123       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2124 
2125       KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2126       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2127       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2128       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2129 
2130       Put put = null;
2131       put = new Put(row1);
2132       put.add(kv14);
2133       put.add(kv24);
2134       region.put(put);
2135       region.flushcache();
2136 
2137       put = new Put(row1);
2138       put.add(kv23);
2139       put.add(kv13);
2140       region.put(put);
2141       region.flushcache();
2142 
2143       put = new Put(row1);
2144       put.add(kv22);
2145       put.add(kv12);
2146       region.put(put);
2147       region.flushcache();
2148 
2149       put = new Put(row1);
2150       put.add(kv21);
2151       put.add(kv11);
2152       region.put(put);
2153 
2154       // Expected
2155       List<Cell> expected = new ArrayList<Cell>();
2156       expected.add(kv14);
2157       expected.add(kv13);
2158       expected.add(kv12);
2159       expected.add(kv24);
2160       expected.add(kv23);
2161       expected.add(kv22);
2162 
2163       Scan scan = new Scan(row1);
2164       scan.addColumn(fam1, qf1);
2165       scan.addColumn(fam1, qf2);
2166       int versions = 3;
2167       scan.setMaxVersions(versions);
2168       List<Cell> actual = new ArrayList<Cell>();
2169       InternalScanner scanner = region.getScanner(scan);
2170 
2171       boolean hasNext = scanner.next(actual);
2172       assertEquals(false, hasNext);
2173 
2174       // Verify result
2175       for (int i = 0; i < expected.size(); i++) {
2176         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2177       }
2178     } finally {
2179       HRegion.closeHRegion(this.region);
2180       this.region = null;
2181     }
2182   }
2183 
2184   @Test
2185   public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2186     byte[] row1 = Bytes.toBytes("row1");
2187     byte[] qf1 = Bytes.toBytes("qualifier1");
2188     byte[] qf2 = Bytes.toBytes("qualifier2");
2189     byte[] fam1 = Bytes.toBytes("fam1");
2190     byte[][] families = { fam1 };
2191 
2192     long ts1 = System.currentTimeMillis();
2193     long ts2 = ts1 + 1;
2194     long ts3 = ts1 + 2;
2195 
2196     // Setting up region
2197     String method = this.getName();
2198     this.region = initHRegion(tableName, method, conf, families);
2199     try {
2200       // Putting data in Region
2201       Put put = null;
2202       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2203       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2204       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2205 
2206       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2207       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2208       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2209 
2210       put = new Put(row1);
2211       put.add(kv13);
2212       put.add(kv12);
2213       put.add(kv11);
2214       put.add(kv23);
2215       put.add(kv22);
2216       put.add(kv21);
2217       region.put(put);
2218 
2219       // Expected
2220       List<Cell> expected = new ArrayList<Cell>();
2221       expected.add(kv13);
2222       expected.add(kv12);
2223       expected.add(kv23);
2224       expected.add(kv22);
2225 
2226       Scan scan = new Scan(row1);
2227       scan.addFamily(fam1);
2228       scan.setMaxVersions(MAX_VERSIONS);
2229       List<Cell> actual = new ArrayList<Cell>();
2230       InternalScanner scanner = region.getScanner(scan);
2231 
2232       boolean hasNext = scanner.next(actual);
2233       assertEquals(false, hasNext);
2234 
2235       // Verify result
2236       for (int i = 0; i < expected.size(); i++) {
2237         assertEquals(expected.get(i), actual.get(i));
2238       }
2239     } finally {
2240       HRegion.closeHRegion(this.region);
2241       this.region = null;
2242     }
2243   }
2244 
2245   @Test
2246   public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
2247     byte[] row1 = Bytes.toBytes("row1");
2248     byte[] qf1 = Bytes.toBytes("qualifier1");
2249     byte[] qf2 = Bytes.toBytes("qualifier2");
2250     byte[] fam1 = Bytes.toBytes("fam1");
2251 
2252     long ts1 = 1; // System.currentTimeMillis();
2253     long ts2 = ts1 + 1;
2254     long ts3 = ts1 + 2;
2255 
2256     // Setting up region
2257     String method = this.getName();
2258     this.region = initHRegion(tableName, method, conf, fam1);
2259     try {
2260       // Putting data in Region
2261       Put put = null;
2262       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2263       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2264       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2265 
2266       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2267       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2268       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2269 
2270       put = new Put(row1);
2271       put.add(kv13);
2272       put.add(kv12);
2273       put.add(kv11);
2274       put.add(kv23);
2275       put.add(kv22);
2276       put.add(kv21);
2277       region.put(put);
2278       region.flushcache();
2279 
2280       // Expected
2281       List<Cell> expected = new ArrayList<Cell>();
2282       expected.add(kv13);
2283       expected.add(kv12);
2284       expected.add(kv23);
2285       expected.add(kv22);
2286 
2287       Scan scan = new Scan(row1);
2288       scan.addFamily(fam1);
2289       scan.setMaxVersions(MAX_VERSIONS);
2290       List<Cell> actual = new ArrayList<Cell>();
2291       InternalScanner scanner = region.getScanner(scan);
2292 
2293       boolean hasNext = scanner.next(actual);
2294       assertEquals(false, hasNext);
2295 
2296       // Verify result
2297       for (int i = 0; i < expected.size(); i++) {
2298         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2299       }
2300     } finally {
2301       HRegion.closeHRegion(this.region);
2302       this.region = null;
2303     }
2304   }
2305 
2306   @Test
2307   public void testScanner_StopRow1542() throws IOException {
2308     byte[] family = Bytes.toBytes("testFamily");
2309     this.region = initHRegion(tableName, getName(), conf, family);
2310     try {
2311       byte[] row1 = Bytes.toBytes("row111");
2312       byte[] row2 = Bytes.toBytes("row222");
2313       byte[] row3 = Bytes.toBytes("row333");
2314       byte[] row4 = Bytes.toBytes("row444");
2315       byte[] row5 = Bytes.toBytes("row555");
2316 
2317       byte[] col1 = Bytes.toBytes("Pub111");
2318       byte[] col2 = Bytes.toBytes("Pub222");
2319 
2320       Put put = new Put(row1);
2321       put.add(family, col1, Bytes.toBytes(10L));
2322       region.put(put);
2323 
2324       put = new Put(row2);
2325       put.add(family, col1, Bytes.toBytes(15L));
2326       region.put(put);
2327 
2328       put = new Put(row3);
2329       put.add(family, col2, Bytes.toBytes(20L));
2330       region.put(put);
2331 
2332       put = new Put(row4);
2333       put.add(family, col2, Bytes.toBytes(30L));
2334       region.put(put);
2335 
2336       put = new Put(row5);
2337       put.add(family, col1, Bytes.toBytes(40L));
2338       region.put(put);
2339 
2340       Scan scan = new Scan(row3, row4);
2341       scan.setMaxVersions();
2342       scan.addColumn(family, col1);
2343       InternalScanner s = region.getScanner(scan);
2344 
2345       List<Cell> results = new ArrayList<Cell>();
2346       assertEquals(false, s.next(results));
2347       assertEquals(0, results.size());
2348     } finally {
2349       HRegion.closeHRegion(this.region);
2350       this.region = null;
2351     }
2352   }
2353 
2354   private void assertICV(byte [] row,
2355                          byte [] familiy,
2356                          byte[] qualifier,
2357                          long amount) throws IOException {
2358     // run a get and see?
2359     Get get = new Get(row);
2360     get.addColumn(familiy, qualifier);
2361     Result result = region.get(get);
2362     assertEquals(1, result.size());
2363 
2364     Cell kv = result.rawCells()[0];
2365     long r = Bytes.toLong(CellUtil.cloneValue(kv));
2366     assertEquals(amount, r);
2367   }
2368 
2369   private void assertICV(byte[] row, byte[] familiy, byte[] qualifier, int amount)
2370       throws IOException {
2371     // run a get and see?
2372     Get get = new Get(row);
2373     get.addColumn(familiy, qualifier);
2374     Result result = region.get(get);
2375     assertEquals(1, result.size());
2376 
2377     Cell kv = result.rawCells()[0];
2378     int r = Bytes.toInt(CellUtil.cloneValue(kv));
2379     assertEquals(amount, r);
2380   }
2381 
2382   @Test
2383   public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2384     byte[] row1 = Bytes.toBytes("row1");
2385     byte[] fam1 = Bytes.toBytes("fam1");
2386     byte[] qf1 = Bytes.toBytes("qualifier1");
2387     byte[] qf2 = Bytes.toBytes("quateslifier2");
2388 
2389     long ts1 = 1;
2390     long ts2 = ts1 + 1;
2391     long ts3 = ts1 + 2;
2392     long ts4 = ts1 + 3;
2393 
2394     // Setting up region
2395     String method = this.getName();
2396     this.region = initHRegion(tableName, method, conf, fam1);
2397     try {
2398       // Putting data in Region
2399       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2400       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2401       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2402       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2403 
2404       KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2405       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2406       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2407       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2408 
2409       Put put = null;
2410       put = new Put(row1);
2411       put.add(kv14);
2412       put.add(kv24);
2413       region.put(put);
2414       region.flushcache();
2415 
2416       put = new Put(row1);
2417       put.add(kv23);
2418       put.add(kv13);
2419       region.put(put);
2420       region.flushcache();
2421 
2422       put = new Put(row1);
2423       put.add(kv22);
2424       put.add(kv12);
2425       region.put(put);
2426       region.flushcache();
2427 
2428       put = new Put(row1);
2429       put.add(kv21);
2430       put.add(kv11);
2431       region.put(put);
2432 
2433       // Expected
2434       List<KeyValue> expected = new ArrayList<KeyValue>();
2435       expected.add(kv14);
2436       expected.add(kv13);
2437       expected.add(kv12);
2438       expected.add(kv24);
2439       expected.add(kv23);
2440       expected.add(kv22);
2441 
2442       Scan scan = new Scan(row1);
2443       int versions = 3;
2444       scan.setMaxVersions(versions);
2445       List<Cell> actual = new ArrayList<Cell>();
2446       InternalScanner scanner = region.getScanner(scan);
2447 
2448       boolean hasNext = scanner.next(actual);
2449       assertEquals(false, hasNext);
2450 
2451       // Verify result
2452       for (int i = 0; i < expected.size(); i++) {
2453         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2454       }
2455     } finally {
2456       HRegion.closeHRegion(this.region);
2457       this.region = null;
2458     }
2459   }
2460 
2461   /**
2462    * Added for HBASE-5416
2463    * 
2464    * Here we test scan optimization when only subset of CFs are used in filter
2465    * conditions.
2466    */
2467   @Test
2468   public void testScanner_JoinedScanners() throws IOException {
2469     byte[] cf_essential = Bytes.toBytes("essential");
2470     byte[] cf_joined = Bytes.toBytes("joined");
2471     byte[] cf_alpha = Bytes.toBytes("alpha");
2472     this.region = initHRegion(tableName, getName(), conf, cf_essential, cf_joined, cf_alpha);
2473     try {
2474       byte[] row1 = Bytes.toBytes("row1");
2475       byte[] row2 = Bytes.toBytes("row2");
2476       byte[] row3 = Bytes.toBytes("row3");
2477 
2478       byte[] col_normal = Bytes.toBytes("d");
2479       byte[] col_alpha = Bytes.toBytes("a");
2480 
2481       byte[] filtered_val = Bytes.toBytes(3);
2482 
2483       Put put = new Put(row1);
2484       put.add(cf_essential, col_normal, Bytes.toBytes(1));
2485       put.add(cf_joined, col_alpha, Bytes.toBytes(1));
2486       region.put(put);
2487 
2488       put = new Put(row2);
2489       put.add(cf_essential, col_alpha, Bytes.toBytes(2));
2490       put.add(cf_joined, col_normal, Bytes.toBytes(2));
2491       put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
2492       region.put(put);
2493 
2494       put = new Put(row3);
2495       put.add(cf_essential, col_normal, filtered_val);
2496       put.add(cf_joined, col_normal, filtered_val);
2497       region.put(put);
2498 
2499       // Check two things:
2500       // 1. result list contains expected values
2501       // 2. result list is sorted properly
2502 
2503       Scan scan = new Scan();
2504       Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
2505           CompareOp.NOT_EQUAL, filtered_val);
2506       scan.setFilter(filter);
2507       scan.setLoadColumnFamiliesOnDemand(true);
2508       InternalScanner s = region.getScanner(scan);
2509 
2510       List<Cell> results = new ArrayList<Cell>();
2511       assertTrue(s.next(results));
2512       assertEquals(results.size(), 1);
2513       results.clear();
2514 
2515       assertTrue(s.next(results));
2516       assertEquals(results.size(), 3);
2517       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
2518       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
2519       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
2520       results.clear();
2521 
2522       assertFalse(s.next(results));
2523       assertEquals(results.size(), 0);
2524     } finally {
2525       HRegion.closeHRegion(this.region);
2526       this.region = null;
2527     }
2528   }
2529 
2530   /**
2531    * HBASE-5416
2532    * 
2533    * Test case when scan limits amount of KVs returned on each next() call.
2534    */
2535   @Test
2536   public void testScanner_JoinedScannersWithLimits() throws IOException {
2537     final byte[] cf_first = Bytes.toBytes("first");
2538     final byte[] cf_second = Bytes.toBytes("second");
2539 
2540     this.region = initHRegion(tableName, getName(), conf, cf_first, cf_second);
2541     try {
2542       final byte[] col_a = Bytes.toBytes("a");
2543       final byte[] col_b = Bytes.toBytes("b");
2544 
2545       Put put;
2546 
2547       for (int i = 0; i < 10; i++) {
2548         put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
2549         put.add(cf_first, col_a, Bytes.toBytes(i));
2550         if (i < 5) {
2551           put.add(cf_first, col_b, Bytes.toBytes(i));
2552           put.add(cf_second, col_a, Bytes.toBytes(i));
2553           put.add(cf_second, col_b, Bytes.toBytes(i));
2554         }
2555         region.put(put);
2556       }
2557 
2558       Scan scan = new Scan();
2559       scan.setLoadColumnFamiliesOnDemand(true);
2560       Filter bogusFilter = new FilterBase() {
2561         @Override
2562         public boolean isFamilyEssential(byte[] name) {
2563           return Bytes.equals(name, cf_first);
2564         }
2565       };
2566 
2567       scan.setFilter(bogusFilter);
2568       InternalScanner s = region.getScanner(scan);
2569 
2570       // Our data looks like this:
2571       // r0: first:a, first:b, second:a, second:b
2572       // r1: first:a, first:b, second:a, second:b
2573       // r2: first:a, first:b, second:a, second:b
2574       // r3: first:a, first:b, second:a, second:b
2575       // r4: first:a, first:b, second:a, second:b
2576       // r5: first:a
2577       // r6: first:a
2578       // r7: first:a
2579       // r8: first:a
2580       // r9: first:a
2581 
2582       // But due to next's limit set to 3, we should get this:
2583       // r0: first:a, first:b, second:a
2584       // r0: second:b
2585       // r1: first:a, first:b, second:a
2586       // r1: second:b
2587       // r2: first:a, first:b, second:a
2588       // r2: second:b
2589       // r3: first:a, first:b, second:a
2590       // r3: second:b
2591       // r4: first:a, first:b, second:a
2592       // r4: second:b
2593       // r5: first:a
2594       // r6: first:a
2595       // r7: first:a
2596       // r8: first:a
2597       // r9: first:a
2598 
2599       List<Cell> results = new ArrayList<Cell>();
2600       int index = 0;
2601       while (true) {
2602         boolean more = s.next(results, 3);
2603         if ((index >> 1) < 5) {
2604           if (index % 2 == 0)
2605             assertEquals(results.size(), 3);
2606           else
2607             assertEquals(results.size(), 1);
2608         } else
2609           assertEquals(results.size(), 1);
2610         results.clear();
2611         index++;
2612         if (!more)
2613           break;
2614       }
2615     } finally {
2616       HRegion.closeHRegion(this.region);
2617       this.region = null;
2618     }
2619   }
2620 
2621   // ////////////////////////////////////////////////////////////////////////////
2622   // Split test
2623   // ////////////////////////////////////////////////////////////////////////////
2624   /**
2625    * Splits twice and verifies getting from each of the split regions.
2626    * 
2627    * @throws Exception
2628    */
2629   @Test
2630   public void testBasicSplit() throws Exception {
2631     byte[][] families = { fam1, fam2, fam3 };
2632 
2633     Configuration hc = initSplit();
2634     // Setting up region
2635     String method = this.getName();
2636     this.region = initHRegion(tableName, method, hc, families);
2637 
2638     try {
2639       LOG.info("" + HBaseTestCase.addContent(region, fam3));
2640       region.flushcache();
2641       region.compactStores();
2642       byte[] splitRow = region.checkSplit();
2643       assertNotNull(splitRow);
2644       LOG.info("SplitRow: " + Bytes.toString(splitRow));
2645       HRegion[] regions = splitRegion(region, splitRow);
2646       try {
2647         // Need to open the regions.
2648         // TODO: Add an 'open' to HRegion... don't do open by constructing
2649         // instance.
2650         for (int i = 0; i < regions.length; i++) {
2651           regions[i] = HRegion.openHRegion(regions[i], null);
2652         }
2653         // Assert can get rows out of new regions. Should be able to get first
2654         // row from first region and the midkey from second region.
2655         assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
2656         assertGet(regions[1], fam3, splitRow);
2657         // Test I can get scanner and that it starts at right place.
2658         assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
2659         assertScan(regions[1], fam3, splitRow);
2660         // Now prove can't split regions that have references.
2661         for (int i = 0; i < regions.length; i++) {
2662           // Add so much data to this region, we create a store file that is >
2663           // than one of our unsplitable references. it will.
2664           for (int j = 0; j < 2; j++) {
2665             HBaseTestCase.addContent(regions[i], fam3);
2666           }
2667           HBaseTestCase.addContent(regions[i], fam2);
2668           HBaseTestCase.addContent(regions[i], fam1);
2669           regions[i].flushcache();
2670         }
2671 
2672         byte[][] midkeys = new byte[regions.length][];
2673         // To make regions splitable force compaction.
2674         for (int i = 0; i < regions.length; i++) {
2675           regions[i].compactStores();
2676           midkeys[i] = regions[i].checkSplit();
2677         }
2678 
2679         TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
2680         // Split these two daughter regions so then I'll have 4 regions. Will
2681         // split because added data above.
2682         for (int i = 0; i < regions.length; i++) {
2683           HRegion[] rs = null;
2684           if (midkeys[i] != null) {
2685             rs = splitRegion(regions[i], midkeys[i]);
2686             for (int j = 0; j < rs.length; j++) {
2687               sortedMap.put(Bytes.toString(rs[j].getRegionName()), HRegion.openHRegion(rs[j], null));
2688             }
2689           }
2690         }
2691         LOG.info("Made 4 regions");
2692         // The splits should have been even. Test I can get some arbitrary row
2693         // out of each.
2694         int interval = (LAST_CHAR - FIRST_CHAR) / 3;
2695         byte[] b = Bytes.toBytes(START_KEY);
2696         for (HRegion r : sortedMap.values()) {
2697           assertGet(r, fam3, b);
2698           b[0] += interval;
2699         }
2700       } finally {
2701         for (int i = 0; i < regions.length; i++) {
2702           try {
2703             regions[i].close();
2704           } catch (IOException e) {
2705             // Ignore.
2706           }
2707         }
2708       }
2709     } finally {
2710       HRegion.closeHRegion(this.region);
2711       this.region = null;
2712     }
2713   }
2714 
2715   @Test
2716   public void testSplitRegion() throws IOException {
2717     byte[] qualifier = Bytes.toBytes("qualifier");
2718     Configuration hc = initSplit();
2719     int numRows = 10;
2720     byte[][] families = { fam1, fam3 };
2721 
2722     // Setting up region
2723     String method = this.getName();
2724     this.region = initHRegion(tableName, method, hc, families);
2725 
2726     // Put data in region
2727     int startRow = 100;
2728     putData(startRow, numRows, qualifier, families);
2729     int splitRow = startRow + numRows;
2730     putData(splitRow, numRows, qualifier, families);
2731     region.flushcache();
2732 
2733     HRegion[] regions = null;
2734     try {
2735       regions = splitRegion(region, Bytes.toBytes("" + splitRow));
2736       // Opening the regions returned.
2737       for (int i = 0; i < regions.length; i++) {
2738         regions[i] = HRegion.openHRegion(regions[i], null);
2739       }
2740       // Verifying that the region has been split
2741       assertEquals(2, regions.length);
2742 
2743       // Verifying that all data is still there and that data is in the right
2744       // place
2745       verifyData(regions[0], startRow, numRows, qualifier, families);
2746       verifyData(regions[1], splitRow, numRows, qualifier, families);
2747 
2748     } finally {
2749       HRegion.closeHRegion(this.region);
2750       this.region = null;
2751     }
2752   }
2753 
2754   /**
2755    * Flushes the cache in a thread while scanning. The tests verify that the
2756    * scan is coherent - e.g. the returned results are always of the same or
2757    * later update as the previous results.
2758    * 
2759    * @throws IOException
2760    *           scan / compact
2761    * @throws InterruptedException
2762    *           thread join
2763    */
2764   @Test
2765   public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
2766     byte[] family = Bytes.toBytes("family");
2767     int numRows = 1000;
2768     int flushAndScanInterval = 10;
2769     int compactInterval = 10 * flushAndScanInterval;
2770 
2771     String method = "testFlushCacheWhileScanning";
2772     this.region = initHRegion(tableName, method, conf, family);
2773     try {
2774       FlushThread flushThread = new FlushThread();
2775       flushThread.start();
2776 
2777       Scan scan = new Scan();
2778       scan.addFamily(family);
2779       scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
2780           new BinaryComparator(Bytes.toBytes(5L))));
2781 
2782       int expectedCount = 0;
2783       List<Cell> res = new ArrayList<Cell>();
2784 
2785       boolean toggle = true;
2786       for (long i = 0; i < numRows; i++) {
2787         Put put = new Put(Bytes.toBytes(i));
2788         put.setDurability(Durability.SKIP_WAL);
2789         put.add(family, qual1, Bytes.toBytes(i % 10));
2790         region.put(put);
2791 
2792         if (i != 0 && i % compactInterval == 0) {
2793           // System.out.println("iteration = " + i);
2794           region.compactStores(true);
2795         }
2796 
2797         if (i % 10 == 5L) {
2798           expectedCount++;
2799         }
2800 
2801         if (i != 0 && i % flushAndScanInterval == 0) {
2802           res.clear();
2803           InternalScanner scanner = region.getScanner(scan);
2804           if (toggle) {
2805             flushThread.flush();
2806           }
2807           while (scanner.next(res))
2808             ;
2809           if (!toggle) {
2810             flushThread.flush();
2811           }
2812           assertEquals("i=" + i, expectedCount, res.size());
2813           toggle = !toggle;
2814         }
2815       }
2816 
2817       flushThread.done();
2818       flushThread.join();
2819       flushThread.checkNoError();
2820     } finally {
2821       HRegion.closeHRegion(this.region);
2822       this.region = null;
2823     }
2824   }
2825 
2826   protected class FlushThread extends Thread {
2827     private volatile boolean done;
2828     private Throwable error = null;
2829 
2830     public void done() {
2831       done = true;
2832       synchronized (this) {
2833         interrupt();
2834       }
2835     }
2836 
2837     public void checkNoError() {
2838       if (error != null) {
2839         assertNull(error);
2840       }
2841     }
2842 
2843     @Override
2844     public void run() {
2845       done = false;
2846       while (!done) {
2847         synchronized (this) {
2848           try {
2849             wait();
2850           } catch (InterruptedException ignored) {
2851             if (done) {
2852               break;
2853             }
2854           }
2855         }
2856         try {
2857           region.flushcache();
2858         } catch (IOException e) {
2859           if (!done) {
2860             LOG.error("Error while flusing cache", e);
2861             error = e;
2862           }
2863           break;
2864         }
2865       }
2866 
2867     }
2868 
2869     public void flush() {
2870       synchronized (this) {
2871         notify();
2872       }
2873 
2874     }
2875   }
2876 
2877   /**
2878    * Writes very wide records and scans for the latest every time.. Flushes and
2879    * compacts the region every now and then to keep things realistic.
2880    * 
2881    * @throws IOException
2882    *           by flush / scan / compaction
2883    * @throws InterruptedException
2884    *           when joining threads
2885    */
2886   @Test
2887   public void testWritesWhileScanning() throws IOException, InterruptedException {
2888     int testCount = 100;
2889     int numRows = 1;
2890     int numFamilies = 10;
2891     int numQualifiers = 100;
2892     int flushInterval = 7;
2893     int compactInterval = 5 * flushInterval;
2894     byte[][] families = new byte[numFamilies][];
2895     for (int i = 0; i < numFamilies; i++) {
2896       families[i] = Bytes.toBytes("family" + i);
2897     }
2898     byte[][] qualifiers = new byte[numQualifiers][];
2899     for (int i = 0; i < numQualifiers; i++) {
2900       qualifiers[i] = Bytes.toBytes("qual" + i);
2901     }
2902 
2903     String method = "testWritesWhileScanning";
2904     this.region = initHRegion(tableName, method, conf, families);
2905     try {
2906       PutThread putThread = new PutThread(numRows, families, qualifiers);
2907       putThread.start();
2908       putThread.waitForFirstPut();
2909 
2910       FlushThread flushThread = new FlushThread();
2911       flushThread.start();
2912 
2913       Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
2914 
2915       int expectedCount = numFamilies * numQualifiers;
2916       List<Cell> res = new ArrayList<Cell>();
2917 
2918       long prevTimestamp = 0L;
2919       for (int i = 0; i < testCount; i++) {
2920 
2921         if (i != 0 && i % compactInterval == 0) {
2922           region.compactStores(true);
2923         }
2924 
2925         if (i != 0 && i % flushInterval == 0) {
2926           flushThread.flush();
2927         }
2928 
2929         boolean previousEmpty = res.isEmpty();
2930         res.clear();
2931         InternalScanner scanner = region.getScanner(scan);
2932         while (scanner.next(res))
2933           ;
2934         if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
2935           assertEquals("i=" + i, expectedCount, res.size());
2936           long timestamp = res.get(0).getTimestamp();
2937           assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
2938               timestamp >= prevTimestamp);
2939           prevTimestamp = timestamp;
2940         }
2941       }
2942 
2943       putThread.done();
2944 
2945       region.flushcache();
2946 
2947       putThread.join();
2948       putThread.checkNoError();
2949 
2950       flushThread.done();
2951       flushThread.join();
2952       flushThread.checkNoError();
2953     } finally {
2954       HRegion.closeHRegion(this.region);
2955       this.region = null;
2956     }
2957   }
2958 
2959   protected class PutThread extends Thread {
2960     private volatile boolean done;
2961     private volatile int numPutsFinished = 0;
2962 
2963     private Throwable error = null;
2964     private int numRows;
2965     private byte[][] families;
2966     private byte[][] qualifiers;
2967 
2968     private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
2969       this.numRows = numRows;
2970       this.families = families;
2971       this.qualifiers = qualifiers;
2972     }
2973 
2974     /**
2975      * Block until this thread has put at least one row.
2976      */
2977     public void waitForFirstPut() throws InterruptedException {
2978       // wait until put thread actually puts some data
2979       while (numPutsFinished == 0) {
2980         checkNoError();
2981         Thread.sleep(50);
2982       }
2983     }
2984 
2985     public void done() {
2986       done = true;
2987       synchronized (this) {
2988         interrupt();
2989       }
2990     }
2991 
2992     public void checkNoError() {
2993       if (error != null) {
2994         assertNull(error);
2995       }
2996     }
2997 
2998     @Override
2999     public void run() {
3000       done = false;
3001       while (!done) {
3002         try {
3003           for (int r = 0; r < numRows; r++) {
3004             byte[] row = Bytes.toBytes("row" + r);
3005             Put put = new Put(row);
3006             put.setDurability(Durability.SKIP_WAL);
3007             byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3008             for (byte[] family : families) {
3009               for (byte[] qualifier : qualifiers) {
3010                 put.add(family, qualifier, (long) numPutsFinished, value);
3011               }
3012             }
3013             region.put(put);
3014             numPutsFinished++;
3015             if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3016               System.out.println("put iteration = " + numPutsFinished);
3017               Delete delete = new Delete(row, (long) numPutsFinished - 30);
3018               region.delete(delete);
3019             }
3020             numPutsFinished++;
3021           }
3022         } catch (InterruptedIOException e) {
3023           // This is fine. It means we are done, or didn't get the lock on time
3024         } catch (IOException e) {
3025           LOG.error("error while putting records", e);
3026           error = e;
3027           break;
3028         }
3029       }
3030 
3031     }
3032 
3033   }
3034 
3035   /**
3036    * Writes very wide records and gets the latest row every time.. Flushes and
3037    * compacts the region aggressivly to catch issues.
3038    * 
3039    * @throws IOException
3040    *           by flush / scan / compaction
3041    * @throws InterruptedException
3042    *           when joining threads
3043    */
3044   @Test
3045   public void testWritesWhileGetting() throws Exception {
3046     int testCount = 100;
3047     int numRows = 1;
3048     int numFamilies = 10;
3049     int numQualifiers = 100;
3050     int compactInterval = 100;
3051     byte[][] families = new byte[numFamilies][];
3052     for (int i = 0; i < numFamilies; i++) {
3053       families[i] = Bytes.toBytes("family" + i);
3054     }
3055     byte[][] qualifiers = new byte[numQualifiers][];
3056     for (int i = 0; i < numQualifiers; i++) {
3057       qualifiers[i] = Bytes.toBytes("qual" + i);
3058     }
3059 
3060 
3061     String method = "testWritesWhileGetting";
3062     // This test flushes constantly and can cause many files to be created,
3063     // possibly
3064     // extending over the ulimit. Make sure compactions are aggressive in
3065     // reducing
3066     // the number of HFiles created.
3067     conf.setInt("hbase.hstore.compaction.min", 1);
3068     conf.setInt("hbase.hstore.compaction.max", 1000);
3069     this.region = initHRegion(tableName, method, conf, families);
3070     PutThread putThread = null;
3071     MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3072     try {
3073       putThread = new PutThread(numRows, families, qualifiers);
3074       putThread.start();
3075       putThread.waitForFirstPut();
3076 
3077       // Add a thread that flushes as fast as possible
3078       ctx.addThread(new RepeatingTestThread(ctx) {
3079         private int flushesSinceCompact = 0;
3080         private final int maxFlushesSinceCompact = 20;
3081 
3082         @Override
3083         public void doAnAction() throws Exception {
3084           if (region.flushcache()) {
3085             ++flushesSinceCompact;
3086           }
3087           // Compact regularly to avoid creating too many files and exceeding
3088           // the ulimit.
3089           if (flushesSinceCompact == maxFlushesSinceCompact) {
3090             region.compactStores(false);
3091             flushesSinceCompact = 0;
3092           }
3093         }
3094       });
3095       ctx.startThreads();
3096 
3097       Get get = new Get(Bytes.toBytes("row0"));
3098       Result result = null;
3099 
3100       int expectedCount = numFamilies * numQualifiers;
3101 
3102       long prevTimestamp = 0L;
3103       for (int i = 0; i < testCount; i++) {
3104 
3105         boolean previousEmpty = result == null || result.isEmpty();
3106         result = region.get(get);
3107         if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3108           assertEquals("i=" + i, expectedCount, result.size());
3109           // TODO this was removed, now what dangit?!
3110           // search looking for the qualifier in question?
3111           long timestamp = 0;
3112           for (Cell kv : result.rawCells()) {
3113             if (CellUtil.matchingFamily(kv, families[0])
3114                 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3115               timestamp = kv.getTimestamp();
3116             }
3117           }
3118           assertTrue(timestamp >= prevTimestamp);
3119           prevTimestamp = timestamp;
3120           Cell previousKV = null;
3121 
3122           for (Cell kv : result.rawCells()) {
3123             byte[] thisValue = CellUtil.cloneValue(kv);
3124             if (previousKV != null) {
3125               if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3126                 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3127                     + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3128                     + "(memStoreTS:" + kv.getMvccVersion() + ")");
3129                 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3130               }
3131             }
3132             previousKV = kv;
3133           }
3134         }
3135       }
3136     } finally {
3137       if (putThread != null)
3138         putThread.done();
3139 
3140       region.flushcache();
3141 
3142       if (putThread != null) {
3143         putThread.join();
3144         putThread.checkNoError();
3145       }
3146 
3147       ctx.stop();
3148       HRegion.closeHRegion(this.region);
3149       this.region = null;
3150     }
3151   }
3152 
3153   @Test
3154   public void testHolesInMeta() throws Exception {
3155     byte[] family = Bytes.toBytes("family");
3156     this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, conf,
3157         false, family);
3158     try {
3159       byte[] rowNotServed = Bytes.toBytes("a");
3160       Get g = new Get(rowNotServed);
3161       try {
3162         region.get(g);
3163         fail();
3164       } catch (WrongRegionException x) {
3165         // OK
3166       }
3167       byte[] row = Bytes.toBytes("y");
3168       g = new Get(row);
3169       region.get(g);
3170     } finally {
3171       HRegion.closeHRegion(this.region);
3172       this.region = null;
3173     }
3174   }
3175 
3176   @Test
3177   public void testIndexesScanWithOneDeletedRow() throws IOException {
3178     byte[] family = Bytes.toBytes("family");
3179 
3180     // Setting up region
3181     String method = "testIndexesScanWithOneDeletedRow";
3182     this.region = initHRegion(tableName, method, conf, family);
3183     try {
3184       Put put = new Put(Bytes.toBytes(1L));
3185       put.add(family, qual1, 1L, Bytes.toBytes(1L));
3186       region.put(put);
3187 
3188       region.flushcache();
3189 
3190       Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3191       region.delete(delete);
3192 
3193       put = new Put(Bytes.toBytes(2L));
3194       put.add(family, qual1, 2L, Bytes.toBytes(2L));
3195       region.put(put);
3196 
3197       Scan idxScan = new Scan();
3198       idxScan.addFamily(family);
3199       idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
3200           new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
3201               new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
3202               CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
3203       InternalScanner scanner = region.getScanner(idxScan);
3204       List<Cell> res = new ArrayList<Cell>();
3205 
3206       while (scanner.next(res))
3207         ;
3208       assertEquals(1L, res.size());
3209     } finally {
3210       HRegion.closeHRegion(this.region);
3211       this.region = null;
3212     }
3213   }
3214 
3215   // ////////////////////////////////////////////////////////////////////////////
3216   // Bloom filter test
3217   // ////////////////////////////////////////////////////////////////////////////
3218   @Test
3219   public void testBloomFilterSize() throws IOException {
3220     byte[] row1 = Bytes.toBytes("row1");
3221     byte[] fam1 = Bytes.toBytes("fam1");
3222     byte[] qf1 = Bytes.toBytes("col");
3223     byte[] val1 = Bytes.toBytes("value1");
3224     // Create Table
3225     HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
3226         .setBloomFilterType(BloomType.ROWCOL);
3227 
3228     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3229     htd.addFamily(hcd);
3230     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3231     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3232     try {
3233       int num_unique_rows = 10;
3234       int duplicate_multiplier = 2;
3235       int num_storefiles = 4;
3236 
3237       int version = 0;
3238       for (int f = 0; f < num_storefiles; f++) {
3239         for (int i = 0; i < duplicate_multiplier; i++) {
3240           for (int j = 0; j < num_unique_rows; j++) {
3241             Put put = new Put(Bytes.toBytes("row" + j));
3242             put.setDurability(Durability.SKIP_WAL);
3243             put.add(fam1, qf1, version++, val1);
3244             region.put(put);
3245           }
3246         }
3247         region.flushcache();
3248       }
3249       // before compaction
3250       HStore store = (HStore) region.getStore(fam1);
3251       Collection<StoreFile> storeFiles = store.getStorefiles();
3252       for (StoreFile storefile : storeFiles) {
3253         StoreFile.Reader reader = storefile.getReader();
3254         reader.loadFileInfo();
3255         reader.loadBloomfilter();
3256         assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
3257         assertEquals(num_unique_rows, reader.getFilterEntries());
3258       }
3259 
3260       region.compactStores(true);
3261 
3262       // after compaction
3263       storeFiles = store.getStorefiles();
3264       for (StoreFile storefile : storeFiles) {
3265         StoreFile.Reader reader = storefile.getReader();
3266         reader.loadFileInfo();
3267         reader.loadBloomfilter();
3268         assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
3269         assertEquals(num_unique_rows, reader.getFilterEntries());
3270       }
3271     } finally {
3272       HRegion.closeHRegion(this.region);
3273       this.region = null;
3274     }
3275   }
3276 
3277   @Test
3278   public void testAllColumnsWithBloomFilter() throws IOException {
3279     byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
3280     byte[] FAMILY = Bytes.toBytes("family");
3281 
3282     // Create table
3283     HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
3284         .setBloomFilterType(BloomType.ROWCOL);
3285     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
3286     htd.addFamily(hcd);
3287     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3288     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3289     try {
3290       // For row:0, col:0: insert versions 1 through 5.
3291       byte row[] = Bytes.toBytes("row:" + 0);
3292       byte column[] = Bytes.toBytes("column:" + 0);
3293       Put put = new Put(row);
3294       put.setDurability(Durability.SKIP_WAL);
3295       for (long idx = 1; idx <= 4; idx++) {
3296         put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
3297       }
3298       region.put(put);
3299 
3300       // Flush
3301       region.flushcache();
3302 
3303       // Get rows
3304       Get get = new Get(row);
3305       get.setMaxVersions();
3306       Cell[] kvs = region.get(get).rawCells();
3307 
3308       // Check if rows are correct
3309       assertEquals(4, kvs.length);
3310       checkOneCell(kvs[0], FAMILY, 0, 0, 4);
3311       checkOneCell(kvs[1], FAMILY, 0, 0, 3);
3312       checkOneCell(kvs[2], FAMILY, 0, 0, 2);
3313       checkOneCell(kvs[3], FAMILY, 0, 0, 1);
3314     } finally {
3315       HRegion.closeHRegion(this.region);
3316       this.region = null;
3317     }
3318   }
3319 
3320   /**
3321    * Testcase to cover bug-fix for HBASE-2823 Ensures correct delete when
3322    * issuing delete row on columns with bloom filter set to row+col
3323    * (BloomType.ROWCOL)
3324    */
3325   @Test
3326   public void testDeleteRowWithBloomFilter() throws IOException {
3327     byte[] familyName = Bytes.toBytes("familyName");
3328 
3329     // Create Table
3330     HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
3331         .setBloomFilterType(BloomType.ROWCOL);
3332 
3333     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3334     htd.addFamily(hcd);
3335     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3336     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3337     try {
3338       // Insert some data
3339       byte row[] = Bytes.toBytes("row1");
3340       byte col[] = Bytes.toBytes("col1");
3341 
3342       Put put = new Put(row);
3343       put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
3344       region.put(put);
3345       region.flushcache();
3346 
3347       Delete del = new Delete(row);
3348       region.delete(del);
3349       region.flushcache();
3350 
3351       // Get remaining rows (should have none)
3352       Get get = new Get(row);
3353       get.addColumn(familyName, col);
3354 
3355       Cell[] keyValues = region.get(get).rawCells();
3356       assertTrue(keyValues.length == 0);
3357     } finally {
3358       HRegion.closeHRegion(this.region);
3359       this.region = null;
3360     }
3361   }
3362 
3363   @Test
3364   public void testgetHDFSBlocksDistribution() throws Exception {
3365     HBaseTestingUtility htu = new HBaseTestingUtility();
3366     final int DEFAULT_BLOCK_SIZE = 1024;
3367     htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
3368     htu.getConfiguration().setInt("dfs.replication", 2);
3369 
3370     // set up a cluster with 3 nodes
3371     MiniHBaseCluster cluster = null;
3372     String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
3373     int regionServersCount = 3;
3374 
3375     try {
3376       cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
3377       byte[][] families = { fam1, fam2 };
3378       HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);
3379 
3380       // Setting up region
3381       byte row[] = Bytes.toBytes("row1");
3382       byte col[] = Bytes.toBytes("col1");
3383 
3384       Put put = new Put(row);
3385       put.add(fam1, col, 1, Bytes.toBytes("test1"));
3386       put.add(fam2, col, 1, Bytes.toBytes("test2"));
3387       ht.put(put);
3388 
3389       HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
3390           .get(0);
3391       firstRegion.flushcache();
3392       HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
3393 
3394       // given the default replication factor is 2 and we have 2 HFiles,
3395       // we will have total of 4 replica of blocks on 3 datanodes; thus there
3396       // must be at least one host that have replica for 2 HFiles. That host's
3397       // weight will be equal to the unique block weight.
3398       long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
3399 
3400       String topHost = blocksDistribution1.getTopHosts().get(0);
3401       long topHostWeight = blocksDistribution1.getWeight(topHost);
3402       assertTrue(uniqueBlocksWeight1 == topHostWeight);
3403 
3404       // use the static method to compute the value, it should be the same.
3405       // static method is used by load balancer or other components
3406       HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
3407           htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
3408       long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
3409 
3410       assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
3411 
3412       ht.close();
3413     } finally {
3414       if (cluster != null) {
3415         htu.shutdownMiniCluster();
3416       }
3417     }
3418   }
3419 
3420   /**
3421    * Testcase to check state of region initialization task set to ABORTED or not
3422    * if any exceptions during initialization
3423    * 
3424    * @throws Exception
3425    */
3426   @Test
3427   public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
3428     TableName tableName = TableName.valueOf(name.getMethodName());
3429     HRegionInfo info = null;
3430     try {
3431       FileSystem fs = Mockito.mock(FileSystem.class);
3432       Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
3433       HTableDescriptor htd = new HTableDescriptor(tableName);
3434       htd.addFamily(new HColumnDescriptor("cf"));
3435       info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
3436           HConstants.EMPTY_BYTE_ARRAY, false);
3437       Path path = new Path(DIR + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
3438       region = HRegion.newHRegion(path, null, fs, conf, info, htd, null);
3439       // region initialization throws IOException and set task state to ABORTED.
3440       region.initialize();
3441       fail("Region initialization should fail due to IOException");
3442     } catch (IOException io) {
3443       List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
3444       for (MonitoredTask monitoredTask : tasks) {
3445         if (!(monitoredTask instanceof MonitoredRPCHandler)
3446             && monitoredTask.getDescription().contains(region.toString())) {
3447           assertTrue("Region state should be ABORTED.",
3448               monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
3449           break;
3450         }
3451       }
3452     } finally {
3453       HRegion.closeHRegion(region);
3454     }
3455   }
3456 
3457   /**
3458    * Verifies that the .regioninfo file is written on region creation and that
3459    * is recreated if missing during region opening.
3460    */
3461   @Test
3462   public void testRegionInfoFileCreation() throws IOException {
3463     Path rootDir = new Path(DIR + "testRegionInfoFileCreation");
3464 
3465     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
3466     htd.addFamily(new HColumnDescriptor("cf"));
3467 
3468     HRegionInfo hri = new HRegionInfo(htd.getTableName());
3469 
3470     // Create a region and skip the initialization (like CreateTableHandler)
3471     HRegion region = HRegion.createHRegion(hri, rootDir, conf, htd, null, false, true);
3472 //    HRegion region = TEST_UTIL.createLocalHRegion(hri, htd);
3473     Path regionDir = region.getRegionFileSystem().getRegionDir();
3474     FileSystem fs = region.getRegionFileSystem().getFileSystem();
3475     HRegion.closeHRegion(region);
3476 
3477     Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
3478 
3479     // Verify that the .regioninfo file is present
3480     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3481         fs.exists(regionInfoFile));
3482 
3483     // Try to open the region
3484     region = HRegion.openHRegion(rootDir, hri, htd, null, conf);
3485     assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3486     HRegion.closeHRegion(region);
3487 
3488     // Verify that the .regioninfo file is still there
3489     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3490         fs.exists(regionInfoFile));
3491 
3492     // Remove the .regioninfo file and verify is recreated on region open
3493     fs.delete(regionInfoFile);
3494     assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
3495         fs.exists(regionInfoFile));
3496 
3497     region = HRegion.openHRegion(rootDir, hri, htd, null, conf);
3498 //    region = TEST_UTIL.openHRegion(hri, htd);
3499     assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3500     HRegion.closeHRegion(region);
3501 
3502     // Verify that the .regioninfo file is still there
3503     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3504         fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
3505   }
3506 
3507   /**
3508    * TestCase for increment
3509    */
3510   private static class Incrementer implements Runnable {
3511     private HRegion region;
3512     private final static byte[] incRow = Bytes.toBytes("incRow");
3513     private final static byte[] family = Bytes.toBytes("family");
3514     private final static byte[] qualifier = Bytes.toBytes("qualifier");
3515     private final static long ONE = 1l;
3516     private int incCounter;
3517 
3518     public Incrementer(HRegion region, int incCounter) {
3519       this.region = region;
3520       this.incCounter = incCounter;
3521     }
3522 
3523     @Override
3524     public void run() {
3525       int count = 0;
3526       while (count < incCounter) {
3527         Increment inc = new Increment(incRow);
3528         inc.addColumn(family, qualifier, ONE);
3529         count++;
3530         try {
3531           region.increment(inc);
3532         } catch (IOException e) {
3533           e.printStackTrace();
3534           break;
3535         }
3536       }
3537     }
3538   }
3539 
3540   /**
3541    * Test case to check increment function with memstore flushing
3542    * @throws Exception
3543    */
3544   @Test
3545   public void testParallelIncrementWithMemStoreFlush() throws Exception {
3546     byte[] family = Incrementer.family;
3547     this.region = initHRegion(tableName, method, conf, family);
3548     final HRegion region = this.region;
3549     final AtomicBoolean incrementDone = new AtomicBoolean(false);
3550     Runnable flusher = new Runnable() {
3551       @Override
3552       public void run() {
3553         while (!incrementDone.get()) {
3554           try {
3555             region.flushcache();
3556           } catch (Exception e) {
3557             e.printStackTrace();
3558           }
3559         }
3560       }
3561     };
3562 
3563     // after all increment finished, the row will increment to 20*100 = 2000
3564     int threadNum = 20;
3565     int incCounter = 100;
3566     long expected = threadNum * incCounter;
3567     Thread[] incrementers = new Thread[threadNum];
3568     Thread flushThread = new Thread(flusher);
3569     for (int i = 0; i < threadNum; i++) {
3570       incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
3571       incrementers[i].start();
3572     }
3573     flushThread.start();
3574     for (int i = 0; i < threadNum; i++) {
3575       incrementers[i].join();
3576     }
3577 
3578     incrementDone.set(true);
3579     flushThread.join();
3580 
3581     Get get = new Get(Incrementer.incRow);
3582     get.addColumn(Incrementer.family, Incrementer.qualifier);
3583     get.setMaxVersions(1);
3584     Result res = this.region.get(get);
3585     List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
3586 
3587     // we just got the latest version
3588     assertEquals(kvs.size(), 1);
3589     Cell kv = kvs.get(0);
3590     assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
3591     this.region = null;
3592   }
3593 
3594   /**
3595    * TestCase for append
3596    */
3597   private static class Appender implements Runnable {
3598     private HRegion region;
3599     private final static byte[] appendRow = Bytes.toBytes("appendRow");
3600     private final static byte[] family = Bytes.toBytes("family");
3601     private final static byte[] qualifier = Bytes.toBytes("qualifier");
3602     private final static byte[] CHAR = Bytes.toBytes("a");
3603     private int appendCounter;
3604 
3605     public Appender(HRegion region, int appendCounter) {
3606       this.region = region;
3607       this.appendCounter = appendCounter;
3608     }
3609 
3610     @Override
3611     public void run() {
3612       int count = 0;
3613       while (count < appendCounter) {
3614         Append app = new Append(appendRow);
3615         app.add(family, qualifier, CHAR);
3616         count++;
3617         try {
3618           region.append(app);
3619         } catch (IOException e) {
3620           e.printStackTrace();
3621           break;
3622         }
3623       }
3624     }
3625   }
3626 
3627   /**
3628    * Test case to check append function with memstore flushing
3629    * @throws Exception
3630    */
3631   @Test
3632   public void testParallelAppendWithMemStoreFlush() throws Exception {
3633     byte[] family = Appender.family;
3634     this.region = initHRegion(tableName, method, conf, family);
3635     final HRegion region = this.region;
3636     final AtomicBoolean appendDone = new AtomicBoolean(false);
3637     Runnable flusher = new Runnable() {
3638       @Override
3639       public void run() {
3640         while (!appendDone.get()) {
3641           try {
3642             region.flushcache();
3643           } catch (Exception e) {
3644             e.printStackTrace();
3645           }
3646         }
3647       }
3648     };
3649 
3650     // after all append finished, the value will append to threadNum *
3651     // appendCounter Appender.CHAR
3652     int threadNum = 20;
3653     int appendCounter = 100;
3654     byte[] expected = new byte[threadNum * appendCounter];
3655     for (int i = 0; i < threadNum * appendCounter; i++) {
3656       System.arraycopy(Appender.CHAR, 0, expected, i, 1);
3657     }
3658     Thread[] appenders = new Thread[threadNum];
3659     Thread flushThread = new Thread(flusher);
3660     for (int i = 0; i < threadNum; i++) {
3661       appenders[i] = new Thread(new Appender(this.region, appendCounter));
3662       appenders[i].start();
3663     }
3664     flushThread.start();
3665     for (int i = 0; i < threadNum; i++) {
3666       appenders[i].join();
3667     }
3668 
3669     appendDone.set(true);
3670     flushThread.join();
3671 
3672     Get get = new Get(Appender.appendRow);
3673     get.addColumn(Appender.family, Appender.qualifier);
3674     get.setMaxVersions(1);
3675     Result res = this.region.get(get);
3676     List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
3677 
3678     // we just got the latest version
3679     assertEquals(kvs.size(), 1);
3680     Cell kv = kvs.get(0);
3681     byte[] appendResult = new byte[kv.getValueLength()];
3682     System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
3683     assertArrayEquals(expected, appendResult);
3684     this.region = null;
3685   }
3686 
3687   /**
3688    * Test case to check put function with memstore flushing for same row, same ts
3689    * @throws Exception
3690    */
3691   @Test
3692   public void testPutWithMemStoreFlush() throws Exception {
3693     byte[] family = Bytes.toBytes("family");
3694     ;
3695     byte[] qualifier = Bytes.toBytes("qualifier");
3696     byte[] row = Bytes.toBytes("putRow");
3697     byte[] value = null;
3698     this.region = initHRegion(tableName, method, conf, family);
3699     Put put = null;
3700     Get get = null;
3701     List<Cell> kvs = null;
3702     Result res = null;
3703 
3704     put = new Put(row);
3705     value = Bytes.toBytes("value0");
3706     put.add(family, qualifier, 1234567l, value);
3707     region.put(put);
3708     get = new Get(row);
3709     get.addColumn(family, qualifier);
3710     get.setMaxVersions();
3711     res = this.region.get(get);
3712     kvs = res.getColumnCells(family, qualifier);
3713     assertEquals(1, kvs.size());
3714     assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
3715 
3716     region.flushcache();
3717     get = new Get(row);
3718     get.addColumn(family, qualifier);
3719     get.setMaxVersions();
3720     res = this.region.get(get);
3721     kvs = res.getColumnCells(family, qualifier);
3722     assertEquals(1, kvs.size());
3723     assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
3724 
3725     put = new Put(row);
3726     value = Bytes.toBytes("value1");
3727     put.add(family, qualifier, 1234567l, value);
3728     region.put(put);
3729     get = new Get(row);
3730     get.addColumn(family, qualifier);
3731     get.setMaxVersions();
3732     res = this.region.get(get);
3733     kvs = res.getColumnCells(family, qualifier);
3734     assertEquals(1, kvs.size());
3735     assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
3736 
3737     region.flushcache();
3738     get = new Get(row);
3739     get.addColumn(family, qualifier);
3740     get.setMaxVersions();
3741     res = this.region.get(get);
3742     kvs = res.getColumnCells(family, qualifier);
3743     assertEquals(1, kvs.size());
3744     assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
3745   }
3746 
3747   @Test
3748   public void testDurability() throws Exception {
3749     String method = "testDurability";
3750     // there are 5 x 5 cases:
3751     // table durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) x mutation
3752     // durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT)
3753 
3754     // expected cases for append and sync wal
3755     durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
3756     durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
3757     durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
3758 
3759     durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
3760     durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
3761     durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
3762 
3763     durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
3764     durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
3765 
3766     durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
3767     durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
3768 
3769     durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
3770     durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
3771     durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
3772 
3773     // expected cases for async wal
3774     // do not sync for deferred flush with large optionallogflushinterval
3775     conf.setLong("hbase.regionserver.optionallogflushinterval", Integer.MAX_VALUE);
3776     durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
3777     durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
3778     durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
3779     durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
3780     durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
3781     durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
3782 
3783     // now small deferred log flush optionallogflushinterval, expect sync
3784     conf.setLong("hbase.regionserver.optionallogflushinterval", 5);
3785     durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
3786     durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
3787     durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
3788     durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
3789     durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
3790     durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
3791 
3792     // expect skip wal cases
3793     durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
3794     durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
3795     durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
3796     durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
3797     durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
3798     durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
3799 
3800   }
3801 
3802   private void durabilityTest(String method, Durability tableDurability,
3803       Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
3804       final boolean expectSyncFromLogSyncer) throws Exception {
3805     method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
3806     TableName tableName = TableName.valueOf(method);
3807     byte[] family = Bytes.toBytes("family");
3808     Path logDir = new Path(new Path(DIR + method), "log");
3809     HLog hlog = HLogFactory.createHLog(fs, logDir, UUID.randomUUID().toString(), conf);
3810     final HLog log = spy(hlog);
3811     this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
3812         HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, log,
3813         new byte[][] { family });
3814 
3815     Put put = new Put(Bytes.toBytes("r1"));
3816     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
3817     put.setDurability(mutationDurability);
3818     region.put(put);
3819 
3820     // verify append called or not
3821     verify(log, expectAppend ? times(1) : never()).appendNoSync((HRegionInfo) any(), eq(tableName),
3822         (WALEdit) any(), (List<UUID>) any(), anyLong(), (HTableDescriptor) any());
3823 
3824     // verify sync called or not
3825     if (expectSync || expectSyncFromLogSyncer) {
3826       TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
3827         @Override
3828         public boolean evaluate() throws Exception {
3829           try {
3830             if (expectSync) {
3831               verify(log, times(1)).sync(anyLong()); // Hregion calls this one
3832             } else if (expectSyncFromLogSyncer) {
3833               verify(log, times(1)).sync(); // log syncer calls this one
3834             }
3835           } catch (Throwable ignore) {
3836           }
3837           return true;
3838         }
3839       });
3840     } else {
3841       verify(log, never()).sync(anyLong());
3842       verify(log, never()).sync();
3843     }
3844 
3845     hlog.close();
3846     region.close();
3847   }
3848 
3849   private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
3850     for (int i = startRow; i < startRow + numRows; i++) {
3851       Put put = new Put(Bytes.toBytes("" + i));
3852       put.setDurability(Durability.SKIP_WAL);
3853       for (byte[] family : families) {
3854         put.add(family, qf, null);
3855       }
3856       region.put(put);
3857     }
3858   }
3859 
3860   private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
3861       throws IOException {
3862     for (int i = startRow; i < startRow + numRows; i++) {
3863       byte[] row = Bytes.toBytes("" + i);
3864       Get get = new Get(row);
3865       for (byte[] family : families) {
3866         get.addColumn(family, qf);
3867       }
3868       Result result = newReg.get(get);
3869       Cell[] raw = result.rawCells();
3870       assertEquals(families.length, result.size());
3871       for (int j = 0; j < families.length; j++) {
3872         assertTrue(CellUtil.matchingRow(raw[j], row));
3873         assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
3874         assertTrue(CellUtil.matchingQualifier(raw[j], qf));
3875       }
3876     }
3877   }
3878 
3879   private void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
3880     // Now I have k, get values out and assert they are as expected.
3881     Get get = new Get(k).addFamily(family).setMaxVersions();
3882     Cell[] results = r.get(get).rawCells();
3883     for (int j = 0; j < results.length; j++) {
3884       byte[] tmp = CellUtil.cloneValue(results[j]);
3885       // Row should be equal to value every time.
3886       assertTrue(Bytes.equals(k, tmp));
3887     }
3888   }
3889 
3890   /*
3891    * Assert first value in the passed region is <code>firstValue</code>.
3892    * 
3893    * @param r
3894    * 
3895    * @param fs
3896    * 
3897    * @param firstValue
3898    * 
3899    * @throws IOException
3900    */
3901   private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
3902       throws IOException {
3903     byte[][] families = { fs };
3904     Scan scan = new Scan();
3905     for (int i = 0; i < families.length; i++)
3906       scan.addFamily(families[i]);
3907     InternalScanner s = r.getScanner(scan);
3908     try {
3909       List<Cell> curVals = new ArrayList<Cell>();
3910       boolean first = true;
3911       OUTER_LOOP: while (s.next(curVals)) {
3912         for (Cell kv : curVals) {
3913           byte[] val = CellUtil.cloneValue(kv);
3914           byte[] curval = val;
3915           if (first) {
3916             first = false;
3917             assertTrue(Bytes.compareTo(curval, firstValue) == 0);
3918           } else {
3919             // Not asserting anything. Might as well break.
3920             break OUTER_LOOP;
3921           }
3922         }
3923       }
3924     } finally {
3925       s.close();
3926     }
3927   }
3928 
3929   private Configuration initSplit() {
3930     // Always compact if there is more than one store file.
3931     conf.setInt("hbase.hstore.compactionThreshold", 2);
3932 
3933     // Make lease timeout longer, lease checks less frequent
3934     conf.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
3935 
3936     conf.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
3937 
3938     // Increase the amount of time between client retries
3939     conf.setLong("hbase.client.pause", 15 * 1000);
3940 
3941     // This size should make it so we always split using the addContent
3942     // below. After adding all data, the first region is 1.3M
3943     conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
3944     return conf;
3945   }
3946 
3947   /**
3948    * @param tableName
3949    * @param callingMethod
3950    * @param conf
3951    * @param families
3952    * @throws IOException
3953    * @return A region on which you must call
3954    *         {@link HRegion#closeHRegion(HRegion)} when done.
3955    */
3956   public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
3957       byte[]... families) throws IOException {
3958     return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
3959   }
3960 
3961   /**
3962    * @param tableName
3963    * @param callingMethod
3964    * @param conf
3965    * @param families
3966    * @throws IOException
3967    * @return A region on which you must call
3968    *         {@link HRegion#closeHRegion(HRegion)} when done.
3969    */
3970   public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
3971       byte[]... families) throws IOException {
3972     return initHRegion(tableName, null, null, callingMethod, conf, false, families);
3973   }
3974 
3975   /**
3976    * @param tableName
3977    * @param callingMethod
3978    * @param conf
3979    * @param isReadOnly
3980    * @param families
3981    * @throws IOException
3982    * @return A region on which you must call
3983    *         {@link HRegion#closeHRegion(HRegion)} when done.
3984    */
3985   public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
3986       boolean isReadOnly, byte[]... families) throws IOException {
3987     return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
3988   }
3989 
3990   private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
3991       String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
3992       throws IOException {
3993     return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
3994         Durability.SYNC_WAL, null, families);
3995   }
3996 
3997   /**
3998    * @param tableName
3999    * @param startKey
4000    * @param stopKey
4001    * @param callingMethod
4002    * @param conf
4003    * @param isReadOnly
4004    * @param families
4005    * @throws IOException
4006    * @return A region on which you must call
4007    *         {@link HRegion#closeHRegion(HRegion)} when done.
4008    */
4009   private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4010       String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4011       HLog hlog, byte[]... families) throws IOException {
4012     return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly, durability, hlog, families);
4013   }
4014 
4015   /**
4016    * Assert that the passed in Cell has expected contents for the specified row,
4017    * column & timestamp.
4018    */
4019   private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
4020     String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4021     assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
4022         Bytes.toString(CellUtil.cloneRow(kv)));
4023     assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
4024         Bytes.toString(CellUtil.cloneFamily(kv)));
4025     assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
4026         Bytes.toString(CellUtil.cloneQualifier(kv)));
4027     assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
4028     assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
4029         Bytes.toString(CellUtil.cloneValue(kv)));
4030   }
4031 }