View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  
22  import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23  import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24  import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25  import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29  import static org.junit.Assert.assertArrayEquals;
30  import static org.junit.Assert.assertEquals;
31  import static org.junit.Assert.assertFalse;
32  import static org.junit.Assert.assertNotNull;
33  import static org.junit.Assert.assertNull;
34  import static org.junit.Assert.assertTrue;
35  import static org.junit.Assert.fail;
36  import static org.mockito.Matchers.any;
37  import static org.mockito.Matchers.anyBoolean;
38  import static org.mockito.Matchers.anyLong;
39  import static org.mockito.Matchers.argThat;
40  import static org.mockito.Mockito.mock;
41  import static org.mockito.Mockito.never;
42  import static org.mockito.Mockito.spy;
43  import static org.mockito.Mockito.times;
44  import static org.mockito.Mockito.verify;
45  import static org.mockito.Mockito.when;
46  
47  import java.io.IOException;
48  import java.io.InterruptedIOException;
49  import java.security.PrivilegedExceptionAction;
50  import java.util.ArrayList;
51  import java.util.Arrays;
52  import java.util.Collection;
53  import java.util.List;
54  import java.util.Map;
55  import java.util.NavigableMap;
56  import java.util.TreeMap;
57  import java.util.UUID;
58  import java.util.concurrent.atomic.AtomicBoolean;
59  import java.util.concurrent.atomic.AtomicInteger;
60  import java.util.concurrent.atomic.AtomicLong;
61  import java.util.concurrent.atomic.AtomicReference;
62  
63  import org.apache.commons.logging.Log;
64  import org.apache.commons.logging.LogFactory;
65  import org.apache.hadoop.conf.Configuration;
66  import org.apache.hadoop.fs.FSDataOutputStream;
67  import org.apache.hadoop.fs.FileStatus;
68  import org.apache.hadoop.fs.FileSystem;
69  import org.apache.hadoop.fs.Path;
70  import org.apache.hadoop.hbase.Cell;
71  import org.apache.hadoop.hbase.CellComparator;
72  import org.apache.hadoop.hbase.CellUtil;
73  import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
74  import org.apache.hadoop.hbase.DroppedSnapshotException;
75  import org.apache.hadoop.hbase.HBaseConfiguration;
76  import org.apache.hadoop.hbase.HBaseTestCase;
77  import org.apache.hadoop.hbase.HBaseTestingUtility;
78  import org.apache.hadoop.hbase.HColumnDescriptor;
79  import org.apache.hadoop.hbase.HConstants;
80  import org.apache.hadoop.hbase.RegionTooBusyException;
81  import org.apache.hadoop.hbase.Tag;
82  import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
83  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
84  import org.apache.hadoop.hbase.HRegionInfo;
85  import org.apache.hadoop.hbase.HTableDescriptor;
86  import org.apache.hadoop.hbase.KeyValue;
87  import org.apache.hadoop.hbase.testclassification.MediumTests;
88  import org.apache.hadoop.hbase.MiniHBaseCluster;
89  import org.apache.hadoop.hbase.MultithreadedTestUtil;
90  import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
91  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
92  import org.apache.hadoop.hbase.NotServingRegionException;
93  import org.apache.hadoop.hbase.ServerName;
94  import org.apache.hadoop.hbase.TableName;
95  import org.apache.hadoop.hbase.TagType;
96  import org.apache.hadoop.hbase.Waiter;
97  import org.apache.hadoop.hbase.client.Append;
98  import org.apache.hadoop.hbase.client.Delete;
99  import org.apache.hadoop.hbase.client.Durability;
100 import org.apache.hadoop.hbase.client.Get;
101 import org.apache.hadoop.hbase.client.Increment;
102 import org.apache.hadoop.hbase.client.Put;
103 import org.apache.hadoop.hbase.client.Result;
104 import org.apache.hadoop.hbase.client.Scan;
105 import org.apache.hadoop.hbase.client.Table;
106 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
107 import org.apache.hadoop.hbase.filter.BinaryComparator;
108 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
109 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
110 import org.apache.hadoop.hbase.filter.Filter;
111 import org.apache.hadoop.hbase.filter.FilterBase;
112 import org.apache.hadoop.hbase.filter.FilterList;
113 import org.apache.hadoop.hbase.filter.NullComparator;
114 import org.apache.hadoop.hbase.filter.PrefixFilter;
115 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
116 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
117 import org.apache.hadoop.hbase.io.hfile.HFile;
118 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
119 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
120 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
121 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
122 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
123 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
124 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
125 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
126 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor;
127 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
128 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
129 import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
130 import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
131 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
132 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
133 import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
134 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
135 import org.apache.hadoop.hbase.wal.DefaultWALProvider;
136 import org.apache.hadoop.hbase.wal.FaultyFSLog;
137 import org.apache.hadoop.hbase.wal.WAL;
138 import org.apache.hadoop.hbase.wal.WALFactory;
139 import org.apache.hadoop.hbase.wal.WALKey;
140 import org.apache.hadoop.hbase.wal.WALProvider;
141 import org.apache.hadoop.hbase.wal.WALSplitter;
142 import org.apache.hadoop.hbase.security.User;
143 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
144 import org.apache.hadoop.hbase.util.Bytes;
145 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
146 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
147 import org.apache.hadoop.hbase.util.FSUtils;
148 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
149 import org.apache.hadoop.hbase.util.PairOfSameType;
150 import org.apache.hadoop.hbase.util.Threads;
151 import org.junit.After;
152 import org.junit.Assert;
153 import org.junit.Before;
154 import org.junit.Rule;
155 import org.junit.Test;
156 import org.junit.experimental.categories.Category;
157 import org.junit.rules.TestName;
158 import org.mockito.ArgumentCaptor;
159 import org.mockito.ArgumentMatcher;
160 import org.mockito.Mockito;
161 
162 import com.google.common.collect.ImmutableList;
163 import com.google.common.collect.Lists;
164 import com.google.protobuf.ByteString;
165 
166 /**
167  * Basic stand-alone testing of HRegion.  No clusters!
168  *
169  * A lot of the meta information for an HRegion now lives inside other HRegions
170  * or in the HBaseMaster, so only basic testing is possible.
171  */
172 @Category(MediumTests.class)
173 @SuppressWarnings("deprecation")
174 public class TestHRegion {
175   // Do not spin up clusters in here. If you need to spin up a cluster, do it
176   // over in TestHRegionOnCluster.
177   static final Log LOG = LogFactory.getLog(TestHRegion.class);
178   @Rule public TestName name = new TestName();
179 
180   private static final String COLUMN_FAMILY = "MyCF";
181   private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
182 
183   HRegion region = null;
184   // Do not run unit tests in parallel (? Why not?  It don't work?  Why not?  St.Ack)
185   private static HBaseTestingUtility TEST_UTIL;
186   public static Configuration CONF ;
187   private String dir;
188   private static FileSystem FILESYSTEM;
189   private final int MAX_VERSIONS = 2;
190 
191   // Test names
192   protected byte[] tableName;
193   protected String method;
194   protected final byte[] qual1 = Bytes.toBytes("qual1");
195   protected final byte[] qual2 = Bytes.toBytes("qual2");
196   protected final byte[] qual3 = Bytes.toBytes("qual3");
197   protected final byte[] value1 = Bytes.toBytes("value1");
198   protected final byte[] value2 = Bytes.toBytes("value2");
199   protected final byte[] row = Bytes.toBytes("rowA");
200   protected final byte[] row2 = Bytes.toBytes("rowB");
201 
202   protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
203       .getInstance(MetricsAssertHelper.class);
204 
205   @Before
206   public void setup() throws IOException {
207     TEST_UTIL = HBaseTestingUtility.createLocalHTU();
208     FILESYSTEM = TEST_UTIL.getTestFileSystem();
209     CONF = TEST_UTIL.getConfiguration();
210     dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
211     method = name.getMethodName();
212     tableName = Bytes.toBytes(name.getMethodName());
213   }
214 
215   @After
216   public void tearDown() throws Exception {
217     EnvironmentEdgeManagerTestHelper.reset();
218     LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
219     TEST_UTIL.cleanupTestDir();
220   }
221 
222   String getName() {
223     return name.getMethodName();
224   }
225 
226   /**
227    * Test for Bug 2 of HBASE-10466.
228    * "Bug 2: Conditions for the first flush of region close (so-called pre-flush) If memstoreSize
229    * is smaller than a certain value, or when region close starts a flush is ongoing, the first
230    * flush is skipped and only the second flush takes place. However, two flushes are required in
231    * case previous flush fails and leaves some data in snapshot. The bug could cause loss of data
232    * in current memstore. The fix is removing all conditions except abort check so we ensure 2
233    * flushes for region close."
234    * @throws IOException
235    */
236   @Test (timeout=60000)
237   public void testCloseCarryingSnapshot() throws IOException {
238     HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
239     Store store = region.getStore(COLUMN_FAMILY_BYTES);
240     // Get some random bytes.
241     byte [] value = Bytes.toBytes(name.getMethodName());
242     // Make a random put against our cf.
243     Put put = new Put(value);
244     put.add(COLUMN_FAMILY_BYTES, null, value);
245     // First put something in current memstore, which will be in snapshot after flusher.prepare()
246     region.put(put);
247     StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
248     storeFlushCtx.prepare();
249     // Second put something in current memstore
250     put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
251     region.put(put);
252     // Close with something in memstore and something in the snapshot.  Make sure all is cleared.
253     region.close();
254     assertEquals(0, region.getMemstoreSize().get());
255     HRegion.closeHRegion(region);
256   }
257 
258   /*
259    * This test is for verifying memstore snapshot size is correctly updated in case of rollback
260    * See HBASE-10845
261    */
262   @Test (timeout=60000)
263   public void testMemstoreSnapshotSize() throws IOException {
264     class MyFaultyFSLog extends FaultyFSLog {
265       StoreFlushContext storeFlushCtx;
266       public MyFaultyFSLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
267           throws IOException {
268         super(fs, rootDir, logName, conf);
269       }
270 
271       void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
272         this.storeFlushCtx = storeFlushCtx;
273       }
274 
275       @Override
276       public void sync(long txid) throws IOException {
277         storeFlushCtx.prepare();
278         super.sync(txid);
279       }
280     }
281 
282     FileSystem fs = FileSystem.get(CONF);
283     Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
284     MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
285     HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
286       CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES);
287 
288     Store store = region.getStore(COLUMN_FAMILY_BYTES);
289     // Get some random bytes.
290     byte [] value = Bytes.toBytes(name.getMethodName());
291     faultyLog.setStoreFlushCtx(store.createFlushContext(12345));
292 
293     Put put = new Put(value);
294     put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
295     faultyLog.setFailureType(FaultyFSLog.FailureType.SYNC);
296 
297     boolean threwIOE = false;
298     try {
299       region.put(put);
300     } catch (IOException ioe) {
301       threwIOE = true;
302     } finally {
303       assertTrue("The regionserver should have thrown an exception", threwIOE);
304     }
305     long sz = store.getFlushableSize();
306     assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
307     HRegion.closeHRegion(region);
308   }
309 
310   /**
311    * Test we do not lose data if we fail a flush and then close.
312    * Part of HBase-10466.  Tests the following from the issue description:
313    * "Bug 1: Wrong calculation of HRegion.memstoreSize: When a flush fails, data to be flushed is
314    * kept in each MemStore's snapshot and wait for next flush attempt to continue on it. But when
315    * the next flush succeeds, the counter of total memstore size in HRegion is always deduced by
316    * the sum of current memstore sizes instead of snapshots left from previous failed flush. This
317    * calculation is problematic that almost every time there is failed flush, HRegion.memstoreSize
318    * gets reduced by a wrong value. If region flush could not proceed for a couple cycles, the size
319    * in current memstore could be much larger than the snapshot. It's likely to drift memstoreSize
320    * much smaller than expected. In extreme case, if the error accumulates to even bigger than
321    * HRegion's memstore size limit, any further flush is skipped because flush does not do anything
322    * if memstoreSize is not larger than 0."
323    * @throws Exception
324    */
325   @Test (timeout=60000)
326   public void testFlushSizeAccounting() throws Exception {
327     final Configuration conf = HBaseConfiguration.create(CONF);
328     // Only retry once.
329     conf.setInt("hbase.hstore.flush.retries.number", 1);
330     final User user =
331       User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
332     // Inject our faulty LocalFileSystem
333     conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
334     user.runAs(new PrivilegedExceptionAction<Object>() {
335       @Override
336       public Object run() throws Exception {
337         // Make sure it worked (above is sensitive to caching details in hadoop core)
338         FileSystem fs = FileSystem.get(conf);
339         Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
340         FaultyFileSystem ffs = (FaultyFileSystem)fs;
341         HRegion region = null;
342         try {
343           // Initialize region
344           region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
345           long size = region.getMemstoreSize().get();
346           Assert.assertEquals(0, size);
347           // Put one item into memstore.  Measure the size of one item in memstore.
348           Put p1 = new Put(row);
349           p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
350           region.put(p1);
351           final long sizeOfOnePut = region.getMemstoreSize().get();
352           // Fail a flush which means the current memstore will hang out as memstore 'snapshot'.
353           try {
354             LOG.info("Flushing");
355             region.flushcache();
356             Assert.fail("Didn't bubble up IOE!");
357           } catch (DroppedSnapshotException dse) {
358             // What we are expecting
359           }
360           // Make it so all writes succeed from here on out
361           ffs.fault.set(false);
362           // Check sizes.  Should still be the one entry.
363           Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize().get());
364           // Now add two entries so that on this next flush that fails, we can see if we
365           // subtract the right amount, the snapshot size only.
366           Put p2 = new Put(row);
367           p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
368           p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
369           region.put(p2);
370           Assert.assertEquals(sizeOfOnePut * 3, region.getMemstoreSize().get());
371           // Do a successful flush.  It will clear the snapshot only.  Thats how flushes work.
372           // If already a snapshot, we clear it else we move the memstore to be snapshot and flush
373           // it
374           region.flushcache();
375           // Make sure our memory accounting is right.
376           Assert.assertEquals(sizeOfOnePut * 2, region.getMemstoreSize().get());
377         } finally {
378           HRegion.closeHRegion(region);
379         }
380         return null;
381       }
382     });
383     FileSystem.closeAllForUGI(user.getUGI());
384   }
385 
386   @Test (timeout=60000)
387   public void testCloseWithFailingFlush() throws Exception {
388     final Configuration conf = HBaseConfiguration.create(CONF);
389     // Only retry once.
390     conf.setInt("hbase.hstore.flush.retries.number", 1);
391     final User user =
392       User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
393     // Inject our faulty LocalFileSystem
394     conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
395     user.runAs(new PrivilegedExceptionAction<Object>() {
396       @Override
397       public Object run() throws Exception {
398         // Make sure it worked (above is sensitive to caching details in hadoop core)
399         FileSystem fs = FileSystem.get(conf);
400         Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
401         FaultyFileSystem ffs = (FaultyFileSystem)fs;
402         HRegion region = null;
403         try {
404           // Initialize region
405           region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
406           long size = region.getMemstoreSize().get();
407           Assert.assertEquals(0, size);
408           // Put one item into memstore.  Measure the size of one item in memstore.
409           Put p1 = new Put(row);
410           p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
411           region.put(p1);
412           // Manufacture an outstanding snapshot -- fake a failed flush by doing prepare step only.
413           Store store = region.getStore(COLUMN_FAMILY_BYTES);
414           StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
415           storeFlushCtx.prepare();
416           // Now add two entries to the foreground memstore.
417           Put p2 = new Put(row);
418           p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
419           p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
420           region.put(p2);
421           // Now try close on top of a failing flush.
422           region.close();
423           fail();
424         } catch (DroppedSnapshotException dse) {
425           // Expected
426           LOG.info("Expected DroppedSnapshotException");
427         } finally {
428           // Make it so all writes succeed from here on out so can close clean
429           ffs.fault.set(false);
430           HRegion.closeHRegion(region);
431         }
432         return null;
433       }
434     });
435     FileSystem.closeAllForUGI(user.getUGI());
436   }
437 
438   @Test
439   public void testCompactionAffectedByScanners() throws Exception {
440     byte[] family = Bytes.toBytes("family");
441     this.region = initHRegion(tableName, method, CONF, family);
442 
443     Put put = new Put(Bytes.toBytes("r1"));
444     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
445     region.put(put);
446     region.flushcache();
447 
448     Scan scan = new Scan();
449     scan.setMaxVersions(3);
450     // open the first scanner
451     RegionScanner scanner1 = region.getScanner(scan);
452 
453     Delete delete = new Delete(Bytes.toBytes("r1"));
454     region.delete(delete);
455     region.flushcache();
456 
457     // open the second scanner
458     RegionScanner scanner2 = region.getScanner(scan);
459 
460     List<Cell> results = new ArrayList<Cell>();
461 
462     System.out.println("Smallest read point:" + region.getSmallestReadPoint());
463 
464     // make a major compaction
465     region.compactStores(true);
466 
467     // open the third scanner
468     RegionScanner scanner3 = region.getScanner(scan);
469 
470     // get data from scanner 1, 2, 3 after major compaction
471     scanner1.next(results);
472     System.out.println(results);
473     assertEquals(1, results.size());
474 
475     results.clear();
476     scanner2.next(results);
477     System.out.println(results);
478     assertEquals(0, results.size());
479 
480     results.clear();
481     scanner3.next(results);
482     System.out.println(results);
483     assertEquals(0, results.size());
484   }
485 
486   @Test
487   public void testToShowNPEOnRegionScannerReseek() throws Exception {
488     byte[] family = Bytes.toBytes("family");
489     this.region = initHRegion(tableName, method, CONF, family);
490 
491     Put put = new Put(Bytes.toBytes("r1"));
492     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
493     region.put(put);
494     put = new Put(Bytes.toBytes("r2"));
495     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
496     region.put(put);
497     region.flushcache();
498 
499     Scan scan = new Scan();
500     scan.setMaxVersions(3);
501     // open the first scanner
502     RegionScanner scanner1 = region.getScanner(scan);
503 
504     System.out.println("Smallest read point:" + region.getSmallestReadPoint());
505 
506     region.compactStores(true);
507 
508     scanner1.reseek(Bytes.toBytes("r2"));
509     List<Cell> results = new ArrayList<Cell>();
510     scanner1.next(results);
511     Cell keyValue = results.get(0);
512     Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
513     scanner1.close();
514   }
515 
516   @Test
517   public void testSkipRecoveredEditsReplay() throws Exception {
518     String method = "testSkipRecoveredEditsReplay";
519     TableName tableName = TableName.valueOf(method);
520     byte[] family = Bytes.toBytes("family");
521     this.region = initHRegion(tableName, method, CONF, family);
522     final WALFactory wals = new WALFactory(CONF, null, method);
523     try {
524       Path regiondir = region.getRegionFileSystem().getRegionDir();
525       FileSystem fs = region.getRegionFileSystem().getFileSystem();
526       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
527 
528       Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
529 
530       long maxSeqId = 1050;
531       long minSeqId = 1000;
532 
533       for (long i = minSeqId; i <= maxSeqId; i += 10) {
534         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
535         fs.create(recoveredEdits);
536         WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
537 
538         long time = System.nanoTime();
539         WALEdit edit = new WALEdit();
540         edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
541             .toBytes(i)));
542         writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time,
543             HConstants.DEFAULT_CLUSTER_ID), edit));
544 
545         writer.close();
546       }
547       MonitoredTask status = TaskMonitor.get().createStatus(method);
548       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
549       for (Store store : region.getStores().values()) {
550         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
551       }
552       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
553       assertEquals(maxSeqId, seqId);
554       region.getMVCC().initialize(seqId);
555       Get get = new Get(row);
556       Result result = region.get(get);
557       for (long i = minSeqId; i <= maxSeqId; i += 10) {
558         List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
559         assertEquals(1, kvs.size());
560         assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
561       }
562     } finally {
563       HRegion.closeHRegion(this.region);
564       this.region = null;
565       wals.close();
566     }
567   }
568 
569   @Test
570   public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
571     String method = "testSkipRecoveredEditsReplaySomeIgnored";
572     TableName tableName = TableName.valueOf(method);
573     byte[] family = Bytes.toBytes("family");
574     this.region = initHRegion(tableName, method, CONF, family);
575     final WALFactory wals = new WALFactory(CONF, null, method);
576     try {
577       Path regiondir = region.getRegionFileSystem().getRegionDir();
578       FileSystem fs = region.getRegionFileSystem().getFileSystem();
579       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
580 
581       Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
582 
583       long maxSeqId = 1050;
584       long minSeqId = 1000;
585 
586       for (long i = minSeqId; i <= maxSeqId; i += 10) {
587         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
588         fs.create(recoveredEdits);
589         WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
590 
591         long time = System.nanoTime();
592         WALEdit edit = new WALEdit();
593         edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
594             .toBytes(i)));
595         writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time,
596             HConstants.DEFAULT_CLUSTER_ID), edit));
597 
598         writer.close();
599       }
600       long recoverSeqId = 1030;
601       MonitoredTask status = TaskMonitor.get().createStatus(method);
602       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
603       for (Store store : region.getStores().values()) {
604         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
605       }
606       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
607       assertEquals(maxSeqId, seqId);
608       region.getMVCC().initialize(seqId);
609       Get get = new Get(row);
610       Result result = region.get(get);
611       for (long i = minSeqId; i <= maxSeqId; i += 10) {
612         List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
613         if (i < recoverSeqId) {
614           assertEquals(0, kvs.size());
615         } else {
616           assertEquals(1, kvs.size());
617           assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
618         }
619       }
620     } finally {
621       HRegion.closeHRegion(this.region);
622       this.region = null;
623       wals.close();
624     }
625   }
626 
627   @Test
628   public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
629     byte[] family = Bytes.toBytes("family");
630     this.region = initHRegion(tableName, method, CONF, family);
631     try {
632       Path regiondir = region.getRegionFileSystem().getRegionDir();
633       FileSystem fs = region.getRegionFileSystem().getFileSystem();
634 
635       Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
636       for (int i = 1000; i < 1050; i += 10) {
637         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
638         FSDataOutputStream dos = fs.create(recoveredEdits);
639         dos.writeInt(i);
640         dos.close();
641       }
642       long minSeqId = 2000;
643       Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
644       FSDataOutputStream dos = fs.create(recoveredEdits);
645       dos.close();
646 
647       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
648       for (Store store : region.getStores().values()) {
649         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
650       }
651       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
652       assertEquals(minSeqId, seqId);
653     } finally {
654       HRegion.closeHRegion(this.region);
655       this.region = null;
656     }
657   }
658 
659   @Test
660   public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
661     String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
662     TableName tableName = TableName.valueOf(method);
663     byte[] family = Bytes.toBytes("family");
664     this.region = initHRegion(tableName, method, CONF, family);
665     final WALFactory wals = new WALFactory(CONF, null, method);
666     try {
667       Path regiondir = region.getRegionFileSystem().getRegionDir();
668       FileSystem fs = region.getRegionFileSystem().getFileSystem();
669       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
670 
671       assertEquals(0, region.getStoreFileList(
672         region.getStores().keySet().toArray(new byte[0][])).size());
673 
674       Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
675 
676       long maxSeqId = 1050;
677       long minSeqId = 1000;
678 
679       for (long i = minSeqId; i <= maxSeqId; i += 10) {
680         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
681         fs.create(recoveredEdits);
682         WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
683 
684         long time = System.nanoTime();
685         WALEdit edit = null;
686         if (i == maxSeqId) {
687           edit = WALEdit.createCompaction(region.getRegionInfo(),
688           CompactionDescriptor.newBuilder()
689           .setTableName(ByteString.copyFrom(tableName.getName()))
690           .setFamilyName(ByteString.copyFrom(regionName))
691           .setEncodedRegionName(ByteString.copyFrom(regionName))
692           .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
693           .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName()))
694           .build());
695         } else {
696           edit = new WALEdit();
697           edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
698             .toBytes(i)));
699         }
700         writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time,
701             HConstants.DEFAULT_CLUSTER_ID), edit));
702         writer.close();
703       }
704 
705       long recoverSeqId = 1030;
706       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
707       MonitoredTask status = TaskMonitor.get().createStatus(method);
708       for (Store store : region.getStores().values()) {
709         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
710       }
711       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
712       assertEquals(maxSeqId, seqId);
713 
714       // assert that the files are flushed
715       assertEquals(1, region.getStoreFileList(
716         region.getStores().keySet().toArray(new byte[0][])).size());
717 
718     } finally {
719       HRegion.closeHRegion(this.region);
720       this.region = null;
721       wals.close();
722     }
723   }
724 
725   @Test
726   public void testRecoveredEditsReplayCompaction() throws Exception {
727     String method = name.getMethodName();
728     TableName tableName = TableName.valueOf(method);
729     byte[] family = Bytes.toBytes("family");
730     this.region = initHRegion(tableName, method, CONF, family);
731     final WALFactory wals = new WALFactory(CONF, null, method);
732     try {
733       Path regiondir = region.getRegionFileSystem().getRegionDir();
734       FileSystem fs = region.getRegionFileSystem().getFileSystem();
735       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
736 
737       long maxSeqId = 3;
738       long minSeqId = 0;
739 
740       for (long i = minSeqId; i < maxSeqId; i++) {
741         Put put = new Put(Bytes.toBytes(i));
742         put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
743         region.put(put);
744         region.flushcache();
745       }
746 
747       // this will create a region with 3 files
748       assertEquals(3, region.getStore(family).getStorefilesCount());
749       List<Path> storeFiles = new ArrayList<Path>(3);
750       for (StoreFile sf : region.getStore(family).getStorefiles()) {
751         storeFiles.add(sf.getPath());
752       }
753 
754       // disable compaction completion
755       CONF.setBoolean("hbase.hstore.compaction.complete", false);
756       region.compactStores();
757 
758       // ensure that nothing changed
759       assertEquals(3, region.getStore(family).getStorefilesCount());
760 
761       // now find the compacted file, and manually add it to the recovered edits
762       Path tmpDir = region.getRegionFileSystem().getTempDir();
763       FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
764       String errorMsg = "Expected to find 1 file in the region temp directory "
765           + "from the compaction, could not find any";
766       assertNotNull(errorMsg, files);
767       assertEquals(errorMsg, 1, files.length);
768       // move the file inside region dir
769       Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
770           files[0].getPath());
771 
772       CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
773           .getRegionInfo(), family, storeFiles, Lists.newArrayList(newFile), region
774           .getRegionFileSystem().getStoreDir(Bytes.toString(family)));
775 
776       WALUtil.writeCompactionMarker(region.getWAL(), this.region.getTableDesc(),
777           this.region.getRegionInfo(), compactionDescriptor, new AtomicLong(1));
778 
779       Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
780 
781       Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
782       fs.create(recoveredEdits);
783       WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
784 
785       long time = System.nanoTime();
786 
787       writer.append(new WAL.Entry(new HLogKey(regionName, tableName, 10, time,
788           HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(),
789           compactionDescriptor)));
790       writer.close();
791 
792       // close the region now, and reopen again
793       region.getTableDesc();
794       region.getRegionInfo();
795       region.close();
796       region = HRegion.openHRegion(region, null);
797 
798       // now check whether we have only one store file, the compacted one
799       Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
800       for (StoreFile sf : sfs) {
801         LOG.info(sf.getPath());
802       }
803       assertEquals(1, region.getStore(family).getStorefilesCount());
804       files = FSUtils.listStatus(fs, tmpDir);
805       assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
806 
807       for (long i = minSeqId; i < maxSeqId; i++) {
808         Get get = new Get(Bytes.toBytes(i));
809         Result result = region.get(get);
810         byte[] value = result.getValue(family, Bytes.toBytes(i));
811         assertArrayEquals(Bytes.toBytes(i), value);
812       }
813     } finally {
814       HRegion.closeHRegion(this.region);
815       this.region = null;
816       wals.close();
817     }
818   }
819 
820   @Test
821   public void testFlushMarkers() throws Exception {
822     // tests that flush markers are written to WAL and handled at recovered edits
823     String method = name.getMethodName();
824     TableName tableName = TableName.valueOf(method);
825     byte[] family = Bytes.toBytes("family");
826     Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log");
827     final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
828     FSUtils.setRootDir(walConf, logDir);
829     final WALFactory wals = new WALFactory(walConf, null, method);
830     final WAL wal = wals.getWAL(tableName.getName());
831 
832     this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
833       HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family);
834     try {
835       Path regiondir = region.getRegionFileSystem().getRegionDir();
836       FileSystem fs = region.getRegionFileSystem().getFileSystem();
837       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
838 
839       long maxSeqId = 3;
840       long minSeqId = 0;
841 
842       for (long i = minSeqId; i < maxSeqId; i++) {
843         Put put = new Put(Bytes.toBytes(i));
844         put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
845         region.put(put);
846         region.flushcache();
847       }
848 
849       // this will create a region with 3 files from flush
850       assertEquals(3, region.getStore(family).getStorefilesCount());
851       List<String> storeFiles = new ArrayList<String>(3);
852       for (StoreFile sf : region.getStore(family).getStorefiles()) {
853         storeFiles.add(sf.getPath().getName());
854       }
855 
856       // now verify that the flush markers are written
857       wal.shutdown();
858       WAL.Reader reader = wals.createReader(fs, DefaultWALProvider.getCurrentFileName(wal),
859         TEST_UTIL.getConfiguration());
860       try {
861         List<WAL.Entry> flushDescriptors = new ArrayList<WAL.Entry>();
862         long lastFlushSeqId = -1;
863         while (true) {
864           WAL.Entry entry = reader.next();
865           if (entry == null) {
866             break;
867           }
868           Cell cell = entry.getEdit().getCells().get(0);
869           if (WALEdit.isMetaEditFamily(cell)) {
870             FlushDescriptor flushDesc = WALEdit.getFlushDescriptor(cell);
871             assertNotNull(flushDesc);
872             assertArrayEquals(tableName.getName(), flushDesc.getTableName().toByteArray());
873             if (flushDesc.getAction() == FlushAction.START_FLUSH) {
874               assertTrue(flushDesc.getFlushSequenceNumber() > lastFlushSeqId);
875             } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) {
876               assertTrue(flushDesc.getFlushSequenceNumber() == lastFlushSeqId);
877             }
878             lastFlushSeqId = flushDesc.getFlushSequenceNumber();
879             assertArrayEquals(regionName, flushDesc.getEncodedRegionName().toByteArray());
880             assertEquals(1, flushDesc.getStoreFlushesCount()); //only one store
881             StoreFlushDescriptor storeFlushDesc = flushDesc.getStoreFlushes(0);
882             assertArrayEquals(family, storeFlushDesc.getFamilyName().toByteArray());
883             assertEquals("family", storeFlushDesc.getStoreHomeDir());
884             if (flushDesc.getAction() == FlushAction.START_FLUSH) {
885               assertEquals(0, storeFlushDesc.getFlushOutputCount());
886             } else {
887               assertEquals(1, storeFlushDesc.getFlushOutputCount()); //only one file from flush
888               assertTrue(storeFiles.contains(storeFlushDesc.getFlushOutput(0)));
889             }
890 
891             flushDescriptors.add(entry);
892           }
893         }
894 
895         assertEquals(3 * 2, flushDescriptors.size()); // START_FLUSH and COMMIT_FLUSH per flush
896 
897         // now write those markers to the recovered edits again.
898 
899         Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
900 
901         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
902         fs.create(recoveredEdits);
903         WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
904 
905         for (WAL.Entry entry : flushDescriptors) {
906           writer.append(entry);
907         }
908         writer.close();
909       } finally {
910         if (null != reader) {
911           try {
912             reader.close();
913           } catch (IOException exception) {
914             LOG.warn("Problem closing wal: " + exception.getMessage());
915             LOG.debug("exception details", exception);
916           }
917         }
918       }
919 
920 
921       // close the region now, and reopen again
922       region.close();
923       region = HRegion.openHRegion(region, null);
924 
925       // now check whether we have can read back the data from region
926       for (long i = minSeqId; i < maxSeqId; i++) {
927         Get get = new Get(Bytes.toBytes(i));
928         Result result = region.get(get);
929         byte[] value = result.getValue(family, Bytes.toBytes(i));
930         assertArrayEquals(Bytes.toBytes(i), value);
931       }
932     } finally {
933       HRegion.closeHRegion(this.region);
934       this.region = null;
935       wals.close();
936     }
937   }
938 
939   class IsFlushWALMarker extends ArgumentMatcher<WALEdit> {
940     volatile FlushAction[] actions;
941     public IsFlushWALMarker(FlushAction... actions) {
942       this.actions = actions;
943     }
944     @Override
945     public boolean matches(Object edit) {
946       List<Cell> cells = ((WALEdit)edit).getCells();
947       if (cells.isEmpty()) {
948         return false;
949       }
950       if (WALEdit.isMetaEditFamily(cells.get(0))) {
951         FlushDescriptor desc = null;
952         try {
953           desc = WALEdit.getFlushDescriptor(cells.get(0));
954         } catch (IOException e) {
955           LOG.warn(e);
956           return false;
957         }
958         if (desc != null) {
959           for (FlushAction action : actions) {
960             if (desc.getAction() == action) {
961               return true;
962             }
963           }
964         }
965       }
966       return false;
967     }
968     public IsFlushWALMarker set(FlushAction... actions) {
969       this.actions = actions;
970       return this;
971     }
972   }
973 
974   @Test
975   @SuppressWarnings("unchecked")
976   public void testFlushMarkersWALFail() throws Exception {
977     // test the cases where the WAL append for flush markers fail.
978     String method = name.getMethodName();
979     TableName tableName = TableName.valueOf(method);
980     byte[] family = Bytes.toBytes("family");
981 
982     // spy an actual WAL implementation to throw exception (was not able to mock)
983     Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + "log");
984 
985     final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
986     FSUtils.setRootDir(walConf, logDir);
987     final WALFactory wals = new WALFactory(walConf, null, method);
988     WAL wal = spy(wals.getWAL(tableName.getName()));
989 
990     this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
991       HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family);
992     try {
993       int i = 0;
994       Put put = new Put(Bytes.toBytes(i));
995       put.setDurability(Durability.SKIP_WAL); // have to skip mocked wal
996       put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
997       region.put(put);
998 
999       // 1. Test case where START_FLUSH throws exception
1000       IsFlushWALMarker isFlushWALMarker = new IsFlushWALMarker(FlushAction.START_FLUSH);
1001 
1002       // throw exceptions if the WalEdit is a start flush action
1003       when(wal.append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any(),
1004         (WALEdit)argThat(isFlushWALMarker), (AtomicLong)any(), Mockito.anyBoolean(),
1005         (List<Cell>)any()))
1006           .thenThrow(new IOException("Fail to append flush marker"));
1007 
1008       // start cache flush will throw exception
1009       try {
1010         region.flushcache();
1011         fail("This should have thrown exception");
1012       } catch (DroppedSnapshotException unexpected) {
1013         // this should not be a dropped snapshot exception. Meaning that RS will not abort
1014         throw unexpected;
1015       } catch (IOException expected) {
1016         // expected
1017       }
1018 
1019       // 2. Test case where START_FLUSH succeeds but COMMIT_FLUSH will throw exception
1020       isFlushWALMarker.set(FlushAction.COMMIT_FLUSH);
1021 
1022       try {
1023         region.flushcache();
1024         fail("This should have thrown exception");
1025       } catch (DroppedSnapshotException expected) {
1026         // we expect this exception, since we were able to write the snapshot, but failed to
1027         // write the flush marker to WAL
1028       } catch (IOException unexpected) {
1029         throw unexpected;
1030       }
1031 
1032       region.close();
1033       this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
1034         HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family);
1035       region.put(put);
1036 
1037       // 3. Test case where ABORT_FLUSH will throw exception.
1038       // Even if ABORT_FLUSH throws exception, we should not fail with IOE, but continue with
1039       // DroppedSnapshotException. Below COMMMIT_FLUSH will cause flush to abort
1040       isFlushWALMarker.set(FlushAction.COMMIT_FLUSH, FlushAction.ABORT_FLUSH);
1041 
1042       try {
1043         region.flushcache();
1044         fail("This should have thrown exception");
1045       } catch (DroppedSnapshotException expected) {
1046         // we expect this exception, since we were able to write the snapshot, but failed to
1047         // write the flush marker to WAL
1048       } catch (IOException unexpected) {
1049         throw unexpected;
1050       }
1051 
1052     } finally {
1053       HRegion.closeHRegion(this.region);
1054       this.region = null;
1055     }
1056   }
1057 
1058   @Test
1059   public void testGetWhileRegionClose() throws IOException {
1060     TableName tableName = TableName.valueOf(name.getMethodName());
1061     Configuration hc = initSplit();
1062     int numRows = 100;
1063     byte[][] families = { fam1, fam2, fam3 };
1064 
1065     // Setting up region
1066     String method = name.getMethodName();
1067     this.region = initHRegion(tableName, method, hc, families);
1068     try {
1069       // Put data in region
1070       final int startRow = 100;
1071       putData(startRow, numRows, qual1, families);
1072       putData(startRow, numRows, qual2, families);
1073       putData(startRow, numRows, qual3, families);
1074       final AtomicBoolean done = new AtomicBoolean(false);
1075       final AtomicInteger gets = new AtomicInteger(0);
1076       GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
1077       try {
1078         // Set ten threads running concurrently getting from the region.
1079         for (int i = 0; i < threads.length / 2; i++) {
1080           threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
1081           threads[i].setDaemon(true);
1082           threads[i].start();
1083         }
1084         // Artificially make the condition by setting closing flag explicitly.
1085         // I can't make the issue happen with a call to region.close().
1086         this.region.closing.set(true);
1087         for (int i = threads.length / 2; i < threads.length; i++) {
1088           threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
1089           threads[i].setDaemon(true);
1090           threads[i].start();
1091         }
1092       } finally {
1093         if (this.region != null) {
1094           HRegion.closeHRegion(this.region);
1095         }
1096       }
1097       done.set(true);
1098       for (GetTillDoneOrException t : threads) {
1099         try {
1100           t.join();
1101         } catch (InterruptedException e) {
1102           e.printStackTrace();
1103         }
1104         if (t.e != null) {
1105           LOG.info("Exception=" + t.e);
1106           assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
1107         }
1108       }
1109     } finally {
1110       HRegion.closeHRegion(this.region);
1111       this.region = null;
1112     }
1113   }
1114 
1115   /*
1116    * Thread that does get on single row until 'done' flag is flipped. If an
1117    * exception causes us to fail, it records it.
1118    */
1119   class GetTillDoneOrException extends Thread {
1120     private final Get g;
1121     private final AtomicBoolean done;
1122     private final AtomicInteger count;
1123     private Exception e;
1124 
1125     GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
1126       super("getter." + i);
1127       this.g = new Get(r);
1128       this.done = d;
1129       this.count = c;
1130     }
1131 
1132     @Override
1133     public void run() {
1134       while (!this.done.get()) {
1135         try {
1136           assertTrue(region.get(g).size() > 0);
1137           this.count.incrementAndGet();
1138         } catch (Exception e) {
1139           this.e = e;
1140           break;
1141         }
1142       }
1143     }
1144   }
1145 
1146   /*
1147    * An involved filter test. Has multiple column families and deletes in mix.
1148    */
1149   @Test
1150   public void testWeirdCacheBehaviour() throws Exception {
1151     byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
1152     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
1153         Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
1154     this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
1155     try {
1156       String value = "this is the value";
1157       String value2 = "this is some other value";
1158       String keyPrefix1 = "prefix1";
1159       String keyPrefix2 = "prefix2";
1160       String keyPrefix3 = "prefix3";
1161       putRows(this.region, 3, value, keyPrefix1);
1162       putRows(this.region, 3, value, keyPrefix2);
1163       putRows(this.region, 3, value, keyPrefix3);
1164       putRows(this.region, 3, value2, keyPrefix1);
1165       putRows(this.region, 3, value2, keyPrefix2);
1166       putRows(this.region, 3, value2, keyPrefix3);
1167       System.out.println("Checking values for key: " + keyPrefix1);
1168       assertEquals("Got back incorrect number of rows from scan", 3,
1169           getNumberOfRows(keyPrefix1, value2, this.region));
1170       System.out.println("Checking values for key: " + keyPrefix2);
1171       assertEquals("Got back incorrect number of rows from scan", 3,
1172           getNumberOfRows(keyPrefix2, value2, this.region));
1173       System.out.println("Checking values for key: " + keyPrefix3);
1174       assertEquals("Got back incorrect number of rows from scan", 3,
1175           getNumberOfRows(keyPrefix3, value2, this.region));
1176       deleteColumns(this.region, value2, keyPrefix1);
1177       deleteColumns(this.region, value2, keyPrefix2);
1178       deleteColumns(this.region, value2, keyPrefix3);
1179       System.out.println("Starting important checks.....");
1180       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
1181           getNumberOfRows(keyPrefix1, value2, this.region));
1182       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
1183           getNumberOfRows(keyPrefix2, value2, this.region));
1184       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
1185           getNumberOfRows(keyPrefix3, value2, this.region));
1186     } finally {
1187       HRegion.closeHRegion(this.region);
1188       this.region = null;
1189     }
1190   }
1191 
1192   @Test
1193   public void testAppendWithReadOnlyTable() throws Exception {
1194     byte[] TABLE = Bytes.toBytes("readOnlyTable");
1195     this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
1196     boolean exceptionCaught = false;
1197     Append append = new Append(Bytes.toBytes("somerow"));
1198     append.setDurability(Durability.SKIP_WAL);
1199     append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
1200         Bytes.toBytes("somevalue"));
1201     try {
1202       region.append(append);
1203     } catch (IOException e) {
1204       exceptionCaught = true;
1205     } finally {
1206       HRegion.closeHRegion(this.region);
1207       this.region = null;
1208     }
1209     assertTrue(exceptionCaught == true);
1210   }
1211 
1212   @Test
1213   public void testIncrWithReadOnlyTable() throws Exception {
1214     byte[] TABLE = Bytes.toBytes("readOnlyTable");
1215     this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
1216     boolean exceptionCaught = false;
1217     Increment inc = new Increment(Bytes.toBytes("somerow"));
1218     inc.setDurability(Durability.SKIP_WAL);
1219     inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
1220     try {
1221       region.increment(inc);
1222     } catch (IOException e) {
1223       exceptionCaught = true;
1224     } finally {
1225       HRegion.closeHRegion(this.region);
1226       this.region = null;
1227     }
1228     assertTrue(exceptionCaught == true);
1229   }
1230 
1231   private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
1232     InternalScanner scanner = buildScanner(keyPrefix, value, r);
1233     int count = 0;
1234     boolean more = false;
1235     List<Cell> results = new ArrayList<Cell>();
1236     do {
1237       more = scanner.next(results);
1238       if (results != null && !results.isEmpty())
1239         count++;
1240       else
1241         break;
1242       Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
1243       delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
1244       r.delete(delete);
1245       results.clear();
1246     } while (more);
1247     assertEquals("Did not perform correct number of deletes", 3, count);
1248   }
1249 
1250   private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
1251     InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
1252     int numberOfResults = 0;
1253     List<Cell> results = new ArrayList<Cell>();
1254     boolean more = false;
1255     do {
1256       more = resultScanner.next(results);
1257       if (results != null && !results.isEmpty())
1258         numberOfResults++;
1259       else
1260         break;
1261       for (Cell kv : results) {
1262         System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
1263       }
1264       results.clear();
1265     } while (more);
1266     return numberOfResults;
1267   }
1268 
1269   private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
1270       throws IOException {
1271     // Defaults FilterList.Operator.MUST_PASS_ALL.
1272     FilterList allFilters = new FilterList();
1273     allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
1274     // Only return rows where this column value exists in the row.
1275     SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
1276         Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
1277     filter.setFilterIfMissing(true);
1278     allFilters.addFilter(filter);
1279     Scan scan = new Scan();
1280     scan.addFamily(Bytes.toBytes("trans-blob"));
1281     scan.addFamily(Bytes.toBytes("trans-type"));
1282     scan.addFamily(Bytes.toBytes("trans-date"));
1283     scan.addFamily(Bytes.toBytes("trans-tags"));
1284     scan.addFamily(Bytes.toBytes("trans-group"));
1285     scan.setFilter(allFilters);
1286     return r.getScanner(scan);
1287   }
1288 
1289   private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
1290     for (int i = 0; i < numRows; i++) {
1291       String row = key + "_" + i/* UUID.randomUUID().toString() */;
1292       System.out.println(String.format("Saving row: %s, with value %s", row, value));
1293       Put put = new Put(Bytes.toBytes(row));
1294       put.setDurability(Durability.SKIP_WAL);
1295       put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
1296       put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
1297       put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
1298       put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
1299       put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
1300       r.put(put);
1301     }
1302   }
1303 
1304   @Test
1305   public void testFamilyWithAndWithoutColon() throws Exception {
1306     byte[] b = Bytes.toBytes(getName());
1307     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1308     this.region = initHRegion(b, getName(), CONF, cf);
1309     try {
1310       Put p = new Put(b);
1311       byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
1312       p.add(cfwithcolon, cfwithcolon, cfwithcolon);
1313       boolean exception = false;
1314       try {
1315         this.region.put(p);
1316       } catch (NoSuchColumnFamilyException e) {
1317         exception = true;
1318       }
1319       assertTrue(exception);
1320     } finally {
1321       HRegion.closeHRegion(this.region);
1322       this.region = null;
1323     }
1324   }
1325 
1326   @Test
1327   public void testBatchPut_whileNoRowLocksHeld() throws IOException {
1328     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1329     byte[] qual = Bytes.toBytes("qual");
1330     byte[] val = Bytes.toBytes("val");
1331     this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1332     MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1333     try {
1334       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1335       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1336 
1337       LOG.info("First a batch put with all valid puts");
1338       final Put[] puts = new Put[10];
1339       for (int i = 0; i < 10; i++) {
1340         puts[i] = new Put(Bytes.toBytes("row_" + i));
1341         puts[i].add(cf, qual, val);
1342       }
1343 
1344       OperationStatus[] codes = this.region.batchMutate(puts);
1345       assertEquals(10, codes.length);
1346       for (int i = 0; i < 10; i++) {
1347         assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1348       }
1349       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1350 
1351       LOG.info("Next a batch put with one invalid family");
1352       puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1353       codes = this.region.batchMutate(puts);
1354       assertEquals(10, codes.length);
1355       for (int i = 0; i < 10; i++) {
1356         assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1357             codes[i].getOperationStatusCode());
1358       }
1359 
1360       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1361     } finally {
1362       HRegion.closeHRegion(this.region);
1363       this.region = null;
1364     }
1365   }
1366 
1367   @Test
1368   public void testBatchPut_whileMultipleRowLocksHeld() throws Exception {
1369     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1370     byte[] qual = Bytes.toBytes("qual");
1371     byte[] val = Bytes.toBytes("val");
1372     this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1373     MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1374     try {
1375       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1376       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1377 
1378       final Put[] puts = new Put[10];
1379       for (int i = 0; i < 10; i++) {
1380         puts[i] = new Put(Bytes.toBytes("row_" + i));
1381         puts[i].add(cf, qual, val);
1382       }
1383       puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1384 
1385       LOG.info("batchPut will have to break into four batches to avoid row locks");
1386       RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2"));
1387       RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_4"));
1388       RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_6"));
1389 
1390       MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1391       final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
1392       TestThread putter = new TestThread(ctx) {
1393         @Override
1394         public void doWork() throws IOException {
1395           retFromThread.set(region.batchMutate(puts));
1396         }
1397       };
1398       LOG.info("...starting put thread while holding locks");
1399       ctx.addThread(putter);
1400       ctx.startThreads();
1401 
1402       LOG.info("...waiting for put thread to sync 1st time");
1403       waitForCounter(source, "syncTimeNumOps", syncs + 1);
1404 
1405       // Now attempt to close the region from another thread.  Prior to HBASE-12565
1406       // this would cause the in-progress batchMutate operation to to fail with
1407       // exception because it use to release and re-acquire the close-guard lock
1408       // between batches.  Caller then didn't get status indicating which writes succeeded.
1409       // We now expect this thread to block until the batchMutate call finishes.
1410       Thread regionCloseThread = new Thread() {
1411         @Override
1412         public void run() {
1413           try {
1414             HRegion.closeHRegion(region);
1415           } catch (IOException e) {
1416             throw new RuntimeException(e);
1417           }
1418         }
1419       };
1420       regionCloseThread.start();
1421 
1422       LOG.info("...releasing row lock 1, which should let put thread continue");
1423       rowLock1.release();
1424 
1425       LOG.info("...waiting for put thread to sync 2nd time");
1426       waitForCounter(source, "syncTimeNumOps", syncs + 2);
1427 
1428       LOG.info("...releasing row lock 2, which should let put thread continue");
1429       rowLock2.release();
1430 
1431       LOG.info("...waiting for put thread to sync 3rd time");
1432       waitForCounter(source, "syncTimeNumOps", syncs + 3);
1433 
1434       LOG.info("...releasing row lock 3, which should let put thread continue");
1435       rowLock3.release();
1436 
1437       LOG.info("...waiting for put thread to sync 4th time");
1438       waitForCounter(source, "syncTimeNumOps", syncs + 4);
1439 
1440       LOG.info("...joining on put thread");
1441       ctx.stop();
1442       regionCloseThread.join();
1443 
1444       OperationStatus[] codes = retFromThread.get();
1445       for (int i = 0; i < codes.length; i++) {
1446         assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1447             codes[i].getOperationStatusCode());
1448       }
1449     } finally {
1450       HRegion.closeHRegion(this.region);
1451       this.region = null;
1452     }
1453   }
1454 
1455   private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount)
1456       throws InterruptedException {
1457     long startWait = System.currentTimeMillis();
1458     long currentCount;
1459     while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) {
1460       Thread.sleep(100);
1461       if (System.currentTimeMillis() - startWait > 10000) {
1462         fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName,
1463           expectedCount, currentCount));
1464       }
1465     }
1466   }
1467 
1468   @Test
1469   public void testBatchPutWithTsSlop() throws Exception {
1470     byte[] b = Bytes.toBytes(getName());
1471     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1472     byte[] qual = Bytes.toBytes("qual");
1473     byte[] val = Bytes.toBytes("val");
1474 
1475     // add data with a timestamp that is too recent for range. Ensure assert
1476     CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1477     this.region = initHRegion(b, getName(), CONF, cf);
1478 
1479     try {
1480       MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1481       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1482       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1483 
1484       final Put[] puts = new Put[10];
1485       for (int i = 0; i < 10; i++) {
1486         puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
1487         puts[i].add(cf, qual, val);
1488       }
1489 
1490       OperationStatus[] codes = this.region.batchMutate(puts);
1491       assertEquals(10, codes.length);
1492       for (int i = 0; i < 10; i++) {
1493         assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1494       }
1495       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1496 
1497     } finally {
1498       HRegion.closeHRegion(this.region);
1499       this.region = null;
1500     }
1501 
1502   }
1503 
1504   // ////////////////////////////////////////////////////////////////////////////
1505   // checkAndMutate tests
1506   // ////////////////////////////////////////////////////////////////////////////
1507   @Test
1508   public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1509     byte[] row1 = Bytes.toBytes("row1");
1510     byte[] fam1 = Bytes.toBytes("fam1");
1511     byte[] qf1 = Bytes.toBytes("qualifier");
1512     byte[] emptyVal = new byte[] {};
1513     byte[] val1 = Bytes.toBytes("value1");
1514     byte[] val2 = Bytes.toBytes("value2");
1515 
1516     // Setting up region
1517     String method = this.getName();
1518     this.region = initHRegion(tableName, method, CONF, fam1);
1519     try {
1520       // Putting empty data in key
1521       Put put = new Put(row1);
1522       put.add(fam1, qf1, emptyVal);
1523 
1524       // checkAndPut with empty value
1525       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1526           emptyVal), put, true);
1527       assertTrue(res);
1528 
1529       // Putting data in key
1530       put = new Put(row1);
1531       put.add(fam1, qf1, val1);
1532 
1533       // checkAndPut with correct value
1534       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1535           put, true);
1536       assertTrue(res);
1537 
1538       // not empty anymore
1539       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1540           put, true);
1541       assertFalse(res);
1542 
1543       Delete delete = new Delete(row1);
1544       delete.deleteColumn(fam1, qf1);
1545       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1546           delete, true);
1547       assertFalse(res);
1548 
1549       put = new Put(row1);
1550       put.add(fam1, qf1, val2);
1551       // checkAndPut with correct value
1552       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1553           put, true);
1554       assertTrue(res);
1555 
1556       // checkAndDelete with correct value
1557       delete = new Delete(row1);
1558       delete.deleteColumn(fam1, qf1);
1559       delete.deleteColumn(fam1, qf1);
1560       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1561           delete, true);
1562       assertTrue(res);
1563 
1564       delete = new Delete(row1);
1565       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1566           delete, true);
1567       assertTrue(res);
1568 
1569       // checkAndPut looking for a null value
1570       put = new Put(row1);
1571       put.add(fam1, qf1, val1);
1572 
1573       res = region
1574           .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
1575       assertTrue(res);
1576     } finally {
1577       HRegion.closeHRegion(this.region);
1578       this.region = null;
1579     }
1580   }
1581 
1582   @Test
1583   public void testCheckAndMutate_WithWrongValue() throws IOException {
1584     byte[] row1 = Bytes.toBytes("row1");
1585     byte[] fam1 = Bytes.toBytes("fam1");
1586     byte[] qf1 = Bytes.toBytes("qualifier");
1587     byte[] val1 = Bytes.toBytes("value1");
1588     byte[] val2 = Bytes.toBytes("value2");
1589 
1590     // Setting up region
1591     String method = this.getName();
1592     this.region = initHRegion(tableName, method, CONF, fam1);
1593     try {
1594       // Putting data in key
1595       Put put = new Put(row1);
1596       put.add(fam1, qf1, val1);
1597       region.put(put);
1598 
1599       // checkAndPut with wrong value
1600       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1601           val2), put, true);
1602       assertEquals(false, res);
1603 
1604       // checkAndDelete with wrong value
1605       Delete delete = new Delete(row1);
1606       delete.deleteFamily(fam1);
1607       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1608           put, true);
1609       assertEquals(false, res);
1610     } finally {
1611       HRegion.closeHRegion(this.region);
1612       this.region = null;
1613     }
1614   }
1615 
1616   @Test
1617   public void testCheckAndMutate_WithCorrectValue() throws IOException {
1618     byte[] row1 = Bytes.toBytes("row1");
1619     byte[] fam1 = Bytes.toBytes("fam1");
1620     byte[] qf1 = Bytes.toBytes("qualifier");
1621     byte[] val1 = Bytes.toBytes("value1");
1622 
1623     // Setting up region
1624     String method = this.getName();
1625     this.region = initHRegion(tableName, method, CONF, fam1);
1626     try {
1627       // Putting data in key
1628       Put put = new Put(row1);
1629       put.add(fam1, qf1, val1);
1630       region.put(put);
1631 
1632       // checkAndPut with correct value
1633       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1634           val1), put, true);
1635       assertEquals(true, res);
1636 
1637       // checkAndDelete with correct value
1638       Delete delete = new Delete(row1);
1639       delete.deleteColumn(fam1, qf1);
1640       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1641           delete, true);
1642       assertEquals(true, res);
1643     } finally {
1644       HRegion.closeHRegion(this.region);
1645       this.region = null;
1646     }
1647   }
1648 
1649   @Test
1650   public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1651     byte[] row1 = Bytes.toBytes("row1");
1652     byte[] fam1 = Bytes.toBytes("fam1");
1653     byte[] qf1 = Bytes.toBytes("qualifier");
1654     byte[] val1 = Bytes.toBytes("value1");
1655     byte[] val2 = Bytes.toBytes("value2");
1656     byte[] val3 = Bytes.toBytes("value3");
1657     byte[] val4 = Bytes.toBytes("value4");
1658 
1659     // Setting up region
1660     String method = this.getName();
1661     this.region = initHRegion(tableName, method, CONF, fam1);
1662     try {
1663       // Putting val3 in key
1664       Put put = new Put(row1);
1665       put.add(fam1, qf1, val3);
1666       region.put(put);
1667 
1668       // Test CompareOp.LESS: original = val3, compare with val3, fail
1669       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1670           new BinaryComparator(val3), put, true);
1671       assertEquals(false, res);
1672 
1673       // Test CompareOp.LESS: original = val3, compare with val4, fail
1674       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1675           new BinaryComparator(val4), put, true);
1676       assertEquals(false, res);
1677 
1678       // Test CompareOp.LESS: original = val3, compare with val2,
1679       // succeed (now value = val2)
1680       put = new Put(row1);
1681       put.add(fam1, qf1, val2);
1682       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1683           new BinaryComparator(val2), put, true);
1684       assertEquals(true, res);
1685 
1686       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail
1687       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1688           new BinaryComparator(val3), put, true);
1689       assertEquals(false, res);
1690 
1691       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2,
1692       // succeed (value still = val2)
1693       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1694           new BinaryComparator(val2), put, true);
1695       assertEquals(true, res);
1696 
1697       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1,
1698       // succeed (now value = val3)
1699       put = new Put(row1);
1700       put.add(fam1, qf1, val3);
1701       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1702           new BinaryComparator(val1), put, true);
1703       assertEquals(true, res);
1704 
1705       // Test CompareOp.GREATER: original = val3, compare with val3, fail
1706       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1707           new BinaryComparator(val3), put, true);
1708       assertEquals(false, res);
1709 
1710       // Test CompareOp.GREATER: original = val3, compare with val2, fail
1711       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1712           new BinaryComparator(val2), put, true);
1713       assertEquals(false, res);
1714 
1715       // Test CompareOp.GREATER: original = val3, compare with val4,
1716       // succeed (now value = val2)
1717       put = new Put(row1);
1718       put.add(fam1, qf1, val2);
1719       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1720           new BinaryComparator(val4), put, true);
1721       assertEquals(true, res);
1722 
1723       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail
1724       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1725           new BinaryComparator(val1), put, true);
1726       assertEquals(false, res);
1727 
1728       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2,
1729       // succeed (value still = val2)
1730       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1731           new BinaryComparator(val2), put, true);
1732       assertEquals(true, res);
1733 
1734       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed
1735       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1736           new BinaryComparator(val3), put, true);
1737       assertEquals(true, res);
1738     } finally {
1739       HRegion.closeHRegion(this.region);
1740       this.region = null;
1741     }
1742   }
1743 
1744   @Test
1745   public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1746     byte[] row1 = Bytes.toBytes("row1");
1747     byte[] fam1 = Bytes.toBytes("fam1");
1748     byte[] fam2 = Bytes.toBytes("fam2");
1749     byte[] qf1 = Bytes.toBytes("qualifier");
1750     byte[] val1 = Bytes.toBytes("value1");
1751     byte[] val2 = Bytes.toBytes("value2");
1752 
1753     byte[][] families = { fam1, fam2 };
1754 
1755     // Setting up region
1756     String method = this.getName();
1757     this.region = initHRegion(tableName, method, CONF, families);
1758     try {
1759       // Putting data in the key to check
1760       Put put = new Put(row1);
1761       put.add(fam1, qf1, val1);
1762       region.put(put);
1763 
1764       // Creating put to add
1765       long ts = System.currentTimeMillis();
1766       KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1767       put = new Put(row1);
1768       put.add(kv);
1769 
1770       // checkAndPut with wrong value
1771       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1772           val1), put, true);
1773       assertEquals(true, res);
1774 
1775       Get get = new Get(row1);
1776       get.addColumn(fam2, qf1);
1777       Cell[] actual = region.get(get).rawCells();
1778 
1779       Cell[] expected = { kv };
1780 
1781       assertEquals(expected.length, actual.length);
1782       for (int i = 0; i < actual.length; i++) {
1783         assertEquals(expected[i], actual[i]);
1784       }
1785     } finally {
1786       HRegion.closeHRegion(this.region);
1787       this.region = null;
1788     }
1789   }
1790 
1791   @Test
1792   public void testCheckAndPut_wrongRowInPut() throws IOException {
1793     TableName tableName = TableName.valueOf(name.getMethodName());
1794     this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
1795     try {
1796       Put put = new Put(row2);
1797       put.add(fam1, qual1, value1);
1798       try {
1799         region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1800             new BinaryComparator(value2), put, false);
1801         fail();
1802       } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1803         // expected exception.
1804       }
1805     } finally {
1806       HRegion.closeHRegion(this.region);
1807       this.region = null;
1808     }
1809   }
1810 
1811   @Test
1812   public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1813     byte[] row1 = Bytes.toBytes("row1");
1814     byte[] fam1 = Bytes.toBytes("fam1");
1815     byte[] fam2 = Bytes.toBytes("fam2");
1816     byte[] qf1 = Bytes.toBytes("qualifier1");
1817     byte[] qf2 = Bytes.toBytes("qualifier2");
1818     byte[] qf3 = Bytes.toBytes("qualifier3");
1819     byte[] val1 = Bytes.toBytes("value1");
1820     byte[] val2 = Bytes.toBytes("value2");
1821     byte[] val3 = Bytes.toBytes("value3");
1822     byte[] emptyVal = new byte[] {};
1823 
1824     byte[][] families = { fam1, fam2 };
1825 
1826     // Setting up region
1827     String method = this.getName();
1828     this.region = initHRegion(tableName, method, CONF, families);
1829     try {
1830       // Put content
1831       Put put = new Put(row1);
1832       put.add(fam1, qf1, val1);
1833       region.put(put);
1834       Threads.sleep(2);
1835 
1836       put = new Put(row1);
1837       put.add(fam1, qf1, val2);
1838       put.add(fam2, qf1, val3);
1839       put.add(fam2, qf2, val2);
1840       put.add(fam2, qf3, val1);
1841       put.add(fam1, qf3, val1);
1842       region.put(put);
1843 
1844       // Multi-column delete
1845       Delete delete = new Delete(row1);
1846       delete.deleteColumn(fam1, qf1);
1847       delete.deleteColumn(fam2, qf1);
1848       delete.deleteColumn(fam1, qf3);
1849       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1850           val2), delete, true);
1851       assertEquals(true, res);
1852 
1853       Get get = new Get(row1);
1854       get.addColumn(fam1, qf1);
1855       get.addColumn(fam1, qf3);
1856       get.addColumn(fam2, qf2);
1857       Result r = region.get(get);
1858       assertEquals(2, r.size());
1859       assertArrayEquals(val1, r.getValue(fam1, qf1));
1860       assertArrayEquals(val2, r.getValue(fam2, qf2));
1861 
1862       // Family delete
1863       delete = new Delete(row1);
1864       delete.deleteFamily(fam2);
1865       res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1866           delete, true);
1867       assertEquals(true, res);
1868 
1869       get = new Get(row1);
1870       r = region.get(get);
1871       assertEquals(1, r.size());
1872       assertArrayEquals(val1, r.getValue(fam1, qf1));
1873 
1874       // Row delete
1875       delete = new Delete(row1);
1876       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1877           delete, true);
1878       assertEquals(true, res);
1879       get = new Get(row1);
1880       r = region.get(get);
1881       assertEquals(0, r.size());
1882     } finally {
1883       HRegion.closeHRegion(this.region);
1884       this.region = null;
1885     }
1886   }
1887 
1888   // ////////////////////////////////////////////////////////////////////////////
1889   // Delete tests
1890   // ////////////////////////////////////////////////////////////////////////////
1891   @Test
1892   public void testDelete_multiDeleteColumn() throws IOException {
1893     byte[] row1 = Bytes.toBytes("row1");
1894     byte[] fam1 = Bytes.toBytes("fam1");
1895     byte[] qual = Bytes.toBytes("qualifier");
1896     byte[] value = Bytes.toBytes("value");
1897 
1898     Put put = new Put(row1);
1899     put.add(fam1, qual, 1, value);
1900     put.add(fam1, qual, 2, value);
1901 
1902     String method = this.getName();
1903     this.region = initHRegion(tableName, method, CONF, fam1);
1904     try {
1905       region.put(put);
1906 
1907       // We do support deleting more than 1 'latest' version
1908       Delete delete = new Delete(row1);
1909       delete.deleteColumn(fam1, qual);
1910       delete.deleteColumn(fam1, qual);
1911       region.delete(delete);
1912 
1913       Get get = new Get(row1);
1914       get.addFamily(fam1);
1915       Result r = region.get(get);
1916       assertEquals(0, r.size());
1917     } finally {
1918       HRegion.closeHRegion(this.region);
1919       this.region = null;
1920     }
1921   }
1922 
1923   @Test
1924   public void testDelete_CheckFamily() throws IOException {
1925     byte[] row1 = Bytes.toBytes("row1");
1926     byte[] fam1 = Bytes.toBytes("fam1");
1927     byte[] fam2 = Bytes.toBytes("fam2");
1928     byte[] fam3 = Bytes.toBytes("fam3");
1929     byte[] fam4 = Bytes.toBytes("fam4");
1930 
1931     // Setting up region
1932     String method = this.getName();
1933     this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
1934     try {
1935       List<Cell> kvs = new ArrayList<Cell>();
1936       kvs.add(new KeyValue(row1, fam4, null, null));
1937 
1938       // testing existing family
1939       byte[] family = fam2;
1940       try {
1941         NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1942             Bytes.BYTES_COMPARATOR);
1943         deleteMap.put(family, kvs);
1944         region.delete(deleteMap, Durability.SYNC_WAL);
1945       } catch (Exception e) {
1946         assertTrue("Family " + new String(family) + " does not exist", false);
1947       }
1948 
1949       // testing non existing family
1950       boolean ok = false;
1951       family = fam4;
1952       try {
1953         NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1954             Bytes.BYTES_COMPARATOR);
1955         deleteMap.put(family, kvs);
1956         region.delete(deleteMap, Durability.SYNC_WAL);
1957       } catch (Exception e) {
1958         ok = true;
1959       }
1960       assertEquals("Family " + new String(family) + " does exist", true, ok);
1961     } finally {
1962       HRegion.closeHRegion(this.region);
1963       this.region = null;
1964     }
1965   }
1966 
1967   @Test
1968   public void testDelete_mixed() throws IOException, InterruptedException {
1969     byte[] fam = Bytes.toBytes("info");
1970     byte[][] families = { fam };
1971     String method = this.getName();
1972     this.region = initHRegion(tableName, method, CONF, families);
1973     try {
1974       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1975 
1976       byte[] row = Bytes.toBytes("table_name");
1977       // column names
1978       byte[] serverinfo = Bytes.toBytes("serverinfo");
1979       byte[] splitA = Bytes.toBytes("splitA");
1980       byte[] splitB = Bytes.toBytes("splitB");
1981 
1982       // add some data:
1983       Put put = new Put(row);
1984       put.add(fam, splitA, Bytes.toBytes("reference_A"));
1985       region.put(put);
1986 
1987       put = new Put(row);
1988       put.add(fam, splitB, Bytes.toBytes("reference_B"));
1989       region.put(put);
1990 
1991       put = new Put(row);
1992       put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1993       region.put(put);
1994 
1995       // ok now delete a split:
1996       Delete delete = new Delete(row);
1997       delete.deleteColumns(fam, splitA);
1998       region.delete(delete);
1999 
2000       // assert some things:
2001       Get get = new Get(row).addColumn(fam, serverinfo);
2002       Result result = region.get(get);
2003       assertEquals(1, result.size());
2004 
2005       get = new Get(row).addColumn(fam, splitA);
2006       result = region.get(get);
2007       assertEquals(0, result.size());
2008 
2009       get = new Get(row).addColumn(fam, splitB);
2010       result = region.get(get);
2011       assertEquals(1, result.size());
2012 
2013       // Assert that after a delete, I can put.
2014       put = new Put(row);
2015       put.add(fam, splitA, Bytes.toBytes("reference_A"));
2016       region.put(put);
2017       get = new Get(row);
2018       result = region.get(get);
2019       assertEquals(3, result.size());
2020 
2021       // Now delete all... then test I can add stuff back
2022       delete = new Delete(row);
2023       region.delete(delete);
2024       assertEquals(0, region.get(get).size());
2025 
2026       region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
2027       result = region.get(get);
2028       assertEquals(1, result.size());
2029     } finally {
2030       HRegion.closeHRegion(this.region);
2031       this.region = null;
2032     }
2033   }
2034 
2035   @Test
2036   public void testDeleteRowWithFutureTs() throws IOException {
2037     byte[] fam = Bytes.toBytes("info");
2038     byte[][] families = { fam };
2039     String method = this.getName();
2040     this.region = initHRegion(tableName, method, CONF, families);
2041     try {
2042       byte[] row = Bytes.toBytes("table_name");
2043       // column names
2044       byte[] serverinfo = Bytes.toBytes("serverinfo");
2045 
2046       // add data in the far future
2047       Put put = new Put(row);
2048       put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
2049       region.put(put);
2050 
2051       // now delete something in the present
2052       Delete delete = new Delete(row);
2053       region.delete(delete);
2054 
2055       // make sure we still see our data
2056       Get get = new Get(row).addColumn(fam, serverinfo);
2057       Result result = region.get(get);
2058       assertEquals(1, result.size());
2059 
2060       // delete the future row
2061       delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
2062       region.delete(delete);
2063 
2064       // make sure it is gone
2065       get = new Get(row).addColumn(fam, serverinfo);
2066       result = region.get(get);
2067       assertEquals(0, result.size());
2068     } finally {
2069       HRegion.closeHRegion(this.region);
2070       this.region = null;
2071     }
2072   }
2073 
2074   /**
2075    * Tests that the special LATEST_TIMESTAMP option for puts gets replaced by
2076    * the actual timestamp
2077    */
2078   @Test
2079   public void testPutWithLatestTS() throws IOException {
2080     byte[] fam = Bytes.toBytes("info");
2081     byte[][] families = { fam };
2082     String method = this.getName();
2083     this.region = initHRegion(tableName, method, CONF, families);
2084     try {
2085       byte[] row = Bytes.toBytes("row1");
2086       // column names
2087       byte[] qual = Bytes.toBytes("qual");
2088 
2089       // add data with LATEST_TIMESTAMP, put without WAL
2090       Put put = new Put(row);
2091       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
2092       region.put(put);
2093 
2094       // Make sure it shows up with an actual timestamp
2095       Get get = new Get(row).addColumn(fam, qual);
2096       Result result = region.get(get);
2097       assertEquals(1, result.size());
2098       Cell kv = result.rawCells()[0];
2099       LOG.info("Got: " + kv);
2100       assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
2101           kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
2102 
2103       // Check same with WAL enabled (historically these took different
2104       // code paths, so check both)
2105       row = Bytes.toBytes("row2");
2106       put = new Put(row);
2107       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
2108       region.put(put);
2109 
2110       // Make sure it shows up with an actual timestamp
2111       get = new Get(row).addColumn(fam, qual);
2112       result = region.get(get);
2113       assertEquals(1, result.size());
2114       kv = result.rawCells()[0];
2115       LOG.info("Got: " + kv);
2116       assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
2117           kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
2118     } finally {
2119       HRegion.closeHRegion(this.region);
2120       this.region = null;
2121     }
2122 
2123   }
2124 
2125   /**
2126    * Tests that there is server-side filtering for invalid timestamp upper
2127    * bound. Note that the timestamp lower bound is automatically handled for us
2128    * by the TTL field.
2129    */
2130   @Test
2131   public void testPutWithTsSlop() throws IOException {
2132     byte[] fam = Bytes.toBytes("info");
2133     byte[][] families = { fam };
2134     String method = this.getName();
2135 
2136     // add data with a timestamp that is too recent for range. Ensure assert
2137     CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
2138     this.region = initHRegion(tableName, method, CONF, families);
2139     boolean caughtExcep = false;
2140     try {
2141       try {
2142         // no TS specified == use latest. should not error
2143         region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
2144         // TS out of range. should error
2145         region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
2146             Bytes.toBytes("value")));
2147         fail("Expected IOE for TS out of configured timerange");
2148       } catch (FailedSanityCheckException ioe) {
2149         LOG.debug("Received expected exception", ioe);
2150         caughtExcep = true;
2151       }
2152       assertTrue("Should catch FailedSanityCheckException", caughtExcep);
2153     } finally {
2154       HRegion.closeHRegion(this.region);
2155       this.region = null;
2156     }
2157   }
2158 
2159   @Test
2160   public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
2161     byte[] fam1 = Bytes.toBytes("columnA");
2162     byte[] fam2 = Bytes.toBytes("columnB");
2163     this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
2164     try {
2165       byte[] rowA = Bytes.toBytes("rowA");
2166       byte[] rowB = Bytes.toBytes("rowB");
2167 
2168       byte[] value = Bytes.toBytes("value");
2169 
2170       Delete delete = new Delete(rowA);
2171       delete.deleteFamily(fam1);
2172 
2173       region.delete(delete);
2174 
2175       // now create data.
2176       Put put = new Put(rowA);
2177       put.add(fam2, null, value);
2178       region.put(put);
2179 
2180       put = new Put(rowB);
2181       put.add(fam1, null, value);
2182       put.add(fam2, null, value);
2183       region.put(put);
2184 
2185       Scan scan = new Scan();
2186       scan.addFamily(fam1).addFamily(fam2);
2187       InternalScanner s = region.getScanner(scan);
2188       List<Cell> results = new ArrayList<Cell>();
2189       s.next(results);
2190       assertTrue(CellUtil.matchingRow(results.get(0), rowA));
2191 
2192       results.clear();
2193       s.next(results);
2194       assertTrue(CellUtil.matchingRow(results.get(0), rowB));
2195     } finally {
2196       HRegion.closeHRegion(this.region);
2197       this.region = null;
2198     }
2199   }
2200 
2201   @Test
2202   public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
2203     Delete delete = new Delete(row);
2204     delete.deleteColumns(fam1, qual1);
2205     doTestDelete_AndPostInsert(delete);
2206   }
2207 
2208   @Test
2209   public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
2210     Delete delete = new Delete(row);
2211     delete.deleteFamily(fam1);
2212     doTestDelete_AndPostInsert(delete);
2213   }
2214 
2215   public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
2216     TableName tableName = TableName.valueOf(name.getMethodName());
2217     this.region = initHRegion(tableName, getName(), CONF, fam1);
2218     try {
2219       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
2220       Put put = new Put(row);
2221       put.add(fam1, qual1, value1);
2222       region.put(put);
2223 
2224       // now delete the value:
2225       region.delete(delete);
2226 
2227       // ok put data:
2228       put = new Put(row);
2229       put.add(fam1, qual1, value2);
2230       region.put(put);
2231 
2232       // ok get:
2233       Get get = new Get(row);
2234       get.addColumn(fam1, qual1);
2235 
2236       Result r = region.get(get);
2237       assertEquals(1, r.size());
2238       assertArrayEquals(value2, r.getValue(fam1, qual1));
2239 
2240       // next:
2241       Scan scan = new Scan(row);
2242       scan.addColumn(fam1, qual1);
2243       InternalScanner s = region.getScanner(scan);
2244 
2245       List<Cell> results = new ArrayList<Cell>();
2246       assertEquals(false, s.next(results));
2247       assertEquals(1, results.size());
2248       Cell kv = results.get(0);
2249 
2250       assertArrayEquals(value2, CellUtil.cloneValue(kv));
2251       assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
2252       assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
2253       assertArrayEquals(row, CellUtil.cloneRow(kv));
2254     } finally {
2255       HRegion.closeHRegion(this.region);
2256       this.region = null;
2257     }
2258   }
2259 
2260   @Test
2261   public void testDelete_CheckTimestampUpdated() throws IOException {
2262     TableName tableName = TableName.valueOf(name.getMethodName());
2263     byte[] row1 = Bytes.toBytes("row1");
2264     byte[] col1 = Bytes.toBytes("col1");
2265     byte[] col2 = Bytes.toBytes("col2");
2266     byte[] col3 = Bytes.toBytes("col3");
2267 
2268     // Setting up region
2269     String method = this.getName();
2270     this.region = initHRegion(tableName, method, CONF, fam1);
2271     try {
2272       // Building checkerList
2273       List<Cell> kvs = new ArrayList<Cell>();
2274       kvs.add(new KeyValue(row1, fam1, col1, null));
2275       kvs.add(new KeyValue(row1, fam1, col2, null));
2276       kvs.add(new KeyValue(row1, fam1, col3, null));
2277 
2278       NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
2279           Bytes.BYTES_COMPARATOR);
2280       deleteMap.put(fam1, kvs);
2281       region.delete(deleteMap, Durability.SYNC_WAL);
2282 
2283       // extract the key values out the memstore:
2284       // This is kinda hacky, but better than nothing...
2285       long now = System.currentTimeMillis();
2286       DefaultMemStore memstore = (DefaultMemStore) ((HStore) region.getStore(fam1)).memstore;
2287       Cell firstCell = memstore.cellSet.first();
2288       assertTrue(firstCell.getTimestamp() <= now);
2289       now = firstCell.getTimestamp();
2290       for (Cell cell : memstore.cellSet) {
2291         assertTrue(cell.getTimestamp() <= now);
2292         now = cell.getTimestamp();
2293       }
2294     } finally {
2295       HRegion.closeHRegion(this.region);
2296       this.region = null;
2297     }
2298   }
2299 
2300   // ////////////////////////////////////////////////////////////////////////////
2301   // Get tests
2302   // ////////////////////////////////////////////////////////////////////////////
2303   @Test
2304   public void testGet_FamilyChecker() throws IOException {
2305     byte[] row1 = Bytes.toBytes("row1");
2306     byte[] fam1 = Bytes.toBytes("fam1");
2307     byte[] fam2 = Bytes.toBytes("False");
2308     byte[] col1 = Bytes.toBytes("col1");
2309 
2310     // Setting up region
2311     String method = this.getName();
2312     this.region = initHRegion(tableName, method, CONF, fam1);
2313     try {
2314       Get get = new Get(row1);
2315       get.addColumn(fam2, col1);
2316 
2317       // Test
2318       try {
2319         region.get(get);
2320       } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
2321         assertFalse(false);
2322         return;
2323       }
2324       assertFalse(true);
2325     } finally {
2326       HRegion.closeHRegion(this.region);
2327       this.region = null;
2328     }
2329   }
2330 
2331   @Test
2332   public void testGet_Basic() throws IOException {
2333     byte[] row1 = Bytes.toBytes("row1");
2334     byte[] fam1 = Bytes.toBytes("fam1");
2335     byte[] col1 = Bytes.toBytes("col1");
2336     byte[] col2 = Bytes.toBytes("col2");
2337     byte[] col3 = Bytes.toBytes("col3");
2338     byte[] col4 = Bytes.toBytes("col4");
2339     byte[] col5 = Bytes.toBytes("col5");
2340 
2341     // Setting up region
2342     String method = this.getName();
2343     this.region = initHRegion(tableName, method, CONF, fam1);
2344     try {
2345       // Add to memstore
2346       Put put = new Put(row1);
2347       put.add(fam1, col1, null);
2348       put.add(fam1, col2, null);
2349       put.add(fam1, col3, null);
2350       put.add(fam1, col4, null);
2351       put.add(fam1, col5, null);
2352       region.put(put);
2353 
2354       Get get = new Get(row1);
2355       get.addColumn(fam1, col2);
2356       get.addColumn(fam1, col4);
2357       // Expected result
2358       KeyValue kv1 = new KeyValue(row1, fam1, col2);
2359       KeyValue kv2 = new KeyValue(row1, fam1, col4);
2360       KeyValue[] expected = { kv1, kv2 };
2361 
2362       // Test
2363       Result res = region.get(get);
2364       assertEquals(expected.length, res.size());
2365       for (int i = 0; i < res.size(); i++) {
2366         assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
2367         assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
2368         assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
2369       }
2370 
2371       // Test using a filter on a Get
2372       Get g = new Get(row1);
2373       final int count = 2;
2374       g.setFilter(new ColumnCountGetFilter(count));
2375       res = region.get(g);
2376       assertEquals(count, res.size());
2377     } finally {
2378       HRegion.closeHRegion(this.region);
2379       this.region = null;
2380     }
2381   }
2382 
2383   @Test
2384   public void testGet_Empty() throws IOException {
2385     byte[] row = Bytes.toBytes("row");
2386     byte[] fam = Bytes.toBytes("fam");
2387 
2388     String method = this.getName();
2389     this.region = initHRegion(tableName, method, CONF, fam);
2390     try {
2391       Get get = new Get(row);
2392       get.addFamily(fam);
2393       Result r = region.get(get);
2394 
2395       assertTrue(r.isEmpty());
2396     } finally {
2397       HRegion.closeHRegion(this.region);
2398       this.region = null;
2399     }
2400   }
2401 
2402   // ////////////////////////////////////////////////////////////////////////////
2403   // Merge test
2404   // ////////////////////////////////////////////////////////////////////////////
2405   @Test
2406   public void testMerge() throws IOException {
2407     byte[][] families = { fam1, fam2, fam3 };
2408     Configuration hc = initSplit();
2409     // Setting up region
2410     String method = this.getName();
2411     this.region = initHRegion(tableName, method, hc, families);
2412     try {
2413       LOG.info("" + HBaseTestCase.addContent(region, fam3));
2414       region.flushcache();
2415       region.compactStores();
2416       byte[] splitRow = region.checkSplit();
2417       assertNotNull(splitRow);
2418       LOG.info("SplitRow: " + Bytes.toString(splitRow));
2419       HRegion[] subregions = splitRegion(region, splitRow);
2420       try {
2421         // Need to open the regions.
2422         for (int i = 0; i < subregions.length; i++) {
2423           HRegion.openHRegion(subregions[i], null);
2424           subregions[i].compactStores();
2425         }
2426         Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
2427         Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
2428         Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
2429         long startTime = System.currentTimeMillis();
2430         region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
2431         LOG.info("Merge regions elapsed time: "
2432             + ((System.currentTimeMillis() - startTime) / 1000.0));
2433         FILESYSTEM.delete(oldRegion1, true);
2434         FILESYSTEM.delete(oldRegion2, true);
2435         FILESYSTEM.delete(oldRegionPath, true);
2436         LOG.info("splitAndMerge completed.");
2437       } finally {
2438         for (int i = 0; i < subregions.length; i++) {
2439           try {
2440             HRegion.closeHRegion(subregions[i]);
2441           } catch (IOException e) {
2442             // Ignore.
2443           }
2444         }
2445       }
2446     } finally {
2447       HRegion.closeHRegion(this.region);
2448       this.region = null;
2449     }
2450   }
2451 
2452   /**
2453    * @param parent
2454    *          Region to split.
2455    * @param midkey
2456    *          Key to split around.
2457    * @return The Regions we created.
2458    * @throws IOException
2459    */
2460   HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
2461     PairOfSameType<HRegion> result = null;
2462     SplitTransaction st = new SplitTransaction(parent, midkey);
2463     // If prepare does not return true, for some reason -- logged inside in
2464     // the prepare call -- we are not ready to split just now. Just return.
2465     if (!st.prepare()) {
2466       parent.clearSplit();
2467       return null;
2468     }
2469     try {
2470       result = st.execute(null, null);
2471     } catch (IOException ioe) {
2472       try {
2473         LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; "
2474             + ioe.getMessage());
2475         st.rollback(null, null);
2476         LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString());
2477         return null;
2478       } catch (RuntimeException e) {
2479         // If failed rollback, kill this server to avoid having a hole in table.
2480         LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString()
2481             + " -- aborting server", e);
2482       }
2483     }
2484     finally {
2485       parent.clearSplit();
2486     }
2487     return new HRegion[] { result.getFirst(), result.getSecond() };
2488   }
2489 
2490   // ////////////////////////////////////////////////////////////////////////////
2491   // Scanner tests
2492   // ////////////////////////////////////////////////////////////////////////////
2493   @Test
2494   public void testGetScanner_WithOkFamilies() throws IOException {
2495     byte[] fam1 = Bytes.toBytes("fam1");
2496     byte[] fam2 = Bytes.toBytes("fam2");
2497 
2498     byte[][] families = { fam1, fam2 };
2499 
2500     // Setting up region
2501     String method = this.getName();
2502     this.region = initHRegion(tableName, method, CONF, families);
2503     try {
2504       Scan scan = new Scan();
2505       scan.addFamily(fam1);
2506       scan.addFamily(fam2);
2507       try {
2508         region.getScanner(scan);
2509       } catch (Exception e) {
2510         assertTrue("Families could not be found in Region", false);
2511       }
2512     } finally {
2513       HRegion.closeHRegion(this.region);
2514       this.region = null;
2515     }
2516   }
2517 
2518   @Test
2519   public void testGetScanner_WithNotOkFamilies() throws IOException {
2520     byte[] fam1 = Bytes.toBytes("fam1");
2521     byte[] fam2 = Bytes.toBytes("fam2");
2522 
2523     byte[][] families = { fam1 };
2524 
2525     // Setting up region
2526     String method = this.getName();
2527     this.region = initHRegion(tableName, method, CONF, families);
2528     try {
2529       Scan scan = new Scan();
2530       scan.addFamily(fam2);
2531       boolean ok = false;
2532       try {
2533         region.getScanner(scan);
2534       } catch (Exception e) {
2535         ok = true;
2536       }
2537       assertTrue("Families could not be found in Region", ok);
2538     } finally {
2539       HRegion.closeHRegion(this.region);
2540       this.region = null;
2541     }
2542   }
2543 
2544   @Test
2545   public void testGetScanner_WithNoFamilies() throws IOException {
2546     byte[] row1 = Bytes.toBytes("row1");
2547     byte[] fam1 = Bytes.toBytes("fam1");
2548     byte[] fam2 = Bytes.toBytes("fam2");
2549     byte[] fam3 = Bytes.toBytes("fam3");
2550     byte[] fam4 = Bytes.toBytes("fam4");
2551 
2552     byte[][] families = { fam1, fam2, fam3, fam4 };
2553 
2554     // Setting up region
2555     String method = this.getName();
2556     this.region = initHRegion(tableName, method, CONF, families);
2557     try {
2558 
2559       // Putting data in Region
2560       Put put = new Put(row1);
2561       put.add(fam1, null, null);
2562       put.add(fam2, null, null);
2563       put.add(fam3, null, null);
2564       put.add(fam4, null, null);
2565       region.put(put);
2566 
2567       Scan scan = null;
2568       HRegion.RegionScannerImpl is = null;
2569 
2570       // Testing to see how many scanners that is produced by getScanner,
2571       // starting
2572       // with known number, 2 - current = 1
2573       scan = new Scan();
2574       scan.addFamily(fam2);
2575       scan.addFamily(fam4);
2576       is = (RegionScannerImpl) region.getScanner(scan);
2577       assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2578 
2579       scan = new Scan();
2580       is = (RegionScannerImpl) region.getScanner(scan);
2581       assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2582     } finally {
2583       HRegion.closeHRegion(this.region);
2584       this.region = null;
2585     }
2586   }
2587 
2588   /**
2589    * This method tests https://issues.apache.org/jira/browse/HBASE-2516.
2590    *
2591    * @throws IOException
2592    */
2593   @Test
2594   public void testGetScanner_WithRegionClosed() throws IOException {
2595     byte[] fam1 = Bytes.toBytes("fam1");
2596     byte[] fam2 = Bytes.toBytes("fam2");
2597 
2598     byte[][] families = { fam1, fam2 };
2599 
2600     // Setting up region
2601     String method = this.getName();
2602     try {
2603       this.region = initHRegion(tableName, method, CONF, families);
2604     } catch (IOException e) {
2605       e.printStackTrace();
2606       fail("Got IOException during initHRegion, " + e.getMessage());
2607     }
2608     try {
2609       region.closed.set(true);
2610       try {
2611         region.getScanner(null);
2612         fail("Expected to get an exception during getScanner on a region that is closed");
2613       } catch (NotServingRegionException e) {
2614         // this is the correct exception that is expected
2615       } catch (IOException e) {
2616         fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2617             + e.getMessage());
2618       }
2619     } finally {
2620       HRegion.closeHRegion(this.region);
2621       this.region = null;
2622     }
2623   }
2624 
2625   @Test
2626   public void testRegionScanner_Next() throws IOException {
2627     byte[] row1 = Bytes.toBytes("row1");
2628     byte[] row2 = Bytes.toBytes("row2");
2629     byte[] fam1 = Bytes.toBytes("fam1");
2630     byte[] fam2 = Bytes.toBytes("fam2");
2631     byte[] fam3 = Bytes.toBytes("fam3");
2632     byte[] fam4 = Bytes.toBytes("fam4");
2633 
2634     byte[][] families = { fam1, fam2, fam3, fam4 };
2635     long ts = System.currentTimeMillis();
2636 
2637     // Setting up region
2638     String method = this.getName();
2639     this.region = initHRegion(tableName, method, CONF, families);
2640     try {
2641       // Putting data in Region
2642       Put put = null;
2643       put = new Put(row1);
2644       put.add(fam1, (byte[]) null, ts, null);
2645       put.add(fam2, (byte[]) null, ts, null);
2646       put.add(fam3, (byte[]) null, ts, null);
2647       put.add(fam4, (byte[]) null, ts, null);
2648       region.put(put);
2649 
2650       put = new Put(row2);
2651       put.add(fam1, (byte[]) null, ts, null);
2652       put.add(fam2, (byte[]) null, ts, null);
2653       put.add(fam3, (byte[]) null, ts, null);
2654       put.add(fam4, (byte[]) null, ts, null);
2655       region.put(put);
2656 
2657       Scan scan = new Scan();
2658       scan.addFamily(fam2);
2659       scan.addFamily(fam4);
2660       InternalScanner is = region.getScanner(scan);
2661 
2662       List<Cell> res = null;
2663 
2664       // Result 1
2665       List<Cell> expected1 = new ArrayList<Cell>();
2666       expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2667       expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2668 
2669       res = new ArrayList<Cell>();
2670       is.next(res);
2671       for (int i = 0; i < res.size(); i++) {
2672         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2673       }
2674 
2675       // Result 2
2676       List<Cell> expected2 = new ArrayList<Cell>();
2677       expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2678       expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2679 
2680       res = new ArrayList<Cell>();
2681       is.next(res);
2682       for (int i = 0; i < res.size(); i++) {
2683         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2684       }
2685     } finally {
2686       HRegion.closeHRegion(this.region);
2687       this.region = null;
2688     }
2689   }
2690 
2691   @Test
2692   public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2693     byte[] row1 = Bytes.toBytes("row1");
2694     byte[] qf1 = Bytes.toBytes("qualifier1");
2695     byte[] qf2 = Bytes.toBytes("qualifier2");
2696     byte[] fam1 = Bytes.toBytes("fam1");
2697     byte[][] families = { fam1 };
2698 
2699     long ts1 = System.currentTimeMillis();
2700     long ts2 = ts1 + 1;
2701     long ts3 = ts1 + 2;
2702 
2703     // Setting up region
2704     String method = this.getName();
2705     this.region = initHRegion(tableName, method, CONF, families);
2706     try {
2707       // Putting data in Region
2708       Put put = null;
2709       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2710       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2711       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2712 
2713       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2714       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2715       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2716 
2717       put = new Put(row1);
2718       put.add(kv13);
2719       put.add(kv12);
2720       put.add(kv11);
2721       put.add(kv23);
2722       put.add(kv22);
2723       put.add(kv21);
2724       region.put(put);
2725 
2726       // Expected
2727       List<Cell> expected = new ArrayList<Cell>();
2728       expected.add(kv13);
2729       expected.add(kv12);
2730 
2731       Scan scan = new Scan(row1);
2732       scan.addColumn(fam1, qf1);
2733       scan.setMaxVersions(MAX_VERSIONS);
2734       List<Cell> actual = new ArrayList<Cell>();
2735       InternalScanner scanner = region.getScanner(scan);
2736 
2737       boolean hasNext = scanner.next(actual);
2738       assertEquals(false, hasNext);
2739 
2740       // Verify result
2741       for (int i = 0; i < expected.size(); i++) {
2742         assertEquals(expected.get(i), actual.get(i));
2743       }
2744     } finally {
2745       HRegion.closeHRegion(this.region);
2746       this.region = null;
2747     }
2748   }
2749 
2750   @Test
2751   public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2752     byte[] row1 = Bytes.toBytes("row1");
2753     byte[] qf1 = Bytes.toBytes("qualifier1");
2754     byte[] qf2 = Bytes.toBytes("qualifier2");
2755     byte[] fam1 = Bytes.toBytes("fam1");
2756     byte[][] families = { fam1 };
2757 
2758     long ts1 = 1; // System.currentTimeMillis();
2759     long ts2 = ts1 + 1;
2760     long ts3 = ts1 + 2;
2761 
2762     // Setting up region
2763     String method = this.getName();
2764     this.region = initHRegion(tableName, method, CONF, families);
2765     try {
2766       // Putting data in Region
2767       Put put = null;
2768       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2769       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2770       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2771 
2772       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2773       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2774       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2775 
2776       put = new Put(row1);
2777       put.add(kv13);
2778       put.add(kv12);
2779       put.add(kv11);
2780       put.add(kv23);
2781       put.add(kv22);
2782       put.add(kv21);
2783       region.put(put);
2784       region.flushcache();
2785 
2786       // Expected
2787       List<Cell> expected = new ArrayList<Cell>();
2788       expected.add(kv13);
2789       expected.add(kv12);
2790       expected.add(kv23);
2791       expected.add(kv22);
2792 
2793       Scan scan = new Scan(row1);
2794       scan.addColumn(fam1, qf1);
2795       scan.addColumn(fam1, qf2);
2796       scan.setMaxVersions(MAX_VERSIONS);
2797       List<Cell> actual = new ArrayList<Cell>();
2798       InternalScanner scanner = region.getScanner(scan);
2799 
2800       boolean hasNext = scanner.next(actual);
2801       assertEquals(false, hasNext);
2802 
2803       // Verify result
2804       for (int i = 0; i < expected.size(); i++) {
2805         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2806       }
2807     } finally {
2808       HRegion.closeHRegion(this.region);
2809       this.region = null;
2810     }
2811   }
2812 
2813   @Test
2814   public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2815     byte[] row1 = Bytes.toBytes("row1");
2816     byte[] fam1 = Bytes.toBytes("fam1");
2817     byte[][] families = { fam1 };
2818     byte[] qf1 = Bytes.toBytes("qualifier1");
2819     byte[] qf2 = Bytes.toBytes("qualifier2");
2820 
2821     long ts1 = 1;
2822     long ts2 = ts1 + 1;
2823     long ts3 = ts1 + 2;
2824     long ts4 = ts1 + 3;
2825 
2826     // Setting up region
2827     String method = this.getName();
2828     this.region = initHRegion(tableName, method, CONF, families);
2829     try {
2830       // Putting data in Region
2831       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2832       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2833       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2834       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2835 
2836       KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2837       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2838       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2839       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2840 
2841       Put put = null;
2842       put = new Put(row1);
2843       put.add(kv14);
2844       put.add(kv24);
2845       region.put(put);
2846       region.flushcache();
2847 
2848       put = new Put(row1);
2849       put.add(kv23);
2850       put.add(kv13);
2851       region.put(put);
2852       region.flushcache();
2853 
2854       put = new Put(row1);
2855       put.add(kv22);
2856       put.add(kv12);
2857       region.put(put);
2858       region.flushcache();
2859 
2860       put = new Put(row1);
2861       put.add(kv21);
2862       put.add(kv11);
2863       region.put(put);
2864 
2865       // Expected
2866       List<Cell> expected = new ArrayList<Cell>();
2867       expected.add(kv14);
2868       expected.add(kv13);
2869       expected.add(kv12);
2870       expected.add(kv24);
2871       expected.add(kv23);
2872       expected.add(kv22);
2873 
2874       Scan scan = new Scan(row1);
2875       scan.addColumn(fam1, qf1);
2876       scan.addColumn(fam1, qf2);
2877       int versions = 3;
2878       scan.setMaxVersions(versions);
2879       List<Cell> actual = new ArrayList<Cell>();
2880       InternalScanner scanner = region.getScanner(scan);
2881 
2882       boolean hasNext = scanner.next(actual);
2883       assertEquals(false, hasNext);
2884 
2885       // Verify result
2886       for (int i = 0; i < expected.size(); i++) {
2887         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2888       }
2889     } finally {
2890       HRegion.closeHRegion(this.region);
2891       this.region = null;
2892     }
2893   }
2894 
2895   @Test
2896   public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2897     byte[] row1 = Bytes.toBytes("row1");
2898     byte[] qf1 = Bytes.toBytes("qualifier1");
2899     byte[] qf2 = Bytes.toBytes("qualifier2");
2900     byte[] fam1 = Bytes.toBytes("fam1");
2901     byte[][] families = { fam1 };
2902 
2903     long ts1 = System.currentTimeMillis();
2904     long ts2 = ts1 + 1;
2905     long ts3 = ts1 + 2;
2906 
2907     // Setting up region
2908     String method = this.getName();
2909     this.region = initHRegion(tableName, method, CONF, families);
2910     try {
2911       // Putting data in Region
2912       Put put = null;
2913       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2914       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2915       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2916 
2917       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2918       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2919       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2920 
2921       put = new Put(row1);
2922       put.add(kv13);
2923       put.add(kv12);
2924       put.add(kv11);
2925       put.add(kv23);
2926       put.add(kv22);
2927       put.add(kv21);
2928       region.put(put);
2929 
2930       // Expected
2931       List<Cell> expected = new ArrayList<Cell>();
2932       expected.add(kv13);
2933       expected.add(kv12);
2934       expected.add(kv23);
2935       expected.add(kv22);
2936 
2937       Scan scan = new Scan(row1);
2938       scan.addFamily(fam1);
2939       scan.setMaxVersions(MAX_VERSIONS);
2940       List<Cell> actual = new ArrayList<Cell>();
2941       InternalScanner scanner = region.getScanner(scan);
2942 
2943       boolean hasNext = scanner.next(actual);
2944       assertEquals(false, hasNext);
2945 
2946       // Verify result
2947       for (int i = 0; i < expected.size(); i++) {
2948         assertEquals(expected.get(i), actual.get(i));
2949       }
2950     } finally {
2951       HRegion.closeHRegion(this.region);
2952       this.region = null;
2953     }
2954   }
2955 
2956   @Test
2957   public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
2958     byte[] row1 = Bytes.toBytes("row1");
2959     byte[] qf1 = Bytes.toBytes("qualifier1");
2960     byte[] qf2 = Bytes.toBytes("qualifier2");
2961     byte[] fam1 = Bytes.toBytes("fam1");
2962 
2963     long ts1 = 1; // System.currentTimeMillis();
2964     long ts2 = ts1 + 1;
2965     long ts3 = ts1 + 2;
2966 
2967     // Setting up region
2968     String method = this.getName();
2969     this.region = initHRegion(tableName, method, CONF, fam1);
2970     try {
2971       // Putting data in Region
2972       Put put = null;
2973       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2974       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2975       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2976 
2977       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2978       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2979       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2980 
2981       put = new Put(row1);
2982       put.add(kv13);
2983       put.add(kv12);
2984       put.add(kv11);
2985       put.add(kv23);
2986       put.add(kv22);
2987       put.add(kv21);
2988       region.put(put);
2989       region.flushcache();
2990 
2991       // Expected
2992       List<Cell> expected = new ArrayList<Cell>();
2993       expected.add(kv13);
2994       expected.add(kv12);
2995       expected.add(kv23);
2996       expected.add(kv22);
2997 
2998       Scan scan = new Scan(row1);
2999       scan.addFamily(fam1);
3000       scan.setMaxVersions(MAX_VERSIONS);
3001       List<Cell> actual = new ArrayList<Cell>();
3002       InternalScanner scanner = region.getScanner(scan);
3003 
3004       boolean hasNext = scanner.next(actual);
3005       assertEquals(false, hasNext);
3006 
3007       // Verify result
3008       for (int i = 0; i < expected.size(); i++) {
3009         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
3010       }
3011     } finally {
3012       HRegion.closeHRegion(this.region);
3013       this.region = null;
3014     }
3015   }
3016 
3017   @Test
3018   public void testScanner_StopRow1542() throws IOException {
3019     byte[] family = Bytes.toBytes("testFamily");
3020     this.region = initHRegion(tableName, getName(), CONF, family);
3021     try {
3022       byte[] row1 = Bytes.toBytes("row111");
3023       byte[] row2 = Bytes.toBytes("row222");
3024       byte[] row3 = Bytes.toBytes("row333");
3025       byte[] row4 = Bytes.toBytes("row444");
3026       byte[] row5 = Bytes.toBytes("row555");
3027 
3028       byte[] col1 = Bytes.toBytes("Pub111");
3029       byte[] col2 = Bytes.toBytes("Pub222");
3030 
3031       Put put = new Put(row1);
3032       put.add(family, col1, Bytes.toBytes(10L));
3033       region.put(put);
3034 
3035       put = new Put(row2);
3036       put.add(family, col1, Bytes.toBytes(15L));
3037       region.put(put);
3038 
3039       put = new Put(row3);
3040       put.add(family, col2, Bytes.toBytes(20L));
3041       region.put(put);
3042 
3043       put = new Put(row4);
3044       put.add(family, col2, Bytes.toBytes(30L));
3045       region.put(put);
3046 
3047       put = new Put(row5);
3048       put.add(family, col1, Bytes.toBytes(40L));
3049       region.put(put);
3050 
3051       Scan scan = new Scan(row3, row4);
3052       scan.setMaxVersions();
3053       scan.addColumn(family, col1);
3054       InternalScanner s = region.getScanner(scan);
3055 
3056       List<Cell> results = new ArrayList<Cell>();
3057       assertEquals(false, s.next(results));
3058       assertEquals(0, results.size());
3059     } finally {
3060       HRegion.closeHRegion(this.region);
3061       this.region = null;
3062     }
3063   }
3064 
3065   @Test
3066   public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
3067     byte[] row1 = Bytes.toBytes("row1");
3068     byte[] fam1 = Bytes.toBytes("fam1");
3069     byte[] qf1 = Bytes.toBytes("qualifier1");
3070     byte[] qf2 = Bytes.toBytes("quateslifier2");
3071 
3072     long ts1 = 1;
3073     long ts2 = ts1 + 1;
3074     long ts3 = ts1 + 2;
3075     long ts4 = ts1 + 3;
3076 
3077     // Setting up region
3078     String method = this.getName();
3079     this.region = initHRegion(tableName, method, CONF, fam1);
3080     try {
3081       // Putting data in Region
3082       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
3083       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
3084       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
3085       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
3086 
3087       KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
3088       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
3089       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
3090       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
3091 
3092       Put put = null;
3093       put = new Put(row1);
3094       put.add(kv14);
3095       put.add(kv24);
3096       region.put(put);
3097       region.flushcache();
3098 
3099       put = new Put(row1);
3100       put.add(kv23);
3101       put.add(kv13);
3102       region.put(put);
3103       region.flushcache();
3104 
3105       put = new Put(row1);
3106       put.add(kv22);
3107       put.add(kv12);
3108       region.put(put);
3109       region.flushcache();
3110 
3111       put = new Put(row1);
3112       put.add(kv21);
3113       put.add(kv11);
3114       region.put(put);
3115 
3116       // Expected
3117       List<KeyValue> expected = new ArrayList<KeyValue>();
3118       expected.add(kv14);
3119       expected.add(kv13);
3120       expected.add(kv12);
3121       expected.add(kv24);
3122       expected.add(kv23);
3123       expected.add(kv22);
3124 
3125       Scan scan = new Scan(row1);
3126       int versions = 3;
3127       scan.setMaxVersions(versions);
3128       List<Cell> actual = new ArrayList<Cell>();
3129       InternalScanner scanner = region.getScanner(scan);
3130 
3131       boolean hasNext = scanner.next(actual);
3132       assertEquals(false, hasNext);
3133 
3134       // Verify result
3135       for (int i = 0; i < expected.size(); i++) {
3136         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
3137       }
3138     } finally {
3139       HRegion.closeHRegion(this.region);
3140       this.region = null;
3141     }
3142   }
3143 
3144   /**
3145    * Added for HBASE-5416
3146    *
3147    * Here we test scan optimization when only subset of CFs are used in filter
3148    * conditions.
3149    */
3150   @Test
3151   public void testScanner_JoinedScanners() throws IOException {
3152     byte[] cf_essential = Bytes.toBytes("essential");
3153     byte[] cf_joined = Bytes.toBytes("joined");
3154     byte[] cf_alpha = Bytes.toBytes("alpha");
3155     this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
3156     try {
3157       byte[] row1 = Bytes.toBytes("row1");
3158       byte[] row2 = Bytes.toBytes("row2");
3159       byte[] row3 = Bytes.toBytes("row3");
3160 
3161       byte[] col_normal = Bytes.toBytes("d");
3162       byte[] col_alpha = Bytes.toBytes("a");
3163 
3164       byte[] filtered_val = Bytes.toBytes(3);
3165 
3166       Put put = new Put(row1);
3167       put.add(cf_essential, col_normal, Bytes.toBytes(1));
3168       put.add(cf_joined, col_alpha, Bytes.toBytes(1));
3169       region.put(put);
3170 
3171       put = new Put(row2);
3172       put.add(cf_essential, col_alpha, Bytes.toBytes(2));
3173       put.add(cf_joined, col_normal, Bytes.toBytes(2));
3174       put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
3175       region.put(put);
3176 
3177       put = new Put(row3);
3178       put.add(cf_essential, col_normal, filtered_val);
3179       put.add(cf_joined, col_normal, filtered_val);
3180       region.put(put);
3181 
3182       // Check two things:
3183       // 1. result list contains expected values
3184       // 2. result list is sorted properly
3185 
3186       Scan scan = new Scan();
3187       Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
3188           CompareOp.NOT_EQUAL, filtered_val);
3189       scan.setFilter(filter);
3190       scan.setLoadColumnFamiliesOnDemand(true);
3191       InternalScanner s = region.getScanner(scan);
3192 
3193       List<Cell> results = new ArrayList<Cell>();
3194       assertTrue(s.next(results));
3195       assertEquals(results.size(), 1);
3196       results.clear();
3197 
3198       assertTrue(s.next(results));
3199       assertEquals(results.size(), 3);
3200       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
3201       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
3202       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
3203       results.clear();
3204 
3205       assertFalse(s.next(results));
3206       assertEquals(results.size(), 0);
3207     } finally {
3208       HRegion.closeHRegion(this.region);
3209       this.region = null;
3210     }
3211   }
3212 
3213   /**
3214    * HBASE-5416
3215    *
3216    * Test case when scan limits amount of KVs returned on each next() call.
3217    */
3218   @Test
3219   public void testScanner_JoinedScannersWithLimits() throws IOException {
3220     final byte[] cf_first = Bytes.toBytes("first");
3221     final byte[] cf_second = Bytes.toBytes("second");
3222 
3223     this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
3224     try {
3225       final byte[] col_a = Bytes.toBytes("a");
3226       final byte[] col_b = Bytes.toBytes("b");
3227 
3228       Put put;
3229 
3230       for (int i = 0; i < 10; i++) {
3231         put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
3232         put.add(cf_first, col_a, Bytes.toBytes(i));
3233         if (i < 5) {
3234           put.add(cf_first, col_b, Bytes.toBytes(i));
3235           put.add(cf_second, col_a, Bytes.toBytes(i));
3236           put.add(cf_second, col_b, Bytes.toBytes(i));
3237         }
3238         region.put(put);
3239       }
3240 
3241       Scan scan = new Scan();
3242       scan.setLoadColumnFamiliesOnDemand(true);
3243       Filter bogusFilter = new FilterBase() {
3244         @Override
3245         public ReturnCode filterKeyValue(Cell ignored) throws IOException {
3246           return ReturnCode.INCLUDE;
3247         }
3248         @Override
3249         public boolean isFamilyEssential(byte[] name) {
3250           return Bytes.equals(name, cf_first);
3251         }
3252       };
3253 
3254       scan.setFilter(bogusFilter);
3255       InternalScanner s = region.getScanner(scan);
3256 
3257       // Our data looks like this:
3258       // r0: first:a, first:b, second:a, second:b
3259       // r1: first:a, first:b, second:a, second:b
3260       // r2: first:a, first:b, second:a, second:b
3261       // r3: first:a, first:b, second:a, second:b
3262       // r4: first:a, first:b, second:a, second:b
3263       // r5: first:a
3264       // r6: first:a
3265       // r7: first:a
3266       // r8: first:a
3267       // r9: first:a
3268 
3269       // But due to next's limit set to 3, we should get this:
3270       // r0: first:a, first:b, second:a
3271       // r0: second:b
3272       // r1: first:a, first:b, second:a
3273       // r1: second:b
3274       // r2: first:a, first:b, second:a
3275       // r2: second:b
3276       // r3: first:a, first:b, second:a
3277       // r3: second:b
3278       // r4: first:a, first:b, second:a
3279       // r4: second:b
3280       // r5: first:a
3281       // r6: first:a
3282       // r7: first:a
3283       // r8: first:a
3284       // r9: first:a
3285 
3286       List<Cell> results = new ArrayList<Cell>();
3287       int index = 0;
3288       while (true) {
3289         boolean more = s.next(results, 3);
3290         if ((index >> 1) < 5) {
3291           if (index % 2 == 0)
3292             assertEquals(results.size(), 3);
3293           else
3294             assertEquals(results.size(), 1);
3295         } else
3296           assertEquals(results.size(), 1);
3297         results.clear();
3298         index++;
3299         if (!more)
3300           break;
3301       }
3302     } finally {
3303       HRegion.closeHRegion(this.region);
3304       this.region = null;
3305     }
3306   }
3307 
3308   // ////////////////////////////////////////////////////////////////////////////
3309   // Split test
3310   // ////////////////////////////////////////////////////////////////////////////
3311   /**
3312    * Splits twice and verifies getting from each of the split regions.
3313    *
3314    * @throws Exception
3315    */
3316   @Test
3317   public void testBasicSplit() throws Exception {
3318     byte[][] families = { fam1, fam2, fam3 };
3319 
3320     Configuration hc = initSplit();
3321     // Setting up region
3322     String method = this.getName();
3323     this.region = initHRegion(tableName, method, hc, families);
3324 
3325     try {
3326       LOG.info("" + HBaseTestCase.addContent(region, fam3));
3327       region.flushcache();
3328       region.compactStores();
3329       byte[] splitRow = region.checkSplit();
3330       assertNotNull(splitRow);
3331       LOG.info("SplitRow: " + Bytes.toString(splitRow));
3332       HRegion[] regions = splitRegion(region, splitRow);
3333       try {
3334         // Need to open the regions.
3335         // TODO: Add an 'open' to HRegion... don't do open by constructing
3336         // instance.
3337         for (int i = 0; i < regions.length; i++) {
3338           regions[i] = HRegion.openHRegion(regions[i], null);
3339         }
3340         // Assert can get rows out of new regions. Should be able to get first
3341         // row from first region and the midkey from second region.
3342         assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
3343         assertGet(regions[1], fam3, splitRow);
3344         // Test I can get scanner and that it starts at right place.
3345         assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
3346         assertScan(regions[1], fam3, splitRow);
3347         // Now prove can't split regions that have references.
3348         for (int i = 0; i < regions.length; i++) {
3349           // Add so much data to this region, we create a store file that is >
3350           // than one of our unsplitable references. it will.
3351           for (int j = 0; j < 2; j++) {
3352             HBaseTestCase.addContent(regions[i], fam3);
3353           }
3354           HBaseTestCase.addContent(regions[i], fam2);
3355           HBaseTestCase.addContent(regions[i], fam1);
3356           regions[i].flushcache();
3357         }
3358 
3359         byte[][] midkeys = new byte[regions.length][];
3360         // To make regions splitable force compaction.
3361         for (int i = 0; i < regions.length; i++) {
3362           regions[i].compactStores();
3363           midkeys[i] = regions[i].checkSplit();
3364         }
3365 
3366         TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
3367         // Split these two daughter regions so then I'll have 4 regions. Will
3368         // split because added data above.
3369         for (int i = 0; i < regions.length; i++) {
3370           HRegion[] rs = null;
3371           if (midkeys[i] != null) {
3372             rs = splitRegion(regions[i], midkeys[i]);
3373             for (int j = 0; j < rs.length; j++) {
3374               sortedMap.put(Bytes.toString(rs[j].getRegionName()), HRegion.openHRegion(rs[j], null));
3375             }
3376           }
3377         }
3378         LOG.info("Made 4 regions");
3379         // The splits should have been even. Test I can get some arbitrary row
3380         // out of each.
3381         int interval = (LAST_CHAR - FIRST_CHAR) / 3;
3382         byte[] b = Bytes.toBytes(START_KEY);
3383         for (HRegion r : sortedMap.values()) {
3384           assertGet(r, fam3, b);
3385           b[0] += interval;
3386         }
3387       } finally {
3388         for (int i = 0; i < regions.length; i++) {
3389           try {
3390             regions[i].close();
3391           } catch (IOException e) {
3392             // Ignore.
3393           }
3394         }
3395       }
3396     } finally {
3397       HRegion.closeHRegion(this.region);
3398       this.region = null;
3399     }
3400   }
3401 
3402   @Test
3403   public void testSplitRegion() throws IOException {
3404     byte[] qualifier = Bytes.toBytes("qualifier");
3405     Configuration hc = initSplit();
3406     int numRows = 10;
3407     byte[][] families = { fam1, fam3 };
3408 
3409     // Setting up region
3410     String method = this.getName();
3411     this.region = initHRegion(tableName, method, hc, families);
3412 
3413     // Put data in region
3414     int startRow = 100;
3415     putData(startRow, numRows, qualifier, families);
3416     int splitRow = startRow + numRows;
3417     putData(splitRow, numRows, qualifier, families);
3418     region.flushcache();
3419 
3420     HRegion[] regions = null;
3421     try {
3422       regions = splitRegion(region, Bytes.toBytes("" + splitRow));
3423       // Opening the regions returned.
3424       for (int i = 0; i < regions.length; i++) {
3425         regions[i] = HRegion.openHRegion(regions[i], null);
3426       }
3427       // Verifying that the region has been split
3428       assertEquals(2, regions.length);
3429 
3430       // Verifying that all data is still there and that data is in the right
3431       // place
3432       verifyData(regions[0], startRow, numRows, qualifier, families);
3433       verifyData(regions[1], splitRow, numRows, qualifier, families);
3434 
3435     } finally {
3436       HRegion.closeHRegion(this.region);
3437       this.region = null;
3438     }
3439   }
3440 
3441   @Test
3442   public void testClearForceSplit() throws IOException {
3443     byte[] qualifier = Bytes.toBytes("qualifier");
3444     Configuration hc = initSplit();
3445     int numRows = 10;
3446     byte[][] families = { fam1, fam3 };
3447 
3448     // Setting up region
3449     String method = this.getName();
3450     this.region = initHRegion(tableName, method, hc, families);
3451 
3452     // Put data in region
3453     int startRow = 100;
3454     putData(startRow, numRows, qualifier, families);
3455     int splitRow = startRow + numRows;
3456     byte[] splitRowBytes = Bytes.toBytes("" + splitRow);
3457     putData(splitRow, numRows, qualifier, families);
3458     region.flushcache();
3459 
3460     HRegion[] regions = null;
3461     try {
3462       // Set force split
3463       region.forceSplit(splitRowBytes);
3464       assertTrue(region.shouldForceSplit());
3465       // Split point should be the force split row
3466       assertTrue(Bytes.equals(splitRowBytes, region.checkSplit()));
3467 
3468       // Add a store that has references.
3469       HStore storeMock = Mockito.mock(HStore.class);
3470       when(storeMock.hasReferences()).thenReturn(true);
3471       when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
3472       when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
3473       when(storeMock.getColumnFamilyName()).thenReturn("cf");
3474       region.stores.put(Bytes.toBytes(storeMock.getColumnFamilyName()), storeMock);
3475       assertTrue(region.hasReferences());
3476 
3477       // Will not split since the store has references.
3478       regions = splitRegion(region, splitRowBytes);
3479       assertNull(regions);
3480 
3481       // Region force split should be cleared after the split try.
3482       assertFalse(region.shouldForceSplit());
3483 
3484       // Remove the store that has references.
3485       region.stores.remove(Bytes.toBytes(storeMock.getColumnFamilyName()));
3486       assertFalse(region.hasReferences());
3487 
3488       // Now we can split.
3489       regions = splitRegion(region, splitRowBytes);
3490 
3491       // Opening the regions returned.
3492       for (int i = 0; i < regions.length; i++) {
3493         regions[i] = HRegion.openHRegion(regions[i], null);
3494       }
3495       // Verifying that the region has been split
3496       assertEquals(2, regions.length);
3497 
3498       // Verifying that all data is still there and that data is in the right
3499       // place
3500       verifyData(regions[0], startRow, numRows, qualifier, families);
3501       verifyData(regions[1], splitRow, numRows, qualifier, families);
3502 
3503     } finally {
3504       HRegion.closeHRegion(this.region);
3505       this.region = null;
3506     }
3507   }
3508 
3509   /**
3510    * Flushes the cache in a thread while scanning. The tests verify that the
3511    * scan is coherent - e.g. the returned results are always of the same or
3512    * later update as the previous results.
3513    *
3514    * @throws IOException
3515    *           scan / compact
3516    * @throws InterruptedException
3517    *           thread join
3518    */
3519   @Test
3520   public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3521     byte[] family = Bytes.toBytes("family");
3522     int numRows = 1000;
3523     int flushAndScanInterval = 10;
3524     int compactInterval = 10 * flushAndScanInterval;
3525 
3526     String method = "testFlushCacheWhileScanning";
3527     this.region = initHRegion(tableName, method, CONF, family);
3528     try {
3529       FlushThread flushThread = new FlushThread();
3530       flushThread.start();
3531 
3532       Scan scan = new Scan();
3533       scan.addFamily(family);
3534       scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
3535           new BinaryComparator(Bytes.toBytes(5L))));
3536 
3537       int expectedCount = 0;
3538       List<Cell> res = new ArrayList<Cell>();
3539 
3540       boolean toggle = true;
3541       for (long i = 0; i < numRows; i++) {
3542         Put put = new Put(Bytes.toBytes(i));
3543         put.setDurability(Durability.SKIP_WAL);
3544         put.add(family, qual1, Bytes.toBytes(i % 10));
3545         region.put(put);
3546 
3547         if (i != 0 && i % compactInterval == 0) {
3548           // System.out.println("iteration = " + i);
3549           region.compactStores(true);
3550         }
3551 
3552         if (i % 10 == 5L) {
3553           expectedCount++;
3554         }
3555 
3556         if (i != 0 && i % flushAndScanInterval == 0) {
3557           res.clear();
3558           InternalScanner scanner = region.getScanner(scan);
3559           if (toggle) {
3560             flushThread.flush();
3561           }
3562           while (scanner.next(res))
3563             ;
3564           if (!toggle) {
3565             flushThread.flush();
3566           }
3567           assertEquals("i=" + i, expectedCount, res.size());
3568           toggle = !toggle;
3569         }
3570       }
3571 
3572       flushThread.done();
3573       flushThread.join();
3574       flushThread.checkNoError();
3575     } finally {
3576       HRegion.closeHRegion(this.region);
3577       this.region = null;
3578     }
3579   }
3580 
3581   protected class FlushThread extends Thread {
3582     private volatile boolean done;
3583     private Throwable error = null;
3584 
3585     public void done() {
3586       done = true;
3587       synchronized (this) {
3588         interrupt();
3589       }
3590     }
3591 
3592     public void checkNoError() {
3593       if (error != null) {
3594         assertNull(error);
3595       }
3596     }
3597 
3598     @Override
3599     public void run() {
3600       done = false;
3601       while (!done) {
3602         synchronized (this) {
3603           try {
3604             wait();
3605           } catch (InterruptedException ignored) {
3606             if (done) {
3607               break;
3608             }
3609           }
3610         }
3611         try {
3612           region.flushcache();
3613         } catch (IOException e) {
3614           if (!done) {
3615             LOG.error("Error while flusing cache", e);
3616             error = e;
3617           }
3618           break;
3619         }
3620       }
3621 
3622     }
3623 
3624     public void flush() {
3625       synchronized (this) {
3626         notify();
3627       }
3628 
3629     }
3630   }
3631 
3632   /**
3633    * Writes very wide records and scans for the latest every time.. Flushes and
3634    * compacts the region every now and then to keep things realistic.
3635    *
3636    * @throws IOException
3637    *           by flush / scan / compaction
3638    * @throws InterruptedException
3639    *           when joining threads
3640    */
3641   @Test
3642   public void testWritesWhileScanning() throws IOException, InterruptedException {
3643     int testCount = 100;
3644     int numRows = 1;
3645     int numFamilies = 10;
3646     int numQualifiers = 100;
3647     int flushInterval = 7;
3648     int compactInterval = 5 * flushInterval;
3649     byte[][] families = new byte[numFamilies][];
3650     for (int i = 0; i < numFamilies; i++) {
3651       families[i] = Bytes.toBytes("family" + i);
3652     }
3653     byte[][] qualifiers = new byte[numQualifiers][];
3654     for (int i = 0; i < numQualifiers; i++) {
3655       qualifiers[i] = Bytes.toBytes("qual" + i);
3656     }
3657 
3658     String method = "testWritesWhileScanning";
3659     this.region = initHRegion(tableName, method, CONF, families);
3660     try {
3661       PutThread putThread = new PutThread(numRows, families, qualifiers);
3662       putThread.start();
3663       putThread.waitForFirstPut();
3664 
3665       FlushThread flushThread = new FlushThread();
3666       flushThread.start();
3667 
3668       Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3669 
3670       int expectedCount = numFamilies * numQualifiers;
3671       List<Cell> res = new ArrayList<Cell>();
3672 
3673       long prevTimestamp = 0L;
3674       for (int i = 0; i < testCount; i++) {
3675 
3676         if (i != 0 && i % compactInterval == 0) {
3677           region.compactStores(true);
3678         }
3679 
3680         if (i != 0 && i % flushInterval == 0) {
3681           flushThread.flush();
3682         }
3683 
3684         boolean previousEmpty = res.isEmpty();
3685         res.clear();
3686         InternalScanner scanner = region.getScanner(scan);
3687         while (scanner.next(res))
3688           ;
3689         if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3690           assertEquals("i=" + i, expectedCount, res.size());
3691           long timestamp = res.get(0).getTimestamp();
3692           assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3693               timestamp >= prevTimestamp);
3694           prevTimestamp = timestamp;
3695         }
3696       }
3697 
3698       putThread.done();
3699 
3700       region.flushcache();
3701 
3702       putThread.join();
3703       putThread.checkNoError();
3704 
3705       flushThread.done();
3706       flushThread.join();
3707       flushThread.checkNoError();
3708     } finally {
3709       try {
3710         HRegion.closeHRegion(this.region);
3711       } catch (DroppedSnapshotException dse) {
3712         // We could get this on way out because we interrupt the background flusher and it could
3713         // fail anywhere causing a DSE over in the background flusher... only it is not properly
3714         // dealt with so could still be memory hanging out when we get to here -- memory we can't
3715         // flush because the accounting is 'off' since original DSE.
3716       }
3717       this.region = null;
3718     }
3719   }
3720 
3721   protected class PutThread extends Thread {
3722     private volatile boolean done;
3723     private volatile int numPutsFinished = 0;
3724 
3725     private Throwable error = null;
3726     private int numRows;
3727     private byte[][] families;
3728     private byte[][] qualifiers;
3729 
3730     private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3731       this.numRows = numRows;
3732       this.families = families;
3733       this.qualifiers = qualifiers;
3734     }
3735 
3736     /**
3737      * Block until this thread has put at least one row.
3738      */
3739     public void waitForFirstPut() throws InterruptedException {
3740       // wait until put thread actually puts some data
3741       while (numPutsFinished == 0) {
3742         checkNoError();
3743         Thread.sleep(50);
3744       }
3745     }
3746 
3747     public void done() {
3748       done = true;
3749       synchronized (this) {
3750         interrupt();
3751       }
3752     }
3753 
3754     public void checkNoError() {
3755       if (error != null) {
3756         assertNull(error);
3757       }
3758     }
3759 
3760     @Override
3761     public void run() {
3762       done = false;
3763       while (!done) {
3764         try {
3765           for (int r = 0; r < numRows; r++) {
3766             byte[] row = Bytes.toBytes("row" + r);
3767             Put put = new Put(row);
3768             put.setDurability(Durability.SKIP_WAL);
3769             byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3770             for (byte[] family : families) {
3771               for (byte[] qualifier : qualifiers) {
3772                 put.add(family, qualifier, (long) numPutsFinished, value);
3773               }
3774             }
3775             region.put(put);
3776             numPutsFinished++;
3777             if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3778               System.out.println("put iteration = " + numPutsFinished);
3779               Delete delete = new Delete(row, (long) numPutsFinished - 30);
3780               region.delete(delete);
3781             }
3782             numPutsFinished++;
3783           }
3784         } catch (InterruptedIOException e) {
3785           // This is fine. It means we are done, or didn't get the lock on time
3786         } catch (IOException e) {
3787           LOG.error("error while putting records", e);
3788           error = e;
3789           break;
3790         }
3791       }
3792 
3793     }
3794 
3795   }
3796 
3797   /**
3798    * Writes very wide records and gets the latest row every time.. Flushes and
3799    * compacts the region aggressivly to catch issues.
3800    *
3801    * @throws IOException
3802    *           by flush / scan / compaction
3803    * @throws InterruptedException
3804    *           when joining threads
3805    */
3806   @Test
3807   public void testWritesWhileGetting() throws Exception {
3808     int testCount = 100;
3809     int numRows = 1;
3810     int numFamilies = 10;
3811     int numQualifiers = 100;
3812     int compactInterval = 100;
3813     byte[][] families = new byte[numFamilies][];
3814     for (int i = 0; i < numFamilies; i++) {
3815       families[i] = Bytes.toBytes("family" + i);
3816     }
3817     byte[][] qualifiers = new byte[numQualifiers][];
3818     for (int i = 0; i < numQualifiers; i++) {
3819       qualifiers[i] = Bytes.toBytes("qual" + i);
3820     }
3821 
3822 
3823     String method = "testWritesWhileGetting";
3824     // This test flushes constantly and can cause many files to be created,
3825     // possibly
3826     // extending over the ulimit. Make sure compactions are aggressive in
3827     // reducing
3828     // the number of HFiles created.
3829     Configuration conf = HBaseConfiguration.create(CONF);
3830     conf.setInt("hbase.hstore.compaction.min", 1);
3831     conf.setInt("hbase.hstore.compaction.max", 1000);
3832     this.region = initHRegion(tableName, method, conf, families);
3833     PutThread putThread = null;
3834     MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3835     try {
3836       putThread = new PutThread(numRows, families, qualifiers);
3837       putThread.start();
3838       putThread.waitForFirstPut();
3839 
3840       // Add a thread that flushes as fast as possible
3841       ctx.addThread(new RepeatingTestThread(ctx) {
3842         private int flushesSinceCompact = 0;
3843         private final int maxFlushesSinceCompact = 20;
3844 
3845         @Override
3846         public void doAnAction() throws Exception {
3847           if (region.flushcache().isCompactionNeeded()) {
3848             ++flushesSinceCompact;
3849           }
3850           // Compact regularly to avoid creating too many files and exceeding
3851           // the ulimit.
3852           if (flushesSinceCompact == maxFlushesSinceCompact) {
3853             region.compactStores(false);
3854             flushesSinceCompact = 0;
3855           }
3856         }
3857       });
3858       ctx.startThreads();
3859 
3860       Get get = new Get(Bytes.toBytes("row0"));
3861       Result result = null;
3862 
3863       int expectedCount = numFamilies * numQualifiers;
3864 
3865       long prevTimestamp = 0L;
3866       for (int i = 0; i < testCount; i++) {
3867 
3868         boolean previousEmpty = result == null || result.isEmpty();
3869         result = region.get(get);
3870         if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3871           assertEquals("i=" + i, expectedCount, result.size());
3872           // TODO this was removed, now what dangit?!
3873           // search looking for the qualifier in question?
3874           long timestamp = 0;
3875           for (Cell kv : result.rawCells()) {
3876             if (CellUtil.matchingFamily(kv, families[0])
3877                 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3878               timestamp = kv.getTimestamp();
3879             }
3880           }
3881           assertTrue(timestamp >= prevTimestamp);
3882           prevTimestamp = timestamp;
3883           Cell previousKV = null;
3884 
3885           for (Cell kv : result.rawCells()) {
3886             byte[] thisValue = CellUtil.cloneValue(kv);
3887             if (previousKV != null) {
3888               if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3889                 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3890                     + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3891                     + "(memStoreTS:" + kv.getMvccVersion() + ")");
3892                 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3893               }
3894             }
3895             previousKV = kv;
3896           }
3897         }
3898       }
3899     } finally {
3900       if (putThread != null)
3901         putThread.done();
3902 
3903       region.flushcache();
3904 
3905       if (putThread != null) {
3906         putThread.join();
3907         putThread.checkNoError();
3908       }
3909 
3910       ctx.stop();
3911       HRegion.closeHRegion(this.region);
3912       this.region = null;
3913     }
3914   }
3915 
3916   @Test
3917   public void testHolesInMeta() throws Exception {
3918     byte[] family = Bytes.toBytes("family");
3919     this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF,
3920         false, family);
3921     try {
3922       byte[] rowNotServed = Bytes.toBytes("a");
3923       Get g = new Get(rowNotServed);
3924       try {
3925         region.get(g);
3926         fail();
3927       } catch (WrongRegionException x) {
3928         // OK
3929       }
3930       byte[] row = Bytes.toBytes("y");
3931       g = new Get(row);
3932       region.get(g);
3933     } finally {
3934       HRegion.closeHRegion(this.region);
3935       this.region = null;
3936     }
3937   }
3938 
3939   @Test
3940   public void testIndexesScanWithOneDeletedRow() throws IOException {
3941     byte[] family = Bytes.toBytes("family");
3942 
3943     // Setting up region
3944     String method = "testIndexesScanWithOneDeletedRow";
3945     this.region = initHRegion(tableName, method, CONF, family);
3946     try {
3947       Put put = new Put(Bytes.toBytes(1L));
3948       put.add(family, qual1, 1L, Bytes.toBytes(1L));
3949       region.put(put);
3950 
3951       region.flushcache();
3952 
3953       Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3954       region.delete(delete);
3955 
3956       put = new Put(Bytes.toBytes(2L));
3957       put.add(family, qual1, 2L, Bytes.toBytes(2L));
3958       region.put(put);
3959 
3960       Scan idxScan = new Scan();
3961       idxScan.addFamily(family);
3962       idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
3963           new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
3964               new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
3965               CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
3966       InternalScanner scanner = region.getScanner(idxScan);
3967       List<Cell> res = new ArrayList<Cell>();
3968 
3969       while (scanner.next(res))
3970         ;
3971       assertEquals(1L, res.size());
3972     } finally {
3973       HRegion.closeHRegion(this.region);
3974       this.region = null;
3975     }
3976   }
3977 
3978   // ////////////////////////////////////////////////////////////////////////////
3979   // Bloom filter test
3980   // ////////////////////////////////////////////////////////////////////////////
3981   @Test
3982   public void testBloomFilterSize() throws IOException {
3983     byte[] fam1 = Bytes.toBytes("fam1");
3984     byte[] qf1 = Bytes.toBytes("col");
3985     byte[] val1 = Bytes.toBytes("value1");
3986     // Create Table
3987     HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
3988         .setBloomFilterType(BloomType.ROWCOL);
3989 
3990     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3991     htd.addFamily(hcd);
3992     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3993     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3994     try {
3995       int num_unique_rows = 10;
3996       int duplicate_multiplier = 2;
3997       int num_storefiles = 4;
3998 
3999       int version = 0;
4000       for (int f = 0; f < num_storefiles; f++) {
4001         for (int i = 0; i < duplicate_multiplier; i++) {
4002           for (int j = 0; j < num_unique_rows; j++) {
4003             Put put = new Put(Bytes.toBytes("row" + j));
4004             put.setDurability(Durability.SKIP_WAL);
4005             put.add(fam1, qf1, version++, val1);
4006             region.put(put);
4007           }
4008         }
4009         region.flushcache();
4010       }
4011       // before compaction
4012       HStore store = (HStore) region.getStore(fam1);
4013       Collection<StoreFile> storeFiles = store.getStorefiles();
4014       for (StoreFile storefile : storeFiles) {
4015         StoreFile.Reader reader = storefile.getReader();
4016         reader.loadFileInfo();
4017         reader.loadBloomfilter();
4018         assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
4019         assertEquals(num_unique_rows, reader.getFilterEntries());
4020       }
4021 
4022       region.compactStores(true);
4023 
4024       // after compaction
4025       storeFiles = store.getStorefiles();
4026       for (StoreFile storefile : storeFiles) {
4027         StoreFile.Reader reader = storefile.getReader();
4028         reader.loadFileInfo();
4029         reader.loadBloomfilter();
4030         assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
4031         assertEquals(num_unique_rows, reader.getFilterEntries());
4032       }
4033     } finally {
4034       HRegion.closeHRegion(this.region);
4035       this.region = null;
4036     }
4037   }
4038 
4039   @Test
4040   public void testAllColumnsWithBloomFilter() throws IOException {
4041     byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
4042     byte[] FAMILY = Bytes.toBytes("family");
4043 
4044     // Create table
4045     HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
4046         .setBloomFilterType(BloomType.ROWCOL);
4047     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
4048     htd.addFamily(hcd);
4049     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
4050     this.region = TEST_UTIL.createLocalHRegion(info, htd);
4051     try {
4052       // For row:0, col:0: insert versions 1 through 5.
4053       byte row[] = Bytes.toBytes("row:" + 0);
4054       byte column[] = Bytes.toBytes("column:" + 0);
4055       Put put = new Put(row);
4056       put.setDurability(Durability.SKIP_WAL);
4057       for (long idx = 1; idx <= 4; idx++) {
4058         put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
4059       }
4060       region.put(put);
4061 
4062       // Flush
4063       region.flushcache();
4064 
4065       // Get rows
4066       Get get = new Get(row);
4067       get.setMaxVersions();
4068       Cell[] kvs = region.get(get).rawCells();
4069 
4070       // Check if rows are correct
4071       assertEquals(4, kvs.length);
4072       checkOneCell(kvs[0], FAMILY, 0, 0, 4);
4073       checkOneCell(kvs[1], FAMILY, 0, 0, 3);
4074       checkOneCell(kvs[2], FAMILY, 0, 0, 2);
4075       checkOneCell(kvs[3], FAMILY, 0, 0, 1);
4076     } finally {
4077       HRegion.closeHRegion(this.region);
4078       this.region = null;
4079     }
4080   }
4081 
4082   /**
4083    * Testcase to cover bug-fix for HBASE-2823 Ensures correct delete when
4084    * issuing delete row on columns with bloom filter set to row+col
4085    * (BloomType.ROWCOL)
4086    */
4087   @Test
4088   public void testDeleteRowWithBloomFilter() throws IOException {
4089     byte[] familyName = Bytes.toBytes("familyName");
4090 
4091     // Create Table
4092     HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
4093         .setBloomFilterType(BloomType.ROWCOL);
4094 
4095     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
4096     htd.addFamily(hcd);
4097     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
4098     this.region = TEST_UTIL.createLocalHRegion(info, htd);
4099     try {
4100       // Insert some data
4101       byte row[] = Bytes.toBytes("row1");
4102       byte col[] = Bytes.toBytes("col1");
4103 
4104       Put put = new Put(row);
4105       put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
4106       region.put(put);
4107       region.flushcache();
4108 
4109       Delete del = new Delete(row);
4110       region.delete(del);
4111       region.flushcache();
4112 
4113       // Get remaining rows (should have none)
4114       Get get = new Get(row);
4115       get.addColumn(familyName, col);
4116 
4117       Cell[] keyValues = region.get(get).rawCells();
4118       assertTrue(keyValues.length == 0);
4119     } finally {
4120       HRegion.closeHRegion(this.region);
4121       this.region = null;
4122     }
4123   }
4124 
4125   @Test
4126   public void testgetHDFSBlocksDistribution() throws Exception {
4127     HBaseTestingUtility htu = new HBaseTestingUtility();
4128     // Why do we set the block size in this test?  If we set it smaller than the kvs, then we'll
4129     // break up the file in to more pieces that can be distributed across the three nodes and we
4130     // won't be able to have the condition this test asserts; that at least one node has
4131     // a copy of all replicas -- if small block size, then blocks are spread evenly across the
4132     // the three nodes.  hfilev3 with tags seems to put us over the block size.  St.Ack.
4133     // final int DEFAULT_BLOCK_SIZE = 1024;
4134     // htu.getConfiguration().setLong("dfs.blocksize", DEFAULT_BLOCK_SIZE);
4135     htu.getConfiguration().setInt("dfs.replication", 2);
4136 
4137     // set up a cluster with 3 nodes
4138     MiniHBaseCluster cluster = null;
4139     String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
4140     int regionServersCount = 3;
4141 
4142     try {
4143       cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
4144       byte[][] families = { fam1, fam2 };
4145       Table ht = htu.createTable(Bytes.toBytes(this.getName()), families);
4146 
4147       // Setting up region
4148       byte row[] = Bytes.toBytes("row1");
4149       byte col[] = Bytes.toBytes("col1");
4150 
4151       Put put = new Put(row);
4152       put.add(fam1, col, 1, Bytes.toBytes("test1"));
4153       put.add(fam2, col, 1, Bytes.toBytes("test2"));
4154       ht.put(put);
4155 
4156       HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
4157           .get(0);
4158       firstRegion.flushcache();
4159       HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
4160 
4161       // Given the default replication factor is 2 and we have 2 HFiles,
4162       // we will have total of 4 replica of blocks on 3 datanodes; thus there
4163       // must be at least one host that have replica for 2 HFiles. That host's
4164       // weight will be equal to the unique block weight.
4165       long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
4166       StringBuilder sb = new StringBuilder();
4167       for (String host: blocksDistribution1.getTopHosts()) {
4168         if (sb.length() > 0) sb.append(", ");
4169         sb.append(host);
4170         sb.append("=");
4171         sb.append(blocksDistribution1.getWeight(host));
4172       }
4173 
4174       String topHost = blocksDistribution1.getTopHosts().get(0);
4175       long topHostWeight = blocksDistribution1.getWeight(topHost);
4176       String msg = "uniqueBlocksWeight=" + uniqueBlocksWeight1 + ", topHostWeight=" +
4177         topHostWeight + ", topHost=" + topHost + "; " + sb.toString();
4178       LOG.info(msg);
4179       assertTrue(msg, uniqueBlocksWeight1 == topHostWeight);
4180 
4181       // use the static method to compute the value, it should be the same.
4182       // static method is used by load balancer or other components
4183       HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
4184           htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
4185       long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
4186 
4187       assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
4188 
4189       ht.close();
4190     } finally {
4191       if (cluster != null) {
4192         htu.shutdownMiniCluster();
4193       }
4194     }
4195   }
4196 
4197   /**
4198    * Testcase to check state of region initialization task set to ABORTED or not
4199    * if any exceptions during initialization
4200    *
4201    * @throws Exception
4202    */
4203   @Test
4204   public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
4205     TableName tableName = TableName.valueOf(name.getMethodName());
4206     HRegionInfo info = null;
4207     try {
4208       FileSystem fs = Mockito.mock(FileSystem.class);
4209       Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
4210       HTableDescriptor htd = new HTableDescriptor(tableName);
4211       htd.addFamily(new HColumnDescriptor("cf"));
4212       info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
4213           HConstants.EMPTY_BYTE_ARRAY, false);
4214       Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
4215       region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
4216       // region initialization throws IOException and set task state to ABORTED.
4217       region.initialize();
4218       fail("Region initialization should fail due to IOException");
4219     } catch (IOException io) {
4220       List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
4221       for (MonitoredTask monitoredTask : tasks) {
4222         if (!(monitoredTask instanceof MonitoredRPCHandler)
4223             && monitoredTask.getDescription().contains(region.toString())) {
4224           assertTrue("Region state should be ABORTED.",
4225               monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
4226           break;
4227         }
4228       }
4229     } finally {
4230       HRegion.closeHRegion(region);
4231     }
4232   }
4233 
4234   /**
4235    * Verifies that the .regioninfo file is written on region creation and that
4236    * is recreated if missing during region opening.
4237    */
4238   @Test
4239   public void testRegionInfoFileCreation() throws IOException {
4240     Path rootDir = new Path(dir + "testRegionInfoFileCreation");
4241 
4242     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
4243     htd.addFamily(new HColumnDescriptor("cf"));
4244 
4245     HRegionInfo hri = new HRegionInfo(htd.getTableName());
4246 
4247     // Create a region and skip the initialization (like CreateTableHandler)
4248     HRegion region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
4249 //    HRegion region = TEST_UTIL.createLocalHRegion(hri, htd);
4250     Path regionDir = region.getRegionFileSystem().getRegionDir();
4251     FileSystem fs = region.getRegionFileSystem().getFileSystem();
4252     HRegion.closeHRegion(region);
4253 
4254     Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
4255 
4256     // Verify that the .regioninfo file is present
4257     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4258         fs.exists(regionInfoFile));
4259 
4260     // Try to open the region
4261     region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
4262     assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
4263     HRegion.closeHRegion(region);
4264 
4265     // Verify that the .regioninfo file is still there
4266     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4267         fs.exists(regionInfoFile));
4268 
4269     // Remove the .regioninfo file and verify is recreated on region open
4270     fs.delete(regionInfoFile, true);
4271     assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
4272         fs.exists(regionInfoFile));
4273 
4274     region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
4275 //    region = TEST_UTIL.openHRegion(hri, htd);
4276     assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
4277     HRegion.closeHRegion(region);
4278 
4279     // Verify that the .regioninfo file is still there
4280     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4281         fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
4282   }
4283 
4284   /**
4285    * TestCase for increment
4286    */
4287   private static class Incrementer implements Runnable {
4288     private HRegion region;
4289     private final static byte[] incRow = Bytes.toBytes("incRow");
4290     private final static byte[] family = Bytes.toBytes("family");
4291     private final static byte[] qualifier = Bytes.toBytes("qualifier");
4292     private final static long ONE = 1l;
4293     private int incCounter;
4294 
4295     public Incrementer(HRegion region, int incCounter) {
4296       this.region = region;
4297       this.incCounter = incCounter;
4298     }
4299 
4300     @Override
4301     public void run() {
4302       int count = 0;
4303       while (count < incCounter) {
4304         Increment inc = new Increment(incRow);
4305         inc.addColumn(family, qualifier, ONE);
4306         count++;
4307         try {
4308           region.increment(inc);
4309         } catch (IOException e) {
4310           e.printStackTrace();
4311           break;
4312         }
4313       }
4314     }
4315   }
4316 
4317   /**
4318    * Test case to check increment function with memstore flushing
4319    * @throws Exception
4320    */
4321   @Test
4322   public void testParallelIncrementWithMemStoreFlush() throws Exception {
4323     byte[] family = Incrementer.family;
4324     this.region = initHRegion(tableName, method, CONF, family);
4325     final HRegion region = this.region;
4326     final AtomicBoolean incrementDone = new AtomicBoolean(false);
4327     Runnable flusher = new Runnable() {
4328       @Override
4329       public void run() {
4330         while (!incrementDone.get()) {
4331           try {
4332             region.flushcache();
4333           } catch (Exception e) {
4334             e.printStackTrace();
4335           }
4336         }
4337       }
4338     };
4339 
4340     // after all increment finished, the row will increment to 20*100 = 2000
4341     int threadNum = 20;
4342     int incCounter = 100;
4343     long expected = threadNum * incCounter;
4344     Thread[] incrementers = new Thread[threadNum];
4345     Thread flushThread = new Thread(flusher);
4346     for (int i = 0; i < threadNum; i++) {
4347       incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
4348       incrementers[i].start();
4349     }
4350     flushThread.start();
4351     for (int i = 0; i < threadNum; i++) {
4352       incrementers[i].join();
4353     }
4354 
4355     incrementDone.set(true);
4356     flushThread.join();
4357 
4358     Get get = new Get(Incrementer.incRow);
4359     get.addColumn(Incrementer.family, Incrementer.qualifier);
4360     get.setMaxVersions(1);
4361     Result res = this.region.get(get);
4362     List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
4363 
4364     // we just got the latest version
4365     assertEquals(kvs.size(), 1);
4366     Cell kv = kvs.get(0);
4367     assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
4368     this.region = null;
4369   }
4370 
4371   /**
4372    * TestCase for append
4373    */
4374   private static class Appender implements Runnable {
4375     private HRegion region;
4376     private final static byte[] appendRow = Bytes.toBytes("appendRow");
4377     private final static byte[] family = Bytes.toBytes("family");
4378     private final static byte[] qualifier = Bytes.toBytes("qualifier");
4379     private final static byte[] CHAR = Bytes.toBytes("a");
4380     private int appendCounter;
4381 
4382     public Appender(HRegion region, int appendCounter) {
4383       this.region = region;
4384       this.appendCounter = appendCounter;
4385     }
4386 
4387     @Override
4388     public void run() {
4389       int count = 0;
4390       while (count < appendCounter) {
4391         Append app = new Append(appendRow);
4392         app.add(family, qualifier, CHAR);
4393         count++;
4394         try {
4395           region.append(app);
4396         } catch (IOException e) {
4397           e.printStackTrace();
4398           break;
4399         }
4400       }
4401     }
4402   }
4403 
4404   /**
4405    * Test case to check append function with memstore flushing
4406    * @throws Exception
4407    */
4408   @Test
4409   public void testParallelAppendWithMemStoreFlush() throws Exception {
4410     byte[] family = Appender.family;
4411     this.region = initHRegion(tableName, method, CONF, family);
4412     final HRegion region = this.region;
4413     final AtomicBoolean appendDone = new AtomicBoolean(false);
4414     Runnable flusher = new Runnable() {
4415       @Override
4416       public void run() {
4417         while (!appendDone.get()) {
4418           try {
4419             region.flushcache();
4420           } catch (Exception e) {
4421             e.printStackTrace();
4422           }
4423         }
4424       }
4425     };
4426 
4427     // after all append finished, the value will append to threadNum *
4428     // appendCounter Appender.CHAR
4429     int threadNum = 20;
4430     int appendCounter = 100;
4431     byte[] expected = new byte[threadNum * appendCounter];
4432     for (int i = 0; i < threadNum * appendCounter; i++) {
4433       System.arraycopy(Appender.CHAR, 0, expected, i, 1);
4434     }
4435     Thread[] appenders = new Thread[threadNum];
4436     Thread flushThread = new Thread(flusher);
4437     for (int i = 0; i < threadNum; i++) {
4438       appenders[i] = new Thread(new Appender(this.region, appendCounter));
4439       appenders[i].start();
4440     }
4441     flushThread.start();
4442     for (int i = 0; i < threadNum; i++) {
4443       appenders[i].join();
4444     }
4445 
4446     appendDone.set(true);
4447     flushThread.join();
4448 
4449     Get get = new Get(Appender.appendRow);
4450     get.addColumn(Appender.family, Appender.qualifier);
4451     get.setMaxVersions(1);
4452     Result res = this.region.get(get);
4453     List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
4454 
4455     // we just got the latest version
4456     assertEquals(kvs.size(), 1);
4457     Cell kv = kvs.get(0);
4458     byte[] appendResult = new byte[kv.getValueLength()];
4459     System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
4460     assertArrayEquals(expected, appendResult);
4461     this.region = null;
4462   }
4463 
4464   /**
4465    * Test case to check put function with memstore flushing for same row, same ts
4466    * @throws Exception
4467    */
4468   @Test
4469   public void testPutWithMemStoreFlush() throws Exception {
4470     byte[] family = Bytes.toBytes("family");
4471     ;
4472     byte[] qualifier = Bytes.toBytes("qualifier");
4473     byte[] row = Bytes.toBytes("putRow");
4474     byte[] value = null;
4475     this.region = initHRegion(tableName, method, CONF, family);
4476     Put put = null;
4477     Get get = null;
4478     List<Cell> kvs = null;
4479     Result res = null;
4480 
4481     put = new Put(row);
4482     value = Bytes.toBytes("value0");
4483     put.add(family, qualifier, 1234567l, value);
4484     region.put(put);
4485     get = new Get(row);
4486     get.addColumn(family, qualifier);
4487     get.setMaxVersions();
4488     res = this.region.get(get);
4489     kvs = res.getColumnCells(family, qualifier);
4490     assertEquals(1, kvs.size());
4491     assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4492 
4493     region.flushcache();
4494     get = new Get(row);
4495     get.addColumn(family, qualifier);
4496     get.setMaxVersions();
4497     res = this.region.get(get);
4498     kvs = res.getColumnCells(family, qualifier);
4499     assertEquals(1, kvs.size());
4500     assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4501 
4502     put = new Put(row);
4503     value = Bytes.toBytes("value1");
4504     put.add(family, qualifier, 1234567l, value);
4505     region.put(put);
4506     get = new Get(row);
4507     get.addColumn(family, qualifier);
4508     get.setMaxVersions();
4509     res = this.region.get(get);
4510     kvs = res.getColumnCells(family, qualifier);
4511     assertEquals(1, kvs.size());
4512     assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4513 
4514     region.flushcache();
4515     get = new Get(row);
4516     get.addColumn(family, qualifier);
4517     get.setMaxVersions();
4518     res = this.region.get(get);
4519     kvs = res.getColumnCells(family, qualifier);
4520     assertEquals(1, kvs.size());
4521     assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4522   }
4523 
4524   @Test
4525   public void testDurability() throws Exception {
4526     String method = "testDurability";
4527     // there are 5 x 5 cases:
4528     // table durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) x mutation
4529     // durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT)
4530 
4531     // expected cases for append and sync wal
4532     durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4533     durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4534     durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4535 
4536     durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4537     durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4538     durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4539 
4540     durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4541     durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4542 
4543     durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
4544     durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4545 
4546     durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
4547     durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
4548     durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
4549 
4550     // expected cases for async wal
4551     durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4552     durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4553     durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4554     durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4555     durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
4556     durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4557 
4558     durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4559     durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4560     durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4561     durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4562     durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
4563     durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
4564 
4565     // expect skip wal cases
4566     durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, true, false, false);
4567     durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, true, false, false);
4568     durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, true, false, false);
4569     durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, true, false, false);
4570     durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, true, false, false);
4571     durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4572 
4573   }
4574 
4575   @SuppressWarnings("unchecked")
4576   private void durabilityTest(String method, Durability tableDurability,
4577       Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
4578       final boolean expectSyncFromLogSyncer) throws Exception {
4579     Configuration conf = HBaseConfiguration.create(CONF);
4580     method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
4581     TableName tableName = TableName.valueOf(method);
4582     byte[] family = Bytes.toBytes("family");
4583     Path logDir = new Path(new Path(dir + method), "log");
4584     final Configuration walConf = new Configuration(conf);
4585     FSUtils.setRootDir(walConf, logDir);
4586     final WALFactory wals = new WALFactory(walConf, null, UUID.randomUUID().toString());
4587     final WAL wal = spy(wals.getWAL(tableName.getName()));
4588     this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
4589         HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, wal,
4590         new byte[][] { family });
4591 
4592     Put put = new Put(Bytes.toBytes("r1"));
4593     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
4594     put.setDurability(mutationDurability);
4595     region.put(put);
4596 
4597     //verify append called or not
4598     verify(wal, expectAppend ? times(1) : never())
4599       .append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any(),
4600           (WALEdit)any(), (AtomicLong)any(), Mockito.anyBoolean(), (List<Cell>)any());
4601 
4602     // verify sync called or not
4603     if (expectSync || expectSyncFromLogSyncer) {
4604       TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
4605         @Override
4606         public boolean evaluate() throws Exception {
4607           try {
4608             if (expectSync) {
4609               verify(wal, times(1)).sync(anyLong()); // Hregion calls this one
4610             } else if (expectSyncFromLogSyncer) {
4611               verify(wal, times(1)).sync(); // wal syncer calls this one
4612             }
4613           } catch (Throwable ignore) {
4614           }
4615           return true;
4616         }
4617       });
4618     } else {
4619       //verify(wal, never()).sync(anyLong());
4620       verify(wal, never()).sync();
4621     }
4622 
4623     HRegion.closeHRegion(this.region);
4624     this.region = null;
4625   }
4626 
4627   @Test
4628   public void testRegionReplicaSecondary() throws IOException {
4629     // create a primary region, load some data and flush
4630     // create a secondary region, and do a get against that
4631     Path rootDir = new Path(dir + "testRegionReplicaSecondary");
4632     FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 
4633 
4634     byte[][] families = new byte[][] {
4635         Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")
4636     };
4637     byte[] cq = Bytes.toBytes("cq");
4638     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionReplicaSecondary"));
4639     for (byte[] family : families) {
4640       htd.addFamily(new HColumnDescriptor(family));
4641     }
4642 
4643     long time = System.currentTimeMillis();
4644     HRegionInfo primaryHri = new HRegionInfo(htd.getTableName(),
4645       HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4646       false, time, 0);
4647     HRegionInfo secondaryHri = new HRegionInfo(htd.getTableName(),
4648       HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4649       false, time, 1);
4650 
4651     HRegion primaryRegion = null, secondaryRegion = null;
4652 
4653     try {
4654       primaryRegion = HRegion.createHRegion(primaryHri,
4655         rootDir, TEST_UTIL.getConfiguration(), htd);
4656 
4657       // load some data
4658       putData(primaryRegion, 0, 1000, cq, families);
4659 
4660       // flush region
4661       primaryRegion.flushcache();
4662 
4663       // open secondary region
4664       secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, htd, null, CONF);
4665 
4666       verifyData(secondaryRegion, 0, 1000, cq, families);
4667     } finally {
4668       if (primaryRegion != null) {
4669         HRegion.closeHRegion(primaryRegion);
4670       }
4671       if (secondaryRegion != null) {
4672         HRegion.closeHRegion(secondaryRegion);
4673       }
4674     }
4675   }
4676 
4677   @Test
4678   public void testRegionReplicaSecondaryIsReadOnly() throws IOException {
4679     // create a primary region, load some data and flush
4680     // create a secondary region, and do a put against that
4681     Path rootDir = new Path(dir + "testRegionReplicaSecondary");
4682     FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
4683 
4684     byte[][] families = new byte[][] {
4685         Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")
4686     };
4687     byte[] cq = Bytes.toBytes("cq");
4688     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionReplicaSecondary"));
4689     for (byte[] family : families) {
4690       htd.addFamily(new HColumnDescriptor(family));
4691     }
4692 
4693     long time = System.currentTimeMillis();
4694     HRegionInfo primaryHri = new HRegionInfo(htd.getTableName(),
4695       HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4696       false, time, 0);
4697     HRegionInfo secondaryHri = new HRegionInfo(htd.getTableName(),
4698       HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4699       false, time, 1);
4700 
4701     HRegion primaryRegion = null, secondaryRegion = null;
4702 
4703     try {
4704       primaryRegion = HRegion.createHRegion(primaryHri,
4705         rootDir, TEST_UTIL.getConfiguration(), htd);
4706 
4707       // load some data
4708       putData(primaryRegion, 0, 1000, cq, families);
4709 
4710       // flush region
4711       primaryRegion.flushcache();
4712 
4713       // open secondary region
4714       secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, htd, null, CONF);
4715 
4716       try {
4717         putData(secondaryRegion, 0, 1000, cq, families);
4718         fail("Should have thrown exception");
4719       } catch (IOException ex) {
4720         // expected
4721       }
4722     } finally {
4723       if (primaryRegion != null) {
4724         HRegion.closeHRegion(primaryRegion);
4725       }
4726       if (secondaryRegion != null) {
4727         HRegion.closeHRegion(secondaryRegion);
4728       }
4729     }
4730   }
4731 
4732   @Test
4733   public void testCompactionFromPrimary() throws IOException {
4734     Path rootDir = new Path(dir + "testRegionReplicaSecondary");
4735     FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
4736 
4737     byte[][] families = new byte[][] {
4738         Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")
4739     };
4740     byte[] cq = Bytes.toBytes("cq");
4741     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionReplicaSecondary"));
4742     for (byte[] family : families) {
4743       htd.addFamily(new HColumnDescriptor(family));
4744     }
4745 
4746     long time = System.currentTimeMillis();
4747     HRegionInfo primaryHri = new HRegionInfo(htd.getTableName(),
4748       HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4749       false, time, 0);
4750     HRegionInfo secondaryHri = new HRegionInfo(htd.getTableName(),
4751       HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4752       false, time, 1);
4753 
4754     HRegion primaryRegion = null, secondaryRegion = null;
4755 
4756     try {
4757       primaryRegion = HRegion.createHRegion(primaryHri,
4758         rootDir, TEST_UTIL.getConfiguration(), htd);
4759 
4760       // load some data
4761       putData(primaryRegion, 0, 1000, cq, families);
4762 
4763       // flush region
4764       primaryRegion.flushcache();
4765 
4766       // open secondary region
4767       secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, htd, null, CONF);
4768 
4769       // move the file of the primary region to the archive, simulating a compaction
4770       Collection<StoreFile> storeFiles = primaryRegion.getStore(families[0]).getStorefiles();
4771       primaryRegion.getRegionFileSystem().removeStoreFiles(Bytes.toString(families[0]), storeFiles);
4772       Collection<StoreFileInfo> storeFileInfos = primaryRegion.getRegionFileSystem().getStoreFiles(families[0]);
4773       Assert.assertTrue(storeFileInfos == null || storeFileInfos.size() == 0);
4774 
4775       verifyData(secondaryRegion, 0, 1000, cq, families);
4776     } finally {
4777       if (primaryRegion != null) {
4778         HRegion.closeHRegion(primaryRegion);
4779       }
4780       if (secondaryRegion != null) {
4781         HRegion.closeHRegion(secondaryRegion);
4782       }
4783     }
4784   }
4785 
4786   private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4787     putData(this.region, startRow, numRows, qf, families);
4788   }
4789 
4790   private void putData(HRegion region,
4791       int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4792     for (int i = startRow; i < startRow + numRows; i++) {
4793       Put put = new Put(Bytes.toBytes("" + i));
4794       put.setDurability(Durability.SKIP_WAL);
4795       for (byte[] family : families) {
4796         put.add(family, qf, null);
4797       }
4798       region.put(put);
4799     }
4800   }
4801 
4802   private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
4803       throws IOException {
4804     for (int i = startRow; i < startRow + numRows; i++) {
4805       byte[] row = Bytes.toBytes("" + i);
4806       Get get = new Get(row);
4807       for (byte[] family : families) {
4808         get.addColumn(family, qf);
4809       }
4810       Result result = newReg.get(get);
4811       Cell[] raw = result.rawCells();
4812       assertEquals(families.length, result.size());
4813       for (int j = 0; j < families.length; j++) {
4814         assertTrue(CellUtil.matchingRow(raw[j], row));
4815         assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
4816         assertTrue(CellUtil.matchingQualifier(raw[j], qf));
4817       }
4818     }
4819   }
4820 
4821   private void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
4822     // Now I have k, get values out and assert they are as expected.
4823     Get get = new Get(k).addFamily(family).setMaxVersions();
4824     Cell[] results = r.get(get).rawCells();
4825     for (int j = 0; j < results.length; j++) {
4826       byte[] tmp = CellUtil.cloneValue(results[j]);
4827       // Row should be equal to value every time.
4828       assertTrue(Bytes.equals(k, tmp));
4829     }
4830   }
4831 
4832   /*
4833    * Assert first value in the passed region is <code>firstValue</code>.
4834    *
4835    * @param r
4836    *
4837    * @param fs
4838    *
4839    * @param firstValue
4840    *
4841    * @throws IOException
4842    */
4843   private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
4844       throws IOException {
4845     byte[][] families = { fs };
4846     Scan scan = new Scan();
4847     for (int i = 0; i < families.length; i++)
4848       scan.addFamily(families[i]);
4849     InternalScanner s = r.getScanner(scan);
4850     try {
4851       List<Cell> curVals = new ArrayList<Cell>();
4852       boolean first = true;
4853       OUTER_LOOP: while (s.next(curVals)) {
4854         for (Cell kv : curVals) {
4855           byte[] val = CellUtil.cloneValue(kv);
4856           byte[] curval = val;
4857           if (first) {
4858             first = false;
4859             assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4860           } else {
4861             // Not asserting anything. Might as well break.
4862             break OUTER_LOOP;
4863           }
4864         }
4865       }
4866     } finally {
4867       s.close();
4868     }
4869   }
4870 
4871   /**
4872    * Test that we get the expected flush results back
4873    * @throws IOException
4874    */
4875   @Test
4876   public void testFlushResult() throws IOException {
4877     String method = name.getMethodName();
4878     byte[] tableName = Bytes.toBytes(method);
4879     byte[] family = Bytes.toBytes("family");
4880 
4881     this.region = initHRegion(tableName, method, family);
4882 
4883     // empty memstore, flush doesn't run
4884     HRegion.FlushResult fr = region.flushcache();
4885     assertFalse(fr.isFlushSucceeded());
4886     assertFalse(fr.isCompactionNeeded());
4887 
4888     // Flush enough files to get up to the threshold, doesn't need compactions
4889     for (int i = 0; i < 2; i++) {
4890       Put put = new Put(tableName).add(family, family, tableName);
4891       region.put(put);
4892       fr = region.flushcache();
4893       assertTrue(fr.isFlushSucceeded());
4894       assertFalse(fr.isCompactionNeeded());
4895     }
4896 
4897     // Two flushes after the threshold, compactions are needed
4898     for (int i = 0; i < 2; i++) {
4899       Put put = new Put(tableName).add(family, family, tableName);
4900       region.put(put);
4901       fr = region.flushcache();
4902       assertTrue(fr.isFlushSucceeded());
4903       assertTrue(fr.isCompactionNeeded());
4904     }
4905   }
4906 
4907   private Configuration initSplit() {
4908     // Always compact if there is more than one store file.
4909     CONF.setInt("hbase.hstore.compactionThreshold", 2);
4910 
4911     // Make lease timeout longer, lease checks less frequent
4912     CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4913 
4914     CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4915 
4916     // Increase the amount of time between client retries
4917     CONF.setLong("hbase.client.pause", 15 * 1000);
4918 
4919     // This size should make it so we always split using the addContent
4920     // below. After adding all data, the first region is 1.3M
4921     CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4922     return CONF;
4923   }
4924 
4925   /**
4926    * @param tableName
4927    * @param callingMethod
4928    * @param conf
4929    * @param families
4930    * @throws IOException
4931    * @return A region on which you must call
4932    *         {@link HRegion#closeHRegion(HRegion)} when done.
4933    */
4934   public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
4935       byte[]... families) throws IOException {
4936     return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
4937   }
4938 
4939   /**
4940    * @param tableName
4941    * @param callingMethod
4942    * @param conf
4943    * @param families
4944    * @throws IOException
4945    * @return A region on which you must call
4946    *         {@link HRegion#closeHRegion(HRegion)} when done.
4947    */
4948   public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4949       byte[]... families) throws IOException {
4950     return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4951   }
4952 
4953   /**
4954    * @param tableName
4955    * @param callingMethod
4956    * @param conf
4957    * @param isReadOnly
4958    * @param families
4959    * @throws IOException
4960    * @return A region on which you must call
4961    *         {@link HRegion#closeHRegion(HRegion)} when done.
4962    */
4963   public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4964       boolean isReadOnly, byte[]... families) throws IOException {
4965     return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4966   }
4967 
4968   private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4969       String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4970       throws IOException {
4971     return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
4972         Durability.SYNC_WAL, null, families);
4973   }
4974 
4975   /**
4976    * @param tableName
4977    * @param startKey
4978    * @param stopKey
4979    * @param callingMethod
4980    * @param conf
4981    * @param isReadOnly
4982    * @param families
4983    * @throws IOException
4984    * @return A region on which you must call
4985    *         {@link HRegion#closeHRegion(HRegion)} when done.
4986    */
4987   private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4988       String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4989       WAL wal, byte[]... families) throws IOException {
4990     return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf,
4991         isReadOnly, durability, wal, families);
4992   }
4993 
4994   /**
4995    * Assert that the passed in Cell has expected contents for the specified row,
4996    * column & timestamp.
4997    */
4998   private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
4999     String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
5000     assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
5001         Bytes.toString(CellUtil.cloneRow(kv)));
5002     assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
5003         Bytes.toString(CellUtil.cloneFamily(kv)));
5004     assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
5005         Bytes.toString(CellUtil.cloneQualifier(kv)));
5006     assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
5007     assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
5008         Bytes.toString(CellUtil.cloneValue(kv)));
5009   }
5010 
5011   @Test (timeout=60000)
5012   public void testReverseScanner_FromMemStore_SingleCF_Normal()
5013       throws IOException {
5014     byte[] rowC = Bytes.toBytes("rowC");
5015     byte[] rowA = Bytes.toBytes("rowA");
5016     byte[] rowB = Bytes.toBytes("rowB");
5017     byte[] cf = Bytes.toBytes("CF");
5018     byte[][] families = { cf };
5019     byte[] col = Bytes.toBytes("C");
5020     long ts = 1;
5021     String method = this.getName();
5022     this.region = initHRegion(tableName, method, families);
5023     try {
5024       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5025       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
5026           null);
5027       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5028       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5029       Put put = null;
5030       put = new Put(rowC);
5031       put.add(kv1);
5032       put.add(kv11);
5033       region.put(put);
5034       put = new Put(rowA);
5035       put.add(kv2);
5036       region.put(put);
5037       put = new Put(rowB);
5038       put.add(kv3);
5039       region.put(put);
5040 
5041       Scan scan = new Scan(rowC);
5042       scan.setMaxVersions(5);
5043       scan.setReversed(true);
5044       InternalScanner scanner = region.getScanner(scan);
5045       List<Cell> currRow = new ArrayList<Cell>();
5046       boolean hasNext = scanner.next(currRow);
5047       assertEquals(2, currRow.size());
5048       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5049       assertTrue(hasNext);
5050       currRow.clear();
5051       hasNext = scanner.next(currRow);
5052       assertEquals(1, currRow.size());
5053       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5054       assertTrue(hasNext);
5055       currRow.clear();
5056       hasNext = scanner.next(currRow);
5057       assertEquals(1, currRow.size());
5058       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
5059       assertFalse(hasNext);
5060       scanner.close();
5061     } finally {
5062       HRegion.closeHRegion(this.region);
5063       this.region = null;
5064     }
5065   }
5066 
5067   @Test (timeout=60000)
5068   public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
5069       throws IOException {
5070     byte[] rowC = Bytes.toBytes("rowC");
5071     byte[] rowA = Bytes.toBytes("rowA");
5072     byte[] rowB = Bytes.toBytes("rowB");
5073     byte[] rowD = Bytes.toBytes("rowD");
5074     byte[] cf = Bytes.toBytes("CF");
5075     byte[][] families = { cf };
5076     byte[] col = Bytes.toBytes("C");
5077     long ts = 1;
5078     String method = this.getName();
5079     this.region = initHRegion(tableName, method, families);
5080     try {
5081       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5082       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
5083           null);
5084       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5085       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5086       Put put = null;
5087       put = new Put(rowC);
5088       put.add(kv1);
5089       put.add(kv11);
5090       region.put(put);
5091       put = new Put(rowA);
5092       put.add(kv2);
5093       region.put(put);
5094       put = new Put(rowB);
5095       put.add(kv3);
5096       region.put(put);
5097 
5098       Scan scan = new Scan(rowD);
5099       List<Cell> currRow = new ArrayList<Cell>();
5100       scan.setReversed(true);
5101       scan.setMaxVersions(5);
5102       InternalScanner scanner = region.getScanner(scan);
5103       boolean hasNext = scanner.next(currRow);
5104       assertEquals(2, currRow.size());
5105       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5106       assertTrue(hasNext);
5107       currRow.clear();
5108       hasNext = scanner.next(currRow);
5109       assertEquals(1, currRow.size());
5110       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5111       assertTrue(hasNext);
5112       currRow.clear();
5113       hasNext = scanner.next(currRow);
5114       assertEquals(1, currRow.size());
5115       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
5116       assertFalse(hasNext);
5117       scanner.close();
5118     } finally {
5119       HRegion.closeHRegion(this.region);
5120       this.region = null;
5121     }
5122   }
5123 
5124   @Test (timeout=60000)
5125   public void testReverseScanner_FromMemStore_SingleCF_FullScan()
5126       throws IOException {
5127     byte[] rowC = Bytes.toBytes("rowC");
5128     byte[] rowA = Bytes.toBytes("rowA");
5129     byte[] rowB = Bytes.toBytes("rowB");
5130     byte[] cf = Bytes.toBytes("CF");
5131     byte[][] families = { cf };
5132     byte[] col = Bytes.toBytes("C");
5133     long ts = 1;
5134     String method = this.getName();
5135     this.region = initHRegion(tableName, method, families);
5136     try {
5137       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5138       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
5139           null);
5140       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5141       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5142       Put put = null;
5143       put = new Put(rowC);
5144       put.add(kv1);
5145       put.add(kv11);
5146       region.put(put);
5147       put = new Put(rowA);
5148       put.add(kv2);
5149       region.put(put);
5150       put = new Put(rowB);
5151       put.add(kv3);
5152       region.put(put);
5153       Scan scan = new Scan();
5154       List<Cell> currRow = new ArrayList<Cell>();
5155       scan.setReversed(true);
5156       InternalScanner scanner = region.getScanner(scan);
5157       boolean hasNext = scanner.next(currRow);
5158       assertEquals(1, currRow.size());
5159       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5160       assertTrue(hasNext);
5161       currRow.clear();
5162       hasNext = scanner.next(currRow);
5163       assertEquals(1, currRow.size());
5164       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5165       assertTrue(hasNext);
5166       currRow.clear();
5167       hasNext = scanner.next(currRow);
5168       assertEquals(1, currRow.size());
5169       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
5170       assertFalse(hasNext);
5171       scanner.close();
5172     } finally {
5173       HRegion.closeHRegion(this.region);
5174       this.region = null;
5175     }
5176   }
5177 
5178   @Test (timeout=60000)
5179   public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
5180     // case for "INCLUDE_AND_SEEK_NEXT_ROW & SEEK_NEXT_ROW" endless loop
5181     byte[] rowA = Bytes.toBytes("rowA");
5182     byte[] rowB = Bytes.toBytes("rowB");
5183     byte[] rowC = Bytes.toBytes("rowC");
5184     byte[] rowD = Bytes.toBytes("rowD");
5185     byte[] rowE = Bytes.toBytes("rowE");
5186     byte[] cf = Bytes.toBytes("CF");
5187     byte[][] families = { cf };
5188     byte[] col1 = Bytes.toBytes("col1");
5189     byte[] col2 = Bytes.toBytes("col2");
5190     long ts = 1;
5191     String method = this.getName();
5192     this.region = initHRegion(tableName, method, families);
5193     try {
5194       KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
5195       KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
5196       KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
5197       KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
5198       KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
5199       KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
5200       Put put = null;
5201       put = new Put(rowA);
5202       put.add(kv1);
5203       region.put(put);
5204       put = new Put(rowB);
5205       put.add(kv2);
5206       region.put(put);
5207       put = new Put(rowC);
5208       put.add(kv3);
5209       region.put(put);
5210       put = new Put(rowD);
5211       put.add(kv4_1);
5212       region.put(put);
5213       put = new Put(rowD);
5214       put.add(kv4_2);
5215       region.put(put);
5216       put = new Put(rowE);
5217       put.add(kv5);
5218       region.put(put);
5219       region.flushcache();
5220       Scan scan = new Scan(rowD, rowA);
5221       scan.addColumn(families[0], col1);
5222       scan.setReversed(true);
5223       List<Cell> currRow = new ArrayList<Cell>();
5224       InternalScanner scanner = region.getScanner(scan);
5225       boolean hasNext = scanner.next(currRow);
5226       assertEquals(1, currRow.size());
5227       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5228       assertTrue(hasNext);
5229       currRow.clear();
5230       hasNext = scanner.next(currRow);
5231       assertEquals(1, currRow.size());
5232       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5233       assertTrue(hasNext);
5234       currRow.clear();
5235       hasNext = scanner.next(currRow);
5236       assertEquals(1, currRow.size());
5237       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5238       assertFalse(hasNext);
5239       scanner.close();
5240 
5241       scan = new Scan(rowD, rowA);
5242       scan.addColumn(families[0], col2);
5243       scan.setReversed(true);
5244       currRow.clear();
5245       scanner = region.getScanner(scan);
5246       hasNext = scanner.next(currRow);
5247       assertEquals(1, currRow.size());
5248       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5249       scanner.close();
5250     } finally {
5251       HRegion.closeHRegion(this.region);
5252       this.region = null;
5253     }
5254   }
5255 
5256   @Test (timeout=60000)
5257   public void testReverseScanner_smaller_blocksize() throws IOException {
5258     // case to ensure no conflict with HFile index optimization
5259     byte[] rowA = Bytes.toBytes("rowA");
5260     byte[] rowB = Bytes.toBytes("rowB");
5261     byte[] rowC = Bytes.toBytes("rowC");
5262     byte[] rowD = Bytes.toBytes("rowD");
5263     byte[] rowE = Bytes.toBytes("rowE");
5264     byte[] cf = Bytes.toBytes("CF");
5265     byte[][] families = { cf };
5266     byte[] col1 = Bytes.toBytes("col1");
5267     byte[] col2 = Bytes.toBytes("col2");
5268     long ts = 1;
5269     String method = this.getName();
5270     HBaseConfiguration config = new HBaseConfiguration();
5271     config.setInt("test.block.size", 1);
5272     this.region = initHRegion(tableName, method, config, families);
5273     try {
5274       KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
5275       KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
5276       KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
5277       KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
5278       KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
5279       KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
5280       Put put = null;
5281       put = new Put(rowA);
5282       put.add(kv1);
5283       region.put(put);
5284       put = new Put(rowB);
5285       put.add(kv2);
5286       region.put(put);
5287       put = new Put(rowC);
5288       put.add(kv3);
5289       region.put(put);
5290       put = new Put(rowD);
5291       put.add(kv4_1);
5292       region.put(put);
5293       put = new Put(rowD);
5294       put.add(kv4_2);
5295       region.put(put);
5296       put = new Put(rowE);
5297       put.add(kv5);
5298       region.put(put);
5299       region.flushcache();
5300       Scan scan = new Scan(rowD, rowA);
5301       scan.addColumn(families[0], col1);
5302       scan.setReversed(true);
5303       List<Cell> currRow = new ArrayList<Cell>();
5304       InternalScanner scanner = region.getScanner(scan);
5305       boolean hasNext = scanner.next(currRow);
5306       assertEquals(1, currRow.size());
5307       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5308       assertTrue(hasNext);
5309       currRow.clear();
5310       hasNext = scanner.next(currRow);
5311       assertEquals(1, currRow.size());
5312       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5313       assertTrue(hasNext);
5314       currRow.clear();
5315       hasNext = scanner.next(currRow);
5316       assertEquals(1, currRow.size());
5317       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5318       assertFalse(hasNext);
5319       scanner.close();
5320 
5321       scan = new Scan(rowD, rowA);
5322       scan.addColumn(families[0], col2);
5323       scan.setReversed(true);
5324       currRow.clear();
5325       scanner = region.getScanner(scan);
5326       hasNext = scanner.next(currRow);
5327       assertEquals(1, currRow.size());
5328       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5329       scanner.close();
5330     } finally {
5331       HRegion.closeHRegion(this.region);
5332       this.region = null;
5333     }
5334   }
5335 
5336   @Test (timeout=60000)
5337   public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
5338       throws IOException {
5339     byte[] row0 = Bytes.toBytes("row0"); // 1 kv
5340     byte[] row1 = Bytes.toBytes("row1"); // 2 kv
5341     byte[] row2 = Bytes.toBytes("row2"); // 4 kv
5342     byte[] row3 = Bytes.toBytes("row3"); // 2 kv
5343     byte[] row4 = Bytes.toBytes("row4"); // 5 kv
5344     byte[] row5 = Bytes.toBytes("row5"); // 2 kv
5345     byte[] cf1 = Bytes.toBytes("CF1");
5346     byte[] cf2 = Bytes.toBytes("CF2");
5347     byte[] cf3 = Bytes.toBytes("CF3");
5348     byte[][] families = { cf1, cf2, cf3 };
5349     byte[] col = Bytes.toBytes("C");
5350     long ts = 1;
5351     String method = this.getName();
5352     HBaseConfiguration conf = new HBaseConfiguration();
5353     // disable compactions in this test.
5354     conf.setInt("hbase.hstore.compactionThreshold", 10000);
5355     this.region = initHRegion(tableName, method, conf, families);
5356     try {
5357       // kv naming style: kv(row number) totalKvCountInThisRow seq no
5358       KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
5359           null);
5360       KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
5361           null);
5362       KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
5363           KeyValue.Type.Put, null);
5364       KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
5365           null);
5366       KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
5367           null);
5368       KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
5369           null);
5370       KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
5371           KeyValue.Type.Put, null);
5372       KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
5373           null);
5374       KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
5375           KeyValue.Type.Put, null);
5376       KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
5377           null);
5378       KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
5379           null);
5380       KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
5381           KeyValue.Type.Put, null);
5382       KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
5383           null);
5384       KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
5385           KeyValue.Type.Put, null);
5386       KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
5387           null);
5388       KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
5389           null);
5390       // hfiles(cf1/cf2) :"row1"(1 kv) / "row2"(1 kv) / "row4"(2 kv)
5391       Put put = null;
5392       put = new Put(row1);
5393       put.add(kv1_2_1);
5394       region.put(put);
5395       put = new Put(row2);
5396       put.add(kv2_4_1);
5397       region.put(put);
5398       put = new Put(row4);
5399       put.add(kv4_5_4);
5400       put.add(kv4_5_5);
5401       region.put(put);
5402       region.flushcache();
5403       // hfiles(cf1/cf3) : "row1" (1 kvs) / "row2" (1 kv) / "row4" (2 kv)
5404       put = new Put(row4);
5405       put.add(kv4_5_1);
5406       put.add(kv4_5_3);
5407       region.put(put);
5408       put = new Put(row1);
5409       put.add(kv1_2_2);
5410       region.put(put);
5411       put = new Put(row2);
5412       put.add(kv2_4_4);
5413       region.put(put);
5414       region.flushcache();
5415       // hfiles(cf1/cf3) : "row2"(2 kv) / "row3"(1 kvs) / "row4" (1 kv)
5416       put = new Put(row4);
5417       put.add(kv4_5_2);
5418       region.put(put);
5419       put = new Put(row2);
5420       put.add(kv2_4_2);
5421       put.add(kv2_4_3);
5422       region.put(put);
5423       put = new Put(row3);
5424       put.add(kv3_2_2);
5425       region.put(put);
5426       region.flushcache();
5427       // memstore(cf1/cf2/cf3) : "row0" (1 kvs) / "row3" ( 1 kv) / "row5" (max)
5428       // ( 2 kv)
5429       put = new Put(row0);
5430       put.add(kv0_1_1);
5431       region.put(put);
5432       put = new Put(row3);
5433       put.add(kv3_2_1);
5434       region.put(put);
5435       put = new Put(row5);
5436       put.add(kv5_2_1);
5437       put.add(kv5_2_2);
5438       region.put(put);
5439       // scan range = ["row4", min), skip the max "row5"
5440       Scan scan = new Scan(row4);
5441       scan.setMaxVersions(5);
5442       scan.setBatch(3);
5443       scan.setReversed(true);
5444       InternalScanner scanner = region.getScanner(scan);
5445       List<Cell> currRow = new ArrayList<Cell>();
5446       boolean hasNext = false;
5447       // 1. scan out "row4" (5 kvs), "row5" can't be scanned out since not
5448       // included in scan range
5449       // "row4" takes 2 next() calls since batch=3
5450       hasNext = scanner.next(currRow);
5451       assertEquals(3, currRow.size());
5452       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5453       assertTrue(hasNext);
5454       currRow.clear();
5455       hasNext = scanner.next(currRow);
5456       assertEquals(2, currRow.size());
5457       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5458       assertTrue(hasNext);
5459       // 2. scan out "row3" (2 kv)
5460       currRow.clear();
5461       hasNext = scanner.next(currRow);
5462       assertEquals(2, currRow.size());
5463       assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5464       assertTrue(hasNext);
5465       // 3. scan out "row2" (4 kvs)
5466       // "row2" takes 2 next() calls since batch=3
5467       currRow.clear();
5468       hasNext = scanner.next(currRow);
5469       assertEquals(3, currRow.size());
5470       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5471       assertTrue(hasNext);
5472       currRow.clear();
5473       hasNext = scanner.next(currRow);
5474       assertEquals(1, currRow.size());
5475       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5476       assertTrue(hasNext);
5477       // 4. scan out "row1" (2 kv)
5478       currRow.clear();
5479       hasNext = scanner.next(currRow);
5480       assertEquals(2, currRow.size());
5481       assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5482       assertTrue(hasNext);
5483       // 5. scan out "row0" (1 kv)
5484       currRow.clear();
5485       hasNext = scanner.next(currRow);
5486       assertEquals(1, currRow.size());
5487       assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
5488       assertFalse(hasNext);
5489 
5490       scanner.close();
5491     } finally {
5492       HRegion.closeHRegion(this.region);
5493       this.region = null;
5494     }
5495   }
5496 
5497   @Test (timeout=60000)
5498   public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
5499       throws IOException {
5500     byte[] row1 = Bytes.toBytes("row1");
5501     byte[] row2 = Bytes.toBytes("row2");
5502     byte[] row3 = Bytes.toBytes("row3");
5503     byte[] row4 = Bytes.toBytes("row4");
5504     byte[] cf1 = Bytes.toBytes("CF1");
5505     byte[] cf2 = Bytes.toBytes("CF2");
5506     byte[] cf3 = Bytes.toBytes("CF3");
5507     byte[] cf4 = Bytes.toBytes("CF4");
5508     byte[][] families = { cf1, cf2, cf3, cf4 };
5509     byte[] col = Bytes.toBytes("C");
5510     long ts = 1;
5511     String method = this.getName();
5512     HBaseConfiguration conf = new HBaseConfiguration();
5513     // disable compactions in this test.
5514     conf.setInt("hbase.hstore.compactionThreshold", 10000);
5515     this.region = initHRegion(tableName, method, conf, families);
5516     try {
5517       KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
5518       KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
5519       KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
5520       KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
5521       // storefile1
5522       Put put = new Put(row1);
5523       put.add(kv1);
5524       region.put(put);
5525       region.flushcache();
5526       // storefile2
5527       put = new Put(row2);
5528       put.add(kv2);
5529       region.put(put);
5530       region.flushcache();
5531       // storefile3
5532       put = new Put(row3);
5533       put.add(kv3);
5534       region.put(put);
5535       region.flushcache();
5536       // memstore
5537       put = new Put(row4);
5538       put.add(kv4);
5539       region.put(put);
5540       // scan range = ["row4", min)
5541       Scan scan = new Scan(row4);
5542       scan.setReversed(true);
5543       scan.setBatch(10);
5544       InternalScanner scanner = region.getScanner(scan);
5545       List<Cell> currRow = new ArrayList<Cell>();
5546       boolean hasNext = scanner.next(currRow);
5547       assertEquals(1, currRow.size());
5548       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5549       assertTrue(hasNext);
5550       currRow.clear();
5551       hasNext = scanner.next(currRow);
5552       assertEquals(1, currRow.size());
5553       assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5554       assertTrue(hasNext);
5555       currRow.clear();
5556       hasNext = scanner.next(currRow);
5557       assertEquals(1, currRow.size());
5558       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5559       assertTrue(hasNext);
5560       currRow.clear();
5561       hasNext = scanner.next(currRow);
5562       assertEquals(1, currRow.size());
5563       assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5564       assertFalse(hasNext);
5565     } finally {
5566       HRegion.closeHRegion(this.region);
5567       this.region = null;
5568     }
5569   }
5570 
5571   @Test (timeout=60000)
5572   public void testSplitRegionWithReverseScan() throws IOException {
5573     byte [] tableName = Bytes.toBytes("testSplitRegionWithReverseScan");
5574     byte [] qualifier = Bytes.toBytes("qualifier");
5575     Configuration hc = initSplit();
5576     int numRows = 3;
5577     byte [][] families = {fam1};
5578 
5579     //Setting up region
5580     String method = this.getName();
5581     this.region = initHRegion(tableName, method, hc, families);
5582 
5583     //Put data in region
5584     int startRow = 100;
5585     putData(startRow, numRows, qualifier, families);
5586     int splitRow = startRow + numRows;
5587     putData(splitRow, numRows, qualifier, families);
5588     int endRow = splitRow + numRows;
5589     region.flushcache();
5590 
5591     HRegion [] regions = null;
5592     try {
5593       regions = splitRegion(region, Bytes.toBytes("" + splitRow));
5594       //Opening the regions returned.
5595       for (int i = 0; i < regions.length; i++) {
5596         regions[i] = HRegion.openHRegion(regions[i], null);
5597       }
5598       //Verifying that the region has been split
5599       assertEquals(2, regions.length);
5600 
5601       //Verifying that all data is still there and that data is in the right
5602       //place
5603       verifyData(regions[0], startRow, numRows, qualifier, families);
5604       verifyData(regions[1], splitRow, numRows, qualifier, families);
5605 
5606       //fire the reverse scan1:  top range, and larger than the last row
5607       Scan scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 10 * numRows)));
5608       scan.setReversed(true);
5609       InternalScanner scanner = regions[1].getScanner(scan);
5610       List<Cell> currRow = new ArrayList<Cell>();
5611       boolean more = false;
5612       int verify = startRow + 2 * numRows - 1;
5613       do {
5614         more = scanner.next(currRow);
5615         assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5616         verify--;
5617         currRow.clear();
5618       } while(more);
5619       assertEquals(verify, startRow + numRows - 1);
5620       scanner.close();
5621       //fire the reverse scan2:  top range, and equals to the last row
5622       scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 2 * numRows - 1)));
5623       scan.setReversed(true);
5624       scanner = regions[1].getScanner(scan);
5625       verify = startRow + 2 * numRows - 1;
5626       do {
5627         more = scanner.next(currRow);
5628         assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5629         verify--;
5630         currRow.clear();
5631       } while(more);
5632       assertEquals(verify, startRow + numRows - 1);
5633       scanner.close();
5634       //fire the reverse scan3:  bottom range, and larger than the last row
5635       scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows)));
5636       scan.setReversed(true);
5637       scanner = regions[0].getScanner(scan);
5638       verify = startRow + numRows - 1;
5639       do {
5640         more = scanner.next(currRow);
5641         assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5642         verify--;
5643         currRow.clear();
5644       } while(more);
5645       assertEquals(verify, 99);
5646       scanner.close();
5647       //fire the reverse scan4:  bottom range, and equals to the last row
5648       scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows - 1)));
5649       scan.setReversed(true);
5650       scanner = regions[0].getScanner(scan);
5651       verify = startRow + numRows - 1;
5652       do {
5653         more = scanner.next(currRow);
5654         assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5655         verify--;
5656         currRow.clear();
5657       } while(more);
5658       assertEquals(verify, startRow - 1);
5659       scanner.close();
5660     } finally {
5661       HRegion.closeHRegion(this.region);
5662       this.region = null;
5663     }
5664   }
5665 
5666   @Test
5667   public void testWriteRequestsCounter() throws IOException {
5668     byte[] fam = Bytes.toBytes("info");
5669     byte[][] families = { fam };
5670     this.region = initHRegion(tableName, method, CONF, families);
5671 
5672     Assert.assertEquals(0L, region.getWriteRequestsCount());
5673 
5674     Put put = new Put(row);
5675     put.add(fam, fam, fam);
5676 
5677     Assert.assertEquals(0L, region.getWriteRequestsCount());
5678     region.put(put);
5679     Assert.assertEquals(1L, region.getWriteRequestsCount());
5680     region.put(put);
5681     Assert.assertEquals(2L, region.getWriteRequestsCount());
5682     region.put(put);
5683     Assert.assertEquals(3L, region.getWriteRequestsCount());
5684 
5685     region.delete(new Delete(row));
5686     Assert.assertEquals(4L, region.getWriteRequestsCount());
5687 
5688     HRegion.closeHRegion(this.region);
5689     this.region = null;
5690   }
5691 
5692   @Test
5693   @SuppressWarnings("unchecked")
5694   public void testOpenRegionWrittenToWAL() throws Exception {
5695     final ServerName serverName = ServerName.valueOf("testOpenRegionWrittenToWAL", 100, 42);
5696     final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
5697 
5698     HTableDescriptor htd
5699         = new HTableDescriptor(TableName.valueOf("testOpenRegionWrittenToWAL"));
5700     htd.addFamily(new HColumnDescriptor(fam1));
5701     htd.addFamily(new HColumnDescriptor(fam2));
5702 
5703     HRegionInfo hri = new HRegionInfo(htd.getTableName(),
5704       HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY);
5705 
5706     // open the region w/o rss and wal and flush some files
5707     HRegion region =
5708          HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL
5709             .getConfiguration(), htd);
5710     assertNotNull(region);
5711 
5712     // create a file in fam1 for the region before opening in OpenRegionHandler
5713     region.put(new Put(Bytes.toBytes("a")).add(fam1, fam1, fam1));
5714     region.flushcache();
5715     region.close();
5716 
5717     ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
5718 
5719     // capture append() calls
5720     WAL wal = mock(WAL.class);
5721     when(rss.getWAL((HRegionInfo) any())).thenReturn(wal);
5722 
5723     try {
5724       region = HRegion.openHRegion(hri, htd, rss.getWAL(hri),
5725         TEST_UTIL.getConfiguration(), rss, null);
5726 
5727       verify(wal, times(1)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any()
5728         , editCaptor.capture(), (AtomicLong)any(), anyBoolean(), (List<Cell>)any());
5729 
5730       WALEdit edit = editCaptor.getValue();
5731       assertNotNull(edit);
5732       assertNotNull(edit.getCells());
5733       assertEquals(1, edit.getCells().size());
5734       RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
5735       assertNotNull(desc);
5736 
5737       LOG.info("RegionEventDescriptor from WAL: " + desc);
5738 
5739       assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType());
5740       assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getName()));
5741       assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(),
5742         hri.getEncodedNameAsBytes()));
5743       assertTrue(desc.getLogSequenceNumber() > 0);
5744       assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
5745       assertEquals(2, desc.getStoresCount());
5746 
5747       StoreDescriptor store = desc.getStores(0);
5748       assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
5749       assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
5750       assertEquals(1, store.getStoreFileCount()); // 1store file
5751       assertFalse(store.getStoreFile(0).contains("/")); // ensure path is relative
5752 
5753       store = desc.getStores(1);
5754       assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
5755       assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
5756       assertEquals(0, store.getStoreFileCount()); // no store files
5757 
5758     } finally {
5759       HRegion.closeHRegion(region);
5760     }
5761   }
5762 
5763   @Test
5764   @SuppressWarnings("unchecked")
5765   public void testCloseRegionWrittenToWAL() throws Exception {
5766     final ServerName serverName = ServerName.valueOf("testCloseRegionWrittenToWAL", 100, 42);
5767     final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
5768 
5769     HTableDescriptor htd
5770     = new HTableDescriptor(TableName.valueOf("testOpenRegionWrittenToWAL"));
5771     htd.addFamily(new HColumnDescriptor(fam1));
5772     htd.addFamily(new HColumnDescriptor(fam2));
5773 
5774     HRegionInfo hri = new HRegionInfo(htd.getTableName(),
5775       HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY);
5776 
5777     ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
5778 
5779     // capture append() calls
5780     WAL wal = mock(WAL.class);
5781     when(rss.getWAL((HRegionInfo) any())).thenReturn(wal);
5782 
5783     // open a region first so that it can be closed later
5784     region = HRegion.openHRegion(hri, htd, rss.getWAL(hri),
5785       TEST_UTIL.getConfiguration(), rss, null);
5786 
5787     // close the region
5788     region.close(false);
5789 
5790     // 2 times, one for region open, the other close region
5791     verify(wal, times(2)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any(),
5792       editCaptor.capture(), (AtomicLong)any(), anyBoolean(), (List<Cell>)any());
5793 
5794     WALEdit edit = editCaptor.getAllValues().get(1);
5795     assertNotNull(edit);
5796     assertNotNull(edit.getCells());
5797     assertEquals(1, edit.getCells().size());
5798     RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
5799     assertNotNull(desc);
5800 
5801     LOG.info("RegionEventDescriptor from WAL: " + desc);
5802 
5803     assertEquals(RegionEventDescriptor.EventType.REGION_CLOSE, desc.getEventType());
5804     assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getName()));
5805     assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(),
5806       hri.getEncodedNameAsBytes()));
5807     assertTrue(desc.getLogSequenceNumber() > 0);
5808     assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
5809     assertEquals(2, desc.getStoresCount());
5810 
5811     StoreDescriptor store = desc.getStores(0);
5812     assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
5813     assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
5814     assertEquals(0, store.getStoreFileCount()); // no store files
5815 
5816     store = desc.getStores(1);
5817     assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
5818     assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
5819     assertEquals(0, store.getStoreFileCount()); // no store files
5820   }
5821 
5822   /**
5823    * Test RegionTooBusyException thrown when region is busy
5824    */
5825   @Test (timeout=24000)
5826   public void testRegionTooBusy() throws IOException {
5827     String method = "testRegionTooBusy";
5828     byte[] tableName = Bytes.toBytes(method);
5829     byte[] family = Bytes.toBytes("family");
5830     long defaultBusyWaitDuration = CONF.getLong("hbase.busy.wait.duration",
5831       HRegion.DEFAULT_BUSY_WAIT_DURATION);
5832     CONF.setLong("hbase.busy.wait.duration", 1000);
5833     region = initHRegion(tableName, method, CONF, family);
5834     final AtomicBoolean stopped = new AtomicBoolean(true);
5835     Thread t = new Thread(new Runnable() {
5836       @Override
5837       public void run() {
5838         try {
5839           region.lock.writeLock().lock();
5840           stopped.set(false);
5841           while (!stopped.get()) {
5842             Thread.sleep(100);
5843           }
5844         } catch (InterruptedException ie) {
5845         } finally {
5846           region.lock.writeLock().unlock();
5847         }
5848       }
5849     });
5850     t.start();
5851     Get get = new Get(row);
5852     try {
5853       while (stopped.get()) {
5854         Thread.sleep(100);
5855       }
5856       region.get(get);
5857       fail("Should throw RegionTooBusyException");
5858     } catch (InterruptedException ie) {
5859       fail("test interrupted");
5860     } catch (RegionTooBusyException e) {
5861       // Good, expected
5862     } finally {
5863       stopped.set(true);
5864       try {
5865         t.join();
5866       } catch (Throwable e) {
5867       }
5868 
5869       HRegion.closeHRegion(region);
5870       region = null;
5871       CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration);
5872     }
5873   }
5874 
5875   @Test
5876   public void testCellTTLs() throws IOException {
5877     IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
5878     EnvironmentEdgeManager.injectEdge(edge);
5879 
5880     final byte[] row = Bytes.toBytes("testRow");
5881     final byte[] q1 = Bytes.toBytes("q1");
5882     final byte[] q2 = Bytes.toBytes("q2");
5883     final byte[] q3 = Bytes.toBytes("q3");
5884     final byte[] q4 = Bytes.toBytes("q4");
5885 
5886     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCellTTLs"));
5887     HColumnDescriptor hcd = new HColumnDescriptor(fam1);
5888     hcd.setTimeToLive(10); // 10 seconds
5889     htd.addFamily(hcd);
5890 
5891     Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
5892     conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS);
5893 
5894     HRegion region = HRegion.createHRegion(new HRegionInfo(htd.getTableName(),
5895         HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY),
5896       TEST_UTIL.getDataTestDir(), conf, htd);
5897     assertNotNull(region);
5898     try {
5899       long now = EnvironmentEdgeManager.currentTime();
5900       // Add a cell that will expire in 5 seconds via cell TTL
5901       region.put(new Put(row).add(new KeyValue(row, fam1, q1, now,
5902         HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
5903           // TTL tags specify ts in milliseconds
5904           new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
5905       // Add a cell that will expire after 10 seconds via family setting
5906       region.put(new Put(row).add(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY));
5907       // Add a cell that will expire in 15 seconds via cell TTL
5908       region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1,
5909         HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
5910           // TTL tags specify ts in milliseconds
5911           new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
5912       // Add a cell that will expire in 20 seconds via family setting
5913       region.put(new Put(row).add(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY));
5914 
5915       // Flush so we are sure store scanning gets this right
5916       region.flushcache();
5917 
5918       // A query at time T+0 should return all cells
5919       Result r = region.get(new Get(row));
5920       assertNotNull(r.getValue(fam1, q1));
5921       assertNotNull(r.getValue(fam1, q2));
5922       assertNotNull(r.getValue(fam1, q3));
5923       assertNotNull(r.getValue(fam1, q4));
5924 
5925       // Increment time to T+5 seconds
5926       edge.incrementTime(5000);
5927 
5928       r = region.get(new Get(row));
5929       assertNull(r.getValue(fam1, q1));
5930       assertNotNull(r.getValue(fam1, q2));
5931       assertNotNull(r.getValue(fam1, q3));
5932       assertNotNull(r.getValue(fam1, q4));
5933 
5934       // Increment time to T+10 seconds
5935       edge.incrementTime(5000);
5936 
5937       r = region.get(new Get(row));
5938       assertNull(r.getValue(fam1, q1));
5939       assertNull(r.getValue(fam1, q2));
5940       assertNotNull(r.getValue(fam1, q3));
5941       assertNotNull(r.getValue(fam1, q4));
5942 
5943       // Increment time to T+15 seconds
5944       edge.incrementTime(5000);
5945 
5946       r = region.get(new Get(row));
5947       assertNull(r.getValue(fam1, q1));
5948       assertNull(r.getValue(fam1, q2));
5949       assertNull(r.getValue(fam1, q3));
5950       assertNotNull(r.getValue(fam1, q4));
5951 
5952       // Increment time to T+20 seconds
5953       edge.incrementTime(10000);
5954 
5955       r = region.get(new Get(row));
5956       assertNull(r.getValue(fam1, q1));
5957       assertNull(r.getValue(fam1, q2));
5958       assertNull(r.getValue(fam1, q3));
5959       assertNull(r.getValue(fam1, q4));
5960 
5961       // Fun with disappearing increments
5962 
5963       // Start at 1
5964       region.put(new Put(row).add(fam1, q1, Bytes.toBytes(1L)));
5965       r = region.get(new Get(row));
5966       byte[] val = r.getValue(fam1, q1);
5967       assertNotNull(val);
5968       assertEquals(Bytes.toLong(val), 1L);
5969 
5970       // Increment with a TTL of 5 seconds
5971       Increment incr = new Increment(row).addColumn(fam1, q1, 1L);
5972       incr.setTTL(5000);
5973       region.increment(incr); // 2
5974 
5975       // New value should be 2
5976       r = region.get(new Get(row));
5977       val = r.getValue(fam1, q1);
5978       assertNotNull(val);
5979       assertEquals(Bytes.toLong(val), 2L);
5980 
5981       // Increment time to T+25 seconds
5982       edge.incrementTime(5000);
5983 
5984       // Value should be back to 1
5985       r = region.get(new Get(row));
5986       val = r.getValue(fam1, q1);
5987       assertNotNull(val);
5988       assertEquals(Bytes.toLong(val), 1L);
5989 
5990       // Increment time to T+30 seconds
5991       edge.incrementTime(5000);
5992 
5993       // Original value written at T+20 should be gone now via family TTL
5994       r = region.get(new Get(row));
5995       assertNull(r.getValue(fam1, q1));
5996 
5997     } finally {
5998       HRegion.closeHRegion(region);
5999     }
6000   }
6001 
6002   private static HRegion initHRegion(byte[] tableName, String callingMethod,
6003       byte[]... families) throws IOException {
6004     return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
6005         families);
6006   }
6007 }