View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  
22  import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23  import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24  import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25  import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29  import static org.junit.Assert.assertArrayEquals;
30  import static org.junit.Assert.assertEquals;
31  import static org.junit.Assert.assertFalse;
32  import static org.junit.Assert.assertNotNull;
33  import static org.junit.Assert.assertNull;
34  import static org.junit.Assert.assertTrue;
35  import static org.junit.Assert.fail;
36  import static org.mockito.Matchers.any;
37  import static org.mockito.Matchers.anyBoolean;
38  import static org.mockito.Matchers.anyLong;
39  import static org.mockito.Matchers.eq;
40  import static org.mockito.Mockito.never;
41  import static org.mockito.Mockito.spy;
42  import static org.mockito.Mockito.times;
43  import static org.mockito.Mockito.verify;
44  import static org.mockito.Mockito.when;
45  
46  import java.io.IOException;
47  import java.io.InterruptedIOException;
48  import java.security.PrivilegedExceptionAction;
49  import java.util.ArrayList;
50  import java.util.Arrays;
51  import java.util.Collection;
52  import java.util.List;
53  import java.util.Map;
54  import java.util.NavigableMap;
55  import java.util.TreeMap;
56  import java.util.UUID;
57  import java.util.concurrent.Callable;
58  import java.util.concurrent.CountDownLatch;
59  import java.util.concurrent.ExecutorService;
60  import java.util.concurrent.Executors;
61  import java.util.concurrent.Future;
62  import java.util.concurrent.TimeUnit;
63  import java.util.concurrent.atomic.AtomicBoolean;
64  import java.util.concurrent.atomic.AtomicInteger;
65  import java.util.concurrent.atomic.AtomicLong;
66  import java.util.concurrent.atomic.AtomicReference;
67  
68  import org.apache.commons.logging.Log;
69  import org.apache.commons.logging.LogFactory;
70  import org.apache.hadoop.conf.Configuration;
71  import org.apache.hadoop.fs.FSDataOutputStream;
72  import org.apache.hadoop.fs.FileStatus;
73  import org.apache.hadoop.fs.FileSystem;
74  import org.apache.hadoop.fs.Path;
75  import org.apache.hadoop.hbase.Cell;
76  import org.apache.hadoop.hbase.CellComparator;
77  import org.apache.hadoop.hbase.CellUtil;
78  import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
79  import org.apache.hadoop.hbase.DroppedSnapshotException;
80  import org.apache.hadoop.hbase.HBaseConfiguration;
81  import org.apache.hadoop.hbase.HBaseTestCase;
82  import org.apache.hadoop.hbase.HBaseTestingUtility;
83  import org.apache.hadoop.hbase.HColumnDescriptor;
84  import org.apache.hadoop.hbase.HConstants;
85  import org.apache.hadoop.hbase.RegionTooBusyException;
86  import org.apache.hadoop.hbase.Tag;
87  import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
88  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
89  import org.apache.hadoop.hbase.HRegionInfo;
90  import org.apache.hadoop.hbase.HTableDescriptor;
91  import org.apache.hadoop.hbase.KeyValue;
92  import org.apache.hadoop.hbase.testclassification.MediumTests;
93  import org.apache.hadoop.hbase.MiniHBaseCluster;
94  import org.apache.hadoop.hbase.MultithreadedTestUtil;
95  import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
96  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
97  import org.apache.hadoop.hbase.NotServingRegionException;
98  import org.apache.hadoop.hbase.TableName;
99  import org.apache.hadoop.hbase.TagType;
100 import org.apache.hadoop.hbase.Waiter;
101 import org.apache.hadoop.hbase.client.Append;
102 import org.apache.hadoop.hbase.client.Delete;
103 import org.apache.hadoop.hbase.client.Durability;
104 import org.apache.hadoop.hbase.client.Get;
105 import org.apache.hadoop.hbase.client.HTable;
106 import org.apache.hadoop.hbase.client.Increment;
107 import org.apache.hadoop.hbase.client.Mutation;
108 import org.apache.hadoop.hbase.client.Put;
109 import org.apache.hadoop.hbase.client.Result;
110 import org.apache.hadoop.hbase.client.RowMutations;
111 import org.apache.hadoop.hbase.client.Scan;
112 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
113 import org.apache.hadoop.hbase.filter.BinaryComparator;
114 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
115 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
116 import org.apache.hadoop.hbase.filter.Filter;
117 import org.apache.hadoop.hbase.filter.FilterBase;
118 import org.apache.hadoop.hbase.filter.FilterList;
119 import org.apache.hadoop.hbase.filter.NullComparator;
120 import org.apache.hadoop.hbase.filter.PrefixFilter;
121 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
122 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
123 import org.apache.hadoop.hbase.io.hfile.HFile;
124 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
125 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
126 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
127 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
128 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
129 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
130 import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
131 import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
132 import org.apache.hadoop.hbase.regionserver.wal.FaultyHLog;
133 import org.apache.hadoop.hbase.regionserver.wal.HLog;
134 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
135 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
136 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
137 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
138 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
139 import org.apache.hadoop.hbase.security.User;
140 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
141 import org.apache.hadoop.hbase.util.Bytes;
142 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
143 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
144 import org.apache.hadoop.hbase.util.FSUtils;
145 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
146 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
147 import org.apache.hadoop.hbase.util.PairOfSameType;
148 import org.apache.hadoop.hbase.util.Threads;
149 import org.junit.After;
150 import org.junit.Assert;
151 import org.junit.Before;
152 import org.junit.Rule;
153 import org.junit.Test;
154 import org.junit.experimental.categories.Category;
155 import org.junit.rules.TestName;
156 import org.mockito.Mockito;
157 
158 import com.google.common.collect.ImmutableList;
159 import com.google.common.collect.Lists;
160 import com.google.protobuf.ByteString;
161 
162 /**
163  * Basic stand-alone testing of HRegion.  No clusters!
164  *
165  * A lot of the meta information for an HRegion now lives inside other HRegions
166  * or in the HBaseMaster, so only basic testing is possible.
167  */
168 @Category(MediumTests.class)
169 @SuppressWarnings("deprecation")
170 public class TestHRegion {
171   // Do not spin up clusters in here. If you need to spin up a cluster, do it
172   // over in TestHRegionOnCluster.
173   static final Log LOG = LogFactory.getLog(TestHRegion.class);
174   @Rule public TestName name = new TestName();
175 
176   private static final String COLUMN_FAMILY = "MyCF";
177   private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
178 
179   HRegion region = null;
180   // Do not run unit tests in parallel (? Why not?  It don't work?  Why not?  St.Ack)
181   private static HBaseTestingUtility TEST_UTIL;
182   public static Configuration CONF ;
183   private String dir;
184   private static FileSystem FILESYSTEM;
185   private final int MAX_VERSIONS = 2;
186 
187   // Test names
188   protected byte[] tableName;
189   protected String method;
190   protected final byte[] qual1 = Bytes.toBytes("qual1");
191   protected final byte[] qual2 = Bytes.toBytes("qual2");
192   protected final byte[] qual3 = Bytes.toBytes("qual3");
193   protected final byte[] value1 = Bytes.toBytes("value1");
194   protected final byte[] value2 = Bytes.toBytes("value2");
195   protected final byte[] row = Bytes.toBytes("rowA");
196   protected final byte[] row2 = Bytes.toBytes("rowB");
197 
198   protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
199       .getInstance(MetricsAssertHelper.class);
200 
201   @Before
202   public void setup() throws IOException {
203     TEST_UTIL = HBaseTestingUtility.createLocalHTU();
204     FILESYSTEM = TEST_UTIL.getTestFileSystem();
205     CONF = TEST_UTIL.getConfiguration();
206     dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
207     method = name.getMethodName();
208     tableName = Bytes.toBytes(name.getMethodName());
209   }
210 
211   @After
212   public void tearDown() throws Exception {
213     EnvironmentEdgeManagerTestHelper.reset();
214     LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
215     TEST_UTIL.cleanupTestDir();
216   }
217 
218   String getName() {
219     return name.getMethodName();
220   }
221 
222   /**
223    * Test for Bug 2 of HBASE-10466.
224    * "Bug 2: Conditions for the first flush of region close (so-called pre-flush) If memstoreSize
225    * is smaller than a certain value, or when region close starts a flush is ongoing, the first
226    * flush is skipped and only the second flush takes place. However, two flushes are required in
227    * case previous flush fails and leaves some data in snapshot. The bug could cause loss of data
228    * in current memstore. The fix is removing all conditions except abort check so we ensure 2
229    * flushes for region close."
230    * @throws IOException
231    */
232   @Test (timeout=60000)
233   public void testCloseCarryingSnapshot() throws IOException {
234     HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
235     Store store = region.getStore(COLUMN_FAMILY_BYTES);
236     // Get some random bytes.
237     byte [] value = Bytes.toBytes(name.getMethodName());
238     // Make a random put against our cf.
239     Put put = new Put(value);
240     put.add(COLUMN_FAMILY_BYTES, null, value);
241     // First put something in current memstore, which will be in snapshot after flusher.prepare()
242     region.put(put);
243     StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
244     storeFlushCtx.prepare();
245     // Second put something in current memstore
246     put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
247     region.put(put);
248     // Close with something in memstore and something in the snapshot.  Make sure all is cleared.
249     region.close();
250     assertEquals(0, region.getMemstoreSize().get());
251     HRegion.closeHRegion(region);
252   }
253 
254   /*
255    * This test is for verifying memstore snapshot size is correctly updated in case of rollback
256    * See HBASE-10845
257    */
258   @Test (timeout=60000)
259   public void testMemstoreSnapshotSize() throws IOException {
260     class MyFaultyHLog extends FaultyHLog {
261       StoreFlushContext storeFlushCtx;
262       public MyFaultyHLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
263           throws IOException {
264         super(fs, rootDir, logName, conf);
265       }
266       
267       void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
268         this.storeFlushCtx = storeFlushCtx;
269       }
270 
271       @Override
272       public void sync(long txid) throws IOException {
273         storeFlushCtx.prepare();
274         super.sync(txid);
275       }
276     }
277     
278     FileSystem fs = FileSystem.get(CONF);
279     Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
280     MyFaultyHLog faultyLog = new MyFaultyHLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
281     HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
282       CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES);
283     
284     Store store = region.getStore(COLUMN_FAMILY_BYTES);
285     // Get some random bytes.
286     byte [] value = Bytes.toBytes(name.getMethodName());
287     faultyLog.setStoreFlushCtx(store.createFlushContext(12345));
288     
289     Put put = new Put(value);
290     put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
291     faultyLog.setFailureType(FaultyHLog.FailureType.SYNC);
292 
293     boolean threwIOE = false;
294     try {
295       region.put(put);
296     } catch (IOException ioe) {
297       threwIOE = true;
298     } finally {
299       assertTrue("The regionserver should have thrown an exception", threwIOE);
300     }
301     long sz = store.getFlushableSize();
302     assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
303     HRegion.closeHRegion(region);
304   }
305 
306   /**
307    * Test for HBASE-14229: Flushing canceled by coprocessor still leads to memstoreSize set down
308    */
309   @Test
310   public void testMemstoreSizeWithFlushCanceling() throws IOException {
311     FileSystem fs = FileSystem.get(CONF);
312     Path rootDir = new Path(dir + "testMemstoreSizeWithFlushCanceling");
313     FaultyHLog hLog = new FaultyHLog(fs, rootDir, "testMemstoreSizeWithFlushCanceling", CONF);
314     HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
315         CONF, false, Durability.SYNC_WAL, hLog, COLUMN_FAMILY_BYTES);
316     Store store = region.getStore(COLUMN_FAMILY_BYTES);
317     assertEquals(0, region.getMemstoreSize().get());
318 
319     // Put some value and make sure flush could be completed normally
320     byte [] value = Bytes.toBytes(name.getMethodName());
321     Put put = new Put(value);
322     put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
323     region.put(put);
324     long onePutSize = region.getMemstoreSize().get();
325     assertTrue(onePutSize > 0);
326     region.flushcache();
327     assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize().get());
328     assertEquals("flushable size should be zero", 0, store.getFlushableSize());
329 
330     // save normalCPHost and replaced by mockedCPHost, which will cancel flush requests
331     RegionCoprocessorHost normalCPHost = region.getCoprocessorHost();
332     RegionCoprocessorHost mockedCPHost = Mockito.mock(RegionCoprocessorHost.class);
333     when(mockedCPHost.preFlush(any(HStore.class), any(InternalScanner.class))).thenReturn(null);
334     region.setCoprocessorHost(mockedCPHost);
335     region.put(put);
336     region.flushcache();
337     assertEquals("memstoreSize should NOT be zero", onePutSize, region.getMemstoreSize().get());
338     assertEquals("flushable size should NOT be zero", onePutSize, store.getFlushableSize());
339 
340     // set normalCPHost and flush again, the snapshot will be flushed
341     region.setCoprocessorHost(normalCPHost);
342     region.flushcache();
343     assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize().get());
344     assertEquals("flushable size should be zero", 0, store.getFlushableSize());
345     HRegion.closeHRegion(region);
346   }
347 
348   /**
349    * Test we do not lose data if we fail a flush and then close.
350    * Part of HBase-10466.  Tests the following from the issue description:
351    * "Bug 1: Wrong calculation of HRegion.memstoreSize: When a flush fails, data to be flushed is
352    * kept in each MemStore's snapshot and wait for next flush attempt to continue on it. But when
353    * the next flush succeeds, the counter of total memstore size in HRegion is always deduced by
354    * the sum of current memstore sizes instead of snapshots left from previous failed flush. This
355    * calculation is problematic that almost every time there is failed flush, HRegion.memstoreSize
356    * gets reduced by a wrong value. If region flush could not proceed for a couple cycles, the size
357    * in current memstore could be much larger than the snapshot. It's likely to drift memstoreSize
358    * much smaller than expected. In extreme case, if the error accumulates to even bigger than
359    * HRegion's memstore size limit, any further flush is skipped because flush does not do anything
360    * if memstoreSize is not larger than 0."
361    * @throws Exception
362    */
363   @Test (timeout=60000)
364   public void testFlushSizeAccounting() throws Exception {
365     final Configuration conf = HBaseConfiguration.create(CONF);
366     // Only retry once.
367     conf.setInt("hbase.hstore.flush.retries.number", 1);
368     final User user =
369       User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
370     // Inject our faulty LocalFileSystem
371     conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
372     user.runAs(new PrivilegedExceptionAction<Object>() {
373       @Override
374       public Object run() throws Exception {
375         // Make sure it worked (above is sensitive to caching details in hadoop core)
376         FileSystem fs = FileSystem.get(conf);
377         Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
378         FaultyFileSystem ffs = (FaultyFileSystem)fs;
379         HRegion region = null;
380         try {
381           // Initialize region
382           region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
383           long size = region.getMemstoreSize().get();
384           Assert.assertEquals(0, size);
385           // Put one item into memstore.  Measure the size of one item in memstore.
386           Put p1 = new Put(row);
387           p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
388           region.put(p1);
389           final long sizeOfOnePut = region.getMemstoreSize().get();
390           // Fail a flush which means the current memstore will hang out as memstore 'snapshot'.
391           try {
392             LOG.info("Flushing");
393             region.flushcache();
394             Assert.fail("Didn't bubble up IOE!");
395           } catch (DroppedSnapshotException dse) {
396             // What we are expecting
397             region.closing.set(false); // this is needed for the rest of the test to work
398           }
399           // Make it so all writes succeed from here on out
400           ffs.fault.set(false);
401           // Check sizes.  Should still be the one entry.
402           Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize().get());
403           // Now add two entries so that on this next flush that fails, we can see if we
404           // subtract the right amount, the snapshot size only.
405           Put p2 = new Put(row);
406           p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
407           p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
408           region.put(p2);
409           Assert.assertEquals(sizeOfOnePut * 3, region.getMemstoreSize().get());
410           // Do a successful flush.  It will clear the snapshot only.  Thats how flushes work.
411           // If already a snapshot, we clear it else we move the memstore to be snapshot and flush
412           // it
413           region.flushcache();
414           // Make sure our memory accounting is right.
415           Assert.assertEquals(sizeOfOnePut * 2, region.getMemstoreSize().get());
416         } finally {
417           HRegion.closeHRegion(region);
418         }
419         return null;
420       }
421     });
422     FileSystem.closeAllForUGI(user.getUGI());
423   }
424 
425   @Test
426   public void testCompactionAffectedByScanners() throws Exception {
427     byte[] family = Bytes.toBytes("family");
428     this.region = initHRegion(tableName, method, CONF, family);
429 
430     Put put = new Put(Bytes.toBytes("r1"));
431     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
432     region.put(put);
433     region.flushcache();
434 
435     Scan scan = new Scan();
436     scan.setMaxVersions(3);
437     // open the first scanner
438     RegionScanner scanner1 = region.getScanner(scan);
439 
440     Delete delete = new Delete(Bytes.toBytes("r1"));
441     region.delete(delete);
442     region.flushcache();
443 
444     // open the second scanner
445     RegionScanner scanner2 = region.getScanner(scan);
446 
447     List<Cell> results = new ArrayList<Cell>();
448 
449     System.out.println("Smallest read point:" + region.getSmallestReadPoint());
450 
451     // make a major compaction
452     region.compactStores(true);
453 
454     // open the third scanner
455     RegionScanner scanner3 = region.getScanner(scan);
456 
457     // get data from scanner 1, 2, 3 after major compaction
458     scanner1.next(results);
459     System.out.println(results);
460     assertEquals(1, results.size());
461 
462     results.clear();
463     scanner2.next(results);
464     System.out.println(results);
465     assertEquals(0, results.size());
466 
467     results.clear();
468     scanner3.next(results);
469     System.out.println(results);
470     assertEquals(0, results.size());
471   }
472 
473   @Test
474   public void testToShowNPEOnRegionScannerReseek() throws Exception {
475     byte[] family = Bytes.toBytes("family");
476     this.region = initHRegion(tableName, method, CONF, family);
477 
478     Put put = new Put(Bytes.toBytes("r1"));
479     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
480     region.put(put);
481     put = new Put(Bytes.toBytes("r2"));
482     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
483     region.put(put);
484     region.flushcache();
485 
486     Scan scan = new Scan();
487     scan.setMaxVersions(3);
488     // open the first scanner
489     RegionScanner scanner1 = region.getScanner(scan);
490 
491     System.out.println("Smallest read point:" + region.getSmallestReadPoint());
492 
493     region.compactStores(true);
494 
495     scanner1.reseek(Bytes.toBytes("r2"));
496     List<Cell> results = new ArrayList<Cell>();
497     scanner1.next(results);
498     Cell keyValue = results.get(0);
499     Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
500     scanner1.close();
501   }
502 
503   @Test
504   public void testSkipRecoveredEditsReplay() throws Exception {
505     String method = "testSkipRecoveredEditsReplay";
506     TableName tableName = TableName.valueOf(method);
507     byte[] family = Bytes.toBytes("family");
508     this.region = initHRegion(tableName, method, CONF, family);
509     try {
510       Path regiondir = region.getRegionFileSystem().getRegionDir();
511       FileSystem fs = region.getRegionFileSystem().getFileSystem();
512       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
513 
514       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
515 
516       long maxSeqId = 1050;
517       long minSeqId = 1000;
518 
519       for (long i = minSeqId; i <= maxSeqId; i += 10) {
520         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
521         fs.create(recoveredEdits);
522         HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
523 
524         long time = System.nanoTime();
525         WALEdit edit = new WALEdit();
526         edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
527             .toBytes(i)));
528         writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
529             HConstants.DEFAULT_CLUSTER_ID), edit));
530 
531         writer.close();
532       }
533       MonitoredTask status = TaskMonitor.get().createStatus(method);
534       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
535       for (Store store : region.getStores().values()) {
536         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
537       }
538       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
539       assertEquals(maxSeqId, seqId);
540       Get get = new Get(row);
541       Result result = region.get(get);
542       for (long i = minSeqId; i <= maxSeqId; i += 10) {
543         List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
544         assertEquals(1, kvs.size());
545         assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
546       }
547     } finally {
548       HRegion.closeHRegion(this.region);
549       this.region = null;
550     }
551   }
552 
553   @Test
554   public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
555     String method = "testSkipRecoveredEditsReplaySomeIgnored";
556     TableName tableName = TableName.valueOf(method);
557     byte[] family = Bytes.toBytes("family");
558     this.region = initHRegion(tableName, method, CONF, family);
559     try {
560       Path regiondir = region.getRegionFileSystem().getRegionDir();
561       FileSystem fs = region.getRegionFileSystem().getFileSystem();
562       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
563 
564       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
565 
566       long maxSeqId = 1050;
567       long minSeqId = 1000;
568 
569       for (long i = minSeqId; i <= maxSeqId; i += 10) {
570         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
571         fs.create(recoveredEdits);
572         HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
573 
574         long time = System.nanoTime();
575         WALEdit edit = new WALEdit();
576         edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
577             .toBytes(i)));
578         writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
579             HConstants.DEFAULT_CLUSTER_ID), edit));
580 
581         writer.close();
582       }
583       long recoverSeqId = 1030;
584       MonitoredTask status = TaskMonitor.get().createStatus(method);
585       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
586       for (Store store : region.getStores().values()) {
587         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
588       }
589       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
590       assertEquals(maxSeqId, seqId);
591       Get get = new Get(row);
592       Result result = region.get(get);
593       for (long i = minSeqId; i <= maxSeqId; i += 10) {
594         List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
595         if (i < recoverSeqId) {
596           assertEquals(0, kvs.size());
597         } else {
598           assertEquals(1, kvs.size());
599           assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
600         }
601       }
602     } finally {
603       HRegion.closeHRegion(this.region);
604       this.region = null;
605     }
606   }
607 
608   @Test
609   public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
610     byte[] family = Bytes.toBytes("family");
611     this.region = initHRegion(tableName, method, CONF, family);
612     try {
613       Path regiondir = region.getRegionFileSystem().getRegionDir();
614       FileSystem fs = region.getRegionFileSystem().getFileSystem();
615 
616       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
617       for (int i = 1000; i < 1050; i += 10) {
618         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
619         FSDataOutputStream dos = fs.create(recoveredEdits);
620         dos.writeInt(i);
621         dos.close();
622       }
623       long minSeqId = 2000;
624       Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
625       FSDataOutputStream dos = fs.create(recoveredEdits);
626       dos.close();
627 
628       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
629       for (Store store : region.getStores().values()) {
630         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
631       }
632       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
633       assertEquals(minSeqId, seqId);
634     } finally {
635       HRegion.closeHRegion(this.region);
636       this.region = null;
637     }
638   }
639 
640   @Test
641   public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
642     String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
643     TableName tableName = TableName.valueOf(method);
644     byte[] family = Bytes.toBytes("family");
645     this.region = initHRegion(tableName, method, CONF, family);
646     try {
647       Path regiondir = region.getRegionFileSystem().getRegionDir();
648       FileSystem fs = region.getRegionFileSystem().getFileSystem();
649       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
650 
651       assertEquals(0, region.getStoreFileList(
652         region.getStores().keySet().toArray(new byte[0][])).size());
653 
654       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
655 
656       long maxSeqId = 1050;
657       long minSeqId = 1000;
658 
659       for (long i = minSeqId; i <= maxSeqId; i += 10) {
660         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
661         fs.create(recoveredEdits);
662         HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
663 
664         long time = System.nanoTime();
665         WALEdit edit = null;
666         if (i == maxSeqId) {
667           edit = WALEdit.createCompaction(region.getRegionInfo(),
668           CompactionDescriptor.newBuilder()
669           .setTableName(ByteString.copyFrom(tableName.getName()))
670           .setFamilyName(ByteString.copyFrom(regionName))
671           .setEncodedRegionName(ByteString.copyFrom(regionName))
672           .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
673           .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName()))
674           .build());
675         } else {
676           edit = new WALEdit();
677           edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
678             .toBytes(i)));
679         }
680         writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
681             HConstants.DEFAULT_CLUSTER_ID), edit));
682         writer.close();
683       }
684 
685       long recoverSeqId = 1030;
686       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
687       MonitoredTask status = TaskMonitor.get().createStatus(method);
688       for (Store store : region.getStores().values()) {
689         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
690       }
691       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
692       assertEquals(maxSeqId, seqId);
693 
694       // assert that the files are flushed
695       assertEquals(1, region.getStoreFileList(
696         region.getStores().keySet().toArray(new byte[0][])).size());
697 
698     } finally {
699       HRegion.closeHRegion(this.region);
700       this.region = null;
701     }  }
702 
703   @Test
704   public void testRecoveredEditsReplayCompaction() throws Exception {
705     testRecoveredEditsReplayCompaction(false);
706     testRecoveredEditsReplayCompaction(true);
707   }
708   public void testRecoveredEditsReplayCompaction(boolean mismatchedRegionName) throws Exception {
709     String method = name.getMethodName();
710     TableName tableName = TableName.valueOf(method);
711     byte[] family = Bytes.toBytes("family");
712     this.region = initHRegion(tableName, method, CONF, family);
713     try {
714       Path regiondir = region.getRegionFileSystem().getRegionDir();
715       FileSystem fs = region.getRegionFileSystem().getFileSystem();
716       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
717 
718       long maxSeqId = 3;
719       long minSeqId = 0;
720 
721       for (long i = minSeqId; i < maxSeqId; i++) {
722         Put put = new Put(Bytes.toBytes(i));
723         put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
724         region.put(put);
725         region.flushcache();
726       }
727 
728       // this will create a region with 3 files
729       assertEquals(3, region.getStore(family).getStorefilesCount());
730       List<Path> storeFiles = new ArrayList<Path>(3);
731       for (StoreFile sf : region.getStore(family).getStorefiles()) {
732         storeFiles.add(sf.getPath());
733       }
734 
735       // disable compaction completion
736       CONF.setBoolean("hbase.hstore.compaction.complete", false);
737       region.compactStores();
738 
739       // ensure that nothing changed
740       assertEquals(3, region.getStore(family).getStorefilesCount());
741 
742       // now find the compacted file, and manually add it to the recovered edits
743       Path tmpDir = region.getRegionFileSystem().getTempDir();
744       FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
745       String errorMsg = "Expected to find 1 file in the region temp directory "
746           + "from the compaction, could not find any";
747       assertNotNull(errorMsg, files);
748       assertEquals(errorMsg, 1, files.length);
749       // move the file inside region dir
750       Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
751           files[0].getPath());
752 
753       byte[] encodedNameAsBytes = this.region.getRegionInfo().getEncodedNameAsBytes();
754       byte[] fakeEncodedNameAsBytes = new byte [encodedNameAsBytes.length];
755       for (int i=0; i < encodedNameAsBytes.length; i++) {
756         // Mix the byte array to have a new encodedName
757         fakeEncodedNameAsBytes[i] = (byte) (encodedNameAsBytes[i] + 1);
758       }
759 
760       CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
761         .getRegionInfo(), mismatchedRegionName ? fakeEncodedNameAsBytes : null, family,
762             storeFiles, Lists.newArrayList(newFile),
763             region.getRegionFileSystem().getStoreDir(Bytes.toString(family)));
764 
765       HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
766           this.region.getRegionInfo(), compactionDescriptor, new AtomicLong(1));
767 
768       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
769 
770       Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
771       fs.create(recoveredEdits);
772       HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
773 
774       long time = System.nanoTime();
775 
776       writer.append(new HLog.Entry(new HLogKey(regionName, tableName, 10, time,
777           HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(),
778           compactionDescriptor)));
779       writer.close();
780 
781       // close the region now, and reopen again
782       region.getTableDesc();
783       region.getRegionInfo();
784       region.close();
785       try {
786         region = HRegion.openHRegion(region, null);
787       } catch (WrongRegionException wre) {
788         fail("Matching encoded region name should not have produced WrongRegionException");
789       }
790 
791       // now check whether we have only one store file, the compacted one
792       Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
793       for (StoreFile sf : sfs) {
794         LOG.info(sf.getPath());
795       }
796       if (!mismatchedRegionName) {
797         assertEquals(1, region.getStore(family).getStorefilesCount());
798       }
799       files = FSUtils.listStatus(fs, tmpDir);
800       assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
801 
802       for (long i = minSeqId; i < maxSeqId; i++) {
803         Get get = new Get(Bytes.toBytes(i));
804         Result result = region.get(get);
805         byte[] value = result.getValue(family, Bytes.toBytes(i));
806         assertArrayEquals(Bytes.toBytes(i), value);
807       }
808     } finally {
809       HRegion.closeHRegion(this.region);
810       this.region = null;
811     }
812   }
813 
814   @Test
815   public void testGetWhileRegionClose() throws IOException {
816     TableName tableName = TableName.valueOf(name.getMethodName());
817     Configuration hc = initSplit();
818     int numRows = 100;
819     byte[][] families = { fam1, fam2, fam3 };
820 
821     // Setting up region
822     String method = name.getMethodName();
823     this.region = initHRegion(tableName, method, hc, families);
824     try {
825       // Put data in region
826       final int startRow = 100;
827       putData(startRow, numRows, qual1, families);
828       putData(startRow, numRows, qual2, families);
829       putData(startRow, numRows, qual3, families);
830       final AtomicBoolean done = new AtomicBoolean(false);
831       final AtomicInteger gets = new AtomicInteger(0);
832       GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
833       try {
834         // Set ten threads running concurrently getting from the region.
835         for (int i = 0; i < threads.length / 2; i++) {
836           threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
837           threads[i].setDaemon(true);
838           threads[i].start();
839         }
840         // Artificially make the condition by setting closing flag explicitly.
841         // I can't make the issue happen with a call to region.close().
842         this.region.closing.set(true);
843         for (int i = threads.length / 2; i < threads.length; i++) {
844           threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
845           threads[i].setDaemon(true);
846           threads[i].start();
847         }
848       } finally {
849         if (this.region != null) {
850           HRegion.closeHRegion(this.region);
851         }
852       }
853       done.set(true);
854       for (GetTillDoneOrException t : threads) {
855         try {
856           t.join();
857         } catch (InterruptedException e) {
858           e.printStackTrace();
859         }
860         if (t.e != null) {
861           LOG.info("Exception=" + t.e);
862           assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
863         }
864       }
865     } finally {
866       HRegion.closeHRegion(this.region);
867       this.region = null;
868     }
869   }
870 
871   /*
872    * Thread that does get on single row until 'done' flag is flipped. If an
873    * exception causes us to fail, it records it.
874    */
875   class GetTillDoneOrException extends Thread {
876     private final Get g;
877     private final AtomicBoolean done;
878     private final AtomicInteger count;
879     private Exception e;
880 
881     GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
882       super("getter." + i);
883       this.g = new Get(r);
884       this.done = d;
885       this.count = c;
886     }
887 
888     @Override
889     public void run() {
890       while (!this.done.get()) {
891         try {
892           assertTrue(region.get(g).size() > 0);
893           this.count.incrementAndGet();
894         } catch (Exception e) {
895           this.e = e;
896           break;
897         }
898       }
899     }
900   }
901 
902   /*
903    * An involved filter test. Has multiple column families and deletes in mix.
904    */
905   @Test
906   public void testWeirdCacheBehaviour() throws Exception {
907     byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
908     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
909         Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
910     this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
911     try {
912       String value = "this is the value";
913       String value2 = "this is some other value";
914       String keyPrefix1 = "prefix1";
915       String keyPrefix2 = "prefix2";
916       String keyPrefix3 = "prefix3";
917       putRows(this.region, 3, value, keyPrefix1);
918       putRows(this.region, 3, value, keyPrefix2);
919       putRows(this.region, 3, value, keyPrefix3);
920       putRows(this.region, 3, value2, keyPrefix1);
921       putRows(this.region, 3, value2, keyPrefix2);
922       putRows(this.region, 3, value2, keyPrefix3);
923       System.out.println("Checking values for key: " + keyPrefix1);
924       assertEquals("Got back incorrect number of rows from scan", 3,
925           getNumberOfRows(keyPrefix1, value2, this.region));
926       System.out.println("Checking values for key: " + keyPrefix2);
927       assertEquals("Got back incorrect number of rows from scan", 3,
928           getNumberOfRows(keyPrefix2, value2, this.region));
929       System.out.println("Checking values for key: " + keyPrefix3);
930       assertEquals("Got back incorrect number of rows from scan", 3,
931           getNumberOfRows(keyPrefix3, value2, this.region));
932       deleteColumns(this.region, value2, keyPrefix1);
933       deleteColumns(this.region, value2, keyPrefix2);
934       deleteColumns(this.region, value2, keyPrefix3);
935       System.out.println("Starting important checks.....");
936       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
937           getNumberOfRows(keyPrefix1, value2, this.region));
938       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
939           getNumberOfRows(keyPrefix2, value2, this.region));
940       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
941           getNumberOfRows(keyPrefix3, value2, this.region));
942     } finally {
943       HRegion.closeHRegion(this.region);
944       this.region = null;
945     }
946   }
947 
948   @Test
949   public void testAppendWithReadOnlyTable() throws Exception {
950     byte[] TABLE = Bytes.toBytes("readOnlyTable");
951     this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
952     boolean exceptionCaught = false;
953     Append append = new Append(Bytes.toBytes("somerow"));
954     append.setDurability(Durability.SKIP_WAL);
955     append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
956         Bytes.toBytes("somevalue"));
957     try {
958       region.append(append);
959     } catch (IOException e) {
960       exceptionCaught = true;
961     } finally {
962       HRegion.closeHRegion(this.region);
963       this.region = null;
964     }
965     assertTrue(exceptionCaught == true);
966   }
967 
968   @Test
969   public void testIncrWithReadOnlyTable() throws Exception {
970     byte[] TABLE = Bytes.toBytes("readOnlyTable");
971     this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
972     boolean exceptionCaught = false;
973     Increment inc = new Increment(Bytes.toBytes("somerow"));
974     inc.setDurability(Durability.SKIP_WAL);
975     inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
976     try {
977       region.increment(inc);
978     } catch (IOException e) {
979       exceptionCaught = true;
980     } finally {
981       HRegion.closeHRegion(this.region);
982       this.region = null;
983     }
984     assertTrue(exceptionCaught == true);
985   }
986 
987   private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
988     InternalScanner scanner = buildScanner(keyPrefix, value, r);
989     int count = 0;
990     boolean more = false;
991     List<Cell> results = new ArrayList<Cell>();
992     do {
993       more = scanner.next(results);
994       if (results != null && !results.isEmpty())
995         count++;
996       else
997         break;
998       Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
999       delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
1000       r.delete(delete);
1001       results.clear();
1002     } while (more);
1003     assertEquals("Did not perform correct number of deletes", 3, count);
1004   }
1005 
1006   private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
1007     InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
1008     int numberOfResults = 0;
1009     List<Cell> results = new ArrayList<Cell>();
1010     boolean more = false;
1011     do {
1012       more = resultScanner.next(results);
1013       if (results != null && !results.isEmpty())
1014         numberOfResults++;
1015       else
1016         break;
1017       for (Cell kv : results) {
1018         System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
1019       }
1020       results.clear();
1021     } while (more);
1022     return numberOfResults;
1023   }
1024 
1025   private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
1026       throws IOException {
1027     // Defaults FilterList.Operator.MUST_PASS_ALL.
1028     FilterList allFilters = new FilterList();
1029     allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
1030     // Only return rows where this column value exists in the row.
1031     SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
1032         Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
1033     filter.setFilterIfMissing(true);
1034     allFilters.addFilter(filter);
1035     Scan scan = new Scan();
1036     scan.addFamily(Bytes.toBytes("trans-blob"));
1037     scan.addFamily(Bytes.toBytes("trans-type"));
1038     scan.addFamily(Bytes.toBytes("trans-date"));
1039     scan.addFamily(Bytes.toBytes("trans-tags"));
1040     scan.addFamily(Bytes.toBytes("trans-group"));
1041     scan.setFilter(allFilters);
1042     return r.getScanner(scan);
1043   }
1044 
1045   private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
1046     for (int i = 0; i < numRows; i++) {
1047       String row = key + "_" + i/* UUID.randomUUID().toString() */;
1048       System.out.println(String.format("Saving row: %s, with value %s", row, value));
1049       Put put = new Put(Bytes.toBytes(row));
1050       put.setDurability(Durability.SKIP_WAL);
1051       put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
1052       put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
1053       put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
1054       put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
1055       put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
1056       r.put(put);
1057     }
1058   }
1059 
1060   @Test
1061   public void testFamilyWithAndWithoutColon() throws Exception {
1062     byte[] b = Bytes.toBytes(getName());
1063     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1064     this.region = initHRegion(b, getName(), CONF, cf);
1065     try {
1066       Put p = new Put(b);
1067       byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
1068       p.add(cfwithcolon, cfwithcolon, cfwithcolon);
1069       boolean exception = false;
1070       try {
1071         this.region.put(p);
1072       } catch (NoSuchColumnFamilyException e) {
1073         exception = true;
1074       }
1075       assertTrue(exception);
1076     } finally {
1077       HRegion.closeHRegion(this.region);
1078       this.region = null;
1079     }
1080   }
1081 
1082   @Test
1083   public void testBatchPut_whileNoRowLocksHeld() throws IOException {
1084     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1085     byte[] qual = Bytes.toBytes("qual");
1086     byte[] val = Bytes.toBytes("val");
1087     this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1088     MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1089     try {
1090       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1091       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1092 
1093       LOG.info("First a batch put with all valid puts");
1094       final Put[] puts = new Put[10];
1095       for (int i = 0; i < 10; i++) {
1096         puts[i] = new Put(Bytes.toBytes("row_" + i));
1097         puts[i].add(cf, qual, val);
1098       }
1099 
1100       OperationStatus[] codes = this.region.batchMutate(puts);
1101       assertEquals(10, codes.length);
1102       for (int i = 0; i < 10; i++) {
1103         assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1104       }
1105       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1106 
1107       LOG.info("Next a batch put with one invalid family");
1108       puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1109       codes = this.region.batchMutate(puts);
1110       assertEquals(10, codes.length);
1111       for (int i = 0; i < 10; i++) {
1112         assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1113             codes[i].getOperationStatusCode());
1114       }
1115 
1116       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1117     } finally {
1118       HRegion.closeHRegion(this.region);
1119       this.region = null;
1120     }
1121   }
1122 
1123   @Test
1124   public void testBatchPut_whileMultipleRowLocksHeld() throws Exception {
1125     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1126     byte[] qual = Bytes.toBytes("qual");
1127     byte[] val = Bytes.toBytes("val");
1128     this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1129     MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1130     try {
1131       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1132       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1133 
1134       final Put[] puts = new Put[10];
1135       for (int i = 0; i < 10; i++) {
1136         puts[i] = new Put(Bytes.toBytes("row_" + i));
1137         puts[i].add(cf, qual, val);
1138       }
1139       puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1140 
1141       LOG.info("batchPut will have to break into four batches to avoid row locks");
1142       RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2"));
1143       RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_4"));
1144       RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_6"));
1145 
1146       MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1147       final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
1148       TestThread putter = new TestThread(ctx) {
1149         @Override
1150         public void doWork() throws IOException {
1151           retFromThread.set(region.batchMutate(puts));
1152         }
1153       };
1154       LOG.info("...starting put thread while holding locks");
1155       ctx.addThread(putter);
1156       ctx.startThreads();
1157 
1158       LOG.info("...waiting for put thread to sync 1st time");
1159       waitForCounter(source, "syncTimeNumOps", syncs + 1);
1160 
1161       // Now attempt to close the region from another thread.  Prior to HBASE-12565
1162       // this would cause the in-progress batchMutate operation to to fail with
1163       // exception because it use to release and re-acquire the close-guard lock
1164       // between batches.  Caller then didn't get status indicating which writes succeeded.
1165       // We now expect this thread to block until the batchMutate call finishes.
1166       Thread regionCloseThread = new Thread() {
1167         @Override
1168         public void run() {
1169           try {
1170             HRegion.closeHRegion(region);
1171           } catch (IOException e) {
1172             throw new RuntimeException(e);
1173           }
1174         }
1175       };
1176       regionCloseThread.start();
1177 
1178       LOG.info("...releasing row lock 1, which should let put thread continue");
1179       rowLock1.release();
1180 
1181       LOG.info("...waiting for put thread to sync 2nd time");
1182       waitForCounter(source, "syncTimeNumOps", syncs + 2);
1183 
1184       LOG.info("...releasing row lock 2, which should let put thread continue");
1185       rowLock2.release();
1186 
1187       LOG.info("...waiting for put thread to sync 3rd time");
1188       waitForCounter(source, "syncTimeNumOps", syncs + 3);
1189 
1190       LOG.info("...releasing row lock 3, which should let put thread continue");
1191       rowLock3.release();
1192 
1193       LOG.info("...waiting for put thread to sync 4th time");
1194       waitForCounter(source, "syncTimeNumOps", syncs + 4);
1195 
1196       LOG.info("...joining on put thread");
1197       ctx.stop();
1198       regionCloseThread.join();
1199 
1200       OperationStatus[] codes = retFromThread.get();
1201       for (int i = 0; i < codes.length; i++) {
1202         assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1203             codes[i].getOperationStatusCode());
1204       }
1205     } finally {
1206       HRegion.closeHRegion(this.region);
1207       this.region = null;
1208     }
1209   }
1210 
1211   private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount)
1212       throws InterruptedException {
1213     long startWait = System.currentTimeMillis();
1214     long currentCount;
1215     while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) {
1216       Thread.sleep(100);
1217       if (System.currentTimeMillis() - startWait > 10000) {
1218         fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName,
1219           expectedCount, currentCount));
1220       }
1221     }
1222   }
1223 
1224 
1225   @Test
1226   public void testBatchPutWithTsSlop() throws Exception {
1227     byte[] b = Bytes.toBytes(getName());
1228     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1229     byte[] qual = Bytes.toBytes("qual");
1230     byte[] val = Bytes.toBytes("val");
1231 
1232     // add data with a timestamp that is too recent for range. Ensure assert
1233     CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1234     this.region = initHRegion(b, getName(), CONF, cf);
1235 
1236     try {
1237       MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1238       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1239       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1240 
1241       final Put[] puts = new Put[10];
1242       for (int i = 0; i < 10; i++) {
1243         puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
1244         puts[i].add(cf, qual, val);
1245       }
1246 
1247       OperationStatus[] codes = this.region.batchMutate(puts);
1248       assertEquals(10, codes.length);
1249       for (int i = 0; i < 10; i++) {
1250         assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1251       }
1252       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1253 
1254     } finally {
1255       HRegion.closeHRegion(this.region);
1256       this.region = null;
1257     }
1258 
1259   }
1260 
1261   // ////////////////////////////////////////////////////////////////////////////
1262   // checkAndMutate tests
1263   // ////////////////////////////////////////////////////////////////////////////
1264   @Test
1265   public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1266     byte[] row1 = Bytes.toBytes("row1");
1267     byte[] fam1 = Bytes.toBytes("fam1");
1268     byte[] qf1 = Bytes.toBytes("qualifier");
1269     byte[] emptyVal = new byte[] {};
1270     byte[] val1 = Bytes.toBytes("value1");
1271     byte[] val2 = Bytes.toBytes("value2");
1272 
1273     // Setting up region
1274     String method = this.getName();
1275     this.region = initHRegion(tableName, method, CONF, fam1);
1276     try {
1277       // Putting empty data in key
1278       Put put = new Put(row1);
1279       put.add(fam1, qf1, emptyVal);
1280 
1281       // checkAndPut with empty value
1282       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1283           emptyVal), put, true);
1284       assertTrue(res);
1285 
1286       // Putting data in key
1287       put = new Put(row1);
1288       put.add(fam1, qf1, val1);
1289 
1290       // checkAndPut with correct value
1291       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1292           put, true);
1293       assertTrue(res);
1294 
1295       // not empty anymore
1296       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1297           put, true);
1298       assertFalse(res);
1299 
1300       Delete delete = new Delete(row1);
1301       delete.deleteColumn(fam1, qf1);
1302       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1303           delete, true);
1304       assertFalse(res);
1305 
1306       put = new Put(row1);
1307       put.add(fam1, qf1, val2);
1308       // checkAndPut with correct value
1309       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1310           put, true);
1311       assertTrue(res);
1312 
1313       // checkAndDelete with correct value
1314       delete = new Delete(row1);
1315       delete.deleteColumn(fam1, qf1);
1316       delete.deleteColumn(fam1, qf1);
1317       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1318           delete, true);
1319       assertTrue(res);
1320 
1321       delete = new Delete(row1);
1322       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1323           delete, true);
1324       assertTrue(res);
1325 
1326       // checkAndPut looking for a null value
1327       put = new Put(row1);
1328       put.add(fam1, qf1, val1);
1329 
1330       res = region
1331           .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
1332       assertTrue(res);
1333     } finally {
1334       HRegion.closeHRegion(this.region);
1335       this.region = null;
1336     }
1337   }
1338 
1339   @Test
1340   public void testCheckAndMutate_WithWrongValue() throws IOException {
1341     byte[] row1 = Bytes.toBytes("row1");
1342     byte[] fam1 = Bytes.toBytes("fam1");
1343     byte[] qf1 = Bytes.toBytes("qualifier");
1344     byte[] val1 = Bytes.toBytes("value1");
1345     byte[] val2 = Bytes.toBytes("value2");
1346 
1347     // Setting up region
1348     String method = this.getName();
1349     this.region = initHRegion(tableName, method, CONF, fam1);
1350     try {
1351       // Putting data in key
1352       Put put = new Put(row1);
1353       put.add(fam1, qf1, val1);
1354       region.put(put);
1355 
1356       // checkAndPut with wrong value
1357       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1358           val2), put, true);
1359       assertEquals(false, res);
1360 
1361       // checkAndDelete with wrong value
1362       Delete delete = new Delete(row1);
1363       delete.deleteFamily(fam1);
1364       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1365           put, true);
1366       assertEquals(false, res);
1367     } finally {
1368       HRegion.closeHRegion(this.region);
1369       this.region = null;
1370     }
1371   }
1372 
1373   @Test
1374   public void testCheckAndMutate_WithCorrectValue() throws IOException {
1375     byte[] row1 = Bytes.toBytes("row1");
1376     byte[] fam1 = Bytes.toBytes("fam1");
1377     byte[] qf1 = Bytes.toBytes("qualifier");
1378     byte[] val1 = Bytes.toBytes("value1");
1379 
1380     // Setting up region
1381     String method = this.getName();
1382     this.region = initHRegion(tableName, method, CONF, fam1);
1383     try {
1384       // Putting data in key
1385       Put put = new Put(row1);
1386       put.add(fam1, qf1, val1);
1387       region.put(put);
1388 
1389       // checkAndPut with correct value
1390       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1391           val1), put, true);
1392       assertEquals(true, res);
1393 
1394       // checkAndDelete with correct value
1395       Delete delete = new Delete(row1);
1396       delete.deleteColumn(fam1, qf1);
1397       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1398           delete, true);
1399       assertEquals(true, res);
1400     } finally {
1401       HRegion.closeHRegion(this.region);
1402       this.region = null;
1403     }
1404   }
1405 
1406   @Test
1407   public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1408     byte[] row1 = Bytes.toBytes("row1");
1409     byte[] fam1 = Bytes.toBytes("fam1");
1410     byte[] qf1 = Bytes.toBytes("qualifier");
1411     byte[] val1 = Bytes.toBytes("value1");
1412     byte[] val2 = Bytes.toBytes("value2");
1413     byte[] val3 = Bytes.toBytes("value3");
1414     byte[] val4 = Bytes.toBytes("value4");
1415 
1416     // Setting up region
1417     String method = this.getName();
1418     this.region = initHRegion(tableName, method, CONF, fam1);
1419     try {
1420       // Putting val3 in key
1421       Put put = new Put(row1);
1422       put.add(fam1, qf1, val3);
1423       region.put(put);
1424 
1425       // Test CompareOp.LESS: original = val3, compare with val3, fail
1426       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1427           new BinaryComparator(val3), put, true);
1428       assertEquals(false, res);
1429 
1430       // Test CompareOp.LESS: original = val3, compare with val4, fail
1431       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1432           new BinaryComparator(val4), put, true);
1433       assertEquals(false, res);
1434 
1435       // Test CompareOp.LESS: original = val3, compare with val2,
1436       // succeed (now value = val2)
1437       put = new Put(row1);
1438       put.add(fam1, qf1, val2);
1439       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1440           new BinaryComparator(val2), put, true);
1441       assertEquals(true, res);
1442 
1443       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail
1444       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1445           new BinaryComparator(val3), put, true);
1446       assertEquals(false, res);
1447 
1448       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2,
1449       // succeed (value still = val2)
1450       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1451           new BinaryComparator(val2), put, true);
1452       assertEquals(true, res);
1453 
1454       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1,
1455       // succeed (now value = val3)
1456       put = new Put(row1);
1457       put.add(fam1, qf1, val3);
1458       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1459           new BinaryComparator(val1), put, true);
1460       assertEquals(true, res);
1461 
1462       // Test CompareOp.GREATER: original = val3, compare with val3, fail
1463       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1464           new BinaryComparator(val3), put, true);
1465       assertEquals(false, res);
1466 
1467       // Test CompareOp.GREATER: original = val3, compare with val2, fail
1468       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1469           new BinaryComparator(val2), put, true);
1470       assertEquals(false, res);
1471 
1472       // Test CompareOp.GREATER: original = val3, compare with val4,
1473       // succeed (now value = val2)
1474       put = new Put(row1);
1475       put.add(fam1, qf1, val2);
1476       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1477           new BinaryComparator(val4), put, true);
1478       assertEquals(true, res);
1479 
1480       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail
1481       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1482           new BinaryComparator(val1), put, true);
1483       assertEquals(false, res);
1484 
1485       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2,
1486       // succeed (value still = val2)
1487       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1488           new BinaryComparator(val2), put, true);
1489       assertEquals(true, res);
1490 
1491       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed
1492       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1493           new BinaryComparator(val3), put, true);
1494       assertEquals(true, res);
1495     } finally {
1496       HRegion.closeHRegion(this.region);
1497       this.region = null;
1498     }
1499   }
1500 
1501   @Test
1502   public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1503     byte[] row1 = Bytes.toBytes("row1");
1504     byte[] fam1 = Bytes.toBytes("fam1");
1505     byte[] fam2 = Bytes.toBytes("fam2");
1506     byte[] qf1 = Bytes.toBytes("qualifier");
1507     byte[] val1 = Bytes.toBytes("value1");
1508     byte[] val2 = Bytes.toBytes("value2");
1509 
1510     byte[][] families = { fam1, fam2 };
1511 
1512     // Setting up region
1513     String method = this.getName();
1514     this.region = initHRegion(tableName, method, CONF, families);
1515     try {
1516       // Putting data in the key to check
1517       Put put = new Put(row1);
1518       put.add(fam1, qf1, val1);
1519       region.put(put);
1520 
1521       // Creating put to add
1522       long ts = System.currentTimeMillis();
1523       KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1524       put = new Put(row1);
1525       put.add(kv);
1526 
1527       // checkAndPut with wrong value
1528       HStore store = (HStore) region.getStore(fam1);
1529       store.memstore.kvset.size();
1530 
1531       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1532           val1), put, true);
1533       assertEquals(true, res);
1534       store.memstore.kvset.size();
1535 
1536       Get get = new Get(row1);
1537       get.addColumn(fam2, qf1);
1538       Cell[] actual = region.get(get).rawCells();
1539 
1540       Cell[] expected = { kv };
1541 
1542       assertEquals(expected.length, actual.length);
1543       for (int i = 0; i < actual.length; i++) {
1544         assertEquals(expected[i], actual[i]);
1545       }
1546     } finally {
1547       HRegion.closeHRegion(this.region);
1548       this.region = null;
1549     }
1550   }
1551 
1552   @Test
1553   public void testCheckAndPut_wrongRowInPut() throws IOException {
1554     TableName tableName = TableName.valueOf(name.getMethodName());
1555     this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
1556     try {
1557       Put put = new Put(row2);
1558       put.add(fam1, qual1, value1);
1559       try {
1560         region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1561             new BinaryComparator(value2), put, false);
1562         fail();
1563       } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1564         // expected exception.
1565       }
1566     } finally {
1567       HRegion.closeHRegion(this.region);
1568       this.region = null;
1569     }
1570   }
1571 
1572   @Test
1573   public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1574     byte[] row1 = Bytes.toBytes("row1");
1575     byte[] fam1 = Bytes.toBytes("fam1");
1576     byte[] fam2 = Bytes.toBytes("fam2");
1577     byte[] qf1 = Bytes.toBytes("qualifier1");
1578     byte[] qf2 = Bytes.toBytes("qualifier2");
1579     byte[] qf3 = Bytes.toBytes("qualifier3");
1580     byte[] val1 = Bytes.toBytes("value1");
1581     byte[] val2 = Bytes.toBytes("value2");
1582     byte[] val3 = Bytes.toBytes("value3");
1583     byte[] emptyVal = new byte[] {};
1584 
1585     byte[][] families = { fam1, fam2 };
1586 
1587     // Setting up region
1588     String method = this.getName();
1589     this.region = initHRegion(tableName, method, CONF, families);
1590     try {
1591       // Put content
1592       Put put = new Put(row1);
1593       put.add(fam1, qf1, val1);
1594       region.put(put);
1595       Threads.sleep(2);
1596 
1597       put = new Put(row1);
1598       put.add(fam1, qf1, val2);
1599       put.add(fam2, qf1, val3);
1600       put.add(fam2, qf2, val2);
1601       put.add(fam2, qf3, val1);
1602       put.add(fam1, qf3, val1);
1603       region.put(put);
1604 
1605       // Multi-column delete
1606       Delete delete = new Delete(row1);
1607       delete.deleteColumn(fam1, qf1);
1608       delete.deleteColumn(fam2, qf1);
1609       delete.deleteColumn(fam1, qf3);
1610       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1611           val2), delete, true);
1612       assertEquals(true, res);
1613 
1614       Get get = new Get(row1);
1615       get.addColumn(fam1, qf1);
1616       get.addColumn(fam1, qf3);
1617       get.addColumn(fam2, qf2);
1618       Result r = region.get(get);
1619       assertEquals(2, r.size());
1620       assertArrayEquals(val1, r.getValue(fam1, qf1));
1621       assertArrayEquals(val2, r.getValue(fam2, qf2));
1622 
1623       // Family delete
1624       delete = new Delete(row1);
1625       delete.deleteFamily(fam2);
1626       res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1627           delete, true);
1628       assertEquals(true, res);
1629 
1630       get = new Get(row1);
1631       r = region.get(get);
1632       assertEquals(1, r.size());
1633       assertArrayEquals(val1, r.getValue(fam1, qf1));
1634 
1635       // Row delete
1636       delete = new Delete(row1);
1637       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1638           delete, true);
1639       assertEquals(true, res);
1640       get = new Get(row1);
1641       r = region.get(get);
1642       assertEquals(0, r.size());
1643     } finally {
1644       HRegion.closeHRegion(this.region);
1645       this.region = null;
1646     }
1647   }
1648 
1649   // ////////////////////////////////////////////////////////////////////////////
1650   // Delete tests
1651   // ////////////////////////////////////////////////////////////////////////////
1652   @Test
1653   public void testDelete_multiDeleteColumn() throws IOException {
1654     byte[] row1 = Bytes.toBytes("row1");
1655     byte[] fam1 = Bytes.toBytes("fam1");
1656     byte[] qual = Bytes.toBytes("qualifier");
1657     byte[] value = Bytes.toBytes("value");
1658 
1659     Put put = new Put(row1);
1660     put.add(fam1, qual, 1, value);
1661     put.add(fam1, qual, 2, value);
1662 
1663     String method = this.getName();
1664     this.region = initHRegion(tableName, method, CONF, fam1);
1665     try {
1666       region.put(put);
1667 
1668       // We do support deleting more than 1 'latest' version
1669       Delete delete = new Delete(row1);
1670       delete.deleteColumn(fam1, qual);
1671       delete.deleteColumn(fam1, qual);
1672       region.delete(delete);
1673 
1674       Get get = new Get(row1);
1675       get.addFamily(fam1);
1676       Result r = region.get(get);
1677       assertEquals(0, r.size());
1678     } finally {
1679       HRegion.closeHRegion(this.region);
1680       this.region = null;
1681     }
1682   }
1683 
1684   @Test
1685   public void testDelete_CheckFamily() throws IOException {
1686     byte[] row1 = Bytes.toBytes("row1");
1687     byte[] fam1 = Bytes.toBytes("fam1");
1688     byte[] fam2 = Bytes.toBytes("fam2");
1689     byte[] fam3 = Bytes.toBytes("fam3");
1690     byte[] fam4 = Bytes.toBytes("fam4");
1691 
1692     // Setting up region
1693     String method = this.getName();
1694     this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
1695     try {
1696       List<Cell> kvs = new ArrayList<Cell>();
1697       kvs.add(new KeyValue(row1, fam4, null, null));
1698 
1699       // testing existing family
1700       byte[] family = fam2;
1701       try {
1702         NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1703             Bytes.BYTES_COMPARATOR);
1704         deleteMap.put(family, kvs);
1705         region.delete(deleteMap, Durability.SYNC_WAL);
1706       } catch (Exception e) {
1707         assertTrue("Family " + new String(family) + " does not exist", false);
1708       }
1709 
1710       // testing non existing family
1711       boolean ok = false;
1712       family = fam4;
1713       try {
1714         NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1715             Bytes.BYTES_COMPARATOR);
1716         deleteMap.put(family, kvs);
1717         region.delete(deleteMap, Durability.SYNC_WAL);
1718       } catch (Exception e) {
1719         ok = true;
1720       }
1721       assertEquals("Family " + new String(family) + " does exist", true, ok);
1722     } finally {
1723       HRegion.closeHRegion(this.region);
1724       this.region = null;
1725     }
1726   }
1727 
1728   @Test
1729   public void testDelete_mixed() throws IOException, InterruptedException {
1730     byte[] fam = Bytes.toBytes("info");
1731     byte[][] families = { fam };
1732     String method = this.getName();
1733     this.region = initHRegion(tableName, method, CONF, families);
1734     try {
1735       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1736 
1737       byte[] row = Bytes.toBytes("table_name");
1738       // column names
1739       byte[] serverinfo = Bytes.toBytes("serverinfo");
1740       byte[] splitA = Bytes.toBytes("splitA");
1741       byte[] splitB = Bytes.toBytes("splitB");
1742 
1743       // add some data:
1744       Put put = new Put(row);
1745       put.add(fam, splitA, Bytes.toBytes("reference_A"));
1746       region.put(put);
1747 
1748       put = new Put(row);
1749       put.add(fam, splitB, Bytes.toBytes("reference_B"));
1750       region.put(put);
1751 
1752       put = new Put(row);
1753       put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1754       region.put(put);
1755 
1756       // ok now delete a split:
1757       Delete delete = new Delete(row);
1758       delete.deleteColumns(fam, splitA);
1759       region.delete(delete);
1760 
1761       // assert some things:
1762       Get get = new Get(row).addColumn(fam, serverinfo);
1763       Result result = region.get(get);
1764       assertEquals(1, result.size());
1765 
1766       get = new Get(row).addColumn(fam, splitA);
1767       result = region.get(get);
1768       assertEquals(0, result.size());
1769 
1770       get = new Get(row).addColumn(fam, splitB);
1771       result = region.get(get);
1772       assertEquals(1, result.size());
1773 
1774       // Assert that after a delete, I can put.
1775       put = new Put(row);
1776       put.add(fam, splitA, Bytes.toBytes("reference_A"));
1777       region.put(put);
1778       get = new Get(row);
1779       result = region.get(get);
1780       assertEquals(3, result.size());
1781 
1782       // Now delete all... then test I can add stuff back
1783       delete = new Delete(row);
1784       region.delete(delete);
1785       assertEquals(0, region.get(get).size());
1786 
1787       region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
1788       result = region.get(get);
1789       assertEquals(1, result.size());
1790     } finally {
1791       HRegion.closeHRegion(this.region);
1792       this.region = null;
1793     }
1794   }
1795 
1796   @Test
1797   public void testDeleteRowWithFutureTs() throws IOException {
1798     byte[] fam = Bytes.toBytes("info");
1799     byte[][] families = { fam };
1800     String method = this.getName();
1801     this.region = initHRegion(tableName, method, CONF, families);
1802     try {
1803       byte[] row = Bytes.toBytes("table_name");
1804       // column names
1805       byte[] serverinfo = Bytes.toBytes("serverinfo");
1806 
1807       // add data in the far future
1808       Put put = new Put(row);
1809       put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
1810       region.put(put);
1811 
1812       // now delete something in the present
1813       Delete delete = new Delete(row);
1814       region.delete(delete);
1815 
1816       // make sure we still see our data
1817       Get get = new Get(row).addColumn(fam, serverinfo);
1818       Result result = region.get(get);
1819       assertEquals(1, result.size());
1820 
1821       // delete the future row
1822       delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
1823       region.delete(delete);
1824 
1825       // make sure it is gone
1826       get = new Get(row).addColumn(fam, serverinfo);
1827       result = region.get(get);
1828       assertEquals(0, result.size());
1829     } finally {
1830       HRegion.closeHRegion(this.region);
1831       this.region = null;
1832     }
1833   }
1834 
1835   /**
1836    * Tests that the special LATEST_TIMESTAMP option for puts gets replaced by
1837    * the actual timestamp
1838    */
1839   @Test
1840   public void testPutWithLatestTS() throws IOException {
1841     byte[] fam = Bytes.toBytes("info");
1842     byte[][] families = { fam };
1843     String method = this.getName();
1844     this.region = initHRegion(tableName, method, CONF, families);
1845     try {
1846       byte[] row = Bytes.toBytes("row1");
1847       // column names
1848       byte[] qual = Bytes.toBytes("qual");
1849 
1850       // add data with LATEST_TIMESTAMP, put without WAL
1851       Put put = new Put(row);
1852       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1853       region.put(put);
1854 
1855       // Make sure it shows up with an actual timestamp
1856       Get get = new Get(row).addColumn(fam, qual);
1857       Result result = region.get(get);
1858       assertEquals(1, result.size());
1859       Cell kv = result.rawCells()[0];
1860       LOG.info("Got: " + kv);
1861       assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1862           kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1863 
1864       // Check same with WAL enabled (historically these took different
1865       // code paths, so check both)
1866       row = Bytes.toBytes("row2");
1867       put = new Put(row);
1868       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1869       region.put(put);
1870 
1871       // Make sure it shows up with an actual timestamp
1872       get = new Get(row).addColumn(fam, qual);
1873       result = region.get(get);
1874       assertEquals(1, result.size());
1875       kv = result.rawCells()[0];
1876       LOG.info("Got: " + kv);
1877       assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1878           kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1879     } finally {
1880       HRegion.closeHRegion(this.region);
1881       this.region = null;
1882     }
1883 
1884   }
1885 
1886   /**
1887    * Tests that there is server-side filtering for invalid timestamp upper
1888    * bound. Note that the timestamp lower bound is automatically handled for us
1889    * by the TTL field.
1890    */
1891   @Test
1892   public void testPutWithTsSlop() throws IOException {
1893     byte[] fam = Bytes.toBytes("info");
1894     byte[][] families = { fam };
1895     String method = this.getName();
1896 
1897     // add data with a timestamp that is too recent for range. Ensure assert
1898     CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1899     this.region = initHRegion(tableName, method, CONF, families);
1900     boolean caughtExcep = false;
1901     try {
1902       try {
1903         // no TS specified == use latest. should not error
1904         region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
1905         // TS out of range. should error
1906         region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
1907             Bytes.toBytes("value")));
1908         fail("Expected IOE for TS out of configured timerange");
1909       } catch (FailedSanityCheckException ioe) {
1910         LOG.debug("Received expected exception", ioe);
1911         caughtExcep = true;
1912       }
1913       assertTrue("Should catch FailedSanityCheckException", caughtExcep);
1914     } finally {
1915       HRegion.closeHRegion(this.region);
1916       this.region = null;
1917     }
1918   }
1919 
1920   @Test
1921   public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
1922     byte[] fam1 = Bytes.toBytes("columnA");
1923     byte[] fam2 = Bytes.toBytes("columnB");
1924     this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
1925     try {
1926       byte[] rowA = Bytes.toBytes("rowA");
1927       byte[] rowB = Bytes.toBytes("rowB");
1928 
1929       byte[] value = Bytes.toBytes("value");
1930 
1931       Delete delete = new Delete(rowA);
1932       delete.deleteFamily(fam1);
1933 
1934       region.delete(delete);
1935 
1936       // now create data.
1937       Put put = new Put(rowA);
1938       put.add(fam2, null, value);
1939       region.put(put);
1940 
1941       put = new Put(rowB);
1942       put.add(fam1, null, value);
1943       put.add(fam2, null, value);
1944       region.put(put);
1945 
1946       Scan scan = new Scan();
1947       scan.addFamily(fam1).addFamily(fam2);
1948       InternalScanner s = region.getScanner(scan);
1949       List<Cell> results = new ArrayList<Cell>();
1950       s.next(results);
1951       assertTrue(CellUtil.matchingRow(results.get(0), rowA));
1952 
1953       results.clear();
1954       s.next(results);
1955       assertTrue(CellUtil.matchingRow(results.get(0), rowB));
1956     } finally {
1957       HRegion.closeHRegion(this.region);
1958       this.region = null;
1959     }
1960   }
1961 
1962   @Test
1963   public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
1964     Delete delete = new Delete(row);
1965     delete.deleteColumns(fam1, qual1);
1966     doTestDelete_AndPostInsert(delete);
1967   }
1968 
1969   @Test
1970   public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
1971     Delete delete = new Delete(row);
1972     delete.deleteFamily(fam1);
1973     doTestDelete_AndPostInsert(delete);
1974   }
1975 
1976   public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
1977     TableName tableName = TableName.valueOf(name.getMethodName());
1978     this.region = initHRegion(tableName, getName(), CONF, fam1);
1979     try {
1980       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1981       Put put = new Put(row);
1982       put.add(fam1, qual1, value1);
1983       region.put(put);
1984 
1985       // now delete the value:
1986       region.delete(delete);
1987 
1988       // ok put data:
1989       put = new Put(row);
1990       put.add(fam1, qual1, value2);
1991       region.put(put);
1992 
1993       // ok get:
1994       Get get = new Get(row);
1995       get.addColumn(fam1, qual1);
1996 
1997       Result r = region.get(get);
1998       assertEquals(1, r.size());
1999       assertArrayEquals(value2, r.getValue(fam1, qual1));
2000 
2001       // next:
2002       Scan scan = new Scan(row);
2003       scan.addColumn(fam1, qual1);
2004       InternalScanner s = region.getScanner(scan);
2005 
2006       List<Cell> results = new ArrayList<Cell>();
2007       assertEquals(false, s.next(results));
2008       assertEquals(1, results.size());
2009       Cell kv = results.get(0);
2010 
2011       assertArrayEquals(value2, CellUtil.cloneValue(kv));
2012       assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
2013       assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
2014       assertArrayEquals(row, CellUtil.cloneRow(kv));
2015     } finally {
2016       HRegion.closeHRegion(this.region);
2017       this.region = null;
2018     }
2019   }
2020 
2021   @Test
2022   public void testDelete_CheckTimestampUpdated() throws IOException {
2023     TableName tableName = TableName.valueOf(name.getMethodName());
2024     byte[] row1 = Bytes.toBytes("row1");
2025     byte[] col1 = Bytes.toBytes("col1");
2026     byte[] col2 = Bytes.toBytes("col2");
2027     byte[] col3 = Bytes.toBytes("col3");
2028 
2029     // Setting up region
2030     String method = this.getName();
2031     this.region = initHRegion(tableName, method, CONF, fam1);
2032     try {
2033       // Building checkerList
2034       List<Cell> kvs = new ArrayList<Cell>();
2035       kvs.add(new KeyValue(row1, fam1, col1, null));
2036       kvs.add(new KeyValue(row1, fam1, col2, null));
2037       kvs.add(new KeyValue(row1, fam1, col3, null));
2038 
2039       NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
2040           Bytes.BYTES_COMPARATOR);
2041       deleteMap.put(fam1, kvs);
2042       region.delete(deleteMap, Durability.SYNC_WAL);
2043 
2044       // extract the key values out the memstore:
2045       // This is kinda hacky, but better than nothing...
2046       long now = System.currentTimeMillis();
2047       KeyValue firstKv = ((HStore) region.getStore(fam1)).memstore.kvset.first();
2048       assertTrue(firstKv.getTimestamp() <= now);
2049       now = firstKv.getTimestamp();
2050       for (KeyValue kv : ((HStore) region.getStore(fam1)).memstore.kvset) {
2051         assertTrue(kv.getTimestamp() <= now);
2052         now = kv.getTimestamp();
2053       }
2054     } finally {
2055       HRegion.closeHRegion(this.region);
2056       this.region = null;
2057     }
2058   }
2059 
2060   // ////////////////////////////////////////////////////////////////////////////
2061   // Get tests
2062   // ////////////////////////////////////////////////////////////////////////////
2063   @Test
2064   public void testGet_FamilyChecker() throws IOException {
2065     byte[] row1 = Bytes.toBytes("row1");
2066     byte[] fam1 = Bytes.toBytes("fam1");
2067     byte[] fam2 = Bytes.toBytes("False");
2068     byte[] col1 = Bytes.toBytes("col1");
2069 
2070     // Setting up region
2071     String method = this.getName();
2072     this.region = initHRegion(tableName, method, CONF, fam1);
2073     try {
2074       Get get = new Get(row1);
2075       get.addColumn(fam2, col1);
2076 
2077       // Test
2078       try {
2079         region.get(get);
2080       } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
2081         assertFalse(false);
2082         return;
2083       }
2084       assertFalse(true);
2085     } finally {
2086       HRegion.closeHRegion(this.region);
2087       this.region = null;
2088     }
2089   }
2090 
2091   @Test
2092   public void testGet_Basic() throws IOException {
2093     byte[] row1 = Bytes.toBytes("row1");
2094     byte[] fam1 = Bytes.toBytes("fam1");
2095     byte[] col1 = Bytes.toBytes("col1");
2096     byte[] col2 = Bytes.toBytes("col2");
2097     byte[] col3 = Bytes.toBytes("col3");
2098     byte[] col4 = Bytes.toBytes("col4");
2099     byte[] col5 = Bytes.toBytes("col5");
2100 
2101     // Setting up region
2102     String method = this.getName();
2103     this.region = initHRegion(tableName, method, CONF, fam1);
2104     try {
2105       // Add to memstore
2106       Put put = new Put(row1);
2107       put.add(fam1, col1, null);
2108       put.add(fam1, col2, null);
2109       put.add(fam1, col3, null);
2110       put.add(fam1, col4, null);
2111       put.add(fam1, col5, null);
2112       region.put(put);
2113 
2114       Get get = new Get(row1);
2115       get.addColumn(fam1, col2);
2116       get.addColumn(fam1, col4);
2117       // Expected result
2118       KeyValue kv1 = new KeyValue(row1, fam1, col2);
2119       KeyValue kv2 = new KeyValue(row1, fam1, col4);
2120       KeyValue[] expected = { kv1, kv2 };
2121 
2122       // Test
2123       Result res = region.get(get);
2124       assertEquals(expected.length, res.size());
2125       for (int i = 0; i < res.size(); i++) {
2126         assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
2127         assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
2128         assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
2129       }
2130 
2131       // Test using a filter on a Get
2132       Get g = new Get(row1);
2133       final int count = 2;
2134       g.setFilter(new ColumnCountGetFilter(count));
2135       res = region.get(g);
2136       assertEquals(count, res.size());
2137     } finally {
2138       HRegion.closeHRegion(this.region);
2139       this.region = null;
2140     }
2141   }
2142 
2143   @Test
2144   public void testGet_Empty() throws IOException {
2145     byte[] row = Bytes.toBytes("row");
2146     byte[] fam = Bytes.toBytes("fam");
2147 
2148     String method = this.getName();
2149     this.region = initHRegion(tableName, method, CONF, fam);
2150     try {
2151       Get get = new Get(row);
2152       get.addFamily(fam);
2153       Result r = region.get(get);
2154 
2155       assertTrue(r.isEmpty());
2156     } finally {
2157       HRegion.closeHRegion(this.region);
2158       this.region = null;
2159     }
2160   }
2161 
2162   // ////////////////////////////////////////////////////////////////////////////
2163   // Merge test
2164   // ////////////////////////////////////////////////////////////////////////////
2165   @Test
2166   public void testMerge() throws IOException {
2167     byte[][] families = { fam1, fam2, fam3 };
2168     Configuration hc = initSplit();
2169     // Setting up region
2170     String method = this.getName();
2171     this.region = initHRegion(tableName, method, hc, families);
2172     try {
2173       LOG.info("" + HBaseTestCase.addContent(region, fam3));
2174       region.flushcache();
2175       region.compactStores();
2176       byte[] splitRow = region.checkSplit();
2177       assertNotNull(splitRow);
2178       LOG.info("SplitRow: " + Bytes.toString(splitRow));
2179       HRegion[] subregions = splitRegion(region, splitRow);
2180       try {
2181         // Need to open the regions.
2182         for (int i = 0; i < subregions.length; i++) {
2183           HRegion.openHRegion(subregions[i], null);
2184           subregions[i].compactStores();
2185         }
2186         Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
2187         Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
2188         Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
2189         long startTime = System.currentTimeMillis();
2190         region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
2191         LOG.info("Merge regions elapsed time: "
2192             + ((System.currentTimeMillis() - startTime) / 1000.0));
2193         FILESYSTEM.delete(oldRegion1, true);
2194         FILESYSTEM.delete(oldRegion2, true);
2195         FILESYSTEM.delete(oldRegionPath, true);
2196         LOG.info("splitAndMerge completed.");
2197       } finally {
2198         for (int i = 0; i < subregions.length; i++) {
2199           try {
2200             HRegion.closeHRegion(subregions[i]);
2201           } catch (IOException e) {
2202             // Ignore.
2203           }
2204         }
2205       }
2206     } finally {
2207       HRegion.closeHRegion(this.region);
2208       this.region = null;
2209     }
2210   }
2211 
2212   /**
2213    * @param parent
2214    *          Region to split.
2215    * @param midkey
2216    *          Key to split around.
2217    * @return The Regions we created.
2218    * @throws IOException
2219    */
2220   HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
2221     PairOfSameType<HRegion> result = null;
2222     SplitTransaction st = new SplitTransaction(parent, midkey);
2223     // If prepare does not return true, for some reason -- logged inside in
2224     // the prepare call -- we are not ready to split just now. Just return.
2225     if (!st.prepare()) {
2226       parent.clearSplit();
2227       return null;
2228     }
2229     try {
2230       result = st.execute(null, null);
2231     } catch (IOException ioe) {
2232       try {
2233         LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; "
2234             + ioe.getMessage());
2235         st.rollback(null, null);
2236         LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString());
2237         return null;
2238       } catch (RuntimeException e) {
2239         // If failed rollback, kill this server to avoid having a hole in table.
2240         LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString()
2241             + " -- aborting server", e);
2242       }
2243     }
2244     finally {
2245       parent.clearSplit();
2246     }
2247     return new HRegion[] { result.getFirst(), result.getSecond() };
2248   }
2249 
2250   // ////////////////////////////////////////////////////////////////////////////
2251   // Scanner tests
2252   // ////////////////////////////////////////////////////////////////////////////
2253   @Test
2254   public void testGetScanner_WithOkFamilies() throws IOException {
2255     byte[] fam1 = Bytes.toBytes("fam1");
2256     byte[] fam2 = Bytes.toBytes("fam2");
2257 
2258     byte[][] families = { fam1, fam2 };
2259 
2260     // Setting up region
2261     String method = this.getName();
2262     this.region = initHRegion(tableName, method, CONF, families);
2263     try {
2264       Scan scan = new Scan();
2265       scan.addFamily(fam1);
2266       scan.addFamily(fam2);
2267       try {
2268         region.getScanner(scan);
2269       } catch (Exception e) {
2270         assertTrue("Families could not be found in Region", false);
2271       }
2272     } finally {
2273       HRegion.closeHRegion(this.region);
2274       this.region = null;
2275     }
2276   }
2277 
2278   @Test
2279   public void testGetScanner_WithNotOkFamilies() throws IOException {
2280     byte[] fam1 = Bytes.toBytes("fam1");
2281     byte[] fam2 = Bytes.toBytes("fam2");
2282 
2283     byte[][] families = { fam1 };
2284 
2285     // Setting up region
2286     String method = this.getName();
2287     this.region = initHRegion(tableName, method, CONF, families);
2288     try {
2289       Scan scan = new Scan();
2290       scan.addFamily(fam2);
2291       boolean ok = false;
2292       try {
2293         region.getScanner(scan);
2294       } catch (Exception e) {
2295         ok = true;
2296       }
2297       assertTrue("Families could not be found in Region", ok);
2298     } finally {
2299       HRegion.closeHRegion(this.region);
2300       this.region = null;
2301     }
2302   }
2303 
2304   @Test
2305   public void testGetScanner_WithNoFamilies() throws IOException {
2306     byte[] row1 = Bytes.toBytes("row1");
2307     byte[] fam1 = Bytes.toBytes("fam1");
2308     byte[] fam2 = Bytes.toBytes("fam2");
2309     byte[] fam3 = Bytes.toBytes("fam3");
2310     byte[] fam4 = Bytes.toBytes("fam4");
2311 
2312     byte[][] families = { fam1, fam2, fam3, fam4 };
2313 
2314     // Setting up region
2315     String method = this.getName();
2316     this.region = initHRegion(tableName, method, CONF, families);
2317     try {
2318 
2319       // Putting data in Region
2320       Put put = new Put(row1);
2321       put.add(fam1, null, null);
2322       put.add(fam2, null, null);
2323       put.add(fam3, null, null);
2324       put.add(fam4, null, null);
2325       region.put(put);
2326 
2327       Scan scan = null;
2328       HRegion.RegionScannerImpl is = null;
2329 
2330       // Testing to see how many scanners that is produced by getScanner,
2331       // starting
2332       // with known number, 2 - current = 1
2333       scan = new Scan();
2334       scan.addFamily(fam2);
2335       scan.addFamily(fam4);
2336       is = (RegionScannerImpl) region.getScanner(scan);
2337       assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2338 
2339       scan = new Scan();
2340       is = (RegionScannerImpl) region.getScanner(scan);
2341       assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2342     } finally {
2343       HRegion.closeHRegion(this.region);
2344       this.region = null;
2345     }
2346   }
2347 
2348   /**
2349    * This method tests https://issues.apache.org/jira/browse/HBASE-2516.
2350    * 
2351    * @throws IOException
2352    */
2353   @Test
2354   public void testGetScanner_WithRegionClosed() throws IOException {
2355     byte[] fam1 = Bytes.toBytes("fam1");
2356     byte[] fam2 = Bytes.toBytes("fam2");
2357 
2358     byte[][] families = { fam1, fam2 };
2359 
2360     // Setting up region
2361     String method = this.getName();
2362     try {
2363       this.region = initHRegion(tableName, method, CONF, families);
2364     } catch (IOException e) {
2365       e.printStackTrace();
2366       fail("Got IOException during initHRegion, " + e.getMessage());
2367     }
2368     try {
2369       region.closed.set(true);
2370       try {
2371         region.getScanner(null);
2372         fail("Expected to get an exception during getScanner on a region that is closed");
2373       } catch (NotServingRegionException e) {
2374         // this is the correct exception that is expected
2375       } catch (IOException e) {
2376         fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2377             + e.getMessage());
2378       }
2379     } finally {
2380       HRegion.closeHRegion(this.region);
2381       this.region = null;
2382     }
2383   }
2384 
2385   @Test
2386   public void testRegionScanner_Next() throws IOException {
2387     byte[] row1 = Bytes.toBytes("row1");
2388     byte[] row2 = Bytes.toBytes("row2");
2389     byte[] fam1 = Bytes.toBytes("fam1");
2390     byte[] fam2 = Bytes.toBytes("fam2");
2391     byte[] fam3 = Bytes.toBytes("fam3");
2392     byte[] fam4 = Bytes.toBytes("fam4");
2393 
2394     byte[][] families = { fam1, fam2, fam3, fam4 };
2395     long ts = System.currentTimeMillis();
2396 
2397     // Setting up region
2398     String method = this.getName();
2399     this.region = initHRegion(tableName, method, CONF, families);
2400     try {
2401       // Putting data in Region
2402       Put put = null;
2403       put = new Put(row1);
2404       put.add(fam1, (byte[]) null, ts, null);
2405       put.add(fam2, (byte[]) null, ts, null);
2406       put.add(fam3, (byte[]) null, ts, null);
2407       put.add(fam4, (byte[]) null, ts, null);
2408       region.put(put);
2409 
2410       put = new Put(row2);
2411       put.add(fam1, (byte[]) null, ts, null);
2412       put.add(fam2, (byte[]) null, ts, null);
2413       put.add(fam3, (byte[]) null, ts, null);
2414       put.add(fam4, (byte[]) null, ts, null);
2415       region.put(put);
2416 
2417       Scan scan = new Scan();
2418       scan.addFamily(fam2);
2419       scan.addFamily(fam4);
2420       InternalScanner is = region.getScanner(scan);
2421 
2422       List<Cell> res = null;
2423 
2424       // Result 1
2425       List<Cell> expected1 = new ArrayList<Cell>();
2426       expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2427       expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2428 
2429       res = new ArrayList<Cell>();
2430       is.next(res);
2431       for (int i = 0; i < res.size(); i++) {
2432         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2433       }
2434 
2435       // Result 2
2436       List<Cell> expected2 = new ArrayList<Cell>();
2437       expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2438       expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2439 
2440       res = new ArrayList<Cell>();
2441       is.next(res);
2442       for (int i = 0; i < res.size(); i++) {
2443         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2444       }
2445     } finally {
2446       HRegion.closeHRegion(this.region);
2447       this.region = null;
2448     }
2449   }
2450 
2451   @Test
2452   public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2453     byte[] row1 = Bytes.toBytes("row1");
2454     byte[] qf1 = Bytes.toBytes("qualifier1");
2455     byte[] qf2 = Bytes.toBytes("qualifier2");
2456     byte[] fam1 = Bytes.toBytes("fam1");
2457     byte[][] families = { fam1 };
2458 
2459     long ts1 = System.currentTimeMillis();
2460     long ts2 = ts1 + 1;
2461     long ts3 = ts1 + 2;
2462 
2463     // Setting up region
2464     String method = this.getName();
2465     this.region = initHRegion(tableName, method, CONF, families);
2466     try {
2467       // Putting data in Region
2468       Put put = null;
2469       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2470       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2471       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2472 
2473       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2474       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2475       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2476 
2477       put = new Put(row1);
2478       put.add(kv13);
2479       put.add(kv12);
2480       put.add(kv11);
2481       put.add(kv23);
2482       put.add(kv22);
2483       put.add(kv21);
2484       region.put(put);
2485 
2486       // Expected
2487       List<Cell> expected = new ArrayList<Cell>();
2488       expected.add(kv13);
2489       expected.add(kv12);
2490 
2491       Scan scan = new Scan(row1);
2492       scan.addColumn(fam1, qf1);
2493       scan.setMaxVersions(MAX_VERSIONS);
2494       List<Cell> actual = new ArrayList<Cell>();
2495       InternalScanner scanner = region.getScanner(scan);
2496 
2497       boolean hasNext = scanner.next(actual);
2498       assertEquals(false, hasNext);
2499 
2500       // Verify result
2501       for (int i = 0; i < expected.size(); i++) {
2502         assertEquals(expected.get(i), actual.get(i));
2503       }
2504     } finally {
2505       HRegion.closeHRegion(this.region);
2506       this.region = null;
2507     }
2508   }
2509 
2510   @Test
2511   public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2512     byte[] row1 = Bytes.toBytes("row1");
2513     byte[] qf1 = Bytes.toBytes("qualifier1");
2514     byte[] qf2 = Bytes.toBytes("qualifier2");
2515     byte[] fam1 = Bytes.toBytes("fam1");
2516     byte[][] families = { fam1 };
2517 
2518     long ts1 = 1; // System.currentTimeMillis();
2519     long ts2 = ts1 + 1;
2520     long ts3 = ts1 + 2;
2521 
2522     // Setting up region
2523     String method = this.getName();
2524     this.region = initHRegion(tableName, method, CONF, families);
2525     try {
2526       // Putting data in Region
2527       Put put = null;
2528       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2529       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2530       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2531 
2532       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2533       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2534       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2535 
2536       put = new Put(row1);
2537       put.add(kv13);
2538       put.add(kv12);
2539       put.add(kv11);
2540       put.add(kv23);
2541       put.add(kv22);
2542       put.add(kv21);
2543       region.put(put);
2544       region.flushcache();
2545 
2546       // Expected
2547       List<Cell> expected = new ArrayList<Cell>();
2548       expected.add(kv13);
2549       expected.add(kv12);
2550       expected.add(kv23);
2551       expected.add(kv22);
2552 
2553       Scan scan = new Scan(row1);
2554       scan.addColumn(fam1, qf1);
2555       scan.addColumn(fam1, qf2);
2556       scan.setMaxVersions(MAX_VERSIONS);
2557       List<Cell> actual = new ArrayList<Cell>();
2558       InternalScanner scanner = region.getScanner(scan);
2559 
2560       boolean hasNext = scanner.next(actual);
2561       assertEquals(false, hasNext);
2562 
2563       // Verify result
2564       for (int i = 0; i < expected.size(); i++) {
2565         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2566       }
2567     } finally {
2568       HRegion.closeHRegion(this.region);
2569       this.region = null;
2570     }
2571   }
2572 
2573   @Test
2574   public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2575     byte[] row1 = Bytes.toBytes("row1");
2576     byte[] fam1 = Bytes.toBytes("fam1");
2577     byte[][] families = { fam1 };
2578     byte[] qf1 = Bytes.toBytes("qualifier1");
2579     byte[] qf2 = Bytes.toBytes("qualifier2");
2580 
2581     long ts1 = 1;
2582     long ts2 = ts1 + 1;
2583     long ts3 = ts1 + 2;
2584     long ts4 = ts1 + 3;
2585 
2586     // Setting up region
2587     String method = this.getName();
2588     this.region = initHRegion(tableName, method, CONF, families);
2589     try {
2590       // Putting data in Region
2591       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2592       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2593       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2594       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2595 
2596       KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2597       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2598       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2599       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2600 
2601       Put put = null;
2602       put = new Put(row1);
2603       put.add(kv14);
2604       put.add(kv24);
2605       region.put(put);
2606       region.flushcache();
2607 
2608       put = new Put(row1);
2609       put.add(kv23);
2610       put.add(kv13);
2611       region.put(put);
2612       region.flushcache();
2613 
2614       put = new Put(row1);
2615       put.add(kv22);
2616       put.add(kv12);
2617       region.put(put);
2618       region.flushcache();
2619 
2620       put = new Put(row1);
2621       put.add(kv21);
2622       put.add(kv11);
2623       region.put(put);
2624 
2625       // Expected
2626       List<Cell> expected = new ArrayList<Cell>();
2627       expected.add(kv14);
2628       expected.add(kv13);
2629       expected.add(kv12);
2630       expected.add(kv24);
2631       expected.add(kv23);
2632       expected.add(kv22);
2633 
2634       Scan scan = new Scan(row1);
2635       scan.addColumn(fam1, qf1);
2636       scan.addColumn(fam1, qf2);
2637       int versions = 3;
2638       scan.setMaxVersions(versions);
2639       List<Cell> actual = new ArrayList<Cell>();
2640       InternalScanner scanner = region.getScanner(scan);
2641 
2642       boolean hasNext = scanner.next(actual);
2643       assertEquals(false, hasNext);
2644 
2645       // Verify result
2646       for (int i = 0; i < expected.size(); i++) {
2647         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2648       }
2649     } finally {
2650       HRegion.closeHRegion(this.region);
2651       this.region = null;
2652     }
2653   }
2654 
2655   @Test
2656   public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2657     byte[] row1 = Bytes.toBytes("row1");
2658     byte[] qf1 = Bytes.toBytes("qualifier1");
2659     byte[] qf2 = Bytes.toBytes("qualifier2");
2660     byte[] fam1 = Bytes.toBytes("fam1");
2661     byte[][] families = { fam1 };
2662 
2663     long ts1 = System.currentTimeMillis();
2664     long ts2 = ts1 + 1;
2665     long ts3 = ts1 + 2;
2666 
2667     // Setting up region
2668     String method = this.getName();
2669     this.region = initHRegion(tableName, method, CONF, families);
2670     try {
2671       // Putting data in Region
2672       Put put = null;
2673       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2674       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2675       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2676 
2677       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2678       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2679       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2680 
2681       put = new Put(row1);
2682       put.add(kv13);
2683       put.add(kv12);
2684       put.add(kv11);
2685       put.add(kv23);
2686       put.add(kv22);
2687       put.add(kv21);
2688       region.put(put);
2689 
2690       // Expected
2691       List<Cell> expected = new ArrayList<Cell>();
2692       expected.add(kv13);
2693       expected.add(kv12);
2694       expected.add(kv23);
2695       expected.add(kv22);
2696 
2697       Scan scan = new Scan(row1);
2698       scan.addFamily(fam1);
2699       scan.setMaxVersions(MAX_VERSIONS);
2700       List<Cell> actual = new ArrayList<Cell>();
2701       InternalScanner scanner = region.getScanner(scan);
2702 
2703       boolean hasNext = scanner.next(actual);
2704       assertEquals(false, hasNext);
2705 
2706       // Verify result
2707       for (int i = 0; i < expected.size(); i++) {
2708         assertEquals(expected.get(i), actual.get(i));
2709       }
2710     } finally {
2711       HRegion.closeHRegion(this.region);
2712       this.region = null;
2713     }
2714   }
2715 
2716   @Test
2717   public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
2718     byte[] row1 = Bytes.toBytes("row1");
2719     byte[] qf1 = Bytes.toBytes("qualifier1");
2720     byte[] qf2 = Bytes.toBytes("qualifier2");
2721     byte[] fam1 = Bytes.toBytes("fam1");
2722 
2723     long ts1 = 1; // System.currentTimeMillis();
2724     long ts2 = ts1 + 1;
2725     long ts3 = ts1 + 2;
2726 
2727     // Setting up region
2728     String method = this.getName();
2729     this.region = initHRegion(tableName, method, CONF, fam1);
2730     try {
2731       // Putting data in Region
2732       Put put = null;
2733       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2734       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2735       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2736 
2737       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2738       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2739       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2740 
2741       put = new Put(row1);
2742       put.add(kv13);
2743       put.add(kv12);
2744       put.add(kv11);
2745       put.add(kv23);
2746       put.add(kv22);
2747       put.add(kv21);
2748       region.put(put);
2749       region.flushcache();
2750 
2751       // Expected
2752       List<Cell> expected = new ArrayList<Cell>();
2753       expected.add(kv13);
2754       expected.add(kv12);
2755       expected.add(kv23);
2756       expected.add(kv22);
2757 
2758       Scan scan = new Scan(row1);
2759       scan.addFamily(fam1);
2760       scan.setMaxVersions(MAX_VERSIONS);
2761       List<Cell> actual = new ArrayList<Cell>();
2762       InternalScanner scanner = region.getScanner(scan);
2763 
2764       boolean hasNext = scanner.next(actual);
2765       assertEquals(false, hasNext);
2766 
2767       // Verify result
2768       for (int i = 0; i < expected.size(); i++) {
2769         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2770       }
2771     } finally {
2772       HRegion.closeHRegion(this.region);
2773       this.region = null;
2774     }
2775   }
2776 
2777   @Test
2778   public void testScanner_StopRow1542() throws IOException {
2779     byte[] family = Bytes.toBytes("testFamily");
2780     this.region = initHRegion(tableName, getName(), CONF, family);
2781     try {
2782       byte[] row1 = Bytes.toBytes("row111");
2783       byte[] row2 = Bytes.toBytes("row222");
2784       byte[] row3 = Bytes.toBytes("row333");
2785       byte[] row4 = Bytes.toBytes("row444");
2786       byte[] row5 = Bytes.toBytes("row555");
2787 
2788       byte[] col1 = Bytes.toBytes("Pub111");
2789       byte[] col2 = Bytes.toBytes("Pub222");
2790 
2791       Put put = new Put(row1);
2792       put.add(family, col1, Bytes.toBytes(10L));
2793       region.put(put);
2794 
2795       put = new Put(row2);
2796       put.add(family, col1, Bytes.toBytes(15L));
2797       region.put(put);
2798 
2799       put = new Put(row3);
2800       put.add(family, col2, Bytes.toBytes(20L));
2801       region.put(put);
2802 
2803       put = new Put(row4);
2804       put.add(family, col2, Bytes.toBytes(30L));
2805       region.put(put);
2806 
2807       put = new Put(row5);
2808       put.add(family, col1, Bytes.toBytes(40L));
2809       region.put(put);
2810 
2811       Scan scan = new Scan(row3, row4);
2812       scan.setMaxVersions();
2813       scan.addColumn(family, col1);
2814       InternalScanner s = region.getScanner(scan);
2815 
2816       List<Cell> results = new ArrayList<Cell>();
2817       assertEquals(false, s.next(results));
2818       assertEquals(0, results.size());
2819     } finally {
2820       HRegion.closeHRegion(this.region);
2821       this.region = null;
2822     }
2823   }
2824 
2825   @Test
2826   public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2827     byte[] row1 = Bytes.toBytes("row1");
2828     byte[] fam1 = Bytes.toBytes("fam1");
2829     byte[] qf1 = Bytes.toBytes("qualifier1");
2830     byte[] qf2 = Bytes.toBytes("quateslifier2");
2831 
2832     long ts1 = 1;
2833     long ts2 = ts1 + 1;
2834     long ts3 = ts1 + 2;
2835     long ts4 = ts1 + 3;
2836 
2837     // Setting up region
2838     String method = this.getName();
2839     this.region = initHRegion(tableName, method, CONF, fam1);
2840     try {
2841       // Putting data in Region
2842       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2843       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2844       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2845       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2846 
2847       KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2848       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2849       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2850       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2851 
2852       Put put = null;
2853       put = new Put(row1);
2854       put.add(kv14);
2855       put.add(kv24);
2856       region.put(put);
2857       region.flushcache();
2858 
2859       put = new Put(row1);
2860       put.add(kv23);
2861       put.add(kv13);
2862       region.put(put);
2863       region.flushcache();
2864 
2865       put = new Put(row1);
2866       put.add(kv22);
2867       put.add(kv12);
2868       region.put(put);
2869       region.flushcache();
2870 
2871       put = new Put(row1);
2872       put.add(kv21);
2873       put.add(kv11);
2874       region.put(put);
2875 
2876       // Expected
2877       List<KeyValue> expected = new ArrayList<KeyValue>();
2878       expected.add(kv14);
2879       expected.add(kv13);
2880       expected.add(kv12);
2881       expected.add(kv24);
2882       expected.add(kv23);
2883       expected.add(kv22);
2884 
2885       Scan scan = new Scan(row1);
2886       int versions = 3;
2887       scan.setMaxVersions(versions);
2888       List<Cell> actual = new ArrayList<Cell>();
2889       InternalScanner scanner = region.getScanner(scan);
2890 
2891       boolean hasNext = scanner.next(actual);
2892       assertEquals(false, hasNext);
2893 
2894       // Verify result
2895       for (int i = 0; i < expected.size(); i++) {
2896         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2897       }
2898     } finally {
2899       HRegion.closeHRegion(this.region);
2900       this.region = null;
2901     }
2902   }
2903 
2904   /**
2905    * Added for HBASE-5416
2906    * 
2907    * Here we test scan optimization when only subset of CFs are used in filter
2908    * conditions.
2909    */
2910   @Test
2911   public void testScanner_JoinedScanners() throws IOException {
2912     byte[] cf_essential = Bytes.toBytes("essential");
2913     byte[] cf_joined = Bytes.toBytes("joined");
2914     byte[] cf_alpha = Bytes.toBytes("alpha");
2915     this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
2916     try {
2917       byte[] row1 = Bytes.toBytes("row1");
2918       byte[] row2 = Bytes.toBytes("row2");
2919       byte[] row3 = Bytes.toBytes("row3");
2920 
2921       byte[] col_normal = Bytes.toBytes("d");
2922       byte[] col_alpha = Bytes.toBytes("a");
2923 
2924       byte[] filtered_val = Bytes.toBytes(3);
2925 
2926       Put put = new Put(row1);
2927       put.add(cf_essential, col_normal, Bytes.toBytes(1));
2928       put.add(cf_joined, col_alpha, Bytes.toBytes(1));
2929       region.put(put);
2930 
2931       put = new Put(row2);
2932       put.add(cf_essential, col_alpha, Bytes.toBytes(2));
2933       put.add(cf_joined, col_normal, Bytes.toBytes(2));
2934       put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
2935       region.put(put);
2936 
2937       put = new Put(row3);
2938       put.add(cf_essential, col_normal, filtered_val);
2939       put.add(cf_joined, col_normal, filtered_val);
2940       region.put(put);
2941 
2942       // Check two things:
2943       // 1. result list contains expected values
2944       // 2. result list is sorted properly
2945 
2946       Scan scan = new Scan();
2947       Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
2948           CompareOp.NOT_EQUAL, filtered_val);
2949       scan.setFilter(filter);
2950       scan.setLoadColumnFamiliesOnDemand(true);
2951       InternalScanner s = region.getScanner(scan);
2952 
2953       List<Cell> results = new ArrayList<Cell>();
2954       assertTrue(s.next(results));
2955       assertEquals(results.size(), 1);
2956       results.clear();
2957 
2958       assertTrue(s.next(results));
2959       assertEquals(results.size(), 3);
2960       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
2961       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
2962       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
2963       results.clear();
2964 
2965       assertFalse(s.next(results));
2966       assertEquals(results.size(), 0);
2967     } finally {
2968       HRegion.closeHRegion(this.region);
2969       this.region = null;
2970     }
2971   }
2972 
2973   /**
2974    * HBASE-5416
2975    * 
2976    * Test case when scan limits amount of KVs returned on each next() call.
2977    */
2978   @Test
2979   public void testScanner_JoinedScannersWithLimits() throws IOException {
2980     final byte[] cf_first = Bytes.toBytes("first");
2981     final byte[] cf_second = Bytes.toBytes("second");
2982 
2983     this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
2984     try {
2985       final byte[] col_a = Bytes.toBytes("a");
2986       final byte[] col_b = Bytes.toBytes("b");
2987 
2988       Put put;
2989 
2990       for (int i = 0; i < 10; i++) {
2991         put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
2992         put.add(cf_first, col_a, Bytes.toBytes(i));
2993         if (i < 5) {
2994           put.add(cf_first, col_b, Bytes.toBytes(i));
2995           put.add(cf_second, col_a, Bytes.toBytes(i));
2996           put.add(cf_second, col_b, Bytes.toBytes(i));
2997         }
2998         region.put(put);
2999       }
3000 
3001       Scan scan = new Scan();
3002       scan.setLoadColumnFamiliesOnDemand(true);
3003       Filter bogusFilter = new FilterBase() {
3004         @Override
3005         public boolean isFamilyEssential(byte[] name) {
3006           return Bytes.equals(name, cf_first);
3007         }
3008       };
3009 
3010       scan.setFilter(bogusFilter);
3011       InternalScanner s = region.getScanner(scan);
3012 
3013       // Our data looks like this:
3014       // r0: first:a, first:b, second:a, second:b
3015       // r1: first:a, first:b, second:a, second:b
3016       // r2: first:a, first:b, second:a, second:b
3017       // r3: first:a, first:b, second:a, second:b
3018       // r4: first:a, first:b, second:a, second:b
3019       // r5: first:a
3020       // r6: first:a
3021       // r7: first:a
3022       // r8: first:a
3023       // r9: first:a
3024 
3025       // But due to next's limit set to 3, we should get this:
3026       // r0: first:a, first:b, second:a
3027       // r0: second:b
3028       // r1: first:a, first:b, second:a
3029       // r1: second:b
3030       // r2: first:a, first:b, second:a
3031       // r2: second:b
3032       // r3: first:a, first:b, second:a
3033       // r3: second:b
3034       // r4: first:a, first:b, second:a
3035       // r4: second:b
3036       // r5: first:a
3037       // r6: first:a
3038       // r7: first:a
3039       // r8: first:a
3040       // r9: first:a
3041 
3042       List<Cell> results = new ArrayList<Cell>();
3043       int index = 0;
3044       while (true) {
3045         boolean more = s.next(results, 3);
3046         if ((index >> 1) < 5) {
3047           if (index % 2 == 0)
3048             assertEquals(results.size(), 3);
3049           else
3050             assertEquals(results.size(), 1);
3051         } else
3052           assertEquals(results.size(), 1);
3053         results.clear();
3054         index++;
3055         if (!more)
3056           break;
3057       }
3058     } finally {
3059       HRegion.closeHRegion(this.region);
3060       this.region = null;
3061     }
3062   }
3063 
3064   // ////////////////////////////////////////////////////////////////////////////
3065   // Split test
3066   // ////////////////////////////////////////////////////////////////////////////
3067   /**
3068    * Splits twice and verifies getting from each of the split regions.
3069    * 
3070    * @throws Exception
3071    */
3072   @Test
3073   public void testBasicSplit() throws Exception {
3074     byte[][] families = { fam1, fam2, fam3 };
3075 
3076     Configuration hc = initSplit();
3077     // Setting up region
3078     String method = this.getName();
3079     this.region = initHRegion(tableName, method, hc, families);
3080 
3081     try {
3082       LOG.info("" + HBaseTestCase.addContent(region, fam3));
3083       region.flushcache();
3084       region.compactStores();
3085       byte[] splitRow = region.checkSplit();
3086       assertNotNull(splitRow);
3087       LOG.info("SplitRow: " + Bytes.toString(splitRow));
3088       HRegion[] regions = splitRegion(region, splitRow);
3089       try {
3090         // Need to open the regions.
3091         // TODO: Add an 'open' to HRegion... don't do open by constructing
3092         // instance.
3093         for (int i = 0; i < regions.length; i++) {
3094           regions[i] = HRegion.openHRegion(regions[i], null);
3095         }
3096         // Assert can get rows out of new regions. Should be able to get first
3097         // row from first region and the midkey from second region.
3098         assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
3099         assertGet(regions[1], fam3, splitRow);
3100         // Test I can get scanner and that it starts at right place.
3101         assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
3102         assertScan(regions[1], fam3, splitRow);
3103         // Now prove can't split regions that have references.
3104         for (int i = 0; i < regions.length; i++) {
3105           // Add so much data to this region, we create a store file that is >
3106           // than one of our unsplitable references. it will.
3107           for (int j = 0; j < 2; j++) {
3108             HBaseTestCase.addContent(regions[i], fam3);
3109           }
3110           HBaseTestCase.addContent(regions[i], fam2);
3111           HBaseTestCase.addContent(regions[i], fam1);
3112           regions[i].flushcache();
3113         }
3114 
3115         byte[][] midkeys = new byte[regions.length][];
3116         // To make regions splitable force compaction.
3117         for (int i = 0; i < regions.length; i++) {
3118           regions[i].compactStores();
3119           midkeys[i] = regions[i].checkSplit();
3120         }
3121 
3122         TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
3123         // Split these two daughter regions so then I'll have 4 regions. Will
3124         // split because added data above.
3125         for (int i = 0; i < regions.length; i++) {
3126           HRegion[] rs = null;
3127           if (midkeys[i] != null) {
3128             rs = splitRegion(regions[i], midkeys[i]);
3129             for (int j = 0; j < rs.length; j++) {
3130               sortedMap.put(Bytes.toString(rs[j].getRegionName()), HRegion.openHRegion(rs[j], null));
3131             }
3132           }
3133         }
3134         LOG.info("Made 4 regions");
3135         // The splits should have been even. Test I can get some arbitrary row
3136         // out of each.
3137         int interval = (LAST_CHAR - FIRST_CHAR) / 3;
3138         byte[] b = Bytes.toBytes(START_KEY);
3139         for (HRegion r : sortedMap.values()) {
3140           assertGet(r, fam3, b);
3141           b[0] += interval;
3142         }
3143       } finally {
3144         for (int i = 0; i < regions.length; i++) {
3145           try {
3146             regions[i].close();
3147           } catch (IOException e) {
3148             // Ignore.
3149           }
3150         }
3151       }
3152     } finally {
3153       HRegion.closeHRegion(this.region);
3154       this.region = null;
3155     }
3156   }
3157 
3158   @Test
3159   public void testSplitRegion() throws IOException {
3160     byte[] qualifier = Bytes.toBytes("qualifier");
3161     Configuration hc = initSplit();
3162     int numRows = 10;
3163     byte[][] families = { fam1, fam3 };
3164 
3165     // Setting up region
3166     String method = this.getName();
3167     this.region = initHRegion(tableName, method, hc, families);
3168 
3169     // Put data in region
3170     int startRow = 100;
3171     putData(startRow, numRows, qualifier, families);
3172     int splitRow = startRow + numRows;
3173     putData(splitRow, numRows, qualifier, families);
3174     region.flushcache();
3175 
3176     HRegion[] regions = null;
3177     try {
3178       regions = splitRegion(region, Bytes.toBytes("" + splitRow));
3179       // Opening the regions returned.
3180       for (int i = 0; i < regions.length; i++) {
3181         regions[i] = HRegion.openHRegion(regions[i], null);
3182       }
3183       // Verifying that the region has been split
3184       assertEquals(2, regions.length);
3185 
3186       // Verifying that all data is still there and that data is in the right
3187       // place
3188       verifyData(regions[0], startRow, numRows, qualifier, families);
3189       verifyData(regions[1], splitRow, numRows, qualifier, families);
3190 
3191     } finally {
3192       HRegion.closeHRegion(this.region);
3193       this.region = null;
3194     }
3195   }
3196 
3197   @Test
3198   public void testClearForceSplit() throws IOException {
3199     byte[] qualifier = Bytes.toBytes("qualifier");
3200     Configuration hc = initSplit();
3201     int numRows = 10;
3202     byte[][] families = { fam1, fam3 };
3203 
3204     // Setting up region
3205     String method = this.getName();
3206     this.region = initHRegion(tableName, method, hc, families);
3207 
3208     // Put data in region
3209     int startRow = 100;
3210     putData(startRow, numRows, qualifier, families);
3211     int splitRow = startRow + numRows;
3212     byte[] splitRowBytes = Bytes.toBytes("" + splitRow);
3213     putData(splitRow, numRows, qualifier, families);
3214     region.flushcache();
3215 
3216     HRegion[] regions = null;
3217     try {
3218       // Set force split
3219       region.forceSplit(splitRowBytes);
3220       assertTrue(region.shouldForceSplit());
3221       // Split point should be the force split row
3222       assertTrue(Bytes.equals(splitRowBytes, region.checkSplit()));
3223 
3224       // Add a store that has references.
3225       HStore storeMock = Mockito.mock(HStore.class);
3226       when(storeMock.hasReferences()).thenReturn(true);
3227       when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
3228       when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
3229       when(storeMock.getColumnFamilyName()).thenReturn("cf");
3230       region.stores.put(Bytes.toBytes(storeMock.getColumnFamilyName()), storeMock);
3231       assertTrue(region.hasReferences());
3232 
3233       // Will not split since the store has references.
3234       regions = splitRegion(region, splitRowBytes);
3235       assertNull(regions);
3236 
3237       // Region force split should be cleared after the split try.
3238       assertFalse(region.shouldForceSplit());
3239 
3240       // Remove the store that has references.
3241       region.stores.remove(Bytes.toBytes(storeMock.getColumnFamilyName()));
3242       assertFalse(region.hasReferences());
3243 
3244       // Now we can split.
3245       regions = splitRegion(region, splitRowBytes);
3246 
3247       // Opening the regions returned.
3248       for (int i = 0; i < regions.length; i++) {
3249         regions[i] = HRegion.openHRegion(regions[i], null);
3250       }
3251       // Verifying that the region has been split
3252       assertEquals(2, regions.length);
3253 
3254       // Verifying that all data is still there and that data is in the right
3255       // place
3256       verifyData(regions[0], startRow, numRows, qualifier, families);
3257       verifyData(regions[1], splitRow, numRows, qualifier, families);
3258 
3259     } finally {
3260       HRegion.closeHRegion(this.region);
3261       this.region = null;
3262     }
3263   }
3264 
3265   /**
3266    * Flushes the cache in a thread while scanning. The tests verify that the
3267    * scan is coherent - e.g. the returned results are always of the same or
3268    * later update as the previous results.
3269    * 
3270    * @throws IOException
3271    *           scan / compact
3272    * @throws InterruptedException
3273    *           thread join
3274    */
3275   @Test
3276   public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3277     byte[] family = Bytes.toBytes("family");
3278     int numRows = 1000;
3279     int flushAndScanInterval = 10;
3280     int compactInterval = 10 * flushAndScanInterval;
3281 
3282     String method = "testFlushCacheWhileScanning";
3283     this.region = initHRegion(tableName, method, CONF, family);
3284     try {
3285       FlushThread flushThread = new FlushThread();
3286       flushThread.start();
3287 
3288       Scan scan = new Scan();
3289       scan.addFamily(family);
3290       scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
3291           new BinaryComparator(Bytes.toBytes(5L))));
3292 
3293       int expectedCount = 0;
3294       List<Cell> res = new ArrayList<Cell>();
3295 
3296       boolean toggle = true;
3297       for (long i = 0; i < numRows; i++) {
3298         Put put = new Put(Bytes.toBytes(i));
3299         put.setDurability(Durability.SKIP_WAL);
3300         put.add(family, qual1, Bytes.toBytes(i % 10));
3301         region.put(put);
3302 
3303         if (i != 0 && i % compactInterval == 0) {
3304           // System.out.println("iteration = " + i);
3305           region.compactStores(true);
3306         }
3307 
3308         if (i % 10 == 5L) {
3309           expectedCount++;
3310         }
3311 
3312         if (i != 0 && i % flushAndScanInterval == 0) {
3313           res.clear();
3314           InternalScanner scanner = region.getScanner(scan);
3315           if (toggle) {
3316             flushThread.flush();
3317           }
3318           while (scanner.next(res))
3319             ;
3320           if (!toggle) {
3321             flushThread.flush();
3322           }
3323           assertEquals("i=" + i, expectedCount, res.size());
3324           toggle = !toggle;
3325         }
3326       }
3327 
3328       flushThread.done();
3329       flushThread.join();
3330       flushThread.checkNoError();
3331     } finally {
3332       HRegion.closeHRegion(this.region);
3333       this.region = null;
3334     }
3335   }
3336 
3337   protected class FlushThread extends Thread {
3338     private volatile boolean done;
3339     private Throwable error = null;
3340 
3341     public void done() {
3342       done = true;
3343       synchronized (this) {
3344         interrupt();
3345       }
3346     }
3347 
3348     public void checkNoError() {
3349       if (error != null) {
3350         assertNull(error);
3351       }
3352     }
3353 
3354     @Override
3355     public void run() {
3356       done = false;
3357       while (!done) {
3358         synchronized (this) {
3359           try {
3360             wait();
3361           } catch (InterruptedException ignored) {
3362             if (done) {
3363               break;
3364             }
3365           }
3366         }
3367         try {
3368           region.flushcache();
3369         } catch (IOException e) {
3370           if (!done) {
3371             LOG.error("Error while flusing cache", e);
3372             error = e;
3373           }
3374           break;
3375         }
3376       }
3377 
3378     }
3379 
3380     public void flush() {
3381       synchronized (this) {
3382         notify();
3383       }
3384 
3385     }
3386   }
3387 
3388   /**
3389    * Writes very wide records and scans for the latest every time.. Flushes and
3390    * compacts the region every now and then to keep things realistic.
3391    * 
3392    * @throws IOException
3393    *           by flush / scan / compaction
3394    * @throws InterruptedException
3395    *           when joining threads
3396    */
3397   @Test
3398   public void testWritesWhileScanning() throws IOException, InterruptedException {
3399     int testCount = 100;
3400     int numRows = 1;
3401     int numFamilies = 10;
3402     int numQualifiers = 100;
3403     int flushInterval = 7;
3404     int compactInterval = 5 * flushInterval;
3405     byte[][] families = new byte[numFamilies][];
3406     for (int i = 0; i < numFamilies; i++) {
3407       families[i] = Bytes.toBytes("family" + i);
3408     }
3409     byte[][] qualifiers = new byte[numQualifiers][];
3410     for (int i = 0; i < numQualifiers; i++) {
3411       qualifiers[i] = Bytes.toBytes("qual" + i);
3412     }
3413 
3414     String method = "testWritesWhileScanning";
3415     this.region = initHRegion(tableName, method, CONF, families);
3416     try {
3417       PutThread putThread = new PutThread(numRows, families, qualifiers);
3418       putThread.start();
3419       putThread.waitForFirstPut();
3420 
3421       FlushThread flushThread = new FlushThread();
3422       flushThread.start();
3423 
3424       Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3425 
3426       int expectedCount = numFamilies * numQualifiers;
3427       List<Cell> res = new ArrayList<Cell>();
3428 
3429       long prevTimestamp = 0L;
3430       for (int i = 0; i < testCount; i++) {
3431 
3432         if (i != 0 && i % compactInterval == 0) {
3433           region.compactStores(true);
3434         }
3435 
3436         if (i != 0 && i % flushInterval == 0) {
3437           flushThread.flush();
3438         }
3439 
3440         boolean previousEmpty = res.isEmpty();
3441         res.clear();
3442         InternalScanner scanner = region.getScanner(scan);
3443         while (scanner.next(res))
3444           ;
3445         if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3446           assertEquals("i=" + i, expectedCount, res.size());
3447           long timestamp = res.get(0).getTimestamp();
3448           assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3449               timestamp >= prevTimestamp);
3450           prevTimestamp = timestamp;
3451         }
3452       }
3453 
3454       putThread.done();
3455 
3456       region.flushcache();
3457 
3458       putThread.join();
3459       putThread.checkNoError();
3460 
3461       flushThread.done();
3462       flushThread.join();
3463       flushThread.checkNoError();
3464     } finally {
3465       try {
3466         HRegion.closeHRegion(this.region);
3467       } catch (DroppedSnapshotException dse) {
3468         // We could get this on way out because we interrupt the background flusher and it could
3469         // fail anywhere causing a DSE over in the background flusher... only it is not properly
3470         // dealt with so could still be memory hanging out when we get to here -- memory we can't
3471         // flush because the accounting is 'off' since original DSE.
3472       }
3473       this.region = null;
3474     }
3475   }
3476 
3477   protected class PutThread extends Thread {
3478     private volatile boolean done;
3479     private volatile int numPutsFinished = 0;
3480 
3481     private Throwable error = null;
3482     private int numRows;
3483     private byte[][] families;
3484     private byte[][] qualifiers;
3485 
3486     private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3487       this.numRows = numRows;
3488       this.families = families;
3489       this.qualifiers = qualifiers;
3490     }
3491 
3492     /**
3493      * Block until this thread has put at least one row.
3494      */
3495     public void waitForFirstPut() throws InterruptedException {
3496       // wait until put thread actually puts some data
3497       while (numPutsFinished == 0) {
3498         checkNoError();
3499         Thread.sleep(50);
3500       }
3501     }
3502 
3503     public void done() {
3504       done = true;
3505       synchronized (this) {
3506         interrupt();
3507       }
3508     }
3509 
3510     public void checkNoError() {
3511       if (error != null) {
3512         assertNull(error);
3513       }
3514     }
3515 
3516     @Override
3517     public void run() {
3518       done = false;
3519       while (!done) {
3520         try {
3521           for (int r = 0; r < numRows; r++) {
3522             byte[] row = Bytes.toBytes("row" + r);
3523             Put put = new Put(row);
3524             put.setDurability(Durability.SKIP_WAL);
3525             byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3526             for (byte[] family : families) {
3527               for (byte[] qualifier : qualifiers) {
3528                 put.add(family, qualifier, (long) numPutsFinished, value);
3529               }
3530             }
3531             region.put(put);
3532             numPutsFinished++;
3533             if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3534               System.out.println("put iteration = " + numPutsFinished);
3535               Delete delete = new Delete(row, (long) numPutsFinished - 30);
3536               region.delete(delete);
3537             }
3538             numPutsFinished++;
3539           }
3540         } catch (InterruptedIOException e) {
3541           // This is fine. It means we are done, or didn't get the lock on time
3542         } catch (IOException e) {
3543           LOG.error("error while putting records", e);
3544           error = e;
3545           break;
3546         }
3547       }
3548 
3549     }
3550 
3551   }
3552 
3553   /**
3554    * Writes very wide records and gets the latest row every time.. Flushes and
3555    * compacts the region aggressivly to catch issues.
3556    * 
3557    * @throws IOException
3558    *           by flush / scan / compaction
3559    * @throws InterruptedException
3560    *           when joining threads
3561    */
3562   @Test
3563   public void testWritesWhileGetting() throws Exception {
3564     int testCount = 100;
3565     int numRows = 1;
3566     int numFamilies = 10;
3567     int numQualifiers = 100;
3568     int compactInterval = 100;
3569     byte[][] families = new byte[numFamilies][];
3570     for (int i = 0; i < numFamilies; i++) {
3571       families[i] = Bytes.toBytes("family" + i);
3572     }
3573     byte[][] qualifiers = new byte[numQualifiers][];
3574     for (int i = 0; i < numQualifiers; i++) {
3575       qualifiers[i] = Bytes.toBytes("qual" + i);
3576     }
3577 
3578 
3579     String method = "testWritesWhileGetting";
3580     // This test flushes constantly and can cause many files to be created,
3581     // possibly
3582     // extending over the ulimit. Make sure compactions are aggressive in
3583     // reducing
3584     // the number of HFiles created.
3585     Configuration conf = HBaseConfiguration.create(CONF);
3586     conf.setInt("hbase.hstore.compaction.min", 1);
3587     conf.setInt("hbase.hstore.compaction.max", 1000);
3588     this.region = initHRegion(tableName, method, conf, families);
3589     PutThread putThread = null;
3590     MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3591     try {
3592       putThread = new PutThread(numRows, families, qualifiers);
3593       putThread.start();
3594       putThread.waitForFirstPut();
3595 
3596       // Add a thread that flushes as fast as possible
3597       ctx.addThread(new RepeatingTestThread(ctx) {
3598         private int flushesSinceCompact = 0;
3599         private final int maxFlushesSinceCompact = 20;
3600 
3601         @Override
3602         public void doAnAction() throws Exception {
3603           if (region.flushcache().isCompactionNeeded()) {
3604             ++flushesSinceCompact;
3605           }
3606           // Compact regularly to avoid creating too many files and exceeding
3607           // the ulimit.
3608           if (flushesSinceCompact == maxFlushesSinceCompact) {
3609             region.compactStores(false);
3610             flushesSinceCompact = 0;
3611           }
3612         }
3613       });
3614       ctx.startThreads();
3615 
3616       Get get = new Get(Bytes.toBytes("row0"));
3617       Result result = null;
3618 
3619       int expectedCount = numFamilies * numQualifiers;
3620 
3621       long prevTimestamp = 0L;
3622       for (int i = 0; i < testCount; i++) {
3623 
3624         boolean previousEmpty = result == null || result.isEmpty();
3625         result = region.get(get);
3626         if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3627           assertEquals("i=" + i, expectedCount, result.size());
3628           // TODO this was removed, now what dangit?!
3629           // search looking for the qualifier in question?
3630           long timestamp = 0;
3631           for (Cell kv : result.rawCells()) {
3632             if (CellUtil.matchingFamily(kv, families[0])
3633                 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3634               timestamp = kv.getTimestamp();
3635             }
3636           }
3637           assertTrue(timestamp >= prevTimestamp);
3638           prevTimestamp = timestamp;
3639           Cell previousKV = null;
3640 
3641           for (Cell kv : result.rawCells()) {
3642             byte[] thisValue = CellUtil.cloneValue(kv);
3643             if (previousKV != null) {
3644               if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3645                 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3646                     + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3647                     + "(memStoreTS:" + kv.getMvccVersion() + ")");
3648                 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3649               }
3650             }
3651             previousKV = kv;
3652           }
3653         }
3654       }
3655     } finally {
3656       if (putThread != null)
3657         putThread.done();
3658 
3659       region.flushcache();
3660 
3661       if (putThread != null) {
3662         putThread.join();
3663         putThread.checkNoError();
3664       }
3665 
3666       ctx.stop();
3667       HRegion.closeHRegion(this.region);
3668       this.region = null;
3669     }
3670   }
3671 
3672   @Test
3673   public void testHolesInMeta() throws Exception {
3674     byte[] family = Bytes.toBytes("family");
3675     this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF,
3676         false, family);
3677     try {
3678       byte[] rowNotServed = Bytes.toBytes("a");
3679       Get g = new Get(rowNotServed);
3680       try {
3681         region.get(g);
3682         fail();
3683       } catch (WrongRegionException x) {
3684         // OK
3685       }
3686       byte[] row = Bytes.toBytes("y");
3687       g = new Get(row);
3688       region.get(g);
3689     } finally {
3690       HRegion.closeHRegion(this.region);
3691       this.region = null;
3692     }
3693   }
3694 
3695   @Test
3696   public void testIndexesScanWithOneDeletedRow() throws IOException {
3697     byte[] family = Bytes.toBytes("family");
3698 
3699     // Setting up region
3700     String method = "testIndexesScanWithOneDeletedRow";
3701     this.region = initHRegion(tableName, method, CONF, family);
3702     try {
3703       Put put = new Put(Bytes.toBytes(1L));
3704       put.add(family, qual1, 1L, Bytes.toBytes(1L));
3705       region.put(put);
3706 
3707       region.flushcache();
3708 
3709       Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3710       region.delete(delete);
3711 
3712       put = new Put(Bytes.toBytes(2L));
3713       put.add(family, qual1, 2L, Bytes.toBytes(2L));
3714       region.put(put);
3715 
3716       Scan idxScan = new Scan();
3717       idxScan.addFamily(family);
3718       idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
3719           new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
3720               new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
3721               CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
3722       InternalScanner scanner = region.getScanner(idxScan);
3723       List<Cell> res = new ArrayList<Cell>();
3724 
3725       while (scanner.next(res))
3726         ;
3727       assertEquals(1L, res.size());
3728     } finally {
3729       HRegion.closeHRegion(this.region);
3730       this.region = null;
3731     }
3732   }
3733 
3734   // ////////////////////////////////////////////////////////////////////////////
3735   // Bloom filter test
3736   // ////////////////////////////////////////////////////////////////////////////
3737   @Test
3738   public void testBloomFilterSize() throws IOException {
3739     byte[] fam1 = Bytes.toBytes("fam1");
3740     byte[] qf1 = Bytes.toBytes("col");
3741     byte[] val1 = Bytes.toBytes("value1");
3742     // Create Table
3743     HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
3744         .setBloomFilterType(BloomType.ROWCOL);
3745 
3746     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3747     htd.addFamily(hcd);
3748     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3749     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3750     try {
3751       int num_unique_rows = 10;
3752       int duplicate_multiplier = 2;
3753       int num_storefiles = 4;
3754 
3755       int version = 0;
3756       for (int f = 0; f < num_storefiles; f++) {
3757         for (int i = 0; i < duplicate_multiplier; i++) {
3758           for (int j = 0; j < num_unique_rows; j++) {
3759             Put put = new Put(Bytes.toBytes("row" + j));
3760             put.setDurability(Durability.SKIP_WAL);
3761             put.add(fam1, qf1, version++, val1);
3762             region.put(put);
3763           }
3764         }
3765         region.flushcache();
3766       }
3767       // before compaction
3768       HStore store = (HStore) region.getStore(fam1);
3769       Collection<StoreFile> storeFiles = store.getStorefiles();
3770       for (StoreFile storefile : storeFiles) {
3771         StoreFile.Reader reader = storefile.getReader();
3772         reader.loadFileInfo();
3773         reader.loadBloomfilter();
3774         assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
3775         assertEquals(num_unique_rows, reader.getFilterEntries());
3776       }
3777 
3778       region.compactStores(true);
3779 
3780       // after compaction
3781       storeFiles = store.getStorefiles();
3782       for (StoreFile storefile : storeFiles) {
3783         StoreFile.Reader reader = storefile.getReader();
3784         reader.loadFileInfo();
3785         reader.loadBloomfilter();
3786         assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
3787         assertEquals(num_unique_rows, reader.getFilterEntries());
3788       }
3789     } finally {
3790       HRegion.closeHRegion(this.region);
3791       this.region = null;
3792     }
3793   }
3794 
3795   @Test
3796   public void testAllColumnsWithBloomFilter() throws IOException {
3797     byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
3798     byte[] FAMILY = Bytes.toBytes("family");
3799 
3800     // Create table
3801     HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
3802         .setBloomFilterType(BloomType.ROWCOL);
3803     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
3804     htd.addFamily(hcd);
3805     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3806     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3807     try {
3808       // For row:0, col:0: insert versions 1 through 5.
3809       byte row[] = Bytes.toBytes("row:" + 0);
3810       byte column[] = Bytes.toBytes("column:" + 0);
3811       Put put = new Put(row);
3812       put.setDurability(Durability.SKIP_WAL);
3813       for (long idx = 1; idx <= 4; idx++) {
3814         put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
3815       }
3816       region.put(put);
3817 
3818       // Flush
3819       region.flushcache();
3820 
3821       // Get rows
3822       Get get = new Get(row);
3823       get.setMaxVersions();
3824       Cell[] kvs = region.get(get).rawCells();
3825 
3826       // Check if rows are correct
3827       assertEquals(4, kvs.length);
3828       checkOneCell(kvs[0], FAMILY, 0, 0, 4);
3829       checkOneCell(kvs[1], FAMILY, 0, 0, 3);
3830       checkOneCell(kvs[2], FAMILY, 0, 0, 2);
3831       checkOneCell(kvs[3], FAMILY, 0, 0, 1);
3832     } finally {
3833       HRegion.closeHRegion(this.region);
3834       this.region = null;
3835     }
3836   }
3837 
3838   /**
3839    * Testcase to cover bug-fix for HBASE-2823 Ensures correct delete when
3840    * issuing delete row on columns with bloom filter set to row+col
3841    * (BloomType.ROWCOL)
3842    */
3843   @Test
3844   public void testDeleteRowWithBloomFilter() throws IOException {
3845     byte[] familyName = Bytes.toBytes("familyName");
3846 
3847     // Create Table
3848     HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
3849         .setBloomFilterType(BloomType.ROWCOL);
3850 
3851     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3852     htd.addFamily(hcd);
3853     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3854     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3855     try {
3856       // Insert some data
3857       byte row[] = Bytes.toBytes("row1");
3858       byte col[] = Bytes.toBytes("col1");
3859 
3860       Put put = new Put(row);
3861       put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
3862       region.put(put);
3863       region.flushcache();
3864 
3865       Delete del = new Delete(row);
3866       region.delete(del);
3867       region.flushcache();
3868 
3869       // Get remaining rows (should have none)
3870       Get get = new Get(row);
3871       get.addColumn(familyName, col);
3872 
3873       Cell[] keyValues = region.get(get).rawCells();
3874       assertTrue(keyValues.length == 0);
3875     } finally {
3876       HRegion.closeHRegion(this.region);
3877       this.region = null;
3878     }
3879   }
3880 
3881   @Test
3882   public void testgetHDFSBlocksDistribution() throws Exception {
3883     HBaseTestingUtility htu = new HBaseTestingUtility();
3884     final int DEFAULT_BLOCK_SIZE = 1024;
3885     htu.getConfiguration().setLong("dfs.namenode.fs-limits.min-block-size", 0);
3886     htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
3887     htu.getConfiguration().setInt("dfs.replication", 2);
3888 
3889     // set up a cluster with 3 nodes
3890     MiniHBaseCluster cluster = null;
3891     String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
3892     int regionServersCount = 3;
3893 
3894     try {
3895       cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
3896       byte[][] families = { fam1, fam2 };
3897       HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);
3898 
3899       // Setting up region
3900       byte row[] = Bytes.toBytes("row1");
3901       byte col[] = Bytes.toBytes("col1");
3902 
3903       Put put = new Put(row);
3904       put.add(fam1, col, 1, Bytes.toBytes("test1"));
3905       put.add(fam2, col, 1, Bytes.toBytes("test2"));
3906       ht.put(put);
3907 
3908       HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
3909           .get(0);
3910       firstRegion.flushcache();
3911       HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
3912 
3913       // given the default replication factor is 2 and we have 2 HFiles,
3914       // we will have total of 4 replica of blocks on 3 datanodes; thus there
3915       // must be at least one host that have replica for 2 HFiles. That host's
3916       // weight will be equal to the unique block weight.
3917       long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
3918 
3919       String topHost = blocksDistribution1.getTopHosts().get(0);
3920       long topHostWeight = blocksDistribution1.getWeight(topHost);
3921       assertTrue(uniqueBlocksWeight1 == topHostWeight);
3922 
3923       // use the static method to compute the value, it should be the same.
3924       // static method is used by load balancer or other components
3925       HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
3926           htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
3927       long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
3928 
3929       assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
3930 
3931       ht.close();
3932     } finally {
3933       if (cluster != null) {
3934         htu.shutdownMiniCluster();
3935       }
3936     }
3937   }
3938 
3939   /**
3940    * Testcase to check state of region initialization task set to ABORTED or not
3941    * if any exceptions during initialization
3942    * 
3943    * @throws Exception
3944    */
3945   @Test
3946   public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
3947     TableName tableName = TableName.valueOf(name.getMethodName());
3948     HRegionInfo info = null;
3949     try {
3950       FileSystem fs = Mockito.mock(FileSystem.class);
3951       when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
3952       HTableDescriptor htd = new HTableDescriptor(tableName);
3953       htd.addFamily(new HColumnDescriptor("cf"));
3954       info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
3955           HConstants.EMPTY_BYTE_ARRAY, false);
3956       Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
3957       region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
3958       // region initialization throws IOException and set task state to ABORTED.
3959       region.initialize();
3960       fail("Region initialization should fail due to IOException");
3961     } catch (IOException io) {
3962       List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
3963       for (MonitoredTask monitoredTask : tasks) {
3964         if (!(monitoredTask instanceof MonitoredRPCHandler)
3965             && monitoredTask.getDescription().contains(region.toString())) {
3966           assertTrue("Region state should be ABORTED.",
3967               monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
3968           break;
3969         }
3970       }
3971     } finally {
3972       HRegion.closeHRegion(region);
3973     }
3974   }
3975 
3976   /**
3977    * Verifies that the .regioninfo file is written on region creation and that
3978    * is recreated if missing during region opening.
3979    */
3980   @Test
3981   public void testRegionInfoFileCreation() throws IOException {
3982     Path rootDir = new Path(dir + "testRegionInfoFileCreation");
3983 
3984     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
3985     htd.addFamily(new HColumnDescriptor("cf"));
3986 
3987     HRegionInfo hri = new HRegionInfo(htd.getTableName());
3988 
3989     // Create a region and skip the initialization (like CreateTableHandler)
3990     HRegion region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
3991 //    HRegion region = TEST_UTIL.createLocalHRegion(hri, htd);
3992     Path regionDir = region.getRegionFileSystem().getRegionDir();
3993     FileSystem fs = region.getRegionFileSystem().getFileSystem();
3994     HRegion.closeHRegion(region);
3995 
3996     Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
3997 
3998     // Verify that the .regioninfo file is present
3999     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4000         fs.exists(regionInfoFile));
4001 
4002     // Try to open the region
4003     region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
4004     assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
4005     HRegion.closeHRegion(region);
4006 
4007     // Verify that the .regioninfo file is still there
4008     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4009         fs.exists(regionInfoFile));
4010 
4011     // Remove the .regioninfo file and verify is recreated on region open
4012     fs.delete(regionInfoFile);
4013     assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
4014         fs.exists(regionInfoFile));
4015 
4016     region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
4017 //    region = TEST_UTIL.openHRegion(hri, htd);
4018     assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
4019     HRegion.closeHRegion(region);
4020 
4021     // Verify that the .regioninfo file is still there
4022     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4023         fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
4024   }
4025 
4026   /**
4027    * TestCase for increment
4028    */
4029   private static class Incrementer implements Runnable {
4030     private HRegion region;
4031     private final static byte[] incRow = Bytes.toBytes("incRow");
4032     private final static byte[] family = Bytes.toBytes("family");
4033     private final static byte[] qualifier = Bytes.toBytes("qualifier");
4034     private final static long ONE = 1l;
4035     private int incCounter;
4036 
4037     public Incrementer(HRegion region, int incCounter) {
4038       this.region = region;
4039       this.incCounter = incCounter;
4040     }
4041 
4042     @Override
4043     public void run() {
4044       int count = 0;
4045       while (count < incCounter) {
4046         Increment inc = new Increment(incRow);
4047         inc.addColumn(family, qualifier, ONE);
4048         count++;
4049         try {
4050           region.increment(inc);
4051         } catch (IOException e) {
4052           e.printStackTrace();
4053           break;
4054         }
4055       }
4056     }
4057   }
4058 
4059   /**
4060    * Test case to check increment function with memstore flushing
4061    * @throws Exception
4062    */
4063   @Test
4064   public void testParallelIncrementWithMemStoreFlush() throws Exception {
4065     byte[] family = Incrementer.family;
4066     this.region = initHRegion(tableName, method, CONF, family);
4067     final HRegion region = this.region;
4068     final AtomicBoolean incrementDone = new AtomicBoolean(false);
4069     Runnable flusher = new Runnable() {
4070       @Override
4071       public void run() {
4072         while (!incrementDone.get()) {
4073           try {
4074             region.flushcache();
4075           } catch (Exception e) {
4076             e.printStackTrace();
4077           }
4078         }
4079       }
4080     };
4081 
4082     // after all increment finished, the row will increment to 20*100 = 2000
4083     int threadNum = 20;
4084     int incCounter = 100;
4085     long expected = threadNum * incCounter;
4086     Thread[] incrementers = new Thread[threadNum];
4087     Thread flushThread = new Thread(flusher);
4088     for (int i = 0; i < threadNum; i++) {
4089       incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
4090       incrementers[i].start();
4091     }
4092     flushThread.start();
4093     for (int i = 0; i < threadNum; i++) {
4094       incrementers[i].join();
4095     }
4096 
4097     incrementDone.set(true);
4098     flushThread.join();
4099 
4100     Get get = new Get(Incrementer.incRow);
4101     get.addColumn(Incrementer.family, Incrementer.qualifier);
4102     get.setMaxVersions(1);
4103     Result res = this.region.get(get);
4104     List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
4105 
4106     // we just got the latest version
4107     assertEquals(kvs.size(), 1);
4108     Cell kv = kvs.get(0);
4109     assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
4110     this.region = null;
4111   }
4112 
4113   /**
4114    * TestCase for append
4115    */
4116   private static class Appender implements Runnable {
4117     private HRegion region;
4118     private final static byte[] appendRow = Bytes.toBytes("appendRow");
4119     private final static byte[] family = Bytes.toBytes("family");
4120     private final static byte[] qualifier = Bytes.toBytes("qualifier");
4121     private final static byte[] CHAR = Bytes.toBytes("a");
4122     private int appendCounter;
4123 
4124     public Appender(HRegion region, int appendCounter) {
4125       this.region = region;
4126       this.appendCounter = appendCounter;
4127     }
4128 
4129     @Override
4130     public void run() {
4131       int count = 0;
4132       while (count < appendCounter) {
4133         Append app = new Append(appendRow);
4134         app.add(family, qualifier, CHAR);
4135         count++;
4136         try {
4137           region.append(app);
4138         } catch (IOException e) {
4139           e.printStackTrace();
4140           break;
4141         }
4142       }
4143     }
4144   }
4145 
4146   /**
4147    * Test case to check append function with memstore flushing
4148    * @throws Exception
4149    */
4150   @Test
4151   public void testParallelAppendWithMemStoreFlush() throws Exception {
4152     byte[] family = Appender.family;
4153     this.region = initHRegion(tableName, method, CONF, family);
4154     final HRegion region = this.region;
4155     final AtomicBoolean appendDone = new AtomicBoolean(false);
4156     Runnable flusher = new Runnable() {
4157       @Override
4158       public void run() {
4159         while (!appendDone.get()) {
4160           try {
4161             region.flushcache();
4162           } catch (Exception e) {
4163             e.printStackTrace();
4164           }
4165         }
4166       }
4167     };
4168 
4169     // after all append finished, the value will append to threadNum *
4170     // appendCounter Appender.CHAR
4171     int threadNum = 20;
4172     int appendCounter = 100;
4173     byte[] expected = new byte[threadNum * appendCounter];
4174     for (int i = 0; i < threadNum * appendCounter; i++) {
4175       System.arraycopy(Appender.CHAR, 0, expected, i, 1);
4176     }
4177     Thread[] appenders = new Thread[threadNum];
4178     Thread flushThread = new Thread(flusher);
4179     for (int i = 0; i < threadNum; i++) {
4180       appenders[i] = new Thread(new Appender(this.region, appendCounter));
4181       appenders[i].start();
4182     }
4183     flushThread.start();
4184     for (int i = 0; i < threadNum; i++) {
4185       appenders[i].join();
4186     }
4187 
4188     appendDone.set(true);
4189     flushThread.join();
4190 
4191     Get get = new Get(Appender.appendRow);
4192     get.addColumn(Appender.family, Appender.qualifier);
4193     get.setMaxVersions(1);
4194     Result res = this.region.get(get);
4195     List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
4196 
4197     // we just got the latest version
4198     assertEquals(kvs.size(), 1);
4199     Cell kv = kvs.get(0);
4200     byte[] appendResult = new byte[kv.getValueLength()];
4201     System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
4202     assertArrayEquals(expected, appendResult);
4203     this.region = null;
4204   }
4205 
4206   /**
4207    * Test case to check put function with memstore flushing for same row, same ts
4208    * @throws Exception
4209    */
4210   @Test
4211   public void testPutWithMemStoreFlush() throws Exception {
4212     byte[] family = Bytes.toBytes("family");
4213     ;
4214     byte[] qualifier = Bytes.toBytes("qualifier");
4215     byte[] row = Bytes.toBytes("putRow");
4216     byte[] value = null;
4217     this.region = initHRegion(tableName, method, CONF, family);
4218     Put put = null;
4219     Get get = null;
4220     List<Cell> kvs = null;
4221     Result res = null;
4222 
4223     put = new Put(row);
4224     value = Bytes.toBytes("value0");
4225     put.add(family, qualifier, 1234567l, value);
4226     region.put(put);
4227     get = new Get(row);
4228     get.addColumn(family, qualifier);
4229     get.setMaxVersions();
4230     res = this.region.get(get);
4231     kvs = res.getColumnCells(family, qualifier);
4232     assertEquals(1, kvs.size());
4233     assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4234 
4235     region.flushcache();
4236     get = new Get(row);
4237     get.addColumn(family, qualifier);
4238     get.setMaxVersions();
4239     res = this.region.get(get);
4240     kvs = res.getColumnCells(family, qualifier);
4241     assertEquals(1, kvs.size());
4242     assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4243 
4244     put = new Put(row);
4245     value = Bytes.toBytes("value1");
4246     put.add(family, qualifier, 1234567l, value);
4247     region.put(put);
4248     get = new Get(row);
4249     get.addColumn(family, qualifier);
4250     get.setMaxVersions();
4251     res = this.region.get(get);
4252     kvs = res.getColumnCells(family, qualifier);
4253     assertEquals(1, kvs.size());
4254     assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4255 
4256     region.flushcache();
4257     get = new Get(row);
4258     get.addColumn(family, qualifier);
4259     get.setMaxVersions();
4260     res = this.region.get(get);
4261     kvs = res.getColumnCells(family, qualifier);
4262     assertEquals(1, kvs.size());
4263     assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4264   }
4265 
4266   @Test
4267   public void testDurability() throws Exception {
4268     String method = "testDurability";
4269     // there are 5 x 5 cases:
4270     // table durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) x mutation
4271     // durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT)
4272 
4273     // expected cases for append and sync wal
4274     durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4275     durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4276     durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4277 
4278     durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4279     durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4280     durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4281 
4282     durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4283     durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4284 
4285     durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
4286     durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4287 
4288     durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
4289     durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
4290     durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
4291 
4292     // expected cases for async wal
4293     durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4294     durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4295     durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4296     durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4297     durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
4298     durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4299 
4300     durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4301     durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4302     durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4303     durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4304     durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
4305     durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
4306 
4307     // expect skip wal cases
4308     durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4309     durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4310     durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4311     durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
4312     durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
4313     durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
4314 
4315   }
4316 
4317   private void durabilityTest(String method, Durability tableDurability,
4318       Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
4319       final boolean expectSyncFromLogSyncer) throws Exception {
4320     Configuration conf = HBaseConfiguration.create(CONF);
4321     method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
4322     TableName tableName = TableName.valueOf(method);
4323     byte[] family = Bytes.toBytes("family");
4324     Path logDir = new Path(new Path(dir + method), "log");
4325     HLog hlog = HLogFactory.createHLog(FILESYSTEM, logDir, UUID.randomUUID().toString(), conf);
4326     final HLog log = spy(hlog);
4327     this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
4328         HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, log,
4329         new byte[][] { family });
4330 
4331     Put put = new Put(Bytes.toBytes("r1"));
4332     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
4333     put.setDurability(mutationDurability);
4334     region.put(put);
4335 
4336     //verify append called or not
4337     verify(log, expectAppend ? times(1) : never())
4338       .appendNoSync((HRegionInfo)any(), eq(tableName), (WALEdit)any(), (List<UUID>)any(),
4339         anyLong(), (HTableDescriptor)any(), (AtomicLong)any(), anyBoolean(), anyLong(), anyLong());
4340 
4341     // verify sync called or not
4342     if (expectSync || expectSyncFromLogSyncer) {
4343       TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
4344         @Override
4345         public boolean evaluate() throws Exception {
4346           try {
4347             if (expectSync) {
4348               verify(log, times(1)).sync(anyLong()); // Hregion calls this one
4349             } else if (expectSyncFromLogSyncer) {
4350               verify(log, times(1)).sync(); // log syncer calls this one
4351             }
4352           } catch (Throwable ignore) {
4353           }
4354           return true;
4355         }
4356       });
4357     } else {
4358       verify(log, never()).sync(anyLong());
4359       verify(log, never()).sync();
4360     }
4361 
4362     HRegion.closeHRegion(this.region);
4363     this.region = null;
4364   }
4365 
4366   private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4367     for (int i = startRow; i < startRow + numRows; i++) {
4368       Put put = new Put(Bytes.toBytes("" + i));
4369       put.setDurability(Durability.SKIP_WAL);
4370       for (byte[] family : families) {
4371         put.add(family, qf, null);
4372       }
4373       region.put(put);
4374     }
4375   }
4376 
4377   private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
4378       throws IOException {
4379     for (int i = startRow; i < startRow + numRows; i++) {
4380       byte[] row = Bytes.toBytes("" + i);
4381       Get get = new Get(row);
4382       for (byte[] family : families) {
4383         get.addColumn(family, qf);
4384       }
4385       Result result = newReg.get(get);
4386       Cell[] raw = result.rawCells();
4387       assertEquals(families.length, result.size());
4388       for (int j = 0; j < families.length; j++) {
4389         assertTrue(CellUtil.matchingRow(raw[j], row));
4390         assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
4391         assertTrue(CellUtil.matchingQualifier(raw[j], qf));
4392       }
4393     }
4394   }
4395 
4396   private void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
4397     // Now I have k, get values out and assert they are as expected.
4398     Get get = new Get(k).addFamily(family).setMaxVersions();
4399     Cell[] results = r.get(get).rawCells();
4400     for (int j = 0; j < results.length; j++) {
4401       byte[] tmp = CellUtil.cloneValue(results[j]);
4402       // Row should be equal to value every time.
4403       assertTrue(Bytes.equals(k, tmp));
4404     }
4405   }
4406 
4407   /*
4408    * Assert first value in the passed region is <code>firstValue</code>.
4409    * 
4410    * @param r
4411    * 
4412    * @param fs
4413    * 
4414    * @param firstValue
4415    * 
4416    * @throws IOException
4417    */
4418   private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
4419       throws IOException {
4420     byte[][] families = { fs };
4421     Scan scan = new Scan();
4422     for (int i = 0; i < families.length; i++)
4423       scan.addFamily(families[i]);
4424     InternalScanner s = r.getScanner(scan);
4425     try {
4426       List<Cell> curVals = new ArrayList<Cell>();
4427       boolean first = true;
4428       OUTER_LOOP: while (s.next(curVals)) {
4429         for (Cell kv : curVals) {
4430           byte[] val = CellUtil.cloneValue(kv);
4431           byte[] curval = val;
4432           if (first) {
4433             first = false;
4434             assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4435           } else {
4436             // Not asserting anything. Might as well break.
4437             break OUTER_LOOP;
4438           }
4439         }
4440       }
4441     } finally {
4442       s.close();
4443     }
4444   }
4445 
4446   /**
4447    * Test that we get the expected flush results back
4448    * @throws IOException
4449    */
4450   @Test
4451   public void testFlushResult() throws IOException {
4452     String method = name.getMethodName();
4453     byte[] tableName = Bytes.toBytes(method);
4454     byte[] family = Bytes.toBytes("family");
4455 
4456     this.region = initHRegion(tableName, method, family);
4457 
4458     // empty memstore, flush doesn't run
4459     HRegion.FlushResult fr = region.flushcache();
4460     assertFalse(fr.isFlushSucceeded());
4461     assertFalse(fr.isCompactionNeeded());
4462 
4463     // Flush enough files to get up to the threshold, doesn't need compactions
4464     for (int i = 0; i < 2; i++) {
4465       Put put = new Put(tableName).add(family, family, tableName);
4466       region.put(put);
4467       fr = region.flushcache();
4468       assertTrue(fr.isFlushSucceeded());
4469       assertFalse(fr.isCompactionNeeded());
4470     }
4471 
4472     // Two flushes after the threshold, compactions are needed
4473     for (int i = 0; i < 2; i++) {
4474       Put put = new Put(tableName).add(family, family, tableName);
4475       region.put(put);
4476       fr = region.flushcache();
4477       assertTrue(fr.isFlushSucceeded());
4478       assertTrue(fr.isCompactionNeeded());
4479     }
4480   }
4481 
4482   private Configuration initSplit() {
4483     // Always compact if there is more than one store file.
4484     CONF.setInt("hbase.hstore.compactionThreshold", 2);
4485 
4486     // Make lease timeout longer, lease checks less frequent
4487     CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4488 
4489     CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4490 
4491     // Increase the amount of time between client retries
4492     CONF.setLong("hbase.client.pause", 15 * 1000);
4493 
4494     // This size should make it so we always split using the addContent
4495     // below. After adding all data, the first region is 1.3M
4496     CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4497     return CONF;
4498   }
4499 
4500   /**
4501    * @param tableName
4502    * @param callingMethod
4503    * @param conf
4504    * @param families
4505    * @throws IOException
4506    * @return A region on which you must call
4507    *         {@link HRegion#closeHRegion(HRegion)} when done.
4508    */
4509   public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
4510       byte[]... families) throws IOException {
4511     return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
4512   }
4513 
4514   /**
4515    * @param tableName
4516    * @param callingMethod
4517    * @param conf
4518    * @param families
4519    * @throws IOException
4520    * @return A region on which you must call
4521    *         {@link HRegion#closeHRegion(HRegion)} when done.
4522    */
4523   public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4524       byte[]... families) throws IOException {
4525     return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4526   }
4527 
4528   /**
4529    * @param tableName
4530    * @param callingMethod
4531    * @param conf
4532    * @param isReadOnly
4533    * @param families
4534    * @throws IOException
4535    * @return A region on which you must call
4536    *         {@link HRegion#closeHRegion(HRegion)} when done.
4537    */
4538   public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4539       boolean isReadOnly, byte[]... families) throws IOException {
4540     return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4541   }
4542 
4543   private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4544       String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4545       throws IOException {
4546     return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
4547         Durability.SYNC_WAL, null, families);
4548   }
4549 
4550   /**
4551    * @param tableName
4552    * @param startKey
4553    * @param stopKey
4554    * @param callingMethod
4555    * @param conf
4556    * @param isReadOnly
4557    * @param families
4558    * @throws IOException
4559    * @return A region on which you must call
4560    *         {@link HRegion#closeHRegion(HRegion)} when done.
4561    */
4562   private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4563       String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4564       HLog hlog, byte[]... families) throws IOException {
4565     return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly, durability, hlog, families);
4566   }
4567 
4568   /**
4569    * Assert that the passed in Cell has expected contents for the specified row,
4570    * column & timestamp.
4571    */
4572   private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
4573     String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4574     assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
4575         Bytes.toString(CellUtil.cloneRow(kv)));
4576     assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
4577         Bytes.toString(CellUtil.cloneFamily(kv)));
4578     assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
4579         Bytes.toString(CellUtil.cloneQualifier(kv)));
4580     assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
4581     assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
4582         Bytes.toString(CellUtil.cloneValue(kv)));
4583   }
4584 
4585   @Test (timeout=60000)
4586   public void testReverseScanner_FromMemStore_SingleCF_Normal()
4587       throws IOException {
4588     byte[] rowC = Bytes.toBytes("rowC");
4589     byte[] rowA = Bytes.toBytes("rowA");
4590     byte[] rowB = Bytes.toBytes("rowB");
4591     byte[] cf = Bytes.toBytes("CF");
4592     byte[][] families = { cf };
4593     byte[] col = Bytes.toBytes("C");
4594     long ts = 1;
4595     String method = this.getName();
4596     this.region = initHRegion(tableName, method, families);
4597     try {
4598       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4599       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4600           null);
4601       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4602       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4603       Put put = null;
4604       put = new Put(rowC);
4605       put.add(kv1);
4606       put.add(kv11);
4607       region.put(put);
4608       put = new Put(rowA);
4609       put.add(kv2);
4610       region.put(put);
4611       put = new Put(rowB);
4612       put.add(kv3);
4613       region.put(put);
4614 
4615       Scan scan = new Scan(rowC);
4616       scan.setMaxVersions(5);
4617       scan.setReversed(true);
4618       InternalScanner scanner = region.getScanner(scan);
4619       List<Cell> currRow = new ArrayList<Cell>();
4620       boolean hasNext = scanner.next(currRow);
4621       assertEquals(2, currRow.size());
4622       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4623       assertTrue(hasNext);
4624       currRow.clear();
4625       hasNext = scanner.next(currRow);
4626       assertEquals(1, currRow.size());
4627       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4628       assertTrue(hasNext);
4629       currRow.clear();
4630       hasNext = scanner.next(currRow);
4631       assertEquals(1, currRow.size());
4632       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4633       assertFalse(hasNext);
4634       scanner.close();
4635     } finally {
4636       HRegion.closeHRegion(this.region);
4637       this.region = null;
4638     }
4639   }
4640 
4641   @Test (timeout=60000)
4642   public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
4643       throws IOException {
4644     byte[] rowC = Bytes.toBytes("rowC");
4645     byte[] rowA = Bytes.toBytes("rowA");
4646     byte[] rowB = Bytes.toBytes("rowB");
4647     byte[] rowD = Bytes.toBytes("rowD");
4648     byte[] cf = Bytes.toBytes("CF");
4649     byte[][] families = { cf };
4650     byte[] col = Bytes.toBytes("C");
4651     long ts = 1;
4652     String method = this.getName();
4653     this.region = initHRegion(tableName, method, families);
4654     try {
4655       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4656       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4657           null);
4658       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4659       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4660       Put put = null;
4661       put = new Put(rowC);
4662       put.add(kv1);
4663       put.add(kv11);
4664       region.put(put);
4665       put = new Put(rowA);
4666       put.add(kv2);
4667       region.put(put);
4668       put = new Put(rowB);
4669       put.add(kv3);
4670       region.put(put);
4671 
4672       Scan scan = new Scan(rowD);
4673       List<Cell> currRow = new ArrayList<Cell>();
4674       scan.setReversed(true);
4675       scan.setMaxVersions(5);
4676       InternalScanner scanner = region.getScanner(scan);
4677       boolean hasNext = scanner.next(currRow);
4678       assertEquals(2, currRow.size());
4679       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4680       assertTrue(hasNext);
4681       currRow.clear();
4682       hasNext = scanner.next(currRow);
4683       assertEquals(1, currRow.size());
4684       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4685       assertTrue(hasNext);
4686       currRow.clear();
4687       hasNext = scanner.next(currRow);
4688       assertEquals(1, currRow.size());
4689       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4690       assertFalse(hasNext);
4691       scanner.close();
4692     } finally {
4693       HRegion.closeHRegion(this.region);
4694       this.region = null;
4695     }
4696   }
4697 
4698   @Test (timeout=60000)
4699   public void testReverseScanner_FromMemStore_SingleCF_FullScan()
4700       throws IOException {
4701     byte[] rowC = Bytes.toBytes("rowC");
4702     byte[] rowA = Bytes.toBytes("rowA");
4703     byte[] rowB = Bytes.toBytes("rowB");
4704     byte[] cf = Bytes.toBytes("CF");
4705     byte[][] families = { cf };
4706     byte[] col = Bytes.toBytes("C");
4707     long ts = 1;
4708     String method = this.getName();
4709     this.region = initHRegion(tableName, method, families);
4710     try {
4711       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4712       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4713           null);
4714       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4715       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4716       Put put = null;
4717       put = new Put(rowC);
4718       put.add(kv1);
4719       put.add(kv11);
4720       region.put(put);
4721       put = new Put(rowA);
4722       put.add(kv2);
4723       region.put(put);
4724       put = new Put(rowB);
4725       put.add(kv3);
4726       region.put(put);
4727       Scan scan = new Scan();
4728       List<Cell> currRow = new ArrayList<Cell>();
4729       scan.setReversed(true);
4730       InternalScanner scanner = region.getScanner(scan);
4731       boolean hasNext = scanner.next(currRow);
4732       assertEquals(1, currRow.size());
4733       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4734       assertTrue(hasNext);
4735       currRow.clear();
4736       hasNext = scanner.next(currRow);
4737       assertEquals(1, currRow.size());
4738       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4739       assertTrue(hasNext);
4740       currRow.clear();
4741       hasNext = scanner.next(currRow);
4742       assertEquals(1, currRow.size());
4743       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4744       assertFalse(hasNext);
4745       scanner.close();
4746     } finally {
4747       HRegion.closeHRegion(this.region);
4748       this.region = null;
4749     }
4750   }
4751 
4752   @Test (timeout=60000)
4753   public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
4754     // case for "INCLUDE_AND_SEEK_NEXT_ROW & SEEK_NEXT_ROW" endless loop
4755     byte[] rowA = Bytes.toBytes("rowA");
4756     byte[] rowB = Bytes.toBytes("rowB");
4757     byte[] rowC = Bytes.toBytes("rowC");
4758     byte[] rowD = Bytes.toBytes("rowD");
4759     byte[] rowE = Bytes.toBytes("rowE");
4760     byte[] cf = Bytes.toBytes("CF");
4761     byte[][] families = { cf };
4762     byte[] col1 = Bytes.toBytes("col1");
4763     byte[] col2 = Bytes.toBytes("col2");
4764     long ts = 1;
4765     String method = this.getName();
4766     this.region = initHRegion(tableName, method, families);
4767     try {
4768       KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4769       KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4770       KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4771       KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4772       KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4773       KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4774       Put put = null;
4775       put = new Put(rowA);
4776       put.add(kv1);
4777       region.put(put);
4778       put = new Put(rowB);
4779       put.add(kv2);
4780       region.put(put);
4781       put = new Put(rowC);
4782       put.add(kv3);
4783       region.put(put);
4784       put = new Put(rowD);
4785       put.add(kv4_1);
4786       region.put(put);
4787       put = new Put(rowD);
4788       put.add(kv4_2);
4789       region.put(put);
4790       put = new Put(rowE);
4791       put.add(kv5);
4792       region.put(put);
4793       region.flushcache();
4794       Scan scan = new Scan(rowD, rowA);
4795       scan.addColumn(families[0], col1);
4796       scan.setReversed(true);
4797       List<Cell> currRow = new ArrayList<Cell>();
4798       InternalScanner scanner = region.getScanner(scan);
4799       boolean hasNext = scanner.next(currRow);
4800       assertEquals(1, currRow.size());
4801       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4802       assertTrue(hasNext);
4803       currRow.clear();
4804       hasNext = scanner.next(currRow);
4805       assertEquals(1, currRow.size());
4806       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4807       assertTrue(hasNext);
4808       currRow.clear();
4809       hasNext = scanner.next(currRow);
4810       assertEquals(1, currRow.size());
4811       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4812       assertFalse(hasNext);
4813       scanner.close();
4814 
4815       scan = new Scan(rowD, rowA);
4816       scan.addColumn(families[0], col2);
4817       scan.setReversed(true);
4818       currRow.clear();
4819       scanner = region.getScanner(scan);
4820       hasNext = scanner.next(currRow);
4821       assertEquals(1, currRow.size());
4822       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4823       scanner.close();
4824     } finally {
4825       HRegion.closeHRegion(this.region);
4826       this.region = null;
4827     }
4828   }
4829 
4830   @Test (timeout=60000)
4831   public void testReverseScanner_smaller_blocksize() throws IOException {
4832     // case to ensure no conflict with HFile index optimization
4833     byte[] rowA = Bytes.toBytes("rowA");
4834     byte[] rowB = Bytes.toBytes("rowB");
4835     byte[] rowC = Bytes.toBytes("rowC");
4836     byte[] rowD = Bytes.toBytes("rowD");
4837     byte[] rowE = Bytes.toBytes("rowE");
4838     byte[] cf = Bytes.toBytes("CF");
4839     byte[][] families = { cf };
4840     byte[] col1 = Bytes.toBytes("col1");
4841     byte[] col2 = Bytes.toBytes("col2");
4842     long ts = 1;
4843     String method = this.getName();
4844     HBaseConfiguration config = new HBaseConfiguration();
4845     config.setInt("test.block.size", 1);
4846     this.region = initHRegion(tableName, method, config, families);
4847     try {
4848       KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4849       KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4850       KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4851       KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4852       KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4853       KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4854       Put put = null;
4855       put = new Put(rowA);
4856       put.add(kv1);
4857       region.put(put);
4858       put = new Put(rowB);
4859       put.add(kv2);
4860       region.put(put);
4861       put = new Put(rowC);
4862       put.add(kv3);
4863       region.put(put);
4864       put = new Put(rowD);
4865       put.add(kv4_1);
4866       region.put(put);
4867       put = new Put(rowD);
4868       put.add(kv4_2);
4869       region.put(put);
4870       put = new Put(rowE);
4871       put.add(kv5);
4872       region.put(put);
4873       region.flushcache();
4874       Scan scan = new Scan(rowD, rowA);
4875       scan.addColumn(families[0], col1);
4876       scan.setReversed(true);
4877       List<Cell> currRow = new ArrayList<Cell>();
4878       InternalScanner scanner = region.getScanner(scan);
4879       boolean hasNext = scanner.next(currRow);
4880       assertEquals(1, currRow.size());
4881       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4882       assertTrue(hasNext);
4883       currRow.clear();
4884       hasNext = scanner.next(currRow);
4885       assertEquals(1, currRow.size());
4886       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4887       assertTrue(hasNext);
4888       currRow.clear();
4889       hasNext = scanner.next(currRow);
4890       assertEquals(1, currRow.size());
4891       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4892       assertFalse(hasNext);
4893       scanner.close();
4894 
4895       scan = new Scan(rowD, rowA);
4896       scan.addColumn(families[0], col2);
4897       scan.setReversed(true);
4898       currRow.clear();
4899       scanner = region.getScanner(scan);
4900       hasNext = scanner.next(currRow);
4901       assertEquals(1, currRow.size());
4902       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4903       scanner.close();
4904     } finally {
4905       HRegion.closeHRegion(this.region);
4906       this.region = null;
4907     }
4908   }
4909 
4910   @Test (timeout=60000)
4911   public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
4912       throws IOException {
4913     byte[] row0 = Bytes.toBytes("row0"); // 1 kv
4914     byte[] row1 = Bytes.toBytes("row1"); // 2 kv
4915     byte[] row2 = Bytes.toBytes("row2"); // 4 kv
4916     byte[] row3 = Bytes.toBytes("row3"); // 2 kv
4917     byte[] row4 = Bytes.toBytes("row4"); // 5 kv
4918     byte[] row5 = Bytes.toBytes("row5"); // 2 kv
4919     byte[] cf1 = Bytes.toBytes("CF1");
4920     byte[] cf2 = Bytes.toBytes("CF2");
4921     byte[] cf3 = Bytes.toBytes("CF3");
4922     byte[][] families = { cf1, cf2, cf3 };
4923     byte[] col = Bytes.toBytes("C");
4924     long ts = 1;
4925     String method = this.getName();
4926     HBaseConfiguration conf = new HBaseConfiguration();
4927     // disable compactions in this test.
4928     conf.setInt("hbase.hstore.compactionThreshold", 10000);
4929     this.region = initHRegion(tableName, method, conf, families);
4930     try {
4931       // kv naming style: kv(row number) totalKvCountInThisRow seq no
4932       KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
4933           null);
4934       KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
4935           null);
4936       KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
4937           KeyValue.Type.Put, null);
4938       KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
4939           null);
4940       KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
4941           null);
4942       KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
4943           null);
4944       KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
4945           KeyValue.Type.Put, null);
4946       KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
4947           null);
4948       KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
4949           KeyValue.Type.Put, null);
4950       KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
4951           null);
4952       KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
4953           null);
4954       KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
4955           KeyValue.Type.Put, null);
4956       KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
4957           null);
4958       KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
4959           KeyValue.Type.Put, null);
4960       KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
4961           null);
4962       KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
4963           null);
4964       // hfiles(cf1/cf2) :"row1"(1 kv) / "row2"(1 kv) / "row4"(2 kv)
4965       Put put = null;
4966       put = new Put(row1);
4967       put.add(kv1_2_1);
4968       region.put(put);
4969       put = new Put(row2);
4970       put.add(kv2_4_1);
4971       region.put(put);
4972       put = new Put(row4);
4973       put.add(kv4_5_4);
4974       put.add(kv4_5_5);
4975       region.put(put);
4976       region.flushcache();
4977       // hfiles(cf1/cf3) : "row1" (1 kvs) / "row2" (1 kv) / "row4" (2 kv)
4978       put = new Put(row4);
4979       put.add(kv4_5_1);
4980       put.add(kv4_5_3);
4981       region.put(put);
4982       put = new Put(row1);
4983       put.add(kv1_2_2);
4984       region.put(put);
4985       put = new Put(row2);
4986       put.add(kv2_4_4);
4987       region.put(put);
4988       region.flushcache();
4989       // hfiles(cf1/cf3) : "row2"(2 kv) / "row3"(1 kvs) / "row4" (1 kv)
4990       put = new Put(row4);
4991       put.add(kv4_5_2);
4992       region.put(put);
4993       put = new Put(row2);
4994       put.add(kv2_4_2);
4995       put.add(kv2_4_3);
4996       region.put(put);
4997       put = new Put(row3);
4998       put.add(kv3_2_2);
4999       region.put(put);
5000       region.flushcache();
5001       // memstore(cf1/cf2/cf3) : "row0" (1 kvs) / "row3" ( 1 kv) / "row5" (max)
5002       // ( 2 kv)
5003       put = new Put(row0);
5004       put.add(kv0_1_1);
5005       region.put(put);
5006       put = new Put(row3);
5007       put.add(kv3_2_1);
5008       region.put(put);
5009       put = new Put(row5);
5010       put.add(kv5_2_1);
5011       put.add(kv5_2_2);
5012       region.put(put);
5013       // scan range = ["row4", min), skip the max "row5"
5014       Scan scan = new Scan(row4);
5015       scan.setMaxVersions(5);
5016       scan.setBatch(3);
5017       scan.setReversed(true);
5018       InternalScanner scanner = region.getScanner(scan);
5019       List<Cell> currRow = new ArrayList<Cell>();
5020       boolean hasNext = false;
5021       // 1. scan out "row4" (5 kvs), "row5" can't be scanned out since not
5022       // included in scan range
5023       // "row4" takes 2 next() calls since batch=3
5024       hasNext = scanner.next(currRow);
5025       assertEquals(3, currRow.size());
5026       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5027       assertTrue(hasNext);
5028       currRow.clear();
5029       hasNext = scanner.next(currRow);
5030       assertEquals(2, currRow.size());
5031       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5032       assertTrue(hasNext);
5033       // 2. scan out "row3" (2 kv)
5034       currRow.clear();
5035       hasNext = scanner.next(currRow);
5036       assertEquals(2, currRow.size());
5037       assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5038       assertTrue(hasNext);
5039       // 3. scan out "row2" (4 kvs)
5040       // "row2" takes 2 next() calls since batch=3
5041       currRow.clear();
5042       hasNext = scanner.next(currRow);
5043       assertEquals(3, currRow.size());
5044       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5045       assertTrue(hasNext);
5046       currRow.clear();
5047       hasNext = scanner.next(currRow);
5048       assertEquals(1, currRow.size());
5049       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5050       assertTrue(hasNext);
5051       // 4. scan out "row1" (2 kv)
5052       currRow.clear();
5053       hasNext = scanner.next(currRow);
5054       assertEquals(2, currRow.size());
5055       assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5056       assertTrue(hasNext);
5057       // 5. scan out "row0" (1 kv)
5058       currRow.clear();
5059       hasNext = scanner.next(currRow);
5060       assertEquals(1, currRow.size());
5061       assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
5062       assertFalse(hasNext);
5063 
5064       scanner.close();
5065     } finally {
5066       HRegion.closeHRegion(this.region);
5067       this.region = null;
5068     }
5069   }
5070 
5071   @Test (timeout=60000)
5072   public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
5073       throws IOException {
5074     byte[] row1 = Bytes.toBytes("row1");
5075     byte[] row2 = Bytes.toBytes("row2");
5076     byte[] row3 = Bytes.toBytes("row3");
5077     byte[] row4 = Bytes.toBytes("row4");
5078     byte[] cf1 = Bytes.toBytes("CF1");
5079     byte[] cf2 = Bytes.toBytes("CF2");
5080     byte[] cf3 = Bytes.toBytes("CF3");
5081     byte[] cf4 = Bytes.toBytes("CF4");
5082     byte[][] families = { cf1, cf2, cf3, cf4 };
5083     byte[] col = Bytes.toBytes("C");
5084     long ts = 1;
5085     String method = this.getName();
5086     HBaseConfiguration conf = new HBaseConfiguration();
5087     // disable compactions in this test.
5088     conf.setInt("hbase.hstore.compactionThreshold", 10000);
5089     this.region = initHRegion(tableName, method, conf, families);
5090     try {
5091       KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
5092       KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
5093       KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
5094       KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
5095       // storefile1
5096       Put put = new Put(row1);
5097       put.add(kv1);
5098       region.put(put);
5099       region.flushcache();
5100       // storefile2
5101       put = new Put(row2);
5102       put.add(kv2);
5103       region.put(put);
5104       region.flushcache();
5105       // storefile3
5106       put = new Put(row3);
5107       put.add(kv3);
5108       region.put(put);
5109       region.flushcache();
5110       // memstore
5111       put = new Put(row4);
5112       put.add(kv4);
5113       region.put(put);
5114       // scan range = ["row4", min)
5115       Scan scan = new Scan(row4);
5116       scan.setReversed(true);
5117       scan.setBatch(10);
5118       InternalScanner scanner = region.getScanner(scan);
5119       List<Cell> currRow = new ArrayList<Cell>();
5120       boolean hasNext = scanner.next(currRow);
5121       assertEquals(1, currRow.size());
5122       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5123       assertTrue(hasNext);
5124       currRow.clear();
5125       hasNext = scanner.next(currRow);
5126       assertEquals(1, currRow.size());
5127       assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5128       assertTrue(hasNext);
5129       currRow.clear();
5130       hasNext = scanner.next(currRow);
5131       assertEquals(1, currRow.size());
5132       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5133       assertTrue(hasNext);
5134       currRow.clear();
5135       hasNext = scanner.next(currRow);
5136       assertEquals(1, currRow.size());
5137       assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5138       assertFalse(hasNext);
5139     } finally {
5140       HRegion.closeHRegion(this.region);
5141       this.region = null;
5142     }
5143   }
5144 
5145   /**
5146    * Test for HBASE-14497: Reverse Scan threw StackOverflow caused by readPt checking
5147    */
5148   @Test (timeout = 60000)
5149   public void testReverseScanner_StackOverflow() throws IOException {
5150     byte[] cf1 = Bytes.toBytes("CF1");
5151     byte[][] families = {cf1};
5152     byte[] col = Bytes.toBytes("C");
5153     String method = this.getName();
5154     HBaseConfiguration conf = new HBaseConfiguration();
5155     // disable compactions in this test.
5156     conf.setInt("hbase.hstore.compactionThreshold", 10000);
5157     this.region = initHRegion(tableName, method, conf, families);
5158 
5159     try {
5160       // setup with one storefile and one memstore, to create scanner and get an earlier readPt
5161       Put put = new Put(Bytes.toBytes("19998"));
5162       put.add(cf1, col, Bytes.toBytes("val"));
5163       region.put(put);
5164       region.flushcache();
5165       Put put2 = new Put(Bytes.toBytes("19997"));
5166       put2.add(cf1, col, Bytes.toBytes("val"));
5167       region.put(put2);
5168 
5169       Scan scan = new Scan(Bytes.toBytes("19998"));
5170       scan.setReversed(true);
5171       InternalScanner scanner = region.getScanner(scan);
5172 
5173       // create one storefile contains many rows will be skipped
5174       // to check StoreFileScanner.seekToPreviousRow
5175       for (int i = 10000; i < 20000; i++) {
5176         Put p = new Put(Bytes.toBytes("" + i));
5177         p.add(cf1, col, Bytes.toBytes("" + i));
5178         region.put(p);
5179       }
5180       region.flushcache();
5181 
5182       // create one memstore contains many rows will be skipped
5183       // to check MemStoreScanner.seekToPreviousRow
5184       for (int i = 10000; i < 20000; i++) {
5185         Put p = new Put(Bytes.toBytes("" + i));
5186         p.add(cf1, col, Bytes.toBytes("" + i));
5187         region.put(p);
5188       }
5189 
5190       List<Cell> currRow = new ArrayList<Cell>();
5191       boolean hasNext;
5192       do {
5193         hasNext = scanner.next(currRow);
5194       } while (hasNext);
5195       assertEquals(2, currRow.size());
5196       assertArrayEquals(Bytes.toBytes("19998"), currRow.get(0).getRow());
5197       assertArrayEquals(Bytes.toBytes("19997"), currRow.get(1).getRow());
5198     } finally {
5199       HRegion.closeHRegion(this.region);
5200       this.region = null;
5201     }
5202   }
5203 
5204   @Test (timeout=60000)
5205   public void testSplitRegionWithReverseScan() throws IOException {
5206     byte [] tableName = Bytes.toBytes("testSplitRegionWithReverseScan");
5207     byte [] qualifier = Bytes.toBytes("qualifier");
5208     Configuration hc = initSplit();
5209     int numRows = 3;
5210     byte [][] families = {fam1};
5211 
5212     //Setting up region
5213     String method = this.getName();
5214     this.region = initHRegion(tableName, method, hc, families);
5215 
5216     //Put data in region
5217     int startRow = 100;
5218     putData(startRow, numRows, qualifier, families);
5219     int splitRow = startRow + numRows;
5220     putData(splitRow, numRows, qualifier, families);
5221     int endRow = splitRow + numRows;
5222     region.flushcache();
5223 
5224     HRegion [] regions = null;
5225     try {
5226       regions = splitRegion(region, Bytes.toBytes("" + splitRow));
5227       //Opening the regions returned.
5228       for (int i = 0; i < regions.length; i++) {
5229         regions[i] = HRegion.openHRegion(regions[i], null);
5230       }
5231       //Verifying that the region has been split
5232       assertEquals(2, regions.length);
5233 
5234       //Verifying that all data is still there and that data is in the right
5235       //place
5236       verifyData(regions[0], startRow, numRows, qualifier, families);
5237       verifyData(regions[1], splitRow, numRows, qualifier, families);
5238 
5239       //fire the reverse scan1:  top range, and larger than the last row
5240       Scan scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 10 * numRows)));
5241       scan.setReversed(true);
5242       InternalScanner scanner = regions[1].getScanner(scan);
5243       List<Cell> currRow = new ArrayList<Cell>();
5244       boolean more = false;
5245       int verify = startRow + 2 * numRows - 1;
5246       do {
5247         more = scanner.next(currRow);
5248         assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5249         verify--;
5250         currRow.clear();
5251       } while(more);
5252       assertEquals(verify, startRow + numRows - 1);
5253       scanner.close();
5254       //fire the reverse scan2:  top range, and equals to the last row
5255       scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 2 * numRows - 1)));
5256       scan.setReversed(true);
5257       scanner = regions[1].getScanner(scan);
5258       verify = startRow + 2 * numRows - 1;
5259       do {
5260         more = scanner.next(currRow);
5261         assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5262         verify--;
5263         currRow.clear();
5264       } while(more);
5265       assertEquals(verify, startRow + numRows - 1);
5266       scanner.close();
5267       //fire the reverse scan3:  bottom range, and larger than the last row
5268       scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows)));
5269       scan.setReversed(true);
5270       scanner = regions[0].getScanner(scan);
5271       verify = startRow + numRows - 1;
5272       do {
5273         more = scanner.next(currRow);
5274         assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5275         verify--;
5276         currRow.clear();
5277       } while(more);
5278       assertEquals(verify, 99);
5279       scanner.close();
5280       //fire the reverse scan4:  bottom range, and equals to the last row
5281       scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows - 1)));
5282       scan.setReversed(true);
5283       scanner = regions[0].getScanner(scan);
5284       verify = startRow + numRows - 1;
5285       do {
5286         more = scanner.next(currRow);
5287         assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5288         verify--;
5289         currRow.clear();
5290       } while(more);
5291       assertEquals(verify, startRow - 1);
5292       scanner.close();
5293     } finally {
5294       HRegion.closeHRegion(this.region);
5295       this.region = null;
5296     }
5297   }
5298 
5299   @Test
5300   public void testWriteRequestsCounter() throws IOException {
5301     byte[] fam = Bytes.toBytes("info");
5302     byte[][] families = { fam };
5303     this.region = initHRegion(tableName, method, CONF, families);
5304 
5305     Assert.assertEquals(0L, region.getWriteRequestsCount());
5306 
5307     Put put = new Put(row);
5308     put.add(fam, fam, fam);
5309 
5310     Assert.assertEquals(0L, region.getWriteRequestsCount());
5311     region.put(put);
5312     Assert.assertEquals(1L, region.getWriteRequestsCount());
5313     region.put(put);
5314     Assert.assertEquals(2L, region.getWriteRequestsCount());
5315     region.put(put);
5316     Assert.assertEquals(3L, region.getWriteRequestsCount());
5317 
5318     region.delete(new Delete(row));
5319     Assert.assertEquals(4L, region.getWriteRequestsCount());
5320 
5321     HRegion.closeHRegion(this.region);
5322     this.region = null;
5323   }
5324 
5325   /**
5326    * Test RegionTooBusyException thrown when region is busy
5327    */
5328   @Test (timeout=24000)
5329   public void testRegionTooBusy() throws IOException {
5330     String method = "testRegionTooBusy";
5331     byte[] tableName = Bytes.toBytes(method);
5332     byte[] family = Bytes.toBytes("family");
5333     long defaultBusyWaitDuration = CONF.getLong("hbase.busy.wait.duration",
5334       HRegion.DEFAULT_BUSY_WAIT_DURATION);
5335     CONF.setLong("hbase.busy.wait.duration", 1000);
5336     region = initHRegion(tableName, method, CONF, family);
5337     final AtomicBoolean stopped = new AtomicBoolean(true);
5338     Thread t = new Thread(new Runnable() {
5339       @Override
5340       public void run() {
5341         try {
5342           region.lock.writeLock().lock();
5343           stopped.set(false);
5344           while (!stopped.get()) {
5345             Thread.sleep(100);
5346           }
5347         } catch (InterruptedException ie) {
5348         } finally {
5349           region.lock.writeLock().unlock();
5350         }
5351       }
5352     });
5353     t.start();
5354     Get get = new Get(row);
5355     try {
5356       while (stopped.get()) {
5357         Thread.sleep(100);
5358       }
5359       region.get(get);
5360       fail("Should throw RegionTooBusyException");
5361     } catch (InterruptedException ie) {
5362       fail("test interrupted");
5363     } catch (RegionTooBusyException e) {
5364       // Good, expected
5365     } finally {
5366       stopped.set(true);
5367       try {
5368         t.join();
5369       } catch (Throwable e) {
5370       }
5371 
5372       HRegion.closeHRegion(region);
5373       region = null;
5374       CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration);
5375     }
5376   }
5377 
5378   @Test
5379   public void testCellTTLs() throws IOException {
5380     IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
5381     EnvironmentEdgeManager.injectEdge(edge);
5382 
5383     final byte[] row = Bytes.toBytes("testRow");
5384     final byte[] q1 = Bytes.toBytes("q1");
5385     final byte[] q2 = Bytes.toBytes("q2");
5386     final byte[] q3 = Bytes.toBytes("q3");
5387     final byte[] q4 = Bytes.toBytes("q4");
5388 
5389     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCellTTLs"));
5390     HColumnDescriptor hcd = new HColumnDescriptor(fam1);
5391     hcd.setTimeToLive(10); // 10 seconds
5392     htd.addFamily(hcd);
5393 
5394     Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
5395     conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS);
5396 
5397     HRegion region = HRegion.createHRegion(new HRegionInfo(htd.getTableName(),
5398         HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY),
5399       TEST_UTIL.getDataTestDir(), conf, htd);
5400     assertNotNull(region);
5401     try {
5402       long now = EnvironmentEdgeManager.currentTimeMillis();
5403       // Add a cell that will expire in 5 seconds via cell TTL
5404       region.put(new Put(row).add(new KeyValue(row, fam1, q1, now,
5405         HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
5406           // TTL tags specify ts in milliseconds
5407           new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
5408       // Add a cell that will expire after 10 seconds via family setting
5409       region.put(new Put(row).add(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY));
5410       // Add a cell that will expire in 15 seconds via cell TTL
5411       region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1,
5412         HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
5413           // TTL tags specify ts in milliseconds
5414           new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
5415       // Add a cell that will expire in 20 seconds via family setting
5416       region.put(new Put(row).add(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY));
5417 
5418       // Flush so we are sure store scanning gets this right
5419       region.flushcache();
5420 
5421       // A query at time T+0 should return all cells
5422       Result r = region.get(new Get(row));
5423       assertNotNull(r.getValue(fam1, q1));
5424       assertNotNull(r.getValue(fam1, q2));
5425       assertNotNull(r.getValue(fam1, q3));
5426       assertNotNull(r.getValue(fam1, q4));
5427 
5428       // Increment time to T+5 seconds
5429       edge.incrementTime(5000);
5430 
5431       r = region.get(new Get(row));
5432       assertNull(r.getValue(fam1, q1));
5433       assertNotNull(r.getValue(fam1, q2));
5434       assertNotNull(r.getValue(fam1, q3));
5435       assertNotNull(r.getValue(fam1, q4));
5436 
5437       // Increment time to T+10 seconds
5438       edge.incrementTime(5000);
5439 
5440       r = region.get(new Get(row));
5441       assertNull(r.getValue(fam1, q1));
5442       assertNull(r.getValue(fam1, q2));
5443       assertNotNull(r.getValue(fam1, q3));
5444       assertNotNull(r.getValue(fam1, q4));
5445 
5446       // Increment time to T+15 seconds
5447       edge.incrementTime(5000);
5448 
5449       r = region.get(new Get(row));
5450       assertNull(r.getValue(fam1, q1));
5451       assertNull(r.getValue(fam1, q2));
5452       assertNull(r.getValue(fam1, q3));
5453       assertNotNull(r.getValue(fam1, q4));
5454 
5455       // Increment time to T+20 seconds
5456       edge.incrementTime(10000);
5457 
5458       r = region.get(new Get(row));
5459       assertNull(r.getValue(fam1, q1));
5460       assertNull(r.getValue(fam1, q2));
5461       assertNull(r.getValue(fam1, q3));
5462       assertNull(r.getValue(fam1, q4));
5463 
5464       // Fun with disappearing increments
5465 
5466       // Start at 1
5467       region.put(new Put(row).add(fam1, q1, Bytes.toBytes(1L)));
5468       r = region.get(new Get(row));
5469       byte[] val = r.getValue(fam1, q1);
5470       assertNotNull(val);
5471       assertEquals(Bytes.toLong(val), 1L);
5472 
5473       // Increment with a TTL of 5 seconds
5474       Increment incr = new Increment(row).addColumn(fam1, q1, 1L);
5475       incr.setTTL(5000);
5476       region.increment(incr); // 2
5477 
5478       // New value should be 2
5479       r = region.get(new Get(row));
5480       val = r.getValue(fam1, q1);
5481       assertNotNull(val);
5482       assertEquals(Bytes.toLong(val), 2L);
5483 
5484       // Increment time to T+25 seconds
5485       edge.incrementTime(5000);
5486 
5487       // Value should be back to 1
5488       r = region.get(new Get(row));
5489       val = r.getValue(fam1, q1);
5490       assertNotNull(val);
5491       assertEquals(Bytes.toLong(val), 1L);
5492 
5493       // Increment time to T+30 seconds
5494       edge.incrementTime(5000);
5495 
5496       // Original value written at T+20 should be gone now via family TTL
5497       r = region.get(new Get(row));
5498       assertNull(r.getValue(fam1, q1));
5499 
5500     } finally {
5501       HRegion.closeHRegion(region);
5502     }
5503   }
5504 
5505   @Test
5506   public void testIncrementTimestampsAreMonotonic() throws IOException {
5507     HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5508     ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5509     EnvironmentEdgeManager.injectEdge(edge);
5510 
5511     edge.setValue(10);
5512     Increment inc = new Increment(row);
5513     inc.setDurability(Durability.SKIP_WAL);
5514     inc.addColumn(fam1, qual1, 1L);
5515     region.increment(inc);
5516 
5517     Result result = region.get(new Get(row));
5518     Cell c = result.getColumnLatestCell(fam1, qual1);
5519     assertNotNull(c);
5520     assertEquals(c.getTimestamp(), 10L);
5521 
5522     edge.setValue(1); // clock goes back
5523     region.increment(inc);
5524     result = region.get(new Get(row));
5525     c = result.getColumnLatestCell(fam1, qual1);
5526     assertEquals(c.getTimestamp(), 10L);
5527     assertEquals(Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength()), 2L);
5528   }
5529 
5530   @Test
5531   public void testAppendTimestampsAreMonotonic() throws IOException {
5532     HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5533     ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5534     EnvironmentEdgeManager.injectEdge(edge);
5535 
5536     edge.setValue(10);
5537     Append a = new Append(row);
5538     a.setDurability(Durability.SKIP_WAL);
5539     a.add(fam1, qual1, qual1);
5540     region.append(a);
5541 
5542     Result result = region.get(new Get(row));
5543     Cell c = result.getColumnLatestCell(fam1, qual1);
5544     assertNotNull(c);
5545     assertEquals(c.getTimestamp(), 10L);
5546 
5547     edge.setValue(1); // clock goes back
5548     region.append(a);
5549     result = region.get(new Get(row));
5550     c = result.getColumnLatestCell(fam1, qual1);
5551     assertEquals(c.getTimestamp(), 10L);
5552 
5553     byte[] expected = new byte[qual1.length*2];
5554     System.arraycopy(qual1, 0, expected, 0, qual1.length);
5555     System.arraycopy(qual1, 0, expected, qual1.length, qual1.length);
5556 
5557     assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
5558       expected, 0, expected.length));
5559   }
5560 
5561   @Test
5562   public void testCheckAndMutateTimestampsAreMonotonic() throws IOException {
5563     HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5564     ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5565     EnvironmentEdgeManager.injectEdge(edge);
5566 
5567     edge.setValue(10);
5568     Put p = new Put(row);
5569     p.setDurability(Durability.SKIP_WAL);
5570     p.add(fam1, qual1, qual1);
5571     region.put(p);
5572 
5573     Result result = region.get(new Get(row));
5574     Cell c = result.getColumnLatestCell(fam1, qual1);
5575     assertNotNull(c);
5576     assertEquals(c.getTimestamp(), 10L);
5577 
5578     edge.setValue(1); // clock goes back
5579     p = new Put(row);
5580     p.setDurability(Durability.SKIP_WAL);
5581     p.add(fam1, qual1, qual2);
5582     region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(qual1), p, false);
5583     result = region.get(new Get(row));
5584     c = result.getColumnLatestCell(fam1, qual1);
5585     assertEquals(c.getTimestamp(), 10L);
5586 
5587     assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
5588       qual2, 0, qual2.length));
5589   }
5590 
5591   @Test(timeout = 60000)
5592   public void testBatchMutateWithWrongRegionException() throws Exception {
5593     final byte[] a = Bytes.toBytes("a");
5594     final byte[] b = Bytes.toBytes("b");
5595     final byte[] c = Bytes.toBytes("c"); // exclusive
5596 
5597     int prevLockTimeout = CONF.getInt("hbase.rowlock.wait.duration", 30000);
5598     CONF.setInt("hbase.rowlock.wait.duration", 1000);
5599     final HRegion region = initHRegion(tableName, a, c, name.getMethodName(), CONF, false, fam1);
5600 
5601     Mutation[] mutations = new Mutation[] {
5602         new Put(a).addImmutable(fam1, null, null),
5603         new Put(c).addImmutable(fam1, null, null), // this is outside the region boundary
5604         new Put(b).addImmutable(fam1, null, null),
5605     };
5606 
5607     OperationStatus[] status = region.batchMutate(mutations);
5608     assertEquals(status[0].getOperationStatusCode(), OperationStatusCode.SUCCESS);
5609     assertEquals(status[1].getOperationStatusCode(), OperationStatusCode.SANITY_CHECK_FAILURE);
5610     assertEquals(status[2].getOperationStatusCode(), OperationStatusCode.SUCCESS);
5611 
5612 
5613     // test with a row lock held for a long time
5614     final CountDownLatch obtainedRowLock = new CountDownLatch(1);
5615     ExecutorService exec = Executors.newFixedThreadPool(2);
5616     Future<Void> f1 = exec.submit(new Callable<Void>() {
5617       @Override
5618       public Void call() throws Exception {
5619         LOG.info("Acquiring row lock");
5620         RowLock rl = region.getRowLock(b);
5621         obtainedRowLock.countDown();
5622         LOG.info("Waiting for 5 seconds before releasing lock");
5623         Threads.sleep(5000);
5624         LOG.info("Releasing row lock");
5625         rl.release();
5626         return null;
5627       }
5628     });
5629     obtainedRowLock.await(30, TimeUnit.SECONDS);
5630 
5631     Future<Void> f2 = exec.submit(new Callable<Void>() {
5632       @Override
5633       public Void call() throws Exception {
5634         Mutation[] mutations = new Mutation[] {
5635             new Put(a).addImmutable(fam1, null, null),
5636             new Put(b).addImmutable(fam1, null, null),
5637         };
5638 
5639         // this will wait for the row lock, and it will eventually succeed
5640         OperationStatus[] status = region.batchMutate(mutations);
5641         assertEquals(status[0].getOperationStatusCode(), OperationStatusCode.SUCCESS);
5642         assertEquals(status[1].getOperationStatusCode(), OperationStatusCode.SUCCESS);
5643         return null;
5644       }
5645     });
5646 
5647     f1.get();
5648     f2.get();
5649 
5650     CONF.setInt("hbase.rowlock.wait.duration", prevLockTimeout);
5651   }
5652 
5653   @Test
5654   public void testCheckAndRowMutateTimestampsAreMonotonic() throws IOException {
5655     HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5656     ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5657     EnvironmentEdgeManager.injectEdge(edge);
5658 
5659     edge.setValue(10);
5660     Put p = new Put(row);
5661     p.setDurability(Durability.SKIP_WAL);
5662     p.add(fam1, qual1, qual1);
5663     region.put(p);
5664 
5665     Result result = region.get(new Get(row));
5666     Cell c = result.getColumnLatestCell(fam1, qual1);
5667     assertNotNull(c);
5668     assertEquals(c.getTimestamp(), 10L);
5669 
5670     edge.setValue(1); // clock goes back
5671     p = new Put(row);
5672     p.setDurability(Durability.SKIP_WAL);
5673     p.add(fam1, qual1, qual2);
5674     RowMutations rm = new RowMutations(row);
5675     rm.add(p);
5676     region.checkAndRowMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(qual1),
5677       rm, false);
5678     result = region.get(new Get(row));
5679     c = result.getColumnLatestCell(fam1, qual1);
5680     assertEquals(c.getTimestamp(), 10L);
5681 
5682     assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
5683       qual2, 0, qual2.length));
5684   }
5685 
5686   private static HRegion initHRegion(byte[] tableName, String callingMethod,
5687       byte[]... families) throws IOException {
5688     return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
5689         families);
5690   }
5691 }