View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  
22  import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23  import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24  import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25  import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29  import static org.junit.Assert.assertArrayEquals;
30  import static org.junit.Assert.assertEquals;
31  import static org.junit.Assert.assertFalse;
32  import static org.junit.Assert.assertNotNull;
33  import static org.junit.Assert.assertNull;
34  import static org.junit.Assert.assertTrue;
35  import static org.junit.Assert.fail;
36  import static org.mockito.Matchers.any;
37  import static org.mockito.Matchers.anyBoolean;
38  import static org.mockito.Matchers.anyLong;
39  import static org.mockito.Matchers.eq;
40  import static org.mockito.Mockito.never;
41  import static org.mockito.Mockito.spy;
42  import static org.mockito.Mockito.times;
43  import static org.mockito.Mockito.verify;
44  import static org.mockito.Mockito.when;
45  
46  import java.io.IOException;
47  import java.io.InterruptedIOException;
48  import java.security.PrivilegedExceptionAction;
49  import java.util.ArrayList;
50  import java.util.Arrays;
51  import java.util.Collection;
52  import java.util.List;
53  import java.util.Map;
54  import java.util.NavigableMap;
55  import java.util.TreeMap;
56  import java.util.UUID;
57  import java.util.concurrent.atomic.AtomicBoolean;
58  import java.util.concurrent.atomic.AtomicInteger;
59  import java.util.concurrent.atomic.AtomicLong;
60  import java.util.concurrent.atomic.AtomicReference;
61  
62  import org.apache.commons.logging.Log;
63  import org.apache.commons.logging.LogFactory;
64  import org.apache.hadoop.conf.Configuration;
65  import org.apache.hadoop.fs.FSDataOutputStream;
66  import org.apache.hadoop.fs.FileStatus;
67  import org.apache.hadoop.fs.FileSystem;
68  import org.apache.hadoop.fs.Path;
69  import org.apache.hadoop.hbase.Cell;
70  import org.apache.hadoop.hbase.CellComparator;
71  import org.apache.hadoop.hbase.CellUtil;
72  import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
73  import org.apache.hadoop.hbase.DroppedSnapshotException;
74  import org.apache.hadoop.hbase.HBaseConfiguration;
75  import org.apache.hadoop.hbase.HBaseTestCase;
76  import org.apache.hadoop.hbase.HBaseTestingUtility;
77  import org.apache.hadoop.hbase.HColumnDescriptor;
78  import org.apache.hadoop.hbase.HConstants;
79  import org.apache.hadoop.hbase.RegionTooBusyException;
80  import org.apache.hadoop.hbase.Tag;
81  import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
82  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
83  import org.apache.hadoop.hbase.HRegionInfo;
84  import org.apache.hadoop.hbase.HTableDescriptor;
85  import org.apache.hadoop.hbase.KeyValue;
86  import org.apache.hadoop.hbase.testclassification.MediumTests;
87  import org.apache.hadoop.hbase.MiniHBaseCluster;
88  import org.apache.hadoop.hbase.MultithreadedTestUtil;
89  import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
90  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
91  import org.apache.hadoop.hbase.NotServingRegionException;
92  import org.apache.hadoop.hbase.TableName;
93  import org.apache.hadoop.hbase.TagType;
94  import org.apache.hadoop.hbase.Waiter;
95  import org.apache.hadoop.hbase.client.Append;
96  import org.apache.hadoop.hbase.client.Delete;
97  import org.apache.hadoop.hbase.client.Durability;
98  import org.apache.hadoop.hbase.client.Get;
99  import org.apache.hadoop.hbase.client.HTable;
100 import org.apache.hadoop.hbase.client.Increment;
101 import org.apache.hadoop.hbase.client.Put;
102 import org.apache.hadoop.hbase.client.Result;
103 import org.apache.hadoop.hbase.client.RowMutations;
104 import org.apache.hadoop.hbase.client.Scan;
105 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
106 import org.apache.hadoop.hbase.filter.BinaryComparator;
107 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
108 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
109 import org.apache.hadoop.hbase.filter.Filter;
110 import org.apache.hadoop.hbase.filter.FilterBase;
111 import org.apache.hadoop.hbase.filter.FilterList;
112 import org.apache.hadoop.hbase.filter.NullComparator;
113 import org.apache.hadoop.hbase.filter.PrefixFilter;
114 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
115 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
116 import org.apache.hadoop.hbase.io.hfile.HFile;
117 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
118 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
119 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
120 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
121 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
122 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
123 import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
124 import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
125 import org.apache.hadoop.hbase.regionserver.wal.FaultyHLog;
126 import org.apache.hadoop.hbase.regionserver.wal.HLog;
127 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
128 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
129 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
130 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
131 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
132 import org.apache.hadoop.hbase.security.User;
133 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
134 import org.apache.hadoop.hbase.util.Bytes;
135 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
136 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
137 import org.apache.hadoop.hbase.util.FSUtils;
138 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
139 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
140 import org.apache.hadoop.hbase.util.PairOfSameType;
141 import org.apache.hadoop.hbase.util.Threads;
142 import org.junit.After;
143 import org.junit.Assert;
144 import org.junit.Before;
145 import org.junit.Rule;
146 import org.junit.Test;
147 import org.junit.experimental.categories.Category;
148 import org.junit.rules.TestName;
149 import org.mockito.Mockito;
150 
151 import com.google.common.collect.ImmutableList;
152 import com.google.common.collect.Lists;
153 import com.google.protobuf.ByteString;
154 
155 /**
156  * Basic stand-alone testing of HRegion.  No clusters!
157  *
158  * A lot of the meta information for an HRegion now lives inside other HRegions
159  * or in the HBaseMaster, so only basic testing is possible.
160  */
161 @Category(MediumTests.class)
162 @SuppressWarnings("deprecation")
163 public class TestHRegion {
164   // Do not spin up clusters in here. If you need to spin up a cluster, do it
165   // over in TestHRegionOnCluster.
166   static final Log LOG = LogFactory.getLog(TestHRegion.class);
167   @Rule public TestName name = new TestName();
168 
169   private static final String COLUMN_FAMILY = "MyCF";
170   private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
171 
172   HRegion region = null;
173   // Do not run unit tests in parallel (? Why not?  It don't work?  Why not?  St.Ack)
174   private static HBaseTestingUtility TEST_UTIL;
175   public static Configuration CONF ;
176   private String dir;
177   private static FileSystem FILESYSTEM;
178   private final int MAX_VERSIONS = 2;
179 
180   // Test names
181   protected byte[] tableName;
182   protected String method;
183   protected final byte[] qual1 = Bytes.toBytes("qual1");
184   protected final byte[] qual2 = Bytes.toBytes("qual2");
185   protected final byte[] qual3 = Bytes.toBytes("qual3");
186   protected final byte[] value1 = Bytes.toBytes("value1");
187   protected final byte[] value2 = Bytes.toBytes("value2");
188   protected final byte[] row = Bytes.toBytes("rowA");
189   protected final byte[] row2 = Bytes.toBytes("rowB");
190 
191   protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
192       .getInstance(MetricsAssertHelper.class);
193 
194   @Before
195   public void setup() throws IOException {
196     TEST_UTIL = HBaseTestingUtility.createLocalHTU();
197     FILESYSTEM = TEST_UTIL.getTestFileSystem();
198     CONF = TEST_UTIL.getConfiguration();
199     dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
200     method = name.getMethodName();
201     tableName = Bytes.toBytes(name.getMethodName());
202   }
203 
204   @After
205   public void tearDown() throws Exception {
206     EnvironmentEdgeManagerTestHelper.reset();
207     LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
208     TEST_UTIL.cleanupTestDir();
209   }
210 
211   String getName() {
212     return name.getMethodName();
213   }
214 
215   /**
216    * Test for Bug 2 of HBASE-10466.
217    * "Bug 2: Conditions for the first flush of region close (so-called pre-flush) If memstoreSize
218    * is smaller than a certain value, or when region close starts a flush is ongoing, the first
219    * flush is skipped and only the second flush takes place. However, two flushes are required in
220    * case previous flush fails and leaves some data in snapshot. The bug could cause loss of data
221    * in current memstore. The fix is removing all conditions except abort check so we ensure 2
222    * flushes for region close."
223    * @throws IOException
224    */
225   @Test (timeout=60000)
226   public void testCloseCarryingSnapshot() throws IOException {
227     HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
228     Store store = region.getStore(COLUMN_FAMILY_BYTES);
229     // Get some random bytes.
230     byte [] value = Bytes.toBytes(name.getMethodName());
231     // Make a random put against our cf.
232     Put put = new Put(value);
233     put.add(COLUMN_FAMILY_BYTES, null, value);
234     // First put something in current memstore, which will be in snapshot after flusher.prepare()
235     region.put(put);
236     StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
237     storeFlushCtx.prepare();
238     // Second put something in current memstore
239     put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
240     region.put(put);
241     // Close with something in memstore and something in the snapshot.  Make sure all is cleared.
242     region.close();
243     assertEquals(0, region.getMemstoreSize().get());
244     HRegion.closeHRegion(region);
245   }
246 
247   /*
248    * This test is for verifying memstore snapshot size is correctly updated in case of rollback
249    * See HBASE-10845
250    */
251   @Test (timeout=60000)
252   public void testMemstoreSnapshotSize() throws IOException {
253     class MyFaultyHLog extends FaultyHLog {
254       StoreFlushContext storeFlushCtx;
255       public MyFaultyHLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
256           throws IOException {
257         super(fs, rootDir, logName, conf);
258       }
259       
260       void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
261         this.storeFlushCtx = storeFlushCtx;
262       }
263 
264       @Override
265       public void sync(long txid) throws IOException {
266         storeFlushCtx.prepare();
267         super.sync(txid);
268       }
269     }
270     
271     FileSystem fs = FileSystem.get(CONF);
272     Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
273     MyFaultyHLog faultyLog = new MyFaultyHLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
274     HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
275       CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES);
276     
277     Store store = region.getStore(COLUMN_FAMILY_BYTES);
278     // Get some random bytes.
279     byte [] value = Bytes.toBytes(name.getMethodName());
280     faultyLog.setStoreFlushCtx(store.createFlushContext(12345));
281     
282     Put put = new Put(value);
283     put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
284     faultyLog.setFailureType(FaultyHLog.FailureType.SYNC);
285 
286     boolean threwIOE = false;
287     try {
288       region.put(put);
289     } catch (IOException ioe) {
290       threwIOE = true;
291     } finally {
292       assertTrue("The regionserver should have thrown an exception", threwIOE);
293     }
294     long sz = store.getFlushableSize();
295     assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
296     HRegion.closeHRegion(region);
297   }
298 
299   /**
300    * Test for HBASE-14229: Flushing canceled by coprocessor still leads to memstoreSize set down
301    */
302   @Test
303   public void testMemstoreSizeWithFlushCanceling() throws IOException {
304     FileSystem fs = FileSystem.get(CONF);
305     Path rootDir = new Path(dir + "testMemstoreSizeWithFlushCanceling");
306     FaultyHLog hLog = new FaultyHLog(fs, rootDir, "testMemstoreSizeWithFlushCanceling", CONF);
307     HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
308         CONF, false, Durability.SYNC_WAL, hLog, COLUMN_FAMILY_BYTES);
309     Store store = region.getStore(COLUMN_FAMILY_BYTES);
310     assertEquals(0, region.getMemstoreSize().get());
311 
312     // Put some value and make sure flush could be completed normally
313     byte [] value = Bytes.toBytes(name.getMethodName());
314     Put put = new Put(value);
315     put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
316     region.put(put);
317     long onePutSize = region.getMemstoreSize().get();
318     assertTrue(onePutSize > 0);
319     region.flushcache();
320     assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize().get());
321     assertEquals("flushable size should be zero", 0, store.getFlushableSize());
322 
323     // save normalCPHost and replaced by mockedCPHost, which will cancel flush requests
324     RegionCoprocessorHost normalCPHost = region.getCoprocessorHost();
325     RegionCoprocessorHost mockedCPHost = Mockito.mock(RegionCoprocessorHost.class);
326     when(mockedCPHost.preFlush(any(HStore.class), any(InternalScanner.class))).thenReturn(null);
327     region.setCoprocessorHost(mockedCPHost);
328     region.put(put);
329     region.flushcache();
330     assertEquals("memstoreSize should NOT be zero", onePutSize, region.getMemstoreSize().get());
331     assertEquals("flushable size should NOT be zero", onePutSize, store.getFlushableSize());
332 
333     // set normalCPHost and flush again, the snapshot will be flushed
334     region.setCoprocessorHost(normalCPHost);
335     region.flushcache();
336     assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize().get());
337     assertEquals("flushable size should be zero", 0, store.getFlushableSize());
338     HRegion.closeHRegion(region);
339   }
340 
341   /**
342    * Test we do not lose data if we fail a flush and then close.
343    * Part of HBase-10466.  Tests the following from the issue description:
344    * "Bug 1: Wrong calculation of HRegion.memstoreSize: When a flush fails, data to be flushed is
345    * kept in each MemStore's snapshot and wait for next flush attempt to continue on it. But when
346    * the next flush succeeds, the counter of total memstore size in HRegion is always deduced by
347    * the sum of current memstore sizes instead of snapshots left from previous failed flush. This
348    * calculation is problematic that almost every time there is failed flush, HRegion.memstoreSize
349    * gets reduced by a wrong value. If region flush could not proceed for a couple cycles, the size
350    * in current memstore could be much larger than the snapshot. It's likely to drift memstoreSize
351    * much smaller than expected. In extreme case, if the error accumulates to even bigger than
352    * HRegion's memstore size limit, any further flush is skipped because flush does not do anything
353    * if memstoreSize is not larger than 0."
354    * @throws Exception
355    */
356   @Test (timeout=60000)
357   public void testFlushSizeAccounting() throws Exception {
358     final Configuration conf = HBaseConfiguration.create(CONF);
359     // Only retry once.
360     conf.setInt("hbase.hstore.flush.retries.number", 1);
361     final User user =
362       User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
363     // Inject our faulty LocalFileSystem
364     conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
365     user.runAs(new PrivilegedExceptionAction<Object>() {
366       @Override
367       public Object run() throws Exception {
368         // Make sure it worked (above is sensitive to caching details in hadoop core)
369         FileSystem fs = FileSystem.get(conf);
370         Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
371         FaultyFileSystem ffs = (FaultyFileSystem)fs;
372         HRegion region = null;
373         try {
374           // Initialize region
375           region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
376           long size = region.getMemstoreSize().get();
377           Assert.assertEquals(0, size);
378           // Put one item into memstore.  Measure the size of one item in memstore.
379           Put p1 = new Put(row);
380           p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
381           region.put(p1);
382           final long sizeOfOnePut = region.getMemstoreSize().get();
383           // Fail a flush which means the current memstore will hang out as memstore 'snapshot'.
384           try {
385             LOG.info("Flushing");
386             region.flushcache();
387             Assert.fail("Didn't bubble up IOE!");
388           } catch (DroppedSnapshotException dse) {
389             // What we are expecting
390             region.closing.set(false); // this is needed for the rest of the test to work
391           }
392           // Make it so all writes succeed from here on out
393           ffs.fault.set(false);
394           // Check sizes.  Should still be the one entry.
395           Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize().get());
396           // Now add two entries so that on this next flush that fails, we can see if we
397           // subtract the right amount, the snapshot size only.
398           Put p2 = new Put(row);
399           p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
400           p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
401           region.put(p2);
402           Assert.assertEquals(sizeOfOnePut * 3, region.getMemstoreSize().get());
403           // Do a successful flush.  It will clear the snapshot only.  Thats how flushes work.
404           // If already a snapshot, we clear it else we move the memstore to be snapshot and flush
405           // it
406           region.flushcache();
407           // Make sure our memory accounting is right.
408           Assert.assertEquals(sizeOfOnePut * 2, region.getMemstoreSize().get());
409         } finally {
410           HRegion.closeHRegion(region);
411         }
412         return null;
413       }
414     });
415     FileSystem.closeAllForUGI(user.getUGI());
416   }
417 
418   @Test
419   public void testCompactionAffectedByScanners() throws Exception {
420     byte[] family = Bytes.toBytes("family");
421     this.region = initHRegion(tableName, method, CONF, family);
422 
423     Put put = new Put(Bytes.toBytes("r1"));
424     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
425     region.put(put);
426     region.flushcache();
427 
428     Scan scan = new Scan();
429     scan.setMaxVersions(3);
430     // open the first scanner
431     RegionScanner scanner1 = region.getScanner(scan);
432 
433     Delete delete = new Delete(Bytes.toBytes("r1"));
434     region.delete(delete);
435     region.flushcache();
436 
437     // open the second scanner
438     RegionScanner scanner2 = region.getScanner(scan);
439 
440     List<Cell> results = new ArrayList<Cell>();
441 
442     System.out.println("Smallest read point:" + region.getSmallestReadPoint());
443 
444     // make a major compaction
445     region.compactStores(true);
446 
447     // open the third scanner
448     RegionScanner scanner3 = region.getScanner(scan);
449 
450     // get data from scanner 1, 2, 3 after major compaction
451     scanner1.next(results);
452     System.out.println(results);
453     assertEquals(1, results.size());
454 
455     results.clear();
456     scanner2.next(results);
457     System.out.println(results);
458     assertEquals(0, results.size());
459 
460     results.clear();
461     scanner3.next(results);
462     System.out.println(results);
463     assertEquals(0, results.size());
464   }
465 
466   @Test
467   public void testToShowNPEOnRegionScannerReseek() throws Exception {
468     byte[] family = Bytes.toBytes("family");
469     this.region = initHRegion(tableName, method, CONF, family);
470 
471     Put put = new Put(Bytes.toBytes("r1"));
472     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
473     region.put(put);
474     put = new Put(Bytes.toBytes("r2"));
475     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
476     region.put(put);
477     region.flushcache();
478 
479     Scan scan = new Scan();
480     scan.setMaxVersions(3);
481     // open the first scanner
482     RegionScanner scanner1 = region.getScanner(scan);
483 
484     System.out.println("Smallest read point:" + region.getSmallestReadPoint());
485 
486     region.compactStores(true);
487 
488     scanner1.reseek(Bytes.toBytes("r2"));
489     List<Cell> results = new ArrayList<Cell>();
490     scanner1.next(results);
491     Cell keyValue = results.get(0);
492     Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
493     scanner1.close();
494   }
495 
496   @Test
497   public void testSkipRecoveredEditsReplay() throws Exception {
498     String method = "testSkipRecoveredEditsReplay";
499     TableName tableName = TableName.valueOf(method);
500     byte[] family = Bytes.toBytes("family");
501     this.region = initHRegion(tableName, method, CONF, family);
502     try {
503       Path regiondir = region.getRegionFileSystem().getRegionDir();
504       FileSystem fs = region.getRegionFileSystem().getFileSystem();
505       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
506 
507       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
508 
509       long maxSeqId = 1050;
510       long minSeqId = 1000;
511 
512       for (long i = minSeqId; i <= maxSeqId; i += 10) {
513         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
514         fs.create(recoveredEdits);
515         HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
516 
517         long time = System.nanoTime();
518         WALEdit edit = new WALEdit();
519         edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
520             .toBytes(i)));
521         writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
522             HConstants.DEFAULT_CLUSTER_ID), edit));
523 
524         writer.close();
525       }
526       MonitoredTask status = TaskMonitor.get().createStatus(method);
527       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
528       for (Store store : region.getStores().values()) {
529         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
530       }
531       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
532       assertEquals(maxSeqId, seqId);
533       Get get = new Get(row);
534       Result result = region.get(get);
535       for (long i = minSeqId; i <= maxSeqId; i += 10) {
536         List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
537         assertEquals(1, kvs.size());
538         assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
539       }
540     } finally {
541       HRegion.closeHRegion(this.region);
542       this.region = null;
543     }
544   }
545 
546   @Test
547   public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
548     String method = "testSkipRecoveredEditsReplaySomeIgnored";
549     TableName tableName = TableName.valueOf(method);
550     byte[] family = Bytes.toBytes("family");
551     this.region = initHRegion(tableName, method, CONF, family);
552     try {
553       Path regiondir = region.getRegionFileSystem().getRegionDir();
554       FileSystem fs = region.getRegionFileSystem().getFileSystem();
555       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
556 
557       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
558 
559       long maxSeqId = 1050;
560       long minSeqId = 1000;
561 
562       for (long i = minSeqId; i <= maxSeqId; i += 10) {
563         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
564         fs.create(recoveredEdits);
565         HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
566 
567         long time = System.nanoTime();
568         WALEdit edit = new WALEdit();
569         edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
570             .toBytes(i)));
571         writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
572             HConstants.DEFAULT_CLUSTER_ID), edit));
573 
574         writer.close();
575       }
576       long recoverSeqId = 1030;
577       MonitoredTask status = TaskMonitor.get().createStatus(method);
578       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
579       for (Store store : region.getStores().values()) {
580         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
581       }
582       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
583       assertEquals(maxSeqId, seqId);
584       Get get = new Get(row);
585       Result result = region.get(get);
586       for (long i = minSeqId; i <= maxSeqId; i += 10) {
587         List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
588         if (i < recoverSeqId) {
589           assertEquals(0, kvs.size());
590         } else {
591           assertEquals(1, kvs.size());
592           assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
593         }
594       }
595     } finally {
596       HRegion.closeHRegion(this.region);
597       this.region = null;
598     }
599   }
600 
601   @Test
602   public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
603     byte[] family = Bytes.toBytes("family");
604     this.region = initHRegion(tableName, method, CONF, family);
605     try {
606       Path regiondir = region.getRegionFileSystem().getRegionDir();
607       FileSystem fs = region.getRegionFileSystem().getFileSystem();
608 
609       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
610       for (int i = 1000; i < 1050; i += 10) {
611         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
612         FSDataOutputStream dos = fs.create(recoveredEdits);
613         dos.writeInt(i);
614         dos.close();
615       }
616       long minSeqId = 2000;
617       Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
618       FSDataOutputStream dos = fs.create(recoveredEdits);
619       dos.close();
620 
621       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
622       for (Store store : region.getStores().values()) {
623         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
624       }
625       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
626       assertEquals(minSeqId, seqId);
627     } finally {
628       HRegion.closeHRegion(this.region);
629       this.region = null;
630     }
631   }
632 
633   @Test
634   public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
635     String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
636     TableName tableName = TableName.valueOf(method);
637     byte[] family = Bytes.toBytes("family");
638     this.region = initHRegion(tableName, method, CONF, family);
639     try {
640       Path regiondir = region.getRegionFileSystem().getRegionDir();
641       FileSystem fs = region.getRegionFileSystem().getFileSystem();
642       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
643 
644       assertEquals(0, region.getStoreFileList(
645         region.getStores().keySet().toArray(new byte[0][])).size());
646 
647       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
648 
649       long maxSeqId = 1050;
650       long minSeqId = 1000;
651 
652       for (long i = minSeqId; i <= maxSeqId; i += 10) {
653         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
654         fs.create(recoveredEdits);
655         HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
656 
657         long time = System.nanoTime();
658         WALEdit edit = null;
659         if (i == maxSeqId) {
660           edit = WALEdit.createCompaction(region.getRegionInfo(),
661           CompactionDescriptor.newBuilder()
662           .setTableName(ByteString.copyFrom(tableName.getName()))
663           .setFamilyName(ByteString.copyFrom(regionName))
664           .setEncodedRegionName(ByteString.copyFrom(regionName))
665           .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
666           .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName()))
667           .build());
668         } else {
669           edit = new WALEdit();
670           edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
671             .toBytes(i)));
672         }
673         writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
674             HConstants.DEFAULT_CLUSTER_ID), edit));
675         writer.close();
676       }
677 
678       long recoverSeqId = 1030;
679       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
680       MonitoredTask status = TaskMonitor.get().createStatus(method);
681       for (Store store : region.getStores().values()) {
682         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
683       }
684       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
685       assertEquals(maxSeqId, seqId);
686 
687       // assert that the files are flushed
688       assertEquals(1, region.getStoreFileList(
689         region.getStores().keySet().toArray(new byte[0][])).size());
690 
691     } finally {
692       HRegion.closeHRegion(this.region);
693       this.region = null;
694     }  }
695 
696   @Test
697   public void testRecoveredEditsReplayCompaction() throws Exception {
698     String method = name.getMethodName();
699     TableName tableName = TableName.valueOf(method);
700     byte[] family = Bytes.toBytes("family");
701     this.region = initHRegion(tableName, method, CONF, family);
702     try {
703       Path regiondir = region.getRegionFileSystem().getRegionDir();
704       FileSystem fs = region.getRegionFileSystem().getFileSystem();
705       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
706 
707       long maxSeqId = 3;
708       long minSeqId = 0;
709 
710       for (long i = minSeqId; i < maxSeqId; i++) {
711         Put put = new Put(Bytes.toBytes(i));
712         put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
713         region.put(put);
714         region.flushcache();
715       }
716 
717       // this will create a region with 3 files
718       assertEquals(3, region.getStore(family).getStorefilesCount());
719       List<Path> storeFiles = new ArrayList<Path>(3);
720       for (StoreFile sf : region.getStore(family).getStorefiles()) {
721         storeFiles.add(sf.getPath());
722       }
723 
724       // disable compaction completion
725       CONF.setBoolean("hbase.hstore.compaction.complete", false);
726       region.compactStores();
727 
728       // ensure that nothing changed
729       assertEquals(3, region.getStore(family).getStorefilesCount());
730 
731       // now find the compacted file, and manually add it to the recovered edits
732       Path tmpDir = region.getRegionFileSystem().getTempDir();
733       FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
734       String errorMsg = "Expected to find 1 file in the region temp directory "
735           + "from the compaction, could not find any";
736       assertNotNull(errorMsg, files);
737       assertEquals(errorMsg, 1, files.length);
738       // move the file inside region dir
739       Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
740           files[0].getPath());
741 
742       CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
743           .getRegionInfo(), family, storeFiles, Lists.newArrayList(newFile), region
744           .getRegionFileSystem().getStoreDir(Bytes.toString(family)));
745 
746       HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
747           this.region.getRegionInfo(), compactionDescriptor, new AtomicLong(1));
748 
749       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
750 
751       Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
752       fs.create(recoveredEdits);
753       HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
754 
755       long time = System.nanoTime();
756 
757       writer.append(new HLog.Entry(new HLogKey(regionName, tableName, 10, time,
758           HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(),
759           compactionDescriptor)));
760       writer.close();
761 
762       // close the region now, and reopen again
763       region.getTableDesc();
764       region.getRegionInfo();
765       region.close();
766       region = HRegion.openHRegion(region, null);
767 
768       // now check whether we have only one store file, the compacted one
769       Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
770       for (StoreFile sf : sfs) {
771         LOG.info(sf.getPath());
772       }
773       assertEquals(1, region.getStore(family).getStorefilesCount());
774       files = FSUtils.listStatus(fs, tmpDir);
775       assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
776 
777       for (long i = minSeqId; i < maxSeqId; i++) {
778         Get get = new Get(Bytes.toBytes(i));
779         Result result = region.get(get);
780         byte[] value = result.getValue(family, Bytes.toBytes(i));
781         assertArrayEquals(Bytes.toBytes(i), value);
782       }
783     } finally {
784       HRegion.closeHRegion(this.region);
785       this.region = null;
786     }
787   }
788 
789   @Test
790   public void testGetWhileRegionClose() throws IOException {
791     TableName tableName = TableName.valueOf(name.getMethodName());
792     Configuration hc = initSplit();
793     int numRows = 100;
794     byte[][] families = { fam1, fam2, fam3 };
795 
796     // Setting up region
797     String method = name.getMethodName();
798     this.region = initHRegion(tableName, method, hc, families);
799     try {
800       // Put data in region
801       final int startRow = 100;
802       putData(startRow, numRows, qual1, families);
803       putData(startRow, numRows, qual2, families);
804       putData(startRow, numRows, qual3, families);
805       final AtomicBoolean done = new AtomicBoolean(false);
806       final AtomicInteger gets = new AtomicInteger(0);
807       GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
808       try {
809         // Set ten threads running concurrently getting from the region.
810         for (int i = 0; i < threads.length / 2; i++) {
811           threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
812           threads[i].setDaemon(true);
813           threads[i].start();
814         }
815         // Artificially make the condition by setting closing flag explicitly.
816         // I can't make the issue happen with a call to region.close().
817         this.region.closing.set(true);
818         for (int i = threads.length / 2; i < threads.length; i++) {
819           threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
820           threads[i].setDaemon(true);
821           threads[i].start();
822         }
823       } finally {
824         if (this.region != null) {
825           HRegion.closeHRegion(this.region);
826         }
827       }
828       done.set(true);
829       for (GetTillDoneOrException t : threads) {
830         try {
831           t.join();
832         } catch (InterruptedException e) {
833           e.printStackTrace();
834         }
835         if (t.e != null) {
836           LOG.info("Exception=" + t.e);
837           assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
838         }
839       }
840     } finally {
841       HRegion.closeHRegion(this.region);
842       this.region = null;
843     }
844   }
845 
846   /*
847    * Thread that does get on single row until 'done' flag is flipped. If an
848    * exception causes us to fail, it records it.
849    */
850   class GetTillDoneOrException extends Thread {
851     private final Get g;
852     private final AtomicBoolean done;
853     private final AtomicInteger count;
854     private Exception e;
855 
856     GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
857       super("getter." + i);
858       this.g = new Get(r);
859       this.done = d;
860       this.count = c;
861     }
862 
863     @Override
864     public void run() {
865       while (!this.done.get()) {
866         try {
867           assertTrue(region.get(g).size() > 0);
868           this.count.incrementAndGet();
869         } catch (Exception e) {
870           this.e = e;
871           break;
872         }
873       }
874     }
875   }
876 
877   /*
878    * An involved filter test. Has multiple column families and deletes in mix.
879    */
880   @Test
881   public void testWeirdCacheBehaviour() throws Exception {
882     byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
883     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
884         Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
885     this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
886     try {
887       String value = "this is the value";
888       String value2 = "this is some other value";
889       String keyPrefix1 = "prefix1";
890       String keyPrefix2 = "prefix2";
891       String keyPrefix3 = "prefix3";
892       putRows(this.region, 3, value, keyPrefix1);
893       putRows(this.region, 3, value, keyPrefix2);
894       putRows(this.region, 3, value, keyPrefix3);
895       putRows(this.region, 3, value2, keyPrefix1);
896       putRows(this.region, 3, value2, keyPrefix2);
897       putRows(this.region, 3, value2, keyPrefix3);
898       System.out.println("Checking values for key: " + keyPrefix1);
899       assertEquals("Got back incorrect number of rows from scan", 3,
900           getNumberOfRows(keyPrefix1, value2, this.region));
901       System.out.println("Checking values for key: " + keyPrefix2);
902       assertEquals("Got back incorrect number of rows from scan", 3,
903           getNumberOfRows(keyPrefix2, value2, this.region));
904       System.out.println("Checking values for key: " + keyPrefix3);
905       assertEquals("Got back incorrect number of rows from scan", 3,
906           getNumberOfRows(keyPrefix3, value2, this.region));
907       deleteColumns(this.region, value2, keyPrefix1);
908       deleteColumns(this.region, value2, keyPrefix2);
909       deleteColumns(this.region, value2, keyPrefix3);
910       System.out.println("Starting important checks.....");
911       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
912           getNumberOfRows(keyPrefix1, value2, this.region));
913       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
914           getNumberOfRows(keyPrefix2, value2, this.region));
915       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
916           getNumberOfRows(keyPrefix3, value2, this.region));
917     } finally {
918       HRegion.closeHRegion(this.region);
919       this.region = null;
920     }
921   }
922 
923   @Test
924   public void testAppendWithReadOnlyTable() throws Exception {
925     byte[] TABLE = Bytes.toBytes("readOnlyTable");
926     this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
927     boolean exceptionCaught = false;
928     Append append = new Append(Bytes.toBytes("somerow"));
929     append.setDurability(Durability.SKIP_WAL);
930     append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
931         Bytes.toBytes("somevalue"));
932     try {
933       region.append(append);
934     } catch (IOException e) {
935       exceptionCaught = true;
936     } finally {
937       HRegion.closeHRegion(this.region);
938       this.region = null;
939     }
940     assertTrue(exceptionCaught == true);
941   }
942 
943   @Test
944   public void testIncrWithReadOnlyTable() throws Exception {
945     byte[] TABLE = Bytes.toBytes("readOnlyTable");
946     this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
947     boolean exceptionCaught = false;
948     Increment inc = new Increment(Bytes.toBytes("somerow"));
949     inc.setDurability(Durability.SKIP_WAL);
950     inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
951     try {
952       region.increment(inc);
953     } catch (IOException e) {
954       exceptionCaught = true;
955     } finally {
956       HRegion.closeHRegion(this.region);
957       this.region = null;
958     }
959     assertTrue(exceptionCaught == true);
960   }
961 
962   private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
963     InternalScanner scanner = buildScanner(keyPrefix, value, r);
964     int count = 0;
965     boolean more = false;
966     List<Cell> results = new ArrayList<Cell>();
967     do {
968       more = scanner.next(results);
969       if (results != null && !results.isEmpty())
970         count++;
971       else
972         break;
973       Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
974       delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
975       r.delete(delete);
976       results.clear();
977     } while (more);
978     assertEquals("Did not perform correct number of deletes", 3, count);
979   }
980 
981   private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
982     InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
983     int numberOfResults = 0;
984     List<Cell> results = new ArrayList<Cell>();
985     boolean more = false;
986     do {
987       more = resultScanner.next(results);
988       if (results != null && !results.isEmpty())
989         numberOfResults++;
990       else
991         break;
992       for (Cell kv : results) {
993         System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
994       }
995       results.clear();
996     } while (more);
997     return numberOfResults;
998   }
999 
1000   private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
1001       throws IOException {
1002     // Defaults FilterList.Operator.MUST_PASS_ALL.
1003     FilterList allFilters = new FilterList();
1004     allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
1005     // Only return rows where this column value exists in the row.
1006     SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
1007         Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
1008     filter.setFilterIfMissing(true);
1009     allFilters.addFilter(filter);
1010     Scan scan = new Scan();
1011     scan.addFamily(Bytes.toBytes("trans-blob"));
1012     scan.addFamily(Bytes.toBytes("trans-type"));
1013     scan.addFamily(Bytes.toBytes("trans-date"));
1014     scan.addFamily(Bytes.toBytes("trans-tags"));
1015     scan.addFamily(Bytes.toBytes("trans-group"));
1016     scan.setFilter(allFilters);
1017     return r.getScanner(scan);
1018   }
1019 
1020   private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
1021     for (int i = 0; i < numRows; i++) {
1022       String row = key + "_" + i/* UUID.randomUUID().toString() */;
1023       System.out.println(String.format("Saving row: %s, with value %s", row, value));
1024       Put put = new Put(Bytes.toBytes(row));
1025       put.setDurability(Durability.SKIP_WAL);
1026       put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
1027       put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
1028       put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
1029       put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
1030       put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
1031       r.put(put);
1032     }
1033   }
1034 
1035   @Test
1036   public void testFamilyWithAndWithoutColon() throws Exception {
1037     byte[] b = Bytes.toBytes(getName());
1038     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1039     this.region = initHRegion(b, getName(), CONF, cf);
1040     try {
1041       Put p = new Put(b);
1042       byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
1043       p.add(cfwithcolon, cfwithcolon, cfwithcolon);
1044       boolean exception = false;
1045       try {
1046         this.region.put(p);
1047       } catch (NoSuchColumnFamilyException e) {
1048         exception = true;
1049       }
1050       assertTrue(exception);
1051     } finally {
1052       HRegion.closeHRegion(this.region);
1053       this.region = null;
1054     }
1055   }
1056 
1057   @Test
1058   public void testBatchPut_whileNoRowLocksHeld() throws IOException {
1059     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1060     byte[] qual = Bytes.toBytes("qual");
1061     byte[] val = Bytes.toBytes("val");
1062     this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1063     MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1064     try {
1065       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1066       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1067 
1068       LOG.info("First a batch put with all valid puts");
1069       final Put[] puts = new Put[10];
1070       for (int i = 0; i < 10; i++) {
1071         puts[i] = new Put(Bytes.toBytes("row_" + i));
1072         puts[i].add(cf, qual, val);
1073       }
1074 
1075       OperationStatus[] codes = this.region.batchMutate(puts);
1076       assertEquals(10, codes.length);
1077       for (int i = 0; i < 10; i++) {
1078         assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1079       }
1080       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1081 
1082       LOG.info("Next a batch put with one invalid family");
1083       puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1084       codes = this.region.batchMutate(puts);
1085       assertEquals(10, codes.length);
1086       for (int i = 0; i < 10; i++) {
1087         assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1088             codes[i].getOperationStatusCode());
1089       }
1090 
1091       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1092     } finally {
1093       HRegion.closeHRegion(this.region);
1094       this.region = null;
1095     }
1096   }
1097 
1098   @Test
1099   public void testBatchPut_whileMultipleRowLocksHeld() throws Exception {
1100     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1101     byte[] qual = Bytes.toBytes("qual");
1102     byte[] val = Bytes.toBytes("val");
1103     this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1104     MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1105     try {
1106       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1107       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1108 
1109       final Put[] puts = new Put[10];
1110       for (int i = 0; i < 10; i++) {
1111         puts[i] = new Put(Bytes.toBytes("row_" + i));
1112         puts[i].add(cf, qual, val);
1113       }
1114       puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1115 
1116       LOG.info("batchPut will have to break into four batches to avoid row locks");
1117       RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2"));
1118       RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_4"));
1119       RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_6"));
1120 
1121       MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1122       final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
1123       TestThread putter = new TestThread(ctx) {
1124         @Override
1125         public void doWork() throws IOException {
1126           retFromThread.set(region.batchMutate(puts));
1127         }
1128       };
1129       LOG.info("...starting put thread while holding locks");
1130       ctx.addThread(putter);
1131       ctx.startThreads();
1132 
1133       LOG.info("...waiting for put thread to sync 1st time");
1134       waitForCounter(source, "syncTimeNumOps", syncs + 1);
1135 
1136       // Now attempt to close the region from another thread.  Prior to HBASE-12565
1137       // this would cause the in-progress batchMutate operation to to fail with
1138       // exception because it use to release and re-acquire the close-guard lock
1139       // between batches.  Caller then didn't get status indicating which writes succeeded.
1140       // We now expect this thread to block until the batchMutate call finishes.
1141       Thread regionCloseThread = new Thread() {
1142         @Override
1143         public void run() {
1144           try {
1145             HRegion.closeHRegion(region);
1146           } catch (IOException e) {
1147             throw new RuntimeException(e);
1148           }
1149         }
1150       };
1151       regionCloseThread.start();
1152 
1153       LOG.info("...releasing row lock 1, which should let put thread continue");
1154       rowLock1.release();
1155 
1156       LOG.info("...waiting for put thread to sync 2nd time");
1157       waitForCounter(source, "syncTimeNumOps", syncs + 2);
1158 
1159       LOG.info("...releasing row lock 2, which should let put thread continue");
1160       rowLock2.release();
1161 
1162       LOG.info("...waiting for put thread to sync 3rd time");
1163       waitForCounter(source, "syncTimeNumOps", syncs + 3);
1164 
1165       LOG.info("...releasing row lock 3, which should let put thread continue");
1166       rowLock3.release();
1167 
1168       LOG.info("...waiting for put thread to sync 4th time");
1169       waitForCounter(source, "syncTimeNumOps", syncs + 4);
1170 
1171       LOG.info("...joining on put thread");
1172       ctx.stop();
1173       regionCloseThread.join();
1174 
1175       OperationStatus[] codes = retFromThread.get();
1176       for (int i = 0; i < codes.length; i++) {
1177         assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1178             codes[i].getOperationStatusCode());
1179       }
1180     } finally {
1181       HRegion.closeHRegion(this.region);
1182       this.region = null;
1183     }
1184   }
1185 
1186   private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount)
1187       throws InterruptedException {
1188     long startWait = System.currentTimeMillis();
1189     long currentCount;
1190     while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) {
1191       Thread.sleep(100);
1192       if (System.currentTimeMillis() - startWait > 10000) {
1193         fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName,
1194           expectedCount, currentCount));
1195       }
1196     }
1197   }
1198 
1199 
1200   @Test
1201   public void testBatchPutWithTsSlop() throws Exception {
1202     byte[] b = Bytes.toBytes(getName());
1203     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1204     byte[] qual = Bytes.toBytes("qual");
1205     byte[] val = Bytes.toBytes("val");
1206 
1207     // add data with a timestamp that is too recent for range. Ensure assert
1208     CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1209     this.region = initHRegion(b, getName(), CONF, cf);
1210 
1211     try {
1212       MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1213       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1214       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1215 
1216       final Put[] puts = new Put[10];
1217       for (int i = 0; i < 10; i++) {
1218         puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
1219         puts[i].add(cf, qual, val);
1220       }
1221 
1222       OperationStatus[] codes = this.region.batchMutate(puts);
1223       assertEquals(10, codes.length);
1224       for (int i = 0; i < 10; i++) {
1225         assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1226       }
1227       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1228 
1229     } finally {
1230       HRegion.closeHRegion(this.region);
1231       this.region = null;
1232     }
1233 
1234   }
1235 
1236   // ////////////////////////////////////////////////////////////////////////////
1237   // checkAndMutate tests
1238   // ////////////////////////////////////////////////////////////////////////////
1239   @Test
1240   public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1241     byte[] row1 = Bytes.toBytes("row1");
1242     byte[] fam1 = Bytes.toBytes("fam1");
1243     byte[] qf1 = Bytes.toBytes("qualifier");
1244     byte[] emptyVal = new byte[] {};
1245     byte[] val1 = Bytes.toBytes("value1");
1246     byte[] val2 = Bytes.toBytes("value2");
1247 
1248     // Setting up region
1249     String method = this.getName();
1250     this.region = initHRegion(tableName, method, CONF, fam1);
1251     try {
1252       // Putting empty data in key
1253       Put put = new Put(row1);
1254       put.add(fam1, qf1, emptyVal);
1255 
1256       // checkAndPut with empty value
1257       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1258           emptyVal), put, true);
1259       assertTrue(res);
1260 
1261       // Putting data in key
1262       put = new Put(row1);
1263       put.add(fam1, qf1, val1);
1264 
1265       // checkAndPut with correct value
1266       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1267           put, true);
1268       assertTrue(res);
1269 
1270       // not empty anymore
1271       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1272           put, true);
1273       assertFalse(res);
1274 
1275       Delete delete = new Delete(row1);
1276       delete.deleteColumn(fam1, qf1);
1277       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1278           delete, true);
1279       assertFalse(res);
1280 
1281       put = new Put(row1);
1282       put.add(fam1, qf1, val2);
1283       // checkAndPut with correct value
1284       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1285           put, true);
1286       assertTrue(res);
1287 
1288       // checkAndDelete with correct value
1289       delete = new Delete(row1);
1290       delete.deleteColumn(fam1, qf1);
1291       delete.deleteColumn(fam1, qf1);
1292       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1293           delete, true);
1294       assertTrue(res);
1295 
1296       delete = new Delete(row1);
1297       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1298           delete, true);
1299       assertTrue(res);
1300 
1301       // checkAndPut looking for a null value
1302       put = new Put(row1);
1303       put.add(fam1, qf1, val1);
1304 
1305       res = region
1306           .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
1307       assertTrue(res);
1308     } finally {
1309       HRegion.closeHRegion(this.region);
1310       this.region = null;
1311     }
1312   }
1313 
1314   @Test
1315   public void testCheckAndMutate_WithWrongValue() throws IOException {
1316     byte[] row1 = Bytes.toBytes("row1");
1317     byte[] fam1 = Bytes.toBytes("fam1");
1318     byte[] qf1 = Bytes.toBytes("qualifier");
1319     byte[] val1 = Bytes.toBytes("value1");
1320     byte[] val2 = Bytes.toBytes("value2");
1321 
1322     // Setting up region
1323     String method = this.getName();
1324     this.region = initHRegion(tableName, method, CONF, fam1);
1325     try {
1326       // Putting data in key
1327       Put put = new Put(row1);
1328       put.add(fam1, qf1, val1);
1329       region.put(put);
1330 
1331       // checkAndPut with wrong value
1332       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1333           val2), put, true);
1334       assertEquals(false, res);
1335 
1336       // checkAndDelete with wrong value
1337       Delete delete = new Delete(row1);
1338       delete.deleteFamily(fam1);
1339       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1340           put, true);
1341       assertEquals(false, res);
1342     } finally {
1343       HRegion.closeHRegion(this.region);
1344       this.region = null;
1345     }
1346   }
1347 
1348   @Test
1349   public void testCheckAndMutate_WithCorrectValue() throws IOException {
1350     byte[] row1 = Bytes.toBytes("row1");
1351     byte[] fam1 = Bytes.toBytes("fam1");
1352     byte[] qf1 = Bytes.toBytes("qualifier");
1353     byte[] val1 = Bytes.toBytes("value1");
1354 
1355     // Setting up region
1356     String method = this.getName();
1357     this.region = initHRegion(tableName, method, CONF, fam1);
1358     try {
1359       // Putting data in key
1360       Put put = new Put(row1);
1361       put.add(fam1, qf1, val1);
1362       region.put(put);
1363 
1364       // checkAndPut with correct value
1365       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1366           val1), put, true);
1367       assertEquals(true, res);
1368 
1369       // checkAndDelete with correct value
1370       Delete delete = new Delete(row1);
1371       delete.deleteColumn(fam1, qf1);
1372       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1373           delete, true);
1374       assertEquals(true, res);
1375     } finally {
1376       HRegion.closeHRegion(this.region);
1377       this.region = null;
1378     }
1379   }
1380 
1381   @Test
1382   public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1383     byte[] row1 = Bytes.toBytes("row1");
1384     byte[] fam1 = Bytes.toBytes("fam1");
1385     byte[] qf1 = Bytes.toBytes("qualifier");
1386     byte[] val1 = Bytes.toBytes("value1");
1387     byte[] val2 = Bytes.toBytes("value2");
1388     byte[] val3 = Bytes.toBytes("value3");
1389     byte[] val4 = Bytes.toBytes("value4");
1390 
1391     // Setting up region
1392     String method = this.getName();
1393     this.region = initHRegion(tableName, method, CONF, fam1);
1394     try {
1395       // Putting val3 in key
1396       Put put = new Put(row1);
1397       put.add(fam1, qf1, val3);
1398       region.put(put);
1399 
1400       // Test CompareOp.LESS: original = val3, compare with val3, fail
1401       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1402           new BinaryComparator(val3), put, true);
1403       assertEquals(false, res);
1404 
1405       // Test CompareOp.LESS: original = val3, compare with val4, fail
1406       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1407           new BinaryComparator(val4), put, true);
1408       assertEquals(false, res);
1409 
1410       // Test CompareOp.LESS: original = val3, compare with val2,
1411       // succeed (now value = val2)
1412       put = new Put(row1);
1413       put.add(fam1, qf1, val2);
1414       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1415           new BinaryComparator(val2), put, true);
1416       assertEquals(true, res);
1417 
1418       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail
1419       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1420           new BinaryComparator(val3), put, true);
1421       assertEquals(false, res);
1422 
1423       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2,
1424       // succeed (value still = val2)
1425       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1426           new BinaryComparator(val2), put, true);
1427       assertEquals(true, res);
1428 
1429       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1,
1430       // succeed (now value = val3)
1431       put = new Put(row1);
1432       put.add(fam1, qf1, val3);
1433       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1434           new BinaryComparator(val1), put, true);
1435       assertEquals(true, res);
1436 
1437       // Test CompareOp.GREATER: original = val3, compare with val3, fail
1438       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1439           new BinaryComparator(val3), put, true);
1440       assertEquals(false, res);
1441 
1442       // Test CompareOp.GREATER: original = val3, compare with val2, fail
1443       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1444           new BinaryComparator(val2), put, true);
1445       assertEquals(false, res);
1446 
1447       // Test CompareOp.GREATER: original = val3, compare with val4,
1448       // succeed (now value = val2)
1449       put = new Put(row1);
1450       put.add(fam1, qf1, val2);
1451       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1452           new BinaryComparator(val4), put, true);
1453       assertEquals(true, res);
1454 
1455       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail
1456       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1457           new BinaryComparator(val1), put, true);
1458       assertEquals(false, res);
1459 
1460       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2,
1461       // succeed (value still = val2)
1462       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1463           new BinaryComparator(val2), put, true);
1464       assertEquals(true, res);
1465 
1466       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed
1467       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1468           new BinaryComparator(val3), put, true);
1469       assertEquals(true, res);
1470     } finally {
1471       HRegion.closeHRegion(this.region);
1472       this.region = null;
1473     }
1474   }
1475 
1476   @Test
1477   public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1478     byte[] row1 = Bytes.toBytes("row1");
1479     byte[] fam1 = Bytes.toBytes("fam1");
1480     byte[] fam2 = Bytes.toBytes("fam2");
1481     byte[] qf1 = Bytes.toBytes("qualifier");
1482     byte[] val1 = Bytes.toBytes("value1");
1483     byte[] val2 = Bytes.toBytes("value2");
1484 
1485     byte[][] families = { fam1, fam2 };
1486 
1487     // Setting up region
1488     String method = this.getName();
1489     this.region = initHRegion(tableName, method, CONF, families);
1490     try {
1491       // Putting data in the key to check
1492       Put put = new Put(row1);
1493       put.add(fam1, qf1, val1);
1494       region.put(put);
1495 
1496       // Creating put to add
1497       long ts = System.currentTimeMillis();
1498       KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1499       put = new Put(row1);
1500       put.add(kv);
1501 
1502       // checkAndPut with wrong value
1503       HStore store = (HStore) region.getStore(fam1);
1504       store.memstore.kvset.size();
1505 
1506       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1507           val1), put, true);
1508       assertEquals(true, res);
1509       store.memstore.kvset.size();
1510 
1511       Get get = new Get(row1);
1512       get.addColumn(fam2, qf1);
1513       Cell[] actual = region.get(get).rawCells();
1514 
1515       Cell[] expected = { kv };
1516 
1517       assertEquals(expected.length, actual.length);
1518       for (int i = 0; i < actual.length; i++) {
1519         assertEquals(expected[i], actual[i]);
1520       }
1521     } finally {
1522       HRegion.closeHRegion(this.region);
1523       this.region = null;
1524     }
1525   }
1526 
1527   @Test
1528   public void testCheckAndPut_wrongRowInPut() throws IOException {
1529     TableName tableName = TableName.valueOf(name.getMethodName());
1530     this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
1531     try {
1532       Put put = new Put(row2);
1533       put.add(fam1, qual1, value1);
1534       try {
1535         region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1536             new BinaryComparator(value2), put, false);
1537         fail();
1538       } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1539         // expected exception.
1540       }
1541     } finally {
1542       HRegion.closeHRegion(this.region);
1543       this.region = null;
1544     }
1545   }
1546 
1547   @Test
1548   public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1549     byte[] row1 = Bytes.toBytes("row1");
1550     byte[] fam1 = Bytes.toBytes("fam1");
1551     byte[] fam2 = Bytes.toBytes("fam2");
1552     byte[] qf1 = Bytes.toBytes("qualifier1");
1553     byte[] qf2 = Bytes.toBytes("qualifier2");
1554     byte[] qf3 = Bytes.toBytes("qualifier3");
1555     byte[] val1 = Bytes.toBytes("value1");
1556     byte[] val2 = Bytes.toBytes("value2");
1557     byte[] val3 = Bytes.toBytes("value3");
1558     byte[] emptyVal = new byte[] {};
1559 
1560     byte[][] families = { fam1, fam2 };
1561 
1562     // Setting up region
1563     String method = this.getName();
1564     this.region = initHRegion(tableName, method, CONF, families);
1565     try {
1566       // Put content
1567       Put put = new Put(row1);
1568       put.add(fam1, qf1, val1);
1569       region.put(put);
1570       Threads.sleep(2);
1571 
1572       put = new Put(row1);
1573       put.add(fam1, qf1, val2);
1574       put.add(fam2, qf1, val3);
1575       put.add(fam2, qf2, val2);
1576       put.add(fam2, qf3, val1);
1577       put.add(fam1, qf3, val1);
1578       region.put(put);
1579 
1580       // Multi-column delete
1581       Delete delete = new Delete(row1);
1582       delete.deleteColumn(fam1, qf1);
1583       delete.deleteColumn(fam2, qf1);
1584       delete.deleteColumn(fam1, qf3);
1585       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1586           val2), delete, true);
1587       assertEquals(true, res);
1588 
1589       Get get = new Get(row1);
1590       get.addColumn(fam1, qf1);
1591       get.addColumn(fam1, qf3);
1592       get.addColumn(fam2, qf2);
1593       Result r = region.get(get);
1594       assertEquals(2, r.size());
1595       assertArrayEquals(val1, r.getValue(fam1, qf1));
1596       assertArrayEquals(val2, r.getValue(fam2, qf2));
1597 
1598       // Family delete
1599       delete = new Delete(row1);
1600       delete.deleteFamily(fam2);
1601       res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1602           delete, true);
1603       assertEquals(true, res);
1604 
1605       get = new Get(row1);
1606       r = region.get(get);
1607       assertEquals(1, r.size());
1608       assertArrayEquals(val1, r.getValue(fam1, qf1));
1609 
1610       // Row delete
1611       delete = new Delete(row1);
1612       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1613           delete, true);
1614       assertEquals(true, res);
1615       get = new Get(row1);
1616       r = region.get(get);
1617       assertEquals(0, r.size());
1618     } finally {
1619       HRegion.closeHRegion(this.region);
1620       this.region = null;
1621     }
1622   }
1623 
1624   // ////////////////////////////////////////////////////////////////////////////
1625   // Delete tests
1626   // ////////////////////////////////////////////////////////////////////////////
1627   @Test
1628   public void testDelete_multiDeleteColumn() throws IOException {
1629     byte[] row1 = Bytes.toBytes("row1");
1630     byte[] fam1 = Bytes.toBytes("fam1");
1631     byte[] qual = Bytes.toBytes("qualifier");
1632     byte[] value = Bytes.toBytes("value");
1633 
1634     Put put = new Put(row1);
1635     put.add(fam1, qual, 1, value);
1636     put.add(fam1, qual, 2, value);
1637 
1638     String method = this.getName();
1639     this.region = initHRegion(tableName, method, CONF, fam1);
1640     try {
1641       region.put(put);
1642 
1643       // We do support deleting more than 1 'latest' version
1644       Delete delete = new Delete(row1);
1645       delete.deleteColumn(fam1, qual);
1646       delete.deleteColumn(fam1, qual);
1647       region.delete(delete);
1648 
1649       Get get = new Get(row1);
1650       get.addFamily(fam1);
1651       Result r = region.get(get);
1652       assertEquals(0, r.size());
1653     } finally {
1654       HRegion.closeHRegion(this.region);
1655       this.region = null;
1656     }
1657   }
1658 
1659   @Test
1660   public void testDelete_CheckFamily() throws IOException {
1661     byte[] row1 = Bytes.toBytes("row1");
1662     byte[] fam1 = Bytes.toBytes("fam1");
1663     byte[] fam2 = Bytes.toBytes("fam2");
1664     byte[] fam3 = Bytes.toBytes("fam3");
1665     byte[] fam4 = Bytes.toBytes("fam4");
1666 
1667     // Setting up region
1668     String method = this.getName();
1669     this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
1670     try {
1671       List<Cell> kvs = new ArrayList<Cell>();
1672       kvs.add(new KeyValue(row1, fam4, null, null));
1673 
1674       // testing existing family
1675       byte[] family = fam2;
1676       try {
1677         NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1678             Bytes.BYTES_COMPARATOR);
1679         deleteMap.put(family, kvs);
1680         region.delete(deleteMap, Durability.SYNC_WAL);
1681       } catch (Exception e) {
1682         assertTrue("Family " + new String(family) + " does not exist", false);
1683       }
1684 
1685       // testing non existing family
1686       boolean ok = false;
1687       family = fam4;
1688       try {
1689         NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1690             Bytes.BYTES_COMPARATOR);
1691         deleteMap.put(family, kvs);
1692         region.delete(deleteMap, Durability.SYNC_WAL);
1693       } catch (Exception e) {
1694         ok = true;
1695       }
1696       assertEquals("Family " + new String(family) + " does exist", true, ok);
1697     } finally {
1698       HRegion.closeHRegion(this.region);
1699       this.region = null;
1700     }
1701   }
1702 
1703   @Test
1704   public void testDelete_mixed() throws IOException, InterruptedException {
1705     byte[] fam = Bytes.toBytes("info");
1706     byte[][] families = { fam };
1707     String method = this.getName();
1708     this.region = initHRegion(tableName, method, CONF, families);
1709     try {
1710       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1711 
1712       byte[] row = Bytes.toBytes("table_name");
1713       // column names
1714       byte[] serverinfo = Bytes.toBytes("serverinfo");
1715       byte[] splitA = Bytes.toBytes("splitA");
1716       byte[] splitB = Bytes.toBytes("splitB");
1717 
1718       // add some data:
1719       Put put = new Put(row);
1720       put.add(fam, splitA, Bytes.toBytes("reference_A"));
1721       region.put(put);
1722 
1723       put = new Put(row);
1724       put.add(fam, splitB, Bytes.toBytes("reference_B"));
1725       region.put(put);
1726 
1727       put = new Put(row);
1728       put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1729       region.put(put);
1730 
1731       // ok now delete a split:
1732       Delete delete = new Delete(row);
1733       delete.deleteColumns(fam, splitA);
1734       region.delete(delete);
1735 
1736       // assert some things:
1737       Get get = new Get(row).addColumn(fam, serverinfo);
1738       Result result = region.get(get);
1739       assertEquals(1, result.size());
1740 
1741       get = new Get(row).addColumn(fam, splitA);
1742       result = region.get(get);
1743       assertEquals(0, result.size());
1744 
1745       get = new Get(row).addColumn(fam, splitB);
1746       result = region.get(get);
1747       assertEquals(1, result.size());
1748 
1749       // Assert that after a delete, I can put.
1750       put = new Put(row);
1751       put.add(fam, splitA, Bytes.toBytes("reference_A"));
1752       region.put(put);
1753       get = new Get(row);
1754       result = region.get(get);
1755       assertEquals(3, result.size());
1756 
1757       // Now delete all... then test I can add stuff back
1758       delete = new Delete(row);
1759       region.delete(delete);
1760       assertEquals(0, region.get(get).size());
1761 
1762       region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
1763       result = region.get(get);
1764       assertEquals(1, result.size());
1765     } finally {
1766       HRegion.closeHRegion(this.region);
1767       this.region = null;
1768     }
1769   }
1770 
1771   @Test
1772   public void testDeleteRowWithFutureTs() throws IOException {
1773     byte[] fam = Bytes.toBytes("info");
1774     byte[][] families = { fam };
1775     String method = this.getName();
1776     this.region = initHRegion(tableName, method, CONF, families);
1777     try {
1778       byte[] row = Bytes.toBytes("table_name");
1779       // column names
1780       byte[] serverinfo = Bytes.toBytes("serverinfo");
1781 
1782       // add data in the far future
1783       Put put = new Put(row);
1784       put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
1785       region.put(put);
1786 
1787       // now delete something in the present
1788       Delete delete = new Delete(row);
1789       region.delete(delete);
1790 
1791       // make sure we still see our data
1792       Get get = new Get(row).addColumn(fam, serverinfo);
1793       Result result = region.get(get);
1794       assertEquals(1, result.size());
1795 
1796       // delete the future row
1797       delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
1798       region.delete(delete);
1799 
1800       // make sure it is gone
1801       get = new Get(row).addColumn(fam, serverinfo);
1802       result = region.get(get);
1803       assertEquals(0, result.size());
1804     } finally {
1805       HRegion.closeHRegion(this.region);
1806       this.region = null;
1807     }
1808   }
1809 
1810   /**
1811    * Tests that the special LATEST_TIMESTAMP option for puts gets replaced by
1812    * the actual timestamp
1813    */
1814   @Test
1815   public void testPutWithLatestTS() throws IOException {
1816     byte[] fam = Bytes.toBytes("info");
1817     byte[][] families = { fam };
1818     String method = this.getName();
1819     this.region = initHRegion(tableName, method, CONF, families);
1820     try {
1821       byte[] row = Bytes.toBytes("row1");
1822       // column names
1823       byte[] qual = Bytes.toBytes("qual");
1824 
1825       // add data with LATEST_TIMESTAMP, put without WAL
1826       Put put = new Put(row);
1827       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1828       region.put(put);
1829 
1830       // Make sure it shows up with an actual timestamp
1831       Get get = new Get(row).addColumn(fam, qual);
1832       Result result = region.get(get);
1833       assertEquals(1, result.size());
1834       Cell kv = result.rawCells()[0];
1835       LOG.info("Got: " + kv);
1836       assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1837           kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1838 
1839       // Check same with WAL enabled (historically these took different
1840       // code paths, so check both)
1841       row = Bytes.toBytes("row2");
1842       put = new Put(row);
1843       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1844       region.put(put);
1845 
1846       // Make sure it shows up with an actual timestamp
1847       get = new Get(row).addColumn(fam, qual);
1848       result = region.get(get);
1849       assertEquals(1, result.size());
1850       kv = result.rawCells()[0];
1851       LOG.info("Got: " + kv);
1852       assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1853           kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1854     } finally {
1855       HRegion.closeHRegion(this.region);
1856       this.region = null;
1857     }
1858 
1859   }
1860 
1861   /**
1862    * Tests that there is server-side filtering for invalid timestamp upper
1863    * bound. Note that the timestamp lower bound is automatically handled for us
1864    * by the TTL field.
1865    */
1866   @Test
1867   public void testPutWithTsSlop() throws IOException {
1868     byte[] fam = Bytes.toBytes("info");
1869     byte[][] families = { fam };
1870     String method = this.getName();
1871 
1872     // add data with a timestamp that is too recent for range. Ensure assert
1873     CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1874     this.region = initHRegion(tableName, method, CONF, families);
1875     boolean caughtExcep = false;
1876     try {
1877       try {
1878         // no TS specified == use latest. should not error
1879         region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
1880         // TS out of range. should error
1881         region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
1882             Bytes.toBytes("value")));
1883         fail("Expected IOE for TS out of configured timerange");
1884       } catch (FailedSanityCheckException ioe) {
1885         LOG.debug("Received expected exception", ioe);
1886         caughtExcep = true;
1887       }
1888       assertTrue("Should catch FailedSanityCheckException", caughtExcep);
1889     } finally {
1890       HRegion.closeHRegion(this.region);
1891       this.region = null;
1892     }
1893   }
1894 
1895   @Test
1896   public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
1897     byte[] fam1 = Bytes.toBytes("columnA");
1898     byte[] fam2 = Bytes.toBytes("columnB");
1899     this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
1900     try {
1901       byte[] rowA = Bytes.toBytes("rowA");
1902       byte[] rowB = Bytes.toBytes("rowB");
1903 
1904       byte[] value = Bytes.toBytes("value");
1905 
1906       Delete delete = new Delete(rowA);
1907       delete.deleteFamily(fam1);
1908 
1909       region.delete(delete);
1910 
1911       // now create data.
1912       Put put = new Put(rowA);
1913       put.add(fam2, null, value);
1914       region.put(put);
1915 
1916       put = new Put(rowB);
1917       put.add(fam1, null, value);
1918       put.add(fam2, null, value);
1919       region.put(put);
1920 
1921       Scan scan = new Scan();
1922       scan.addFamily(fam1).addFamily(fam2);
1923       InternalScanner s = region.getScanner(scan);
1924       List<Cell> results = new ArrayList<Cell>();
1925       s.next(results);
1926       assertTrue(CellUtil.matchingRow(results.get(0), rowA));
1927 
1928       results.clear();
1929       s.next(results);
1930       assertTrue(CellUtil.matchingRow(results.get(0), rowB));
1931     } finally {
1932       HRegion.closeHRegion(this.region);
1933       this.region = null;
1934     }
1935   }
1936 
1937   @Test
1938   public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
1939     Delete delete = new Delete(row);
1940     delete.deleteColumns(fam1, qual1);
1941     doTestDelete_AndPostInsert(delete);
1942   }
1943 
1944   @Test
1945   public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
1946     Delete delete = new Delete(row);
1947     delete.deleteFamily(fam1);
1948     doTestDelete_AndPostInsert(delete);
1949   }
1950 
1951   public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
1952     TableName tableName = TableName.valueOf(name.getMethodName());
1953     this.region = initHRegion(tableName, getName(), CONF, fam1);
1954     try {
1955       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1956       Put put = new Put(row);
1957       put.add(fam1, qual1, value1);
1958       region.put(put);
1959 
1960       // now delete the value:
1961       region.delete(delete);
1962 
1963       // ok put data:
1964       put = new Put(row);
1965       put.add(fam1, qual1, value2);
1966       region.put(put);
1967 
1968       // ok get:
1969       Get get = new Get(row);
1970       get.addColumn(fam1, qual1);
1971 
1972       Result r = region.get(get);
1973       assertEquals(1, r.size());
1974       assertArrayEquals(value2, r.getValue(fam1, qual1));
1975 
1976       // next:
1977       Scan scan = new Scan(row);
1978       scan.addColumn(fam1, qual1);
1979       InternalScanner s = region.getScanner(scan);
1980 
1981       List<Cell> results = new ArrayList<Cell>();
1982       assertEquals(false, s.next(results));
1983       assertEquals(1, results.size());
1984       Cell kv = results.get(0);
1985 
1986       assertArrayEquals(value2, CellUtil.cloneValue(kv));
1987       assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
1988       assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
1989       assertArrayEquals(row, CellUtil.cloneRow(kv));
1990     } finally {
1991       HRegion.closeHRegion(this.region);
1992       this.region = null;
1993     }
1994   }
1995 
1996   @Test
1997   public void testDelete_CheckTimestampUpdated() throws IOException {
1998     TableName tableName = TableName.valueOf(name.getMethodName());
1999     byte[] row1 = Bytes.toBytes("row1");
2000     byte[] col1 = Bytes.toBytes("col1");
2001     byte[] col2 = Bytes.toBytes("col2");
2002     byte[] col3 = Bytes.toBytes("col3");
2003 
2004     // Setting up region
2005     String method = this.getName();
2006     this.region = initHRegion(tableName, method, CONF, fam1);
2007     try {
2008       // Building checkerList
2009       List<Cell> kvs = new ArrayList<Cell>();
2010       kvs.add(new KeyValue(row1, fam1, col1, null));
2011       kvs.add(new KeyValue(row1, fam1, col2, null));
2012       kvs.add(new KeyValue(row1, fam1, col3, null));
2013 
2014       NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
2015           Bytes.BYTES_COMPARATOR);
2016       deleteMap.put(fam1, kvs);
2017       region.delete(deleteMap, Durability.SYNC_WAL);
2018 
2019       // extract the key values out the memstore:
2020       // This is kinda hacky, but better than nothing...
2021       long now = System.currentTimeMillis();
2022       KeyValue firstKv = ((HStore) region.getStore(fam1)).memstore.kvset.first();
2023       assertTrue(firstKv.getTimestamp() <= now);
2024       now = firstKv.getTimestamp();
2025       for (KeyValue kv : ((HStore) region.getStore(fam1)).memstore.kvset) {
2026         assertTrue(kv.getTimestamp() <= now);
2027         now = kv.getTimestamp();
2028       }
2029     } finally {
2030       HRegion.closeHRegion(this.region);
2031       this.region = null;
2032     }
2033   }
2034 
2035   // ////////////////////////////////////////////////////////////////////////////
2036   // Get tests
2037   // ////////////////////////////////////////////////////////////////////////////
2038   @Test
2039   public void testGet_FamilyChecker() throws IOException {
2040     byte[] row1 = Bytes.toBytes("row1");
2041     byte[] fam1 = Bytes.toBytes("fam1");
2042     byte[] fam2 = Bytes.toBytes("False");
2043     byte[] col1 = Bytes.toBytes("col1");
2044 
2045     // Setting up region
2046     String method = this.getName();
2047     this.region = initHRegion(tableName, method, CONF, fam1);
2048     try {
2049       Get get = new Get(row1);
2050       get.addColumn(fam2, col1);
2051 
2052       // Test
2053       try {
2054         region.get(get);
2055       } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
2056         assertFalse(false);
2057         return;
2058       }
2059       assertFalse(true);
2060     } finally {
2061       HRegion.closeHRegion(this.region);
2062       this.region = null;
2063     }
2064   }
2065 
2066   @Test
2067   public void testGet_Basic() throws IOException {
2068     byte[] row1 = Bytes.toBytes("row1");
2069     byte[] fam1 = Bytes.toBytes("fam1");
2070     byte[] col1 = Bytes.toBytes("col1");
2071     byte[] col2 = Bytes.toBytes("col2");
2072     byte[] col3 = Bytes.toBytes("col3");
2073     byte[] col4 = Bytes.toBytes("col4");
2074     byte[] col5 = Bytes.toBytes("col5");
2075 
2076     // Setting up region
2077     String method = this.getName();
2078     this.region = initHRegion(tableName, method, CONF, fam1);
2079     try {
2080       // Add to memstore
2081       Put put = new Put(row1);
2082       put.add(fam1, col1, null);
2083       put.add(fam1, col2, null);
2084       put.add(fam1, col3, null);
2085       put.add(fam1, col4, null);
2086       put.add(fam1, col5, null);
2087       region.put(put);
2088 
2089       Get get = new Get(row1);
2090       get.addColumn(fam1, col2);
2091       get.addColumn(fam1, col4);
2092       // Expected result
2093       KeyValue kv1 = new KeyValue(row1, fam1, col2);
2094       KeyValue kv2 = new KeyValue(row1, fam1, col4);
2095       KeyValue[] expected = { kv1, kv2 };
2096 
2097       // Test
2098       Result res = region.get(get);
2099       assertEquals(expected.length, res.size());
2100       for (int i = 0; i < res.size(); i++) {
2101         assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
2102         assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
2103         assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
2104       }
2105 
2106       // Test using a filter on a Get
2107       Get g = new Get(row1);
2108       final int count = 2;
2109       g.setFilter(new ColumnCountGetFilter(count));
2110       res = region.get(g);
2111       assertEquals(count, res.size());
2112     } finally {
2113       HRegion.closeHRegion(this.region);
2114       this.region = null;
2115     }
2116   }
2117 
2118   @Test
2119   public void testGet_Empty() throws IOException {
2120     byte[] row = Bytes.toBytes("row");
2121     byte[] fam = Bytes.toBytes("fam");
2122 
2123     String method = this.getName();
2124     this.region = initHRegion(tableName, method, CONF, fam);
2125     try {
2126       Get get = new Get(row);
2127       get.addFamily(fam);
2128       Result r = region.get(get);
2129 
2130       assertTrue(r.isEmpty());
2131     } finally {
2132       HRegion.closeHRegion(this.region);
2133       this.region = null;
2134     }
2135   }
2136 
2137   // ////////////////////////////////////////////////////////////////////////////
2138   // Merge test
2139   // ////////////////////////////////////////////////////////////////////////////
2140   @Test
2141   public void testMerge() throws IOException {
2142     byte[][] families = { fam1, fam2, fam3 };
2143     Configuration hc = initSplit();
2144     // Setting up region
2145     String method = this.getName();
2146     this.region = initHRegion(tableName, method, hc, families);
2147     try {
2148       LOG.info("" + HBaseTestCase.addContent(region, fam3));
2149       region.flushcache();
2150       region.compactStores();
2151       byte[] splitRow = region.checkSplit();
2152       assertNotNull(splitRow);
2153       LOG.info("SplitRow: " + Bytes.toString(splitRow));
2154       HRegion[] subregions = splitRegion(region, splitRow);
2155       try {
2156         // Need to open the regions.
2157         for (int i = 0; i < subregions.length; i++) {
2158           HRegion.openHRegion(subregions[i], null);
2159           subregions[i].compactStores();
2160         }
2161         Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
2162         Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
2163         Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
2164         long startTime = System.currentTimeMillis();
2165         region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
2166         LOG.info("Merge regions elapsed time: "
2167             + ((System.currentTimeMillis() - startTime) / 1000.0));
2168         FILESYSTEM.delete(oldRegion1, true);
2169         FILESYSTEM.delete(oldRegion2, true);
2170         FILESYSTEM.delete(oldRegionPath, true);
2171         LOG.info("splitAndMerge completed.");
2172       } finally {
2173         for (int i = 0; i < subregions.length; i++) {
2174           try {
2175             HRegion.closeHRegion(subregions[i]);
2176           } catch (IOException e) {
2177             // Ignore.
2178           }
2179         }
2180       }
2181     } finally {
2182       HRegion.closeHRegion(this.region);
2183       this.region = null;
2184     }
2185   }
2186 
2187   /**
2188    * @param parent
2189    *          Region to split.
2190    * @param midkey
2191    *          Key to split around.
2192    * @return The Regions we created.
2193    * @throws IOException
2194    */
2195   HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
2196     PairOfSameType<HRegion> result = null;
2197     SplitTransaction st = new SplitTransaction(parent, midkey);
2198     // If prepare does not return true, for some reason -- logged inside in
2199     // the prepare call -- we are not ready to split just now. Just return.
2200     if (!st.prepare()) {
2201       parent.clearSplit();
2202       return null;
2203     }
2204     try {
2205       result = st.execute(null, null);
2206     } catch (IOException ioe) {
2207       try {
2208         LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; "
2209             + ioe.getMessage());
2210         st.rollback(null, null);
2211         LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString());
2212         return null;
2213       } catch (RuntimeException e) {
2214         // If failed rollback, kill this server to avoid having a hole in table.
2215         LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString()
2216             + " -- aborting server", e);
2217       }
2218     }
2219     finally {
2220       parent.clearSplit();
2221     }
2222     return new HRegion[] { result.getFirst(), result.getSecond() };
2223   }
2224 
2225   // ////////////////////////////////////////////////////////////////////////////
2226   // Scanner tests
2227   // ////////////////////////////////////////////////////////////////////////////
2228   @Test
2229   public void testGetScanner_WithOkFamilies() throws IOException {
2230     byte[] fam1 = Bytes.toBytes("fam1");
2231     byte[] fam2 = Bytes.toBytes("fam2");
2232 
2233     byte[][] families = { fam1, fam2 };
2234 
2235     // Setting up region
2236     String method = this.getName();
2237     this.region = initHRegion(tableName, method, CONF, families);
2238     try {
2239       Scan scan = new Scan();
2240       scan.addFamily(fam1);
2241       scan.addFamily(fam2);
2242       try {
2243         region.getScanner(scan);
2244       } catch (Exception e) {
2245         assertTrue("Families could not be found in Region", false);
2246       }
2247     } finally {
2248       HRegion.closeHRegion(this.region);
2249       this.region = null;
2250     }
2251   }
2252 
2253   @Test
2254   public void testGetScanner_WithNotOkFamilies() throws IOException {
2255     byte[] fam1 = Bytes.toBytes("fam1");
2256     byte[] fam2 = Bytes.toBytes("fam2");
2257 
2258     byte[][] families = { fam1 };
2259 
2260     // Setting up region
2261     String method = this.getName();
2262     this.region = initHRegion(tableName, method, CONF, families);
2263     try {
2264       Scan scan = new Scan();
2265       scan.addFamily(fam2);
2266       boolean ok = false;
2267       try {
2268         region.getScanner(scan);
2269       } catch (Exception e) {
2270         ok = true;
2271       }
2272       assertTrue("Families could not be found in Region", ok);
2273     } finally {
2274       HRegion.closeHRegion(this.region);
2275       this.region = null;
2276     }
2277   }
2278 
2279   @Test
2280   public void testGetScanner_WithNoFamilies() throws IOException {
2281     byte[] row1 = Bytes.toBytes("row1");
2282     byte[] fam1 = Bytes.toBytes("fam1");
2283     byte[] fam2 = Bytes.toBytes("fam2");
2284     byte[] fam3 = Bytes.toBytes("fam3");
2285     byte[] fam4 = Bytes.toBytes("fam4");
2286 
2287     byte[][] families = { fam1, fam2, fam3, fam4 };
2288 
2289     // Setting up region
2290     String method = this.getName();
2291     this.region = initHRegion(tableName, method, CONF, families);
2292     try {
2293 
2294       // Putting data in Region
2295       Put put = new Put(row1);
2296       put.add(fam1, null, null);
2297       put.add(fam2, null, null);
2298       put.add(fam3, null, null);
2299       put.add(fam4, null, null);
2300       region.put(put);
2301 
2302       Scan scan = null;
2303       HRegion.RegionScannerImpl is = null;
2304 
2305       // Testing to see how many scanners that is produced by getScanner,
2306       // starting
2307       // with known number, 2 - current = 1
2308       scan = new Scan();
2309       scan.addFamily(fam2);
2310       scan.addFamily(fam4);
2311       is = (RegionScannerImpl) region.getScanner(scan);
2312       assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2313 
2314       scan = new Scan();
2315       is = (RegionScannerImpl) region.getScanner(scan);
2316       assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2317     } finally {
2318       HRegion.closeHRegion(this.region);
2319       this.region = null;
2320     }
2321   }
2322 
2323   /**
2324    * This method tests https://issues.apache.org/jira/browse/HBASE-2516.
2325    * 
2326    * @throws IOException
2327    */
2328   @Test
2329   public void testGetScanner_WithRegionClosed() throws IOException {
2330     byte[] fam1 = Bytes.toBytes("fam1");
2331     byte[] fam2 = Bytes.toBytes("fam2");
2332 
2333     byte[][] families = { fam1, fam2 };
2334 
2335     // Setting up region
2336     String method = this.getName();
2337     try {
2338       this.region = initHRegion(tableName, method, CONF, families);
2339     } catch (IOException e) {
2340       e.printStackTrace();
2341       fail("Got IOException during initHRegion, " + e.getMessage());
2342     }
2343     try {
2344       region.closed.set(true);
2345       try {
2346         region.getScanner(null);
2347         fail("Expected to get an exception during getScanner on a region that is closed");
2348       } catch (NotServingRegionException e) {
2349         // this is the correct exception that is expected
2350       } catch (IOException e) {
2351         fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2352             + e.getMessage());
2353       }
2354     } finally {
2355       HRegion.closeHRegion(this.region);
2356       this.region = null;
2357     }
2358   }
2359 
2360   @Test
2361   public void testRegionScanner_Next() throws IOException {
2362     byte[] row1 = Bytes.toBytes("row1");
2363     byte[] row2 = Bytes.toBytes("row2");
2364     byte[] fam1 = Bytes.toBytes("fam1");
2365     byte[] fam2 = Bytes.toBytes("fam2");
2366     byte[] fam3 = Bytes.toBytes("fam3");
2367     byte[] fam4 = Bytes.toBytes("fam4");
2368 
2369     byte[][] families = { fam1, fam2, fam3, fam4 };
2370     long ts = System.currentTimeMillis();
2371 
2372     // Setting up region
2373     String method = this.getName();
2374     this.region = initHRegion(tableName, method, CONF, families);
2375     try {
2376       // Putting data in Region
2377       Put put = null;
2378       put = new Put(row1);
2379       put.add(fam1, (byte[]) null, ts, null);
2380       put.add(fam2, (byte[]) null, ts, null);
2381       put.add(fam3, (byte[]) null, ts, null);
2382       put.add(fam4, (byte[]) null, ts, null);
2383       region.put(put);
2384 
2385       put = new Put(row2);
2386       put.add(fam1, (byte[]) null, ts, null);
2387       put.add(fam2, (byte[]) null, ts, null);
2388       put.add(fam3, (byte[]) null, ts, null);
2389       put.add(fam4, (byte[]) null, ts, null);
2390       region.put(put);
2391 
2392       Scan scan = new Scan();
2393       scan.addFamily(fam2);
2394       scan.addFamily(fam4);
2395       InternalScanner is = region.getScanner(scan);
2396 
2397       List<Cell> res = null;
2398 
2399       // Result 1
2400       List<Cell> expected1 = new ArrayList<Cell>();
2401       expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2402       expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2403 
2404       res = new ArrayList<Cell>();
2405       is.next(res);
2406       for (int i = 0; i < res.size(); i++) {
2407         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2408       }
2409 
2410       // Result 2
2411       List<Cell> expected2 = new ArrayList<Cell>();
2412       expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2413       expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2414 
2415       res = new ArrayList<Cell>();
2416       is.next(res);
2417       for (int i = 0; i < res.size(); i++) {
2418         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2419       }
2420     } finally {
2421       HRegion.closeHRegion(this.region);
2422       this.region = null;
2423     }
2424   }
2425 
2426   @Test
2427   public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2428     byte[] row1 = Bytes.toBytes("row1");
2429     byte[] qf1 = Bytes.toBytes("qualifier1");
2430     byte[] qf2 = Bytes.toBytes("qualifier2");
2431     byte[] fam1 = Bytes.toBytes("fam1");
2432     byte[][] families = { fam1 };
2433 
2434     long ts1 = System.currentTimeMillis();
2435     long ts2 = ts1 + 1;
2436     long ts3 = ts1 + 2;
2437 
2438     // Setting up region
2439     String method = this.getName();
2440     this.region = initHRegion(tableName, method, CONF, families);
2441     try {
2442       // Putting data in Region
2443       Put put = null;
2444       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2445       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2446       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2447 
2448       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2449       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2450       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2451 
2452       put = new Put(row1);
2453       put.add(kv13);
2454       put.add(kv12);
2455       put.add(kv11);
2456       put.add(kv23);
2457       put.add(kv22);
2458       put.add(kv21);
2459       region.put(put);
2460 
2461       // Expected
2462       List<Cell> expected = new ArrayList<Cell>();
2463       expected.add(kv13);
2464       expected.add(kv12);
2465 
2466       Scan scan = new Scan(row1);
2467       scan.addColumn(fam1, qf1);
2468       scan.setMaxVersions(MAX_VERSIONS);
2469       List<Cell> actual = new ArrayList<Cell>();
2470       InternalScanner scanner = region.getScanner(scan);
2471 
2472       boolean hasNext = scanner.next(actual);
2473       assertEquals(false, hasNext);
2474 
2475       // Verify result
2476       for (int i = 0; i < expected.size(); i++) {
2477         assertEquals(expected.get(i), actual.get(i));
2478       }
2479     } finally {
2480       HRegion.closeHRegion(this.region);
2481       this.region = null;
2482     }
2483   }
2484 
2485   @Test
2486   public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2487     byte[] row1 = Bytes.toBytes("row1");
2488     byte[] qf1 = Bytes.toBytes("qualifier1");
2489     byte[] qf2 = Bytes.toBytes("qualifier2");
2490     byte[] fam1 = Bytes.toBytes("fam1");
2491     byte[][] families = { fam1 };
2492 
2493     long ts1 = 1; // System.currentTimeMillis();
2494     long ts2 = ts1 + 1;
2495     long ts3 = ts1 + 2;
2496 
2497     // Setting up region
2498     String method = this.getName();
2499     this.region = initHRegion(tableName, method, CONF, families);
2500     try {
2501       // Putting data in Region
2502       Put put = null;
2503       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2504       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2505       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2506 
2507       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2508       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2509       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2510 
2511       put = new Put(row1);
2512       put.add(kv13);
2513       put.add(kv12);
2514       put.add(kv11);
2515       put.add(kv23);
2516       put.add(kv22);
2517       put.add(kv21);
2518       region.put(put);
2519       region.flushcache();
2520 
2521       // Expected
2522       List<Cell> expected = new ArrayList<Cell>();
2523       expected.add(kv13);
2524       expected.add(kv12);
2525       expected.add(kv23);
2526       expected.add(kv22);
2527 
2528       Scan scan = new Scan(row1);
2529       scan.addColumn(fam1, qf1);
2530       scan.addColumn(fam1, qf2);
2531       scan.setMaxVersions(MAX_VERSIONS);
2532       List<Cell> actual = new ArrayList<Cell>();
2533       InternalScanner scanner = region.getScanner(scan);
2534 
2535       boolean hasNext = scanner.next(actual);
2536       assertEquals(false, hasNext);
2537 
2538       // Verify result
2539       for (int i = 0; i < expected.size(); i++) {
2540         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2541       }
2542     } finally {
2543       HRegion.closeHRegion(this.region);
2544       this.region = null;
2545     }
2546   }
2547 
2548   @Test
2549   public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2550     byte[] row1 = Bytes.toBytes("row1");
2551     byte[] fam1 = Bytes.toBytes("fam1");
2552     byte[][] families = { fam1 };
2553     byte[] qf1 = Bytes.toBytes("qualifier1");
2554     byte[] qf2 = Bytes.toBytes("qualifier2");
2555 
2556     long ts1 = 1;
2557     long ts2 = ts1 + 1;
2558     long ts3 = ts1 + 2;
2559     long ts4 = ts1 + 3;
2560 
2561     // Setting up region
2562     String method = this.getName();
2563     this.region = initHRegion(tableName, method, CONF, families);
2564     try {
2565       // Putting data in Region
2566       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2567       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2568       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2569       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2570 
2571       KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2572       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2573       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2574       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2575 
2576       Put put = null;
2577       put = new Put(row1);
2578       put.add(kv14);
2579       put.add(kv24);
2580       region.put(put);
2581       region.flushcache();
2582 
2583       put = new Put(row1);
2584       put.add(kv23);
2585       put.add(kv13);
2586       region.put(put);
2587       region.flushcache();
2588 
2589       put = new Put(row1);
2590       put.add(kv22);
2591       put.add(kv12);
2592       region.put(put);
2593       region.flushcache();
2594 
2595       put = new Put(row1);
2596       put.add(kv21);
2597       put.add(kv11);
2598       region.put(put);
2599 
2600       // Expected
2601       List<Cell> expected = new ArrayList<Cell>();
2602       expected.add(kv14);
2603       expected.add(kv13);
2604       expected.add(kv12);
2605       expected.add(kv24);
2606       expected.add(kv23);
2607       expected.add(kv22);
2608 
2609       Scan scan = new Scan(row1);
2610       scan.addColumn(fam1, qf1);
2611       scan.addColumn(fam1, qf2);
2612       int versions = 3;
2613       scan.setMaxVersions(versions);
2614       List<Cell> actual = new ArrayList<Cell>();
2615       InternalScanner scanner = region.getScanner(scan);
2616 
2617       boolean hasNext = scanner.next(actual);
2618       assertEquals(false, hasNext);
2619 
2620       // Verify result
2621       for (int i = 0; i < expected.size(); i++) {
2622         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2623       }
2624     } finally {
2625       HRegion.closeHRegion(this.region);
2626       this.region = null;
2627     }
2628   }
2629 
2630   @Test
2631   public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2632     byte[] row1 = Bytes.toBytes("row1");
2633     byte[] qf1 = Bytes.toBytes("qualifier1");
2634     byte[] qf2 = Bytes.toBytes("qualifier2");
2635     byte[] fam1 = Bytes.toBytes("fam1");
2636     byte[][] families = { fam1 };
2637 
2638     long ts1 = System.currentTimeMillis();
2639     long ts2 = ts1 + 1;
2640     long ts3 = ts1 + 2;
2641 
2642     // Setting up region
2643     String method = this.getName();
2644     this.region = initHRegion(tableName, method, CONF, families);
2645     try {
2646       // Putting data in Region
2647       Put put = null;
2648       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2649       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2650       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2651 
2652       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2653       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2654       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2655 
2656       put = new Put(row1);
2657       put.add(kv13);
2658       put.add(kv12);
2659       put.add(kv11);
2660       put.add(kv23);
2661       put.add(kv22);
2662       put.add(kv21);
2663       region.put(put);
2664 
2665       // Expected
2666       List<Cell> expected = new ArrayList<Cell>();
2667       expected.add(kv13);
2668       expected.add(kv12);
2669       expected.add(kv23);
2670       expected.add(kv22);
2671 
2672       Scan scan = new Scan(row1);
2673       scan.addFamily(fam1);
2674       scan.setMaxVersions(MAX_VERSIONS);
2675       List<Cell> actual = new ArrayList<Cell>();
2676       InternalScanner scanner = region.getScanner(scan);
2677 
2678       boolean hasNext = scanner.next(actual);
2679       assertEquals(false, hasNext);
2680 
2681       // Verify result
2682       for (int i = 0; i < expected.size(); i++) {
2683         assertEquals(expected.get(i), actual.get(i));
2684       }
2685     } finally {
2686       HRegion.closeHRegion(this.region);
2687       this.region = null;
2688     }
2689   }
2690 
2691   @Test
2692   public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
2693     byte[] row1 = Bytes.toBytes("row1");
2694     byte[] qf1 = Bytes.toBytes("qualifier1");
2695     byte[] qf2 = Bytes.toBytes("qualifier2");
2696     byte[] fam1 = Bytes.toBytes("fam1");
2697 
2698     long ts1 = 1; // System.currentTimeMillis();
2699     long ts2 = ts1 + 1;
2700     long ts3 = ts1 + 2;
2701 
2702     // Setting up region
2703     String method = this.getName();
2704     this.region = initHRegion(tableName, method, CONF, fam1);
2705     try {
2706       // Putting data in Region
2707       Put put = null;
2708       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2709       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2710       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2711 
2712       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2713       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2714       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2715 
2716       put = new Put(row1);
2717       put.add(kv13);
2718       put.add(kv12);
2719       put.add(kv11);
2720       put.add(kv23);
2721       put.add(kv22);
2722       put.add(kv21);
2723       region.put(put);
2724       region.flushcache();
2725 
2726       // Expected
2727       List<Cell> expected = new ArrayList<Cell>();
2728       expected.add(kv13);
2729       expected.add(kv12);
2730       expected.add(kv23);
2731       expected.add(kv22);
2732 
2733       Scan scan = new Scan(row1);
2734       scan.addFamily(fam1);
2735       scan.setMaxVersions(MAX_VERSIONS);
2736       List<Cell> actual = new ArrayList<Cell>();
2737       InternalScanner scanner = region.getScanner(scan);
2738 
2739       boolean hasNext = scanner.next(actual);
2740       assertEquals(false, hasNext);
2741 
2742       // Verify result
2743       for (int i = 0; i < expected.size(); i++) {
2744         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2745       }
2746     } finally {
2747       HRegion.closeHRegion(this.region);
2748       this.region = null;
2749     }
2750   }
2751 
2752   @Test
2753   public void testScanner_StopRow1542() throws IOException {
2754     byte[] family = Bytes.toBytes("testFamily");
2755     this.region = initHRegion(tableName, getName(), CONF, family);
2756     try {
2757       byte[] row1 = Bytes.toBytes("row111");
2758       byte[] row2 = Bytes.toBytes("row222");
2759       byte[] row3 = Bytes.toBytes("row333");
2760       byte[] row4 = Bytes.toBytes("row444");
2761       byte[] row5 = Bytes.toBytes("row555");
2762 
2763       byte[] col1 = Bytes.toBytes("Pub111");
2764       byte[] col2 = Bytes.toBytes("Pub222");
2765 
2766       Put put = new Put(row1);
2767       put.add(family, col1, Bytes.toBytes(10L));
2768       region.put(put);
2769 
2770       put = new Put(row2);
2771       put.add(family, col1, Bytes.toBytes(15L));
2772       region.put(put);
2773 
2774       put = new Put(row3);
2775       put.add(family, col2, Bytes.toBytes(20L));
2776       region.put(put);
2777 
2778       put = new Put(row4);
2779       put.add(family, col2, Bytes.toBytes(30L));
2780       region.put(put);
2781 
2782       put = new Put(row5);
2783       put.add(family, col1, Bytes.toBytes(40L));
2784       region.put(put);
2785 
2786       Scan scan = new Scan(row3, row4);
2787       scan.setMaxVersions();
2788       scan.addColumn(family, col1);
2789       InternalScanner s = region.getScanner(scan);
2790 
2791       List<Cell> results = new ArrayList<Cell>();
2792       assertEquals(false, s.next(results));
2793       assertEquals(0, results.size());
2794     } finally {
2795       HRegion.closeHRegion(this.region);
2796       this.region = null;
2797     }
2798   }
2799 
2800   @Test
2801   public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2802     byte[] row1 = Bytes.toBytes("row1");
2803     byte[] fam1 = Bytes.toBytes("fam1");
2804     byte[] qf1 = Bytes.toBytes("qualifier1");
2805     byte[] qf2 = Bytes.toBytes("quateslifier2");
2806 
2807     long ts1 = 1;
2808     long ts2 = ts1 + 1;
2809     long ts3 = ts1 + 2;
2810     long ts4 = ts1 + 3;
2811 
2812     // Setting up region
2813     String method = this.getName();
2814     this.region = initHRegion(tableName, method, CONF, fam1);
2815     try {
2816       // Putting data in Region
2817       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2818       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2819       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2820       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2821 
2822       KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2823       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2824       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2825       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2826 
2827       Put put = null;
2828       put = new Put(row1);
2829       put.add(kv14);
2830       put.add(kv24);
2831       region.put(put);
2832       region.flushcache();
2833 
2834       put = new Put(row1);
2835       put.add(kv23);
2836       put.add(kv13);
2837       region.put(put);
2838       region.flushcache();
2839 
2840       put = new Put(row1);
2841       put.add(kv22);
2842       put.add(kv12);
2843       region.put(put);
2844       region.flushcache();
2845 
2846       put = new Put(row1);
2847       put.add(kv21);
2848       put.add(kv11);
2849       region.put(put);
2850 
2851       // Expected
2852       List<KeyValue> expected = new ArrayList<KeyValue>();
2853       expected.add(kv14);
2854       expected.add(kv13);
2855       expected.add(kv12);
2856       expected.add(kv24);
2857       expected.add(kv23);
2858       expected.add(kv22);
2859 
2860       Scan scan = new Scan(row1);
2861       int versions = 3;
2862       scan.setMaxVersions(versions);
2863       List<Cell> actual = new ArrayList<Cell>();
2864       InternalScanner scanner = region.getScanner(scan);
2865 
2866       boolean hasNext = scanner.next(actual);
2867       assertEquals(false, hasNext);
2868 
2869       // Verify result
2870       for (int i = 0; i < expected.size(); i++) {
2871         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2872       }
2873     } finally {
2874       HRegion.closeHRegion(this.region);
2875       this.region = null;
2876     }
2877   }
2878 
2879   /**
2880    * Added for HBASE-5416
2881    * 
2882    * Here we test scan optimization when only subset of CFs are used in filter
2883    * conditions.
2884    */
2885   @Test
2886   public void testScanner_JoinedScanners() throws IOException {
2887     byte[] cf_essential = Bytes.toBytes("essential");
2888     byte[] cf_joined = Bytes.toBytes("joined");
2889     byte[] cf_alpha = Bytes.toBytes("alpha");
2890     this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
2891     try {
2892       byte[] row1 = Bytes.toBytes("row1");
2893       byte[] row2 = Bytes.toBytes("row2");
2894       byte[] row3 = Bytes.toBytes("row3");
2895 
2896       byte[] col_normal = Bytes.toBytes("d");
2897       byte[] col_alpha = Bytes.toBytes("a");
2898 
2899       byte[] filtered_val = Bytes.toBytes(3);
2900 
2901       Put put = new Put(row1);
2902       put.add(cf_essential, col_normal, Bytes.toBytes(1));
2903       put.add(cf_joined, col_alpha, Bytes.toBytes(1));
2904       region.put(put);
2905 
2906       put = new Put(row2);
2907       put.add(cf_essential, col_alpha, Bytes.toBytes(2));
2908       put.add(cf_joined, col_normal, Bytes.toBytes(2));
2909       put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
2910       region.put(put);
2911 
2912       put = new Put(row3);
2913       put.add(cf_essential, col_normal, filtered_val);
2914       put.add(cf_joined, col_normal, filtered_val);
2915       region.put(put);
2916 
2917       // Check two things:
2918       // 1. result list contains expected values
2919       // 2. result list is sorted properly
2920 
2921       Scan scan = new Scan();
2922       Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
2923           CompareOp.NOT_EQUAL, filtered_val);
2924       scan.setFilter(filter);
2925       scan.setLoadColumnFamiliesOnDemand(true);
2926       InternalScanner s = region.getScanner(scan);
2927 
2928       List<Cell> results = new ArrayList<Cell>();
2929       assertTrue(s.next(results));
2930       assertEquals(results.size(), 1);
2931       results.clear();
2932 
2933       assertTrue(s.next(results));
2934       assertEquals(results.size(), 3);
2935       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
2936       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
2937       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
2938       results.clear();
2939 
2940       assertFalse(s.next(results));
2941       assertEquals(results.size(), 0);
2942     } finally {
2943       HRegion.closeHRegion(this.region);
2944       this.region = null;
2945     }
2946   }
2947 
2948   /**
2949    * HBASE-5416
2950    * 
2951    * Test case when scan limits amount of KVs returned on each next() call.
2952    */
2953   @Test
2954   public void testScanner_JoinedScannersWithLimits() throws IOException {
2955     final byte[] cf_first = Bytes.toBytes("first");
2956     final byte[] cf_second = Bytes.toBytes("second");
2957 
2958     this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
2959     try {
2960       final byte[] col_a = Bytes.toBytes("a");
2961       final byte[] col_b = Bytes.toBytes("b");
2962 
2963       Put put;
2964 
2965       for (int i = 0; i < 10; i++) {
2966         put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
2967         put.add(cf_first, col_a, Bytes.toBytes(i));
2968         if (i < 5) {
2969           put.add(cf_first, col_b, Bytes.toBytes(i));
2970           put.add(cf_second, col_a, Bytes.toBytes(i));
2971           put.add(cf_second, col_b, Bytes.toBytes(i));
2972         }
2973         region.put(put);
2974       }
2975 
2976       Scan scan = new Scan();
2977       scan.setLoadColumnFamiliesOnDemand(true);
2978       Filter bogusFilter = new FilterBase() {
2979         @Override
2980         public boolean isFamilyEssential(byte[] name) {
2981           return Bytes.equals(name, cf_first);
2982         }
2983       };
2984 
2985       scan.setFilter(bogusFilter);
2986       InternalScanner s = region.getScanner(scan);
2987 
2988       // Our data looks like this:
2989       // r0: first:a, first:b, second:a, second:b
2990       // r1: first:a, first:b, second:a, second:b
2991       // r2: first:a, first:b, second:a, second:b
2992       // r3: first:a, first:b, second:a, second:b
2993       // r4: first:a, first:b, second:a, second:b
2994       // r5: first:a
2995       // r6: first:a
2996       // r7: first:a
2997       // r8: first:a
2998       // r9: first:a
2999 
3000       // But due to next's limit set to 3, we should get this:
3001       // r0: first:a, first:b, second:a
3002       // r0: second:b
3003       // r1: first:a, first:b, second:a
3004       // r1: second:b
3005       // r2: first:a, first:b, second:a
3006       // r2: second:b
3007       // r3: first:a, first:b, second:a
3008       // r3: second:b
3009       // r4: first:a, first:b, second:a
3010       // r4: second:b
3011       // r5: first:a
3012       // r6: first:a
3013       // r7: first:a
3014       // r8: first:a
3015       // r9: first:a
3016 
3017       List<Cell> results = new ArrayList<Cell>();
3018       int index = 0;
3019       while (true) {
3020         boolean more = s.next(results, 3);
3021         if ((index >> 1) < 5) {
3022           if (index % 2 == 0)
3023             assertEquals(results.size(), 3);
3024           else
3025             assertEquals(results.size(), 1);
3026         } else
3027           assertEquals(results.size(), 1);
3028         results.clear();
3029         index++;
3030         if (!more)
3031           break;
3032       }
3033     } finally {
3034       HRegion.closeHRegion(this.region);
3035       this.region = null;
3036     }
3037   }
3038 
3039   // ////////////////////////////////////////////////////////////////////////////
3040   // Split test
3041   // ////////////////////////////////////////////////////////////////////////////
3042   /**
3043    * Splits twice and verifies getting from each of the split regions.
3044    * 
3045    * @throws Exception
3046    */
3047   @Test
3048   public void testBasicSplit() throws Exception {
3049     byte[][] families = { fam1, fam2, fam3 };
3050 
3051     Configuration hc = initSplit();
3052     // Setting up region
3053     String method = this.getName();
3054     this.region = initHRegion(tableName, method, hc, families);
3055 
3056     try {
3057       LOG.info("" + HBaseTestCase.addContent(region, fam3));
3058       region.flushcache();
3059       region.compactStores();
3060       byte[] splitRow = region.checkSplit();
3061       assertNotNull(splitRow);
3062       LOG.info("SplitRow: " + Bytes.toString(splitRow));
3063       HRegion[] regions = splitRegion(region, splitRow);
3064       try {
3065         // Need to open the regions.
3066         // TODO: Add an 'open' to HRegion... don't do open by constructing
3067         // instance.
3068         for (int i = 0; i < regions.length; i++) {
3069           regions[i] = HRegion.openHRegion(regions[i], null);
3070         }
3071         // Assert can get rows out of new regions. Should be able to get first
3072         // row from first region and the midkey from second region.
3073         assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
3074         assertGet(regions[1], fam3, splitRow);
3075         // Test I can get scanner and that it starts at right place.
3076         assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
3077         assertScan(regions[1], fam3, splitRow);
3078         // Now prove can't split regions that have references.
3079         for (int i = 0; i < regions.length; i++) {
3080           // Add so much data to this region, we create a store file that is >
3081           // than one of our unsplitable references. it will.
3082           for (int j = 0; j < 2; j++) {
3083             HBaseTestCase.addContent(regions[i], fam3);
3084           }
3085           HBaseTestCase.addContent(regions[i], fam2);
3086           HBaseTestCase.addContent(regions[i], fam1);
3087           regions[i].flushcache();
3088         }
3089 
3090         byte[][] midkeys = new byte[regions.length][];
3091         // To make regions splitable force compaction.
3092         for (int i = 0; i < regions.length; i++) {
3093           regions[i].compactStores();
3094           midkeys[i] = regions[i].checkSplit();
3095         }
3096 
3097         TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
3098         // Split these two daughter regions so then I'll have 4 regions. Will
3099         // split because added data above.
3100         for (int i = 0; i < regions.length; i++) {
3101           HRegion[] rs = null;
3102           if (midkeys[i] != null) {
3103             rs = splitRegion(regions[i], midkeys[i]);
3104             for (int j = 0; j < rs.length; j++) {
3105               sortedMap.put(Bytes.toString(rs[j].getRegionName()), HRegion.openHRegion(rs[j], null));
3106             }
3107           }
3108         }
3109         LOG.info("Made 4 regions");
3110         // The splits should have been even. Test I can get some arbitrary row
3111         // out of each.
3112         int interval = (LAST_CHAR - FIRST_CHAR) / 3;
3113         byte[] b = Bytes.toBytes(START_KEY);
3114         for (HRegion r : sortedMap.values()) {
3115           assertGet(r, fam3, b);
3116           b[0] += interval;
3117         }
3118       } finally {
3119         for (int i = 0; i < regions.length; i++) {
3120           try {
3121             regions[i].close();
3122           } catch (IOException e) {
3123             // Ignore.
3124           }
3125         }
3126       }
3127     } finally {
3128       HRegion.closeHRegion(this.region);
3129       this.region = null;
3130     }
3131   }
3132 
3133   @Test
3134   public void testSplitRegion() throws IOException {
3135     byte[] qualifier = Bytes.toBytes("qualifier");
3136     Configuration hc = initSplit();
3137     int numRows = 10;
3138     byte[][] families = { fam1, fam3 };
3139 
3140     // Setting up region
3141     String method = this.getName();
3142     this.region = initHRegion(tableName, method, hc, families);
3143 
3144     // Put data in region
3145     int startRow = 100;
3146     putData(startRow, numRows, qualifier, families);
3147     int splitRow = startRow + numRows;
3148     putData(splitRow, numRows, qualifier, families);
3149     region.flushcache();
3150 
3151     HRegion[] regions = null;
3152     try {
3153       regions = splitRegion(region, Bytes.toBytes("" + splitRow));
3154       // Opening the regions returned.
3155       for (int i = 0; i < regions.length; i++) {
3156         regions[i] = HRegion.openHRegion(regions[i], null);
3157       }
3158       // Verifying that the region has been split
3159       assertEquals(2, regions.length);
3160 
3161       // Verifying that all data is still there and that data is in the right
3162       // place
3163       verifyData(regions[0], startRow, numRows, qualifier, families);
3164       verifyData(regions[1], splitRow, numRows, qualifier, families);
3165 
3166     } finally {
3167       HRegion.closeHRegion(this.region);
3168       this.region = null;
3169     }
3170   }
3171 
3172   @Test
3173   public void testClearForceSplit() throws IOException {
3174     byte[] qualifier = Bytes.toBytes("qualifier");
3175     Configuration hc = initSplit();
3176     int numRows = 10;
3177     byte[][] families = { fam1, fam3 };
3178 
3179     // Setting up region
3180     String method = this.getName();
3181     this.region = initHRegion(tableName, method, hc, families);
3182 
3183     // Put data in region
3184     int startRow = 100;
3185     putData(startRow, numRows, qualifier, families);
3186     int splitRow = startRow + numRows;
3187     byte[] splitRowBytes = Bytes.toBytes("" + splitRow);
3188     putData(splitRow, numRows, qualifier, families);
3189     region.flushcache();
3190 
3191     HRegion[] regions = null;
3192     try {
3193       // Set force split
3194       region.forceSplit(splitRowBytes);
3195       assertTrue(region.shouldForceSplit());
3196       // Split point should be the force split row
3197       assertTrue(Bytes.equals(splitRowBytes, region.checkSplit()));
3198 
3199       // Add a store that has references.
3200       HStore storeMock = Mockito.mock(HStore.class);
3201       when(storeMock.hasReferences()).thenReturn(true);
3202       when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
3203       when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
3204       when(storeMock.getColumnFamilyName()).thenReturn("cf");
3205       region.stores.put(Bytes.toBytes(storeMock.getColumnFamilyName()), storeMock);
3206       assertTrue(region.hasReferences());
3207 
3208       // Will not split since the store has references.
3209       regions = splitRegion(region, splitRowBytes);
3210       assertNull(regions);
3211 
3212       // Region force split should be cleared after the split try.
3213       assertFalse(region.shouldForceSplit());
3214 
3215       // Remove the store that has references.
3216       region.stores.remove(Bytes.toBytes(storeMock.getColumnFamilyName()));
3217       assertFalse(region.hasReferences());
3218 
3219       // Now we can split.
3220       regions = splitRegion(region, splitRowBytes);
3221 
3222       // Opening the regions returned.
3223       for (int i = 0; i < regions.length; i++) {
3224         regions[i] = HRegion.openHRegion(regions[i], null);
3225       }
3226       // Verifying that the region has been split
3227       assertEquals(2, regions.length);
3228 
3229       // Verifying that all data is still there and that data is in the right
3230       // place
3231       verifyData(regions[0], startRow, numRows, qualifier, families);
3232       verifyData(regions[1], splitRow, numRows, qualifier, families);
3233 
3234     } finally {
3235       HRegion.closeHRegion(this.region);
3236       this.region = null;
3237     }
3238   }
3239 
3240   /**
3241    * Flushes the cache in a thread while scanning. The tests verify that the
3242    * scan is coherent - e.g. the returned results are always of the same or
3243    * later update as the previous results.
3244    * 
3245    * @throws IOException
3246    *           scan / compact
3247    * @throws InterruptedException
3248    *           thread join
3249    */
3250   @Test
3251   public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3252     byte[] family = Bytes.toBytes("family");
3253     int numRows = 1000;
3254     int flushAndScanInterval = 10;
3255     int compactInterval = 10 * flushAndScanInterval;
3256 
3257     String method = "testFlushCacheWhileScanning";
3258     this.region = initHRegion(tableName, method, CONF, family);
3259     try {
3260       FlushThread flushThread = new FlushThread();
3261       flushThread.start();
3262 
3263       Scan scan = new Scan();
3264       scan.addFamily(family);
3265       scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
3266           new BinaryComparator(Bytes.toBytes(5L))));
3267 
3268       int expectedCount = 0;
3269       List<Cell> res = new ArrayList<Cell>();
3270 
3271       boolean toggle = true;
3272       for (long i = 0; i < numRows; i++) {
3273         Put put = new Put(Bytes.toBytes(i));
3274         put.setDurability(Durability.SKIP_WAL);
3275         put.add(family, qual1, Bytes.toBytes(i % 10));
3276         region.put(put);
3277 
3278         if (i != 0 && i % compactInterval == 0) {
3279           // System.out.println("iteration = " + i);
3280           region.compactStores(true);
3281         }
3282 
3283         if (i % 10 == 5L) {
3284           expectedCount++;
3285         }
3286 
3287         if (i != 0 && i % flushAndScanInterval == 0) {
3288           res.clear();
3289           InternalScanner scanner = region.getScanner(scan);
3290           if (toggle) {
3291             flushThread.flush();
3292           }
3293           while (scanner.next(res))
3294             ;
3295           if (!toggle) {
3296             flushThread.flush();
3297           }
3298           assertEquals("i=" + i, expectedCount, res.size());
3299           toggle = !toggle;
3300         }
3301       }
3302 
3303       flushThread.done();
3304       flushThread.join();
3305       flushThread.checkNoError();
3306     } finally {
3307       HRegion.closeHRegion(this.region);
3308       this.region = null;
3309     }
3310   }
3311 
3312   protected class FlushThread extends Thread {
3313     private volatile boolean done;
3314     private Throwable error = null;
3315 
3316     public void done() {
3317       done = true;
3318       synchronized (this) {
3319         interrupt();
3320       }
3321     }
3322 
3323     public void checkNoError() {
3324       if (error != null) {
3325         assertNull(error);
3326       }
3327     }
3328 
3329     @Override
3330     public void run() {
3331       done = false;
3332       while (!done) {
3333         synchronized (this) {
3334           try {
3335             wait();
3336           } catch (InterruptedException ignored) {
3337             if (done) {
3338               break;
3339             }
3340           }
3341         }
3342         try {
3343           region.flushcache();
3344         } catch (IOException e) {
3345           if (!done) {
3346             LOG.error("Error while flusing cache", e);
3347             error = e;
3348           }
3349           break;
3350         }
3351       }
3352 
3353     }
3354 
3355     public void flush() {
3356       synchronized (this) {
3357         notify();
3358       }
3359 
3360     }
3361   }
3362 
3363   /**
3364    * Writes very wide records and scans for the latest every time.. Flushes and
3365    * compacts the region every now and then to keep things realistic.
3366    * 
3367    * @throws IOException
3368    *           by flush / scan / compaction
3369    * @throws InterruptedException
3370    *           when joining threads
3371    */
3372   @Test
3373   public void testWritesWhileScanning() throws IOException, InterruptedException {
3374     int testCount = 100;
3375     int numRows = 1;
3376     int numFamilies = 10;
3377     int numQualifiers = 100;
3378     int flushInterval = 7;
3379     int compactInterval = 5 * flushInterval;
3380     byte[][] families = new byte[numFamilies][];
3381     for (int i = 0; i < numFamilies; i++) {
3382       families[i] = Bytes.toBytes("family" + i);
3383     }
3384     byte[][] qualifiers = new byte[numQualifiers][];
3385     for (int i = 0; i < numQualifiers; i++) {
3386       qualifiers[i] = Bytes.toBytes("qual" + i);
3387     }
3388 
3389     String method = "testWritesWhileScanning";
3390     this.region = initHRegion(tableName, method, CONF, families);
3391     try {
3392       PutThread putThread = new PutThread(numRows, families, qualifiers);
3393       putThread.start();
3394       putThread.waitForFirstPut();
3395 
3396       FlushThread flushThread = new FlushThread();
3397       flushThread.start();
3398 
3399       Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3400 
3401       int expectedCount = numFamilies * numQualifiers;
3402       List<Cell> res = new ArrayList<Cell>();
3403 
3404       long prevTimestamp = 0L;
3405       for (int i = 0; i < testCount; i++) {
3406 
3407         if (i != 0 && i % compactInterval == 0) {
3408           region.compactStores(true);
3409         }
3410 
3411         if (i != 0 && i % flushInterval == 0) {
3412           flushThread.flush();
3413         }
3414 
3415         boolean previousEmpty = res.isEmpty();
3416         res.clear();
3417         InternalScanner scanner = region.getScanner(scan);
3418         while (scanner.next(res))
3419           ;
3420         if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3421           assertEquals("i=" + i, expectedCount, res.size());
3422           long timestamp = res.get(0).getTimestamp();
3423           assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3424               timestamp >= prevTimestamp);
3425           prevTimestamp = timestamp;
3426         }
3427       }
3428 
3429       putThread.done();
3430 
3431       region.flushcache();
3432 
3433       putThread.join();
3434       putThread.checkNoError();
3435 
3436       flushThread.done();
3437       flushThread.join();
3438       flushThread.checkNoError();
3439     } finally {
3440       try {
3441         HRegion.closeHRegion(this.region);
3442       } catch (DroppedSnapshotException dse) {
3443         // We could get this on way out because we interrupt the background flusher and it could
3444         // fail anywhere causing a DSE over in the background flusher... only it is not properly
3445         // dealt with so could still be memory hanging out when we get to here -- memory we can't
3446         // flush because the accounting is 'off' since original DSE.
3447       }
3448       this.region = null;
3449     }
3450   }
3451 
3452   protected class PutThread extends Thread {
3453     private volatile boolean done;
3454     private volatile int numPutsFinished = 0;
3455 
3456     private Throwable error = null;
3457     private int numRows;
3458     private byte[][] families;
3459     private byte[][] qualifiers;
3460 
3461     private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3462       this.numRows = numRows;
3463       this.families = families;
3464       this.qualifiers = qualifiers;
3465     }
3466 
3467     /**
3468      * Block until this thread has put at least one row.
3469      */
3470     public void waitForFirstPut() throws InterruptedException {
3471       // wait until put thread actually puts some data
3472       while (numPutsFinished == 0) {
3473         checkNoError();
3474         Thread.sleep(50);
3475       }
3476     }
3477 
3478     public void done() {
3479       done = true;
3480       synchronized (this) {
3481         interrupt();
3482       }
3483     }
3484 
3485     public void checkNoError() {
3486       if (error != null) {
3487         assertNull(error);
3488       }
3489     }
3490 
3491     @Override
3492     public void run() {
3493       done = false;
3494       while (!done) {
3495         try {
3496           for (int r = 0; r < numRows; r++) {
3497             byte[] row = Bytes.toBytes("row" + r);
3498             Put put = new Put(row);
3499             put.setDurability(Durability.SKIP_WAL);
3500             byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3501             for (byte[] family : families) {
3502               for (byte[] qualifier : qualifiers) {
3503                 put.add(family, qualifier, (long) numPutsFinished, value);
3504               }
3505             }
3506             region.put(put);
3507             numPutsFinished++;
3508             if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3509               System.out.println("put iteration = " + numPutsFinished);
3510               Delete delete = new Delete(row, (long) numPutsFinished - 30);
3511               region.delete(delete);
3512             }
3513             numPutsFinished++;
3514           }
3515         } catch (InterruptedIOException e) {
3516           // This is fine. It means we are done, or didn't get the lock on time
3517         } catch (IOException e) {
3518           LOG.error("error while putting records", e);
3519           error = e;
3520           break;
3521         }
3522       }
3523 
3524     }
3525 
3526   }
3527 
3528   /**
3529    * Writes very wide records and gets the latest row every time.. Flushes and
3530    * compacts the region aggressivly to catch issues.
3531    * 
3532    * @throws IOException
3533    *           by flush / scan / compaction
3534    * @throws InterruptedException
3535    *           when joining threads
3536    */
3537   @Test
3538   public void testWritesWhileGetting() throws Exception {
3539     int testCount = 100;
3540     int numRows = 1;
3541     int numFamilies = 10;
3542     int numQualifiers = 100;
3543     int compactInterval = 100;
3544     byte[][] families = new byte[numFamilies][];
3545     for (int i = 0; i < numFamilies; i++) {
3546       families[i] = Bytes.toBytes("family" + i);
3547     }
3548     byte[][] qualifiers = new byte[numQualifiers][];
3549     for (int i = 0; i < numQualifiers; i++) {
3550       qualifiers[i] = Bytes.toBytes("qual" + i);
3551     }
3552 
3553 
3554     String method = "testWritesWhileGetting";
3555     // This test flushes constantly and can cause many files to be created,
3556     // possibly
3557     // extending over the ulimit. Make sure compactions are aggressive in
3558     // reducing
3559     // the number of HFiles created.
3560     Configuration conf = HBaseConfiguration.create(CONF);
3561     conf.setInt("hbase.hstore.compaction.min", 1);
3562     conf.setInt("hbase.hstore.compaction.max", 1000);
3563     this.region = initHRegion(tableName, method, conf, families);
3564     PutThread putThread = null;
3565     MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3566     try {
3567       putThread = new PutThread(numRows, families, qualifiers);
3568       putThread.start();
3569       putThread.waitForFirstPut();
3570 
3571       // Add a thread that flushes as fast as possible
3572       ctx.addThread(new RepeatingTestThread(ctx) {
3573         private int flushesSinceCompact = 0;
3574         private final int maxFlushesSinceCompact = 20;
3575 
3576         @Override
3577         public void doAnAction() throws Exception {
3578           if (region.flushcache().isCompactionNeeded()) {
3579             ++flushesSinceCompact;
3580           }
3581           // Compact regularly to avoid creating too many files and exceeding
3582           // the ulimit.
3583           if (flushesSinceCompact == maxFlushesSinceCompact) {
3584             region.compactStores(false);
3585             flushesSinceCompact = 0;
3586           }
3587         }
3588       });
3589       ctx.startThreads();
3590 
3591       Get get = new Get(Bytes.toBytes("row0"));
3592       Result result = null;
3593 
3594       int expectedCount = numFamilies * numQualifiers;
3595 
3596       long prevTimestamp = 0L;
3597       for (int i = 0; i < testCount; i++) {
3598 
3599         boolean previousEmpty = result == null || result.isEmpty();
3600         result = region.get(get);
3601         if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3602           assertEquals("i=" + i, expectedCount, result.size());
3603           // TODO this was removed, now what dangit?!
3604           // search looking for the qualifier in question?
3605           long timestamp = 0;
3606           for (Cell kv : result.rawCells()) {
3607             if (CellUtil.matchingFamily(kv, families[0])
3608                 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3609               timestamp = kv.getTimestamp();
3610             }
3611           }
3612           assertTrue(timestamp >= prevTimestamp);
3613           prevTimestamp = timestamp;
3614           Cell previousKV = null;
3615 
3616           for (Cell kv : result.rawCells()) {
3617             byte[] thisValue = CellUtil.cloneValue(kv);
3618             if (previousKV != null) {
3619               if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3620                 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3621                     + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3622                     + "(memStoreTS:" + kv.getMvccVersion() + ")");
3623                 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3624               }
3625             }
3626             previousKV = kv;
3627           }
3628         }
3629       }
3630     } finally {
3631       if (putThread != null)
3632         putThread.done();
3633 
3634       region.flushcache();
3635 
3636       if (putThread != null) {
3637         putThread.join();
3638         putThread.checkNoError();
3639       }
3640 
3641       ctx.stop();
3642       HRegion.closeHRegion(this.region);
3643       this.region = null;
3644     }
3645   }
3646 
3647   @Test
3648   public void testHolesInMeta() throws Exception {
3649     byte[] family = Bytes.toBytes("family");
3650     this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF,
3651         false, family);
3652     try {
3653       byte[] rowNotServed = Bytes.toBytes("a");
3654       Get g = new Get(rowNotServed);
3655       try {
3656         region.get(g);
3657         fail();
3658       } catch (WrongRegionException x) {
3659         // OK
3660       }
3661       byte[] row = Bytes.toBytes("y");
3662       g = new Get(row);
3663       region.get(g);
3664     } finally {
3665       HRegion.closeHRegion(this.region);
3666       this.region = null;
3667     }
3668   }
3669 
3670   @Test
3671   public void testIndexesScanWithOneDeletedRow() throws IOException {
3672     byte[] family = Bytes.toBytes("family");
3673 
3674     // Setting up region
3675     String method = "testIndexesScanWithOneDeletedRow";
3676     this.region = initHRegion(tableName, method, CONF, family);
3677     try {
3678       Put put = new Put(Bytes.toBytes(1L));
3679       put.add(family, qual1, 1L, Bytes.toBytes(1L));
3680       region.put(put);
3681 
3682       region.flushcache();
3683 
3684       Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3685       region.delete(delete);
3686 
3687       put = new Put(Bytes.toBytes(2L));
3688       put.add(family, qual1, 2L, Bytes.toBytes(2L));
3689       region.put(put);
3690 
3691       Scan idxScan = new Scan();
3692       idxScan.addFamily(family);
3693       idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
3694           new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
3695               new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
3696               CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
3697       InternalScanner scanner = region.getScanner(idxScan);
3698       List<Cell> res = new ArrayList<Cell>();
3699 
3700       while (scanner.next(res))
3701         ;
3702       assertEquals(1L, res.size());
3703     } finally {
3704       HRegion.closeHRegion(this.region);
3705       this.region = null;
3706     }
3707   }
3708 
3709   // ////////////////////////////////////////////////////////////////////////////
3710   // Bloom filter test
3711   // ////////////////////////////////////////////////////////////////////////////
3712   @Test
3713   public void testBloomFilterSize() throws IOException {
3714     byte[] fam1 = Bytes.toBytes("fam1");
3715     byte[] qf1 = Bytes.toBytes("col");
3716     byte[] val1 = Bytes.toBytes("value1");
3717     // Create Table
3718     HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
3719         .setBloomFilterType(BloomType.ROWCOL);
3720 
3721     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3722     htd.addFamily(hcd);
3723     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3724     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3725     try {
3726       int num_unique_rows = 10;
3727       int duplicate_multiplier = 2;
3728       int num_storefiles = 4;
3729 
3730       int version = 0;
3731       for (int f = 0; f < num_storefiles; f++) {
3732         for (int i = 0; i < duplicate_multiplier; i++) {
3733           for (int j = 0; j < num_unique_rows; j++) {
3734             Put put = new Put(Bytes.toBytes("row" + j));
3735             put.setDurability(Durability.SKIP_WAL);
3736             put.add(fam1, qf1, version++, val1);
3737             region.put(put);
3738           }
3739         }
3740         region.flushcache();
3741       }
3742       // before compaction
3743       HStore store = (HStore) region.getStore(fam1);
3744       Collection<StoreFile> storeFiles = store.getStorefiles();
3745       for (StoreFile storefile : storeFiles) {
3746         StoreFile.Reader reader = storefile.getReader();
3747         reader.loadFileInfo();
3748         reader.loadBloomfilter();
3749         assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
3750         assertEquals(num_unique_rows, reader.getFilterEntries());
3751       }
3752 
3753       region.compactStores(true);
3754 
3755       // after compaction
3756       storeFiles = store.getStorefiles();
3757       for (StoreFile storefile : storeFiles) {
3758         StoreFile.Reader reader = storefile.getReader();
3759         reader.loadFileInfo();
3760         reader.loadBloomfilter();
3761         assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
3762         assertEquals(num_unique_rows, reader.getFilterEntries());
3763       }
3764     } finally {
3765       HRegion.closeHRegion(this.region);
3766       this.region = null;
3767     }
3768   }
3769 
3770   @Test
3771   public void testAllColumnsWithBloomFilter() throws IOException {
3772     byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
3773     byte[] FAMILY = Bytes.toBytes("family");
3774 
3775     // Create table
3776     HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
3777         .setBloomFilterType(BloomType.ROWCOL);
3778     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
3779     htd.addFamily(hcd);
3780     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3781     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3782     try {
3783       // For row:0, col:0: insert versions 1 through 5.
3784       byte row[] = Bytes.toBytes("row:" + 0);
3785       byte column[] = Bytes.toBytes("column:" + 0);
3786       Put put = new Put(row);
3787       put.setDurability(Durability.SKIP_WAL);
3788       for (long idx = 1; idx <= 4; idx++) {
3789         put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
3790       }
3791       region.put(put);
3792 
3793       // Flush
3794       region.flushcache();
3795 
3796       // Get rows
3797       Get get = new Get(row);
3798       get.setMaxVersions();
3799       Cell[] kvs = region.get(get).rawCells();
3800 
3801       // Check if rows are correct
3802       assertEquals(4, kvs.length);
3803       checkOneCell(kvs[0], FAMILY, 0, 0, 4);
3804       checkOneCell(kvs[1], FAMILY, 0, 0, 3);
3805       checkOneCell(kvs[2], FAMILY, 0, 0, 2);
3806       checkOneCell(kvs[3], FAMILY, 0, 0, 1);
3807     } finally {
3808       HRegion.closeHRegion(this.region);
3809       this.region = null;
3810     }
3811   }
3812 
3813   /**
3814    * Testcase to cover bug-fix for HBASE-2823 Ensures correct delete when
3815    * issuing delete row on columns with bloom filter set to row+col
3816    * (BloomType.ROWCOL)
3817    */
3818   @Test
3819   public void testDeleteRowWithBloomFilter() throws IOException {
3820     byte[] familyName = Bytes.toBytes("familyName");
3821 
3822     // Create Table
3823     HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
3824         .setBloomFilterType(BloomType.ROWCOL);
3825 
3826     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3827     htd.addFamily(hcd);
3828     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3829     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3830     try {
3831       // Insert some data
3832       byte row[] = Bytes.toBytes("row1");
3833       byte col[] = Bytes.toBytes("col1");
3834 
3835       Put put = new Put(row);
3836       put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
3837       region.put(put);
3838       region.flushcache();
3839 
3840       Delete del = new Delete(row);
3841       region.delete(del);
3842       region.flushcache();
3843 
3844       // Get remaining rows (should have none)
3845       Get get = new Get(row);
3846       get.addColumn(familyName, col);
3847 
3848       Cell[] keyValues = region.get(get).rawCells();
3849       assertTrue(keyValues.length == 0);
3850     } finally {
3851       HRegion.closeHRegion(this.region);
3852       this.region = null;
3853     }
3854   }
3855 
3856   @Test
3857   public void testgetHDFSBlocksDistribution() throws Exception {
3858     HBaseTestingUtility htu = new HBaseTestingUtility();
3859     final int DEFAULT_BLOCK_SIZE = 1024;
3860     htu.getConfiguration().setLong("dfs.namenode.fs-limits.min-block-size", 0);
3861     htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
3862     htu.getConfiguration().setInt("dfs.replication", 2);
3863 
3864     // set up a cluster with 3 nodes
3865     MiniHBaseCluster cluster = null;
3866     String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
3867     int regionServersCount = 3;
3868 
3869     try {
3870       cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
3871       byte[][] families = { fam1, fam2 };
3872       HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);
3873 
3874       // Setting up region
3875       byte row[] = Bytes.toBytes("row1");
3876       byte col[] = Bytes.toBytes("col1");
3877 
3878       Put put = new Put(row);
3879       put.add(fam1, col, 1, Bytes.toBytes("test1"));
3880       put.add(fam2, col, 1, Bytes.toBytes("test2"));
3881       ht.put(put);
3882 
3883       HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
3884           .get(0);
3885       firstRegion.flushcache();
3886       HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
3887 
3888       // given the default replication factor is 2 and we have 2 HFiles,
3889       // we will have total of 4 replica of blocks on 3 datanodes; thus there
3890       // must be at least one host that have replica for 2 HFiles. That host's
3891       // weight will be equal to the unique block weight.
3892       long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
3893 
3894       String topHost = blocksDistribution1.getTopHosts().get(0);
3895       long topHostWeight = blocksDistribution1.getWeight(topHost);
3896       assertTrue(uniqueBlocksWeight1 == topHostWeight);
3897 
3898       // use the static method to compute the value, it should be the same.
3899       // static method is used by load balancer or other components
3900       HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
3901           htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
3902       long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
3903 
3904       assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
3905 
3906       ht.close();
3907     } finally {
3908       if (cluster != null) {
3909         htu.shutdownMiniCluster();
3910       }
3911     }
3912   }
3913 
3914   /**
3915    * Testcase to check state of region initialization task set to ABORTED or not
3916    * if any exceptions during initialization
3917    * 
3918    * @throws Exception
3919    */
3920   @Test
3921   public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
3922     TableName tableName = TableName.valueOf(name.getMethodName());
3923     HRegionInfo info = null;
3924     try {
3925       FileSystem fs = Mockito.mock(FileSystem.class);
3926       when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
3927       HTableDescriptor htd = new HTableDescriptor(tableName);
3928       htd.addFamily(new HColumnDescriptor("cf"));
3929       info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
3930           HConstants.EMPTY_BYTE_ARRAY, false);
3931       Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
3932       region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
3933       // region initialization throws IOException and set task state to ABORTED.
3934       region.initialize();
3935       fail("Region initialization should fail due to IOException");
3936     } catch (IOException io) {
3937       List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
3938       for (MonitoredTask monitoredTask : tasks) {
3939         if (!(monitoredTask instanceof MonitoredRPCHandler)
3940             && monitoredTask.getDescription().contains(region.toString())) {
3941           assertTrue("Region state should be ABORTED.",
3942               monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
3943           break;
3944         }
3945       }
3946     } finally {
3947       HRegion.closeHRegion(region);
3948     }
3949   }
3950 
3951   /**
3952    * Verifies that the .regioninfo file is written on region creation and that
3953    * is recreated if missing during region opening.
3954    */
3955   @Test
3956   public void testRegionInfoFileCreation() throws IOException {
3957     Path rootDir = new Path(dir + "testRegionInfoFileCreation");
3958 
3959     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
3960     htd.addFamily(new HColumnDescriptor("cf"));
3961 
3962     HRegionInfo hri = new HRegionInfo(htd.getTableName());
3963 
3964     // Create a region and skip the initialization (like CreateTableHandler)
3965     HRegion region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
3966 //    HRegion region = TEST_UTIL.createLocalHRegion(hri, htd);
3967     Path regionDir = region.getRegionFileSystem().getRegionDir();
3968     FileSystem fs = region.getRegionFileSystem().getFileSystem();
3969     HRegion.closeHRegion(region);
3970 
3971     Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
3972 
3973     // Verify that the .regioninfo file is present
3974     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3975         fs.exists(regionInfoFile));
3976 
3977     // Try to open the region
3978     region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3979     assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3980     HRegion.closeHRegion(region);
3981 
3982     // Verify that the .regioninfo file is still there
3983     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3984         fs.exists(regionInfoFile));
3985 
3986     // Remove the .regioninfo file and verify is recreated on region open
3987     fs.delete(regionInfoFile);
3988     assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
3989         fs.exists(regionInfoFile));
3990 
3991     region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3992 //    region = TEST_UTIL.openHRegion(hri, htd);
3993     assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3994     HRegion.closeHRegion(region);
3995 
3996     // Verify that the .regioninfo file is still there
3997     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3998         fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
3999   }
4000 
4001   /**
4002    * TestCase for increment
4003    */
4004   private static class Incrementer implements Runnable {
4005     private HRegion region;
4006     private final static byte[] incRow = Bytes.toBytes("incRow");
4007     private final static byte[] family = Bytes.toBytes("family");
4008     private final static byte[] qualifier = Bytes.toBytes("qualifier");
4009     private final static long ONE = 1l;
4010     private int incCounter;
4011 
4012     public Incrementer(HRegion region, int incCounter) {
4013       this.region = region;
4014       this.incCounter = incCounter;
4015     }
4016 
4017     @Override
4018     public void run() {
4019       int count = 0;
4020       while (count < incCounter) {
4021         Increment inc = new Increment(incRow);
4022         inc.addColumn(family, qualifier, ONE);
4023         count++;
4024         try {
4025           region.increment(inc);
4026         } catch (IOException e) {
4027           e.printStackTrace();
4028           break;
4029         }
4030       }
4031     }
4032   }
4033 
4034   /**
4035    * Test case to check increment function with memstore flushing
4036    * @throws Exception
4037    */
4038   @Test
4039   public void testParallelIncrementWithMemStoreFlush() throws Exception {
4040     byte[] family = Incrementer.family;
4041     this.region = initHRegion(tableName, method, CONF, family);
4042     final HRegion region = this.region;
4043     final AtomicBoolean incrementDone = new AtomicBoolean(false);
4044     Runnable flusher = new Runnable() {
4045       @Override
4046       public void run() {
4047         while (!incrementDone.get()) {
4048           try {
4049             region.flushcache();
4050           } catch (Exception e) {
4051             e.printStackTrace();
4052           }
4053         }
4054       }
4055     };
4056 
4057     // after all increment finished, the row will increment to 20*100 = 2000
4058     int threadNum = 20;
4059     int incCounter = 100;
4060     long expected = threadNum * incCounter;
4061     Thread[] incrementers = new Thread[threadNum];
4062     Thread flushThread = new Thread(flusher);
4063     for (int i = 0; i < threadNum; i++) {
4064       incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
4065       incrementers[i].start();
4066     }
4067     flushThread.start();
4068     for (int i = 0; i < threadNum; i++) {
4069       incrementers[i].join();
4070     }
4071 
4072     incrementDone.set(true);
4073     flushThread.join();
4074 
4075     Get get = new Get(Incrementer.incRow);
4076     get.addColumn(Incrementer.family, Incrementer.qualifier);
4077     get.setMaxVersions(1);
4078     Result res = this.region.get(get);
4079     List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
4080 
4081     // we just got the latest version
4082     assertEquals(kvs.size(), 1);
4083     Cell kv = kvs.get(0);
4084     assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
4085     this.region = null;
4086   }
4087 
4088   /**
4089    * TestCase for append
4090    */
4091   private static class Appender implements Runnable {
4092     private HRegion region;
4093     private final static byte[] appendRow = Bytes.toBytes("appendRow");
4094     private final static byte[] family = Bytes.toBytes("family");
4095     private final static byte[] qualifier = Bytes.toBytes("qualifier");
4096     private final static byte[] CHAR = Bytes.toBytes("a");
4097     private int appendCounter;
4098 
4099     public Appender(HRegion region, int appendCounter) {
4100       this.region = region;
4101       this.appendCounter = appendCounter;
4102     }
4103 
4104     @Override
4105     public void run() {
4106       int count = 0;
4107       while (count < appendCounter) {
4108         Append app = new Append(appendRow);
4109         app.add(family, qualifier, CHAR);
4110         count++;
4111         try {
4112           region.append(app);
4113         } catch (IOException e) {
4114           e.printStackTrace();
4115           break;
4116         }
4117       }
4118     }
4119   }
4120 
4121   /**
4122    * Test case to check append function with memstore flushing
4123    * @throws Exception
4124    */
4125   @Test
4126   public void testParallelAppendWithMemStoreFlush() throws Exception {
4127     byte[] family = Appender.family;
4128     this.region = initHRegion(tableName, method, CONF, family);
4129     final HRegion region = this.region;
4130     final AtomicBoolean appendDone = new AtomicBoolean(false);
4131     Runnable flusher = new Runnable() {
4132       @Override
4133       public void run() {
4134         while (!appendDone.get()) {
4135           try {
4136             region.flushcache();
4137           } catch (Exception e) {
4138             e.printStackTrace();
4139           }
4140         }
4141       }
4142     };
4143 
4144     // after all append finished, the value will append to threadNum *
4145     // appendCounter Appender.CHAR
4146     int threadNum = 20;
4147     int appendCounter = 100;
4148     byte[] expected = new byte[threadNum * appendCounter];
4149     for (int i = 0; i < threadNum * appendCounter; i++) {
4150       System.arraycopy(Appender.CHAR, 0, expected, i, 1);
4151     }
4152     Thread[] appenders = new Thread[threadNum];
4153     Thread flushThread = new Thread(flusher);
4154     for (int i = 0; i < threadNum; i++) {
4155       appenders[i] = new Thread(new Appender(this.region, appendCounter));
4156       appenders[i].start();
4157     }
4158     flushThread.start();
4159     for (int i = 0; i < threadNum; i++) {
4160       appenders[i].join();
4161     }
4162 
4163     appendDone.set(true);
4164     flushThread.join();
4165 
4166     Get get = new Get(Appender.appendRow);
4167     get.addColumn(Appender.family, Appender.qualifier);
4168     get.setMaxVersions(1);
4169     Result res = this.region.get(get);
4170     List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
4171 
4172     // we just got the latest version
4173     assertEquals(kvs.size(), 1);
4174     Cell kv = kvs.get(0);
4175     byte[] appendResult = new byte[kv.getValueLength()];
4176     System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
4177     assertArrayEquals(expected, appendResult);
4178     this.region = null;
4179   }
4180 
4181   /**
4182    * Test case to check put function with memstore flushing for same row, same ts
4183    * @throws Exception
4184    */
4185   @Test
4186   public void testPutWithMemStoreFlush() throws Exception {
4187     byte[] family = Bytes.toBytes("family");
4188     ;
4189     byte[] qualifier = Bytes.toBytes("qualifier");
4190     byte[] row = Bytes.toBytes("putRow");
4191     byte[] value = null;
4192     this.region = initHRegion(tableName, method, CONF, family);
4193     Put put = null;
4194     Get get = null;
4195     List<Cell> kvs = null;
4196     Result res = null;
4197 
4198     put = new Put(row);
4199     value = Bytes.toBytes("value0");
4200     put.add(family, qualifier, 1234567l, value);
4201     region.put(put);
4202     get = new Get(row);
4203     get.addColumn(family, qualifier);
4204     get.setMaxVersions();
4205     res = this.region.get(get);
4206     kvs = res.getColumnCells(family, qualifier);
4207     assertEquals(1, kvs.size());
4208     assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4209 
4210     region.flushcache();
4211     get = new Get(row);
4212     get.addColumn(family, qualifier);
4213     get.setMaxVersions();
4214     res = this.region.get(get);
4215     kvs = res.getColumnCells(family, qualifier);
4216     assertEquals(1, kvs.size());
4217     assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4218 
4219     put = new Put(row);
4220     value = Bytes.toBytes("value1");
4221     put.add(family, qualifier, 1234567l, value);
4222     region.put(put);
4223     get = new Get(row);
4224     get.addColumn(family, qualifier);
4225     get.setMaxVersions();
4226     res = this.region.get(get);
4227     kvs = res.getColumnCells(family, qualifier);
4228     assertEquals(1, kvs.size());
4229     assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4230 
4231     region.flushcache();
4232     get = new Get(row);
4233     get.addColumn(family, qualifier);
4234     get.setMaxVersions();
4235     res = this.region.get(get);
4236     kvs = res.getColumnCells(family, qualifier);
4237     assertEquals(1, kvs.size());
4238     assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4239   }
4240 
4241   @Test
4242   public void testDurability() throws Exception {
4243     String method = "testDurability";
4244     // there are 5 x 5 cases:
4245     // table durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) x mutation
4246     // durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT)
4247 
4248     // expected cases for append and sync wal
4249     durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4250     durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4251     durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4252 
4253     durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4254     durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4255     durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4256 
4257     durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4258     durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4259 
4260     durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
4261     durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4262 
4263     durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
4264     durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
4265     durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
4266 
4267     // expected cases for async wal
4268     durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4269     durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4270     durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4271     durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4272     durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
4273     durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4274 
4275     durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4276     durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4277     durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4278     durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4279     durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
4280     durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
4281 
4282     // expect skip wal cases
4283     durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4284     durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4285     durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4286     durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
4287     durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
4288     durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
4289 
4290   }
4291 
4292   private void durabilityTest(String method, Durability tableDurability,
4293       Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
4294       final boolean expectSyncFromLogSyncer) throws Exception {
4295     Configuration conf = HBaseConfiguration.create(CONF);
4296     method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
4297     TableName tableName = TableName.valueOf(method);
4298     byte[] family = Bytes.toBytes("family");
4299     Path logDir = new Path(new Path(dir + method), "log");
4300     HLog hlog = HLogFactory.createHLog(FILESYSTEM, logDir, UUID.randomUUID().toString(), conf);
4301     final HLog log = spy(hlog);
4302     this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
4303         HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, log,
4304         new byte[][] { family });
4305 
4306     Put put = new Put(Bytes.toBytes("r1"));
4307     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
4308     put.setDurability(mutationDurability);
4309     region.put(put);
4310 
4311     //verify append called or not
4312     verify(log, expectAppend ? times(1) : never())
4313       .appendNoSync((HRegionInfo)any(), eq(tableName), (WALEdit)any(), (List<UUID>)any(),
4314         anyLong(), (HTableDescriptor)any(), (AtomicLong)any(), anyBoolean(), anyLong(), anyLong());
4315 
4316     // verify sync called or not
4317     if (expectSync || expectSyncFromLogSyncer) {
4318       TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
4319         @Override
4320         public boolean evaluate() throws Exception {
4321           try {
4322             if (expectSync) {
4323               verify(log, times(1)).sync(anyLong()); // Hregion calls this one
4324             } else if (expectSyncFromLogSyncer) {
4325               verify(log, times(1)).sync(); // log syncer calls this one
4326             }
4327           } catch (Throwable ignore) {
4328           }
4329           return true;
4330         }
4331       });
4332     } else {
4333       verify(log, never()).sync(anyLong());
4334       verify(log, never()).sync();
4335     }
4336 
4337     HRegion.closeHRegion(this.region);
4338     this.region = null;
4339   }
4340 
4341   private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4342     for (int i = startRow; i < startRow + numRows; i++) {
4343       Put put = new Put(Bytes.toBytes("" + i));
4344       put.setDurability(Durability.SKIP_WAL);
4345       for (byte[] family : families) {
4346         put.add(family, qf, null);
4347       }
4348       region.put(put);
4349     }
4350   }
4351 
4352   private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
4353       throws IOException {
4354     for (int i = startRow; i < startRow + numRows; i++) {
4355       byte[] row = Bytes.toBytes("" + i);
4356       Get get = new Get(row);
4357       for (byte[] family : families) {
4358         get.addColumn(family, qf);
4359       }
4360       Result result = newReg.get(get);
4361       Cell[] raw = result.rawCells();
4362       assertEquals(families.length, result.size());
4363       for (int j = 0; j < families.length; j++) {
4364         assertTrue(CellUtil.matchingRow(raw[j], row));
4365         assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
4366         assertTrue(CellUtil.matchingQualifier(raw[j], qf));
4367       }
4368     }
4369   }
4370 
4371   private void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
4372     // Now I have k, get values out and assert they are as expected.
4373     Get get = new Get(k).addFamily(family).setMaxVersions();
4374     Cell[] results = r.get(get).rawCells();
4375     for (int j = 0; j < results.length; j++) {
4376       byte[] tmp = CellUtil.cloneValue(results[j]);
4377       // Row should be equal to value every time.
4378       assertTrue(Bytes.equals(k, tmp));
4379     }
4380   }
4381 
4382   /*
4383    * Assert first value in the passed region is <code>firstValue</code>.
4384    * 
4385    * @param r
4386    * 
4387    * @param fs
4388    * 
4389    * @param firstValue
4390    * 
4391    * @throws IOException
4392    */
4393   private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
4394       throws IOException {
4395     byte[][] families = { fs };
4396     Scan scan = new Scan();
4397     for (int i = 0; i < families.length; i++)
4398       scan.addFamily(families[i]);
4399     InternalScanner s = r.getScanner(scan);
4400     try {
4401       List<Cell> curVals = new ArrayList<Cell>();
4402       boolean first = true;
4403       OUTER_LOOP: while (s.next(curVals)) {
4404         for (Cell kv : curVals) {
4405           byte[] val = CellUtil.cloneValue(kv);
4406           byte[] curval = val;
4407           if (first) {
4408             first = false;
4409             assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4410           } else {
4411             // Not asserting anything. Might as well break.
4412             break OUTER_LOOP;
4413           }
4414         }
4415       }
4416     } finally {
4417       s.close();
4418     }
4419   }
4420 
4421   /**
4422    * Test that we get the expected flush results back
4423    * @throws IOException
4424    */
4425   @Test
4426   public void testFlushResult() throws IOException {
4427     String method = name.getMethodName();
4428     byte[] tableName = Bytes.toBytes(method);
4429     byte[] family = Bytes.toBytes("family");
4430 
4431     this.region = initHRegion(tableName, method, family);
4432 
4433     // empty memstore, flush doesn't run
4434     HRegion.FlushResult fr = region.flushcache();
4435     assertFalse(fr.isFlushSucceeded());
4436     assertFalse(fr.isCompactionNeeded());
4437 
4438     // Flush enough files to get up to the threshold, doesn't need compactions
4439     for (int i = 0; i < 2; i++) {
4440       Put put = new Put(tableName).add(family, family, tableName);
4441       region.put(put);
4442       fr = region.flushcache();
4443       assertTrue(fr.isFlushSucceeded());
4444       assertFalse(fr.isCompactionNeeded());
4445     }
4446 
4447     // Two flushes after the threshold, compactions are needed
4448     for (int i = 0; i < 2; i++) {
4449       Put put = new Put(tableName).add(family, family, tableName);
4450       region.put(put);
4451       fr = region.flushcache();
4452       assertTrue(fr.isFlushSucceeded());
4453       assertTrue(fr.isCompactionNeeded());
4454     }
4455   }
4456 
4457   private Configuration initSplit() {
4458     // Always compact if there is more than one store file.
4459     CONF.setInt("hbase.hstore.compactionThreshold", 2);
4460 
4461     // Make lease timeout longer, lease checks less frequent
4462     CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4463 
4464     CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4465 
4466     // Increase the amount of time between client retries
4467     CONF.setLong("hbase.client.pause", 15 * 1000);
4468 
4469     // This size should make it so we always split using the addContent
4470     // below. After adding all data, the first region is 1.3M
4471     CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4472     return CONF;
4473   }
4474 
4475   /**
4476    * @param tableName
4477    * @param callingMethod
4478    * @param conf
4479    * @param families
4480    * @throws IOException
4481    * @return A region on which you must call
4482    *         {@link HRegion#closeHRegion(HRegion)} when done.
4483    */
4484   public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
4485       byte[]... families) throws IOException {
4486     return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
4487   }
4488 
4489   /**
4490    * @param tableName
4491    * @param callingMethod
4492    * @param conf
4493    * @param families
4494    * @throws IOException
4495    * @return A region on which you must call
4496    *         {@link HRegion#closeHRegion(HRegion)} when done.
4497    */
4498   public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4499       byte[]... families) throws IOException {
4500     return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4501   }
4502 
4503   /**
4504    * @param tableName
4505    * @param callingMethod
4506    * @param conf
4507    * @param isReadOnly
4508    * @param families
4509    * @throws IOException
4510    * @return A region on which you must call
4511    *         {@link HRegion#closeHRegion(HRegion)} when done.
4512    */
4513   public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4514       boolean isReadOnly, byte[]... families) throws IOException {
4515     return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4516   }
4517 
4518   private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4519       String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4520       throws IOException {
4521     return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
4522         Durability.SYNC_WAL, null, families);
4523   }
4524 
4525   /**
4526    * @param tableName
4527    * @param startKey
4528    * @param stopKey
4529    * @param callingMethod
4530    * @param conf
4531    * @param isReadOnly
4532    * @param families
4533    * @throws IOException
4534    * @return A region on which you must call
4535    *         {@link HRegion#closeHRegion(HRegion)} when done.
4536    */
4537   private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4538       String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4539       HLog hlog, byte[]... families) throws IOException {
4540     return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly, durability, hlog, families);
4541   }
4542 
4543   /**
4544    * Assert that the passed in Cell has expected contents for the specified row,
4545    * column & timestamp.
4546    */
4547   private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
4548     String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4549     assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
4550         Bytes.toString(CellUtil.cloneRow(kv)));
4551     assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
4552         Bytes.toString(CellUtil.cloneFamily(kv)));
4553     assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
4554         Bytes.toString(CellUtil.cloneQualifier(kv)));
4555     assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
4556     assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
4557         Bytes.toString(CellUtil.cloneValue(kv)));
4558   }
4559 
4560   @Test (timeout=60000)
4561   public void testReverseScanner_FromMemStore_SingleCF_Normal()
4562       throws IOException {
4563     byte[] rowC = Bytes.toBytes("rowC");
4564     byte[] rowA = Bytes.toBytes("rowA");
4565     byte[] rowB = Bytes.toBytes("rowB");
4566     byte[] cf = Bytes.toBytes("CF");
4567     byte[][] families = { cf };
4568     byte[] col = Bytes.toBytes("C");
4569     long ts = 1;
4570     String method = this.getName();
4571     this.region = initHRegion(tableName, method, families);
4572     try {
4573       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4574       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4575           null);
4576       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4577       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4578       Put put = null;
4579       put = new Put(rowC);
4580       put.add(kv1);
4581       put.add(kv11);
4582       region.put(put);
4583       put = new Put(rowA);
4584       put.add(kv2);
4585       region.put(put);
4586       put = new Put(rowB);
4587       put.add(kv3);
4588       region.put(put);
4589 
4590       Scan scan = new Scan(rowC);
4591       scan.setMaxVersions(5);
4592       scan.setReversed(true);
4593       InternalScanner scanner = region.getScanner(scan);
4594       List<Cell> currRow = new ArrayList<Cell>();
4595       boolean hasNext = scanner.next(currRow);
4596       assertEquals(2, currRow.size());
4597       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4598       assertTrue(hasNext);
4599       currRow.clear();
4600       hasNext = scanner.next(currRow);
4601       assertEquals(1, currRow.size());
4602       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4603       assertTrue(hasNext);
4604       currRow.clear();
4605       hasNext = scanner.next(currRow);
4606       assertEquals(1, currRow.size());
4607       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4608       assertFalse(hasNext);
4609       scanner.close();
4610     } finally {
4611       HRegion.closeHRegion(this.region);
4612       this.region = null;
4613     }
4614   }
4615 
4616   @Test (timeout=60000)
4617   public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
4618       throws IOException {
4619     byte[] rowC = Bytes.toBytes("rowC");
4620     byte[] rowA = Bytes.toBytes("rowA");
4621     byte[] rowB = Bytes.toBytes("rowB");
4622     byte[] rowD = Bytes.toBytes("rowD");
4623     byte[] cf = Bytes.toBytes("CF");
4624     byte[][] families = { cf };
4625     byte[] col = Bytes.toBytes("C");
4626     long ts = 1;
4627     String method = this.getName();
4628     this.region = initHRegion(tableName, method, families);
4629     try {
4630       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4631       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4632           null);
4633       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4634       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4635       Put put = null;
4636       put = new Put(rowC);
4637       put.add(kv1);
4638       put.add(kv11);
4639       region.put(put);
4640       put = new Put(rowA);
4641       put.add(kv2);
4642       region.put(put);
4643       put = new Put(rowB);
4644       put.add(kv3);
4645       region.put(put);
4646 
4647       Scan scan = new Scan(rowD);
4648       List<Cell> currRow = new ArrayList<Cell>();
4649       scan.setReversed(true);
4650       scan.setMaxVersions(5);
4651       InternalScanner scanner = region.getScanner(scan);
4652       boolean hasNext = scanner.next(currRow);
4653       assertEquals(2, currRow.size());
4654       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4655       assertTrue(hasNext);
4656       currRow.clear();
4657       hasNext = scanner.next(currRow);
4658       assertEquals(1, currRow.size());
4659       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4660       assertTrue(hasNext);
4661       currRow.clear();
4662       hasNext = scanner.next(currRow);
4663       assertEquals(1, currRow.size());
4664       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4665       assertFalse(hasNext);
4666       scanner.close();
4667     } finally {
4668       HRegion.closeHRegion(this.region);
4669       this.region = null;
4670     }
4671   }
4672 
4673   @Test (timeout=60000)
4674   public void testReverseScanner_FromMemStore_SingleCF_FullScan()
4675       throws IOException {
4676     byte[] rowC = Bytes.toBytes("rowC");
4677     byte[] rowA = Bytes.toBytes("rowA");
4678     byte[] rowB = Bytes.toBytes("rowB");
4679     byte[] cf = Bytes.toBytes("CF");
4680     byte[][] families = { cf };
4681     byte[] col = Bytes.toBytes("C");
4682     long ts = 1;
4683     String method = this.getName();
4684     this.region = initHRegion(tableName, method, families);
4685     try {
4686       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4687       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4688           null);
4689       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4690       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4691       Put put = null;
4692       put = new Put(rowC);
4693       put.add(kv1);
4694       put.add(kv11);
4695       region.put(put);
4696       put = new Put(rowA);
4697       put.add(kv2);
4698       region.put(put);
4699       put = new Put(rowB);
4700       put.add(kv3);
4701       region.put(put);
4702       Scan scan = new Scan();
4703       List<Cell> currRow = new ArrayList<Cell>();
4704       scan.setReversed(true);
4705       InternalScanner scanner = region.getScanner(scan);
4706       boolean hasNext = scanner.next(currRow);
4707       assertEquals(1, currRow.size());
4708       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4709       assertTrue(hasNext);
4710       currRow.clear();
4711       hasNext = scanner.next(currRow);
4712       assertEquals(1, currRow.size());
4713       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4714       assertTrue(hasNext);
4715       currRow.clear();
4716       hasNext = scanner.next(currRow);
4717       assertEquals(1, currRow.size());
4718       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4719       assertFalse(hasNext);
4720       scanner.close();
4721     } finally {
4722       HRegion.closeHRegion(this.region);
4723       this.region = null;
4724     }
4725   }
4726 
4727   @Test (timeout=60000)
4728   public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
4729     // case for "INCLUDE_AND_SEEK_NEXT_ROW & SEEK_NEXT_ROW" endless loop
4730     byte[] rowA = Bytes.toBytes("rowA");
4731     byte[] rowB = Bytes.toBytes("rowB");
4732     byte[] rowC = Bytes.toBytes("rowC");
4733     byte[] rowD = Bytes.toBytes("rowD");
4734     byte[] rowE = Bytes.toBytes("rowE");
4735     byte[] cf = Bytes.toBytes("CF");
4736     byte[][] families = { cf };
4737     byte[] col1 = Bytes.toBytes("col1");
4738     byte[] col2 = Bytes.toBytes("col2");
4739     long ts = 1;
4740     String method = this.getName();
4741     this.region = initHRegion(tableName, method, families);
4742     try {
4743       KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4744       KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4745       KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4746       KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4747       KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4748       KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4749       Put put = null;
4750       put = new Put(rowA);
4751       put.add(kv1);
4752       region.put(put);
4753       put = new Put(rowB);
4754       put.add(kv2);
4755       region.put(put);
4756       put = new Put(rowC);
4757       put.add(kv3);
4758       region.put(put);
4759       put = new Put(rowD);
4760       put.add(kv4_1);
4761       region.put(put);
4762       put = new Put(rowD);
4763       put.add(kv4_2);
4764       region.put(put);
4765       put = new Put(rowE);
4766       put.add(kv5);
4767       region.put(put);
4768       region.flushcache();
4769       Scan scan = new Scan(rowD, rowA);
4770       scan.addColumn(families[0], col1);
4771       scan.setReversed(true);
4772       List<Cell> currRow = new ArrayList<Cell>();
4773       InternalScanner scanner = region.getScanner(scan);
4774       boolean hasNext = scanner.next(currRow);
4775       assertEquals(1, currRow.size());
4776       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4777       assertTrue(hasNext);
4778       currRow.clear();
4779       hasNext = scanner.next(currRow);
4780       assertEquals(1, currRow.size());
4781       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4782       assertTrue(hasNext);
4783       currRow.clear();
4784       hasNext = scanner.next(currRow);
4785       assertEquals(1, currRow.size());
4786       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4787       assertFalse(hasNext);
4788       scanner.close();
4789 
4790       scan = new Scan(rowD, rowA);
4791       scan.addColumn(families[0], col2);
4792       scan.setReversed(true);
4793       currRow.clear();
4794       scanner = region.getScanner(scan);
4795       hasNext = scanner.next(currRow);
4796       assertEquals(1, currRow.size());
4797       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4798       scanner.close();
4799     } finally {
4800       HRegion.closeHRegion(this.region);
4801       this.region = null;
4802     }
4803   }
4804 
4805   @Test (timeout=60000)
4806   public void testReverseScanner_smaller_blocksize() throws IOException {
4807     // case to ensure no conflict with HFile index optimization
4808     byte[] rowA = Bytes.toBytes("rowA");
4809     byte[] rowB = Bytes.toBytes("rowB");
4810     byte[] rowC = Bytes.toBytes("rowC");
4811     byte[] rowD = Bytes.toBytes("rowD");
4812     byte[] rowE = Bytes.toBytes("rowE");
4813     byte[] cf = Bytes.toBytes("CF");
4814     byte[][] families = { cf };
4815     byte[] col1 = Bytes.toBytes("col1");
4816     byte[] col2 = Bytes.toBytes("col2");
4817     long ts = 1;
4818     String method = this.getName();
4819     HBaseConfiguration config = new HBaseConfiguration();
4820     config.setInt("test.block.size", 1);
4821     this.region = initHRegion(tableName, method, config, families);
4822     try {
4823       KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4824       KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4825       KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4826       KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4827       KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4828       KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4829       Put put = null;
4830       put = new Put(rowA);
4831       put.add(kv1);
4832       region.put(put);
4833       put = new Put(rowB);
4834       put.add(kv2);
4835       region.put(put);
4836       put = new Put(rowC);
4837       put.add(kv3);
4838       region.put(put);
4839       put = new Put(rowD);
4840       put.add(kv4_1);
4841       region.put(put);
4842       put = new Put(rowD);
4843       put.add(kv4_2);
4844       region.put(put);
4845       put = new Put(rowE);
4846       put.add(kv5);
4847       region.put(put);
4848       region.flushcache();
4849       Scan scan = new Scan(rowD, rowA);
4850       scan.addColumn(families[0], col1);
4851       scan.setReversed(true);
4852       List<Cell> currRow = new ArrayList<Cell>();
4853       InternalScanner scanner = region.getScanner(scan);
4854       boolean hasNext = scanner.next(currRow);
4855       assertEquals(1, currRow.size());
4856       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4857       assertTrue(hasNext);
4858       currRow.clear();
4859       hasNext = scanner.next(currRow);
4860       assertEquals(1, currRow.size());
4861       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4862       assertTrue(hasNext);
4863       currRow.clear();
4864       hasNext = scanner.next(currRow);
4865       assertEquals(1, currRow.size());
4866       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4867       assertFalse(hasNext);
4868       scanner.close();
4869 
4870       scan = new Scan(rowD, rowA);
4871       scan.addColumn(families[0], col2);
4872       scan.setReversed(true);
4873       currRow.clear();
4874       scanner = region.getScanner(scan);
4875       hasNext = scanner.next(currRow);
4876       assertEquals(1, currRow.size());
4877       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4878       scanner.close();
4879     } finally {
4880       HRegion.closeHRegion(this.region);
4881       this.region = null;
4882     }
4883   }
4884 
4885   @Test (timeout=60000)
4886   public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
4887       throws IOException {
4888     byte[] row0 = Bytes.toBytes("row0"); // 1 kv
4889     byte[] row1 = Bytes.toBytes("row1"); // 2 kv
4890     byte[] row2 = Bytes.toBytes("row2"); // 4 kv
4891     byte[] row3 = Bytes.toBytes("row3"); // 2 kv
4892     byte[] row4 = Bytes.toBytes("row4"); // 5 kv
4893     byte[] row5 = Bytes.toBytes("row5"); // 2 kv
4894     byte[] cf1 = Bytes.toBytes("CF1");
4895     byte[] cf2 = Bytes.toBytes("CF2");
4896     byte[] cf3 = Bytes.toBytes("CF3");
4897     byte[][] families = { cf1, cf2, cf3 };
4898     byte[] col = Bytes.toBytes("C");
4899     long ts = 1;
4900     String method = this.getName();
4901     HBaseConfiguration conf = new HBaseConfiguration();
4902     // disable compactions in this test.
4903     conf.setInt("hbase.hstore.compactionThreshold", 10000);
4904     this.region = initHRegion(tableName, method, conf, families);
4905     try {
4906       // kv naming style: kv(row number) totalKvCountInThisRow seq no
4907       KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
4908           null);
4909       KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
4910           null);
4911       KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
4912           KeyValue.Type.Put, null);
4913       KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
4914           null);
4915       KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
4916           null);
4917       KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
4918           null);
4919       KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
4920           KeyValue.Type.Put, null);
4921       KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
4922           null);
4923       KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
4924           KeyValue.Type.Put, null);
4925       KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
4926           null);
4927       KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
4928           null);
4929       KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
4930           KeyValue.Type.Put, null);
4931       KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
4932           null);
4933       KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
4934           KeyValue.Type.Put, null);
4935       KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
4936           null);
4937       KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
4938           null);
4939       // hfiles(cf1/cf2) :"row1"(1 kv) / "row2"(1 kv) / "row4"(2 kv)
4940       Put put = null;
4941       put = new Put(row1);
4942       put.add(kv1_2_1);
4943       region.put(put);
4944       put = new Put(row2);
4945       put.add(kv2_4_1);
4946       region.put(put);
4947       put = new Put(row4);
4948       put.add(kv4_5_4);
4949       put.add(kv4_5_5);
4950       region.put(put);
4951       region.flushcache();
4952       // hfiles(cf1/cf3) : "row1" (1 kvs) / "row2" (1 kv) / "row4" (2 kv)
4953       put = new Put(row4);
4954       put.add(kv4_5_1);
4955       put.add(kv4_5_3);
4956       region.put(put);
4957       put = new Put(row1);
4958       put.add(kv1_2_2);
4959       region.put(put);
4960       put = new Put(row2);
4961       put.add(kv2_4_4);
4962       region.put(put);
4963       region.flushcache();
4964       // hfiles(cf1/cf3) : "row2"(2 kv) / "row3"(1 kvs) / "row4" (1 kv)
4965       put = new Put(row4);
4966       put.add(kv4_5_2);
4967       region.put(put);
4968       put = new Put(row2);
4969       put.add(kv2_4_2);
4970       put.add(kv2_4_3);
4971       region.put(put);
4972       put = new Put(row3);
4973       put.add(kv3_2_2);
4974       region.put(put);
4975       region.flushcache();
4976       // memstore(cf1/cf2/cf3) : "row0" (1 kvs) / "row3" ( 1 kv) / "row5" (max)
4977       // ( 2 kv)
4978       put = new Put(row0);
4979       put.add(kv0_1_1);
4980       region.put(put);
4981       put = new Put(row3);
4982       put.add(kv3_2_1);
4983       region.put(put);
4984       put = new Put(row5);
4985       put.add(kv5_2_1);
4986       put.add(kv5_2_2);
4987       region.put(put);
4988       // scan range = ["row4", min), skip the max "row5"
4989       Scan scan = new Scan(row4);
4990       scan.setMaxVersions(5);
4991       scan.setBatch(3);
4992       scan.setReversed(true);
4993       InternalScanner scanner = region.getScanner(scan);
4994       List<Cell> currRow = new ArrayList<Cell>();
4995       boolean hasNext = false;
4996       // 1. scan out "row4" (5 kvs), "row5" can't be scanned out since not
4997       // included in scan range
4998       // "row4" takes 2 next() calls since batch=3
4999       hasNext = scanner.next(currRow);
5000       assertEquals(3, currRow.size());
5001       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5002       assertTrue(hasNext);
5003       currRow.clear();
5004       hasNext = scanner.next(currRow);
5005       assertEquals(2, currRow.size());
5006       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5007       assertTrue(hasNext);
5008       // 2. scan out "row3" (2 kv)
5009       currRow.clear();
5010       hasNext = scanner.next(currRow);
5011       assertEquals(2, currRow.size());
5012       assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5013       assertTrue(hasNext);
5014       // 3. scan out "row2" (4 kvs)
5015       // "row2" takes 2 next() calls since batch=3
5016       currRow.clear();
5017       hasNext = scanner.next(currRow);
5018       assertEquals(3, currRow.size());
5019       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5020       assertTrue(hasNext);
5021       currRow.clear();
5022       hasNext = scanner.next(currRow);
5023       assertEquals(1, currRow.size());
5024       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5025       assertTrue(hasNext);
5026       // 4. scan out "row1" (2 kv)
5027       currRow.clear();
5028       hasNext = scanner.next(currRow);
5029       assertEquals(2, currRow.size());
5030       assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5031       assertTrue(hasNext);
5032       // 5. scan out "row0" (1 kv)
5033       currRow.clear();
5034       hasNext = scanner.next(currRow);
5035       assertEquals(1, currRow.size());
5036       assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
5037       assertFalse(hasNext);
5038 
5039       scanner.close();
5040     } finally {
5041       HRegion.closeHRegion(this.region);
5042       this.region = null;
5043     }
5044   }
5045 
5046   @Test (timeout=60000)
5047   public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
5048       throws IOException {
5049     byte[] row1 = Bytes.toBytes("row1");
5050     byte[] row2 = Bytes.toBytes("row2");
5051     byte[] row3 = Bytes.toBytes("row3");
5052     byte[] row4 = Bytes.toBytes("row4");
5053     byte[] cf1 = Bytes.toBytes("CF1");
5054     byte[] cf2 = Bytes.toBytes("CF2");
5055     byte[] cf3 = Bytes.toBytes("CF3");
5056     byte[] cf4 = Bytes.toBytes("CF4");
5057     byte[][] families = { cf1, cf2, cf3, cf4 };
5058     byte[] col = Bytes.toBytes("C");
5059     long ts = 1;
5060     String method = this.getName();
5061     HBaseConfiguration conf = new HBaseConfiguration();
5062     // disable compactions in this test.
5063     conf.setInt("hbase.hstore.compactionThreshold", 10000);
5064     this.region = initHRegion(tableName, method, conf, families);
5065     try {
5066       KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
5067       KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
5068       KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
5069       KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
5070       // storefile1
5071       Put put = new Put(row1);
5072       put.add(kv1);
5073       region.put(put);
5074       region.flushcache();
5075       // storefile2
5076       put = new Put(row2);
5077       put.add(kv2);
5078       region.put(put);
5079       region.flushcache();
5080       // storefile3
5081       put = new Put(row3);
5082       put.add(kv3);
5083       region.put(put);
5084       region.flushcache();
5085       // memstore
5086       put = new Put(row4);
5087       put.add(kv4);
5088       region.put(put);
5089       // scan range = ["row4", min)
5090       Scan scan = new Scan(row4);
5091       scan.setReversed(true);
5092       scan.setBatch(10);
5093       InternalScanner scanner = region.getScanner(scan);
5094       List<Cell> currRow = new ArrayList<Cell>();
5095       boolean hasNext = scanner.next(currRow);
5096       assertEquals(1, currRow.size());
5097       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5098       assertTrue(hasNext);
5099       currRow.clear();
5100       hasNext = scanner.next(currRow);
5101       assertEquals(1, currRow.size());
5102       assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5103       assertTrue(hasNext);
5104       currRow.clear();
5105       hasNext = scanner.next(currRow);
5106       assertEquals(1, currRow.size());
5107       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5108       assertTrue(hasNext);
5109       currRow.clear();
5110       hasNext = scanner.next(currRow);
5111       assertEquals(1, currRow.size());
5112       assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5113       assertFalse(hasNext);
5114     } finally {
5115       HRegion.closeHRegion(this.region);
5116       this.region = null;
5117     }
5118   }
5119 
5120   /**
5121    * Test for HBASE-14497: Reverse Scan threw StackOverflow caused by readPt checking
5122    */
5123   @Test (timeout = 60000)
5124   public void testReverseScanner_StackOverflow() throws IOException {
5125     byte[] cf1 = Bytes.toBytes("CF1");
5126     byte[][] families = {cf1};
5127     byte[] col = Bytes.toBytes("C");
5128     String method = this.getName();
5129     HBaseConfiguration conf = new HBaseConfiguration();
5130     // disable compactions in this test.
5131     conf.setInt("hbase.hstore.compactionThreshold", 10000);
5132     this.region = initHRegion(tableName, method, conf, families);
5133 
5134     try {
5135       // setup with one storefile and one memstore, to create scanner and get an earlier readPt
5136       Put put = new Put(Bytes.toBytes("19998"));
5137       put.add(cf1, col, Bytes.toBytes("val"));
5138       region.put(put);
5139       region.flushcache();
5140       Put put2 = new Put(Bytes.toBytes("19997"));
5141       put2.add(cf1, col, Bytes.toBytes("val"));
5142       region.put(put2);
5143 
5144       Scan scan = new Scan(Bytes.toBytes("19998"));
5145       scan.setReversed(true);
5146       InternalScanner scanner = region.getScanner(scan);
5147 
5148       // create one storefile contains many rows will be skipped
5149       // to check StoreFileScanner.seekToPreviousRow
5150       for (int i = 10000; i < 20000; i++) {
5151         Put p = new Put(Bytes.toBytes("" + i));
5152         p.add(cf1, col, Bytes.toBytes("" + i));
5153         region.put(p);
5154       }
5155       region.flushcache();
5156 
5157       // create one memstore contains many rows will be skipped
5158       // to check MemStoreScanner.seekToPreviousRow
5159       for (int i = 10000; i < 20000; i++) {
5160         Put p = new Put(Bytes.toBytes("" + i));
5161         p.add(cf1, col, Bytes.toBytes("" + i));
5162         region.put(p);
5163       }
5164 
5165       List<Cell> currRow = new ArrayList<Cell>();
5166       boolean hasNext;
5167       do {
5168         hasNext = scanner.next(currRow);
5169       } while (hasNext);
5170       assertEquals(2, currRow.size());
5171       assertArrayEquals(Bytes.toBytes("19998"), currRow.get(0).getRow());
5172       assertArrayEquals(Bytes.toBytes("19997"), currRow.get(1).getRow());
5173     } finally {
5174       HRegion.closeHRegion(this.region);
5175       this.region = null;
5176     }
5177   }
5178 
5179   @Test (timeout=60000)
5180   public void testSplitRegionWithReverseScan() throws IOException {
5181     byte [] tableName = Bytes.toBytes("testSplitRegionWithReverseScan");
5182     byte [] qualifier = Bytes.toBytes("qualifier");
5183     Configuration hc = initSplit();
5184     int numRows = 3;
5185     byte [][] families = {fam1};
5186 
5187     //Setting up region
5188     String method = this.getName();
5189     this.region = initHRegion(tableName, method, hc, families);
5190 
5191     //Put data in region
5192     int startRow = 100;
5193     putData(startRow, numRows, qualifier, families);
5194     int splitRow = startRow + numRows;
5195     putData(splitRow, numRows, qualifier, families);
5196     int endRow = splitRow + numRows;
5197     region.flushcache();
5198 
5199     HRegion [] regions = null;
5200     try {
5201       regions = splitRegion(region, Bytes.toBytes("" + splitRow));
5202       //Opening the regions returned.
5203       for (int i = 0; i < regions.length; i++) {
5204         regions[i] = HRegion.openHRegion(regions[i], null);
5205       }
5206       //Verifying that the region has been split
5207       assertEquals(2, regions.length);
5208 
5209       //Verifying that all data is still there and that data is in the right
5210       //place
5211       verifyData(regions[0], startRow, numRows, qualifier, families);
5212       verifyData(regions[1], splitRow, numRows, qualifier, families);
5213 
5214       //fire the reverse scan1:  top range, and larger than the last row
5215       Scan scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 10 * numRows)));
5216       scan.setReversed(true);
5217       InternalScanner scanner = regions[1].getScanner(scan);
5218       List<Cell> currRow = new ArrayList<Cell>();
5219       boolean more = false;
5220       int verify = startRow + 2 * numRows - 1;
5221       do {
5222         more = scanner.next(currRow);
5223         assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5224         verify--;
5225         currRow.clear();
5226       } while(more);
5227       assertEquals(verify, startRow + numRows - 1);
5228       scanner.close();
5229       //fire the reverse scan2:  top range, and equals to the last row
5230       scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 2 * numRows - 1)));
5231       scan.setReversed(true);
5232       scanner = regions[1].getScanner(scan);
5233       verify = startRow + 2 * numRows - 1;
5234       do {
5235         more = scanner.next(currRow);
5236         assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5237         verify--;
5238         currRow.clear();
5239       } while(more);
5240       assertEquals(verify, startRow + numRows - 1);
5241       scanner.close();
5242       //fire the reverse scan3:  bottom range, and larger than the last row
5243       scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows)));
5244       scan.setReversed(true);
5245       scanner = regions[0].getScanner(scan);
5246       verify = startRow + numRows - 1;
5247       do {
5248         more = scanner.next(currRow);
5249         assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5250         verify--;
5251         currRow.clear();
5252       } while(more);
5253       assertEquals(verify, 99);
5254       scanner.close();
5255       //fire the reverse scan4:  bottom range, and equals to the last row
5256       scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows - 1)));
5257       scan.setReversed(true);
5258       scanner = regions[0].getScanner(scan);
5259       verify = startRow + numRows - 1;
5260       do {
5261         more = scanner.next(currRow);
5262         assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5263         verify--;
5264         currRow.clear();
5265       } while(more);
5266       assertEquals(verify, startRow - 1);
5267       scanner.close();
5268     } finally {
5269       HRegion.closeHRegion(this.region);
5270       this.region = null;
5271     }
5272   }
5273 
5274   @Test
5275   public void testWriteRequestsCounter() throws IOException {
5276     byte[] fam = Bytes.toBytes("info");
5277     byte[][] families = { fam };
5278     this.region = initHRegion(tableName, method, CONF, families);
5279 
5280     Assert.assertEquals(0L, region.getWriteRequestsCount());
5281 
5282     Put put = new Put(row);
5283     put.add(fam, fam, fam);
5284 
5285     Assert.assertEquals(0L, region.getWriteRequestsCount());
5286     region.put(put);
5287     Assert.assertEquals(1L, region.getWriteRequestsCount());
5288     region.put(put);
5289     Assert.assertEquals(2L, region.getWriteRequestsCount());
5290     region.put(put);
5291     Assert.assertEquals(3L, region.getWriteRequestsCount());
5292 
5293     region.delete(new Delete(row));
5294     Assert.assertEquals(4L, region.getWriteRequestsCount());
5295 
5296     HRegion.closeHRegion(this.region);
5297     this.region = null;
5298   }
5299 
5300   /**
5301    * Test RegionTooBusyException thrown when region is busy
5302    */
5303   @Test (timeout=24000)
5304   public void testRegionTooBusy() throws IOException {
5305     String method = "testRegionTooBusy";
5306     byte[] tableName = Bytes.toBytes(method);
5307     byte[] family = Bytes.toBytes("family");
5308     long defaultBusyWaitDuration = CONF.getLong("hbase.busy.wait.duration",
5309       HRegion.DEFAULT_BUSY_WAIT_DURATION);
5310     CONF.setLong("hbase.busy.wait.duration", 1000);
5311     region = initHRegion(tableName, method, CONF, family);
5312     final AtomicBoolean stopped = new AtomicBoolean(true);
5313     Thread t = new Thread(new Runnable() {
5314       @Override
5315       public void run() {
5316         try {
5317           region.lock.writeLock().lock();
5318           stopped.set(false);
5319           while (!stopped.get()) {
5320             Thread.sleep(100);
5321           }
5322         } catch (InterruptedException ie) {
5323         } finally {
5324           region.lock.writeLock().unlock();
5325         }
5326       }
5327     });
5328     t.start();
5329     Get get = new Get(row);
5330     try {
5331       while (stopped.get()) {
5332         Thread.sleep(100);
5333       }
5334       region.get(get);
5335       fail("Should throw RegionTooBusyException");
5336     } catch (InterruptedException ie) {
5337       fail("test interrupted");
5338     } catch (RegionTooBusyException e) {
5339       // Good, expected
5340     } finally {
5341       stopped.set(true);
5342       try {
5343         t.join();
5344       } catch (Throwable e) {
5345       }
5346 
5347       HRegion.closeHRegion(region);
5348       region = null;
5349       CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration);
5350     }
5351   }
5352 
5353   @Test
5354   public void testCellTTLs() throws IOException {
5355     IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
5356     EnvironmentEdgeManager.injectEdge(edge);
5357 
5358     final byte[] row = Bytes.toBytes("testRow");
5359     final byte[] q1 = Bytes.toBytes("q1");
5360     final byte[] q2 = Bytes.toBytes("q2");
5361     final byte[] q3 = Bytes.toBytes("q3");
5362     final byte[] q4 = Bytes.toBytes("q4");
5363 
5364     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCellTTLs"));
5365     HColumnDescriptor hcd = new HColumnDescriptor(fam1);
5366     hcd.setTimeToLive(10); // 10 seconds
5367     htd.addFamily(hcd);
5368 
5369     Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
5370     conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS);
5371 
5372     HRegion region = HRegion.createHRegion(new HRegionInfo(htd.getTableName(),
5373         HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY),
5374       TEST_UTIL.getDataTestDir(), conf, htd);
5375     assertNotNull(region);
5376     try {
5377       long now = EnvironmentEdgeManager.currentTimeMillis();
5378       // Add a cell that will expire in 5 seconds via cell TTL
5379       region.put(new Put(row).add(new KeyValue(row, fam1, q1, now,
5380         HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
5381           // TTL tags specify ts in milliseconds
5382           new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
5383       // Add a cell that will expire after 10 seconds via family setting
5384       region.put(new Put(row).add(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY));
5385       // Add a cell that will expire in 15 seconds via cell TTL
5386       region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1,
5387         HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
5388           // TTL tags specify ts in milliseconds
5389           new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
5390       // Add a cell that will expire in 20 seconds via family setting
5391       region.put(new Put(row).add(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY));
5392 
5393       // Flush so we are sure store scanning gets this right
5394       region.flushcache();
5395 
5396       // A query at time T+0 should return all cells
5397       Result r = region.get(new Get(row));
5398       assertNotNull(r.getValue(fam1, q1));
5399       assertNotNull(r.getValue(fam1, q2));
5400       assertNotNull(r.getValue(fam1, q3));
5401       assertNotNull(r.getValue(fam1, q4));
5402 
5403       // Increment time to T+5 seconds
5404       edge.incrementTime(5000);
5405 
5406       r = region.get(new Get(row));
5407       assertNull(r.getValue(fam1, q1));
5408       assertNotNull(r.getValue(fam1, q2));
5409       assertNotNull(r.getValue(fam1, q3));
5410       assertNotNull(r.getValue(fam1, q4));
5411 
5412       // Increment time to T+10 seconds
5413       edge.incrementTime(5000);
5414 
5415       r = region.get(new Get(row));
5416       assertNull(r.getValue(fam1, q1));
5417       assertNull(r.getValue(fam1, q2));
5418       assertNotNull(r.getValue(fam1, q3));
5419       assertNotNull(r.getValue(fam1, q4));
5420 
5421       // Increment time to T+15 seconds
5422       edge.incrementTime(5000);
5423 
5424       r = region.get(new Get(row));
5425       assertNull(r.getValue(fam1, q1));
5426       assertNull(r.getValue(fam1, q2));
5427       assertNull(r.getValue(fam1, q3));
5428       assertNotNull(r.getValue(fam1, q4));
5429 
5430       // Increment time to T+20 seconds
5431       edge.incrementTime(10000);
5432 
5433       r = region.get(new Get(row));
5434       assertNull(r.getValue(fam1, q1));
5435       assertNull(r.getValue(fam1, q2));
5436       assertNull(r.getValue(fam1, q3));
5437       assertNull(r.getValue(fam1, q4));
5438 
5439       // Fun with disappearing increments
5440 
5441       // Start at 1
5442       region.put(new Put(row).add(fam1, q1, Bytes.toBytes(1L)));
5443       r = region.get(new Get(row));
5444       byte[] val = r.getValue(fam1, q1);
5445       assertNotNull(val);
5446       assertEquals(Bytes.toLong(val), 1L);
5447 
5448       // Increment with a TTL of 5 seconds
5449       Increment incr = new Increment(row).addColumn(fam1, q1, 1L);
5450       incr.setTTL(5000);
5451       region.increment(incr); // 2
5452 
5453       // New value should be 2
5454       r = region.get(new Get(row));
5455       val = r.getValue(fam1, q1);
5456       assertNotNull(val);
5457       assertEquals(Bytes.toLong(val), 2L);
5458 
5459       // Increment time to T+25 seconds
5460       edge.incrementTime(5000);
5461 
5462       // Value should be back to 1
5463       r = region.get(new Get(row));
5464       val = r.getValue(fam1, q1);
5465       assertNotNull(val);
5466       assertEquals(Bytes.toLong(val), 1L);
5467 
5468       // Increment time to T+30 seconds
5469       edge.incrementTime(5000);
5470 
5471       // Original value written at T+20 should be gone now via family TTL
5472       r = region.get(new Get(row));
5473       assertNull(r.getValue(fam1, q1));
5474 
5475     } finally {
5476       HRegion.closeHRegion(region);
5477     }
5478   }
5479 
5480   @Test
5481   public void testIncrementTimestampsAreMonotonic() throws IOException {
5482     HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5483     ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5484     EnvironmentEdgeManager.injectEdge(edge);
5485 
5486     edge.setValue(10);
5487     Increment inc = new Increment(row);
5488     inc.setDurability(Durability.SKIP_WAL);
5489     inc.addColumn(fam1, qual1, 1L);
5490     region.increment(inc);
5491 
5492     Result result = region.get(new Get(row));
5493     Cell c = result.getColumnLatestCell(fam1, qual1);
5494     assertNotNull(c);
5495     assertEquals(c.getTimestamp(), 10L);
5496 
5497     edge.setValue(1); // clock goes back
5498     region.increment(inc);
5499     result = region.get(new Get(row));
5500     c = result.getColumnLatestCell(fam1, qual1);
5501     assertEquals(c.getTimestamp(), 10L);
5502     assertEquals(Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength()), 2L);
5503   }
5504 
5505   @Test
5506   public void testAppendTimestampsAreMonotonic() throws IOException {
5507     HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5508     ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5509     EnvironmentEdgeManager.injectEdge(edge);
5510 
5511     edge.setValue(10);
5512     Append a = new Append(row);
5513     a.setDurability(Durability.SKIP_WAL);
5514     a.add(fam1, qual1, qual1);
5515     region.append(a);
5516 
5517     Result result = region.get(new Get(row));
5518     Cell c = result.getColumnLatestCell(fam1, qual1);
5519     assertNotNull(c);
5520     assertEquals(c.getTimestamp(), 10L);
5521 
5522     edge.setValue(1); // clock goes back
5523     region.append(a);
5524     result = region.get(new Get(row));
5525     c = result.getColumnLatestCell(fam1, qual1);
5526     assertEquals(c.getTimestamp(), 10L);
5527 
5528     byte[] expected = new byte[qual1.length*2];
5529     System.arraycopy(qual1, 0, expected, 0, qual1.length);
5530     System.arraycopy(qual1, 0, expected, qual1.length, qual1.length);
5531 
5532     assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
5533       expected, 0, expected.length));
5534   }
5535 
5536   @Test
5537   public void testCheckAndMutateTimestampsAreMonotonic() throws IOException {
5538     HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5539     ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5540     EnvironmentEdgeManager.injectEdge(edge);
5541 
5542     edge.setValue(10);
5543     Put p = new Put(row);
5544     p.setDurability(Durability.SKIP_WAL);
5545     p.add(fam1, qual1, qual1);
5546     region.put(p);
5547 
5548     Result result = region.get(new Get(row));
5549     Cell c = result.getColumnLatestCell(fam1, qual1);
5550     assertNotNull(c);
5551     assertEquals(c.getTimestamp(), 10L);
5552 
5553     edge.setValue(1); // clock goes back
5554     p = new Put(row);
5555     p.setDurability(Durability.SKIP_WAL);
5556     p.add(fam1, qual1, qual2);
5557     region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(qual1), p, false);
5558     result = region.get(new Get(row));
5559     c = result.getColumnLatestCell(fam1, qual1);
5560     assertEquals(c.getTimestamp(), 10L);
5561 
5562     assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
5563       qual2, 0, qual2.length));
5564   }
5565 
5566   @Test
5567   public void testCheckAndRowMutateTimestampsAreMonotonic() throws IOException {
5568     HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5569     ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5570     EnvironmentEdgeManager.injectEdge(edge);
5571 
5572     edge.setValue(10);
5573     Put p = new Put(row);
5574     p.setDurability(Durability.SKIP_WAL);
5575     p.add(fam1, qual1, qual1);
5576     region.put(p);
5577 
5578     Result result = region.get(new Get(row));
5579     Cell c = result.getColumnLatestCell(fam1, qual1);
5580     assertNotNull(c);
5581     assertEquals(c.getTimestamp(), 10L);
5582 
5583     edge.setValue(1); // clock goes back
5584     p = new Put(row);
5585     p.setDurability(Durability.SKIP_WAL);
5586     p.add(fam1, qual1, qual2);
5587     RowMutations rm = new RowMutations(row);
5588     rm.add(p);
5589     region.checkAndRowMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(qual1),
5590       rm, false);
5591     result = region.get(new Get(row));
5592     c = result.getColumnLatestCell(fam1, qual1);
5593     assertEquals(c.getTimestamp(), 10L);
5594 
5595     assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
5596       qual2, 0, qual2.length));
5597   }
5598 
5599   private static HRegion initHRegion(byte[] tableName, String callingMethod,
5600       byte[]... families) throws IOException {
5601     return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
5602         families);
5603   }
5604 }