View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  
22  import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23  import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24  import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25  import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29  import static org.junit.Assert.assertArrayEquals;
30  import static org.junit.Assert.assertEquals;
31  import static org.junit.Assert.assertFalse;
32  import static org.junit.Assert.assertNotNull;
33  import static org.junit.Assert.assertNull;
34  import static org.junit.Assert.assertTrue;
35  import static org.junit.Assert.fail;
36  import static org.mockito.Matchers.any;
37  import static org.mockito.Matchers.anyBoolean;
38  import static org.mockito.Matchers.anyLong;
39  import static org.mockito.Matchers.eq;
40  import static org.mockito.Mockito.never;
41  import static org.mockito.Mockito.spy;
42  import static org.mockito.Mockito.times;
43  import static org.mockito.Mockito.verify;
44  
45  import java.io.IOException;
46  import java.io.InterruptedIOException;
47  import java.security.PrivilegedExceptionAction;
48  import java.util.ArrayList;
49  import java.util.Arrays;
50  import java.util.Collection;
51  import java.util.List;
52  import java.util.Map;
53  import java.util.NavigableMap;
54  import java.util.TreeMap;
55  import java.util.UUID;
56  import java.util.concurrent.atomic.AtomicBoolean;
57  import java.util.concurrent.atomic.AtomicInteger;
58  import java.util.concurrent.atomic.AtomicLong;
59  import java.util.concurrent.atomic.AtomicReference;
60  
61  import org.apache.commons.logging.Log;
62  import org.apache.commons.logging.LogFactory;
63  import org.apache.hadoop.conf.Configuration;
64  import org.apache.hadoop.fs.FSDataOutputStream;
65  import org.apache.hadoop.fs.FileStatus;
66  import org.apache.hadoop.fs.FileSystem;
67  import org.apache.hadoop.fs.Path;
68  import org.apache.hadoop.hbase.Cell;
69  import org.apache.hadoop.hbase.CellComparator;
70  import org.apache.hadoop.hbase.CellUtil;
71  import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
72  import org.apache.hadoop.hbase.DroppedSnapshotException;
73  import org.apache.hadoop.hbase.HBaseConfiguration;
74  import org.apache.hadoop.hbase.HBaseTestCase;
75  import org.apache.hadoop.hbase.HBaseTestingUtility;
76  import org.apache.hadoop.hbase.HColumnDescriptor;
77  import org.apache.hadoop.hbase.HConstants;
78  import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
79  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
80  import org.apache.hadoop.hbase.HRegionInfo;
81  import org.apache.hadoop.hbase.HTableDescriptor;
82  import org.apache.hadoop.hbase.KeyValue;
83  import org.apache.hadoop.hbase.MediumTests;
84  import org.apache.hadoop.hbase.MiniHBaseCluster;
85  import org.apache.hadoop.hbase.MultithreadedTestUtil;
86  import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
87  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
88  import org.apache.hadoop.hbase.NotServingRegionException;
89  import org.apache.hadoop.hbase.TableName;
90  import org.apache.hadoop.hbase.Waiter;
91  import org.apache.hadoop.hbase.client.Append;
92  import org.apache.hadoop.hbase.client.Delete;
93  import org.apache.hadoop.hbase.client.Durability;
94  import org.apache.hadoop.hbase.client.Get;
95  import org.apache.hadoop.hbase.client.HTable;
96  import org.apache.hadoop.hbase.client.Increment;
97  import org.apache.hadoop.hbase.client.Put;
98  import org.apache.hadoop.hbase.client.Result;
99  import org.apache.hadoop.hbase.client.Scan;
100 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
101 import org.apache.hadoop.hbase.filter.BinaryComparator;
102 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
103 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
104 import org.apache.hadoop.hbase.filter.Filter;
105 import org.apache.hadoop.hbase.filter.FilterBase;
106 import org.apache.hadoop.hbase.filter.FilterList;
107 import org.apache.hadoop.hbase.filter.NullComparator;
108 import org.apache.hadoop.hbase.filter.PrefixFilter;
109 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
110 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
111 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
112 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
113 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
114 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
115 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
116 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
117 import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
118 import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
119 import org.apache.hadoop.hbase.regionserver.wal.FaultyHLog;
120 import org.apache.hadoop.hbase.regionserver.wal.HLog;
121 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
122 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
123 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
124 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
125 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
126 import org.apache.hadoop.hbase.security.User;
127 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
128 import org.apache.hadoop.hbase.util.Bytes;
129 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
130 import org.apache.hadoop.hbase.util.FSUtils;
131 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
132 import org.apache.hadoop.hbase.util.PairOfSameType;
133 import org.apache.hadoop.hbase.util.Threads;
134 import org.junit.After;
135 import org.junit.Assert;
136 import org.junit.Before;
137 import org.junit.Rule;
138 import org.junit.Test;
139 import org.junit.experimental.categories.Category;
140 import org.junit.rules.TestName;
141 import org.mockito.Mockito;
142 
143 import com.google.common.collect.ImmutableList;
144 import com.google.common.collect.Lists;
145 import com.google.protobuf.ByteString;
146 
147 /**
148  * Basic stand-alone testing of HRegion.  No clusters!
149  *
150  * A lot of the meta information for an HRegion now lives inside other HRegions
151  * or in the HBaseMaster, so only basic testing is possible.
152  */
153 @Category(MediumTests.class)
154 @SuppressWarnings("deprecation")
155 public class TestHRegion {
156   // Do not spin up clusters in here. If you need to spin up a cluster, do it
157   // over in TestHRegionOnCluster.
158   static final Log LOG = LogFactory.getLog(TestHRegion.class);
159   @Rule public TestName name = new TestName();
160 
161   private static final String COLUMN_FAMILY = "MyCF";
162   private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
163 
164   HRegion region = null;
165   // Do not run unit tests in parallel (? Why not?  It don't work?  Why not?  St.Ack)
166   private static HBaseTestingUtility TEST_UTIL;
167   public static Configuration CONF ;
168   private String dir;
169   private static FileSystem FILESYSTEM;
170   private final int MAX_VERSIONS = 2;
171 
172   // Test names
173   protected byte[] tableName;
174   protected String method;
175   protected final byte[] qual1 = Bytes.toBytes("qual1");
176   protected final byte[] qual2 = Bytes.toBytes("qual2");
177   protected final byte[] qual3 = Bytes.toBytes("qual3");
178   protected final byte[] value1 = Bytes.toBytes("value1");
179   protected final byte[] value2 = Bytes.toBytes("value2");
180   protected final byte[] row = Bytes.toBytes("rowA");
181   protected final byte[] row2 = Bytes.toBytes("rowB");
182 
183   protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
184       .getInstance(MetricsAssertHelper.class);
185 
186   @Before
187   public void setup() throws IOException {
188     TEST_UTIL = HBaseTestingUtility.createLocalHTU();
189     FILESYSTEM = TEST_UTIL.getTestFileSystem();
190     CONF = TEST_UTIL.getConfiguration();
191     dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
192     method = name.getMethodName();
193     tableName = Bytes.toBytes(name.getMethodName());
194   }
195 
196   @After
197   public void tearDown() throws Exception {
198     EnvironmentEdgeManagerTestHelper.reset();
199     LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
200     TEST_UTIL.cleanupTestDir();
201   }
202 
203   String getName() {
204     return name.getMethodName();
205   }
206 
207   /**
208    * Test for Bug 2 of HBASE-10466.
209    * "Bug 2: Conditions for the first flush of region close (so-called pre-flush) If memstoreSize
210    * is smaller than a certain value, or when region close starts a flush is ongoing, the first
211    * flush is skipped and only the second flush takes place. However, two flushes are required in
212    * case previous flush fails and leaves some data in snapshot. The bug could cause loss of data
213    * in current memstore. The fix is removing all conditions except abort check so we ensure 2
214    * flushes for region close."
215    * @throws IOException
216    */
217   @Test (timeout=60000)
218   public void testCloseCarryingSnapshot() throws IOException {
219     HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
220     Store store = region.getStore(COLUMN_FAMILY_BYTES);
221     // Get some random bytes.
222     byte [] value = Bytes.toBytes(name.getMethodName());
223     // Make a random put against our cf.
224     Put put = new Put(value);
225     put.add(COLUMN_FAMILY_BYTES, null, value);
226     // First put something in current memstore, which will be in snapshot after flusher.prepare()
227     region.put(put);
228     StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
229     storeFlushCtx.prepare();
230     // Second put something in current memstore
231     put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
232     region.put(put);
233     // Close with something in memstore and something in the snapshot.  Make sure all is cleared.
234     region.close();
235     assertEquals(0, region.getMemstoreSize().get());
236     HRegion.closeHRegion(region);
237   }
238 
239   /*
240    * This test is for verifying memstore snapshot size is correctly updated in case of rollback
241    * See HBASE-10845
242    */
243   @Test (timeout=60000)
244   public void testMemstoreSnapshotSize() throws IOException {
245     class MyFaultyHLog extends FaultyHLog {
246       StoreFlushContext storeFlushCtx;
247       public MyFaultyHLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
248           throws IOException {
249         super(fs, rootDir, logName, conf);
250       }
251       
252       void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
253         this.storeFlushCtx = storeFlushCtx;
254       }
255 
256       @Override
257       public void sync(long txid) throws IOException {
258         storeFlushCtx.prepare();
259         super.sync(txid);
260       }
261     }
262     
263     FileSystem fs = FileSystem.get(CONF);
264     Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
265     MyFaultyHLog faultyLog = new MyFaultyHLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
266     HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
267       CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES);
268     
269     Store store = region.getStore(COLUMN_FAMILY_BYTES);
270     // Get some random bytes.
271     byte [] value = Bytes.toBytes(name.getMethodName());
272     faultyLog.setStoreFlushCtx(store.createFlushContext(12345));
273     
274     Put put = new Put(value);
275     put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
276     faultyLog.setFailureType(FaultyHLog.FailureType.SYNC);
277 
278     boolean threwIOE = false;
279     try {
280       region.put(put);
281     } catch (IOException ioe) {
282       threwIOE = true;
283     } finally {
284       assertTrue("The regionserver should have thrown an exception", threwIOE);
285     }
286     long sz = store.getFlushableSize();
287     assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
288     HRegion.closeHRegion(region);
289   }
290   
291   /**
292    * Test we do not lose data if we fail a flush and then close.
293    * Part of HBase-10466.  Tests the following from the issue description:
294    * "Bug 1: Wrong calculation of HRegion.memstoreSize: When a flush fails, data to be flushed is
295    * kept in each MemStore's snapshot and wait for next flush attempt to continue on it. But when
296    * the next flush succeeds, the counter of total memstore size in HRegion is always deduced by
297    * the sum of current memstore sizes instead of snapshots left from previous failed flush. This
298    * calculation is problematic that almost every time there is failed flush, HRegion.memstoreSize
299    * gets reduced by a wrong value. If region flush could not proceed for a couple cycles, the size
300    * in current memstore could be much larger than the snapshot. It's likely to drift memstoreSize
301    * much smaller than expected. In extreme case, if the error accumulates to even bigger than
302    * HRegion's memstore size limit, any further flush is skipped because flush does not do anything
303    * if memstoreSize is not larger than 0."
304    * @throws Exception
305    */
306   @Test (timeout=60000)
307   public void testFlushSizeAccounting() throws Exception {
308     final Configuration conf = HBaseConfiguration.create(CONF);
309     // Only retry once.
310     conf.setInt("hbase.hstore.flush.retries.number", 1);
311     final User user =
312       User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
313     // Inject our faulty LocalFileSystem
314     conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
315     user.runAs(new PrivilegedExceptionAction<Object>() {
316       @Override
317       public Object run() throws Exception {
318         // Make sure it worked (above is sensitive to caching details in hadoop core)
319         FileSystem fs = FileSystem.get(conf);
320         Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
321         FaultyFileSystem ffs = (FaultyFileSystem)fs;
322         HRegion region = null;
323         try {
324           // Initialize region
325           region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
326           long size = region.getMemstoreSize().get();
327           Assert.assertEquals(0, size);
328           // Put one item into memstore.  Measure the size of one item in memstore.
329           Put p1 = new Put(row);
330           p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
331           region.put(p1);
332           final long sizeOfOnePut = region.getMemstoreSize().get();
333           // Fail a flush which means the current memstore will hang out as memstore 'snapshot'.
334           try {
335             LOG.info("Flushing");
336             region.flushcache();
337             Assert.fail("Didn't bubble up IOE!");
338           } catch (DroppedSnapshotException dse) {
339             // What we are expecting
340           }
341           // Make it so all writes succeed from here on out
342           ffs.fault.set(false);
343           // Check sizes.  Should still be the one entry.
344           Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize().get());
345           // Now add two entries so that on this next flush that fails, we can see if we
346           // subtract the right amount, the snapshot size only.
347           Put p2 = new Put(row);
348           p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
349           p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
350           region.put(p2);
351           Assert.assertEquals(sizeOfOnePut * 3, region.getMemstoreSize().get());
352           // Do a successful flush.  It will clear the snapshot only.  Thats how flushes work.
353           // If already a snapshot, we clear it else we move the memstore to be snapshot and flush
354           // it
355           region.flushcache();
356           // Make sure our memory accounting is right.
357           Assert.assertEquals(sizeOfOnePut * 2, region.getMemstoreSize().get());
358         } finally {
359           HRegion.closeHRegion(region);
360         }
361         return null;
362       }
363     });
364     FileSystem.closeAllForUGI(user.getUGI());
365   }
366 
367   @Test
368   public void testCompactionAffectedByScanners() throws Exception {
369     byte[] family = Bytes.toBytes("family");
370     this.region = initHRegion(tableName, method, CONF, family);
371 
372     Put put = new Put(Bytes.toBytes("r1"));
373     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
374     region.put(put);
375     region.flushcache();
376 
377     Scan scan = new Scan();
378     scan.setMaxVersions(3);
379     // open the first scanner
380     RegionScanner scanner1 = region.getScanner(scan);
381 
382     Delete delete = new Delete(Bytes.toBytes("r1"));
383     region.delete(delete);
384     region.flushcache();
385 
386     // open the second scanner
387     RegionScanner scanner2 = region.getScanner(scan);
388 
389     List<Cell> results = new ArrayList<Cell>();
390 
391     System.out.println("Smallest read point:" + region.getSmallestReadPoint());
392 
393     // make a major compaction
394     region.compactStores(true);
395 
396     // open the third scanner
397     RegionScanner scanner3 = region.getScanner(scan);
398 
399     // get data from scanner 1, 2, 3 after major compaction
400     scanner1.next(results);
401     System.out.println(results);
402     assertEquals(1, results.size());
403 
404     results.clear();
405     scanner2.next(results);
406     System.out.println(results);
407     assertEquals(0, results.size());
408 
409     results.clear();
410     scanner3.next(results);
411     System.out.println(results);
412     assertEquals(0, results.size());
413   }
414 
415   @Test
416   public void testToShowNPEOnRegionScannerReseek() throws Exception {
417     byte[] family = Bytes.toBytes("family");
418     this.region = initHRegion(tableName, method, CONF, family);
419 
420     Put put = new Put(Bytes.toBytes("r1"));
421     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
422     region.put(put);
423     put = new Put(Bytes.toBytes("r2"));
424     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
425     region.put(put);
426     region.flushcache();
427 
428     Scan scan = new Scan();
429     scan.setMaxVersions(3);
430     // open the first scanner
431     RegionScanner scanner1 = region.getScanner(scan);
432 
433     System.out.println("Smallest read point:" + region.getSmallestReadPoint());
434 
435     region.compactStores(true);
436 
437     scanner1.reseek(Bytes.toBytes("r2"));
438     List<Cell> results = new ArrayList<Cell>();
439     scanner1.next(results);
440     Cell keyValue = results.get(0);
441     Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
442     scanner1.close();
443   }
444 
445   @Test
446   public void testSkipRecoveredEditsReplay() throws Exception {
447     String method = "testSkipRecoveredEditsReplay";
448     TableName tableName = TableName.valueOf(method);
449     byte[] family = Bytes.toBytes("family");
450     this.region = initHRegion(tableName, method, CONF, family);
451     try {
452       Path regiondir = region.getRegionFileSystem().getRegionDir();
453       FileSystem fs = region.getRegionFileSystem().getFileSystem();
454       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
455 
456       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
457 
458       long maxSeqId = 1050;
459       long minSeqId = 1000;
460 
461       for (long i = minSeqId; i <= maxSeqId; i += 10) {
462         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
463         fs.create(recoveredEdits);
464         HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
465 
466         long time = System.nanoTime();
467         WALEdit edit = new WALEdit();
468         edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
469             .toBytes(i)));
470         writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
471             HConstants.DEFAULT_CLUSTER_ID), edit));
472 
473         writer.close();
474       }
475       MonitoredTask status = TaskMonitor.get().createStatus(method);
476       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
477       for (Store store : region.getStores().values()) {
478         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
479       }
480       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
481       assertEquals(maxSeqId, seqId);
482       Get get = new Get(row);
483       Result result = region.get(get);
484       for (long i = minSeqId; i <= maxSeqId; i += 10) {
485         List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
486         assertEquals(1, kvs.size());
487         assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
488       }
489     } finally {
490       HRegion.closeHRegion(this.region);
491       this.region = null;
492     }
493   }
494 
495   @Test
496   public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
497     String method = "testSkipRecoveredEditsReplaySomeIgnored";
498     TableName tableName = TableName.valueOf(method);
499     byte[] family = Bytes.toBytes("family");
500     this.region = initHRegion(tableName, method, CONF, family);
501     try {
502       Path regiondir = region.getRegionFileSystem().getRegionDir();
503       FileSystem fs = region.getRegionFileSystem().getFileSystem();
504       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
505 
506       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
507 
508       long maxSeqId = 1050;
509       long minSeqId = 1000;
510 
511       for (long i = minSeqId; i <= maxSeqId; i += 10) {
512         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
513         fs.create(recoveredEdits);
514         HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
515 
516         long time = System.nanoTime();
517         WALEdit edit = new WALEdit();
518         edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
519             .toBytes(i)));
520         writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
521             HConstants.DEFAULT_CLUSTER_ID), edit));
522 
523         writer.close();
524       }
525       long recoverSeqId = 1030;
526       MonitoredTask status = TaskMonitor.get().createStatus(method);
527       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
528       for (Store store : region.getStores().values()) {
529         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
530       }
531       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
532       assertEquals(maxSeqId, seqId);
533       Get get = new Get(row);
534       Result result = region.get(get);
535       for (long i = minSeqId; i <= maxSeqId; i += 10) {
536         List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
537         if (i < recoverSeqId) {
538           assertEquals(0, kvs.size());
539         } else {
540           assertEquals(1, kvs.size());
541           assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
542         }
543       }
544     } finally {
545       HRegion.closeHRegion(this.region);
546       this.region = null;
547     }
548   }
549 
550   @Test
551   public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
552     byte[] family = Bytes.toBytes("family");
553     this.region = initHRegion(tableName, method, CONF, family);
554     try {
555       Path regiondir = region.getRegionFileSystem().getRegionDir();
556       FileSystem fs = region.getRegionFileSystem().getFileSystem();
557 
558       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
559       for (int i = 1000; i < 1050; i += 10) {
560         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
561         FSDataOutputStream dos = fs.create(recoveredEdits);
562         dos.writeInt(i);
563         dos.close();
564       }
565       long minSeqId = 2000;
566       Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
567       FSDataOutputStream dos = fs.create(recoveredEdits);
568       dos.close();
569 
570       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
571       for (Store store : region.getStores().values()) {
572         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
573       }
574       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
575       assertEquals(minSeqId, seqId);
576     } finally {
577       HRegion.closeHRegion(this.region);
578       this.region = null;
579     }
580   }
581 
582   @Test
583   public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
584     String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
585     TableName tableName = TableName.valueOf(method);
586     byte[] family = Bytes.toBytes("family");
587     this.region = initHRegion(tableName, method, CONF, family);
588     try {
589       Path regiondir = region.getRegionFileSystem().getRegionDir();
590       FileSystem fs = region.getRegionFileSystem().getFileSystem();
591       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
592 
593       assertEquals(0, region.getStoreFileList(
594         region.getStores().keySet().toArray(new byte[0][])).size());
595 
596       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
597 
598       long maxSeqId = 1050;
599       long minSeqId = 1000;
600 
601       for (long i = minSeqId; i <= maxSeqId; i += 10) {
602         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
603         fs.create(recoveredEdits);
604         HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
605 
606         long time = System.nanoTime();
607         WALEdit edit = null;
608         if (i == maxSeqId) {
609           edit = WALEdit.createCompaction(region.getRegionInfo(),
610           CompactionDescriptor.newBuilder()
611           .setTableName(ByteString.copyFrom(tableName.getName()))
612           .setFamilyName(ByteString.copyFrom(regionName))
613           .setEncodedRegionName(ByteString.copyFrom(regionName))
614           .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
615           .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName()))
616           .build());
617         } else {
618           edit = new WALEdit();
619           edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
620             .toBytes(i)));
621         }
622         writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
623             HConstants.DEFAULT_CLUSTER_ID), edit));
624         writer.close();
625       }
626 
627       long recoverSeqId = 1030;
628       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
629       MonitoredTask status = TaskMonitor.get().createStatus(method);
630       for (Store store : region.getStores().values()) {
631         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
632       }
633       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
634       assertEquals(maxSeqId, seqId);
635 
636       // assert that the files are flushed
637       assertEquals(1, region.getStoreFileList(
638         region.getStores().keySet().toArray(new byte[0][])).size());
639 
640     } finally {
641       HRegion.closeHRegion(this.region);
642       this.region = null;
643     }  }
644 
645   @Test
646   public void testRecoveredEditsReplayCompaction() throws Exception {
647     String method = name.getMethodName();
648     TableName tableName = TableName.valueOf(method);
649     byte[] family = Bytes.toBytes("family");
650     this.region = initHRegion(tableName, method, CONF, family);
651     try {
652       Path regiondir = region.getRegionFileSystem().getRegionDir();
653       FileSystem fs = region.getRegionFileSystem().getFileSystem();
654       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
655 
656       long maxSeqId = 3;
657       long minSeqId = 0;
658 
659       for (long i = minSeqId; i < maxSeqId; i++) {
660         Put put = new Put(Bytes.toBytes(i));
661         put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
662         region.put(put);
663         region.flushcache();
664       }
665 
666       // this will create a region with 3 files
667       assertEquals(3, region.getStore(family).getStorefilesCount());
668       List<Path> storeFiles = new ArrayList<Path>(3);
669       for (StoreFile sf : region.getStore(family).getStorefiles()) {
670         storeFiles.add(sf.getPath());
671       }
672 
673       // disable compaction completion
674       CONF.setBoolean("hbase.hstore.compaction.complete", false);
675       region.compactStores();
676 
677       // ensure that nothing changed
678       assertEquals(3, region.getStore(family).getStorefilesCount());
679 
680       // now find the compacted file, and manually add it to the recovered edits
681       Path tmpDir = region.getRegionFileSystem().getTempDir();
682       FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
683       String errorMsg = "Expected to find 1 file in the region temp directory "
684           + "from the compaction, could not find any";
685       assertNotNull(errorMsg, files);
686       assertEquals(errorMsg, 1, files.length);
687       // move the file inside region dir
688       Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
689           files[0].getPath());
690 
691       CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
692           .getRegionInfo(), family, storeFiles, Lists.newArrayList(newFile), region
693           .getRegionFileSystem().getStoreDir(Bytes.toString(family)));
694 
695       HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
696           this.region.getRegionInfo(), compactionDescriptor, new AtomicLong(1));
697 
698       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
699 
700       Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
701       fs.create(recoveredEdits);
702       HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
703 
704       long time = System.nanoTime();
705 
706       writer.append(new HLog.Entry(new HLogKey(regionName, tableName, 10, time,
707           HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(),
708           compactionDescriptor)));
709       writer.close();
710 
711       // close the region now, and reopen again
712       region.getTableDesc();
713       region.getRegionInfo();
714       region.close();
715       region = HRegion.openHRegion(region, null);
716 
717       // now check whether we have only one store file, the compacted one
718       Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
719       for (StoreFile sf : sfs) {
720         LOG.info(sf.getPath());
721       }
722       assertEquals(1, region.getStore(family).getStorefilesCount());
723       files = FSUtils.listStatus(fs, tmpDir);
724       assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
725 
726       for (long i = minSeqId; i < maxSeqId; i++) {
727         Get get = new Get(Bytes.toBytes(i));
728         Result result = region.get(get);
729         byte[] value = result.getValue(family, Bytes.toBytes(i));
730         assertArrayEquals(Bytes.toBytes(i), value);
731       }
732     } finally {
733       HRegion.closeHRegion(this.region);
734       this.region = null;
735     }
736   }
737 
738   @Test
739   public void testGetWhileRegionClose() throws IOException {
740     TableName tableName = TableName.valueOf(name.getMethodName());
741     Configuration hc = initSplit();
742     int numRows = 100;
743     byte[][] families = { fam1, fam2, fam3 };
744 
745     // Setting up region
746     String method = name.getMethodName();
747     this.region = initHRegion(tableName, method, hc, families);
748     try {
749       // Put data in region
750       final int startRow = 100;
751       putData(startRow, numRows, qual1, families);
752       putData(startRow, numRows, qual2, families);
753       putData(startRow, numRows, qual3, families);
754       final AtomicBoolean done = new AtomicBoolean(false);
755       final AtomicInteger gets = new AtomicInteger(0);
756       GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
757       try {
758         // Set ten threads running concurrently getting from the region.
759         for (int i = 0; i < threads.length / 2; i++) {
760           threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
761           threads[i].setDaemon(true);
762           threads[i].start();
763         }
764         // Artificially make the condition by setting closing flag explicitly.
765         // I can't make the issue happen with a call to region.close().
766         this.region.closing.set(true);
767         for (int i = threads.length / 2; i < threads.length; i++) {
768           threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
769           threads[i].setDaemon(true);
770           threads[i].start();
771         }
772       } finally {
773         if (this.region != null) {
774           HRegion.closeHRegion(this.region);
775         }
776       }
777       done.set(true);
778       for (GetTillDoneOrException t : threads) {
779         try {
780           t.join();
781         } catch (InterruptedException e) {
782           e.printStackTrace();
783         }
784         if (t.e != null) {
785           LOG.info("Exception=" + t.e);
786           assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
787         }
788       }
789     } finally {
790       HRegion.closeHRegion(this.region);
791       this.region = null;
792     }
793   }
794 
795   /*
796    * Thread that does get on single row until 'done' flag is flipped. If an
797    * exception causes us to fail, it records it.
798    */
799   class GetTillDoneOrException extends Thread {
800     private final Get g;
801     private final AtomicBoolean done;
802     private final AtomicInteger count;
803     private Exception e;
804 
805     GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
806       super("getter." + i);
807       this.g = new Get(r);
808       this.done = d;
809       this.count = c;
810     }
811 
812     @Override
813     public void run() {
814       while (!this.done.get()) {
815         try {
816           assertTrue(region.get(g).size() > 0);
817           this.count.incrementAndGet();
818         } catch (Exception e) {
819           this.e = e;
820           break;
821         }
822       }
823     }
824   }
825 
826   /*
827    * An involved filter test. Has multiple column families and deletes in mix.
828    */
829   @Test
830   public void testWeirdCacheBehaviour() throws Exception {
831     byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
832     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
833         Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
834     this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
835     try {
836       String value = "this is the value";
837       String value2 = "this is some other value";
838       String keyPrefix1 = "prefix1";
839       String keyPrefix2 = "prefix2";
840       String keyPrefix3 = "prefix3";
841       putRows(this.region, 3, value, keyPrefix1);
842       putRows(this.region, 3, value, keyPrefix2);
843       putRows(this.region, 3, value, keyPrefix3);
844       putRows(this.region, 3, value2, keyPrefix1);
845       putRows(this.region, 3, value2, keyPrefix2);
846       putRows(this.region, 3, value2, keyPrefix3);
847       System.out.println("Checking values for key: " + keyPrefix1);
848       assertEquals("Got back incorrect number of rows from scan", 3,
849           getNumberOfRows(keyPrefix1, value2, this.region));
850       System.out.println("Checking values for key: " + keyPrefix2);
851       assertEquals("Got back incorrect number of rows from scan", 3,
852           getNumberOfRows(keyPrefix2, value2, this.region));
853       System.out.println("Checking values for key: " + keyPrefix3);
854       assertEquals("Got back incorrect number of rows from scan", 3,
855           getNumberOfRows(keyPrefix3, value2, this.region));
856       deleteColumns(this.region, value2, keyPrefix1);
857       deleteColumns(this.region, value2, keyPrefix2);
858       deleteColumns(this.region, value2, keyPrefix3);
859       System.out.println("Starting important checks.....");
860       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
861           getNumberOfRows(keyPrefix1, value2, this.region));
862       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
863           getNumberOfRows(keyPrefix2, value2, this.region));
864       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
865           getNumberOfRows(keyPrefix3, value2, this.region));
866     } finally {
867       HRegion.closeHRegion(this.region);
868       this.region = null;
869     }
870   }
871 
872   @Test
873   public void testAppendWithReadOnlyTable() throws Exception {
874     byte[] TABLE = Bytes.toBytes("readOnlyTable");
875     this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
876     boolean exceptionCaught = false;
877     Append append = new Append(Bytes.toBytes("somerow"));
878     append.setDurability(Durability.SKIP_WAL);
879     append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
880         Bytes.toBytes("somevalue"));
881     try {
882       region.append(append);
883     } catch (IOException e) {
884       exceptionCaught = true;
885     } finally {
886       HRegion.closeHRegion(this.region);
887       this.region = null;
888     }
889     assertTrue(exceptionCaught == true);
890   }
891 
892   @Test
893   public void testIncrWithReadOnlyTable() throws Exception {
894     byte[] TABLE = Bytes.toBytes("readOnlyTable");
895     this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
896     boolean exceptionCaught = false;
897     Increment inc = new Increment(Bytes.toBytes("somerow"));
898     inc.setDurability(Durability.SKIP_WAL);
899     inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
900     try {
901       region.increment(inc);
902     } catch (IOException e) {
903       exceptionCaught = true;
904     } finally {
905       HRegion.closeHRegion(this.region);
906       this.region = null;
907     }
908     assertTrue(exceptionCaught == true);
909   }
910 
911   private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
912     InternalScanner scanner = buildScanner(keyPrefix, value, r);
913     int count = 0;
914     boolean more = false;
915     List<Cell> results = new ArrayList<Cell>();
916     do {
917       more = scanner.next(results);
918       if (results != null && !results.isEmpty())
919         count++;
920       else
921         break;
922       Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
923       delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
924       r.delete(delete);
925       results.clear();
926     } while (more);
927     assertEquals("Did not perform correct number of deletes", 3, count);
928   }
929 
930   private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
931     InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
932     int numberOfResults = 0;
933     List<Cell> results = new ArrayList<Cell>();
934     boolean more = false;
935     do {
936       more = resultScanner.next(results);
937       if (results != null && !results.isEmpty())
938         numberOfResults++;
939       else
940         break;
941       for (Cell kv : results) {
942         System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
943       }
944       results.clear();
945     } while (more);
946     return numberOfResults;
947   }
948 
949   private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
950       throws IOException {
951     // Defaults FilterList.Operator.MUST_PASS_ALL.
952     FilterList allFilters = new FilterList();
953     allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
954     // Only return rows where this column value exists in the row.
955     SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
956         Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
957     filter.setFilterIfMissing(true);
958     allFilters.addFilter(filter);
959     Scan scan = new Scan();
960     scan.addFamily(Bytes.toBytes("trans-blob"));
961     scan.addFamily(Bytes.toBytes("trans-type"));
962     scan.addFamily(Bytes.toBytes("trans-date"));
963     scan.addFamily(Bytes.toBytes("trans-tags"));
964     scan.addFamily(Bytes.toBytes("trans-group"));
965     scan.setFilter(allFilters);
966     return r.getScanner(scan);
967   }
968 
969   private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
970     for (int i = 0; i < numRows; i++) {
971       String row = key + "_" + i/* UUID.randomUUID().toString() */;
972       System.out.println(String.format("Saving row: %s, with value %s", row, value));
973       Put put = new Put(Bytes.toBytes(row));
974       put.setDurability(Durability.SKIP_WAL);
975       put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
976       put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
977       put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
978       put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
979       put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
980       r.put(put);
981     }
982   }
983 
984   @Test
985   public void testFamilyWithAndWithoutColon() throws Exception {
986     byte[] b = Bytes.toBytes(getName());
987     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
988     this.region = initHRegion(b, getName(), CONF, cf);
989     try {
990       Put p = new Put(b);
991       byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
992       p.add(cfwithcolon, cfwithcolon, cfwithcolon);
993       boolean exception = false;
994       try {
995         this.region.put(p);
996       } catch (NoSuchColumnFamilyException e) {
997         exception = true;
998       }
999       assertTrue(exception);
1000     } finally {
1001       HRegion.closeHRegion(this.region);
1002       this.region = null;
1003     }
1004   }
1005 
1006   @Test
1007   public void testBatchPut() throws Exception {
1008     byte[] b = Bytes.toBytes(getName());
1009     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1010     byte[] qual = Bytes.toBytes("qual");
1011     byte[] val = Bytes.toBytes("val");
1012     this.region = initHRegion(b, getName(), CONF, cf);
1013     MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1014     try {
1015       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1016       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1017 
1018       LOG.info("First a batch put with all valid puts");
1019       final Put[] puts = new Put[10];
1020       for (int i = 0; i < 10; i++) {
1021         puts[i] = new Put(Bytes.toBytes("row_" + i));
1022         puts[i].add(cf, qual, val);
1023       }
1024 
1025       OperationStatus[] codes = this.region.batchMutate(puts);
1026       assertEquals(10, codes.length);
1027       for (int i = 0; i < 10; i++) {
1028         assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1029       }
1030       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1031 
1032       LOG.info("Next a batch put with one invalid family");
1033       puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1034       codes = this.region.batchMutate(puts);
1035       assertEquals(10, codes.length);
1036       for (int i = 0; i < 10; i++) {
1037         assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1038             codes[i].getOperationStatusCode());
1039       }
1040 
1041       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1042 
1043       LOG.info("Next a batch put that has to break into two batches to avoid a lock");
1044       RowLock rowLock = region.getRowLock(Bytes.toBytes("row_2"));
1045 
1046       MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1047       final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
1048       TestThread putter = new TestThread(ctx) {
1049         @Override
1050         public void doWork() throws IOException {
1051           retFromThread.set(region.batchMutate(puts));
1052         }
1053       };
1054       LOG.info("...starting put thread while holding lock");
1055       ctx.addThread(putter);
1056       ctx.startThreads();
1057 
1058       LOG.info("...waiting for put thread to sync first time");
1059       long startWait = System.currentTimeMillis();
1060       while (metricsAssertHelper.getCounter("syncTimeNumOps", source) == syncs + 2) {
1061         Thread.sleep(100);
1062         if (System.currentTimeMillis() - startWait > 10000) {
1063           fail("Timed out waiting for thread to sync first minibatch");
1064         }
1065       }
1066       LOG.info("...releasing row lock, which should let put thread continue");
1067       rowLock.release();
1068       LOG.info("...joining on thread");
1069       ctx.stop();
1070       LOG.info("...checking that next batch was synced");
1071       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 4, source);
1072       codes = retFromThread.get();
1073       for (int i = 0; i < 10; i++) {
1074         assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1075             codes[i].getOperationStatusCode());
1076       }
1077 
1078     } finally {
1079       HRegion.closeHRegion(this.region);
1080       this.region = null;
1081     }
1082   }
1083 
1084   @Test
1085   public void testBatchPutWithTsSlop() throws Exception {
1086     byte[] b = Bytes.toBytes(getName());
1087     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1088     byte[] qual = Bytes.toBytes("qual");
1089     byte[] val = Bytes.toBytes("val");
1090 
1091     // add data with a timestamp that is too recent for range. Ensure assert
1092     CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1093     this.region = initHRegion(b, getName(), CONF, cf);
1094 
1095     try {
1096       MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1097       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1098       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1099 
1100       final Put[] puts = new Put[10];
1101       for (int i = 0; i < 10; i++) {
1102         puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
1103         puts[i].add(cf, qual, val);
1104       }
1105 
1106       OperationStatus[] codes = this.region.batchMutate(puts);
1107       assertEquals(10, codes.length);
1108       for (int i = 0; i < 10; i++) {
1109         assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1110       }
1111       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1112 
1113     } finally {
1114       HRegion.closeHRegion(this.region);
1115       this.region = null;
1116     }
1117 
1118   }
1119 
1120   // ////////////////////////////////////////////////////////////////////////////
1121   // checkAndMutate tests
1122   // ////////////////////////////////////////////////////////////////////////////
1123   @Test
1124   public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1125     byte[] row1 = Bytes.toBytes("row1");
1126     byte[] fam1 = Bytes.toBytes("fam1");
1127     byte[] qf1 = Bytes.toBytes("qualifier");
1128     byte[] emptyVal = new byte[] {};
1129     byte[] val1 = Bytes.toBytes("value1");
1130     byte[] val2 = Bytes.toBytes("value2");
1131 
1132     // Setting up region
1133     String method = this.getName();
1134     this.region = initHRegion(tableName, method, CONF, fam1);
1135     try {
1136       // Putting empty data in key
1137       Put put = new Put(row1);
1138       put.add(fam1, qf1, emptyVal);
1139 
1140       // checkAndPut with empty value
1141       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1142           emptyVal), put, true);
1143       assertTrue(res);
1144 
1145       // Putting data in key
1146       put = new Put(row1);
1147       put.add(fam1, qf1, val1);
1148 
1149       // checkAndPut with correct value
1150       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1151           put, true);
1152       assertTrue(res);
1153 
1154       // not empty anymore
1155       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1156           put, true);
1157       assertFalse(res);
1158 
1159       Delete delete = new Delete(row1);
1160       delete.deleteColumn(fam1, qf1);
1161       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1162           delete, true);
1163       assertFalse(res);
1164 
1165       put = new Put(row1);
1166       put.add(fam1, qf1, val2);
1167       // checkAndPut with correct value
1168       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1169           put, true);
1170       assertTrue(res);
1171 
1172       // checkAndDelete with correct value
1173       delete = new Delete(row1);
1174       delete.deleteColumn(fam1, qf1);
1175       delete.deleteColumn(fam1, qf1);
1176       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1177           delete, true);
1178       assertTrue(res);
1179 
1180       delete = new Delete(row1);
1181       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1182           delete, true);
1183       assertTrue(res);
1184 
1185       // checkAndPut looking for a null value
1186       put = new Put(row1);
1187       put.add(fam1, qf1, val1);
1188 
1189       res = region
1190           .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
1191       assertTrue(res);
1192     } finally {
1193       HRegion.closeHRegion(this.region);
1194       this.region = null;
1195     }
1196   }
1197 
1198   @Test
1199   public void testCheckAndMutate_WithWrongValue() throws IOException {
1200     byte[] row1 = Bytes.toBytes("row1");
1201     byte[] fam1 = Bytes.toBytes("fam1");
1202     byte[] qf1 = Bytes.toBytes("qualifier");
1203     byte[] val1 = Bytes.toBytes("value1");
1204     byte[] val2 = Bytes.toBytes("value2");
1205 
1206     // Setting up region
1207     String method = this.getName();
1208     this.region = initHRegion(tableName, method, CONF, fam1);
1209     try {
1210       // Putting data in key
1211       Put put = new Put(row1);
1212       put.add(fam1, qf1, val1);
1213       region.put(put);
1214 
1215       // checkAndPut with wrong value
1216       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1217           val2), put, true);
1218       assertEquals(false, res);
1219 
1220       // checkAndDelete with wrong value
1221       Delete delete = new Delete(row1);
1222       delete.deleteFamily(fam1);
1223       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1224           delete, true);
1225       assertEquals(false, res);
1226     } finally {
1227       HRegion.closeHRegion(this.region);
1228       this.region = null;
1229     }
1230   }
1231 
1232   @Test
1233   public void testCheckAndMutate_WithCorrectValue() throws IOException {
1234     byte[] row1 = Bytes.toBytes("row1");
1235     byte[] fam1 = Bytes.toBytes("fam1");
1236     byte[] qf1 = Bytes.toBytes("qualifier");
1237     byte[] val1 = Bytes.toBytes("value1");
1238 
1239     // Setting up region
1240     String method = this.getName();
1241     this.region = initHRegion(tableName, method, CONF, fam1);
1242     try {
1243       // Putting data in key
1244       Put put = new Put(row1);
1245       put.add(fam1, qf1, val1);
1246       region.put(put);
1247 
1248       // checkAndPut with correct value
1249       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1250           val1), put, true);
1251       assertEquals(true, res);
1252 
1253       // checkAndDelete with correct value
1254       Delete delete = new Delete(row1);
1255       delete.deleteColumn(fam1, qf1);
1256       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1257           put, true);
1258       assertEquals(true, res);
1259     } finally {
1260       HRegion.closeHRegion(this.region);
1261       this.region = null;
1262     }
1263   }
1264 
1265   @Test
1266   public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1267     byte[] row1 = Bytes.toBytes("row1");
1268     byte[] fam1 = Bytes.toBytes("fam1");
1269     byte[] qf1 = Bytes.toBytes("qualifier");
1270     byte[] val1 = Bytes.toBytes("value1");
1271     byte[] val2 = Bytes.toBytes("value2");
1272     byte[] val3 = Bytes.toBytes("value3");
1273     byte[] val4 = Bytes.toBytes("value4");
1274 
1275     // Setting up region
1276     String method = this.getName();
1277     this.region = initHRegion(tableName, method, CONF, fam1);
1278     try {
1279       // Putting val3 in key
1280       Put put = new Put(row1);
1281       put.add(fam1, qf1, val3);
1282       region.put(put);
1283 
1284       // Test CompareOp.LESS: original = val3, compare with val3, fail
1285       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1286           new BinaryComparator(val3), put, true);
1287       assertEquals(false, res);
1288 
1289       // Test CompareOp.LESS: original = val3, compare with val4, fail
1290       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1291           new BinaryComparator(val4), put, true);
1292       assertEquals(false, res);
1293 
1294       // Test CompareOp.LESS: original = val3, compare with val2,
1295       // succeed (now value = val2)
1296       put = new Put(row1);
1297       put.add(fam1, qf1, val2);
1298       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1299           new BinaryComparator(val2), put, true);
1300       assertEquals(true, res);
1301 
1302       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail
1303       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1304           new BinaryComparator(val3), put, true);
1305       assertEquals(false, res);
1306 
1307       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2,
1308       // succeed (value still = val2)
1309       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1310           new BinaryComparator(val2), put, true);
1311       assertEquals(true, res);
1312 
1313       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1,
1314       // succeed (now value = val3)
1315       put = new Put(row1);
1316       put.add(fam1, qf1, val3);
1317       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1318           new BinaryComparator(val1), put, true);
1319       assertEquals(true, res);
1320 
1321       // Test CompareOp.GREATER: original = val3, compare with val3, fail
1322       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1323           new BinaryComparator(val3), put, true);
1324       assertEquals(false, res);
1325 
1326       // Test CompareOp.GREATER: original = val3, compare with val2, fail
1327       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1328           new BinaryComparator(val2), put, true);
1329       assertEquals(false, res);
1330 
1331       // Test CompareOp.GREATER: original = val3, compare with val4,
1332       // succeed (now value = val2)
1333       put = new Put(row1);
1334       put.add(fam1, qf1, val2);
1335       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1336           new BinaryComparator(val4), put, true);
1337       assertEquals(true, res);
1338 
1339       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail
1340       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1341           new BinaryComparator(val1), put, true);
1342       assertEquals(false, res);
1343 
1344       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2,
1345       // succeed (value still = val2)
1346       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1347           new BinaryComparator(val2), put, true);
1348       assertEquals(true, res);
1349 
1350       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed
1351       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1352           new BinaryComparator(val3), put, true);
1353       assertEquals(true, res);
1354     } finally {
1355       HRegion.closeHRegion(this.region);
1356       this.region = null;
1357     }
1358   }
1359 
1360   @Test
1361   public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1362     byte[] row1 = Bytes.toBytes("row1");
1363     byte[] fam1 = Bytes.toBytes("fam1");
1364     byte[] fam2 = Bytes.toBytes("fam2");
1365     byte[] qf1 = Bytes.toBytes("qualifier");
1366     byte[] val1 = Bytes.toBytes("value1");
1367     byte[] val2 = Bytes.toBytes("value2");
1368 
1369     byte[][] families = { fam1, fam2 };
1370 
1371     // Setting up region
1372     String method = this.getName();
1373     this.region = initHRegion(tableName, method, CONF, families);
1374     try {
1375       // Putting data in the key to check
1376       Put put = new Put(row1);
1377       put.add(fam1, qf1, val1);
1378       region.put(put);
1379 
1380       // Creating put to add
1381       long ts = System.currentTimeMillis();
1382       KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1383       put = new Put(row1);
1384       put.add(kv);
1385 
1386       // checkAndPut with wrong value
1387       HStore store = (HStore) region.getStore(fam1);
1388       store.memstore.kvset.size();
1389 
1390       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1391           val1), put, true);
1392       assertEquals(true, res);
1393       store.memstore.kvset.size();
1394 
1395       Get get = new Get(row1);
1396       get.addColumn(fam2, qf1);
1397       Cell[] actual = region.get(get).rawCells();
1398 
1399       Cell[] expected = { kv };
1400 
1401       assertEquals(expected.length, actual.length);
1402       for (int i = 0; i < actual.length; i++) {
1403         assertEquals(expected[i], actual[i]);
1404       }
1405     } finally {
1406       HRegion.closeHRegion(this.region);
1407       this.region = null;
1408     }
1409   }
1410 
1411   @Test
1412   public void testCheckAndPut_wrongRowInPut() throws IOException {
1413     TableName tableName = TableName.valueOf(name.getMethodName());
1414     this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
1415     try {
1416       Put put = new Put(row2);
1417       put.add(fam1, qual1, value1);
1418       try {
1419         region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1420             new BinaryComparator(value2), put, false);
1421         fail();
1422       } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1423         // expected exception.
1424       }
1425     } finally {
1426       HRegion.closeHRegion(this.region);
1427       this.region = null;
1428     }
1429   }
1430 
1431   @Test
1432   public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1433     byte[] row1 = Bytes.toBytes("row1");
1434     byte[] fam1 = Bytes.toBytes("fam1");
1435     byte[] fam2 = Bytes.toBytes("fam2");
1436     byte[] qf1 = Bytes.toBytes("qualifier1");
1437     byte[] qf2 = Bytes.toBytes("qualifier2");
1438     byte[] qf3 = Bytes.toBytes("qualifier3");
1439     byte[] val1 = Bytes.toBytes("value1");
1440     byte[] val2 = Bytes.toBytes("value2");
1441     byte[] val3 = Bytes.toBytes("value3");
1442     byte[] emptyVal = new byte[] {};
1443 
1444     byte[][] families = { fam1, fam2 };
1445 
1446     // Setting up region
1447     String method = this.getName();
1448     this.region = initHRegion(tableName, method, CONF, families);
1449     try {
1450       // Put content
1451       Put put = new Put(row1);
1452       put.add(fam1, qf1, val1);
1453       region.put(put);
1454       Threads.sleep(2);
1455 
1456       put = new Put(row1);
1457       put.add(fam1, qf1, val2);
1458       put.add(fam2, qf1, val3);
1459       put.add(fam2, qf2, val2);
1460       put.add(fam2, qf3, val1);
1461       put.add(fam1, qf3, val1);
1462       region.put(put);
1463 
1464       // Multi-column delete
1465       Delete delete = new Delete(row1);
1466       delete.deleteColumn(fam1, qf1);
1467       delete.deleteColumn(fam2, qf1);
1468       delete.deleteColumn(fam1, qf3);
1469       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1470           val2), delete, true);
1471       assertEquals(true, res);
1472 
1473       Get get = new Get(row1);
1474       get.addColumn(fam1, qf1);
1475       get.addColumn(fam1, qf3);
1476       get.addColumn(fam2, qf2);
1477       Result r = region.get(get);
1478       assertEquals(2, r.size());
1479       assertArrayEquals(val1, r.getValue(fam1, qf1));
1480       assertArrayEquals(val2, r.getValue(fam2, qf2));
1481 
1482       // Family delete
1483       delete = new Delete(row1);
1484       delete.deleteFamily(fam2);
1485       res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1486           delete, true);
1487       assertEquals(true, res);
1488 
1489       get = new Get(row1);
1490       r = region.get(get);
1491       assertEquals(1, r.size());
1492       assertArrayEquals(val1, r.getValue(fam1, qf1));
1493 
1494       // Row delete
1495       delete = new Delete(row1);
1496       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1497           delete, true);
1498       assertEquals(true, res);
1499       get = new Get(row1);
1500       r = region.get(get);
1501       assertEquals(0, r.size());
1502     } finally {
1503       HRegion.closeHRegion(this.region);
1504       this.region = null;
1505     }
1506   }
1507 
1508   // ////////////////////////////////////////////////////////////////////////////
1509   // Delete tests
1510   // ////////////////////////////////////////////////////////////////////////////
1511   @Test
1512   public void testDelete_multiDeleteColumn() throws IOException {
1513     byte[] row1 = Bytes.toBytes("row1");
1514     byte[] fam1 = Bytes.toBytes("fam1");
1515     byte[] qual = Bytes.toBytes("qualifier");
1516     byte[] value = Bytes.toBytes("value");
1517 
1518     Put put = new Put(row1);
1519     put.add(fam1, qual, 1, value);
1520     put.add(fam1, qual, 2, value);
1521 
1522     String method = this.getName();
1523     this.region = initHRegion(tableName, method, CONF, fam1);
1524     try {
1525       region.put(put);
1526 
1527       // We do support deleting more than 1 'latest' version
1528       Delete delete = new Delete(row1);
1529       delete.deleteColumn(fam1, qual);
1530       delete.deleteColumn(fam1, qual);
1531       region.delete(delete);
1532 
1533       Get get = new Get(row1);
1534       get.addFamily(fam1);
1535       Result r = region.get(get);
1536       assertEquals(0, r.size());
1537     } finally {
1538       HRegion.closeHRegion(this.region);
1539       this.region = null;
1540     }
1541   }
1542 
1543   @Test
1544   public void testDelete_CheckFamily() throws IOException {
1545     byte[] row1 = Bytes.toBytes("row1");
1546     byte[] fam1 = Bytes.toBytes("fam1");
1547     byte[] fam2 = Bytes.toBytes("fam2");
1548     byte[] fam3 = Bytes.toBytes("fam3");
1549     byte[] fam4 = Bytes.toBytes("fam4");
1550 
1551     // Setting up region
1552     String method = this.getName();
1553     this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
1554     try {
1555       List<Cell> kvs = new ArrayList<Cell>();
1556       kvs.add(new KeyValue(row1, fam4, null, null));
1557 
1558       // testing existing family
1559       byte[] family = fam2;
1560       try {
1561         NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1562             Bytes.BYTES_COMPARATOR);
1563         deleteMap.put(family, kvs);
1564         region.delete(deleteMap, Durability.SYNC_WAL);
1565       } catch (Exception e) {
1566         assertTrue("Family " + new String(family) + " does not exist", false);
1567       }
1568 
1569       // testing non existing family
1570       boolean ok = false;
1571       family = fam4;
1572       try {
1573         NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1574             Bytes.BYTES_COMPARATOR);
1575         deleteMap.put(family, kvs);
1576         region.delete(deleteMap, Durability.SYNC_WAL);
1577       } catch (Exception e) {
1578         ok = true;
1579       }
1580       assertEquals("Family " + new String(family) + " does exist", true, ok);
1581     } finally {
1582       HRegion.closeHRegion(this.region);
1583       this.region = null;
1584     }
1585   }
1586 
1587   @Test
1588   public void testDelete_mixed() throws IOException, InterruptedException {
1589     byte[] fam = Bytes.toBytes("info");
1590     byte[][] families = { fam };
1591     String method = this.getName();
1592     this.region = initHRegion(tableName, method, CONF, families);
1593     try {
1594       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1595 
1596       byte[] row = Bytes.toBytes("table_name");
1597       // column names
1598       byte[] serverinfo = Bytes.toBytes("serverinfo");
1599       byte[] splitA = Bytes.toBytes("splitA");
1600       byte[] splitB = Bytes.toBytes("splitB");
1601 
1602       // add some data:
1603       Put put = new Put(row);
1604       put.add(fam, splitA, Bytes.toBytes("reference_A"));
1605       region.put(put);
1606 
1607       put = new Put(row);
1608       put.add(fam, splitB, Bytes.toBytes("reference_B"));
1609       region.put(put);
1610 
1611       put = new Put(row);
1612       put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1613       region.put(put);
1614 
1615       // ok now delete a split:
1616       Delete delete = new Delete(row);
1617       delete.deleteColumns(fam, splitA);
1618       region.delete(delete);
1619 
1620       // assert some things:
1621       Get get = new Get(row).addColumn(fam, serverinfo);
1622       Result result = region.get(get);
1623       assertEquals(1, result.size());
1624 
1625       get = new Get(row).addColumn(fam, splitA);
1626       result = region.get(get);
1627       assertEquals(0, result.size());
1628 
1629       get = new Get(row).addColumn(fam, splitB);
1630       result = region.get(get);
1631       assertEquals(1, result.size());
1632 
1633       // Assert that after a delete, I can put.
1634       put = new Put(row);
1635       put.add(fam, splitA, Bytes.toBytes("reference_A"));
1636       region.put(put);
1637       get = new Get(row);
1638       result = region.get(get);
1639       assertEquals(3, result.size());
1640 
1641       // Now delete all... then test I can add stuff back
1642       delete = new Delete(row);
1643       region.delete(delete);
1644       assertEquals(0, region.get(get).size());
1645 
1646       region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
1647       result = region.get(get);
1648       assertEquals(1, result.size());
1649     } finally {
1650       HRegion.closeHRegion(this.region);
1651       this.region = null;
1652     }
1653   }
1654 
1655   @Test
1656   public void testDeleteRowWithFutureTs() throws IOException {
1657     byte[] fam = Bytes.toBytes("info");
1658     byte[][] families = { fam };
1659     String method = this.getName();
1660     this.region = initHRegion(tableName, method, CONF, families);
1661     try {
1662       byte[] row = Bytes.toBytes("table_name");
1663       // column names
1664       byte[] serverinfo = Bytes.toBytes("serverinfo");
1665 
1666       // add data in the far future
1667       Put put = new Put(row);
1668       put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
1669       region.put(put);
1670 
1671       // now delete something in the present
1672       Delete delete = new Delete(row);
1673       region.delete(delete);
1674 
1675       // make sure we still see our data
1676       Get get = new Get(row).addColumn(fam, serverinfo);
1677       Result result = region.get(get);
1678       assertEquals(1, result.size());
1679 
1680       // delete the future row
1681       delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
1682       region.delete(delete);
1683 
1684       // make sure it is gone
1685       get = new Get(row).addColumn(fam, serverinfo);
1686       result = region.get(get);
1687       assertEquals(0, result.size());
1688     } finally {
1689       HRegion.closeHRegion(this.region);
1690       this.region = null;
1691     }
1692   }
1693 
1694   /**
1695    * Tests that the special LATEST_TIMESTAMP option for puts gets replaced by
1696    * the actual timestamp
1697    */
1698   @Test
1699   public void testPutWithLatestTS() throws IOException {
1700     byte[] fam = Bytes.toBytes("info");
1701     byte[][] families = { fam };
1702     String method = this.getName();
1703     this.region = initHRegion(tableName, method, CONF, families);
1704     try {
1705       byte[] row = Bytes.toBytes("row1");
1706       // column names
1707       byte[] qual = Bytes.toBytes("qual");
1708 
1709       // add data with LATEST_TIMESTAMP, put without WAL
1710       Put put = new Put(row);
1711       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1712       region.put(put);
1713 
1714       // Make sure it shows up with an actual timestamp
1715       Get get = new Get(row).addColumn(fam, qual);
1716       Result result = region.get(get);
1717       assertEquals(1, result.size());
1718       Cell kv = result.rawCells()[0];
1719       LOG.info("Got: " + kv);
1720       assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1721           kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1722 
1723       // Check same with WAL enabled (historically these took different
1724       // code paths, so check both)
1725       row = Bytes.toBytes("row2");
1726       put = new Put(row);
1727       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1728       region.put(put);
1729 
1730       // Make sure it shows up with an actual timestamp
1731       get = new Get(row).addColumn(fam, qual);
1732       result = region.get(get);
1733       assertEquals(1, result.size());
1734       kv = result.rawCells()[0];
1735       LOG.info("Got: " + kv);
1736       assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1737           kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1738     } finally {
1739       HRegion.closeHRegion(this.region);
1740       this.region = null;
1741     }
1742 
1743   }
1744 
1745   /**
1746    * Tests that there is server-side filtering for invalid timestamp upper
1747    * bound. Note that the timestamp lower bound is automatically handled for us
1748    * by the TTL field.
1749    */
1750   @Test
1751   public void testPutWithTsSlop() throws IOException {
1752     byte[] fam = Bytes.toBytes("info");
1753     byte[][] families = { fam };
1754     String method = this.getName();
1755 
1756     // add data with a timestamp that is too recent for range. Ensure assert
1757     CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1758     this.region = initHRegion(tableName, method, CONF, families);
1759     boolean caughtExcep = false;
1760     try {
1761       try {
1762         // no TS specified == use latest. should not error
1763         region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
1764         // TS out of range. should error
1765         region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
1766             Bytes.toBytes("value")));
1767         fail("Expected IOE for TS out of configured timerange");
1768       } catch (FailedSanityCheckException ioe) {
1769         LOG.debug("Received expected exception", ioe);
1770         caughtExcep = true;
1771       }
1772       assertTrue("Should catch FailedSanityCheckException", caughtExcep);
1773     } finally {
1774       HRegion.closeHRegion(this.region);
1775       this.region = null;
1776     }
1777   }
1778 
1779   @Test
1780   public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
1781     byte[] fam1 = Bytes.toBytes("columnA");
1782     byte[] fam2 = Bytes.toBytes("columnB");
1783     this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
1784     try {
1785       byte[] rowA = Bytes.toBytes("rowA");
1786       byte[] rowB = Bytes.toBytes("rowB");
1787 
1788       byte[] value = Bytes.toBytes("value");
1789 
1790       Delete delete = new Delete(rowA);
1791       delete.deleteFamily(fam1);
1792 
1793       region.delete(delete);
1794 
1795       // now create data.
1796       Put put = new Put(rowA);
1797       put.add(fam2, null, value);
1798       region.put(put);
1799 
1800       put = new Put(rowB);
1801       put.add(fam1, null, value);
1802       put.add(fam2, null, value);
1803       region.put(put);
1804 
1805       Scan scan = new Scan();
1806       scan.addFamily(fam1).addFamily(fam2);
1807       InternalScanner s = region.getScanner(scan);
1808       List<Cell> results = new ArrayList<Cell>();
1809       s.next(results);
1810       assertTrue(CellUtil.matchingRow(results.get(0), rowA));
1811 
1812       results.clear();
1813       s.next(results);
1814       assertTrue(CellUtil.matchingRow(results.get(0), rowB));
1815     } finally {
1816       HRegion.closeHRegion(this.region);
1817       this.region = null;
1818     }
1819   }
1820 
1821   @Test
1822   public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
1823     Delete delete = new Delete(row);
1824     delete.deleteColumns(fam1, qual1);
1825     doTestDelete_AndPostInsert(delete);
1826   }
1827 
1828   @Test
1829   public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
1830     Delete delete = new Delete(row);
1831     delete.deleteFamily(fam1);
1832     doTestDelete_AndPostInsert(delete);
1833   }
1834 
1835   public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
1836     TableName tableName = TableName.valueOf(name.getMethodName());
1837     this.region = initHRegion(tableName, getName(), CONF, fam1);
1838     try {
1839       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1840       Put put = new Put(row);
1841       put.add(fam1, qual1, value1);
1842       region.put(put);
1843 
1844       // now delete the value:
1845       region.delete(delete);
1846 
1847       // ok put data:
1848       put = new Put(row);
1849       put.add(fam1, qual1, value2);
1850       region.put(put);
1851 
1852       // ok get:
1853       Get get = new Get(row);
1854       get.addColumn(fam1, qual1);
1855 
1856       Result r = region.get(get);
1857       assertEquals(1, r.size());
1858       assertArrayEquals(value2, r.getValue(fam1, qual1));
1859 
1860       // next:
1861       Scan scan = new Scan(row);
1862       scan.addColumn(fam1, qual1);
1863       InternalScanner s = region.getScanner(scan);
1864 
1865       List<Cell> results = new ArrayList<Cell>();
1866       assertEquals(false, s.next(results));
1867       assertEquals(1, results.size());
1868       Cell kv = results.get(0);
1869 
1870       assertArrayEquals(value2, CellUtil.cloneValue(kv));
1871       assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
1872       assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
1873       assertArrayEquals(row, CellUtil.cloneRow(kv));
1874     } finally {
1875       HRegion.closeHRegion(this.region);
1876       this.region = null;
1877     }
1878   }
1879 
1880   @Test
1881   public void testDelete_CheckTimestampUpdated() throws IOException {
1882     TableName tableName = TableName.valueOf(name.getMethodName());
1883     byte[] row1 = Bytes.toBytes("row1");
1884     byte[] col1 = Bytes.toBytes("col1");
1885     byte[] col2 = Bytes.toBytes("col2");
1886     byte[] col3 = Bytes.toBytes("col3");
1887 
1888     // Setting up region
1889     String method = this.getName();
1890     this.region = initHRegion(tableName, method, CONF, fam1);
1891     try {
1892       // Building checkerList
1893       List<Cell> kvs = new ArrayList<Cell>();
1894       kvs.add(new KeyValue(row1, fam1, col1, null));
1895       kvs.add(new KeyValue(row1, fam1, col2, null));
1896       kvs.add(new KeyValue(row1, fam1, col3, null));
1897 
1898       NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1899           Bytes.BYTES_COMPARATOR);
1900       deleteMap.put(fam1, kvs);
1901       region.delete(deleteMap, Durability.SYNC_WAL);
1902 
1903       // extract the key values out the memstore:
1904       // This is kinda hacky, but better than nothing...
1905       long now = System.currentTimeMillis();
1906       KeyValue firstKv = ((HStore) region.getStore(fam1)).memstore.kvset.first();
1907       assertTrue(firstKv.getTimestamp() <= now);
1908       now = firstKv.getTimestamp();
1909       for (KeyValue kv : ((HStore) region.getStore(fam1)).memstore.kvset) {
1910         assertTrue(kv.getTimestamp() <= now);
1911         now = kv.getTimestamp();
1912       }
1913     } finally {
1914       HRegion.closeHRegion(this.region);
1915       this.region = null;
1916     }
1917   }
1918 
1919   // ////////////////////////////////////////////////////////////////////////////
1920   // Get tests
1921   // ////////////////////////////////////////////////////////////////////////////
1922   @Test
1923   public void testGet_FamilyChecker() throws IOException {
1924     byte[] row1 = Bytes.toBytes("row1");
1925     byte[] fam1 = Bytes.toBytes("fam1");
1926     byte[] fam2 = Bytes.toBytes("False");
1927     byte[] col1 = Bytes.toBytes("col1");
1928 
1929     // Setting up region
1930     String method = this.getName();
1931     this.region = initHRegion(tableName, method, CONF, fam1);
1932     try {
1933       Get get = new Get(row1);
1934       get.addColumn(fam2, col1);
1935 
1936       // Test
1937       try {
1938         region.get(get);
1939       } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
1940         assertFalse(false);
1941         return;
1942       }
1943       assertFalse(true);
1944     } finally {
1945       HRegion.closeHRegion(this.region);
1946       this.region = null;
1947     }
1948   }
1949 
1950   @Test
1951   public void testGet_Basic() throws IOException {
1952     byte[] row1 = Bytes.toBytes("row1");
1953     byte[] fam1 = Bytes.toBytes("fam1");
1954     byte[] col1 = Bytes.toBytes("col1");
1955     byte[] col2 = Bytes.toBytes("col2");
1956     byte[] col3 = Bytes.toBytes("col3");
1957     byte[] col4 = Bytes.toBytes("col4");
1958     byte[] col5 = Bytes.toBytes("col5");
1959 
1960     // Setting up region
1961     String method = this.getName();
1962     this.region = initHRegion(tableName, method, CONF, fam1);
1963     try {
1964       // Add to memstore
1965       Put put = new Put(row1);
1966       put.add(fam1, col1, null);
1967       put.add(fam1, col2, null);
1968       put.add(fam1, col3, null);
1969       put.add(fam1, col4, null);
1970       put.add(fam1, col5, null);
1971       region.put(put);
1972 
1973       Get get = new Get(row1);
1974       get.addColumn(fam1, col2);
1975       get.addColumn(fam1, col4);
1976       // Expected result
1977       KeyValue kv1 = new KeyValue(row1, fam1, col2);
1978       KeyValue kv2 = new KeyValue(row1, fam1, col4);
1979       KeyValue[] expected = { kv1, kv2 };
1980 
1981       // Test
1982       Result res = region.get(get);
1983       assertEquals(expected.length, res.size());
1984       for (int i = 0; i < res.size(); i++) {
1985         assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
1986         assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
1987         assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
1988       }
1989 
1990       // Test using a filter on a Get
1991       Get g = new Get(row1);
1992       final int count = 2;
1993       g.setFilter(new ColumnCountGetFilter(count));
1994       res = region.get(g);
1995       assertEquals(count, res.size());
1996     } finally {
1997       HRegion.closeHRegion(this.region);
1998       this.region = null;
1999     }
2000   }
2001 
2002   @Test
2003   public void testGet_Empty() throws IOException {
2004     byte[] row = Bytes.toBytes("row");
2005     byte[] fam = Bytes.toBytes("fam");
2006 
2007     String method = this.getName();
2008     this.region = initHRegion(tableName, method, CONF, fam);
2009     try {
2010       Get get = new Get(row);
2011       get.addFamily(fam);
2012       Result r = region.get(get);
2013 
2014       assertTrue(r.isEmpty());
2015     } finally {
2016       HRegion.closeHRegion(this.region);
2017       this.region = null;
2018     }
2019   }
2020 
2021   // ////////////////////////////////////////////////////////////////////////////
2022   // Merge test
2023   // ////////////////////////////////////////////////////////////////////////////
2024   @Test
2025   public void testMerge() throws IOException {
2026     byte[][] families = { fam1, fam2, fam3 };
2027     Configuration hc = initSplit();
2028     // Setting up region
2029     String method = this.getName();
2030     this.region = initHRegion(tableName, method, hc, families);
2031     try {
2032       LOG.info("" + HBaseTestCase.addContent(region, fam3));
2033       region.flushcache();
2034       region.compactStores();
2035       byte[] splitRow = region.checkSplit();
2036       assertNotNull(splitRow);
2037       LOG.info("SplitRow: " + Bytes.toString(splitRow));
2038       HRegion[] subregions = splitRegion(region, splitRow);
2039       try {
2040         // Need to open the regions.
2041         for (int i = 0; i < subregions.length; i++) {
2042           HRegion.openHRegion(subregions[i], null);
2043           subregions[i].compactStores();
2044         }
2045         Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
2046         Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
2047         Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
2048         long startTime = System.currentTimeMillis();
2049         region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
2050         LOG.info("Merge regions elapsed time: "
2051             + ((System.currentTimeMillis() - startTime) / 1000.0));
2052         FILESYSTEM.delete(oldRegion1, true);
2053         FILESYSTEM.delete(oldRegion2, true);
2054         FILESYSTEM.delete(oldRegionPath, true);
2055         LOG.info("splitAndMerge completed.");
2056       } finally {
2057         for (int i = 0; i < subregions.length; i++) {
2058           try {
2059             HRegion.closeHRegion(subregions[i]);
2060           } catch (IOException e) {
2061             // Ignore.
2062           }
2063         }
2064       }
2065     } finally {
2066       HRegion.closeHRegion(this.region);
2067       this.region = null;
2068     }
2069   }
2070 
2071   /**
2072    * @param parent
2073    *          Region to split.
2074    * @param midkey
2075    *          Key to split around.
2076    * @return The Regions we created.
2077    * @throws IOException
2078    */
2079   HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
2080     PairOfSameType<HRegion> result = null;
2081     SplitTransaction st = new SplitTransaction(parent, midkey);
2082     // If prepare does not return true, for some reason -- logged inside in
2083     // the prepare call -- we are not ready to split just now. Just return.
2084     if (!st.prepare()) {
2085       parent.clearSplit();
2086       return null;
2087     }
2088     try {
2089       result = st.execute(null, null);
2090     } catch (IOException ioe) {
2091       try {
2092         LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; "
2093             + ioe.getMessage());
2094         st.rollback(null, null);
2095         LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString());
2096         return null;
2097       } catch (RuntimeException e) {
2098         // If failed rollback, kill this server to avoid having a hole in table.
2099         LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString()
2100             + " -- aborting server", e);
2101       }
2102     }
2103     finally {
2104       parent.clearSplit();
2105     }
2106     return new HRegion[] { result.getFirst(), result.getSecond() };
2107   }
2108 
2109   // ////////////////////////////////////////////////////////////////////////////
2110   // Scanner tests
2111   // ////////////////////////////////////////////////////////////////////////////
2112   @Test
2113   public void testGetScanner_WithOkFamilies() throws IOException {
2114     byte[] fam1 = Bytes.toBytes("fam1");
2115     byte[] fam2 = Bytes.toBytes("fam2");
2116 
2117     byte[][] families = { fam1, fam2 };
2118 
2119     // Setting up region
2120     String method = this.getName();
2121     this.region = initHRegion(tableName, method, CONF, families);
2122     try {
2123       Scan scan = new Scan();
2124       scan.addFamily(fam1);
2125       scan.addFamily(fam2);
2126       try {
2127         region.getScanner(scan);
2128       } catch (Exception e) {
2129         assertTrue("Families could not be found in Region", false);
2130       }
2131     } finally {
2132       HRegion.closeHRegion(this.region);
2133       this.region = null;
2134     }
2135   }
2136 
2137   @Test
2138   public void testGetScanner_WithNotOkFamilies() throws IOException {
2139     byte[] fam1 = Bytes.toBytes("fam1");
2140     byte[] fam2 = Bytes.toBytes("fam2");
2141 
2142     byte[][] families = { fam1 };
2143 
2144     // Setting up region
2145     String method = this.getName();
2146     this.region = initHRegion(tableName, method, CONF, families);
2147     try {
2148       Scan scan = new Scan();
2149       scan.addFamily(fam2);
2150       boolean ok = false;
2151       try {
2152         region.getScanner(scan);
2153       } catch (Exception e) {
2154         ok = true;
2155       }
2156       assertTrue("Families could not be found in Region", ok);
2157     } finally {
2158       HRegion.closeHRegion(this.region);
2159       this.region = null;
2160     }
2161   }
2162 
2163   @Test
2164   public void testGetScanner_WithNoFamilies() throws IOException {
2165     byte[] row1 = Bytes.toBytes("row1");
2166     byte[] fam1 = Bytes.toBytes("fam1");
2167     byte[] fam2 = Bytes.toBytes("fam2");
2168     byte[] fam3 = Bytes.toBytes("fam3");
2169     byte[] fam4 = Bytes.toBytes("fam4");
2170 
2171     byte[][] families = { fam1, fam2, fam3, fam4 };
2172 
2173     // Setting up region
2174     String method = this.getName();
2175     this.region = initHRegion(tableName, method, CONF, families);
2176     try {
2177 
2178       // Putting data in Region
2179       Put put = new Put(row1);
2180       put.add(fam1, null, null);
2181       put.add(fam2, null, null);
2182       put.add(fam3, null, null);
2183       put.add(fam4, null, null);
2184       region.put(put);
2185 
2186       Scan scan = null;
2187       HRegion.RegionScannerImpl is = null;
2188 
2189       // Testing to see how many scanners that is produced by getScanner,
2190       // starting
2191       // with known number, 2 - current = 1
2192       scan = new Scan();
2193       scan.addFamily(fam2);
2194       scan.addFamily(fam4);
2195       is = (RegionScannerImpl) region.getScanner(scan);
2196       assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2197 
2198       scan = new Scan();
2199       is = (RegionScannerImpl) region.getScanner(scan);
2200       assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2201     } finally {
2202       HRegion.closeHRegion(this.region);
2203       this.region = null;
2204     }
2205   }
2206 
2207   /**
2208    * This method tests https://issues.apache.org/jira/browse/HBASE-2516.
2209    * 
2210    * @throws IOException
2211    */
2212   @Test
2213   public void testGetScanner_WithRegionClosed() throws IOException {
2214     byte[] fam1 = Bytes.toBytes("fam1");
2215     byte[] fam2 = Bytes.toBytes("fam2");
2216 
2217     byte[][] families = { fam1, fam2 };
2218 
2219     // Setting up region
2220     String method = this.getName();
2221     try {
2222       this.region = initHRegion(tableName, method, CONF, families);
2223     } catch (IOException e) {
2224       e.printStackTrace();
2225       fail("Got IOException during initHRegion, " + e.getMessage());
2226     }
2227     try {
2228       region.closed.set(true);
2229       try {
2230         region.getScanner(null);
2231         fail("Expected to get an exception during getScanner on a region that is closed");
2232       } catch (NotServingRegionException e) {
2233         // this is the correct exception that is expected
2234       } catch (IOException e) {
2235         fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2236             + e.getMessage());
2237       }
2238     } finally {
2239       HRegion.closeHRegion(this.region);
2240       this.region = null;
2241     }
2242   }
2243 
2244   @Test
2245   public void testRegionScanner_Next() throws IOException {
2246     byte[] row1 = Bytes.toBytes("row1");
2247     byte[] row2 = Bytes.toBytes("row2");
2248     byte[] fam1 = Bytes.toBytes("fam1");
2249     byte[] fam2 = Bytes.toBytes("fam2");
2250     byte[] fam3 = Bytes.toBytes("fam3");
2251     byte[] fam4 = Bytes.toBytes("fam4");
2252 
2253     byte[][] families = { fam1, fam2, fam3, fam4 };
2254     long ts = System.currentTimeMillis();
2255 
2256     // Setting up region
2257     String method = this.getName();
2258     this.region = initHRegion(tableName, method, CONF, families);
2259     try {
2260       // Putting data in Region
2261       Put put = null;
2262       put = new Put(row1);
2263       put.add(fam1, (byte[]) null, ts, null);
2264       put.add(fam2, (byte[]) null, ts, null);
2265       put.add(fam3, (byte[]) null, ts, null);
2266       put.add(fam4, (byte[]) null, ts, null);
2267       region.put(put);
2268 
2269       put = new Put(row2);
2270       put.add(fam1, (byte[]) null, ts, null);
2271       put.add(fam2, (byte[]) null, ts, null);
2272       put.add(fam3, (byte[]) null, ts, null);
2273       put.add(fam4, (byte[]) null, ts, null);
2274       region.put(put);
2275 
2276       Scan scan = new Scan();
2277       scan.addFamily(fam2);
2278       scan.addFamily(fam4);
2279       InternalScanner is = region.getScanner(scan);
2280 
2281       List<Cell> res = null;
2282 
2283       // Result 1
2284       List<Cell> expected1 = new ArrayList<Cell>();
2285       expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2286       expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2287 
2288       res = new ArrayList<Cell>();
2289       is.next(res);
2290       for (int i = 0; i < res.size(); i++) {
2291         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2292       }
2293 
2294       // Result 2
2295       List<Cell> expected2 = new ArrayList<Cell>();
2296       expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2297       expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2298 
2299       res = new ArrayList<Cell>();
2300       is.next(res);
2301       for (int i = 0; i < res.size(); i++) {
2302         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2303       }
2304     } finally {
2305       HRegion.closeHRegion(this.region);
2306       this.region = null;
2307     }
2308   }
2309 
2310   @Test
2311   public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2312     byte[] row1 = Bytes.toBytes("row1");
2313     byte[] qf1 = Bytes.toBytes("qualifier1");
2314     byte[] qf2 = Bytes.toBytes("qualifier2");
2315     byte[] fam1 = Bytes.toBytes("fam1");
2316     byte[][] families = { fam1 };
2317 
2318     long ts1 = System.currentTimeMillis();
2319     long ts2 = ts1 + 1;
2320     long ts3 = ts1 + 2;
2321 
2322     // Setting up region
2323     String method = this.getName();
2324     this.region = initHRegion(tableName, method, CONF, families);
2325     try {
2326       // Putting data in Region
2327       Put put = null;
2328       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2329       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2330       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2331 
2332       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2333       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2334       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2335 
2336       put = new Put(row1);
2337       put.add(kv13);
2338       put.add(kv12);
2339       put.add(kv11);
2340       put.add(kv23);
2341       put.add(kv22);
2342       put.add(kv21);
2343       region.put(put);
2344 
2345       // Expected
2346       List<Cell> expected = new ArrayList<Cell>();
2347       expected.add(kv13);
2348       expected.add(kv12);
2349 
2350       Scan scan = new Scan(row1);
2351       scan.addColumn(fam1, qf1);
2352       scan.setMaxVersions(MAX_VERSIONS);
2353       List<Cell> actual = new ArrayList<Cell>();
2354       InternalScanner scanner = region.getScanner(scan);
2355 
2356       boolean hasNext = scanner.next(actual);
2357       assertEquals(false, hasNext);
2358 
2359       // Verify result
2360       for (int i = 0; i < expected.size(); i++) {
2361         assertEquals(expected.get(i), actual.get(i));
2362       }
2363     } finally {
2364       HRegion.closeHRegion(this.region);
2365       this.region = null;
2366     }
2367   }
2368 
2369   @Test
2370   public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2371     byte[] row1 = Bytes.toBytes("row1");
2372     byte[] qf1 = Bytes.toBytes("qualifier1");
2373     byte[] qf2 = Bytes.toBytes("qualifier2");
2374     byte[] fam1 = Bytes.toBytes("fam1");
2375     byte[][] families = { fam1 };
2376 
2377     long ts1 = 1; // System.currentTimeMillis();
2378     long ts2 = ts1 + 1;
2379     long ts3 = ts1 + 2;
2380 
2381     // Setting up region
2382     String method = this.getName();
2383     this.region = initHRegion(tableName, method, CONF, families);
2384     try {
2385       // Putting data in Region
2386       Put put = null;
2387       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2388       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2389       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2390 
2391       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2392       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2393       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2394 
2395       put = new Put(row1);
2396       put.add(kv13);
2397       put.add(kv12);
2398       put.add(kv11);
2399       put.add(kv23);
2400       put.add(kv22);
2401       put.add(kv21);
2402       region.put(put);
2403       region.flushcache();
2404 
2405       // Expected
2406       List<Cell> expected = new ArrayList<Cell>();
2407       expected.add(kv13);
2408       expected.add(kv12);
2409       expected.add(kv23);
2410       expected.add(kv22);
2411 
2412       Scan scan = new Scan(row1);
2413       scan.addColumn(fam1, qf1);
2414       scan.addColumn(fam1, qf2);
2415       scan.setMaxVersions(MAX_VERSIONS);
2416       List<Cell> actual = new ArrayList<Cell>();
2417       InternalScanner scanner = region.getScanner(scan);
2418 
2419       boolean hasNext = scanner.next(actual);
2420       assertEquals(false, hasNext);
2421 
2422       // Verify result
2423       for (int i = 0; i < expected.size(); i++) {
2424         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2425       }
2426     } finally {
2427       HRegion.closeHRegion(this.region);
2428       this.region = null;
2429     }
2430   }
2431 
2432   @Test
2433   public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2434     byte[] row1 = Bytes.toBytes("row1");
2435     byte[] fam1 = Bytes.toBytes("fam1");
2436     byte[][] families = { fam1 };
2437     byte[] qf1 = Bytes.toBytes("qualifier1");
2438     byte[] qf2 = Bytes.toBytes("qualifier2");
2439 
2440     long ts1 = 1;
2441     long ts2 = ts1 + 1;
2442     long ts3 = ts1 + 2;
2443     long ts4 = ts1 + 3;
2444 
2445     // Setting up region
2446     String method = this.getName();
2447     this.region = initHRegion(tableName, method, CONF, families);
2448     try {
2449       // Putting data in Region
2450       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2451       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2452       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2453       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2454 
2455       KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2456       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2457       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2458       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2459 
2460       Put put = null;
2461       put = new Put(row1);
2462       put.add(kv14);
2463       put.add(kv24);
2464       region.put(put);
2465       region.flushcache();
2466 
2467       put = new Put(row1);
2468       put.add(kv23);
2469       put.add(kv13);
2470       region.put(put);
2471       region.flushcache();
2472 
2473       put = new Put(row1);
2474       put.add(kv22);
2475       put.add(kv12);
2476       region.put(put);
2477       region.flushcache();
2478 
2479       put = new Put(row1);
2480       put.add(kv21);
2481       put.add(kv11);
2482       region.put(put);
2483 
2484       // Expected
2485       List<Cell> expected = new ArrayList<Cell>();
2486       expected.add(kv14);
2487       expected.add(kv13);
2488       expected.add(kv12);
2489       expected.add(kv24);
2490       expected.add(kv23);
2491       expected.add(kv22);
2492 
2493       Scan scan = new Scan(row1);
2494       scan.addColumn(fam1, qf1);
2495       scan.addColumn(fam1, qf2);
2496       int versions = 3;
2497       scan.setMaxVersions(versions);
2498       List<Cell> actual = new ArrayList<Cell>();
2499       InternalScanner scanner = region.getScanner(scan);
2500 
2501       boolean hasNext = scanner.next(actual);
2502       assertEquals(false, hasNext);
2503 
2504       // Verify result
2505       for (int i = 0; i < expected.size(); i++) {
2506         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2507       }
2508     } finally {
2509       HRegion.closeHRegion(this.region);
2510       this.region = null;
2511     }
2512   }
2513 
2514   @Test
2515   public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2516     byte[] row1 = Bytes.toBytes("row1");
2517     byte[] qf1 = Bytes.toBytes("qualifier1");
2518     byte[] qf2 = Bytes.toBytes("qualifier2");
2519     byte[] fam1 = Bytes.toBytes("fam1");
2520     byte[][] families = { fam1 };
2521 
2522     long ts1 = System.currentTimeMillis();
2523     long ts2 = ts1 + 1;
2524     long ts3 = ts1 + 2;
2525 
2526     // Setting up region
2527     String method = this.getName();
2528     this.region = initHRegion(tableName, method, CONF, families);
2529     try {
2530       // Putting data in Region
2531       Put put = null;
2532       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2533       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2534       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2535 
2536       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2537       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2538       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2539 
2540       put = new Put(row1);
2541       put.add(kv13);
2542       put.add(kv12);
2543       put.add(kv11);
2544       put.add(kv23);
2545       put.add(kv22);
2546       put.add(kv21);
2547       region.put(put);
2548 
2549       // Expected
2550       List<Cell> expected = new ArrayList<Cell>();
2551       expected.add(kv13);
2552       expected.add(kv12);
2553       expected.add(kv23);
2554       expected.add(kv22);
2555 
2556       Scan scan = new Scan(row1);
2557       scan.addFamily(fam1);
2558       scan.setMaxVersions(MAX_VERSIONS);
2559       List<Cell> actual = new ArrayList<Cell>();
2560       InternalScanner scanner = region.getScanner(scan);
2561 
2562       boolean hasNext = scanner.next(actual);
2563       assertEquals(false, hasNext);
2564 
2565       // Verify result
2566       for (int i = 0; i < expected.size(); i++) {
2567         assertEquals(expected.get(i), actual.get(i));
2568       }
2569     } finally {
2570       HRegion.closeHRegion(this.region);
2571       this.region = null;
2572     }
2573   }
2574 
2575   @Test
2576   public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
2577     byte[] row1 = Bytes.toBytes("row1");
2578     byte[] qf1 = Bytes.toBytes("qualifier1");
2579     byte[] qf2 = Bytes.toBytes("qualifier2");
2580     byte[] fam1 = Bytes.toBytes("fam1");
2581 
2582     long ts1 = 1; // System.currentTimeMillis();
2583     long ts2 = ts1 + 1;
2584     long ts3 = ts1 + 2;
2585 
2586     // Setting up region
2587     String method = this.getName();
2588     this.region = initHRegion(tableName, method, CONF, fam1);
2589     try {
2590       // Putting data in Region
2591       Put put = null;
2592       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2593       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2594       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2595 
2596       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2597       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2598       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2599 
2600       put = new Put(row1);
2601       put.add(kv13);
2602       put.add(kv12);
2603       put.add(kv11);
2604       put.add(kv23);
2605       put.add(kv22);
2606       put.add(kv21);
2607       region.put(put);
2608       region.flushcache();
2609 
2610       // Expected
2611       List<Cell> expected = new ArrayList<Cell>();
2612       expected.add(kv13);
2613       expected.add(kv12);
2614       expected.add(kv23);
2615       expected.add(kv22);
2616 
2617       Scan scan = new Scan(row1);
2618       scan.addFamily(fam1);
2619       scan.setMaxVersions(MAX_VERSIONS);
2620       List<Cell> actual = new ArrayList<Cell>();
2621       InternalScanner scanner = region.getScanner(scan);
2622 
2623       boolean hasNext = scanner.next(actual);
2624       assertEquals(false, hasNext);
2625 
2626       // Verify result
2627       for (int i = 0; i < expected.size(); i++) {
2628         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2629       }
2630     } finally {
2631       HRegion.closeHRegion(this.region);
2632       this.region = null;
2633     }
2634   }
2635 
2636   @Test
2637   public void testScanner_StopRow1542() throws IOException {
2638     byte[] family = Bytes.toBytes("testFamily");
2639     this.region = initHRegion(tableName, getName(), CONF, family);
2640     try {
2641       byte[] row1 = Bytes.toBytes("row111");
2642       byte[] row2 = Bytes.toBytes("row222");
2643       byte[] row3 = Bytes.toBytes("row333");
2644       byte[] row4 = Bytes.toBytes("row444");
2645       byte[] row5 = Bytes.toBytes("row555");
2646 
2647       byte[] col1 = Bytes.toBytes("Pub111");
2648       byte[] col2 = Bytes.toBytes("Pub222");
2649 
2650       Put put = new Put(row1);
2651       put.add(family, col1, Bytes.toBytes(10L));
2652       region.put(put);
2653 
2654       put = new Put(row2);
2655       put.add(family, col1, Bytes.toBytes(15L));
2656       region.put(put);
2657 
2658       put = new Put(row3);
2659       put.add(family, col2, Bytes.toBytes(20L));
2660       region.put(put);
2661 
2662       put = new Put(row4);
2663       put.add(family, col2, Bytes.toBytes(30L));
2664       region.put(put);
2665 
2666       put = new Put(row5);
2667       put.add(family, col1, Bytes.toBytes(40L));
2668       region.put(put);
2669 
2670       Scan scan = new Scan(row3, row4);
2671       scan.setMaxVersions();
2672       scan.addColumn(family, col1);
2673       InternalScanner s = region.getScanner(scan);
2674 
2675       List<Cell> results = new ArrayList<Cell>();
2676       assertEquals(false, s.next(results));
2677       assertEquals(0, results.size());
2678     } finally {
2679       HRegion.closeHRegion(this.region);
2680       this.region = null;
2681     }
2682   }
2683 
2684   @Test
2685   public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2686     byte[] row1 = Bytes.toBytes("row1");
2687     byte[] fam1 = Bytes.toBytes("fam1");
2688     byte[] qf1 = Bytes.toBytes("qualifier1");
2689     byte[] qf2 = Bytes.toBytes("quateslifier2");
2690 
2691     long ts1 = 1;
2692     long ts2 = ts1 + 1;
2693     long ts3 = ts1 + 2;
2694     long ts4 = ts1 + 3;
2695 
2696     // Setting up region
2697     String method = this.getName();
2698     this.region = initHRegion(tableName, method, CONF, fam1);
2699     try {
2700       // Putting data in Region
2701       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2702       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2703       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2704       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2705 
2706       KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2707       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2708       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2709       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2710 
2711       Put put = null;
2712       put = new Put(row1);
2713       put.add(kv14);
2714       put.add(kv24);
2715       region.put(put);
2716       region.flushcache();
2717 
2718       put = new Put(row1);
2719       put.add(kv23);
2720       put.add(kv13);
2721       region.put(put);
2722       region.flushcache();
2723 
2724       put = new Put(row1);
2725       put.add(kv22);
2726       put.add(kv12);
2727       region.put(put);
2728       region.flushcache();
2729 
2730       put = new Put(row1);
2731       put.add(kv21);
2732       put.add(kv11);
2733       region.put(put);
2734 
2735       // Expected
2736       List<KeyValue> expected = new ArrayList<KeyValue>();
2737       expected.add(kv14);
2738       expected.add(kv13);
2739       expected.add(kv12);
2740       expected.add(kv24);
2741       expected.add(kv23);
2742       expected.add(kv22);
2743 
2744       Scan scan = new Scan(row1);
2745       int versions = 3;
2746       scan.setMaxVersions(versions);
2747       List<Cell> actual = new ArrayList<Cell>();
2748       InternalScanner scanner = region.getScanner(scan);
2749 
2750       boolean hasNext = scanner.next(actual);
2751       assertEquals(false, hasNext);
2752 
2753       // Verify result
2754       for (int i = 0; i < expected.size(); i++) {
2755         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2756       }
2757     } finally {
2758       HRegion.closeHRegion(this.region);
2759       this.region = null;
2760     }
2761   }
2762 
2763   /**
2764    * Added for HBASE-5416
2765    * 
2766    * Here we test scan optimization when only subset of CFs are used in filter
2767    * conditions.
2768    */
2769   @Test
2770   public void testScanner_JoinedScanners() throws IOException {
2771     byte[] cf_essential = Bytes.toBytes("essential");
2772     byte[] cf_joined = Bytes.toBytes("joined");
2773     byte[] cf_alpha = Bytes.toBytes("alpha");
2774     this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
2775     try {
2776       byte[] row1 = Bytes.toBytes("row1");
2777       byte[] row2 = Bytes.toBytes("row2");
2778       byte[] row3 = Bytes.toBytes("row3");
2779 
2780       byte[] col_normal = Bytes.toBytes("d");
2781       byte[] col_alpha = Bytes.toBytes("a");
2782 
2783       byte[] filtered_val = Bytes.toBytes(3);
2784 
2785       Put put = new Put(row1);
2786       put.add(cf_essential, col_normal, Bytes.toBytes(1));
2787       put.add(cf_joined, col_alpha, Bytes.toBytes(1));
2788       region.put(put);
2789 
2790       put = new Put(row2);
2791       put.add(cf_essential, col_alpha, Bytes.toBytes(2));
2792       put.add(cf_joined, col_normal, Bytes.toBytes(2));
2793       put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
2794       region.put(put);
2795 
2796       put = new Put(row3);
2797       put.add(cf_essential, col_normal, filtered_val);
2798       put.add(cf_joined, col_normal, filtered_val);
2799       region.put(put);
2800 
2801       // Check two things:
2802       // 1. result list contains expected values
2803       // 2. result list is sorted properly
2804 
2805       Scan scan = new Scan();
2806       Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
2807           CompareOp.NOT_EQUAL, filtered_val);
2808       scan.setFilter(filter);
2809       scan.setLoadColumnFamiliesOnDemand(true);
2810       InternalScanner s = region.getScanner(scan);
2811 
2812       List<Cell> results = new ArrayList<Cell>();
2813       assertTrue(s.next(results));
2814       assertEquals(results.size(), 1);
2815       results.clear();
2816 
2817       assertTrue(s.next(results));
2818       assertEquals(results.size(), 3);
2819       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
2820       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
2821       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
2822       results.clear();
2823 
2824       assertFalse(s.next(results));
2825       assertEquals(results.size(), 0);
2826     } finally {
2827       HRegion.closeHRegion(this.region);
2828       this.region = null;
2829     }
2830   }
2831 
2832   /**
2833    * HBASE-5416
2834    * 
2835    * Test case when scan limits amount of KVs returned on each next() call.
2836    */
2837   @Test
2838   public void testScanner_JoinedScannersWithLimits() throws IOException {
2839     final byte[] cf_first = Bytes.toBytes("first");
2840     final byte[] cf_second = Bytes.toBytes("second");
2841 
2842     this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
2843     try {
2844       final byte[] col_a = Bytes.toBytes("a");
2845       final byte[] col_b = Bytes.toBytes("b");
2846 
2847       Put put;
2848 
2849       for (int i = 0; i < 10; i++) {
2850         put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
2851         put.add(cf_first, col_a, Bytes.toBytes(i));
2852         if (i < 5) {
2853           put.add(cf_first, col_b, Bytes.toBytes(i));
2854           put.add(cf_second, col_a, Bytes.toBytes(i));
2855           put.add(cf_second, col_b, Bytes.toBytes(i));
2856         }
2857         region.put(put);
2858       }
2859 
2860       Scan scan = new Scan();
2861       scan.setLoadColumnFamiliesOnDemand(true);
2862       Filter bogusFilter = new FilterBase() {
2863         @Override
2864         public boolean isFamilyEssential(byte[] name) {
2865           return Bytes.equals(name, cf_first);
2866         }
2867       };
2868 
2869       scan.setFilter(bogusFilter);
2870       InternalScanner s = region.getScanner(scan);
2871 
2872       // Our data looks like this:
2873       // r0: first:a, first:b, second:a, second:b
2874       // r1: first:a, first:b, second:a, second:b
2875       // r2: first:a, first:b, second:a, second:b
2876       // r3: first:a, first:b, second:a, second:b
2877       // r4: first:a, first:b, second:a, second:b
2878       // r5: first:a
2879       // r6: first:a
2880       // r7: first:a
2881       // r8: first:a
2882       // r9: first:a
2883 
2884       // But due to next's limit set to 3, we should get this:
2885       // r0: first:a, first:b, second:a
2886       // r0: second:b
2887       // r1: first:a, first:b, second:a
2888       // r1: second:b
2889       // r2: first:a, first:b, second:a
2890       // r2: second:b
2891       // r3: first:a, first:b, second:a
2892       // r3: second:b
2893       // r4: first:a, first:b, second:a
2894       // r4: second:b
2895       // r5: first:a
2896       // r6: first:a
2897       // r7: first:a
2898       // r8: first:a
2899       // r9: first:a
2900 
2901       List<Cell> results = new ArrayList<Cell>();
2902       int index = 0;
2903       while (true) {
2904         boolean more = s.next(results, 3);
2905         if ((index >> 1) < 5) {
2906           if (index % 2 == 0)
2907             assertEquals(results.size(), 3);
2908           else
2909             assertEquals(results.size(), 1);
2910         } else
2911           assertEquals(results.size(), 1);
2912         results.clear();
2913         index++;
2914         if (!more)
2915           break;
2916       }
2917     } finally {
2918       HRegion.closeHRegion(this.region);
2919       this.region = null;
2920     }
2921   }
2922 
2923   // ////////////////////////////////////////////////////////////////////////////
2924   // Split test
2925   // ////////////////////////////////////////////////////////////////////////////
2926   /**
2927    * Splits twice and verifies getting from each of the split regions.
2928    * 
2929    * @throws Exception
2930    */
2931   @Test
2932   public void testBasicSplit() throws Exception {
2933     byte[][] families = { fam1, fam2, fam3 };
2934 
2935     Configuration hc = initSplit();
2936     // Setting up region
2937     String method = this.getName();
2938     this.region = initHRegion(tableName, method, hc, families);
2939 
2940     try {
2941       LOG.info("" + HBaseTestCase.addContent(region, fam3));
2942       region.flushcache();
2943       region.compactStores();
2944       byte[] splitRow = region.checkSplit();
2945       assertNotNull(splitRow);
2946       LOG.info("SplitRow: " + Bytes.toString(splitRow));
2947       HRegion[] regions = splitRegion(region, splitRow);
2948       try {
2949         // Need to open the regions.
2950         // TODO: Add an 'open' to HRegion... don't do open by constructing
2951         // instance.
2952         for (int i = 0; i < regions.length; i++) {
2953           regions[i] = HRegion.openHRegion(regions[i], null);
2954         }
2955         // Assert can get rows out of new regions. Should be able to get first
2956         // row from first region and the midkey from second region.
2957         assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
2958         assertGet(regions[1], fam3, splitRow);
2959         // Test I can get scanner and that it starts at right place.
2960         assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
2961         assertScan(regions[1], fam3, splitRow);
2962         // Now prove can't split regions that have references.
2963         for (int i = 0; i < regions.length; i++) {
2964           // Add so much data to this region, we create a store file that is >
2965           // than one of our unsplitable references. it will.
2966           for (int j = 0; j < 2; j++) {
2967             HBaseTestCase.addContent(regions[i], fam3);
2968           }
2969           HBaseTestCase.addContent(regions[i], fam2);
2970           HBaseTestCase.addContent(regions[i], fam1);
2971           regions[i].flushcache();
2972         }
2973 
2974         byte[][] midkeys = new byte[regions.length][];
2975         // To make regions splitable force compaction.
2976         for (int i = 0; i < regions.length; i++) {
2977           regions[i].compactStores();
2978           midkeys[i] = regions[i].checkSplit();
2979         }
2980 
2981         TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
2982         // Split these two daughter regions so then I'll have 4 regions. Will
2983         // split because added data above.
2984         for (int i = 0; i < regions.length; i++) {
2985           HRegion[] rs = null;
2986           if (midkeys[i] != null) {
2987             rs = splitRegion(regions[i], midkeys[i]);
2988             for (int j = 0; j < rs.length; j++) {
2989               sortedMap.put(Bytes.toString(rs[j].getRegionName()), HRegion.openHRegion(rs[j], null));
2990             }
2991           }
2992         }
2993         LOG.info("Made 4 regions");
2994         // The splits should have been even. Test I can get some arbitrary row
2995         // out of each.
2996         int interval = (LAST_CHAR - FIRST_CHAR) / 3;
2997         byte[] b = Bytes.toBytes(START_KEY);
2998         for (HRegion r : sortedMap.values()) {
2999           assertGet(r, fam3, b);
3000           b[0] += interval;
3001         }
3002       } finally {
3003         for (int i = 0; i < regions.length; i++) {
3004           try {
3005             regions[i].close();
3006           } catch (IOException e) {
3007             // Ignore.
3008           }
3009         }
3010       }
3011     } finally {
3012       HRegion.closeHRegion(this.region);
3013       this.region = null;
3014     }
3015   }
3016 
3017   @Test
3018   public void testSplitRegion() throws IOException {
3019     byte[] qualifier = Bytes.toBytes("qualifier");
3020     Configuration hc = initSplit();
3021     int numRows = 10;
3022     byte[][] families = { fam1, fam3 };
3023 
3024     // Setting up region
3025     String method = this.getName();
3026     this.region = initHRegion(tableName, method, hc, families);
3027 
3028     // Put data in region
3029     int startRow = 100;
3030     putData(startRow, numRows, qualifier, families);
3031     int splitRow = startRow + numRows;
3032     putData(splitRow, numRows, qualifier, families);
3033     region.flushcache();
3034 
3035     HRegion[] regions = null;
3036     try {
3037       regions = splitRegion(region, Bytes.toBytes("" + splitRow));
3038       // Opening the regions returned.
3039       for (int i = 0; i < regions.length; i++) {
3040         regions[i] = HRegion.openHRegion(regions[i], null);
3041       }
3042       // Verifying that the region has been split
3043       assertEquals(2, regions.length);
3044 
3045       // Verifying that all data is still there and that data is in the right
3046       // place
3047       verifyData(regions[0], startRow, numRows, qualifier, families);
3048       verifyData(regions[1], splitRow, numRows, qualifier, families);
3049 
3050     } finally {
3051       HRegion.closeHRegion(this.region);
3052       this.region = null;
3053     }
3054   }
3055 
3056   @Test
3057   public void testClearForceSplit() throws IOException {
3058     byte[] qualifier = Bytes.toBytes("qualifier");
3059     Configuration hc = initSplit();
3060     int numRows = 10;
3061     byte[][] families = { fam1, fam3 };
3062 
3063     // Setting up region
3064     String method = this.getName();
3065     this.region = initHRegion(tableName, method, hc, families);
3066 
3067     // Put data in region
3068     int startRow = 100;
3069     putData(startRow, numRows, qualifier, families);
3070     int splitRow = startRow + numRows;
3071     byte[] splitRowBytes = Bytes.toBytes("" + splitRow);
3072     putData(splitRow, numRows, qualifier, families);
3073     region.flushcache();
3074 
3075     HRegion[] regions = null;
3076     try {
3077       // Set force split
3078       region.forceSplit(splitRowBytes);
3079       assertTrue(region.shouldForceSplit());
3080       // Split point should be the force split row
3081       assertTrue(Bytes.equals(splitRowBytes, region.checkSplit()));
3082 
3083       // Add a store that has references.
3084       HStore storeMock = Mockito.mock(HStore.class);
3085       Mockito.when(storeMock.hasReferences()).thenReturn(true);
3086       Mockito.when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
3087       Mockito.when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
3088       Mockito.when(storeMock.getColumnFamilyName()).thenReturn("cf");
3089       region.stores.put(Bytes.toBytes(storeMock.getColumnFamilyName()), storeMock);
3090       assertTrue(region.hasReferences());
3091 
3092       // Will not split since the store has references.
3093       regions = splitRegion(region, splitRowBytes);
3094       assertNull(regions);
3095 
3096       // Region force split should be cleared after the split try.
3097       assertFalse(region.shouldForceSplit());
3098 
3099       // Remove the store that has references.
3100       region.stores.remove(Bytes.toBytes(storeMock.getColumnFamilyName()));
3101       assertFalse(region.hasReferences());
3102 
3103       // Now we can split.
3104       regions = splitRegion(region, splitRowBytes);
3105 
3106       // Opening the regions returned.
3107       for (int i = 0; i < regions.length; i++) {
3108         regions[i] = HRegion.openHRegion(regions[i], null);
3109       }
3110       // Verifying that the region has been split
3111       assertEquals(2, regions.length);
3112 
3113       // Verifying that all data is still there and that data is in the right
3114       // place
3115       verifyData(regions[0], startRow, numRows, qualifier, families);
3116       verifyData(regions[1], splitRow, numRows, qualifier, families);
3117 
3118     } finally {
3119       HRegion.closeHRegion(this.region);
3120       this.region = null;
3121     }
3122   }
3123 
3124   /**
3125    * Flushes the cache in a thread while scanning. The tests verify that the
3126    * scan is coherent - e.g. the returned results are always of the same or
3127    * later update as the previous results.
3128    * 
3129    * @throws IOException
3130    *           scan / compact
3131    * @throws InterruptedException
3132    *           thread join
3133    */
3134   @Test
3135   public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3136     byte[] family = Bytes.toBytes("family");
3137     int numRows = 1000;
3138     int flushAndScanInterval = 10;
3139     int compactInterval = 10 * flushAndScanInterval;
3140 
3141     String method = "testFlushCacheWhileScanning";
3142     this.region = initHRegion(tableName, method, CONF, family);
3143     try {
3144       FlushThread flushThread = new FlushThread();
3145       flushThread.start();
3146 
3147       Scan scan = new Scan();
3148       scan.addFamily(family);
3149       scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
3150           new BinaryComparator(Bytes.toBytes(5L))));
3151 
3152       int expectedCount = 0;
3153       List<Cell> res = new ArrayList<Cell>();
3154 
3155       boolean toggle = true;
3156       for (long i = 0; i < numRows; i++) {
3157         Put put = new Put(Bytes.toBytes(i));
3158         put.setDurability(Durability.SKIP_WAL);
3159         put.add(family, qual1, Bytes.toBytes(i % 10));
3160         region.put(put);
3161 
3162         if (i != 0 && i % compactInterval == 0) {
3163           // System.out.println("iteration = " + i);
3164           region.compactStores(true);
3165         }
3166 
3167         if (i % 10 == 5L) {
3168           expectedCount++;
3169         }
3170 
3171         if (i != 0 && i % flushAndScanInterval == 0) {
3172           res.clear();
3173           InternalScanner scanner = region.getScanner(scan);
3174           if (toggle) {
3175             flushThread.flush();
3176           }
3177           while (scanner.next(res))
3178             ;
3179           if (!toggle) {
3180             flushThread.flush();
3181           }
3182           assertEquals("i=" + i, expectedCount, res.size());
3183           toggle = !toggle;
3184         }
3185       }
3186 
3187       flushThread.done();
3188       flushThread.join();
3189       flushThread.checkNoError();
3190     } finally {
3191       HRegion.closeHRegion(this.region);
3192       this.region = null;
3193     }
3194   }
3195 
3196   protected class FlushThread extends Thread {
3197     private volatile boolean done;
3198     private Throwable error = null;
3199 
3200     public void done() {
3201       done = true;
3202       synchronized (this) {
3203         interrupt();
3204       }
3205     }
3206 
3207     public void checkNoError() {
3208       if (error != null) {
3209         assertNull(error);
3210       }
3211     }
3212 
3213     @Override
3214     public void run() {
3215       done = false;
3216       while (!done) {
3217         synchronized (this) {
3218           try {
3219             wait();
3220           } catch (InterruptedException ignored) {
3221             if (done) {
3222               break;
3223             }
3224           }
3225         }
3226         try {
3227           region.flushcache();
3228         } catch (IOException e) {
3229           if (!done) {
3230             LOG.error("Error while flusing cache", e);
3231             error = e;
3232           }
3233           break;
3234         }
3235       }
3236 
3237     }
3238 
3239     public void flush() {
3240       synchronized (this) {
3241         notify();
3242       }
3243 
3244     }
3245   }
3246 
3247   /**
3248    * Writes very wide records and scans for the latest every time.. Flushes and
3249    * compacts the region every now and then to keep things realistic.
3250    * 
3251    * @throws IOException
3252    *           by flush / scan / compaction
3253    * @throws InterruptedException
3254    *           when joining threads
3255    */
3256   @Test
3257   public void testWritesWhileScanning() throws IOException, InterruptedException {
3258     int testCount = 100;
3259     int numRows = 1;
3260     int numFamilies = 10;
3261     int numQualifiers = 100;
3262     int flushInterval = 7;
3263     int compactInterval = 5 * flushInterval;
3264     byte[][] families = new byte[numFamilies][];
3265     for (int i = 0; i < numFamilies; i++) {
3266       families[i] = Bytes.toBytes("family" + i);
3267     }
3268     byte[][] qualifiers = new byte[numQualifiers][];
3269     for (int i = 0; i < numQualifiers; i++) {
3270       qualifiers[i] = Bytes.toBytes("qual" + i);
3271     }
3272 
3273     String method = "testWritesWhileScanning";
3274     this.region = initHRegion(tableName, method, CONF, families);
3275     try {
3276       PutThread putThread = new PutThread(numRows, families, qualifiers);
3277       putThread.start();
3278       putThread.waitForFirstPut();
3279 
3280       FlushThread flushThread = new FlushThread();
3281       flushThread.start();
3282 
3283       Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3284 
3285       int expectedCount = numFamilies * numQualifiers;
3286       List<Cell> res = new ArrayList<Cell>();
3287 
3288       long prevTimestamp = 0L;
3289       for (int i = 0; i < testCount; i++) {
3290 
3291         if (i != 0 && i % compactInterval == 0) {
3292           region.compactStores(true);
3293         }
3294 
3295         if (i != 0 && i % flushInterval == 0) {
3296           flushThread.flush();
3297         }
3298 
3299         boolean previousEmpty = res.isEmpty();
3300         res.clear();
3301         InternalScanner scanner = region.getScanner(scan);
3302         while (scanner.next(res))
3303           ;
3304         if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3305           assertEquals("i=" + i, expectedCount, res.size());
3306           long timestamp = res.get(0).getTimestamp();
3307           assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3308               timestamp >= prevTimestamp);
3309           prevTimestamp = timestamp;
3310         }
3311       }
3312 
3313       putThread.done();
3314 
3315       region.flushcache();
3316 
3317       putThread.join();
3318       putThread.checkNoError();
3319 
3320       flushThread.done();
3321       flushThread.join();
3322       flushThread.checkNoError();
3323     } finally {
3324       try {
3325         HRegion.closeHRegion(this.region);
3326       } catch (DroppedSnapshotException dse) {
3327         // We could get this on way out because we interrupt the background flusher and it could
3328         // fail anywhere causing a DSE over in the background flusher... only it is not properly
3329         // dealt with so could still be memory hanging out when we get to here -- memory we can't
3330         // flush because the accounting is 'off' since original DSE.
3331       }
3332       this.region = null;
3333     }
3334   }
3335 
3336   protected class PutThread extends Thread {
3337     private volatile boolean done;
3338     private volatile int numPutsFinished = 0;
3339 
3340     private Throwable error = null;
3341     private int numRows;
3342     private byte[][] families;
3343     private byte[][] qualifiers;
3344 
3345     private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3346       this.numRows = numRows;
3347       this.families = families;
3348       this.qualifiers = qualifiers;
3349     }
3350 
3351     /**
3352      * Block until this thread has put at least one row.
3353      */
3354     public void waitForFirstPut() throws InterruptedException {
3355       // wait until put thread actually puts some data
3356       while (numPutsFinished == 0) {
3357         checkNoError();
3358         Thread.sleep(50);
3359       }
3360     }
3361 
3362     public void done() {
3363       done = true;
3364       synchronized (this) {
3365         interrupt();
3366       }
3367     }
3368 
3369     public void checkNoError() {
3370       if (error != null) {
3371         assertNull(error);
3372       }
3373     }
3374 
3375     @Override
3376     public void run() {
3377       done = false;
3378       while (!done) {
3379         try {
3380           for (int r = 0; r < numRows; r++) {
3381             byte[] row = Bytes.toBytes("row" + r);
3382             Put put = new Put(row);
3383             put.setDurability(Durability.SKIP_WAL);
3384             byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3385             for (byte[] family : families) {
3386               for (byte[] qualifier : qualifiers) {
3387                 put.add(family, qualifier, (long) numPutsFinished, value);
3388               }
3389             }
3390             region.put(put);
3391             numPutsFinished++;
3392             if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3393               System.out.println("put iteration = " + numPutsFinished);
3394               Delete delete = new Delete(row, (long) numPutsFinished - 30);
3395               region.delete(delete);
3396             }
3397             numPutsFinished++;
3398           }
3399         } catch (InterruptedIOException e) {
3400           // This is fine. It means we are done, or didn't get the lock on time
3401         } catch (IOException e) {
3402           LOG.error("error while putting records", e);
3403           error = e;
3404           break;
3405         }
3406       }
3407 
3408     }
3409 
3410   }
3411 
3412   /**
3413    * Writes very wide records and gets the latest row every time.. Flushes and
3414    * compacts the region aggressivly to catch issues.
3415    * 
3416    * @throws IOException
3417    *           by flush / scan / compaction
3418    * @throws InterruptedException
3419    *           when joining threads
3420    */
3421   @Test
3422   public void testWritesWhileGetting() throws Exception {
3423     int testCount = 100;
3424     int numRows = 1;
3425     int numFamilies = 10;
3426     int numQualifiers = 100;
3427     int compactInterval = 100;
3428     byte[][] families = new byte[numFamilies][];
3429     for (int i = 0; i < numFamilies; i++) {
3430       families[i] = Bytes.toBytes("family" + i);
3431     }
3432     byte[][] qualifiers = new byte[numQualifiers][];
3433     for (int i = 0; i < numQualifiers; i++) {
3434       qualifiers[i] = Bytes.toBytes("qual" + i);
3435     }
3436 
3437 
3438     String method = "testWritesWhileGetting";
3439     // This test flushes constantly and can cause many files to be created,
3440     // possibly
3441     // extending over the ulimit. Make sure compactions are aggressive in
3442     // reducing
3443     // the number of HFiles created.
3444     Configuration conf = HBaseConfiguration.create(CONF);
3445     conf.setInt("hbase.hstore.compaction.min", 1);
3446     conf.setInt("hbase.hstore.compaction.max", 1000);
3447     this.region = initHRegion(tableName, method, conf, families);
3448     PutThread putThread = null;
3449     MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3450     try {
3451       putThread = new PutThread(numRows, families, qualifiers);
3452       putThread.start();
3453       putThread.waitForFirstPut();
3454 
3455       // Add a thread that flushes as fast as possible
3456       ctx.addThread(new RepeatingTestThread(ctx) {
3457         private int flushesSinceCompact = 0;
3458         private final int maxFlushesSinceCompact = 20;
3459 
3460         @Override
3461         public void doAnAction() throws Exception {
3462           if (region.flushcache().isCompactionNeeded()) {
3463             ++flushesSinceCompact;
3464           }
3465           // Compact regularly to avoid creating too many files and exceeding
3466           // the ulimit.
3467           if (flushesSinceCompact == maxFlushesSinceCompact) {
3468             region.compactStores(false);
3469             flushesSinceCompact = 0;
3470           }
3471         }
3472       });
3473       ctx.startThreads();
3474 
3475       Get get = new Get(Bytes.toBytes("row0"));
3476       Result result = null;
3477 
3478       int expectedCount = numFamilies * numQualifiers;
3479 
3480       long prevTimestamp = 0L;
3481       for (int i = 0; i < testCount; i++) {
3482 
3483         boolean previousEmpty = result == null || result.isEmpty();
3484         result = region.get(get);
3485         if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3486           assertEquals("i=" + i, expectedCount, result.size());
3487           // TODO this was removed, now what dangit?!
3488           // search looking for the qualifier in question?
3489           long timestamp = 0;
3490           for (Cell kv : result.rawCells()) {
3491             if (CellUtil.matchingFamily(kv, families[0])
3492                 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3493               timestamp = kv.getTimestamp();
3494             }
3495           }
3496           assertTrue(timestamp >= prevTimestamp);
3497           prevTimestamp = timestamp;
3498           Cell previousKV = null;
3499 
3500           for (Cell kv : result.rawCells()) {
3501             byte[] thisValue = CellUtil.cloneValue(kv);
3502             if (previousKV != null) {
3503               if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3504                 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3505                     + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3506                     + "(memStoreTS:" + kv.getMvccVersion() + ")");
3507                 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3508               }
3509             }
3510             previousKV = kv;
3511           }
3512         }
3513       }
3514     } finally {
3515       if (putThread != null)
3516         putThread.done();
3517 
3518       region.flushcache();
3519 
3520       if (putThread != null) {
3521         putThread.join();
3522         putThread.checkNoError();
3523       }
3524 
3525       ctx.stop();
3526       HRegion.closeHRegion(this.region);
3527       this.region = null;
3528     }
3529   }
3530 
3531   @Test
3532   public void testHolesInMeta() throws Exception {
3533     byte[] family = Bytes.toBytes("family");
3534     this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF,
3535         false, family);
3536     try {
3537       byte[] rowNotServed = Bytes.toBytes("a");
3538       Get g = new Get(rowNotServed);
3539       try {
3540         region.get(g);
3541         fail();
3542       } catch (WrongRegionException x) {
3543         // OK
3544       }
3545       byte[] row = Bytes.toBytes("y");
3546       g = new Get(row);
3547       region.get(g);
3548     } finally {
3549       HRegion.closeHRegion(this.region);
3550       this.region = null;
3551     }
3552   }
3553 
3554   @Test
3555   public void testIndexesScanWithOneDeletedRow() throws IOException {
3556     byte[] family = Bytes.toBytes("family");
3557 
3558     // Setting up region
3559     String method = "testIndexesScanWithOneDeletedRow";
3560     this.region = initHRegion(tableName, method, CONF, family);
3561     try {
3562       Put put = new Put(Bytes.toBytes(1L));
3563       put.add(family, qual1, 1L, Bytes.toBytes(1L));
3564       region.put(put);
3565 
3566       region.flushcache();
3567 
3568       Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3569       region.delete(delete);
3570 
3571       put = new Put(Bytes.toBytes(2L));
3572       put.add(family, qual1, 2L, Bytes.toBytes(2L));
3573       region.put(put);
3574 
3575       Scan idxScan = new Scan();
3576       idxScan.addFamily(family);
3577       idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
3578           new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
3579               new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
3580               CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
3581       InternalScanner scanner = region.getScanner(idxScan);
3582       List<Cell> res = new ArrayList<Cell>();
3583 
3584       while (scanner.next(res))
3585         ;
3586       assertEquals(1L, res.size());
3587     } finally {
3588       HRegion.closeHRegion(this.region);
3589       this.region = null;
3590     }
3591   }
3592 
3593   // ////////////////////////////////////////////////////////////////////////////
3594   // Bloom filter test
3595   // ////////////////////////////////////////////////////////////////////////////
3596   @Test
3597   public void testBloomFilterSize() throws IOException {
3598     byte[] fam1 = Bytes.toBytes("fam1");
3599     byte[] qf1 = Bytes.toBytes("col");
3600     byte[] val1 = Bytes.toBytes("value1");
3601     // Create Table
3602     HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
3603         .setBloomFilterType(BloomType.ROWCOL);
3604 
3605     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3606     htd.addFamily(hcd);
3607     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3608     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3609     try {
3610       int num_unique_rows = 10;
3611       int duplicate_multiplier = 2;
3612       int num_storefiles = 4;
3613 
3614       int version = 0;
3615       for (int f = 0; f < num_storefiles; f++) {
3616         for (int i = 0; i < duplicate_multiplier; i++) {
3617           for (int j = 0; j < num_unique_rows; j++) {
3618             Put put = new Put(Bytes.toBytes("row" + j));
3619             put.setDurability(Durability.SKIP_WAL);
3620             put.add(fam1, qf1, version++, val1);
3621             region.put(put);
3622           }
3623         }
3624         region.flushcache();
3625       }
3626       // before compaction
3627       HStore store = (HStore) region.getStore(fam1);
3628       Collection<StoreFile> storeFiles = store.getStorefiles();
3629       for (StoreFile storefile : storeFiles) {
3630         StoreFile.Reader reader = storefile.getReader();
3631         reader.loadFileInfo();
3632         reader.loadBloomfilter();
3633         assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
3634         assertEquals(num_unique_rows, reader.getFilterEntries());
3635       }
3636 
3637       region.compactStores(true);
3638 
3639       // after compaction
3640       storeFiles = store.getStorefiles();
3641       for (StoreFile storefile : storeFiles) {
3642         StoreFile.Reader reader = storefile.getReader();
3643         reader.loadFileInfo();
3644         reader.loadBloomfilter();
3645         assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
3646         assertEquals(num_unique_rows, reader.getFilterEntries());
3647       }
3648     } finally {
3649       HRegion.closeHRegion(this.region);
3650       this.region = null;
3651     }
3652   }
3653 
3654   @Test
3655   public void testAllColumnsWithBloomFilter() throws IOException {
3656     byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
3657     byte[] FAMILY = Bytes.toBytes("family");
3658 
3659     // Create table
3660     HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
3661         .setBloomFilterType(BloomType.ROWCOL);
3662     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
3663     htd.addFamily(hcd);
3664     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3665     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3666     try {
3667       // For row:0, col:0: insert versions 1 through 5.
3668       byte row[] = Bytes.toBytes("row:" + 0);
3669       byte column[] = Bytes.toBytes("column:" + 0);
3670       Put put = new Put(row);
3671       put.setDurability(Durability.SKIP_WAL);
3672       for (long idx = 1; idx <= 4; idx++) {
3673         put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
3674       }
3675       region.put(put);
3676 
3677       // Flush
3678       region.flushcache();
3679 
3680       // Get rows
3681       Get get = new Get(row);
3682       get.setMaxVersions();
3683       Cell[] kvs = region.get(get).rawCells();
3684 
3685       // Check if rows are correct
3686       assertEquals(4, kvs.length);
3687       checkOneCell(kvs[0], FAMILY, 0, 0, 4);
3688       checkOneCell(kvs[1], FAMILY, 0, 0, 3);
3689       checkOneCell(kvs[2], FAMILY, 0, 0, 2);
3690       checkOneCell(kvs[3], FAMILY, 0, 0, 1);
3691     } finally {
3692       HRegion.closeHRegion(this.region);
3693       this.region = null;
3694     }
3695   }
3696 
3697   /**
3698    * Testcase to cover bug-fix for HBASE-2823 Ensures correct delete when
3699    * issuing delete row on columns with bloom filter set to row+col
3700    * (BloomType.ROWCOL)
3701    */
3702   @Test
3703   public void testDeleteRowWithBloomFilter() throws IOException {
3704     byte[] familyName = Bytes.toBytes("familyName");
3705 
3706     // Create Table
3707     HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
3708         .setBloomFilterType(BloomType.ROWCOL);
3709 
3710     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3711     htd.addFamily(hcd);
3712     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3713     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3714     try {
3715       // Insert some data
3716       byte row[] = Bytes.toBytes("row1");
3717       byte col[] = Bytes.toBytes("col1");
3718 
3719       Put put = new Put(row);
3720       put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
3721       region.put(put);
3722       region.flushcache();
3723 
3724       Delete del = new Delete(row);
3725       region.delete(del);
3726       region.flushcache();
3727 
3728       // Get remaining rows (should have none)
3729       Get get = new Get(row);
3730       get.addColumn(familyName, col);
3731 
3732       Cell[] keyValues = region.get(get).rawCells();
3733       assertTrue(keyValues.length == 0);
3734     } finally {
3735       HRegion.closeHRegion(this.region);
3736       this.region = null;
3737     }
3738   }
3739 
3740   @Test
3741   public void testgetHDFSBlocksDistribution() throws Exception {
3742     HBaseTestingUtility htu = new HBaseTestingUtility();
3743     final int DEFAULT_BLOCK_SIZE = 1024;
3744     htu.getConfiguration().setLong("dfs.namenode.fs-limits.min-block-size", 0);
3745     htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
3746     htu.getConfiguration().setInt("dfs.replication", 2);
3747 
3748     // set up a cluster with 3 nodes
3749     MiniHBaseCluster cluster = null;
3750     String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
3751     int regionServersCount = 3;
3752 
3753     try {
3754       cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
3755       byte[][] families = { fam1, fam2 };
3756       HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);
3757 
3758       // Setting up region
3759       byte row[] = Bytes.toBytes("row1");
3760       byte col[] = Bytes.toBytes("col1");
3761 
3762       Put put = new Put(row);
3763       put.add(fam1, col, 1, Bytes.toBytes("test1"));
3764       put.add(fam2, col, 1, Bytes.toBytes("test2"));
3765       ht.put(put);
3766 
3767       HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
3768           .get(0);
3769       firstRegion.flushcache();
3770       HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
3771 
3772       // given the default replication factor is 2 and we have 2 HFiles,
3773       // we will have total of 4 replica of blocks on 3 datanodes; thus there
3774       // must be at least one host that have replica for 2 HFiles. That host's
3775       // weight will be equal to the unique block weight.
3776       long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
3777 
3778       String topHost = blocksDistribution1.getTopHosts().get(0);
3779       long topHostWeight = blocksDistribution1.getWeight(topHost);
3780       assertTrue(uniqueBlocksWeight1 == topHostWeight);
3781 
3782       // use the static method to compute the value, it should be the same.
3783       // static method is used by load balancer or other components
3784       HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
3785           htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
3786       long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
3787 
3788       assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
3789 
3790       ht.close();
3791     } finally {
3792       if (cluster != null) {
3793         htu.shutdownMiniCluster();
3794       }
3795     }
3796   }
3797 
3798   /**
3799    * Testcase to check state of region initialization task set to ABORTED or not
3800    * if any exceptions during initialization
3801    * 
3802    * @throws Exception
3803    */
3804   @Test
3805   public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
3806     TableName tableName = TableName.valueOf(name.getMethodName());
3807     HRegionInfo info = null;
3808     try {
3809       FileSystem fs = Mockito.mock(FileSystem.class);
3810       Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
3811       HTableDescriptor htd = new HTableDescriptor(tableName);
3812       htd.addFamily(new HColumnDescriptor("cf"));
3813       info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
3814           HConstants.EMPTY_BYTE_ARRAY, false);
3815       Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
3816       region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
3817       // region initialization throws IOException and set task state to ABORTED.
3818       region.initialize();
3819       fail("Region initialization should fail due to IOException");
3820     } catch (IOException io) {
3821       List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
3822       for (MonitoredTask monitoredTask : tasks) {
3823         if (!(monitoredTask instanceof MonitoredRPCHandler)
3824             && monitoredTask.getDescription().contains(region.toString())) {
3825           assertTrue("Region state should be ABORTED.",
3826               monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
3827           break;
3828         }
3829       }
3830     } finally {
3831       HRegion.closeHRegion(region);
3832     }
3833   }
3834 
3835   /**
3836    * Verifies that the .regioninfo file is written on region creation and that
3837    * is recreated if missing during region opening.
3838    */
3839   @Test
3840   public void testRegionInfoFileCreation() throws IOException {
3841     Path rootDir = new Path(dir + "testRegionInfoFileCreation");
3842 
3843     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
3844     htd.addFamily(new HColumnDescriptor("cf"));
3845 
3846     HRegionInfo hri = new HRegionInfo(htd.getTableName());
3847 
3848     // Create a region and skip the initialization (like CreateTableHandler)
3849     HRegion region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
3850 //    HRegion region = TEST_UTIL.createLocalHRegion(hri, htd);
3851     Path regionDir = region.getRegionFileSystem().getRegionDir();
3852     FileSystem fs = region.getRegionFileSystem().getFileSystem();
3853     HRegion.closeHRegion(region);
3854 
3855     Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
3856 
3857     // Verify that the .regioninfo file is present
3858     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3859         fs.exists(regionInfoFile));
3860 
3861     // Try to open the region
3862     region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3863     assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3864     HRegion.closeHRegion(region);
3865 
3866     // Verify that the .regioninfo file is still there
3867     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3868         fs.exists(regionInfoFile));
3869 
3870     // Remove the .regioninfo file and verify is recreated on region open
3871     fs.delete(regionInfoFile);
3872     assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
3873         fs.exists(regionInfoFile));
3874 
3875     region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3876 //    region = TEST_UTIL.openHRegion(hri, htd);
3877     assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3878     HRegion.closeHRegion(region);
3879 
3880     // Verify that the .regioninfo file is still there
3881     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3882         fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
3883   }
3884 
3885   /**
3886    * TestCase for increment
3887    */
3888   private static class Incrementer implements Runnable {
3889     private HRegion region;
3890     private final static byte[] incRow = Bytes.toBytes("incRow");
3891     private final static byte[] family = Bytes.toBytes("family");
3892     private final static byte[] qualifier = Bytes.toBytes("qualifier");
3893     private final static long ONE = 1l;
3894     private int incCounter;
3895 
3896     public Incrementer(HRegion region, int incCounter) {
3897       this.region = region;
3898       this.incCounter = incCounter;
3899     }
3900 
3901     @Override
3902     public void run() {
3903       int count = 0;
3904       while (count < incCounter) {
3905         Increment inc = new Increment(incRow);
3906         inc.addColumn(family, qualifier, ONE);
3907         count++;
3908         try {
3909           region.increment(inc);
3910         } catch (IOException e) {
3911           e.printStackTrace();
3912           break;
3913         }
3914       }
3915     }
3916   }
3917 
3918   /**
3919    * Test case to check increment function with memstore flushing
3920    * @throws Exception
3921    */
3922   @Test
3923   public void testParallelIncrementWithMemStoreFlush() throws Exception {
3924     byte[] family = Incrementer.family;
3925     this.region = initHRegion(tableName, method, CONF, family);
3926     final HRegion region = this.region;
3927     final AtomicBoolean incrementDone = new AtomicBoolean(false);
3928     Runnable flusher = new Runnable() {
3929       @Override
3930       public void run() {
3931         while (!incrementDone.get()) {
3932           try {
3933             region.flushcache();
3934           } catch (Exception e) {
3935             e.printStackTrace();
3936           }
3937         }
3938       }
3939     };
3940 
3941     // after all increment finished, the row will increment to 20*100 = 2000
3942     int threadNum = 20;
3943     int incCounter = 100;
3944     long expected = threadNum * incCounter;
3945     Thread[] incrementers = new Thread[threadNum];
3946     Thread flushThread = new Thread(flusher);
3947     for (int i = 0; i < threadNum; i++) {
3948       incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
3949       incrementers[i].start();
3950     }
3951     flushThread.start();
3952     for (int i = 0; i < threadNum; i++) {
3953       incrementers[i].join();
3954     }
3955 
3956     incrementDone.set(true);
3957     flushThread.join();
3958 
3959     Get get = new Get(Incrementer.incRow);
3960     get.addColumn(Incrementer.family, Incrementer.qualifier);
3961     get.setMaxVersions(1);
3962     Result res = this.region.get(get);
3963     List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
3964 
3965     // we just got the latest version
3966     assertEquals(kvs.size(), 1);
3967     Cell kv = kvs.get(0);
3968     assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
3969     this.region = null;
3970   }
3971 
3972   /**
3973    * TestCase for append
3974    */
3975   private static class Appender implements Runnable {
3976     private HRegion region;
3977     private final static byte[] appendRow = Bytes.toBytes("appendRow");
3978     private final static byte[] family = Bytes.toBytes("family");
3979     private final static byte[] qualifier = Bytes.toBytes("qualifier");
3980     private final static byte[] CHAR = Bytes.toBytes("a");
3981     private int appendCounter;
3982 
3983     public Appender(HRegion region, int appendCounter) {
3984       this.region = region;
3985       this.appendCounter = appendCounter;
3986     }
3987 
3988     @Override
3989     public void run() {
3990       int count = 0;
3991       while (count < appendCounter) {
3992         Append app = new Append(appendRow);
3993         app.add(family, qualifier, CHAR);
3994         count++;
3995         try {
3996           region.append(app);
3997         } catch (IOException e) {
3998           e.printStackTrace();
3999           break;
4000         }
4001       }
4002     }
4003   }
4004 
4005   /**
4006    * Test case to check append function with memstore flushing
4007    * @throws Exception
4008    */
4009   @Test
4010   public void testParallelAppendWithMemStoreFlush() throws Exception {
4011     byte[] family = Appender.family;
4012     this.region = initHRegion(tableName, method, CONF, family);
4013     final HRegion region = this.region;
4014     final AtomicBoolean appendDone = new AtomicBoolean(false);
4015     Runnable flusher = new Runnable() {
4016       @Override
4017       public void run() {
4018         while (!appendDone.get()) {
4019           try {
4020             region.flushcache();
4021           } catch (Exception e) {
4022             e.printStackTrace();
4023           }
4024         }
4025       }
4026     };
4027 
4028     // after all append finished, the value will append to threadNum *
4029     // appendCounter Appender.CHAR
4030     int threadNum = 20;
4031     int appendCounter = 100;
4032     byte[] expected = new byte[threadNum * appendCounter];
4033     for (int i = 0; i < threadNum * appendCounter; i++) {
4034       System.arraycopy(Appender.CHAR, 0, expected, i, 1);
4035     }
4036     Thread[] appenders = new Thread[threadNum];
4037     Thread flushThread = new Thread(flusher);
4038     for (int i = 0; i < threadNum; i++) {
4039       appenders[i] = new Thread(new Appender(this.region, appendCounter));
4040       appenders[i].start();
4041     }
4042     flushThread.start();
4043     for (int i = 0; i < threadNum; i++) {
4044       appenders[i].join();
4045     }
4046 
4047     appendDone.set(true);
4048     flushThread.join();
4049 
4050     Get get = new Get(Appender.appendRow);
4051     get.addColumn(Appender.family, Appender.qualifier);
4052     get.setMaxVersions(1);
4053     Result res = this.region.get(get);
4054     List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
4055 
4056     // we just got the latest version
4057     assertEquals(kvs.size(), 1);
4058     Cell kv = kvs.get(0);
4059     byte[] appendResult = new byte[kv.getValueLength()];
4060     System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
4061     assertArrayEquals(expected, appendResult);
4062     this.region = null;
4063   }
4064 
4065   /**
4066    * Test case to check put function with memstore flushing for same row, same ts
4067    * @throws Exception
4068    */
4069   @Test
4070   public void testPutWithMemStoreFlush() throws Exception {
4071     byte[] family = Bytes.toBytes("family");
4072     ;
4073     byte[] qualifier = Bytes.toBytes("qualifier");
4074     byte[] row = Bytes.toBytes("putRow");
4075     byte[] value = null;
4076     this.region = initHRegion(tableName, method, CONF, family);
4077     Put put = null;
4078     Get get = null;
4079     List<Cell> kvs = null;
4080     Result res = null;
4081 
4082     put = new Put(row);
4083     value = Bytes.toBytes("value0");
4084     put.add(family, qualifier, 1234567l, value);
4085     region.put(put);
4086     get = new Get(row);
4087     get.addColumn(family, qualifier);
4088     get.setMaxVersions();
4089     res = this.region.get(get);
4090     kvs = res.getColumnCells(family, qualifier);
4091     assertEquals(1, kvs.size());
4092     assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4093 
4094     region.flushcache();
4095     get = new Get(row);
4096     get.addColumn(family, qualifier);
4097     get.setMaxVersions();
4098     res = this.region.get(get);
4099     kvs = res.getColumnCells(family, qualifier);
4100     assertEquals(1, kvs.size());
4101     assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4102 
4103     put = new Put(row);
4104     value = Bytes.toBytes("value1");
4105     put.add(family, qualifier, 1234567l, value);
4106     region.put(put);
4107     get = new Get(row);
4108     get.addColumn(family, qualifier);
4109     get.setMaxVersions();
4110     res = this.region.get(get);
4111     kvs = res.getColumnCells(family, qualifier);
4112     assertEquals(1, kvs.size());
4113     assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4114 
4115     region.flushcache();
4116     get = new Get(row);
4117     get.addColumn(family, qualifier);
4118     get.setMaxVersions();
4119     res = this.region.get(get);
4120     kvs = res.getColumnCells(family, qualifier);
4121     assertEquals(1, kvs.size());
4122     assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4123   }
4124 
4125   @Test
4126   public void testDurability() throws Exception {
4127     String method = "testDurability";
4128     // there are 5 x 5 cases:
4129     // table durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) x mutation
4130     // durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT)
4131 
4132     // expected cases for append and sync wal
4133     durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4134     durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4135     durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4136 
4137     durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4138     durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4139     durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4140 
4141     durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4142     durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4143 
4144     durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
4145     durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4146 
4147     durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
4148     durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
4149     durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
4150 
4151     // expected cases for async wal
4152     durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4153     durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4154     durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4155     durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4156     durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
4157     durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4158 
4159     durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4160     durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4161     durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4162     durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4163     durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
4164     durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
4165 
4166     // expect skip wal cases
4167     durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4168     durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4169     durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4170     durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
4171     durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
4172     durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
4173 
4174   }
4175 
4176   private void durabilityTest(String method, Durability tableDurability,
4177       Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
4178       final boolean expectSyncFromLogSyncer) throws Exception {
4179     Configuration conf = HBaseConfiguration.create(CONF);
4180     method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
4181     TableName tableName = TableName.valueOf(method);
4182     byte[] family = Bytes.toBytes("family");
4183     Path logDir = new Path(new Path(dir + method), "log");
4184     HLog hlog = HLogFactory.createHLog(FILESYSTEM, logDir, UUID.randomUUID().toString(), conf);
4185     final HLog log = spy(hlog);
4186     this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
4187         HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, log,
4188         new byte[][] { family });
4189 
4190     Put put = new Put(Bytes.toBytes("r1"));
4191     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
4192     put.setDurability(mutationDurability);
4193     region.put(put);
4194 
4195     //verify append called or not
4196     verify(log, expectAppend ? times(1) : never())
4197       .appendNoSync((HRegionInfo)any(), eq(tableName), (WALEdit)any(), (List<UUID>)any(),
4198         anyLong(), (HTableDescriptor)any(), (AtomicLong)any(), anyBoolean(), anyLong(), anyLong());
4199 
4200     // verify sync called or not
4201     if (expectSync || expectSyncFromLogSyncer) {
4202       TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
4203         @Override
4204         public boolean evaluate() throws Exception {
4205           try {
4206             if (expectSync) {
4207               verify(log, times(1)).sync(anyLong()); // Hregion calls this one
4208             } else if (expectSyncFromLogSyncer) {
4209               verify(log, times(1)).sync(); // log syncer calls this one
4210             }
4211           } catch (Throwable ignore) {
4212           }
4213           return true;
4214         }
4215       });
4216     } else {
4217       verify(log, never()).sync(anyLong());
4218       verify(log, never()).sync();
4219     }
4220 
4221     HRegion.closeHRegion(this.region);
4222     this.region = null;
4223   }
4224 
4225   private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4226     for (int i = startRow; i < startRow + numRows; i++) {
4227       Put put = new Put(Bytes.toBytes("" + i));
4228       put.setDurability(Durability.SKIP_WAL);
4229       for (byte[] family : families) {
4230         put.add(family, qf, null);
4231       }
4232       region.put(put);
4233     }
4234   }
4235 
4236   private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
4237       throws IOException {
4238     for (int i = startRow; i < startRow + numRows; i++) {
4239       byte[] row = Bytes.toBytes("" + i);
4240       Get get = new Get(row);
4241       for (byte[] family : families) {
4242         get.addColumn(family, qf);
4243       }
4244       Result result = newReg.get(get);
4245       Cell[] raw = result.rawCells();
4246       assertEquals(families.length, result.size());
4247       for (int j = 0; j < families.length; j++) {
4248         assertTrue(CellUtil.matchingRow(raw[j], row));
4249         assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
4250         assertTrue(CellUtil.matchingQualifier(raw[j], qf));
4251       }
4252     }
4253   }
4254 
4255   private void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
4256     // Now I have k, get values out and assert they are as expected.
4257     Get get = new Get(k).addFamily(family).setMaxVersions();
4258     Cell[] results = r.get(get).rawCells();
4259     for (int j = 0; j < results.length; j++) {
4260       byte[] tmp = CellUtil.cloneValue(results[j]);
4261       // Row should be equal to value every time.
4262       assertTrue(Bytes.equals(k, tmp));
4263     }
4264   }
4265 
4266   /*
4267    * Assert first value in the passed region is <code>firstValue</code>.
4268    * 
4269    * @param r
4270    * 
4271    * @param fs
4272    * 
4273    * @param firstValue
4274    * 
4275    * @throws IOException
4276    */
4277   private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
4278       throws IOException {
4279     byte[][] families = { fs };
4280     Scan scan = new Scan();
4281     for (int i = 0; i < families.length; i++)
4282       scan.addFamily(families[i]);
4283     InternalScanner s = r.getScanner(scan);
4284     try {
4285       List<Cell> curVals = new ArrayList<Cell>();
4286       boolean first = true;
4287       OUTER_LOOP: while (s.next(curVals)) {
4288         for (Cell kv : curVals) {
4289           byte[] val = CellUtil.cloneValue(kv);
4290           byte[] curval = val;
4291           if (first) {
4292             first = false;
4293             assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4294           } else {
4295             // Not asserting anything. Might as well break.
4296             break OUTER_LOOP;
4297           }
4298         }
4299       }
4300     } finally {
4301       s.close();
4302     }
4303   }
4304 
4305   /**
4306    * Test that we get the expected flush results back
4307    * @throws IOException
4308    */
4309   @Test
4310   public void testFlushResult() throws IOException {
4311     String method = name.getMethodName();
4312     byte[] tableName = Bytes.toBytes(method);
4313     byte[] family = Bytes.toBytes("family");
4314 
4315     this.region = initHRegion(tableName, method, family);
4316 
4317     // empty memstore, flush doesn't run
4318     HRegion.FlushResult fr = region.flushcache();
4319     assertFalse(fr.isFlushSucceeded());
4320     assertFalse(fr.isCompactionNeeded());
4321 
4322     // Flush enough files to get up to the threshold, doesn't need compactions
4323     for (int i = 0; i < 2; i++) {
4324       Put put = new Put(tableName).add(family, family, tableName);
4325       region.put(put);
4326       fr = region.flushcache();
4327       assertTrue(fr.isFlushSucceeded());
4328       assertFalse(fr.isCompactionNeeded());
4329     }
4330 
4331     // Two flushes after the threshold, compactions are needed
4332     for (int i = 0; i < 2; i++) {
4333       Put put = new Put(tableName).add(family, family, tableName);
4334       region.put(put);
4335       fr = region.flushcache();
4336       assertTrue(fr.isFlushSucceeded());
4337       assertTrue(fr.isCompactionNeeded());
4338     }
4339   }
4340 
4341   private Configuration initSplit() {
4342     // Always compact if there is more than one store file.
4343     CONF.setInt("hbase.hstore.compactionThreshold", 2);
4344 
4345     // Make lease timeout longer, lease checks less frequent
4346     CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4347 
4348     CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4349 
4350     // Increase the amount of time between client retries
4351     CONF.setLong("hbase.client.pause", 15 * 1000);
4352 
4353     // This size should make it so we always split using the addContent
4354     // below. After adding all data, the first region is 1.3M
4355     CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4356     return CONF;
4357   }
4358 
4359   /**
4360    * @param tableName
4361    * @param callingMethod
4362    * @param conf
4363    * @param families
4364    * @throws IOException
4365    * @return A region on which you must call
4366    *         {@link HRegion#closeHRegion(HRegion)} when done.
4367    */
4368   public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
4369       byte[]... families) throws IOException {
4370     return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
4371   }
4372 
4373   /**
4374    * @param tableName
4375    * @param callingMethod
4376    * @param conf
4377    * @param families
4378    * @throws IOException
4379    * @return A region on which you must call
4380    *         {@link HRegion#closeHRegion(HRegion)} when done.
4381    */
4382   public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4383       byte[]... families) throws IOException {
4384     return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4385   }
4386 
4387   /**
4388    * @param tableName
4389    * @param callingMethod
4390    * @param conf
4391    * @param isReadOnly
4392    * @param families
4393    * @throws IOException
4394    * @return A region on which you must call
4395    *         {@link HRegion#closeHRegion(HRegion)} when done.
4396    */
4397   public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4398       boolean isReadOnly, byte[]... families) throws IOException {
4399     return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4400   }
4401 
4402   private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4403       String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4404       throws IOException {
4405     return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
4406         Durability.SYNC_WAL, null, families);
4407   }
4408 
4409   /**
4410    * @param tableName
4411    * @param startKey
4412    * @param stopKey
4413    * @param callingMethod
4414    * @param conf
4415    * @param isReadOnly
4416    * @param families
4417    * @throws IOException
4418    * @return A region on which you must call
4419    *         {@link HRegion#closeHRegion(HRegion)} when done.
4420    */
4421   private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4422       String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4423       HLog hlog, byte[]... families) throws IOException {
4424     return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly, durability, hlog, families);
4425   }
4426 
4427   /**
4428    * Assert that the passed in Cell has expected contents for the specified row,
4429    * column & timestamp.
4430    */
4431   private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
4432     String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4433     assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
4434         Bytes.toString(CellUtil.cloneRow(kv)));
4435     assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
4436         Bytes.toString(CellUtil.cloneFamily(kv)));
4437     assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
4438         Bytes.toString(CellUtil.cloneQualifier(kv)));
4439     assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
4440     assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
4441         Bytes.toString(CellUtil.cloneValue(kv)));
4442   }
4443 
4444   public void testReverseScanner_FromMemStore_SingleCF_Normal()
4445       throws IOException {
4446     byte[] rowC = Bytes.toBytes("rowC");
4447     byte[] rowA = Bytes.toBytes("rowA");
4448     byte[] rowB = Bytes.toBytes("rowB");
4449     byte[] cf = Bytes.toBytes("CF");
4450     byte[][] families = { cf };
4451     byte[] col = Bytes.toBytes("C");
4452     long ts = 1;
4453     String method = this.getName();
4454     this.region = initHRegion(tableName, method, families);
4455     try {
4456       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4457       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4458           null);
4459       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4460       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4461       Put put = null;
4462       put = new Put(rowC);
4463       put.add(kv1);
4464       put.add(kv11);
4465       region.put(put);
4466       put = new Put(rowA);
4467       put.add(kv2);
4468       region.put(put);
4469       put = new Put(rowB);
4470       put.add(kv3);
4471       region.put(put);
4472 
4473       Scan scan = new Scan(rowC);
4474       scan.setMaxVersions(5);
4475       scan.setReversed(true);
4476       InternalScanner scanner = region.getScanner(scan);
4477       List<Cell> currRow = new ArrayList<Cell>();
4478       boolean hasNext = scanner.next(currRow);
4479       assertEquals(2, currRow.size());
4480       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4481       assertTrue(hasNext);
4482       currRow.clear();
4483       hasNext = scanner.next(currRow);
4484       assertEquals(1, currRow.size());
4485       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4486       assertTrue(hasNext);
4487       currRow.clear();
4488       hasNext = scanner.next(currRow);
4489       assertEquals(1, currRow.size());
4490       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4491       assertFalse(hasNext);
4492       scanner.close();
4493     } finally {
4494       HRegion.closeHRegion(this.region);
4495       this.region = null;
4496     }
4497   }
4498 
4499   public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
4500       throws IOException {
4501     byte[] rowC = Bytes.toBytes("rowC");
4502     byte[] rowA = Bytes.toBytes("rowA");
4503     byte[] rowB = Bytes.toBytes("rowB");
4504     byte[] rowD = Bytes.toBytes("rowD");
4505     byte[] cf = Bytes.toBytes("CF");
4506     byte[][] families = { cf };
4507     byte[] col = Bytes.toBytes("C");
4508     long ts = 1;
4509     String method = this.getName();
4510     this.region = initHRegion(tableName, method, families);
4511     try {
4512       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4513       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4514           null);
4515       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4516       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4517       Put put = null;
4518       put = new Put(rowC);
4519       put.add(kv1);
4520       put.add(kv11);
4521       region.put(put);
4522       put = new Put(rowA);
4523       put.add(kv2);
4524       region.put(put);
4525       put = new Put(rowB);
4526       put.add(kv3);
4527       region.put(put);
4528 
4529       Scan scan = new Scan(rowD);
4530       List<Cell> currRow = new ArrayList<Cell>();
4531       scan.setReversed(true);
4532       scan.setMaxVersions(5);
4533       InternalScanner scanner = region.getScanner(scan);
4534       boolean hasNext = scanner.next(currRow);
4535       assertEquals(2, currRow.size());
4536       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4537       assertTrue(hasNext);
4538       currRow.clear();
4539       hasNext = scanner.next(currRow);
4540       assertEquals(1, currRow.size());
4541       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4542       assertTrue(hasNext);
4543       currRow.clear();
4544       hasNext = scanner.next(currRow);
4545       assertEquals(1, currRow.size());
4546       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4547       assertFalse(hasNext);
4548       scanner.close();
4549     } finally {
4550       HRegion.closeHRegion(this.region);
4551       this.region = null;
4552     }
4553   }
4554 
4555   public void testReverseScanner_FromMemStore_SingleCF_FullScan()
4556       throws IOException {
4557     byte[] rowC = Bytes.toBytes("rowC");
4558     byte[] rowA = Bytes.toBytes("rowA");
4559     byte[] rowB = Bytes.toBytes("rowB");
4560     byte[] cf = Bytes.toBytes("CF");
4561     byte[][] families = { cf };
4562     byte[] col = Bytes.toBytes("C");
4563     long ts = 1;
4564     String method = this.getName();
4565     this.region = initHRegion(tableName, method, families);
4566     try {
4567       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4568       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4569           null);
4570       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4571       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4572       Put put = null;
4573       put = new Put(rowC);
4574       put.add(kv1);
4575       put.add(kv11);
4576       region.put(put);
4577       put = new Put(rowA);
4578       put.add(kv2);
4579       region.put(put);
4580       put = new Put(rowB);
4581       put.add(kv3);
4582       region.put(put);
4583       Scan scan = new Scan();
4584       List<Cell> currRow = new ArrayList<Cell>();
4585       scan.setReversed(true);
4586       InternalScanner scanner = region.getScanner(scan);
4587       boolean hasNext = scanner.next(currRow);
4588       assertEquals(1, currRow.size());
4589       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4590       assertTrue(hasNext);
4591       currRow.clear();
4592       hasNext = scanner.next(currRow);
4593       assertEquals(1, currRow.size());
4594       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4595       assertTrue(hasNext);
4596       currRow.clear();
4597       hasNext = scanner.next(currRow);
4598       assertEquals(1, currRow.size());
4599       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4600       assertFalse(hasNext);
4601       scanner.close();
4602     } finally {
4603       HRegion.closeHRegion(this.region);
4604       this.region = null;
4605     }
4606   }
4607 
4608   public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
4609     // case for "INCLUDE_AND_SEEK_NEXT_ROW & SEEK_NEXT_ROW" endless loop
4610     byte[] rowA = Bytes.toBytes("rowA");
4611     byte[] rowB = Bytes.toBytes("rowB");
4612     byte[] rowC = Bytes.toBytes("rowC");
4613     byte[] rowD = Bytes.toBytes("rowD");
4614     byte[] rowE = Bytes.toBytes("rowE");
4615     byte[] cf = Bytes.toBytes("CF");
4616     byte[][] families = { cf };
4617     byte[] col1 = Bytes.toBytes("col1");
4618     byte[] col2 = Bytes.toBytes("col2");
4619     long ts = 1;
4620     String method = this.getName();
4621     this.region = initHRegion(tableName, method, families);
4622     try {
4623       KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4624       KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4625       KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4626       KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4627       KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4628       KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4629       Put put = null;
4630       put = new Put(rowA);
4631       put.add(kv1);
4632       region.put(put);
4633       put = new Put(rowB);
4634       put.add(kv2);
4635       region.put(put);
4636       put = new Put(rowC);
4637       put.add(kv3);
4638       region.put(put);
4639       put = new Put(rowD);
4640       put.add(kv4_1);
4641       region.put(put);
4642       put = new Put(rowD);
4643       put.add(kv4_2);
4644       region.put(put);
4645       put = new Put(rowE);
4646       put.add(kv5);
4647       region.put(put);
4648       region.flushcache();
4649       Scan scan = new Scan(rowD, rowA);
4650       scan.addColumn(families[0], col1);
4651       scan.setReversed(true);
4652       List<Cell> currRow = new ArrayList<Cell>();
4653       InternalScanner scanner = region.getScanner(scan);
4654       boolean hasNext = scanner.next(currRow);
4655       assertEquals(1, currRow.size());
4656       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4657       assertTrue(hasNext);
4658       currRow.clear();
4659       hasNext = scanner.next(currRow);
4660       assertEquals(1, currRow.size());
4661       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4662       assertTrue(hasNext);
4663       currRow.clear();
4664       hasNext = scanner.next(currRow);
4665       assertEquals(1, currRow.size());
4666       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4667       assertFalse(hasNext);
4668       scanner.close();
4669 
4670       scan = new Scan(rowD, rowA);
4671       scan.addColumn(families[0], col2);
4672       scan.setReversed(true);
4673       currRow.clear();
4674       scanner = region.getScanner(scan);
4675       hasNext = scanner.next(currRow);
4676       assertEquals(1, currRow.size());
4677       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4678       scanner.close();
4679     } finally {
4680       HRegion.closeHRegion(this.region);
4681       this.region = null;
4682     }
4683   }
4684 
4685   public void testReverseScanner_smaller_blocksize() throws IOException {
4686     // case to ensure no conflict with HFile index optimization
4687     byte[] rowA = Bytes.toBytes("rowA");
4688     byte[] rowB = Bytes.toBytes("rowB");
4689     byte[] rowC = Bytes.toBytes("rowC");
4690     byte[] rowD = Bytes.toBytes("rowD");
4691     byte[] rowE = Bytes.toBytes("rowE");
4692     byte[] cf = Bytes.toBytes("CF");
4693     byte[][] families = { cf };
4694     byte[] col1 = Bytes.toBytes("col1");
4695     byte[] col2 = Bytes.toBytes("col2");
4696     long ts = 1;
4697     String method = this.getName();
4698     HBaseConfiguration config = new HBaseConfiguration();
4699     config.setInt("test.block.size", 1);
4700     this.region = initHRegion(tableName, method, config, families);
4701     try {
4702       KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4703       KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4704       KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4705       KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4706       KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4707       KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4708       Put put = null;
4709       put = new Put(rowA);
4710       put.add(kv1);
4711       region.put(put);
4712       put = new Put(rowB);
4713       put.add(kv2);
4714       region.put(put);
4715       put = new Put(rowC);
4716       put.add(kv3);
4717       region.put(put);
4718       put = new Put(rowD);
4719       put.add(kv4_1);
4720       region.put(put);
4721       put = new Put(rowD);
4722       put.add(kv4_2);
4723       region.put(put);
4724       put = new Put(rowE);
4725       put.add(kv5);
4726       region.put(put);
4727       region.flushcache();
4728       Scan scan = new Scan(rowD, rowA);
4729       scan.addColumn(families[0], col1);
4730       scan.setReversed(true);
4731       List<Cell> currRow = new ArrayList<Cell>();
4732       InternalScanner scanner = region.getScanner(scan);
4733       boolean hasNext = scanner.next(currRow);
4734       assertEquals(1, currRow.size());
4735       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4736       assertTrue(hasNext);
4737       currRow.clear();
4738       hasNext = scanner.next(currRow);
4739       assertEquals(1, currRow.size());
4740       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4741       assertTrue(hasNext);
4742       currRow.clear();
4743       hasNext = scanner.next(currRow);
4744       assertEquals(1, currRow.size());
4745       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4746       assertFalse(hasNext);
4747       scanner.close();
4748 
4749       scan = new Scan(rowD, rowA);
4750       scan.addColumn(families[0], col2);
4751       scan.setReversed(true);
4752       currRow.clear();
4753       scanner = region.getScanner(scan);
4754       hasNext = scanner.next(currRow);
4755       assertEquals(1, currRow.size());
4756       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4757       scanner.close();
4758     } finally {
4759       HRegion.closeHRegion(this.region);
4760       this.region = null;
4761     }
4762   }
4763 
4764   public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
4765       throws IOException {
4766     byte[] row0 = Bytes.toBytes("row0"); // 1 kv
4767     byte[] row1 = Bytes.toBytes("row1"); // 2 kv
4768     byte[] row2 = Bytes.toBytes("row2"); // 4 kv
4769     byte[] row3 = Bytes.toBytes("row3"); // 2 kv
4770     byte[] row4 = Bytes.toBytes("row4"); // 5 kv
4771     byte[] row5 = Bytes.toBytes("row5"); // 2 kv
4772     byte[] cf1 = Bytes.toBytes("CF1");
4773     byte[] cf2 = Bytes.toBytes("CF2");
4774     byte[] cf3 = Bytes.toBytes("CF3");
4775     byte[][] families = { cf1, cf2, cf3 };
4776     byte[] col = Bytes.toBytes("C");
4777     long ts = 1;
4778     String method = this.getName();
4779     HBaseConfiguration conf = new HBaseConfiguration();
4780     // disable compactions in this test.
4781     conf.setInt("hbase.hstore.compactionThreshold", 10000);
4782     this.region = initHRegion(tableName, method, conf, families);
4783     try {
4784       // kv naming style: kv(row number) totalKvCountInThisRow seq no
4785       KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
4786           null);
4787       KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
4788           null);
4789       KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
4790           KeyValue.Type.Put, null);
4791       KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
4792           null);
4793       KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
4794           null);
4795       KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
4796           null);
4797       KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
4798           KeyValue.Type.Put, null);
4799       KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
4800           null);
4801       KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
4802           KeyValue.Type.Put, null);
4803       KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
4804           null);
4805       KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
4806           null);
4807       KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
4808           KeyValue.Type.Put, null);
4809       KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
4810           null);
4811       KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
4812           KeyValue.Type.Put, null);
4813       KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
4814           null);
4815       KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
4816           null);
4817       // hfiles(cf1/cf2) :"row1"(1 kv) / "row2"(1 kv) / "row4"(2 kv)
4818       Put put = null;
4819       put = new Put(row1);
4820       put.add(kv1_2_1);
4821       region.put(put);
4822       put = new Put(row2);
4823       put.add(kv2_4_1);
4824       region.put(put);
4825       put = new Put(row4);
4826       put.add(kv4_5_4);
4827       put.add(kv4_5_5);
4828       region.put(put);
4829       region.flushcache();
4830       // hfiles(cf1/cf3) : "row1" (1 kvs) / "row2" (1 kv) / "row4" (2 kv)
4831       put = new Put(row4);
4832       put.add(kv4_5_1);
4833       put.add(kv4_5_3);
4834       region.put(put);
4835       put = new Put(row1);
4836       put.add(kv1_2_2);
4837       region.put(put);
4838       put = new Put(row2);
4839       put.add(kv2_4_4);
4840       region.put(put);
4841       region.flushcache();
4842       // hfiles(cf1/cf3) : "row2"(2 kv) / "row3"(1 kvs) / "row4" (1 kv)
4843       put = new Put(row4);
4844       put.add(kv4_5_2);
4845       region.put(put);
4846       put = new Put(row2);
4847       put.add(kv2_4_2);
4848       put.add(kv2_4_3);
4849       region.put(put);
4850       put = new Put(row3);
4851       put.add(kv3_2_2);
4852       region.put(put);
4853       region.flushcache();
4854       // memstore(cf1/cf2/cf3) : "row0" (1 kvs) / "row3" ( 1 kv) / "row5" (max)
4855       // ( 2 kv)
4856       put = new Put(row0);
4857       put.add(kv0_1_1);
4858       region.put(put);
4859       put = new Put(row3);
4860       put.add(kv3_2_1);
4861       region.put(put);
4862       put = new Put(row5);
4863       put.add(kv5_2_1);
4864       put.add(kv5_2_2);
4865       region.put(put);
4866       // scan range = ["row4", min), skip the max "row5"
4867       Scan scan = new Scan(row4);
4868       scan.setMaxVersions(5);
4869       scan.setBatch(3);
4870       scan.setReversed(true);
4871       InternalScanner scanner = region.getScanner(scan);
4872       List<Cell> currRow = new ArrayList<Cell>();
4873       boolean hasNext = false;
4874       // 1. scan out "row4" (5 kvs), "row5" can't be scanned out since not
4875       // included in scan range
4876       // "row4" takes 2 next() calls since batch=3
4877       hasNext = scanner.next(currRow);
4878       assertEquals(3, currRow.size());
4879       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4880       assertTrue(hasNext);
4881       currRow.clear();
4882       hasNext = scanner.next(currRow);
4883       assertEquals(2, currRow.size());
4884       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4885       assertTrue(hasNext);
4886       // 2. scan out "row3" (2 kv)
4887       currRow.clear();
4888       hasNext = scanner.next(currRow);
4889       assertEquals(2, currRow.size());
4890       assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
4891       assertTrue(hasNext);
4892       // 3. scan out "row2" (4 kvs)
4893       // "row2" takes 2 next() calls since batch=3
4894       currRow.clear();
4895       hasNext = scanner.next(currRow);
4896       assertEquals(3, currRow.size());
4897       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4898       assertTrue(hasNext);
4899       currRow.clear();
4900       hasNext = scanner.next(currRow);
4901       assertEquals(1, currRow.size());
4902       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4903       assertTrue(hasNext);
4904       // 4. scan out "row1" (2 kv)
4905       currRow.clear();
4906       hasNext = scanner.next(currRow);
4907       assertEquals(2, currRow.size());
4908       assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
4909       assertTrue(hasNext);
4910       // 5. scan out "row0" (1 kv)
4911       currRow.clear();
4912       hasNext = scanner.next(currRow);
4913       assertEquals(1, currRow.size());
4914       assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
4915       assertFalse(hasNext);
4916 
4917       scanner.close();
4918     } finally {
4919       HRegion.closeHRegion(this.region);
4920       this.region = null;
4921     }
4922   }
4923 
4924   public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
4925       throws IOException {
4926     byte[] row1 = Bytes.toBytes("row1");
4927     byte[] row2 = Bytes.toBytes("row2");
4928     byte[] row3 = Bytes.toBytes("row3");
4929     byte[] row4 = Bytes.toBytes("row4");
4930     byte[] cf1 = Bytes.toBytes("CF1");
4931     byte[] cf2 = Bytes.toBytes("CF2");
4932     byte[] cf3 = Bytes.toBytes("CF3");
4933     byte[] cf4 = Bytes.toBytes("CF4");
4934     byte[][] families = { cf1, cf2, cf3, cf4 };
4935     byte[] col = Bytes.toBytes("C");
4936     long ts = 1;
4937     String method = this.getName();
4938     HBaseConfiguration conf = new HBaseConfiguration();
4939     // disable compactions in this test.
4940     conf.setInt("hbase.hstore.compactionThreshold", 10000);
4941     this.region = initHRegion(tableName, method, conf, families);
4942     try {
4943       KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
4944       KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
4945       KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
4946       KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
4947       // storefile1
4948       Put put = new Put(row1);
4949       put.add(kv1);
4950       region.put(put);
4951       region.flushcache();
4952       // storefile2
4953       put = new Put(row2);
4954       put.add(kv2);
4955       region.put(put);
4956       region.flushcache();
4957       // storefile3
4958       put = new Put(row3);
4959       put.add(kv3);
4960       region.put(put);
4961       region.flushcache();
4962       // memstore
4963       put = new Put(row4);
4964       put.add(kv4);
4965       region.put(put);
4966       // scan range = ["row4", min)
4967       Scan scan = new Scan(row4);
4968       scan.setReversed(true);
4969       scan.setBatch(10);
4970       InternalScanner scanner = region.getScanner(scan);
4971       List<Cell> currRow = new ArrayList<Cell>();
4972       boolean hasNext = scanner.next(currRow);
4973       assertEquals(1, currRow.size());
4974       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4975       assertTrue(hasNext);
4976       currRow.clear();
4977       hasNext = scanner.next(currRow);
4978       assertEquals(1, currRow.size());
4979       assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
4980       assertTrue(hasNext);
4981       currRow.clear();
4982       hasNext = scanner.next(currRow);
4983       assertEquals(1, currRow.size());
4984       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4985       assertTrue(hasNext);
4986       currRow.clear();
4987       hasNext = scanner.next(currRow);
4988       assertEquals(1, currRow.size());
4989       assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
4990       assertFalse(hasNext);
4991     } finally {
4992       HRegion.closeHRegion(this.region);
4993       this.region = null;
4994     }
4995   }
4996 
4997   @Test
4998   public void testWriteRequestsCounter() throws IOException {
4999     byte[] fam = Bytes.toBytes("info");
5000     byte[][] families = { fam };
5001     this.region = initHRegion(tableName, method, CONF, families);
5002 
5003     Assert.assertEquals(0L, region.getWriteRequestsCount());
5004 
5005     Put put = new Put(row);
5006     put.add(fam, fam, fam);
5007 
5008     Assert.assertEquals(0L, region.getWriteRequestsCount());
5009     region.put(put);
5010     Assert.assertEquals(1L, region.getWriteRequestsCount());
5011     region.put(put);
5012     Assert.assertEquals(2L, region.getWriteRequestsCount());
5013     region.put(put);
5014     Assert.assertEquals(3L, region.getWriteRequestsCount());
5015 
5016     region.delete(new Delete(row));
5017     Assert.assertEquals(4L, region.getWriteRequestsCount());
5018 
5019     HRegion.closeHRegion(this.region);
5020     this.region = null;
5021   }
5022 
5023   private static HRegion initHRegion(byte[] tableName, String callingMethod,
5024       byte[]... families) throws IOException {
5025     return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
5026         families);
5027   }
5028 }