View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  
22  import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23  import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24  import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25  import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29  import static org.junit.Assert.assertArrayEquals;
30  import static org.junit.Assert.assertEquals;
31  import static org.junit.Assert.assertFalse;
32  import static org.junit.Assert.assertNotNull;
33  import static org.junit.Assert.assertNull;
34  import static org.junit.Assert.assertTrue;
35  import static org.junit.Assert.fail;
36  import static org.mockito.Matchers.any;
37  import static org.mockito.Matchers.anyBoolean;
38  import static org.mockito.Matchers.anyLong;
39  import static org.mockito.Matchers.eq;
40  import static org.mockito.Mockito.never;
41  import static org.mockito.Mockito.spy;
42  import static org.mockito.Mockito.times;
43  import static org.mockito.Mockito.verify;
44  
45  import java.io.IOException;
46  import java.io.InterruptedIOException;
47  import java.security.PrivilegedExceptionAction;
48  import java.util.ArrayList;
49  import java.util.Arrays;
50  import java.util.Collection;
51  import java.util.List;
52  import java.util.Map;
53  import java.util.NavigableMap;
54  import java.util.TreeMap;
55  import java.util.UUID;
56  import java.util.concurrent.atomic.AtomicBoolean;
57  import java.util.concurrent.atomic.AtomicInteger;
58  import java.util.concurrent.atomic.AtomicLong;
59  import java.util.concurrent.atomic.AtomicReference;
60  
61  import org.apache.commons.logging.Log;
62  import org.apache.commons.logging.LogFactory;
63  import org.apache.hadoop.conf.Configuration;
64  import org.apache.hadoop.fs.FSDataOutputStream;
65  import org.apache.hadoop.fs.FileStatus;
66  import org.apache.hadoop.fs.FileSystem;
67  import org.apache.hadoop.fs.Path;
68  import org.apache.hadoop.hbase.Cell;
69  import org.apache.hadoop.hbase.CellComparator;
70  import org.apache.hadoop.hbase.CellUtil;
71  import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
72  import org.apache.hadoop.hbase.DroppedSnapshotException;
73  import org.apache.hadoop.hbase.HBaseConfiguration;
74  import org.apache.hadoop.hbase.HBaseTestCase;
75  import org.apache.hadoop.hbase.HBaseTestingUtility;
76  import org.apache.hadoop.hbase.HColumnDescriptor;
77  import org.apache.hadoop.hbase.HConstants;
78  import org.apache.hadoop.hbase.RegionTooBusyException;
79  import org.apache.hadoop.hbase.Tag;
80  import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
81  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
82  import org.apache.hadoop.hbase.HRegionInfo;
83  import org.apache.hadoop.hbase.HTableDescriptor;
84  import org.apache.hadoop.hbase.KeyValue;
85  import org.apache.hadoop.hbase.testclassification.MediumTests;
86  import org.apache.hadoop.hbase.MiniHBaseCluster;
87  import org.apache.hadoop.hbase.MultithreadedTestUtil;
88  import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
89  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
90  import org.apache.hadoop.hbase.NotServingRegionException;
91  import org.apache.hadoop.hbase.TableName;
92  import org.apache.hadoop.hbase.TagType;
93  import org.apache.hadoop.hbase.Waiter;
94  import org.apache.hadoop.hbase.client.Append;
95  import org.apache.hadoop.hbase.client.Delete;
96  import org.apache.hadoop.hbase.client.Durability;
97  import org.apache.hadoop.hbase.client.Get;
98  import org.apache.hadoop.hbase.client.HTable;
99  import org.apache.hadoop.hbase.client.Increment;
100 import org.apache.hadoop.hbase.client.Put;
101 import org.apache.hadoop.hbase.client.Result;
102 import org.apache.hadoop.hbase.client.RowMutations;
103 import org.apache.hadoop.hbase.client.Scan;
104 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
105 import org.apache.hadoop.hbase.filter.BinaryComparator;
106 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
107 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
108 import org.apache.hadoop.hbase.filter.Filter;
109 import org.apache.hadoop.hbase.filter.FilterBase;
110 import org.apache.hadoop.hbase.filter.FilterList;
111 import org.apache.hadoop.hbase.filter.NullComparator;
112 import org.apache.hadoop.hbase.filter.PrefixFilter;
113 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
114 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
115 import org.apache.hadoop.hbase.io.hfile.HFile;
116 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
117 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
118 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
119 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
120 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
121 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
122 import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
123 import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
124 import org.apache.hadoop.hbase.regionserver.wal.FaultyHLog;
125 import org.apache.hadoop.hbase.regionserver.wal.HLog;
126 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
127 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
128 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
129 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
130 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
131 import org.apache.hadoop.hbase.security.User;
132 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
133 import org.apache.hadoop.hbase.util.Bytes;
134 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
135 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
136 import org.apache.hadoop.hbase.util.FSUtils;
137 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
138 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
139 import org.apache.hadoop.hbase.util.PairOfSameType;
140 import org.apache.hadoop.hbase.util.Threads;
141 import org.junit.After;
142 import org.junit.Assert;
143 import org.junit.Before;
144 import org.junit.Rule;
145 import org.junit.Test;
146 import org.junit.experimental.categories.Category;
147 import org.junit.rules.TestName;
148 import org.mockito.Mockito;
149 
150 import com.google.common.collect.ImmutableList;
151 import com.google.common.collect.Lists;
152 import com.google.protobuf.ByteString;
153 
154 /**
155  * Basic stand-alone testing of HRegion.  No clusters!
156  *
157  * A lot of the meta information for an HRegion now lives inside other HRegions
158  * or in the HBaseMaster, so only basic testing is possible.
159  */
160 @Category(MediumTests.class)
161 @SuppressWarnings("deprecation")
162 public class TestHRegion {
163   // Do not spin up clusters in here. If you need to spin up a cluster, do it
164   // over in TestHRegionOnCluster.
165   static final Log LOG = LogFactory.getLog(TestHRegion.class);
166   @Rule public TestName name = new TestName();
167 
168   private static final String COLUMN_FAMILY = "MyCF";
169   private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
170 
171   HRegion region = null;
172   // Do not run unit tests in parallel (? Why not?  It don't work?  Why not?  St.Ack)
173   private static HBaseTestingUtility TEST_UTIL;
174   public static Configuration CONF ;
175   private String dir;
176   private static FileSystem FILESYSTEM;
177   private final int MAX_VERSIONS = 2;
178 
179   // Test names
180   protected byte[] tableName;
181   protected String method;
182   protected final byte[] qual1 = Bytes.toBytes("qual1");
183   protected final byte[] qual2 = Bytes.toBytes("qual2");
184   protected final byte[] qual3 = Bytes.toBytes("qual3");
185   protected final byte[] value1 = Bytes.toBytes("value1");
186   protected final byte[] value2 = Bytes.toBytes("value2");
187   protected final byte[] row = Bytes.toBytes("rowA");
188   protected final byte[] row2 = Bytes.toBytes("rowB");
189 
190   protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
191       .getInstance(MetricsAssertHelper.class);
192 
193   @Before
194   public void setup() throws IOException {
195     TEST_UTIL = HBaseTestingUtility.createLocalHTU();
196     FILESYSTEM = TEST_UTIL.getTestFileSystem();
197     CONF = TEST_UTIL.getConfiguration();
198     dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
199     method = name.getMethodName();
200     tableName = Bytes.toBytes(name.getMethodName());
201   }
202 
203   @After
204   public void tearDown() throws Exception {
205     EnvironmentEdgeManagerTestHelper.reset();
206     LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
207     TEST_UTIL.cleanupTestDir();
208   }
209 
210   String getName() {
211     return name.getMethodName();
212   }
213 
214   /**
215    * Test for Bug 2 of HBASE-10466.
216    * "Bug 2: Conditions for the first flush of region close (so-called pre-flush) If memstoreSize
217    * is smaller than a certain value, or when region close starts a flush is ongoing, the first
218    * flush is skipped and only the second flush takes place. However, two flushes are required in
219    * case previous flush fails and leaves some data in snapshot. The bug could cause loss of data
220    * in current memstore. The fix is removing all conditions except abort check so we ensure 2
221    * flushes for region close."
222    * @throws IOException
223    */
224   @Test (timeout=60000)
225   public void testCloseCarryingSnapshot() throws IOException {
226     HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
227     Store store = region.getStore(COLUMN_FAMILY_BYTES);
228     // Get some random bytes.
229     byte [] value = Bytes.toBytes(name.getMethodName());
230     // Make a random put against our cf.
231     Put put = new Put(value);
232     put.add(COLUMN_FAMILY_BYTES, null, value);
233     // First put something in current memstore, which will be in snapshot after flusher.prepare()
234     region.put(put);
235     StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
236     storeFlushCtx.prepare();
237     // Second put something in current memstore
238     put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
239     region.put(put);
240     // Close with something in memstore and something in the snapshot.  Make sure all is cleared.
241     region.close();
242     assertEquals(0, region.getMemstoreSize().get());
243     HRegion.closeHRegion(region);
244   }
245 
246   /*
247    * This test is for verifying memstore snapshot size is correctly updated in case of rollback
248    * See HBASE-10845
249    */
250   @Test (timeout=60000)
251   public void testMemstoreSnapshotSize() throws IOException {
252     class MyFaultyHLog extends FaultyHLog {
253       StoreFlushContext storeFlushCtx;
254       public MyFaultyHLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
255           throws IOException {
256         super(fs, rootDir, logName, conf);
257       }
258       
259       void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
260         this.storeFlushCtx = storeFlushCtx;
261       }
262 
263       @Override
264       public void sync(long txid) throws IOException {
265         storeFlushCtx.prepare();
266         super.sync(txid);
267       }
268     }
269     
270     FileSystem fs = FileSystem.get(CONF);
271     Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
272     MyFaultyHLog faultyLog = new MyFaultyHLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
273     HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
274       CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES);
275     
276     Store store = region.getStore(COLUMN_FAMILY_BYTES);
277     // Get some random bytes.
278     byte [] value = Bytes.toBytes(name.getMethodName());
279     faultyLog.setStoreFlushCtx(store.createFlushContext(12345));
280     
281     Put put = new Put(value);
282     put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
283     faultyLog.setFailureType(FaultyHLog.FailureType.SYNC);
284 
285     boolean threwIOE = false;
286     try {
287       region.put(put);
288     } catch (IOException ioe) {
289       threwIOE = true;
290     } finally {
291       assertTrue("The regionserver should have thrown an exception", threwIOE);
292     }
293     long sz = store.getFlushableSize();
294     assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
295     HRegion.closeHRegion(region);
296   }
297   
298   /**
299    * Test we do not lose data if we fail a flush and then close.
300    * Part of HBase-10466.  Tests the following from the issue description:
301    * "Bug 1: Wrong calculation of HRegion.memstoreSize: When a flush fails, data to be flushed is
302    * kept in each MemStore's snapshot and wait for next flush attempt to continue on it. But when
303    * the next flush succeeds, the counter of total memstore size in HRegion is always deduced by
304    * the sum of current memstore sizes instead of snapshots left from previous failed flush. This
305    * calculation is problematic that almost every time there is failed flush, HRegion.memstoreSize
306    * gets reduced by a wrong value. If region flush could not proceed for a couple cycles, the size
307    * in current memstore could be much larger than the snapshot. It's likely to drift memstoreSize
308    * much smaller than expected. In extreme case, if the error accumulates to even bigger than
309    * HRegion's memstore size limit, any further flush is skipped because flush does not do anything
310    * if memstoreSize is not larger than 0."
311    * @throws Exception
312    */
313   @Test (timeout=60000)
314   public void testFlushSizeAccounting() throws Exception {
315     final Configuration conf = HBaseConfiguration.create(CONF);
316     // Only retry once.
317     conf.setInt("hbase.hstore.flush.retries.number", 1);
318     final User user =
319       User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
320     // Inject our faulty LocalFileSystem
321     conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
322     user.runAs(new PrivilegedExceptionAction<Object>() {
323       @Override
324       public Object run() throws Exception {
325         // Make sure it worked (above is sensitive to caching details in hadoop core)
326         FileSystem fs = FileSystem.get(conf);
327         Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
328         FaultyFileSystem ffs = (FaultyFileSystem)fs;
329         HRegion region = null;
330         try {
331           // Initialize region
332           region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
333           long size = region.getMemstoreSize().get();
334           Assert.assertEquals(0, size);
335           // Put one item into memstore.  Measure the size of one item in memstore.
336           Put p1 = new Put(row);
337           p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
338           region.put(p1);
339           final long sizeOfOnePut = region.getMemstoreSize().get();
340           // Fail a flush which means the current memstore will hang out as memstore 'snapshot'.
341           try {
342             LOG.info("Flushing");
343             region.flushcache();
344             Assert.fail("Didn't bubble up IOE!");
345           } catch (DroppedSnapshotException dse) {
346             // What we are expecting
347             region.closing.set(false); // this is needed for the rest of the test to work
348           }
349           // Make it so all writes succeed from here on out
350           ffs.fault.set(false);
351           // Check sizes.  Should still be the one entry.
352           Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize().get());
353           // Now add two entries so that on this next flush that fails, we can see if we
354           // subtract the right amount, the snapshot size only.
355           Put p2 = new Put(row);
356           p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
357           p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
358           region.put(p2);
359           Assert.assertEquals(sizeOfOnePut * 3, region.getMemstoreSize().get());
360           // Do a successful flush.  It will clear the snapshot only.  Thats how flushes work.
361           // If already a snapshot, we clear it else we move the memstore to be snapshot and flush
362           // it
363           region.flushcache();
364           // Make sure our memory accounting is right.
365           Assert.assertEquals(sizeOfOnePut * 2, region.getMemstoreSize().get());
366         } finally {
367           HRegion.closeHRegion(region);
368         }
369         return null;
370       }
371     });
372     FileSystem.closeAllForUGI(user.getUGI());
373   }
374 
375   @Test
376   public void testCompactionAffectedByScanners() throws Exception {
377     byte[] family = Bytes.toBytes("family");
378     this.region = initHRegion(tableName, method, CONF, family);
379 
380     Put put = new Put(Bytes.toBytes("r1"));
381     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
382     region.put(put);
383     region.flushcache();
384 
385     Scan scan = new Scan();
386     scan.setMaxVersions(3);
387     // open the first scanner
388     RegionScanner scanner1 = region.getScanner(scan);
389 
390     Delete delete = new Delete(Bytes.toBytes("r1"));
391     region.delete(delete);
392     region.flushcache();
393 
394     // open the second scanner
395     RegionScanner scanner2 = region.getScanner(scan);
396 
397     List<Cell> results = new ArrayList<Cell>();
398 
399     System.out.println("Smallest read point:" + region.getSmallestReadPoint());
400 
401     // make a major compaction
402     region.compactStores(true);
403 
404     // open the third scanner
405     RegionScanner scanner3 = region.getScanner(scan);
406 
407     // get data from scanner 1, 2, 3 after major compaction
408     scanner1.next(results);
409     System.out.println(results);
410     assertEquals(1, results.size());
411 
412     results.clear();
413     scanner2.next(results);
414     System.out.println(results);
415     assertEquals(0, results.size());
416 
417     results.clear();
418     scanner3.next(results);
419     System.out.println(results);
420     assertEquals(0, results.size());
421   }
422 
423   @Test
424   public void testToShowNPEOnRegionScannerReseek() throws Exception {
425     byte[] family = Bytes.toBytes("family");
426     this.region = initHRegion(tableName, method, CONF, family);
427 
428     Put put = new Put(Bytes.toBytes("r1"));
429     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
430     region.put(put);
431     put = new Put(Bytes.toBytes("r2"));
432     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
433     region.put(put);
434     region.flushcache();
435 
436     Scan scan = new Scan();
437     scan.setMaxVersions(3);
438     // open the first scanner
439     RegionScanner scanner1 = region.getScanner(scan);
440 
441     System.out.println("Smallest read point:" + region.getSmallestReadPoint());
442 
443     region.compactStores(true);
444 
445     scanner1.reseek(Bytes.toBytes("r2"));
446     List<Cell> results = new ArrayList<Cell>();
447     scanner1.next(results);
448     Cell keyValue = results.get(0);
449     Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
450     scanner1.close();
451   }
452 
453   @Test
454   public void testSkipRecoveredEditsReplay() throws Exception {
455     String method = "testSkipRecoveredEditsReplay";
456     TableName tableName = TableName.valueOf(method);
457     byte[] family = Bytes.toBytes("family");
458     this.region = initHRegion(tableName, method, CONF, family);
459     try {
460       Path regiondir = region.getRegionFileSystem().getRegionDir();
461       FileSystem fs = region.getRegionFileSystem().getFileSystem();
462       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
463 
464       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
465 
466       long maxSeqId = 1050;
467       long minSeqId = 1000;
468 
469       for (long i = minSeqId; i <= maxSeqId; i += 10) {
470         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
471         fs.create(recoveredEdits);
472         HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
473 
474         long time = System.nanoTime();
475         WALEdit edit = new WALEdit();
476         edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
477             .toBytes(i)));
478         writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
479             HConstants.DEFAULT_CLUSTER_ID), edit));
480 
481         writer.close();
482       }
483       MonitoredTask status = TaskMonitor.get().createStatus(method);
484       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
485       for (Store store : region.getStores().values()) {
486         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
487       }
488       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
489       assertEquals(maxSeqId, seqId);
490       Get get = new Get(row);
491       Result result = region.get(get);
492       for (long i = minSeqId; i <= maxSeqId; i += 10) {
493         List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
494         assertEquals(1, kvs.size());
495         assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
496       }
497     } finally {
498       HRegion.closeHRegion(this.region);
499       this.region = null;
500     }
501   }
502 
503   @Test
504   public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
505     String method = "testSkipRecoveredEditsReplaySomeIgnored";
506     TableName tableName = TableName.valueOf(method);
507     byte[] family = Bytes.toBytes("family");
508     this.region = initHRegion(tableName, method, CONF, family);
509     try {
510       Path regiondir = region.getRegionFileSystem().getRegionDir();
511       FileSystem fs = region.getRegionFileSystem().getFileSystem();
512       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
513 
514       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
515 
516       long maxSeqId = 1050;
517       long minSeqId = 1000;
518 
519       for (long i = minSeqId; i <= maxSeqId; i += 10) {
520         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
521         fs.create(recoveredEdits);
522         HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
523 
524         long time = System.nanoTime();
525         WALEdit edit = new WALEdit();
526         edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
527             .toBytes(i)));
528         writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
529             HConstants.DEFAULT_CLUSTER_ID), edit));
530 
531         writer.close();
532       }
533       long recoverSeqId = 1030;
534       MonitoredTask status = TaskMonitor.get().createStatus(method);
535       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
536       for (Store store : region.getStores().values()) {
537         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
538       }
539       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
540       assertEquals(maxSeqId, seqId);
541       Get get = new Get(row);
542       Result result = region.get(get);
543       for (long i = minSeqId; i <= maxSeqId; i += 10) {
544         List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
545         if (i < recoverSeqId) {
546           assertEquals(0, kvs.size());
547         } else {
548           assertEquals(1, kvs.size());
549           assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
550         }
551       }
552     } finally {
553       HRegion.closeHRegion(this.region);
554       this.region = null;
555     }
556   }
557 
558   @Test
559   public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
560     byte[] family = Bytes.toBytes("family");
561     this.region = initHRegion(tableName, method, CONF, family);
562     try {
563       Path regiondir = region.getRegionFileSystem().getRegionDir();
564       FileSystem fs = region.getRegionFileSystem().getFileSystem();
565 
566       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
567       for (int i = 1000; i < 1050; i += 10) {
568         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
569         FSDataOutputStream dos = fs.create(recoveredEdits);
570         dos.writeInt(i);
571         dos.close();
572       }
573       long minSeqId = 2000;
574       Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
575       FSDataOutputStream dos = fs.create(recoveredEdits);
576       dos.close();
577 
578       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
579       for (Store store : region.getStores().values()) {
580         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
581       }
582       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
583       assertEquals(minSeqId, seqId);
584     } finally {
585       HRegion.closeHRegion(this.region);
586       this.region = null;
587     }
588   }
589 
590   @Test
591   public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
592     String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
593     TableName tableName = TableName.valueOf(method);
594     byte[] family = Bytes.toBytes("family");
595     this.region = initHRegion(tableName, method, CONF, family);
596     try {
597       Path regiondir = region.getRegionFileSystem().getRegionDir();
598       FileSystem fs = region.getRegionFileSystem().getFileSystem();
599       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
600 
601       assertEquals(0, region.getStoreFileList(
602         region.getStores().keySet().toArray(new byte[0][])).size());
603 
604       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
605 
606       long maxSeqId = 1050;
607       long minSeqId = 1000;
608 
609       for (long i = minSeqId; i <= maxSeqId; i += 10) {
610         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
611         fs.create(recoveredEdits);
612         HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
613 
614         long time = System.nanoTime();
615         WALEdit edit = null;
616         if (i == maxSeqId) {
617           edit = WALEdit.createCompaction(region.getRegionInfo(),
618           CompactionDescriptor.newBuilder()
619           .setTableName(ByteString.copyFrom(tableName.getName()))
620           .setFamilyName(ByteString.copyFrom(regionName))
621           .setEncodedRegionName(ByteString.copyFrom(regionName))
622           .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
623           .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName()))
624           .build());
625         } else {
626           edit = new WALEdit();
627           edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
628             .toBytes(i)));
629         }
630         writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
631             HConstants.DEFAULT_CLUSTER_ID), edit));
632         writer.close();
633       }
634 
635       long recoverSeqId = 1030;
636       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
637       MonitoredTask status = TaskMonitor.get().createStatus(method);
638       for (Store store : region.getStores().values()) {
639         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
640       }
641       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
642       assertEquals(maxSeqId, seqId);
643 
644       // assert that the files are flushed
645       assertEquals(1, region.getStoreFileList(
646         region.getStores().keySet().toArray(new byte[0][])).size());
647 
648     } finally {
649       HRegion.closeHRegion(this.region);
650       this.region = null;
651     }  }
652 
653   @Test
654   public void testRecoveredEditsReplayCompaction() throws Exception {
655     String method = name.getMethodName();
656     TableName tableName = TableName.valueOf(method);
657     byte[] family = Bytes.toBytes("family");
658     this.region = initHRegion(tableName, method, CONF, family);
659     try {
660       Path regiondir = region.getRegionFileSystem().getRegionDir();
661       FileSystem fs = region.getRegionFileSystem().getFileSystem();
662       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
663 
664       long maxSeqId = 3;
665       long minSeqId = 0;
666 
667       for (long i = minSeqId; i < maxSeqId; i++) {
668         Put put = new Put(Bytes.toBytes(i));
669         put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
670         region.put(put);
671         region.flushcache();
672       }
673 
674       // this will create a region with 3 files
675       assertEquals(3, region.getStore(family).getStorefilesCount());
676       List<Path> storeFiles = new ArrayList<Path>(3);
677       for (StoreFile sf : region.getStore(family).getStorefiles()) {
678         storeFiles.add(sf.getPath());
679       }
680 
681       // disable compaction completion
682       CONF.setBoolean("hbase.hstore.compaction.complete", false);
683       region.compactStores();
684 
685       // ensure that nothing changed
686       assertEquals(3, region.getStore(family).getStorefilesCount());
687 
688       // now find the compacted file, and manually add it to the recovered edits
689       Path tmpDir = region.getRegionFileSystem().getTempDir();
690       FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
691       String errorMsg = "Expected to find 1 file in the region temp directory "
692           + "from the compaction, could not find any";
693       assertNotNull(errorMsg, files);
694       assertEquals(errorMsg, 1, files.length);
695       // move the file inside region dir
696       Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
697           files[0].getPath());
698 
699       CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
700           .getRegionInfo(), family, storeFiles, Lists.newArrayList(newFile), region
701           .getRegionFileSystem().getStoreDir(Bytes.toString(family)));
702 
703       HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
704           this.region.getRegionInfo(), compactionDescriptor, new AtomicLong(1));
705 
706       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
707 
708       Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
709       fs.create(recoveredEdits);
710       HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
711 
712       long time = System.nanoTime();
713 
714       writer.append(new HLog.Entry(new HLogKey(regionName, tableName, 10, time,
715           HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(),
716           compactionDescriptor)));
717       writer.close();
718 
719       // close the region now, and reopen again
720       region.getTableDesc();
721       region.getRegionInfo();
722       region.close();
723       region = HRegion.openHRegion(region, null);
724 
725       // now check whether we have only one store file, the compacted one
726       Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
727       for (StoreFile sf : sfs) {
728         LOG.info(sf.getPath());
729       }
730       assertEquals(1, region.getStore(family).getStorefilesCount());
731       files = FSUtils.listStatus(fs, tmpDir);
732       assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
733 
734       for (long i = minSeqId; i < maxSeqId; i++) {
735         Get get = new Get(Bytes.toBytes(i));
736         Result result = region.get(get);
737         byte[] value = result.getValue(family, Bytes.toBytes(i));
738         assertArrayEquals(Bytes.toBytes(i), value);
739       }
740     } finally {
741       HRegion.closeHRegion(this.region);
742       this.region = null;
743     }
744   }
745 
746   @Test
747   public void testGetWhileRegionClose() throws IOException {
748     TableName tableName = TableName.valueOf(name.getMethodName());
749     Configuration hc = initSplit();
750     int numRows = 100;
751     byte[][] families = { fam1, fam2, fam3 };
752 
753     // Setting up region
754     String method = name.getMethodName();
755     this.region = initHRegion(tableName, method, hc, families);
756     try {
757       // Put data in region
758       final int startRow = 100;
759       putData(startRow, numRows, qual1, families);
760       putData(startRow, numRows, qual2, families);
761       putData(startRow, numRows, qual3, families);
762       final AtomicBoolean done = new AtomicBoolean(false);
763       final AtomicInteger gets = new AtomicInteger(0);
764       GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
765       try {
766         // Set ten threads running concurrently getting from the region.
767         for (int i = 0; i < threads.length / 2; i++) {
768           threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
769           threads[i].setDaemon(true);
770           threads[i].start();
771         }
772         // Artificially make the condition by setting closing flag explicitly.
773         // I can't make the issue happen with a call to region.close().
774         this.region.closing.set(true);
775         for (int i = threads.length / 2; i < threads.length; i++) {
776           threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
777           threads[i].setDaemon(true);
778           threads[i].start();
779         }
780       } finally {
781         if (this.region != null) {
782           HRegion.closeHRegion(this.region);
783         }
784       }
785       done.set(true);
786       for (GetTillDoneOrException t : threads) {
787         try {
788           t.join();
789         } catch (InterruptedException e) {
790           e.printStackTrace();
791         }
792         if (t.e != null) {
793           LOG.info("Exception=" + t.e);
794           assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
795         }
796       }
797     } finally {
798       HRegion.closeHRegion(this.region);
799       this.region = null;
800     }
801   }
802 
803   /*
804    * Thread that does get on single row until 'done' flag is flipped. If an
805    * exception causes us to fail, it records it.
806    */
807   class GetTillDoneOrException extends Thread {
808     private final Get g;
809     private final AtomicBoolean done;
810     private final AtomicInteger count;
811     private Exception e;
812 
813     GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
814       super("getter." + i);
815       this.g = new Get(r);
816       this.done = d;
817       this.count = c;
818     }
819 
820     @Override
821     public void run() {
822       while (!this.done.get()) {
823         try {
824           assertTrue(region.get(g).size() > 0);
825           this.count.incrementAndGet();
826         } catch (Exception e) {
827           this.e = e;
828           break;
829         }
830       }
831     }
832   }
833 
834   /*
835    * An involved filter test. Has multiple column families and deletes in mix.
836    */
837   @Test
838   public void testWeirdCacheBehaviour() throws Exception {
839     byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
840     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
841         Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
842     this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
843     try {
844       String value = "this is the value";
845       String value2 = "this is some other value";
846       String keyPrefix1 = "prefix1";
847       String keyPrefix2 = "prefix2";
848       String keyPrefix3 = "prefix3";
849       putRows(this.region, 3, value, keyPrefix1);
850       putRows(this.region, 3, value, keyPrefix2);
851       putRows(this.region, 3, value, keyPrefix3);
852       putRows(this.region, 3, value2, keyPrefix1);
853       putRows(this.region, 3, value2, keyPrefix2);
854       putRows(this.region, 3, value2, keyPrefix3);
855       System.out.println("Checking values for key: " + keyPrefix1);
856       assertEquals("Got back incorrect number of rows from scan", 3,
857           getNumberOfRows(keyPrefix1, value2, this.region));
858       System.out.println("Checking values for key: " + keyPrefix2);
859       assertEquals("Got back incorrect number of rows from scan", 3,
860           getNumberOfRows(keyPrefix2, value2, this.region));
861       System.out.println("Checking values for key: " + keyPrefix3);
862       assertEquals("Got back incorrect number of rows from scan", 3,
863           getNumberOfRows(keyPrefix3, value2, this.region));
864       deleteColumns(this.region, value2, keyPrefix1);
865       deleteColumns(this.region, value2, keyPrefix2);
866       deleteColumns(this.region, value2, keyPrefix3);
867       System.out.println("Starting important checks.....");
868       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
869           getNumberOfRows(keyPrefix1, value2, this.region));
870       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
871           getNumberOfRows(keyPrefix2, value2, this.region));
872       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
873           getNumberOfRows(keyPrefix3, value2, this.region));
874     } finally {
875       HRegion.closeHRegion(this.region);
876       this.region = null;
877     }
878   }
879 
880   @Test
881   public void testAppendWithReadOnlyTable() throws Exception {
882     byte[] TABLE = Bytes.toBytes("readOnlyTable");
883     this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
884     boolean exceptionCaught = false;
885     Append append = new Append(Bytes.toBytes("somerow"));
886     append.setDurability(Durability.SKIP_WAL);
887     append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
888         Bytes.toBytes("somevalue"));
889     try {
890       region.append(append);
891     } catch (IOException e) {
892       exceptionCaught = true;
893     } finally {
894       HRegion.closeHRegion(this.region);
895       this.region = null;
896     }
897     assertTrue(exceptionCaught == true);
898   }
899 
900   @Test
901   public void testIncrWithReadOnlyTable() throws Exception {
902     byte[] TABLE = Bytes.toBytes("readOnlyTable");
903     this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
904     boolean exceptionCaught = false;
905     Increment inc = new Increment(Bytes.toBytes("somerow"));
906     inc.setDurability(Durability.SKIP_WAL);
907     inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
908     try {
909       region.increment(inc);
910     } catch (IOException e) {
911       exceptionCaught = true;
912     } finally {
913       HRegion.closeHRegion(this.region);
914       this.region = null;
915     }
916     assertTrue(exceptionCaught == true);
917   }
918 
919   private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
920     InternalScanner scanner = buildScanner(keyPrefix, value, r);
921     int count = 0;
922     boolean more = false;
923     List<Cell> results = new ArrayList<Cell>();
924     do {
925       more = scanner.next(results);
926       if (results != null && !results.isEmpty())
927         count++;
928       else
929         break;
930       Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
931       delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
932       r.delete(delete);
933       results.clear();
934     } while (more);
935     assertEquals("Did not perform correct number of deletes", 3, count);
936   }
937 
938   private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
939     InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
940     int numberOfResults = 0;
941     List<Cell> results = new ArrayList<Cell>();
942     boolean more = false;
943     do {
944       more = resultScanner.next(results);
945       if (results != null && !results.isEmpty())
946         numberOfResults++;
947       else
948         break;
949       for (Cell kv : results) {
950         System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
951       }
952       results.clear();
953     } while (more);
954     return numberOfResults;
955   }
956 
957   private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
958       throws IOException {
959     // Defaults FilterList.Operator.MUST_PASS_ALL.
960     FilterList allFilters = new FilterList();
961     allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
962     // Only return rows where this column value exists in the row.
963     SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
964         Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
965     filter.setFilterIfMissing(true);
966     allFilters.addFilter(filter);
967     Scan scan = new Scan();
968     scan.addFamily(Bytes.toBytes("trans-blob"));
969     scan.addFamily(Bytes.toBytes("trans-type"));
970     scan.addFamily(Bytes.toBytes("trans-date"));
971     scan.addFamily(Bytes.toBytes("trans-tags"));
972     scan.addFamily(Bytes.toBytes("trans-group"));
973     scan.setFilter(allFilters);
974     return r.getScanner(scan);
975   }
976 
977   private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
978     for (int i = 0; i < numRows; i++) {
979       String row = key + "_" + i/* UUID.randomUUID().toString() */;
980       System.out.println(String.format("Saving row: %s, with value %s", row, value));
981       Put put = new Put(Bytes.toBytes(row));
982       put.setDurability(Durability.SKIP_WAL);
983       put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
984       put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
985       put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
986       put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
987       put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
988       r.put(put);
989     }
990   }
991 
992   @Test
993   public void testFamilyWithAndWithoutColon() throws Exception {
994     byte[] b = Bytes.toBytes(getName());
995     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
996     this.region = initHRegion(b, getName(), CONF, cf);
997     try {
998       Put p = new Put(b);
999       byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
1000       p.add(cfwithcolon, cfwithcolon, cfwithcolon);
1001       boolean exception = false;
1002       try {
1003         this.region.put(p);
1004       } catch (NoSuchColumnFamilyException e) {
1005         exception = true;
1006       }
1007       assertTrue(exception);
1008     } finally {
1009       HRegion.closeHRegion(this.region);
1010       this.region = null;
1011     }
1012   }
1013 
1014   @Test
1015   public void testBatchPut_whileNoRowLocksHeld() throws IOException {
1016     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1017     byte[] qual = Bytes.toBytes("qual");
1018     byte[] val = Bytes.toBytes("val");
1019     this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1020     MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1021     try {
1022       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1023       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1024 
1025       LOG.info("First a batch put with all valid puts");
1026       final Put[] puts = new Put[10];
1027       for (int i = 0; i < 10; i++) {
1028         puts[i] = new Put(Bytes.toBytes("row_" + i));
1029         puts[i].add(cf, qual, val);
1030       }
1031 
1032       OperationStatus[] codes = this.region.batchMutate(puts);
1033       assertEquals(10, codes.length);
1034       for (int i = 0; i < 10; i++) {
1035         assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1036       }
1037       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1038 
1039       LOG.info("Next a batch put with one invalid family");
1040       puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1041       codes = this.region.batchMutate(puts);
1042       assertEquals(10, codes.length);
1043       for (int i = 0; i < 10; i++) {
1044         assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1045             codes[i].getOperationStatusCode());
1046       }
1047 
1048       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1049     } finally {
1050       HRegion.closeHRegion(this.region);
1051       this.region = null;
1052     }
1053   }
1054 
1055   @Test
1056   public void testBatchPut_whileMultipleRowLocksHeld() throws Exception {
1057     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1058     byte[] qual = Bytes.toBytes("qual");
1059     byte[] val = Bytes.toBytes("val");
1060     this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1061     MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1062     try {
1063       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1064       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1065 
1066       final Put[] puts = new Put[10];
1067       for (int i = 0; i < 10; i++) {
1068         puts[i] = new Put(Bytes.toBytes("row_" + i));
1069         puts[i].add(cf, qual, val);
1070       }
1071       puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1072 
1073       LOG.info("batchPut will have to break into four batches to avoid row locks");
1074       RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2"));
1075       RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_4"));
1076       RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_6"));
1077 
1078       MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1079       final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
1080       TestThread putter = new TestThread(ctx) {
1081         @Override
1082         public void doWork() throws IOException {
1083           retFromThread.set(region.batchMutate(puts));
1084         }
1085       };
1086       LOG.info("...starting put thread while holding locks");
1087       ctx.addThread(putter);
1088       ctx.startThreads();
1089 
1090       LOG.info("...waiting for put thread to sync 1st time");
1091       waitForCounter(source, "syncTimeNumOps", syncs + 1);
1092 
1093       // Now attempt to close the region from another thread.  Prior to HBASE-12565
1094       // this would cause the in-progress batchMutate operation to to fail with
1095       // exception because it use to release and re-acquire the close-guard lock
1096       // between batches.  Caller then didn't get status indicating which writes succeeded.
1097       // We now expect this thread to block until the batchMutate call finishes.
1098       Thread regionCloseThread = new Thread() {
1099         @Override
1100         public void run() {
1101           try {
1102             HRegion.closeHRegion(region);
1103           } catch (IOException e) {
1104             throw new RuntimeException(e);
1105           }
1106         }
1107       };
1108       regionCloseThread.start();
1109 
1110       LOG.info("...releasing row lock 1, which should let put thread continue");
1111       rowLock1.release();
1112 
1113       LOG.info("...waiting for put thread to sync 2nd time");
1114       waitForCounter(source, "syncTimeNumOps", syncs + 2);
1115 
1116       LOG.info("...releasing row lock 2, which should let put thread continue");
1117       rowLock2.release();
1118 
1119       LOG.info("...waiting for put thread to sync 3rd time");
1120       waitForCounter(source, "syncTimeNumOps", syncs + 3);
1121 
1122       LOG.info("...releasing row lock 3, which should let put thread continue");
1123       rowLock3.release();
1124 
1125       LOG.info("...waiting for put thread to sync 4th time");
1126       waitForCounter(source, "syncTimeNumOps", syncs + 4);
1127 
1128       LOG.info("...joining on put thread");
1129       ctx.stop();
1130       regionCloseThread.join();
1131 
1132       OperationStatus[] codes = retFromThread.get();
1133       for (int i = 0; i < codes.length; i++) {
1134         assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1135             codes[i].getOperationStatusCode());
1136       }
1137     } finally {
1138       HRegion.closeHRegion(this.region);
1139       this.region = null;
1140     }
1141   }
1142 
1143   private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount)
1144       throws InterruptedException {
1145     long startWait = System.currentTimeMillis();
1146     long currentCount;
1147     while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) {
1148       Thread.sleep(100);
1149       if (System.currentTimeMillis() - startWait > 10000) {
1150         fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName,
1151           expectedCount, currentCount));
1152       }
1153     }
1154   }
1155 
1156 
1157   @Test
1158   public void testBatchPutWithTsSlop() throws Exception {
1159     byte[] b = Bytes.toBytes(getName());
1160     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1161     byte[] qual = Bytes.toBytes("qual");
1162     byte[] val = Bytes.toBytes("val");
1163 
1164     // add data with a timestamp that is too recent for range. Ensure assert
1165     CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1166     this.region = initHRegion(b, getName(), CONF, cf);
1167 
1168     try {
1169       MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1170       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1171       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1172 
1173       final Put[] puts = new Put[10];
1174       for (int i = 0; i < 10; i++) {
1175         puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
1176         puts[i].add(cf, qual, val);
1177       }
1178 
1179       OperationStatus[] codes = this.region.batchMutate(puts);
1180       assertEquals(10, codes.length);
1181       for (int i = 0; i < 10; i++) {
1182         assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1183       }
1184       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1185 
1186     } finally {
1187       HRegion.closeHRegion(this.region);
1188       this.region = null;
1189     }
1190 
1191   }
1192 
1193   // ////////////////////////////////////////////////////////////////////////////
1194   // checkAndMutate tests
1195   // ////////////////////////////////////////////////////////////////////////////
1196   @Test
1197   public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1198     byte[] row1 = Bytes.toBytes("row1");
1199     byte[] fam1 = Bytes.toBytes("fam1");
1200     byte[] qf1 = Bytes.toBytes("qualifier");
1201     byte[] emptyVal = new byte[] {};
1202     byte[] val1 = Bytes.toBytes("value1");
1203     byte[] val2 = Bytes.toBytes("value2");
1204 
1205     // Setting up region
1206     String method = this.getName();
1207     this.region = initHRegion(tableName, method, CONF, fam1);
1208     try {
1209       // Putting empty data in key
1210       Put put = new Put(row1);
1211       put.add(fam1, qf1, emptyVal);
1212 
1213       // checkAndPut with empty value
1214       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1215           emptyVal), put, true);
1216       assertTrue(res);
1217 
1218       // Putting data in key
1219       put = new Put(row1);
1220       put.add(fam1, qf1, val1);
1221 
1222       // checkAndPut with correct value
1223       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1224           put, true);
1225       assertTrue(res);
1226 
1227       // not empty anymore
1228       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1229           put, true);
1230       assertFalse(res);
1231 
1232       Delete delete = new Delete(row1);
1233       delete.deleteColumn(fam1, qf1);
1234       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1235           delete, true);
1236       assertFalse(res);
1237 
1238       put = new Put(row1);
1239       put.add(fam1, qf1, val2);
1240       // checkAndPut with correct value
1241       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1242           put, true);
1243       assertTrue(res);
1244 
1245       // checkAndDelete with correct value
1246       delete = new Delete(row1);
1247       delete.deleteColumn(fam1, qf1);
1248       delete.deleteColumn(fam1, qf1);
1249       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1250           delete, true);
1251       assertTrue(res);
1252 
1253       delete = new Delete(row1);
1254       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1255           delete, true);
1256       assertTrue(res);
1257 
1258       // checkAndPut looking for a null value
1259       put = new Put(row1);
1260       put.add(fam1, qf1, val1);
1261 
1262       res = region
1263           .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
1264       assertTrue(res);
1265     } finally {
1266       HRegion.closeHRegion(this.region);
1267       this.region = null;
1268     }
1269   }
1270 
1271   @Test
1272   public void testCheckAndMutate_WithWrongValue() throws IOException {
1273     byte[] row1 = Bytes.toBytes("row1");
1274     byte[] fam1 = Bytes.toBytes("fam1");
1275     byte[] qf1 = Bytes.toBytes("qualifier");
1276     byte[] val1 = Bytes.toBytes("value1");
1277     byte[] val2 = Bytes.toBytes("value2");
1278 
1279     // Setting up region
1280     String method = this.getName();
1281     this.region = initHRegion(tableName, method, CONF, fam1);
1282     try {
1283       // Putting data in key
1284       Put put = new Put(row1);
1285       put.add(fam1, qf1, val1);
1286       region.put(put);
1287 
1288       // checkAndPut with wrong value
1289       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1290           val2), put, true);
1291       assertEquals(false, res);
1292 
1293       // checkAndDelete with wrong value
1294       Delete delete = new Delete(row1);
1295       delete.deleteFamily(fam1);
1296       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1297           put, true);
1298       assertEquals(false, res);
1299     } finally {
1300       HRegion.closeHRegion(this.region);
1301       this.region = null;
1302     }
1303   }
1304 
1305   @Test
1306   public void testCheckAndMutate_WithCorrectValue() throws IOException {
1307     byte[] row1 = Bytes.toBytes("row1");
1308     byte[] fam1 = Bytes.toBytes("fam1");
1309     byte[] qf1 = Bytes.toBytes("qualifier");
1310     byte[] val1 = Bytes.toBytes("value1");
1311 
1312     // Setting up region
1313     String method = this.getName();
1314     this.region = initHRegion(tableName, method, CONF, fam1);
1315     try {
1316       // Putting data in key
1317       Put put = new Put(row1);
1318       put.add(fam1, qf1, val1);
1319       region.put(put);
1320 
1321       // checkAndPut with correct value
1322       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1323           val1), put, true);
1324       assertEquals(true, res);
1325 
1326       // checkAndDelete with correct value
1327       Delete delete = new Delete(row1);
1328       delete.deleteColumn(fam1, qf1);
1329       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1330           delete, true);
1331       assertEquals(true, res);
1332     } finally {
1333       HRegion.closeHRegion(this.region);
1334       this.region = null;
1335     }
1336   }
1337 
1338   @Test
1339   public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1340     byte[] row1 = Bytes.toBytes("row1");
1341     byte[] fam1 = Bytes.toBytes("fam1");
1342     byte[] qf1 = Bytes.toBytes("qualifier");
1343     byte[] val1 = Bytes.toBytes("value1");
1344     byte[] val2 = Bytes.toBytes("value2");
1345     byte[] val3 = Bytes.toBytes("value3");
1346     byte[] val4 = Bytes.toBytes("value4");
1347 
1348     // Setting up region
1349     String method = this.getName();
1350     this.region = initHRegion(tableName, method, CONF, fam1);
1351     try {
1352       // Putting val3 in key
1353       Put put = new Put(row1);
1354       put.add(fam1, qf1, val3);
1355       region.put(put);
1356 
1357       // Test CompareOp.LESS: original = val3, compare with val3, fail
1358       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1359           new BinaryComparator(val3), put, true);
1360       assertEquals(false, res);
1361 
1362       // Test CompareOp.LESS: original = val3, compare with val4, fail
1363       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1364           new BinaryComparator(val4), put, true);
1365       assertEquals(false, res);
1366 
1367       // Test CompareOp.LESS: original = val3, compare with val2,
1368       // succeed (now value = val2)
1369       put = new Put(row1);
1370       put.add(fam1, qf1, val2);
1371       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1372           new BinaryComparator(val2), put, true);
1373       assertEquals(true, res);
1374 
1375       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail
1376       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1377           new BinaryComparator(val3), put, true);
1378       assertEquals(false, res);
1379 
1380       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2,
1381       // succeed (value still = val2)
1382       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1383           new BinaryComparator(val2), put, true);
1384       assertEquals(true, res);
1385 
1386       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1,
1387       // succeed (now value = val3)
1388       put = new Put(row1);
1389       put.add(fam1, qf1, val3);
1390       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1391           new BinaryComparator(val1), put, true);
1392       assertEquals(true, res);
1393 
1394       // Test CompareOp.GREATER: original = val3, compare with val3, fail
1395       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1396           new BinaryComparator(val3), put, true);
1397       assertEquals(false, res);
1398 
1399       // Test CompareOp.GREATER: original = val3, compare with val2, fail
1400       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1401           new BinaryComparator(val2), put, true);
1402       assertEquals(false, res);
1403 
1404       // Test CompareOp.GREATER: original = val3, compare with val4,
1405       // succeed (now value = val2)
1406       put = new Put(row1);
1407       put.add(fam1, qf1, val2);
1408       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1409           new BinaryComparator(val4), put, true);
1410       assertEquals(true, res);
1411 
1412       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail
1413       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1414           new BinaryComparator(val1), put, true);
1415       assertEquals(false, res);
1416 
1417       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2,
1418       // succeed (value still = val2)
1419       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1420           new BinaryComparator(val2), put, true);
1421       assertEquals(true, res);
1422 
1423       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed
1424       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1425           new BinaryComparator(val3), put, true);
1426       assertEquals(true, res);
1427     } finally {
1428       HRegion.closeHRegion(this.region);
1429       this.region = null;
1430     }
1431   }
1432 
1433   @Test
1434   public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1435     byte[] row1 = Bytes.toBytes("row1");
1436     byte[] fam1 = Bytes.toBytes("fam1");
1437     byte[] fam2 = Bytes.toBytes("fam2");
1438     byte[] qf1 = Bytes.toBytes("qualifier");
1439     byte[] val1 = Bytes.toBytes("value1");
1440     byte[] val2 = Bytes.toBytes("value2");
1441 
1442     byte[][] families = { fam1, fam2 };
1443 
1444     // Setting up region
1445     String method = this.getName();
1446     this.region = initHRegion(tableName, method, CONF, families);
1447     try {
1448       // Putting data in the key to check
1449       Put put = new Put(row1);
1450       put.add(fam1, qf1, val1);
1451       region.put(put);
1452 
1453       // Creating put to add
1454       long ts = System.currentTimeMillis();
1455       KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1456       put = new Put(row1);
1457       put.add(kv);
1458 
1459       // checkAndPut with wrong value
1460       HStore store = (HStore) region.getStore(fam1);
1461       store.memstore.kvset.size();
1462 
1463       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1464           val1), put, true);
1465       assertEquals(true, res);
1466       store.memstore.kvset.size();
1467 
1468       Get get = new Get(row1);
1469       get.addColumn(fam2, qf1);
1470       Cell[] actual = region.get(get).rawCells();
1471 
1472       Cell[] expected = { kv };
1473 
1474       assertEquals(expected.length, actual.length);
1475       for (int i = 0; i < actual.length; i++) {
1476         assertEquals(expected[i], actual[i]);
1477       }
1478     } finally {
1479       HRegion.closeHRegion(this.region);
1480       this.region = null;
1481     }
1482   }
1483 
1484   @Test
1485   public void testCheckAndPut_wrongRowInPut() throws IOException {
1486     TableName tableName = TableName.valueOf(name.getMethodName());
1487     this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
1488     try {
1489       Put put = new Put(row2);
1490       put.add(fam1, qual1, value1);
1491       try {
1492         region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1493             new BinaryComparator(value2), put, false);
1494         fail();
1495       } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1496         // expected exception.
1497       }
1498     } finally {
1499       HRegion.closeHRegion(this.region);
1500       this.region = null;
1501     }
1502   }
1503 
1504   @Test
1505   public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1506     byte[] row1 = Bytes.toBytes("row1");
1507     byte[] fam1 = Bytes.toBytes("fam1");
1508     byte[] fam2 = Bytes.toBytes("fam2");
1509     byte[] qf1 = Bytes.toBytes("qualifier1");
1510     byte[] qf2 = Bytes.toBytes("qualifier2");
1511     byte[] qf3 = Bytes.toBytes("qualifier3");
1512     byte[] val1 = Bytes.toBytes("value1");
1513     byte[] val2 = Bytes.toBytes("value2");
1514     byte[] val3 = Bytes.toBytes("value3");
1515     byte[] emptyVal = new byte[] {};
1516 
1517     byte[][] families = { fam1, fam2 };
1518 
1519     // Setting up region
1520     String method = this.getName();
1521     this.region = initHRegion(tableName, method, CONF, families);
1522     try {
1523       // Put content
1524       Put put = new Put(row1);
1525       put.add(fam1, qf1, val1);
1526       region.put(put);
1527       Threads.sleep(2);
1528 
1529       put = new Put(row1);
1530       put.add(fam1, qf1, val2);
1531       put.add(fam2, qf1, val3);
1532       put.add(fam2, qf2, val2);
1533       put.add(fam2, qf3, val1);
1534       put.add(fam1, qf3, val1);
1535       region.put(put);
1536 
1537       // Multi-column delete
1538       Delete delete = new Delete(row1);
1539       delete.deleteColumn(fam1, qf1);
1540       delete.deleteColumn(fam2, qf1);
1541       delete.deleteColumn(fam1, qf3);
1542       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1543           val2), delete, true);
1544       assertEquals(true, res);
1545 
1546       Get get = new Get(row1);
1547       get.addColumn(fam1, qf1);
1548       get.addColumn(fam1, qf3);
1549       get.addColumn(fam2, qf2);
1550       Result r = region.get(get);
1551       assertEquals(2, r.size());
1552       assertArrayEquals(val1, r.getValue(fam1, qf1));
1553       assertArrayEquals(val2, r.getValue(fam2, qf2));
1554 
1555       // Family delete
1556       delete = new Delete(row1);
1557       delete.deleteFamily(fam2);
1558       res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1559           delete, true);
1560       assertEquals(true, res);
1561 
1562       get = new Get(row1);
1563       r = region.get(get);
1564       assertEquals(1, r.size());
1565       assertArrayEquals(val1, r.getValue(fam1, qf1));
1566 
1567       // Row delete
1568       delete = new Delete(row1);
1569       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1570           delete, true);
1571       assertEquals(true, res);
1572       get = new Get(row1);
1573       r = region.get(get);
1574       assertEquals(0, r.size());
1575     } finally {
1576       HRegion.closeHRegion(this.region);
1577       this.region = null;
1578     }
1579   }
1580 
1581   // ////////////////////////////////////////////////////////////////////////////
1582   // Delete tests
1583   // ////////////////////////////////////////////////////////////////////////////
1584   @Test
1585   public void testDelete_multiDeleteColumn() throws IOException {
1586     byte[] row1 = Bytes.toBytes("row1");
1587     byte[] fam1 = Bytes.toBytes("fam1");
1588     byte[] qual = Bytes.toBytes("qualifier");
1589     byte[] value = Bytes.toBytes("value");
1590 
1591     Put put = new Put(row1);
1592     put.add(fam1, qual, 1, value);
1593     put.add(fam1, qual, 2, value);
1594 
1595     String method = this.getName();
1596     this.region = initHRegion(tableName, method, CONF, fam1);
1597     try {
1598       region.put(put);
1599 
1600       // We do support deleting more than 1 'latest' version
1601       Delete delete = new Delete(row1);
1602       delete.deleteColumn(fam1, qual);
1603       delete.deleteColumn(fam1, qual);
1604       region.delete(delete);
1605 
1606       Get get = new Get(row1);
1607       get.addFamily(fam1);
1608       Result r = region.get(get);
1609       assertEquals(0, r.size());
1610     } finally {
1611       HRegion.closeHRegion(this.region);
1612       this.region = null;
1613     }
1614   }
1615 
1616   @Test
1617   public void testDelete_CheckFamily() throws IOException {
1618     byte[] row1 = Bytes.toBytes("row1");
1619     byte[] fam1 = Bytes.toBytes("fam1");
1620     byte[] fam2 = Bytes.toBytes("fam2");
1621     byte[] fam3 = Bytes.toBytes("fam3");
1622     byte[] fam4 = Bytes.toBytes("fam4");
1623 
1624     // Setting up region
1625     String method = this.getName();
1626     this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
1627     try {
1628       List<Cell> kvs = new ArrayList<Cell>();
1629       kvs.add(new KeyValue(row1, fam4, null, null));
1630 
1631       // testing existing family
1632       byte[] family = fam2;
1633       try {
1634         NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1635             Bytes.BYTES_COMPARATOR);
1636         deleteMap.put(family, kvs);
1637         region.delete(deleteMap, Durability.SYNC_WAL);
1638       } catch (Exception e) {
1639         assertTrue("Family " + new String(family) + " does not exist", false);
1640       }
1641 
1642       // testing non existing family
1643       boolean ok = false;
1644       family = fam4;
1645       try {
1646         NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1647             Bytes.BYTES_COMPARATOR);
1648         deleteMap.put(family, kvs);
1649         region.delete(deleteMap, Durability.SYNC_WAL);
1650       } catch (Exception e) {
1651         ok = true;
1652       }
1653       assertEquals("Family " + new String(family) + " does exist", true, ok);
1654     } finally {
1655       HRegion.closeHRegion(this.region);
1656       this.region = null;
1657     }
1658   }
1659 
1660   @Test
1661   public void testDelete_mixed() throws IOException, InterruptedException {
1662     byte[] fam = Bytes.toBytes("info");
1663     byte[][] families = { fam };
1664     String method = this.getName();
1665     this.region = initHRegion(tableName, method, CONF, families);
1666     try {
1667       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1668 
1669       byte[] row = Bytes.toBytes("table_name");
1670       // column names
1671       byte[] serverinfo = Bytes.toBytes("serverinfo");
1672       byte[] splitA = Bytes.toBytes("splitA");
1673       byte[] splitB = Bytes.toBytes("splitB");
1674 
1675       // add some data:
1676       Put put = new Put(row);
1677       put.add(fam, splitA, Bytes.toBytes("reference_A"));
1678       region.put(put);
1679 
1680       put = new Put(row);
1681       put.add(fam, splitB, Bytes.toBytes("reference_B"));
1682       region.put(put);
1683 
1684       put = new Put(row);
1685       put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1686       region.put(put);
1687 
1688       // ok now delete a split:
1689       Delete delete = new Delete(row);
1690       delete.deleteColumns(fam, splitA);
1691       region.delete(delete);
1692 
1693       // assert some things:
1694       Get get = new Get(row).addColumn(fam, serverinfo);
1695       Result result = region.get(get);
1696       assertEquals(1, result.size());
1697 
1698       get = new Get(row).addColumn(fam, splitA);
1699       result = region.get(get);
1700       assertEquals(0, result.size());
1701 
1702       get = new Get(row).addColumn(fam, splitB);
1703       result = region.get(get);
1704       assertEquals(1, result.size());
1705 
1706       // Assert that after a delete, I can put.
1707       put = new Put(row);
1708       put.add(fam, splitA, Bytes.toBytes("reference_A"));
1709       region.put(put);
1710       get = new Get(row);
1711       result = region.get(get);
1712       assertEquals(3, result.size());
1713 
1714       // Now delete all... then test I can add stuff back
1715       delete = new Delete(row);
1716       region.delete(delete);
1717       assertEquals(0, region.get(get).size());
1718 
1719       region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
1720       result = region.get(get);
1721       assertEquals(1, result.size());
1722     } finally {
1723       HRegion.closeHRegion(this.region);
1724       this.region = null;
1725     }
1726   }
1727 
1728   @Test
1729   public void testDeleteRowWithFutureTs() throws IOException {
1730     byte[] fam = Bytes.toBytes("info");
1731     byte[][] families = { fam };
1732     String method = this.getName();
1733     this.region = initHRegion(tableName, method, CONF, families);
1734     try {
1735       byte[] row = Bytes.toBytes("table_name");
1736       // column names
1737       byte[] serverinfo = Bytes.toBytes("serverinfo");
1738 
1739       // add data in the far future
1740       Put put = new Put(row);
1741       put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
1742       region.put(put);
1743 
1744       // now delete something in the present
1745       Delete delete = new Delete(row);
1746       region.delete(delete);
1747 
1748       // make sure we still see our data
1749       Get get = new Get(row).addColumn(fam, serverinfo);
1750       Result result = region.get(get);
1751       assertEquals(1, result.size());
1752 
1753       // delete the future row
1754       delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
1755       region.delete(delete);
1756 
1757       // make sure it is gone
1758       get = new Get(row).addColumn(fam, serverinfo);
1759       result = region.get(get);
1760       assertEquals(0, result.size());
1761     } finally {
1762       HRegion.closeHRegion(this.region);
1763       this.region = null;
1764     }
1765   }
1766 
1767   /**
1768    * Tests that the special LATEST_TIMESTAMP option for puts gets replaced by
1769    * the actual timestamp
1770    */
1771   @Test
1772   public void testPutWithLatestTS() throws IOException {
1773     byte[] fam = Bytes.toBytes("info");
1774     byte[][] families = { fam };
1775     String method = this.getName();
1776     this.region = initHRegion(tableName, method, CONF, families);
1777     try {
1778       byte[] row = Bytes.toBytes("row1");
1779       // column names
1780       byte[] qual = Bytes.toBytes("qual");
1781 
1782       // add data with LATEST_TIMESTAMP, put without WAL
1783       Put put = new Put(row);
1784       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1785       region.put(put);
1786 
1787       // Make sure it shows up with an actual timestamp
1788       Get get = new Get(row).addColumn(fam, qual);
1789       Result result = region.get(get);
1790       assertEquals(1, result.size());
1791       Cell kv = result.rawCells()[0];
1792       LOG.info("Got: " + kv);
1793       assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1794           kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1795 
1796       // Check same with WAL enabled (historically these took different
1797       // code paths, so check both)
1798       row = Bytes.toBytes("row2");
1799       put = new Put(row);
1800       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1801       region.put(put);
1802 
1803       // Make sure it shows up with an actual timestamp
1804       get = new Get(row).addColumn(fam, qual);
1805       result = region.get(get);
1806       assertEquals(1, result.size());
1807       kv = result.rawCells()[0];
1808       LOG.info("Got: " + kv);
1809       assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1810           kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1811     } finally {
1812       HRegion.closeHRegion(this.region);
1813       this.region = null;
1814     }
1815 
1816   }
1817 
1818   /**
1819    * Tests that there is server-side filtering for invalid timestamp upper
1820    * bound. Note that the timestamp lower bound is automatically handled for us
1821    * by the TTL field.
1822    */
1823   @Test
1824   public void testPutWithTsSlop() throws IOException {
1825     byte[] fam = Bytes.toBytes("info");
1826     byte[][] families = { fam };
1827     String method = this.getName();
1828 
1829     // add data with a timestamp that is too recent for range. Ensure assert
1830     CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1831     this.region = initHRegion(tableName, method, CONF, families);
1832     boolean caughtExcep = false;
1833     try {
1834       try {
1835         // no TS specified == use latest. should not error
1836         region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
1837         // TS out of range. should error
1838         region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
1839             Bytes.toBytes("value")));
1840         fail("Expected IOE for TS out of configured timerange");
1841       } catch (FailedSanityCheckException ioe) {
1842         LOG.debug("Received expected exception", ioe);
1843         caughtExcep = true;
1844       }
1845       assertTrue("Should catch FailedSanityCheckException", caughtExcep);
1846     } finally {
1847       HRegion.closeHRegion(this.region);
1848       this.region = null;
1849     }
1850   }
1851 
1852   @Test
1853   public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
1854     byte[] fam1 = Bytes.toBytes("columnA");
1855     byte[] fam2 = Bytes.toBytes("columnB");
1856     this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
1857     try {
1858       byte[] rowA = Bytes.toBytes("rowA");
1859       byte[] rowB = Bytes.toBytes("rowB");
1860 
1861       byte[] value = Bytes.toBytes("value");
1862 
1863       Delete delete = new Delete(rowA);
1864       delete.deleteFamily(fam1);
1865 
1866       region.delete(delete);
1867 
1868       // now create data.
1869       Put put = new Put(rowA);
1870       put.add(fam2, null, value);
1871       region.put(put);
1872 
1873       put = new Put(rowB);
1874       put.add(fam1, null, value);
1875       put.add(fam2, null, value);
1876       region.put(put);
1877 
1878       Scan scan = new Scan();
1879       scan.addFamily(fam1).addFamily(fam2);
1880       InternalScanner s = region.getScanner(scan);
1881       List<Cell> results = new ArrayList<Cell>();
1882       s.next(results);
1883       assertTrue(CellUtil.matchingRow(results.get(0), rowA));
1884 
1885       results.clear();
1886       s.next(results);
1887       assertTrue(CellUtil.matchingRow(results.get(0), rowB));
1888     } finally {
1889       HRegion.closeHRegion(this.region);
1890       this.region = null;
1891     }
1892   }
1893 
1894   @Test
1895   public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
1896     Delete delete = new Delete(row);
1897     delete.deleteColumns(fam1, qual1);
1898     doTestDelete_AndPostInsert(delete);
1899   }
1900 
1901   @Test
1902   public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
1903     Delete delete = new Delete(row);
1904     delete.deleteFamily(fam1);
1905     doTestDelete_AndPostInsert(delete);
1906   }
1907 
1908   public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
1909     TableName tableName = TableName.valueOf(name.getMethodName());
1910     this.region = initHRegion(tableName, getName(), CONF, fam1);
1911     try {
1912       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1913       Put put = new Put(row);
1914       put.add(fam1, qual1, value1);
1915       region.put(put);
1916 
1917       // now delete the value:
1918       region.delete(delete);
1919 
1920       // ok put data:
1921       put = new Put(row);
1922       put.add(fam1, qual1, value2);
1923       region.put(put);
1924 
1925       // ok get:
1926       Get get = new Get(row);
1927       get.addColumn(fam1, qual1);
1928 
1929       Result r = region.get(get);
1930       assertEquals(1, r.size());
1931       assertArrayEquals(value2, r.getValue(fam1, qual1));
1932 
1933       // next:
1934       Scan scan = new Scan(row);
1935       scan.addColumn(fam1, qual1);
1936       InternalScanner s = region.getScanner(scan);
1937 
1938       List<Cell> results = new ArrayList<Cell>();
1939       assertEquals(false, s.next(results));
1940       assertEquals(1, results.size());
1941       Cell kv = results.get(0);
1942 
1943       assertArrayEquals(value2, CellUtil.cloneValue(kv));
1944       assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
1945       assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
1946       assertArrayEquals(row, CellUtil.cloneRow(kv));
1947     } finally {
1948       HRegion.closeHRegion(this.region);
1949       this.region = null;
1950     }
1951   }
1952 
1953   @Test
1954   public void testDelete_CheckTimestampUpdated() throws IOException {
1955     TableName tableName = TableName.valueOf(name.getMethodName());
1956     byte[] row1 = Bytes.toBytes("row1");
1957     byte[] col1 = Bytes.toBytes("col1");
1958     byte[] col2 = Bytes.toBytes("col2");
1959     byte[] col3 = Bytes.toBytes("col3");
1960 
1961     // Setting up region
1962     String method = this.getName();
1963     this.region = initHRegion(tableName, method, CONF, fam1);
1964     try {
1965       // Building checkerList
1966       List<Cell> kvs = new ArrayList<Cell>();
1967       kvs.add(new KeyValue(row1, fam1, col1, null));
1968       kvs.add(new KeyValue(row1, fam1, col2, null));
1969       kvs.add(new KeyValue(row1, fam1, col3, null));
1970 
1971       NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1972           Bytes.BYTES_COMPARATOR);
1973       deleteMap.put(fam1, kvs);
1974       region.delete(deleteMap, Durability.SYNC_WAL);
1975 
1976       // extract the key values out the memstore:
1977       // This is kinda hacky, but better than nothing...
1978       long now = System.currentTimeMillis();
1979       KeyValue firstKv = ((HStore) region.getStore(fam1)).memstore.kvset.first();
1980       assertTrue(firstKv.getTimestamp() <= now);
1981       now = firstKv.getTimestamp();
1982       for (KeyValue kv : ((HStore) region.getStore(fam1)).memstore.kvset) {
1983         assertTrue(kv.getTimestamp() <= now);
1984         now = kv.getTimestamp();
1985       }
1986     } finally {
1987       HRegion.closeHRegion(this.region);
1988       this.region = null;
1989     }
1990   }
1991 
1992   // ////////////////////////////////////////////////////////////////////////////
1993   // Get tests
1994   // ////////////////////////////////////////////////////////////////////////////
1995   @Test
1996   public void testGet_FamilyChecker() throws IOException {
1997     byte[] row1 = Bytes.toBytes("row1");
1998     byte[] fam1 = Bytes.toBytes("fam1");
1999     byte[] fam2 = Bytes.toBytes("False");
2000     byte[] col1 = Bytes.toBytes("col1");
2001 
2002     // Setting up region
2003     String method = this.getName();
2004     this.region = initHRegion(tableName, method, CONF, fam1);
2005     try {
2006       Get get = new Get(row1);
2007       get.addColumn(fam2, col1);
2008 
2009       // Test
2010       try {
2011         region.get(get);
2012       } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
2013         assertFalse(false);
2014         return;
2015       }
2016       assertFalse(true);
2017     } finally {
2018       HRegion.closeHRegion(this.region);
2019       this.region = null;
2020     }
2021   }
2022 
2023   @Test
2024   public void testGet_Basic() throws IOException {
2025     byte[] row1 = Bytes.toBytes("row1");
2026     byte[] fam1 = Bytes.toBytes("fam1");
2027     byte[] col1 = Bytes.toBytes("col1");
2028     byte[] col2 = Bytes.toBytes("col2");
2029     byte[] col3 = Bytes.toBytes("col3");
2030     byte[] col4 = Bytes.toBytes("col4");
2031     byte[] col5 = Bytes.toBytes("col5");
2032 
2033     // Setting up region
2034     String method = this.getName();
2035     this.region = initHRegion(tableName, method, CONF, fam1);
2036     try {
2037       // Add to memstore
2038       Put put = new Put(row1);
2039       put.add(fam1, col1, null);
2040       put.add(fam1, col2, null);
2041       put.add(fam1, col3, null);
2042       put.add(fam1, col4, null);
2043       put.add(fam1, col5, null);
2044       region.put(put);
2045 
2046       Get get = new Get(row1);
2047       get.addColumn(fam1, col2);
2048       get.addColumn(fam1, col4);
2049       // Expected result
2050       KeyValue kv1 = new KeyValue(row1, fam1, col2);
2051       KeyValue kv2 = new KeyValue(row1, fam1, col4);
2052       KeyValue[] expected = { kv1, kv2 };
2053 
2054       // Test
2055       Result res = region.get(get);
2056       assertEquals(expected.length, res.size());
2057       for (int i = 0; i < res.size(); i++) {
2058         assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
2059         assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
2060         assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
2061       }
2062 
2063       // Test using a filter on a Get
2064       Get g = new Get(row1);
2065       final int count = 2;
2066       g.setFilter(new ColumnCountGetFilter(count));
2067       res = region.get(g);
2068       assertEquals(count, res.size());
2069     } finally {
2070       HRegion.closeHRegion(this.region);
2071       this.region = null;
2072     }
2073   }
2074 
2075   @Test
2076   public void testGet_Empty() throws IOException {
2077     byte[] row = Bytes.toBytes("row");
2078     byte[] fam = Bytes.toBytes("fam");
2079 
2080     String method = this.getName();
2081     this.region = initHRegion(tableName, method, CONF, fam);
2082     try {
2083       Get get = new Get(row);
2084       get.addFamily(fam);
2085       Result r = region.get(get);
2086 
2087       assertTrue(r.isEmpty());
2088     } finally {
2089       HRegion.closeHRegion(this.region);
2090       this.region = null;
2091     }
2092   }
2093 
2094   // ////////////////////////////////////////////////////////////////////////////
2095   // Merge test
2096   // ////////////////////////////////////////////////////////////////////////////
2097   @Test
2098   public void testMerge() throws IOException {
2099     byte[][] families = { fam1, fam2, fam3 };
2100     Configuration hc = initSplit();
2101     // Setting up region
2102     String method = this.getName();
2103     this.region = initHRegion(tableName, method, hc, families);
2104     try {
2105       LOG.info("" + HBaseTestCase.addContent(region, fam3));
2106       region.flushcache();
2107       region.compactStores();
2108       byte[] splitRow = region.checkSplit();
2109       assertNotNull(splitRow);
2110       LOG.info("SplitRow: " + Bytes.toString(splitRow));
2111       HRegion[] subregions = splitRegion(region, splitRow);
2112       try {
2113         // Need to open the regions.
2114         for (int i = 0; i < subregions.length; i++) {
2115           HRegion.openHRegion(subregions[i], null);
2116           subregions[i].compactStores();
2117         }
2118         Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
2119         Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
2120         Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
2121         long startTime = System.currentTimeMillis();
2122         region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
2123         LOG.info("Merge regions elapsed time: "
2124             + ((System.currentTimeMillis() - startTime) / 1000.0));
2125         FILESYSTEM.delete(oldRegion1, true);
2126         FILESYSTEM.delete(oldRegion2, true);
2127         FILESYSTEM.delete(oldRegionPath, true);
2128         LOG.info("splitAndMerge completed.");
2129       } finally {
2130         for (int i = 0; i < subregions.length; i++) {
2131           try {
2132             HRegion.closeHRegion(subregions[i]);
2133           } catch (IOException e) {
2134             // Ignore.
2135           }
2136         }
2137       }
2138     } finally {
2139       HRegion.closeHRegion(this.region);
2140       this.region = null;
2141     }
2142   }
2143 
2144   /**
2145    * @param parent
2146    *          Region to split.
2147    * @param midkey
2148    *          Key to split around.
2149    * @return The Regions we created.
2150    * @throws IOException
2151    */
2152   HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
2153     PairOfSameType<HRegion> result = null;
2154     SplitTransaction st = new SplitTransaction(parent, midkey);
2155     // If prepare does not return true, for some reason -- logged inside in
2156     // the prepare call -- we are not ready to split just now. Just return.
2157     if (!st.prepare()) {
2158       parent.clearSplit();
2159       return null;
2160     }
2161     try {
2162       result = st.execute(null, null);
2163     } catch (IOException ioe) {
2164       try {
2165         LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; "
2166             + ioe.getMessage());
2167         st.rollback(null, null);
2168         LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString());
2169         return null;
2170       } catch (RuntimeException e) {
2171         // If failed rollback, kill this server to avoid having a hole in table.
2172         LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString()
2173             + " -- aborting server", e);
2174       }
2175     }
2176     finally {
2177       parent.clearSplit();
2178     }
2179     return new HRegion[] { result.getFirst(), result.getSecond() };
2180   }
2181 
2182   // ////////////////////////////////////////////////////////////////////////////
2183   // Scanner tests
2184   // ////////////////////////////////////////////////////////////////////////////
2185   @Test
2186   public void testGetScanner_WithOkFamilies() throws IOException {
2187     byte[] fam1 = Bytes.toBytes("fam1");
2188     byte[] fam2 = Bytes.toBytes("fam2");
2189 
2190     byte[][] families = { fam1, fam2 };
2191 
2192     // Setting up region
2193     String method = this.getName();
2194     this.region = initHRegion(tableName, method, CONF, families);
2195     try {
2196       Scan scan = new Scan();
2197       scan.addFamily(fam1);
2198       scan.addFamily(fam2);
2199       try {
2200         region.getScanner(scan);
2201       } catch (Exception e) {
2202         assertTrue("Families could not be found in Region", false);
2203       }
2204     } finally {
2205       HRegion.closeHRegion(this.region);
2206       this.region = null;
2207     }
2208   }
2209 
2210   @Test
2211   public void testGetScanner_WithNotOkFamilies() throws IOException {
2212     byte[] fam1 = Bytes.toBytes("fam1");
2213     byte[] fam2 = Bytes.toBytes("fam2");
2214 
2215     byte[][] families = { fam1 };
2216 
2217     // Setting up region
2218     String method = this.getName();
2219     this.region = initHRegion(tableName, method, CONF, families);
2220     try {
2221       Scan scan = new Scan();
2222       scan.addFamily(fam2);
2223       boolean ok = false;
2224       try {
2225         region.getScanner(scan);
2226       } catch (Exception e) {
2227         ok = true;
2228       }
2229       assertTrue("Families could not be found in Region", ok);
2230     } finally {
2231       HRegion.closeHRegion(this.region);
2232       this.region = null;
2233     }
2234   }
2235 
2236   @Test
2237   public void testGetScanner_WithNoFamilies() throws IOException {
2238     byte[] row1 = Bytes.toBytes("row1");
2239     byte[] fam1 = Bytes.toBytes("fam1");
2240     byte[] fam2 = Bytes.toBytes("fam2");
2241     byte[] fam3 = Bytes.toBytes("fam3");
2242     byte[] fam4 = Bytes.toBytes("fam4");
2243 
2244     byte[][] families = { fam1, fam2, fam3, fam4 };
2245 
2246     // Setting up region
2247     String method = this.getName();
2248     this.region = initHRegion(tableName, method, CONF, families);
2249     try {
2250 
2251       // Putting data in Region
2252       Put put = new Put(row1);
2253       put.add(fam1, null, null);
2254       put.add(fam2, null, null);
2255       put.add(fam3, null, null);
2256       put.add(fam4, null, null);
2257       region.put(put);
2258 
2259       Scan scan = null;
2260       HRegion.RegionScannerImpl is = null;
2261 
2262       // Testing to see how many scanners that is produced by getScanner,
2263       // starting
2264       // with known number, 2 - current = 1
2265       scan = new Scan();
2266       scan.addFamily(fam2);
2267       scan.addFamily(fam4);
2268       is = (RegionScannerImpl) region.getScanner(scan);
2269       assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2270 
2271       scan = new Scan();
2272       is = (RegionScannerImpl) region.getScanner(scan);
2273       assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2274     } finally {
2275       HRegion.closeHRegion(this.region);
2276       this.region = null;
2277     }
2278   }
2279 
2280   /**
2281    * This method tests https://issues.apache.org/jira/browse/HBASE-2516.
2282    * 
2283    * @throws IOException
2284    */
2285   @Test
2286   public void testGetScanner_WithRegionClosed() throws IOException {
2287     byte[] fam1 = Bytes.toBytes("fam1");
2288     byte[] fam2 = Bytes.toBytes("fam2");
2289 
2290     byte[][] families = { fam1, fam2 };
2291 
2292     // Setting up region
2293     String method = this.getName();
2294     try {
2295       this.region = initHRegion(tableName, method, CONF, families);
2296     } catch (IOException e) {
2297       e.printStackTrace();
2298       fail("Got IOException during initHRegion, " + e.getMessage());
2299     }
2300     try {
2301       region.closed.set(true);
2302       try {
2303         region.getScanner(null);
2304         fail("Expected to get an exception during getScanner on a region that is closed");
2305       } catch (NotServingRegionException e) {
2306         // this is the correct exception that is expected
2307       } catch (IOException e) {
2308         fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2309             + e.getMessage());
2310       }
2311     } finally {
2312       HRegion.closeHRegion(this.region);
2313       this.region = null;
2314     }
2315   }
2316 
2317   @Test
2318   public void testRegionScanner_Next() throws IOException {
2319     byte[] row1 = Bytes.toBytes("row1");
2320     byte[] row2 = Bytes.toBytes("row2");
2321     byte[] fam1 = Bytes.toBytes("fam1");
2322     byte[] fam2 = Bytes.toBytes("fam2");
2323     byte[] fam3 = Bytes.toBytes("fam3");
2324     byte[] fam4 = Bytes.toBytes("fam4");
2325 
2326     byte[][] families = { fam1, fam2, fam3, fam4 };
2327     long ts = System.currentTimeMillis();
2328 
2329     // Setting up region
2330     String method = this.getName();
2331     this.region = initHRegion(tableName, method, CONF, families);
2332     try {
2333       // Putting data in Region
2334       Put put = null;
2335       put = new Put(row1);
2336       put.add(fam1, (byte[]) null, ts, null);
2337       put.add(fam2, (byte[]) null, ts, null);
2338       put.add(fam3, (byte[]) null, ts, null);
2339       put.add(fam4, (byte[]) null, ts, null);
2340       region.put(put);
2341 
2342       put = new Put(row2);
2343       put.add(fam1, (byte[]) null, ts, null);
2344       put.add(fam2, (byte[]) null, ts, null);
2345       put.add(fam3, (byte[]) null, ts, null);
2346       put.add(fam4, (byte[]) null, ts, null);
2347       region.put(put);
2348 
2349       Scan scan = new Scan();
2350       scan.addFamily(fam2);
2351       scan.addFamily(fam4);
2352       InternalScanner is = region.getScanner(scan);
2353 
2354       List<Cell> res = null;
2355 
2356       // Result 1
2357       List<Cell> expected1 = new ArrayList<Cell>();
2358       expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2359       expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2360 
2361       res = new ArrayList<Cell>();
2362       is.next(res);
2363       for (int i = 0; i < res.size(); i++) {
2364         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2365       }
2366 
2367       // Result 2
2368       List<Cell> expected2 = new ArrayList<Cell>();
2369       expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2370       expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2371 
2372       res = new ArrayList<Cell>();
2373       is.next(res);
2374       for (int i = 0; i < res.size(); i++) {
2375         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2376       }
2377     } finally {
2378       HRegion.closeHRegion(this.region);
2379       this.region = null;
2380     }
2381   }
2382 
2383   @Test
2384   public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2385     byte[] row1 = Bytes.toBytes("row1");
2386     byte[] qf1 = Bytes.toBytes("qualifier1");
2387     byte[] qf2 = Bytes.toBytes("qualifier2");
2388     byte[] fam1 = Bytes.toBytes("fam1");
2389     byte[][] families = { fam1 };
2390 
2391     long ts1 = System.currentTimeMillis();
2392     long ts2 = ts1 + 1;
2393     long ts3 = ts1 + 2;
2394 
2395     // Setting up region
2396     String method = this.getName();
2397     this.region = initHRegion(tableName, method, CONF, families);
2398     try {
2399       // Putting data in Region
2400       Put put = null;
2401       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2402       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2403       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2404 
2405       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2406       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2407       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2408 
2409       put = new Put(row1);
2410       put.add(kv13);
2411       put.add(kv12);
2412       put.add(kv11);
2413       put.add(kv23);
2414       put.add(kv22);
2415       put.add(kv21);
2416       region.put(put);
2417 
2418       // Expected
2419       List<Cell> expected = new ArrayList<Cell>();
2420       expected.add(kv13);
2421       expected.add(kv12);
2422 
2423       Scan scan = new Scan(row1);
2424       scan.addColumn(fam1, qf1);
2425       scan.setMaxVersions(MAX_VERSIONS);
2426       List<Cell> actual = new ArrayList<Cell>();
2427       InternalScanner scanner = region.getScanner(scan);
2428 
2429       boolean hasNext = scanner.next(actual);
2430       assertEquals(false, hasNext);
2431 
2432       // Verify result
2433       for (int i = 0; i < expected.size(); i++) {
2434         assertEquals(expected.get(i), actual.get(i));
2435       }
2436     } finally {
2437       HRegion.closeHRegion(this.region);
2438       this.region = null;
2439     }
2440   }
2441 
2442   @Test
2443   public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2444     byte[] row1 = Bytes.toBytes("row1");
2445     byte[] qf1 = Bytes.toBytes("qualifier1");
2446     byte[] qf2 = Bytes.toBytes("qualifier2");
2447     byte[] fam1 = Bytes.toBytes("fam1");
2448     byte[][] families = { fam1 };
2449 
2450     long ts1 = 1; // System.currentTimeMillis();
2451     long ts2 = ts1 + 1;
2452     long ts3 = ts1 + 2;
2453 
2454     // Setting up region
2455     String method = this.getName();
2456     this.region = initHRegion(tableName, method, CONF, families);
2457     try {
2458       // Putting data in Region
2459       Put put = null;
2460       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2461       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2462       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2463 
2464       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2465       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2466       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2467 
2468       put = new Put(row1);
2469       put.add(kv13);
2470       put.add(kv12);
2471       put.add(kv11);
2472       put.add(kv23);
2473       put.add(kv22);
2474       put.add(kv21);
2475       region.put(put);
2476       region.flushcache();
2477 
2478       // Expected
2479       List<Cell> expected = new ArrayList<Cell>();
2480       expected.add(kv13);
2481       expected.add(kv12);
2482       expected.add(kv23);
2483       expected.add(kv22);
2484 
2485       Scan scan = new Scan(row1);
2486       scan.addColumn(fam1, qf1);
2487       scan.addColumn(fam1, qf2);
2488       scan.setMaxVersions(MAX_VERSIONS);
2489       List<Cell> actual = new ArrayList<Cell>();
2490       InternalScanner scanner = region.getScanner(scan);
2491 
2492       boolean hasNext = scanner.next(actual);
2493       assertEquals(false, hasNext);
2494 
2495       // Verify result
2496       for (int i = 0; i < expected.size(); i++) {
2497         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2498       }
2499     } finally {
2500       HRegion.closeHRegion(this.region);
2501       this.region = null;
2502     }
2503   }
2504 
2505   @Test
2506   public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2507     byte[] row1 = Bytes.toBytes("row1");
2508     byte[] fam1 = Bytes.toBytes("fam1");
2509     byte[][] families = { fam1 };
2510     byte[] qf1 = Bytes.toBytes("qualifier1");
2511     byte[] qf2 = Bytes.toBytes("qualifier2");
2512 
2513     long ts1 = 1;
2514     long ts2 = ts1 + 1;
2515     long ts3 = ts1 + 2;
2516     long ts4 = ts1 + 3;
2517 
2518     // Setting up region
2519     String method = this.getName();
2520     this.region = initHRegion(tableName, method, CONF, families);
2521     try {
2522       // Putting data in Region
2523       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2524       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2525       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2526       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2527 
2528       KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2529       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2530       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2531       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2532 
2533       Put put = null;
2534       put = new Put(row1);
2535       put.add(kv14);
2536       put.add(kv24);
2537       region.put(put);
2538       region.flushcache();
2539 
2540       put = new Put(row1);
2541       put.add(kv23);
2542       put.add(kv13);
2543       region.put(put);
2544       region.flushcache();
2545 
2546       put = new Put(row1);
2547       put.add(kv22);
2548       put.add(kv12);
2549       region.put(put);
2550       region.flushcache();
2551 
2552       put = new Put(row1);
2553       put.add(kv21);
2554       put.add(kv11);
2555       region.put(put);
2556 
2557       // Expected
2558       List<Cell> expected = new ArrayList<Cell>();
2559       expected.add(kv14);
2560       expected.add(kv13);
2561       expected.add(kv12);
2562       expected.add(kv24);
2563       expected.add(kv23);
2564       expected.add(kv22);
2565 
2566       Scan scan = new Scan(row1);
2567       scan.addColumn(fam1, qf1);
2568       scan.addColumn(fam1, qf2);
2569       int versions = 3;
2570       scan.setMaxVersions(versions);
2571       List<Cell> actual = new ArrayList<Cell>();
2572       InternalScanner scanner = region.getScanner(scan);
2573 
2574       boolean hasNext = scanner.next(actual);
2575       assertEquals(false, hasNext);
2576 
2577       // Verify result
2578       for (int i = 0; i < expected.size(); i++) {
2579         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2580       }
2581     } finally {
2582       HRegion.closeHRegion(this.region);
2583       this.region = null;
2584     }
2585   }
2586 
2587   @Test
2588   public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2589     byte[] row1 = Bytes.toBytes("row1");
2590     byte[] qf1 = Bytes.toBytes("qualifier1");
2591     byte[] qf2 = Bytes.toBytes("qualifier2");
2592     byte[] fam1 = Bytes.toBytes("fam1");
2593     byte[][] families = { fam1 };
2594 
2595     long ts1 = System.currentTimeMillis();
2596     long ts2 = ts1 + 1;
2597     long ts3 = ts1 + 2;
2598 
2599     // Setting up region
2600     String method = this.getName();
2601     this.region = initHRegion(tableName, method, CONF, families);
2602     try {
2603       // Putting data in Region
2604       Put put = null;
2605       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2606       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2607       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2608 
2609       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2610       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2611       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2612 
2613       put = new Put(row1);
2614       put.add(kv13);
2615       put.add(kv12);
2616       put.add(kv11);
2617       put.add(kv23);
2618       put.add(kv22);
2619       put.add(kv21);
2620       region.put(put);
2621 
2622       // Expected
2623       List<Cell> expected = new ArrayList<Cell>();
2624       expected.add(kv13);
2625       expected.add(kv12);
2626       expected.add(kv23);
2627       expected.add(kv22);
2628 
2629       Scan scan = new Scan(row1);
2630       scan.addFamily(fam1);
2631       scan.setMaxVersions(MAX_VERSIONS);
2632       List<Cell> actual = new ArrayList<Cell>();
2633       InternalScanner scanner = region.getScanner(scan);
2634 
2635       boolean hasNext = scanner.next(actual);
2636       assertEquals(false, hasNext);
2637 
2638       // Verify result
2639       for (int i = 0; i < expected.size(); i++) {
2640         assertEquals(expected.get(i), actual.get(i));
2641       }
2642     } finally {
2643       HRegion.closeHRegion(this.region);
2644       this.region = null;
2645     }
2646   }
2647 
2648   @Test
2649   public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
2650     byte[] row1 = Bytes.toBytes("row1");
2651     byte[] qf1 = Bytes.toBytes("qualifier1");
2652     byte[] qf2 = Bytes.toBytes("qualifier2");
2653     byte[] fam1 = Bytes.toBytes("fam1");
2654 
2655     long ts1 = 1; // System.currentTimeMillis();
2656     long ts2 = ts1 + 1;
2657     long ts3 = ts1 + 2;
2658 
2659     // Setting up region
2660     String method = this.getName();
2661     this.region = initHRegion(tableName, method, CONF, fam1);
2662     try {
2663       // Putting data in Region
2664       Put put = null;
2665       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2666       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2667       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2668 
2669       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2670       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2671       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2672 
2673       put = new Put(row1);
2674       put.add(kv13);
2675       put.add(kv12);
2676       put.add(kv11);
2677       put.add(kv23);
2678       put.add(kv22);
2679       put.add(kv21);
2680       region.put(put);
2681       region.flushcache();
2682 
2683       // Expected
2684       List<Cell> expected = new ArrayList<Cell>();
2685       expected.add(kv13);
2686       expected.add(kv12);
2687       expected.add(kv23);
2688       expected.add(kv22);
2689 
2690       Scan scan = new Scan(row1);
2691       scan.addFamily(fam1);
2692       scan.setMaxVersions(MAX_VERSIONS);
2693       List<Cell> actual = new ArrayList<Cell>();
2694       InternalScanner scanner = region.getScanner(scan);
2695 
2696       boolean hasNext = scanner.next(actual);
2697       assertEquals(false, hasNext);
2698 
2699       // Verify result
2700       for (int i = 0; i < expected.size(); i++) {
2701         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2702       }
2703     } finally {
2704       HRegion.closeHRegion(this.region);
2705       this.region = null;
2706     }
2707   }
2708 
2709   @Test
2710   public void testScanner_StopRow1542() throws IOException {
2711     byte[] family = Bytes.toBytes("testFamily");
2712     this.region = initHRegion(tableName, getName(), CONF, family);
2713     try {
2714       byte[] row1 = Bytes.toBytes("row111");
2715       byte[] row2 = Bytes.toBytes("row222");
2716       byte[] row3 = Bytes.toBytes("row333");
2717       byte[] row4 = Bytes.toBytes("row444");
2718       byte[] row5 = Bytes.toBytes("row555");
2719 
2720       byte[] col1 = Bytes.toBytes("Pub111");
2721       byte[] col2 = Bytes.toBytes("Pub222");
2722 
2723       Put put = new Put(row1);
2724       put.add(family, col1, Bytes.toBytes(10L));
2725       region.put(put);
2726 
2727       put = new Put(row2);
2728       put.add(family, col1, Bytes.toBytes(15L));
2729       region.put(put);
2730 
2731       put = new Put(row3);
2732       put.add(family, col2, Bytes.toBytes(20L));
2733       region.put(put);
2734 
2735       put = new Put(row4);
2736       put.add(family, col2, Bytes.toBytes(30L));
2737       region.put(put);
2738 
2739       put = new Put(row5);
2740       put.add(family, col1, Bytes.toBytes(40L));
2741       region.put(put);
2742 
2743       Scan scan = new Scan(row3, row4);
2744       scan.setMaxVersions();
2745       scan.addColumn(family, col1);
2746       InternalScanner s = region.getScanner(scan);
2747 
2748       List<Cell> results = new ArrayList<Cell>();
2749       assertEquals(false, s.next(results));
2750       assertEquals(0, results.size());
2751     } finally {
2752       HRegion.closeHRegion(this.region);
2753       this.region = null;
2754     }
2755   }
2756 
2757   @Test
2758   public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2759     byte[] row1 = Bytes.toBytes("row1");
2760     byte[] fam1 = Bytes.toBytes("fam1");
2761     byte[] qf1 = Bytes.toBytes("qualifier1");
2762     byte[] qf2 = Bytes.toBytes("quateslifier2");
2763 
2764     long ts1 = 1;
2765     long ts2 = ts1 + 1;
2766     long ts3 = ts1 + 2;
2767     long ts4 = ts1 + 3;
2768 
2769     // Setting up region
2770     String method = this.getName();
2771     this.region = initHRegion(tableName, method, CONF, fam1);
2772     try {
2773       // Putting data in Region
2774       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2775       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2776       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2777       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2778 
2779       KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2780       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2781       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2782       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2783 
2784       Put put = null;
2785       put = new Put(row1);
2786       put.add(kv14);
2787       put.add(kv24);
2788       region.put(put);
2789       region.flushcache();
2790 
2791       put = new Put(row1);
2792       put.add(kv23);
2793       put.add(kv13);
2794       region.put(put);
2795       region.flushcache();
2796 
2797       put = new Put(row1);
2798       put.add(kv22);
2799       put.add(kv12);
2800       region.put(put);
2801       region.flushcache();
2802 
2803       put = new Put(row1);
2804       put.add(kv21);
2805       put.add(kv11);
2806       region.put(put);
2807 
2808       // Expected
2809       List<KeyValue> expected = new ArrayList<KeyValue>();
2810       expected.add(kv14);
2811       expected.add(kv13);
2812       expected.add(kv12);
2813       expected.add(kv24);
2814       expected.add(kv23);
2815       expected.add(kv22);
2816 
2817       Scan scan = new Scan(row1);
2818       int versions = 3;
2819       scan.setMaxVersions(versions);
2820       List<Cell> actual = new ArrayList<Cell>();
2821       InternalScanner scanner = region.getScanner(scan);
2822 
2823       boolean hasNext = scanner.next(actual);
2824       assertEquals(false, hasNext);
2825 
2826       // Verify result
2827       for (int i = 0; i < expected.size(); i++) {
2828         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2829       }
2830     } finally {
2831       HRegion.closeHRegion(this.region);
2832       this.region = null;
2833     }
2834   }
2835 
2836   /**
2837    * Added for HBASE-5416
2838    * 
2839    * Here we test scan optimization when only subset of CFs are used in filter
2840    * conditions.
2841    */
2842   @Test
2843   public void testScanner_JoinedScanners() throws IOException {
2844     byte[] cf_essential = Bytes.toBytes("essential");
2845     byte[] cf_joined = Bytes.toBytes("joined");
2846     byte[] cf_alpha = Bytes.toBytes("alpha");
2847     this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
2848     try {
2849       byte[] row1 = Bytes.toBytes("row1");
2850       byte[] row2 = Bytes.toBytes("row2");
2851       byte[] row3 = Bytes.toBytes("row3");
2852 
2853       byte[] col_normal = Bytes.toBytes("d");
2854       byte[] col_alpha = Bytes.toBytes("a");
2855 
2856       byte[] filtered_val = Bytes.toBytes(3);
2857 
2858       Put put = new Put(row1);
2859       put.add(cf_essential, col_normal, Bytes.toBytes(1));
2860       put.add(cf_joined, col_alpha, Bytes.toBytes(1));
2861       region.put(put);
2862 
2863       put = new Put(row2);
2864       put.add(cf_essential, col_alpha, Bytes.toBytes(2));
2865       put.add(cf_joined, col_normal, Bytes.toBytes(2));
2866       put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
2867       region.put(put);
2868 
2869       put = new Put(row3);
2870       put.add(cf_essential, col_normal, filtered_val);
2871       put.add(cf_joined, col_normal, filtered_val);
2872       region.put(put);
2873 
2874       // Check two things:
2875       // 1. result list contains expected values
2876       // 2. result list is sorted properly
2877 
2878       Scan scan = new Scan();
2879       Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
2880           CompareOp.NOT_EQUAL, filtered_val);
2881       scan.setFilter(filter);
2882       scan.setLoadColumnFamiliesOnDemand(true);
2883       InternalScanner s = region.getScanner(scan);
2884 
2885       List<Cell> results = new ArrayList<Cell>();
2886       assertTrue(s.next(results));
2887       assertEquals(results.size(), 1);
2888       results.clear();
2889 
2890       assertTrue(s.next(results));
2891       assertEquals(results.size(), 3);
2892       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
2893       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
2894       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
2895       results.clear();
2896 
2897       assertFalse(s.next(results));
2898       assertEquals(results.size(), 0);
2899     } finally {
2900       HRegion.closeHRegion(this.region);
2901       this.region = null;
2902     }
2903   }
2904 
2905   /**
2906    * HBASE-5416
2907    * 
2908    * Test case when scan limits amount of KVs returned on each next() call.
2909    */
2910   @Test
2911   public void testScanner_JoinedScannersWithLimits() throws IOException {
2912     final byte[] cf_first = Bytes.toBytes("first");
2913     final byte[] cf_second = Bytes.toBytes("second");
2914 
2915     this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
2916     try {
2917       final byte[] col_a = Bytes.toBytes("a");
2918       final byte[] col_b = Bytes.toBytes("b");
2919 
2920       Put put;
2921 
2922       for (int i = 0; i < 10; i++) {
2923         put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
2924         put.add(cf_first, col_a, Bytes.toBytes(i));
2925         if (i < 5) {
2926           put.add(cf_first, col_b, Bytes.toBytes(i));
2927           put.add(cf_second, col_a, Bytes.toBytes(i));
2928           put.add(cf_second, col_b, Bytes.toBytes(i));
2929         }
2930         region.put(put);
2931       }
2932 
2933       Scan scan = new Scan();
2934       scan.setLoadColumnFamiliesOnDemand(true);
2935       Filter bogusFilter = new FilterBase() {
2936         @Override
2937         public boolean isFamilyEssential(byte[] name) {
2938           return Bytes.equals(name, cf_first);
2939         }
2940       };
2941 
2942       scan.setFilter(bogusFilter);
2943       InternalScanner s = region.getScanner(scan);
2944 
2945       // Our data looks like this:
2946       // r0: first:a, first:b, second:a, second:b
2947       // r1: first:a, first:b, second:a, second:b
2948       // r2: first:a, first:b, second:a, second:b
2949       // r3: first:a, first:b, second:a, second:b
2950       // r4: first:a, first:b, second:a, second:b
2951       // r5: first:a
2952       // r6: first:a
2953       // r7: first:a
2954       // r8: first:a
2955       // r9: first:a
2956 
2957       // But due to next's limit set to 3, we should get this:
2958       // r0: first:a, first:b, second:a
2959       // r0: second:b
2960       // r1: first:a, first:b, second:a
2961       // r1: second:b
2962       // r2: first:a, first:b, second:a
2963       // r2: second:b
2964       // r3: first:a, first:b, second:a
2965       // r3: second:b
2966       // r4: first:a, first:b, second:a
2967       // r4: second:b
2968       // r5: first:a
2969       // r6: first:a
2970       // r7: first:a
2971       // r8: first:a
2972       // r9: first:a
2973 
2974       List<Cell> results = new ArrayList<Cell>();
2975       int index = 0;
2976       while (true) {
2977         boolean more = s.next(results, 3);
2978         if ((index >> 1) < 5) {
2979           if (index % 2 == 0)
2980             assertEquals(results.size(), 3);
2981           else
2982             assertEquals(results.size(), 1);
2983         } else
2984           assertEquals(results.size(), 1);
2985         results.clear();
2986         index++;
2987         if (!more)
2988           break;
2989       }
2990     } finally {
2991       HRegion.closeHRegion(this.region);
2992       this.region = null;
2993     }
2994   }
2995 
2996   // ////////////////////////////////////////////////////////////////////////////
2997   // Split test
2998   // ////////////////////////////////////////////////////////////////////////////
2999   /**
3000    * Splits twice and verifies getting from each of the split regions.
3001    * 
3002    * @throws Exception
3003    */
3004   @Test
3005   public void testBasicSplit() throws Exception {
3006     byte[][] families = { fam1, fam2, fam3 };
3007 
3008     Configuration hc = initSplit();
3009     // Setting up region
3010     String method = this.getName();
3011     this.region = initHRegion(tableName, method, hc, families);
3012 
3013     try {
3014       LOG.info("" + HBaseTestCase.addContent(region, fam3));
3015       region.flushcache();
3016       region.compactStores();
3017       byte[] splitRow = region.checkSplit();
3018       assertNotNull(splitRow);
3019       LOG.info("SplitRow: " + Bytes.toString(splitRow));
3020       HRegion[] regions = splitRegion(region, splitRow);
3021       try {
3022         // Need to open the regions.
3023         // TODO: Add an 'open' to HRegion... don't do open by constructing
3024         // instance.
3025         for (int i = 0; i < regions.length; i++) {
3026           regions[i] = HRegion.openHRegion(regions[i], null);
3027         }
3028         // Assert can get rows out of new regions. Should be able to get first
3029         // row from first region and the midkey from second region.
3030         assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
3031         assertGet(regions[1], fam3, splitRow);
3032         // Test I can get scanner and that it starts at right place.
3033         assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
3034         assertScan(regions[1], fam3, splitRow);
3035         // Now prove can't split regions that have references.
3036         for (int i = 0; i < regions.length; i++) {
3037           // Add so much data to this region, we create a store file that is >
3038           // than one of our unsplitable references. it will.
3039           for (int j = 0; j < 2; j++) {
3040             HBaseTestCase.addContent(regions[i], fam3);
3041           }
3042           HBaseTestCase.addContent(regions[i], fam2);
3043           HBaseTestCase.addContent(regions[i], fam1);
3044           regions[i].flushcache();
3045         }
3046 
3047         byte[][] midkeys = new byte[regions.length][];
3048         // To make regions splitable force compaction.
3049         for (int i = 0; i < regions.length; i++) {
3050           regions[i].compactStores();
3051           midkeys[i] = regions[i].checkSplit();
3052         }
3053 
3054         TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
3055         // Split these two daughter regions so then I'll have 4 regions. Will
3056         // split because added data above.
3057         for (int i = 0; i < regions.length; i++) {
3058           HRegion[] rs = null;
3059           if (midkeys[i] != null) {
3060             rs = splitRegion(regions[i], midkeys[i]);
3061             for (int j = 0; j < rs.length; j++) {
3062               sortedMap.put(Bytes.toString(rs[j].getRegionName()), HRegion.openHRegion(rs[j], null));
3063             }
3064           }
3065         }
3066         LOG.info("Made 4 regions");
3067         // The splits should have been even. Test I can get some arbitrary row
3068         // out of each.
3069         int interval = (LAST_CHAR - FIRST_CHAR) / 3;
3070         byte[] b = Bytes.toBytes(START_KEY);
3071         for (HRegion r : sortedMap.values()) {
3072           assertGet(r, fam3, b);
3073           b[0] += interval;
3074         }
3075       } finally {
3076         for (int i = 0; i < regions.length; i++) {
3077           try {
3078             regions[i].close();
3079           } catch (IOException e) {
3080             // Ignore.
3081           }
3082         }
3083       }
3084     } finally {
3085       HRegion.closeHRegion(this.region);
3086       this.region = null;
3087     }
3088   }
3089 
3090   @Test
3091   public void testSplitRegion() throws IOException {
3092     byte[] qualifier = Bytes.toBytes("qualifier");
3093     Configuration hc = initSplit();
3094     int numRows = 10;
3095     byte[][] families = { fam1, fam3 };
3096 
3097     // Setting up region
3098     String method = this.getName();
3099     this.region = initHRegion(tableName, method, hc, families);
3100 
3101     // Put data in region
3102     int startRow = 100;
3103     putData(startRow, numRows, qualifier, families);
3104     int splitRow = startRow + numRows;
3105     putData(splitRow, numRows, qualifier, families);
3106     region.flushcache();
3107 
3108     HRegion[] regions = null;
3109     try {
3110       regions = splitRegion(region, Bytes.toBytes("" + splitRow));
3111       // Opening the regions returned.
3112       for (int i = 0; i < regions.length; i++) {
3113         regions[i] = HRegion.openHRegion(regions[i], null);
3114       }
3115       // Verifying that the region has been split
3116       assertEquals(2, regions.length);
3117 
3118       // Verifying that all data is still there and that data is in the right
3119       // place
3120       verifyData(regions[0], startRow, numRows, qualifier, families);
3121       verifyData(regions[1], splitRow, numRows, qualifier, families);
3122 
3123     } finally {
3124       HRegion.closeHRegion(this.region);
3125       this.region = null;
3126     }
3127   }
3128 
3129   @Test
3130   public void testClearForceSplit() throws IOException {
3131     byte[] qualifier = Bytes.toBytes("qualifier");
3132     Configuration hc = initSplit();
3133     int numRows = 10;
3134     byte[][] families = { fam1, fam3 };
3135 
3136     // Setting up region
3137     String method = this.getName();
3138     this.region = initHRegion(tableName, method, hc, families);
3139 
3140     // Put data in region
3141     int startRow = 100;
3142     putData(startRow, numRows, qualifier, families);
3143     int splitRow = startRow + numRows;
3144     byte[] splitRowBytes = Bytes.toBytes("" + splitRow);
3145     putData(splitRow, numRows, qualifier, families);
3146     region.flushcache();
3147 
3148     HRegion[] regions = null;
3149     try {
3150       // Set force split
3151       region.forceSplit(splitRowBytes);
3152       assertTrue(region.shouldForceSplit());
3153       // Split point should be the force split row
3154       assertTrue(Bytes.equals(splitRowBytes, region.checkSplit()));
3155 
3156       // Add a store that has references.
3157       HStore storeMock = Mockito.mock(HStore.class);
3158       Mockito.when(storeMock.hasReferences()).thenReturn(true);
3159       Mockito.when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
3160       Mockito.when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
3161       Mockito.when(storeMock.getColumnFamilyName()).thenReturn("cf");
3162       region.stores.put(Bytes.toBytes(storeMock.getColumnFamilyName()), storeMock);
3163       assertTrue(region.hasReferences());
3164 
3165       // Will not split since the store has references.
3166       regions = splitRegion(region, splitRowBytes);
3167       assertNull(regions);
3168 
3169       // Region force split should be cleared after the split try.
3170       assertFalse(region.shouldForceSplit());
3171 
3172       // Remove the store that has references.
3173       region.stores.remove(Bytes.toBytes(storeMock.getColumnFamilyName()));
3174       assertFalse(region.hasReferences());
3175 
3176       // Now we can split.
3177       regions = splitRegion(region, splitRowBytes);
3178 
3179       // Opening the regions returned.
3180       for (int i = 0; i < regions.length; i++) {
3181         regions[i] = HRegion.openHRegion(regions[i], null);
3182       }
3183       // Verifying that the region has been split
3184       assertEquals(2, regions.length);
3185 
3186       // Verifying that all data is still there and that data is in the right
3187       // place
3188       verifyData(regions[0], startRow, numRows, qualifier, families);
3189       verifyData(regions[1], splitRow, numRows, qualifier, families);
3190 
3191     } finally {
3192       HRegion.closeHRegion(this.region);
3193       this.region = null;
3194     }
3195   }
3196 
3197   /**
3198    * Flushes the cache in a thread while scanning. The tests verify that the
3199    * scan is coherent - e.g. the returned results are always of the same or
3200    * later update as the previous results.
3201    * 
3202    * @throws IOException
3203    *           scan / compact
3204    * @throws InterruptedException
3205    *           thread join
3206    */
3207   @Test
3208   public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3209     byte[] family = Bytes.toBytes("family");
3210     int numRows = 1000;
3211     int flushAndScanInterval = 10;
3212     int compactInterval = 10 * flushAndScanInterval;
3213 
3214     String method = "testFlushCacheWhileScanning";
3215     this.region = initHRegion(tableName, method, CONF, family);
3216     try {
3217       FlushThread flushThread = new FlushThread();
3218       flushThread.start();
3219 
3220       Scan scan = new Scan();
3221       scan.addFamily(family);
3222       scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
3223           new BinaryComparator(Bytes.toBytes(5L))));
3224 
3225       int expectedCount = 0;
3226       List<Cell> res = new ArrayList<Cell>();
3227 
3228       boolean toggle = true;
3229       for (long i = 0; i < numRows; i++) {
3230         Put put = new Put(Bytes.toBytes(i));
3231         put.setDurability(Durability.SKIP_WAL);
3232         put.add(family, qual1, Bytes.toBytes(i % 10));
3233         region.put(put);
3234 
3235         if (i != 0 && i % compactInterval == 0) {
3236           // System.out.println("iteration = " + i);
3237           region.compactStores(true);
3238         }
3239 
3240         if (i % 10 == 5L) {
3241           expectedCount++;
3242         }
3243 
3244         if (i != 0 && i % flushAndScanInterval == 0) {
3245           res.clear();
3246           InternalScanner scanner = region.getScanner(scan);
3247           if (toggle) {
3248             flushThread.flush();
3249           }
3250           while (scanner.next(res))
3251             ;
3252           if (!toggle) {
3253             flushThread.flush();
3254           }
3255           assertEquals("i=" + i, expectedCount, res.size());
3256           toggle = !toggle;
3257         }
3258       }
3259 
3260       flushThread.done();
3261       flushThread.join();
3262       flushThread.checkNoError();
3263     } finally {
3264       HRegion.closeHRegion(this.region);
3265       this.region = null;
3266     }
3267   }
3268 
3269   protected class FlushThread extends Thread {
3270     private volatile boolean done;
3271     private Throwable error = null;
3272 
3273     public void done() {
3274       done = true;
3275       synchronized (this) {
3276         interrupt();
3277       }
3278     }
3279 
3280     public void checkNoError() {
3281       if (error != null) {
3282         assertNull(error);
3283       }
3284     }
3285 
3286     @Override
3287     public void run() {
3288       done = false;
3289       while (!done) {
3290         synchronized (this) {
3291           try {
3292             wait();
3293           } catch (InterruptedException ignored) {
3294             if (done) {
3295               break;
3296             }
3297           }
3298         }
3299         try {
3300           region.flushcache();
3301         } catch (IOException e) {
3302           if (!done) {
3303             LOG.error("Error while flusing cache", e);
3304             error = e;
3305           }
3306           break;
3307         }
3308       }
3309 
3310     }
3311 
3312     public void flush() {
3313       synchronized (this) {
3314         notify();
3315       }
3316 
3317     }
3318   }
3319 
3320   /**
3321    * Writes very wide records and scans for the latest every time.. Flushes and
3322    * compacts the region every now and then to keep things realistic.
3323    * 
3324    * @throws IOException
3325    *           by flush / scan / compaction
3326    * @throws InterruptedException
3327    *           when joining threads
3328    */
3329   @Test
3330   public void testWritesWhileScanning() throws IOException, InterruptedException {
3331     int testCount = 100;
3332     int numRows = 1;
3333     int numFamilies = 10;
3334     int numQualifiers = 100;
3335     int flushInterval = 7;
3336     int compactInterval = 5 * flushInterval;
3337     byte[][] families = new byte[numFamilies][];
3338     for (int i = 0; i < numFamilies; i++) {
3339       families[i] = Bytes.toBytes("family" + i);
3340     }
3341     byte[][] qualifiers = new byte[numQualifiers][];
3342     for (int i = 0; i < numQualifiers; i++) {
3343       qualifiers[i] = Bytes.toBytes("qual" + i);
3344     }
3345 
3346     String method = "testWritesWhileScanning";
3347     this.region = initHRegion(tableName, method, CONF, families);
3348     try {
3349       PutThread putThread = new PutThread(numRows, families, qualifiers);
3350       putThread.start();
3351       putThread.waitForFirstPut();
3352 
3353       FlushThread flushThread = new FlushThread();
3354       flushThread.start();
3355 
3356       Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3357 
3358       int expectedCount = numFamilies * numQualifiers;
3359       List<Cell> res = new ArrayList<Cell>();
3360 
3361       long prevTimestamp = 0L;
3362       for (int i = 0; i < testCount; i++) {
3363 
3364         if (i != 0 && i % compactInterval == 0) {
3365           region.compactStores(true);
3366         }
3367 
3368         if (i != 0 && i % flushInterval == 0) {
3369           flushThread.flush();
3370         }
3371 
3372         boolean previousEmpty = res.isEmpty();
3373         res.clear();
3374         InternalScanner scanner = region.getScanner(scan);
3375         while (scanner.next(res))
3376           ;
3377         if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3378           assertEquals("i=" + i, expectedCount, res.size());
3379           long timestamp = res.get(0).getTimestamp();
3380           assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3381               timestamp >= prevTimestamp);
3382           prevTimestamp = timestamp;
3383         }
3384       }
3385 
3386       putThread.done();
3387 
3388       region.flushcache();
3389 
3390       putThread.join();
3391       putThread.checkNoError();
3392 
3393       flushThread.done();
3394       flushThread.join();
3395       flushThread.checkNoError();
3396     } finally {
3397       try {
3398         HRegion.closeHRegion(this.region);
3399       } catch (DroppedSnapshotException dse) {
3400         // We could get this on way out because we interrupt the background flusher and it could
3401         // fail anywhere causing a DSE over in the background flusher... only it is not properly
3402         // dealt with so could still be memory hanging out when we get to here -- memory we can't
3403         // flush because the accounting is 'off' since original DSE.
3404       }
3405       this.region = null;
3406     }
3407   }
3408 
3409   protected class PutThread extends Thread {
3410     private volatile boolean done;
3411     private volatile int numPutsFinished = 0;
3412 
3413     private Throwable error = null;
3414     private int numRows;
3415     private byte[][] families;
3416     private byte[][] qualifiers;
3417 
3418     private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3419       this.numRows = numRows;
3420       this.families = families;
3421       this.qualifiers = qualifiers;
3422     }
3423 
3424     /**
3425      * Block until this thread has put at least one row.
3426      */
3427     public void waitForFirstPut() throws InterruptedException {
3428       // wait until put thread actually puts some data
3429       while (numPutsFinished == 0) {
3430         checkNoError();
3431         Thread.sleep(50);
3432       }
3433     }
3434 
3435     public void done() {
3436       done = true;
3437       synchronized (this) {
3438         interrupt();
3439       }
3440     }
3441 
3442     public void checkNoError() {
3443       if (error != null) {
3444         assertNull(error);
3445       }
3446     }
3447 
3448     @Override
3449     public void run() {
3450       done = false;
3451       while (!done) {
3452         try {
3453           for (int r = 0; r < numRows; r++) {
3454             byte[] row = Bytes.toBytes("row" + r);
3455             Put put = new Put(row);
3456             put.setDurability(Durability.SKIP_WAL);
3457             byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3458             for (byte[] family : families) {
3459               for (byte[] qualifier : qualifiers) {
3460                 put.add(family, qualifier, (long) numPutsFinished, value);
3461               }
3462             }
3463             region.put(put);
3464             numPutsFinished++;
3465             if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3466               System.out.println("put iteration = " + numPutsFinished);
3467               Delete delete = new Delete(row, (long) numPutsFinished - 30);
3468               region.delete(delete);
3469             }
3470             numPutsFinished++;
3471           }
3472         } catch (InterruptedIOException e) {
3473           // This is fine. It means we are done, or didn't get the lock on time
3474         } catch (IOException e) {
3475           LOG.error("error while putting records", e);
3476           error = e;
3477           break;
3478         }
3479       }
3480 
3481     }
3482 
3483   }
3484 
3485   /**
3486    * Writes very wide records and gets the latest row every time.. Flushes and
3487    * compacts the region aggressivly to catch issues.
3488    * 
3489    * @throws IOException
3490    *           by flush / scan / compaction
3491    * @throws InterruptedException
3492    *           when joining threads
3493    */
3494   @Test
3495   public void testWritesWhileGetting() throws Exception {
3496     int testCount = 100;
3497     int numRows = 1;
3498     int numFamilies = 10;
3499     int numQualifiers = 100;
3500     int compactInterval = 100;
3501     byte[][] families = new byte[numFamilies][];
3502     for (int i = 0; i < numFamilies; i++) {
3503       families[i] = Bytes.toBytes("family" + i);
3504     }
3505     byte[][] qualifiers = new byte[numQualifiers][];
3506     for (int i = 0; i < numQualifiers; i++) {
3507       qualifiers[i] = Bytes.toBytes("qual" + i);
3508     }
3509 
3510 
3511     String method = "testWritesWhileGetting";
3512     // This test flushes constantly and can cause many files to be created,
3513     // possibly
3514     // extending over the ulimit. Make sure compactions are aggressive in
3515     // reducing
3516     // the number of HFiles created.
3517     Configuration conf = HBaseConfiguration.create(CONF);
3518     conf.setInt("hbase.hstore.compaction.min", 1);
3519     conf.setInt("hbase.hstore.compaction.max", 1000);
3520     this.region = initHRegion(tableName, method, conf, families);
3521     PutThread putThread = null;
3522     MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3523     try {
3524       putThread = new PutThread(numRows, families, qualifiers);
3525       putThread.start();
3526       putThread.waitForFirstPut();
3527 
3528       // Add a thread that flushes as fast as possible
3529       ctx.addThread(new RepeatingTestThread(ctx) {
3530         private int flushesSinceCompact = 0;
3531         private final int maxFlushesSinceCompact = 20;
3532 
3533         @Override
3534         public void doAnAction() throws Exception {
3535           if (region.flushcache().isCompactionNeeded()) {
3536             ++flushesSinceCompact;
3537           }
3538           // Compact regularly to avoid creating too many files and exceeding
3539           // the ulimit.
3540           if (flushesSinceCompact == maxFlushesSinceCompact) {
3541             region.compactStores(false);
3542             flushesSinceCompact = 0;
3543           }
3544         }
3545       });
3546       ctx.startThreads();
3547 
3548       Get get = new Get(Bytes.toBytes("row0"));
3549       Result result = null;
3550 
3551       int expectedCount = numFamilies * numQualifiers;
3552 
3553       long prevTimestamp = 0L;
3554       for (int i = 0; i < testCount; i++) {
3555 
3556         boolean previousEmpty = result == null || result.isEmpty();
3557         result = region.get(get);
3558         if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3559           assertEquals("i=" + i, expectedCount, result.size());
3560           // TODO this was removed, now what dangit?!
3561           // search looking for the qualifier in question?
3562           long timestamp = 0;
3563           for (Cell kv : result.rawCells()) {
3564             if (CellUtil.matchingFamily(kv, families[0])
3565                 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3566               timestamp = kv.getTimestamp();
3567             }
3568           }
3569           assertTrue(timestamp >= prevTimestamp);
3570           prevTimestamp = timestamp;
3571           Cell previousKV = null;
3572 
3573           for (Cell kv : result.rawCells()) {
3574             byte[] thisValue = CellUtil.cloneValue(kv);
3575             if (previousKV != null) {
3576               if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3577                 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3578                     + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3579                     + "(memStoreTS:" + kv.getMvccVersion() + ")");
3580                 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3581               }
3582             }
3583             previousKV = kv;
3584           }
3585         }
3586       }
3587     } finally {
3588       if (putThread != null)
3589         putThread.done();
3590 
3591       region.flushcache();
3592 
3593       if (putThread != null) {
3594         putThread.join();
3595         putThread.checkNoError();
3596       }
3597 
3598       ctx.stop();
3599       HRegion.closeHRegion(this.region);
3600       this.region = null;
3601     }
3602   }
3603 
3604   @Test
3605   public void testHolesInMeta() throws Exception {
3606     byte[] family = Bytes.toBytes("family");
3607     this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF,
3608         false, family);
3609     try {
3610       byte[] rowNotServed = Bytes.toBytes("a");
3611       Get g = new Get(rowNotServed);
3612       try {
3613         region.get(g);
3614         fail();
3615       } catch (WrongRegionException x) {
3616         // OK
3617       }
3618       byte[] row = Bytes.toBytes("y");
3619       g = new Get(row);
3620       region.get(g);
3621     } finally {
3622       HRegion.closeHRegion(this.region);
3623       this.region = null;
3624     }
3625   }
3626 
3627   @Test
3628   public void testIndexesScanWithOneDeletedRow() throws IOException {
3629     byte[] family = Bytes.toBytes("family");
3630 
3631     // Setting up region
3632     String method = "testIndexesScanWithOneDeletedRow";
3633     this.region = initHRegion(tableName, method, CONF, family);
3634     try {
3635       Put put = new Put(Bytes.toBytes(1L));
3636       put.add(family, qual1, 1L, Bytes.toBytes(1L));
3637       region.put(put);
3638 
3639       region.flushcache();
3640 
3641       Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3642       region.delete(delete);
3643 
3644       put = new Put(Bytes.toBytes(2L));
3645       put.add(family, qual1, 2L, Bytes.toBytes(2L));
3646       region.put(put);
3647 
3648       Scan idxScan = new Scan();
3649       idxScan.addFamily(family);
3650       idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
3651           new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
3652               new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
3653               CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
3654       InternalScanner scanner = region.getScanner(idxScan);
3655       List<Cell> res = new ArrayList<Cell>();
3656 
3657       while (scanner.next(res))
3658         ;
3659       assertEquals(1L, res.size());
3660     } finally {
3661       HRegion.closeHRegion(this.region);
3662       this.region = null;
3663     }
3664   }
3665 
3666   // ////////////////////////////////////////////////////////////////////////////
3667   // Bloom filter test
3668   // ////////////////////////////////////////////////////////////////////////////
3669   @Test
3670   public void testBloomFilterSize() throws IOException {
3671     byte[] fam1 = Bytes.toBytes("fam1");
3672     byte[] qf1 = Bytes.toBytes("col");
3673     byte[] val1 = Bytes.toBytes("value1");
3674     // Create Table
3675     HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
3676         .setBloomFilterType(BloomType.ROWCOL);
3677 
3678     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3679     htd.addFamily(hcd);
3680     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3681     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3682     try {
3683       int num_unique_rows = 10;
3684       int duplicate_multiplier = 2;
3685       int num_storefiles = 4;
3686 
3687       int version = 0;
3688       for (int f = 0; f < num_storefiles; f++) {
3689         for (int i = 0; i < duplicate_multiplier; i++) {
3690           for (int j = 0; j < num_unique_rows; j++) {
3691             Put put = new Put(Bytes.toBytes("row" + j));
3692             put.setDurability(Durability.SKIP_WAL);
3693             put.add(fam1, qf1, version++, val1);
3694             region.put(put);
3695           }
3696         }
3697         region.flushcache();
3698       }
3699       // before compaction
3700       HStore store = (HStore) region.getStore(fam1);
3701       Collection<StoreFile> storeFiles = store.getStorefiles();
3702       for (StoreFile storefile : storeFiles) {
3703         StoreFile.Reader reader = storefile.getReader();
3704         reader.loadFileInfo();
3705         reader.loadBloomfilter();
3706         assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
3707         assertEquals(num_unique_rows, reader.getFilterEntries());
3708       }
3709 
3710       region.compactStores(true);
3711 
3712       // after compaction
3713       storeFiles = store.getStorefiles();
3714       for (StoreFile storefile : storeFiles) {
3715         StoreFile.Reader reader = storefile.getReader();
3716         reader.loadFileInfo();
3717         reader.loadBloomfilter();
3718         assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
3719         assertEquals(num_unique_rows, reader.getFilterEntries());
3720       }
3721     } finally {
3722       HRegion.closeHRegion(this.region);
3723       this.region = null;
3724     }
3725   }
3726 
3727   @Test
3728   public void testAllColumnsWithBloomFilter() throws IOException {
3729     byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
3730     byte[] FAMILY = Bytes.toBytes("family");
3731 
3732     // Create table
3733     HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
3734         .setBloomFilterType(BloomType.ROWCOL);
3735     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
3736     htd.addFamily(hcd);
3737     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3738     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3739     try {
3740       // For row:0, col:0: insert versions 1 through 5.
3741       byte row[] = Bytes.toBytes("row:" + 0);
3742       byte column[] = Bytes.toBytes("column:" + 0);
3743       Put put = new Put(row);
3744       put.setDurability(Durability.SKIP_WAL);
3745       for (long idx = 1; idx <= 4; idx++) {
3746         put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
3747       }
3748       region.put(put);
3749 
3750       // Flush
3751       region.flushcache();
3752 
3753       // Get rows
3754       Get get = new Get(row);
3755       get.setMaxVersions();
3756       Cell[] kvs = region.get(get).rawCells();
3757 
3758       // Check if rows are correct
3759       assertEquals(4, kvs.length);
3760       checkOneCell(kvs[0], FAMILY, 0, 0, 4);
3761       checkOneCell(kvs[1], FAMILY, 0, 0, 3);
3762       checkOneCell(kvs[2], FAMILY, 0, 0, 2);
3763       checkOneCell(kvs[3], FAMILY, 0, 0, 1);
3764     } finally {
3765       HRegion.closeHRegion(this.region);
3766       this.region = null;
3767     }
3768   }
3769 
3770   /**
3771    * Testcase to cover bug-fix for HBASE-2823 Ensures correct delete when
3772    * issuing delete row on columns with bloom filter set to row+col
3773    * (BloomType.ROWCOL)
3774    */
3775   @Test
3776   public void testDeleteRowWithBloomFilter() throws IOException {
3777     byte[] familyName = Bytes.toBytes("familyName");
3778 
3779     // Create Table
3780     HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
3781         .setBloomFilterType(BloomType.ROWCOL);
3782 
3783     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3784     htd.addFamily(hcd);
3785     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3786     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3787     try {
3788       // Insert some data
3789       byte row[] = Bytes.toBytes("row1");
3790       byte col[] = Bytes.toBytes("col1");
3791 
3792       Put put = new Put(row);
3793       put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
3794       region.put(put);
3795       region.flushcache();
3796 
3797       Delete del = new Delete(row);
3798       region.delete(del);
3799       region.flushcache();
3800 
3801       // Get remaining rows (should have none)
3802       Get get = new Get(row);
3803       get.addColumn(familyName, col);
3804 
3805       Cell[] keyValues = region.get(get).rawCells();
3806       assertTrue(keyValues.length == 0);
3807     } finally {
3808       HRegion.closeHRegion(this.region);
3809       this.region = null;
3810     }
3811   }
3812 
3813   @Test
3814   public void testgetHDFSBlocksDistribution() throws Exception {
3815     HBaseTestingUtility htu = new HBaseTestingUtility();
3816     final int DEFAULT_BLOCK_SIZE = 1024;
3817     htu.getConfiguration().setLong("dfs.namenode.fs-limits.min-block-size", 0);
3818     htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
3819     htu.getConfiguration().setInt("dfs.replication", 2);
3820 
3821     // set up a cluster with 3 nodes
3822     MiniHBaseCluster cluster = null;
3823     String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
3824     int regionServersCount = 3;
3825 
3826     try {
3827       cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
3828       byte[][] families = { fam1, fam2 };
3829       HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);
3830 
3831       // Setting up region
3832       byte row[] = Bytes.toBytes("row1");
3833       byte col[] = Bytes.toBytes("col1");
3834 
3835       Put put = new Put(row);
3836       put.add(fam1, col, 1, Bytes.toBytes("test1"));
3837       put.add(fam2, col, 1, Bytes.toBytes("test2"));
3838       ht.put(put);
3839 
3840       HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
3841           .get(0);
3842       firstRegion.flushcache();
3843       HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
3844 
3845       // given the default replication factor is 2 and we have 2 HFiles,
3846       // we will have total of 4 replica of blocks on 3 datanodes; thus there
3847       // must be at least one host that have replica for 2 HFiles. That host's
3848       // weight will be equal to the unique block weight.
3849       long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
3850 
3851       String topHost = blocksDistribution1.getTopHosts().get(0);
3852       long topHostWeight = blocksDistribution1.getWeight(topHost);
3853       assertTrue(uniqueBlocksWeight1 == topHostWeight);
3854 
3855       // use the static method to compute the value, it should be the same.
3856       // static method is used by load balancer or other components
3857       HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
3858           htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
3859       long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
3860 
3861       assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
3862 
3863       ht.close();
3864     } finally {
3865       if (cluster != null) {
3866         htu.shutdownMiniCluster();
3867       }
3868     }
3869   }
3870 
3871   /**
3872    * Testcase to check state of region initialization task set to ABORTED or not
3873    * if any exceptions during initialization
3874    * 
3875    * @throws Exception
3876    */
3877   @Test
3878   public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
3879     TableName tableName = TableName.valueOf(name.getMethodName());
3880     HRegionInfo info = null;
3881     try {
3882       FileSystem fs = Mockito.mock(FileSystem.class);
3883       Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
3884       HTableDescriptor htd = new HTableDescriptor(tableName);
3885       htd.addFamily(new HColumnDescriptor("cf"));
3886       info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
3887           HConstants.EMPTY_BYTE_ARRAY, false);
3888       Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
3889       region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
3890       // region initialization throws IOException and set task state to ABORTED.
3891       region.initialize();
3892       fail("Region initialization should fail due to IOException");
3893     } catch (IOException io) {
3894       List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
3895       for (MonitoredTask monitoredTask : tasks) {
3896         if (!(monitoredTask instanceof MonitoredRPCHandler)
3897             && monitoredTask.getDescription().contains(region.toString())) {
3898           assertTrue("Region state should be ABORTED.",
3899               monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
3900           break;
3901         }
3902       }
3903     } finally {
3904       HRegion.closeHRegion(region);
3905     }
3906   }
3907 
3908   /**
3909    * Verifies that the .regioninfo file is written on region creation and that
3910    * is recreated if missing during region opening.
3911    */
3912   @Test
3913   public void testRegionInfoFileCreation() throws IOException {
3914     Path rootDir = new Path(dir + "testRegionInfoFileCreation");
3915 
3916     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
3917     htd.addFamily(new HColumnDescriptor("cf"));
3918 
3919     HRegionInfo hri = new HRegionInfo(htd.getTableName());
3920 
3921     // Create a region and skip the initialization (like CreateTableHandler)
3922     HRegion region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
3923 //    HRegion region = TEST_UTIL.createLocalHRegion(hri, htd);
3924     Path regionDir = region.getRegionFileSystem().getRegionDir();
3925     FileSystem fs = region.getRegionFileSystem().getFileSystem();
3926     HRegion.closeHRegion(region);
3927 
3928     Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
3929 
3930     // Verify that the .regioninfo file is present
3931     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3932         fs.exists(regionInfoFile));
3933 
3934     // Try to open the region
3935     region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3936     assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3937     HRegion.closeHRegion(region);
3938 
3939     // Verify that the .regioninfo file is still there
3940     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3941         fs.exists(regionInfoFile));
3942 
3943     // Remove the .regioninfo file and verify is recreated on region open
3944     fs.delete(regionInfoFile);
3945     assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
3946         fs.exists(regionInfoFile));
3947 
3948     region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3949 //    region = TEST_UTIL.openHRegion(hri, htd);
3950     assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3951     HRegion.closeHRegion(region);
3952 
3953     // Verify that the .regioninfo file is still there
3954     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3955         fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
3956   }
3957 
3958   /**
3959    * TestCase for increment
3960    */
3961   private static class Incrementer implements Runnable {
3962     private HRegion region;
3963     private final static byte[] incRow = Bytes.toBytes("incRow");
3964     private final static byte[] family = Bytes.toBytes("family");
3965     private final static byte[] qualifier = Bytes.toBytes("qualifier");
3966     private final static long ONE = 1l;
3967     private int incCounter;
3968 
3969     public Incrementer(HRegion region, int incCounter) {
3970       this.region = region;
3971       this.incCounter = incCounter;
3972     }
3973 
3974     @Override
3975     public void run() {
3976       int count = 0;
3977       while (count < incCounter) {
3978         Increment inc = new Increment(incRow);
3979         inc.addColumn(family, qualifier, ONE);
3980         count++;
3981         try {
3982           region.increment(inc);
3983         } catch (IOException e) {
3984           e.printStackTrace();
3985           break;
3986         }
3987       }
3988     }
3989   }
3990 
3991   /**
3992    * Test case to check increment function with memstore flushing
3993    * @throws Exception
3994    */
3995   @Test
3996   public void testParallelIncrementWithMemStoreFlush() throws Exception {
3997     byte[] family = Incrementer.family;
3998     this.region = initHRegion(tableName, method, CONF, family);
3999     final HRegion region = this.region;
4000     final AtomicBoolean incrementDone = new AtomicBoolean(false);
4001     Runnable flusher = new Runnable() {
4002       @Override
4003       public void run() {
4004         while (!incrementDone.get()) {
4005           try {
4006             region.flushcache();
4007           } catch (Exception e) {
4008             e.printStackTrace();
4009           }
4010         }
4011       }
4012     };
4013 
4014     // after all increment finished, the row will increment to 20*100 = 2000
4015     int threadNum = 20;
4016     int incCounter = 100;
4017     long expected = threadNum * incCounter;
4018     Thread[] incrementers = new Thread[threadNum];
4019     Thread flushThread = new Thread(flusher);
4020     for (int i = 0; i < threadNum; i++) {
4021       incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
4022       incrementers[i].start();
4023     }
4024     flushThread.start();
4025     for (int i = 0; i < threadNum; i++) {
4026       incrementers[i].join();
4027     }
4028 
4029     incrementDone.set(true);
4030     flushThread.join();
4031 
4032     Get get = new Get(Incrementer.incRow);
4033     get.addColumn(Incrementer.family, Incrementer.qualifier);
4034     get.setMaxVersions(1);
4035     Result res = this.region.get(get);
4036     List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
4037 
4038     // we just got the latest version
4039     assertEquals(kvs.size(), 1);
4040     Cell kv = kvs.get(0);
4041     assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
4042     this.region = null;
4043   }
4044 
4045   /**
4046    * TestCase for append
4047    */
4048   private static class Appender implements Runnable {
4049     private HRegion region;
4050     private final static byte[] appendRow = Bytes.toBytes("appendRow");
4051     private final static byte[] family = Bytes.toBytes("family");
4052     private final static byte[] qualifier = Bytes.toBytes("qualifier");
4053     private final static byte[] CHAR = Bytes.toBytes("a");
4054     private int appendCounter;
4055 
4056     public Appender(HRegion region, int appendCounter) {
4057       this.region = region;
4058       this.appendCounter = appendCounter;
4059     }
4060 
4061     @Override
4062     public void run() {
4063       int count = 0;
4064       while (count < appendCounter) {
4065         Append app = new Append(appendRow);
4066         app.add(family, qualifier, CHAR);
4067         count++;
4068         try {
4069           region.append(app);
4070         } catch (IOException e) {
4071           e.printStackTrace();
4072           break;
4073         }
4074       }
4075     }
4076   }
4077 
4078   /**
4079    * Test case to check append function with memstore flushing
4080    * @throws Exception
4081    */
4082   @Test
4083   public void testParallelAppendWithMemStoreFlush() throws Exception {
4084     byte[] family = Appender.family;
4085     this.region = initHRegion(tableName, method, CONF, family);
4086     final HRegion region = this.region;
4087     final AtomicBoolean appendDone = new AtomicBoolean(false);
4088     Runnable flusher = new Runnable() {
4089       @Override
4090       public void run() {
4091         while (!appendDone.get()) {
4092           try {
4093             region.flushcache();
4094           } catch (Exception e) {
4095             e.printStackTrace();
4096           }
4097         }
4098       }
4099     };
4100 
4101     // after all append finished, the value will append to threadNum *
4102     // appendCounter Appender.CHAR
4103     int threadNum = 20;
4104     int appendCounter = 100;
4105     byte[] expected = new byte[threadNum * appendCounter];
4106     for (int i = 0; i < threadNum * appendCounter; i++) {
4107       System.arraycopy(Appender.CHAR, 0, expected, i, 1);
4108     }
4109     Thread[] appenders = new Thread[threadNum];
4110     Thread flushThread = new Thread(flusher);
4111     for (int i = 0; i < threadNum; i++) {
4112       appenders[i] = new Thread(new Appender(this.region, appendCounter));
4113       appenders[i].start();
4114     }
4115     flushThread.start();
4116     for (int i = 0; i < threadNum; i++) {
4117       appenders[i].join();
4118     }
4119 
4120     appendDone.set(true);
4121     flushThread.join();
4122 
4123     Get get = new Get(Appender.appendRow);
4124     get.addColumn(Appender.family, Appender.qualifier);
4125     get.setMaxVersions(1);
4126     Result res = this.region.get(get);
4127     List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
4128 
4129     // we just got the latest version
4130     assertEquals(kvs.size(), 1);
4131     Cell kv = kvs.get(0);
4132     byte[] appendResult = new byte[kv.getValueLength()];
4133     System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
4134     assertArrayEquals(expected, appendResult);
4135     this.region = null;
4136   }
4137 
4138   /**
4139    * Test case to check put function with memstore flushing for same row, same ts
4140    * @throws Exception
4141    */
4142   @Test
4143   public void testPutWithMemStoreFlush() throws Exception {
4144     byte[] family = Bytes.toBytes("family");
4145     ;
4146     byte[] qualifier = Bytes.toBytes("qualifier");
4147     byte[] row = Bytes.toBytes("putRow");
4148     byte[] value = null;
4149     this.region = initHRegion(tableName, method, CONF, family);
4150     Put put = null;
4151     Get get = null;
4152     List<Cell> kvs = null;
4153     Result res = null;
4154 
4155     put = new Put(row);
4156     value = Bytes.toBytes("value0");
4157     put.add(family, qualifier, 1234567l, value);
4158     region.put(put);
4159     get = new Get(row);
4160     get.addColumn(family, qualifier);
4161     get.setMaxVersions();
4162     res = this.region.get(get);
4163     kvs = res.getColumnCells(family, qualifier);
4164     assertEquals(1, kvs.size());
4165     assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4166 
4167     region.flushcache();
4168     get = new Get(row);
4169     get.addColumn(family, qualifier);
4170     get.setMaxVersions();
4171     res = this.region.get(get);
4172     kvs = res.getColumnCells(family, qualifier);
4173     assertEquals(1, kvs.size());
4174     assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4175 
4176     put = new Put(row);
4177     value = Bytes.toBytes("value1");
4178     put.add(family, qualifier, 1234567l, value);
4179     region.put(put);
4180     get = new Get(row);
4181     get.addColumn(family, qualifier);
4182     get.setMaxVersions();
4183     res = this.region.get(get);
4184     kvs = res.getColumnCells(family, qualifier);
4185     assertEquals(1, kvs.size());
4186     assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4187 
4188     region.flushcache();
4189     get = new Get(row);
4190     get.addColumn(family, qualifier);
4191     get.setMaxVersions();
4192     res = this.region.get(get);
4193     kvs = res.getColumnCells(family, qualifier);
4194     assertEquals(1, kvs.size());
4195     assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4196   }
4197 
4198   @Test
4199   public void testDurability() throws Exception {
4200     String method = "testDurability";
4201     // there are 5 x 5 cases:
4202     // table durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) x mutation
4203     // durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT)
4204 
4205     // expected cases for append and sync wal
4206     durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4207     durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4208     durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4209 
4210     durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4211     durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4212     durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4213 
4214     durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4215     durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4216 
4217     durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
4218     durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4219 
4220     durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
4221     durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
4222     durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
4223 
4224     // expected cases for async wal
4225     durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4226     durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4227     durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4228     durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4229     durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
4230     durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4231 
4232     durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4233     durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4234     durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4235     durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4236     durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
4237     durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
4238 
4239     // expect skip wal cases
4240     durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4241     durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4242     durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4243     durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
4244     durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
4245     durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
4246 
4247   }
4248 
4249   private void durabilityTest(String method, Durability tableDurability,
4250       Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
4251       final boolean expectSyncFromLogSyncer) throws Exception {
4252     Configuration conf = HBaseConfiguration.create(CONF);
4253     method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
4254     TableName tableName = TableName.valueOf(method);
4255     byte[] family = Bytes.toBytes("family");
4256     Path logDir = new Path(new Path(dir + method), "log");
4257     HLog hlog = HLogFactory.createHLog(FILESYSTEM, logDir, UUID.randomUUID().toString(), conf);
4258     final HLog log = spy(hlog);
4259     this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
4260         HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, log,
4261         new byte[][] { family });
4262 
4263     Put put = new Put(Bytes.toBytes("r1"));
4264     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
4265     put.setDurability(mutationDurability);
4266     region.put(put);
4267 
4268     //verify append called or not
4269     verify(log, expectAppend ? times(1) : never())
4270       .appendNoSync((HRegionInfo)any(), eq(tableName), (WALEdit)any(), (List<UUID>)any(),
4271         anyLong(), (HTableDescriptor)any(), (AtomicLong)any(), anyBoolean(), anyLong(), anyLong());
4272 
4273     // verify sync called or not
4274     if (expectSync || expectSyncFromLogSyncer) {
4275       TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
4276         @Override
4277         public boolean evaluate() throws Exception {
4278           try {
4279             if (expectSync) {
4280               verify(log, times(1)).sync(anyLong()); // Hregion calls this one
4281             } else if (expectSyncFromLogSyncer) {
4282               verify(log, times(1)).sync(); // log syncer calls this one
4283             }
4284           } catch (Throwable ignore) {
4285           }
4286           return true;
4287         }
4288       });
4289     } else {
4290       verify(log, never()).sync(anyLong());
4291       verify(log, never()).sync();
4292     }
4293 
4294     HRegion.closeHRegion(this.region);
4295     this.region = null;
4296   }
4297 
4298   private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4299     for (int i = startRow; i < startRow + numRows; i++) {
4300       Put put = new Put(Bytes.toBytes("" + i));
4301       put.setDurability(Durability.SKIP_WAL);
4302       for (byte[] family : families) {
4303         put.add(family, qf, null);
4304       }
4305       region.put(put);
4306     }
4307   }
4308 
4309   private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
4310       throws IOException {
4311     for (int i = startRow; i < startRow + numRows; i++) {
4312       byte[] row = Bytes.toBytes("" + i);
4313       Get get = new Get(row);
4314       for (byte[] family : families) {
4315         get.addColumn(family, qf);
4316       }
4317       Result result = newReg.get(get);
4318       Cell[] raw = result.rawCells();
4319       assertEquals(families.length, result.size());
4320       for (int j = 0; j < families.length; j++) {
4321         assertTrue(CellUtil.matchingRow(raw[j], row));
4322         assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
4323         assertTrue(CellUtil.matchingQualifier(raw[j], qf));
4324       }
4325     }
4326   }
4327 
4328   private void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
4329     // Now I have k, get values out and assert they are as expected.
4330     Get get = new Get(k).addFamily(family).setMaxVersions();
4331     Cell[] results = r.get(get).rawCells();
4332     for (int j = 0; j < results.length; j++) {
4333       byte[] tmp = CellUtil.cloneValue(results[j]);
4334       // Row should be equal to value every time.
4335       assertTrue(Bytes.equals(k, tmp));
4336     }
4337   }
4338 
4339   /*
4340    * Assert first value in the passed region is <code>firstValue</code>.
4341    * 
4342    * @param r
4343    * 
4344    * @param fs
4345    * 
4346    * @param firstValue
4347    * 
4348    * @throws IOException
4349    */
4350   private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
4351       throws IOException {
4352     byte[][] families = { fs };
4353     Scan scan = new Scan();
4354     for (int i = 0; i < families.length; i++)
4355       scan.addFamily(families[i]);
4356     InternalScanner s = r.getScanner(scan);
4357     try {
4358       List<Cell> curVals = new ArrayList<Cell>();
4359       boolean first = true;
4360       OUTER_LOOP: while (s.next(curVals)) {
4361         for (Cell kv : curVals) {
4362           byte[] val = CellUtil.cloneValue(kv);
4363           byte[] curval = val;
4364           if (first) {
4365             first = false;
4366             assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4367           } else {
4368             // Not asserting anything. Might as well break.
4369             break OUTER_LOOP;
4370           }
4371         }
4372       }
4373     } finally {
4374       s.close();
4375     }
4376   }
4377 
4378   /**
4379    * Test that we get the expected flush results back
4380    * @throws IOException
4381    */
4382   @Test
4383   public void testFlushResult() throws IOException {
4384     String method = name.getMethodName();
4385     byte[] tableName = Bytes.toBytes(method);
4386     byte[] family = Bytes.toBytes("family");
4387 
4388     this.region = initHRegion(tableName, method, family);
4389 
4390     // empty memstore, flush doesn't run
4391     HRegion.FlushResult fr = region.flushcache();
4392     assertFalse(fr.isFlushSucceeded());
4393     assertFalse(fr.isCompactionNeeded());
4394 
4395     // Flush enough files to get up to the threshold, doesn't need compactions
4396     for (int i = 0; i < 2; i++) {
4397       Put put = new Put(tableName).add(family, family, tableName);
4398       region.put(put);
4399       fr = region.flushcache();
4400       assertTrue(fr.isFlushSucceeded());
4401       assertFalse(fr.isCompactionNeeded());
4402     }
4403 
4404     // Two flushes after the threshold, compactions are needed
4405     for (int i = 0; i < 2; i++) {
4406       Put put = new Put(tableName).add(family, family, tableName);
4407       region.put(put);
4408       fr = region.flushcache();
4409       assertTrue(fr.isFlushSucceeded());
4410       assertTrue(fr.isCompactionNeeded());
4411     }
4412   }
4413 
4414   private Configuration initSplit() {
4415     // Always compact if there is more than one store file.
4416     CONF.setInt("hbase.hstore.compactionThreshold", 2);
4417 
4418     // Make lease timeout longer, lease checks less frequent
4419     CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4420 
4421     CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4422 
4423     // Increase the amount of time between client retries
4424     CONF.setLong("hbase.client.pause", 15 * 1000);
4425 
4426     // This size should make it so we always split using the addContent
4427     // below. After adding all data, the first region is 1.3M
4428     CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4429     return CONF;
4430   }
4431 
4432   /**
4433    * @param tableName
4434    * @param callingMethod
4435    * @param conf
4436    * @param families
4437    * @throws IOException
4438    * @return A region on which you must call
4439    *         {@link HRegion#closeHRegion(HRegion)} when done.
4440    */
4441   public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
4442       byte[]... families) throws IOException {
4443     return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
4444   }
4445 
4446   /**
4447    * @param tableName
4448    * @param callingMethod
4449    * @param conf
4450    * @param families
4451    * @throws IOException
4452    * @return A region on which you must call
4453    *         {@link HRegion#closeHRegion(HRegion)} when done.
4454    */
4455   public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4456       byte[]... families) throws IOException {
4457     return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4458   }
4459 
4460   /**
4461    * @param tableName
4462    * @param callingMethod
4463    * @param conf
4464    * @param isReadOnly
4465    * @param families
4466    * @throws IOException
4467    * @return A region on which you must call
4468    *         {@link HRegion#closeHRegion(HRegion)} when done.
4469    */
4470   public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4471       boolean isReadOnly, byte[]... families) throws IOException {
4472     return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4473   }
4474 
4475   private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4476       String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4477       throws IOException {
4478     return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
4479         Durability.SYNC_WAL, null, families);
4480   }
4481 
4482   /**
4483    * @param tableName
4484    * @param startKey
4485    * @param stopKey
4486    * @param callingMethod
4487    * @param conf
4488    * @param isReadOnly
4489    * @param families
4490    * @throws IOException
4491    * @return A region on which you must call
4492    *         {@link HRegion#closeHRegion(HRegion)} when done.
4493    */
4494   private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4495       String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4496       HLog hlog, byte[]... families) throws IOException {
4497     return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly, durability, hlog, families);
4498   }
4499 
4500   /**
4501    * Assert that the passed in Cell has expected contents for the specified row,
4502    * column & timestamp.
4503    */
4504   private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
4505     String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4506     assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
4507         Bytes.toString(CellUtil.cloneRow(kv)));
4508     assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
4509         Bytes.toString(CellUtil.cloneFamily(kv)));
4510     assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
4511         Bytes.toString(CellUtil.cloneQualifier(kv)));
4512     assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
4513     assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
4514         Bytes.toString(CellUtil.cloneValue(kv)));
4515   }
4516 
4517   @Test (timeout=60000)
4518   public void testReverseScanner_FromMemStore_SingleCF_Normal()
4519       throws IOException {
4520     byte[] rowC = Bytes.toBytes("rowC");
4521     byte[] rowA = Bytes.toBytes("rowA");
4522     byte[] rowB = Bytes.toBytes("rowB");
4523     byte[] cf = Bytes.toBytes("CF");
4524     byte[][] families = { cf };
4525     byte[] col = Bytes.toBytes("C");
4526     long ts = 1;
4527     String method = this.getName();
4528     this.region = initHRegion(tableName, method, families);
4529     try {
4530       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4531       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4532           null);
4533       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4534       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4535       Put put = null;
4536       put = new Put(rowC);
4537       put.add(kv1);
4538       put.add(kv11);
4539       region.put(put);
4540       put = new Put(rowA);
4541       put.add(kv2);
4542       region.put(put);
4543       put = new Put(rowB);
4544       put.add(kv3);
4545       region.put(put);
4546 
4547       Scan scan = new Scan(rowC);
4548       scan.setMaxVersions(5);
4549       scan.setReversed(true);
4550       InternalScanner scanner = region.getScanner(scan);
4551       List<Cell> currRow = new ArrayList<Cell>();
4552       boolean hasNext = scanner.next(currRow);
4553       assertEquals(2, currRow.size());
4554       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4555       assertTrue(hasNext);
4556       currRow.clear();
4557       hasNext = scanner.next(currRow);
4558       assertEquals(1, currRow.size());
4559       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4560       assertTrue(hasNext);
4561       currRow.clear();
4562       hasNext = scanner.next(currRow);
4563       assertEquals(1, currRow.size());
4564       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4565       assertFalse(hasNext);
4566       scanner.close();
4567     } finally {
4568       HRegion.closeHRegion(this.region);
4569       this.region = null;
4570     }
4571   }
4572 
4573   @Test (timeout=60000)
4574   public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
4575       throws IOException {
4576     byte[] rowC = Bytes.toBytes("rowC");
4577     byte[] rowA = Bytes.toBytes("rowA");
4578     byte[] rowB = Bytes.toBytes("rowB");
4579     byte[] rowD = Bytes.toBytes("rowD");
4580     byte[] cf = Bytes.toBytes("CF");
4581     byte[][] families = { cf };
4582     byte[] col = Bytes.toBytes("C");
4583     long ts = 1;
4584     String method = this.getName();
4585     this.region = initHRegion(tableName, method, families);
4586     try {
4587       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4588       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4589           null);
4590       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4591       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4592       Put put = null;
4593       put = new Put(rowC);
4594       put.add(kv1);
4595       put.add(kv11);
4596       region.put(put);
4597       put = new Put(rowA);
4598       put.add(kv2);
4599       region.put(put);
4600       put = new Put(rowB);
4601       put.add(kv3);
4602       region.put(put);
4603 
4604       Scan scan = new Scan(rowD);
4605       List<Cell> currRow = new ArrayList<Cell>();
4606       scan.setReversed(true);
4607       scan.setMaxVersions(5);
4608       InternalScanner scanner = region.getScanner(scan);
4609       boolean hasNext = scanner.next(currRow);
4610       assertEquals(2, currRow.size());
4611       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4612       assertTrue(hasNext);
4613       currRow.clear();
4614       hasNext = scanner.next(currRow);
4615       assertEquals(1, currRow.size());
4616       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4617       assertTrue(hasNext);
4618       currRow.clear();
4619       hasNext = scanner.next(currRow);
4620       assertEquals(1, currRow.size());
4621       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4622       assertFalse(hasNext);
4623       scanner.close();
4624     } finally {
4625       HRegion.closeHRegion(this.region);
4626       this.region = null;
4627     }
4628   }
4629 
4630   @Test (timeout=60000)
4631   public void testReverseScanner_FromMemStore_SingleCF_FullScan()
4632       throws IOException {
4633     byte[] rowC = Bytes.toBytes("rowC");
4634     byte[] rowA = Bytes.toBytes("rowA");
4635     byte[] rowB = Bytes.toBytes("rowB");
4636     byte[] cf = Bytes.toBytes("CF");
4637     byte[][] families = { cf };
4638     byte[] col = Bytes.toBytes("C");
4639     long ts = 1;
4640     String method = this.getName();
4641     this.region = initHRegion(tableName, method, families);
4642     try {
4643       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4644       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4645           null);
4646       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4647       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4648       Put put = null;
4649       put = new Put(rowC);
4650       put.add(kv1);
4651       put.add(kv11);
4652       region.put(put);
4653       put = new Put(rowA);
4654       put.add(kv2);
4655       region.put(put);
4656       put = new Put(rowB);
4657       put.add(kv3);
4658       region.put(put);
4659       Scan scan = new Scan();
4660       List<Cell> currRow = new ArrayList<Cell>();
4661       scan.setReversed(true);
4662       InternalScanner scanner = region.getScanner(scan);
4663       boolean hasNext = scanner.next(currRow);
4664       assertEquals(1, currRow.size());
4665       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4666       assertTrue(hasNext);
4667       currRow.clear();
4668       hasNext = scanner.next(currRow);
4669       assertEquals(1, currRow.size());
4670       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4671       assertTrue(hasNext);
4672       currRow.clear();
4673       hasNext = scanner.next(currRow);
4674       assertEquals(1, currRow.size());
4675       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4676       assertFalse(hasNext);
4677       scanner.close();
4678     } finally {
4679       HRegion.closeHRegion(this.region);
4680       this.region = null;
4681     }
4682   }
4683 
4684   @Test (timeout=60000)
4685   public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
4686     // case for "INCLUDE_AND_SEEK_NEXT_ROW & SEEK_NEXT_ROW" endless loop
4687     byte[] rowA = Bytes.toBytes("rowA");
4688     byte[] rowB = Bytes.toBytes("rowB");
4689     byte[] rowC = Bytes.toBytes("rowC");
4690     byte[] rowD = Bytes.toBytes("rowD");
4691     byte[] rowE = Bytes.toBytes("rowE");
4692     byte[] cf = Bytes.toBytes("CF");
4693     byte[][] families = { cf };
4694     byte[] col1 = Bytes.toBytes("col1");
4695     byte[] col2 = Bytes.toBytes("col2");
4696     long ts = 1;
4697     String method = this.getName();
4698     this.region = initHRegion(tableName, method, families);
4699     try {
4700       KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4701       KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4702       KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4703       KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4704       KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4705       KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4706       Put put = null;
4707       put = new Put(rowA);
4708       put.add(kv1);
4709       region.put(put);
4710       put = new Put(rowB);
4711       put.add(kv2);
4712       region.put(put);
4713       put = new Put(rowC);
4714       put.add(kv3);
4715       region.put(put);
4716       put = new Put(rowD);
4717       put.add(kv4_1);
4718       region.put(put);
4719       put = new Put(rowD);
4720       put.add(kv4_2);
4721       region.put(put);
4722       put = new Put(rowE);
4723       put.add(kv5);
4724       region.put(put);
4725       region.flushcache();
4726       Scan scan = new Scan(rowD, rowA);
4727       scan.addColumn(families[0], col1);
4728       scan.setReversed(true);
4729       List<Cell> currRow = new ArrayList<Cell>();
4730       InternalScanner scanner = region.getScanner(scan);
4731       boolean hasNext = scanner.next(currRow);
4732       assertEquals(1, currRow.size());
4733       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4734       assertTrue(hasNext);
4735       currRow.clear();
4736       hasNext = scanner.next(currRow);
4737       assertEquals(1, currRow.size());
4738       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4739       assertTrue(hasNext);
4740       currRow.clear();
4741       hasNext = scanner.next(currRow);
4742       assertEquals(1, currRow.size());
4743       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4744       assertFalse(hasNext);
4745       scanner.close();
4746 
4747       scan = new Scan(rowD, rowA);
4748       scan.addColumn(families[0], col2);
4749       scan.setReversed(true);
4750       currRow.clear();
4751       scanner = region.getScanner(scan);
4752       hasNext = scanner.next(currRow);
4753       assertEquals(1, currRow.size());
4754       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4755       scanner.close();
4756     } finally {
4757       HRegion.closeHRegion(this.region);
4758       this.region = null;
4759     }
4760   }
4761 
4762   @Test (timeout=60000)
4763   public void testReverseScanner_smaller_blocksize() throws IOException {
4764     // case to ensure no conflict with HFile index optimization
4765     byte[] rowA = Bytes.toBytes("rowA");
4766     byte[] rowB = Bytes.toBytes("rowB");
4767     byte[] rowC = Bytes.toBytes("rowC");
4768     byte[] rowD = Bytes.toBytes("rowD");
4769     byte[] rowE = Bytes.toBytes("rowE");
4770     byte[] cf = Bytes.toBytes("CF");
4771     byte[][] families = { cf };
4772     byte[] col1 = Bytes.toBytes("col1");
4773     byte[] col2 = Bytes.toBytes("col2");
4774     long ts = 1;
4775     String method = this.getName();
4776     HBaseConfiguration config = new HBaseConfiguration();
4777     config.setInt("test.block.size", 1);
4778     this.region = initHRegion(tableName, method, config, families);
4779     try {
4780       KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4781       KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4782       KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4783       KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4784       KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4785       KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4786       Put put = null;
4787       put = new Put(rowA);
4788       put.add(kv1);
4789       region.put(put);
4790       put = new Put(rowB);
4791       put.add(kv2);
4792       region.put(put);
4793       put = new Put(rowC);
4794       put.add(kv3);
4795       region.put(put);
4796       put = new Put(rowD);
4797       put.add(kv4_1);
4798       region.put(put);
4799       put = new Put(rowD);
4800       put.add(kv4_2);
4801       region.put(put);
4802       put = new Put(rowE);
4803       put.add(kv5);
4804       region.put(put);
4805       region.flushcache();
4806       Scan scan = new Scan(rowD, rowA);
4807       scan.addColumn(families[0], col1);
4808       scan.setReversed(true);
4809       List<Cell> currRow = new ArrayList<Cell>();
4810       InternalScanner scanner = region.getScanner(scan);
4811       boolean hasNext = scanner.next(currRow);
4812       assertEquals(1, currRow.size());
4813       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4814       assertTrue(hasNext);
4815       currRow.clear();
4816       hasNext = scanner.next(currRow);
4817       assertEquals(1, currRow.size());
4818       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4819       assertTrue(hasNext);
4820       currRow.clear();
4821       hasNext = scanner.next(currRow);
4822       assertEquals(1, currRow.size());
4823       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4824       assertFalse(hasNext);
4825       scanner.close();
4826 
4827       scan = new Scan(rowD, rowA);
4828       scan.addColumn(families[0], col2);
4829       scan.setReversed(true);
4830       currRow.clear();
4831       scanner = region.getScanner(scan);
4832       hasNext = scanner.next(currRow);
4833       assertEquals(1, currRow.size());
4834       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4835       scanner.close();
4836     } finally {
4837       HRegion.closeHRegion(this.region);
4838       this.region = null;
4839     }
4840   }
4841 
4842   @Test (timeout=60000)
4843   public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
4844       throws IOException {
4845     byte[] row0 = Bytes.toBytes("row0"); // 1 kv
4846     byte[] row1 = Bytes.toBytes("row1"); // 2 kv
4847     byte[] row2 = Bytes.toBytes("row2"); // 4 kv
4848     byte[] row3 = Bytes.toBytes("row3"); // 2 kv
4849     byte[] row4 = Bytes.toBytes("row4"); // 5 kv
4850     byte[] row5 = Bytes.toBytes("row5"); // 2 kv
4851     byte[] cf1 = Bytes.toBytes("CF1");
4852     byte[] cf2 = Bytes.toBytes("CF2");
4853     byte[] cf3 = Bytes.toBytes("CF3");
4854     byte[][] families = { cf1, cf2, cf3 };
4855     byte[] col = Bytes.toBytes("C");
4856     long ts = 1;
4857     String method = this.getName();
4858     HBaseConfiguration conf = new HBaseConfiguration();
4859     // disable compactions in this test.
4860     conf.setInt("hbase.hstore.compactionThreshold", 10000);
4861     this.region = initHRegion(tableName, method, conf, families);
4862     try {
4863       // kv naming style: kv(row number) totalKvCountInThisRow seq no
4864       KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
4865           null);
4866       KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
4867           null);
4868       KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
4869           KeyValue.Type.Put, null);
4870       KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
4871           null);
4872       KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
4873           null);
4874       KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
4875           null);
4876       KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
4877           KeyValue.Type.Put, null);
4878       KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
4879           null);
4880       KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
4881           KeyValue.Type.Put, null);
4882       KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
4883           null);
4884       KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
4885           null);
4886       KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
4887           KeyValue.Type.Put, null);
4888       KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
4889           null);
4890       KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
4891           KeyValue.Type.Put, null);
4892       KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
4893           null);
4894       KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
4895           null);
4896       // hfiles(cf1/cf2) :"row1"(1 kv) / "row2"(1 kv) / "row4"(2 kv)
4897       Put put = null;
4898       put = new Put(row1);
4899       put.add(kv1_2_1);
4900       region.put(put);
4901       put = new Put(row2);
4902       put.add(kv2_4_1);
4903       region.put(put);
4904       put = new Put(row4);
4905       put.add(kv4_5_4);
4906       put.add(kv4_5_5);
4907       region.put(put);
4908       region.flushcache();
4909       // hfiles(cf1/cf3) : "row1" (1 kvs) / "row2" (1 kv) / "row4" (2 kv)
4910       put = new Put(row4);
4911       put.add(kv4_5_1);
4912       put.add(kv4_5_3);
4913       region.put(put);
4914       put = new Put(row1);
4915       put.add(kv1_2_2);
4916       region.put(put);
4917       put = new Put(row2);
4918       put.add(kv2_4_4);
4919       region.put(put);
4920       region.flushcache();
4921       // hfiles(cf1/cf3) : "row2"(2 kv) / "row3"(1 kvs) / "row4" (1 kv)
4922       put = new Put(row4);
4923       put.add(kv4_5_2);
4924       region.put(put);
4925       put = new Put(row2);
4926       put.add(kv2_4_2);
4927       put.add(kv2_4_3);
4928       region.put(put);
4929       put = new Put(row3);
4930       put.add(kv3_2_2);
4931       region.put(put);
4932       region.flushcache();
4933       // memstore(cf1/cf2/cf3) : "row0" (1 kvs) / "row3" ( 1 kv) / "row5" (max)
4934       // ( 2 kv)
4935       put = new Put(row0);
4936       put.add(kv0_1_1);
4937       region.put(put);
4938       put = new Put(row3);
4939       put.add(kv3_2_1);
4940       region.put(put);
4941       put = new Put(row5);
4942       put.add(kv5_2_1);
4943       put.add(kv5_2_2);
4944       region.put(put);
4945       // scan range = ["row4", min), skip the max "row5"
4946       Scan scan = new Scan(row4);
4947       scan.setMaxVersions(5);
4948       scan.setBatch(3);
4949       scan.setReversed(true);
4950       InternalScanner scanner = region.getScanner(scan);
4951       List<Cell> currRow = new ArrayList<Cell>();
4952       boolean hasNext = false;
4953       // 1. scan out "row4" (5 kvs), "row5" can't be scanned out since not
4954       // included in scan range
4955       // "row4" takes 2 next() calls since batch=3
4956       hasNext = scanner.next(currRow);
4957       assertEquals(3, currRow.size());
4958       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4959       assertTrue(hasNext);
4960       currRow.clear();
4961       hasNext = scanner.next(currRow);
4962       assertEquals(2, currRow.size());
4963       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4964       assertTrue(hasNext);
4965       // 2. scan out "row3" (2 kv)
4966       currRow.clear();
4967       hasNext = scanner.next(currRow);
4968       assertEquals(2, currRow.size());
4969       assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
4970       assertTrue(hasNext);
4971       // 3. scan out "row2" (4 kvs)
4972       // "row2" takes 2 next() calls since batch=3
4973       currRow.clear();
4974       hasNext = scanner.next(currRow);
4975       assertEquals(3, currRow.size());
4976       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4977       assertTrue(hasNext);
4978       currRow.clear();
4979       hasNext = scanner.next(currRow);
4980       assertEquals(1, currRow.size());
4981       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4982       assertTrue(hasNext);
4983       // 4. scan out "row1" (2 kv)
4984       currRow.clear();
4985       hasNext = scanner.next(currRow);
4986       assertEquals(2, currRow.size());
4987       assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
4988       assertTrue(hasNext);
4989       // 5. scan out "row0" (1 kv)
4990       currRow.clear();
4991       hasNext = scanner.next(currRow);
4992       assertEquals(1, currRow.size());
4993       assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
4994       assertFalse(hasNext);
4995 
4996       scanner.close();
4997     } finally {
4998       HRegion.closeHRegion(this.region);
4999       this.region = null;
5000     }
5001   }
5002 
5003   @Test (timeout=60000)
5004   public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
5005       throws IOException {
5006     byte[] row1 = Bytes.toBytes("row1");
5007     byte[] row2 = Bytes.toBytes("row2");
5008     byte[] row3 = Bytes.toBytes("row3");
5009     byte[] row4 = Bytes.toBytes("row4");
5010     byte[] cf1 = Bytes.toBytes("CF1");
5011     byte[] cf2 = Bytes.toBytes("CF2");
5012     byte[] cf3 = Bytes.toBytes("CF3");
5013     byte[] cf4 = Bytes.toBytes("CF4");
5014     byte[][] families = { cf1, cf2, cf3, cf4 };
5015     byte[] col = Bytes.toBytes("C");
5016     long ts = 1;
5017     String method = this.getName();
5018     HBaseConfiguration conf = new HBaseConfiguration();
5019     // disable compactions in this test.
5020     conf.setInt("hbase.hstore.compactionThreshold", 10000);
5021     this.region = initHRegion(tableName, method, conf, families);
5022     try {
5023       KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
5024       KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
5025       KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
5026       KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
5027       // storefile1
5028       Put put = new Put(row1);
5029       put.add(kv1);
5030       region.put(put);
5031       region.flushcache();
5032       // storefile2
5033       put = new Put(row2);
5034       put.add(kv2);
5035       region.put(put);
5036       region.flushcache();
5037       // storefile3
5038       put = new Put(row3);
5039       put.add(kv3);
5040       region.put(put);
5041       region.flushcache();
5042       // memstore
5043       put = new Put(row4);
5044       put.add(kv4);
5045       region.put(put);
5046       // scan range = ["row4", min)
5047       Scan scan = new Scan(row4);
5048       scan.setReversed(true);
5049       scan.setBatch(10);
5050       InternalScanner scanner = region.getScanner(scan);
5051       List<Cell> currRow = new ArrayList<Cell>();
5052       boolean hasNext = scanner.next(currRow);
5053       assertEquals(1, currRow.size());
5054       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5055       assertTrue(hasNext);
5056       currRow.clear();
5057       hasNext = scanner.next(currRow);
5058       assertEquals(1, currRow.size());
5059       assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5060       assertTrue(hasNext);
5061       currRow.clear();
5062       hasNext = scanner.next(currRow);
5063       assertEquals(1, currRow.size());
5064       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5065       assertTrue(hasNext);
5066       currRow.clear();
5067       hasNext = scanner.next(currRow);
5068       assertEquals(1, currRow.size());
5069       assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5070       assertFalse(hasNext);
5071     } finally {
5072       HRegion.closeHRegion(this.region);
5073       this.region = null;
5074     }
5075   }
5076 
5077   @Test (timeout=60000)
5078   public void testSplitRegionWithReverseScan() throws IOException {
5079     byte [] tableName = Bytes.toBytes("testSplitRegionWithReverseScan");
5080     byte [] qualifier = Bytes.toBytes("qualifier");
5081     Configuration hc = initSplit();
5082     int numRows = 3;
5083     byte [][] families = {fam1};
5084 
5085     //Setting up region
5086     String method = this.getName();
5087     this.region = initHRegion(tableName, method, hc, families);
5088 
5089     //Put data in region
5090     int startRow = 100;
5091     putData(startRow, numRows, qualifier, families);
5092     int splitRow = startRow + numRows;
5093     putData(splitRow, numRows, qualifier, families);
5094     int endRow = splitRow + numRows;
5095     region.flushcache();
5096 
5097     HRegion [] regions = null;
5098     try {
5099       regions = splitRegion(region, Bytes.toBytes("" + splitRow));
5100       //Opening the regions returned.
5101       for (int i = 0; i < regions.length; i++) {
5102         regions[i] = HRegion.openHRegion(regions[i], null);
5103       }
5104       //Verifying that the region has been split
5105       assertEquals(2, regions.length);
5106 
5107       //Verifying that all data is still there and that data is in the right
5108       //place
5109       verifyData(regions[0], startRow, numRows, qualifier, families);
5110       verifyData(regions[1], splitRow, numRows, qualifier, families);
5111 
5112       //fire the reverse scan1:  top range, and larger than the last row
5113       Scan scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 10 * numRows)));
5114       scan.setReversed(true);
5115       InternalScanner scanner = regions[1].getScanner(scan);
5116       List<Cell> currRow = new ArrayList<Cell>();
5117       boolean more = false;
5118       int verify = startRow + 2 * numRows - 1;
5119       do {
5120         more = scanner.next(currRow);
5121         assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5122         verify--;
5123         currRow.clear();
5124       } while(more);
5125       assertEquals(verify, startRow + numRows - 1);
5126       scanner.close();
5127       //fire the reverse scan2:  top range, and equals to the last row
5128       scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 2 * numRows - 1)));
5129       scan.setReversed(true);
5130       scanner = regions[1].getScanner(scan);
5131       verify = startRow + 2 * numRows - 1;
5132       do {
5133         more = scanner.next(currRow);
5134         assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5135         verify--;
5136         currRow.clear();
5137       } while(more);
5138       assertEquals(verify, startRow + numRows - 1);
5139       scanner.close();
5140       //fire the reverse scan3:  bottom range, and larger than the last row
5141       scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows)));
5142       scan.setReversed(true);
5143       scanner = regions[0].getScanner(scan);
5144       verify = startRow + numRows - 1;
5145       do {
5146         more = scanner.next(currRow);
5147         assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5148         verify--;
5149         currRow.clear();
5150       } while(more);
5151       assertEquals(verify, 99);
5152       scanner.close();
5153       //fire the reverse scan4:  bottom range, and equals to the last row
5154       scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows - 1)));
5155       scan.setReversed(true);
5156       scanner = regions[0].getScanner(scan);
5157       verify = startRow + numRows - 1;
5158       do {
5159         more = scanner.next(currRow);
5160         assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5161         verify--;
5162         currRow.clear();
5163       } while(more);
5164       assertEquals(verify, startRow - 1);
5165       scanner.close();
5166     } finally {
5167       HRegion.closeHRegion(this.region);
5168       this.region = null;
5169     }
5170   }
5171 
5172   @Test
5173   public void testWriteRequestsCounter() throws IOException {
5174     byte[] fam = Bytes.toBytes("info");
5175     byte[][] families = { fam };
5176     this.region = initHRegion(tableName, method, CONF, families);
5177 
5178     Assert.assertEquals(0L, region.getWriteRequestsCount());
5179 
5180     Put put = new Put(row);
5181     put.add(fam, fam, fam);
5182 
5183     Assert.assertEquals(0L, region.getWriteRequestsCount());
5184     region.put(put);
5185     Assert.assertEquals(1L, region.getWriteRequestsCount());
5186     region.put(put);
5187     Assert.assertEquals(2L, region.getWriteRequestsCount());
5188     region.put(put);
5189     Assert.assertEquals(3L, region.getWriteRequestsCount());
5190 
5191     region.delete(new Delete(row));
5192     Assert.assertEquals(4L, region.getWriteRequestsCount());
5193 
5194     HRegion.closeHRegion(this.region);
5195     this.region = null;
5196   }
5197 
5198   /**
5199    * Test RegionTooBusyException thrown when region is busy
5200    */
5201   @Test (timeout=24000)
5202   public void testRegionTooBusy() throws IOException {
5203     String method = "testRegionTooBusy";
5204     byte[] tableName = Bytes.toBytes(method);
5205     byte[] family = Bytes.toBytes("family");
5206     long defaultBusyWaitDuration = CONF.getLong("hbase.busy.wait.duration",
5207       HRegion.DEFAULT_BUSY_WAIT_DURATION);
5208     CONF.setLong("hbase.busy.wait.duration", 1000);
5209     region = initHRegion(tableName, method, CONF, family);
5210     final AtomicBoolean stopped = new AtomicBoolean(true);
5211     Thread t = new Thread(new Runnable() {
5212       @Override
5213       public void run() {
5214         try {
5215           region.lock.writeLock().lock();
5216           stopped.set(false);
5217           while (!stopped.get()) {
5218             Thread.sleep(100);
5219           }
5220         } catch (InterruptedException ie) {
5221         } finally {
5222           region.lock.writeLock().unlock();
5223         }
5224       }
5225     });
5226     t.start();
5227     Get get = new Get(row);
5228     try {
5229       while (stopped.get()) {
5230         Thread.sleep(100);
5231       }
5232       region.get(get);
5233       fail("Should throw RegionTooBusyException");
5234     } catch (InterruptedException ie) {
5235       fail("test interrupted");
5236     } catch (RegionTooBusyException e) {
5237       // Good, expected
5238     } finally {
5239       stopped.set(true);
5240       try {
5241         t.join();
5242       } catch (Throwable e) {
5243       }
5244 
5245       HRegion.closeHRegion(region);
5246       region = null;
5247       CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration);
5248     }
5249   }
5250 
5251   @Test
5252   public void testCellTTLs() throws IOException {
5253     IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
5254     EnvironmentEdgeManager.injectEdge(edge);
5255 
5256     final byte[] row = Bytes.toBytes("testRow");
5257     final byte[] q1 = Bytes.toBytes("q1");
5258     final byte[] q2 = Bytes.toBytes("q2");
5259     final byte[] q3 = Bytes.toBytes("q3");
5260     final byte[] q4 = Bytes.toBytes("q4");
5261 
5262     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCellTTLs"));
5263     HColumnDescriptor hcd = new HColumnDescriptor(fam1);
5264     hcd.setTimeToLive(10); // 10 seconds
5265     htd.addFamily(hcd);
5266 
5267     Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
5268     conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS);
5269 
5270     HRegion region = HRegion.createHRegion(new HRegionInfo(htd.getTableName(),
5271         HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY),
5272       TEST_UTIL.getDataTestDir(), conf, htd);
5273     assertNotNull(region);
5274     try {
5275       long now = EnvironmentEdgeManager.currentTimeMillis();
5276       // Add a cell that will expire in 5 seconds via cell TTL
5277       region.put(new Put(row).add(new KeyValue(row, fam1, q1, now,
5278         HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
5279           // TTL tags specify ts in milliseconds
5280           new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
5281       // Add a cell that will expire after 10 seconds via family setting
5282       region.put(new Put(row).add(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY));
5283       // Add a cell that will expire in 15 seconds via cell TTL
5284       region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1,
5285         HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
5286           // TTL tags specify ts in milliseconds
5287           new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
5288       // Add a cell that will expire in 20 seconds via family setting
5289       region.put(new Put(row).add(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY));
5290 
5291       // Flush so we are sure store scanning gets this right
5292       region.flushcache();
5293 
5294       // A query at time T+0 should return all cells
5295       Result r = region.get(new Get(row));
5296       assertNotNull(r.getValue(fam1, q1));
5297       assertNotNull(r.getValue(fam1, q2));
5298       assertNotNull(r.getValue(fam1, q3));
5299       assertNotNull(r.getValue(fam1, q4));
5300 
5301       // Increment time to T+5 seconds
5302       edge.incrementTime(5000);
5303 
5304       r = region.get(new Get(row));
5305       assertNull(r.getValue(fam1, q1));
5306       assertNotNull(r.getValue(fam1, q2));
5307       assertNotNull(r.getValue(fam1, q3));
5308       assertNotNull(r.getValue(fam1, q4));
5309 
5310       // Increment time to T+10 seconds
5311       edge.incrementTime(5000);
5312 
5313       r = region.get(new Get(row));
5314       assertNull(r.getValue(fam1, q1));
5315       assertNull(r.getValue(fam1, q2));
5316       assertNotNull(r.getValue(fam1, q3));
5317       assertNotNull(r.getValue(fam1, q4));
5318 
5319       // Increment time to T+15 seconds
5320       edge.incrementTime(5000);
5321 
5322       r = region.get(new Get(row));
5323       assertNull(r.getValue(fam1, q1));
5324       assertNull(r.getValue(fam1, q2));
5325       assertNull(r.getValue(fam1, q3));
5326       assertNotNull(r.getValue(fam1, q4));
5327 
5328       // Increment time to T+20 seconds
5329       edge.incrementTime(10000);
5330 
5331       r = region.get(new Get(row));
5332       assertNull(r.getValue(fam1, q1));
5333       assertNull(r.getValue(fam1, q2));
5334       assertNull(r.getValue(fam1, q3));
5335       assertNull(r.getValue(fam1, q4));
5336 
5337       // Fun with disappearing increments
5338 
5339       // Start at 1
5340       region.put(new Put(row).add(fam1, q1, Bytes.toBytes(1L)));
5341       r = region.get(new Get(row));
5342       byte[] val = r.getValue(fam1, q1);
5343       assertNotNull(val);
5344       assertEquals(Bytes.toLong(val), 1L);
5345 
5346       // Increment with a TTL of 5 seconds
5347       Increment incr = new Increment(row).addColumn(fam1, q1, 1L);
5348       incr.setTTL(5000);
5349       region.increment(incr); // 2
5350 
5351       // New value should be 2
5352       r = region.get(new Get(row));
5353       val = r.getValue(fam1, q1);
5354       assertNotNull(val);
5355       assertEquals(Bytes.toLong(val), 2L);
5356 
5357       // Increment time to T+25 seconds
5358       edge.incrementTime(5000);
5359 
5360       // Value should be back to 1
5361       r = region.get(new Get(row));
5362       val = r.getValue(fam1, q1);
5363       assertNotNull(val);
5364       assertEquals(Bytes.toLong(val), 1L);
5365 
5366       // Increment time to T+30 seconds
5367       edge.incrementTime(5000);
5368 
5369       // Original value written at T+20 should be gone now via family TTL
5370       r = region.get(new Get(row));
5371       assertNull(r.getValue(fam1, q1));
5372 
5373     } finally {
5374       HRegion.closeHRegion(region);
5375     }
5376   }
5377 
5378   @Test
5379   public void testIncrementTimestampsAreMonotonic() throws IOException {
5380     HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5381     ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5382     EnvironmentEdgeManager.injectEdge(edge);
5383 
5384     edge.setValue(10);
5385     Increment inc = new Increment(row);
5386     inc.setDurability(Durability.SKIP_WAL);
5387     inc.addColumn(fam1, qual1, 1L);
5388     region.increment(inc);
5389 
5390     Result result = region.get(new Get(row));
5391     Cell c = result.getColumnLatestCell(fam1, qual1);
5392     assertNotNull(c);
5393     assertEquals(c.getTimestamp(), 10L);
5394 
5395     edge.setValue(1); // clock goes back
5396     region.increment(inc);
5397     result = region.get(new Get(row));
5398     c = result.getColumnLatestCell(fam1, qual1);
5399     assertEquals(c.getTimestamp(), 10L);
5400     assertEquals(Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength()), 2L);
5401   }
5402 
5403   @Test
5404   public void testAppendTimestampsAreMonotonic() throws IOException {
5405     HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5406     ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5407     EnvironmentEdgeManager.injectEdge(edge);
5408 
5409     edge.setValue(10);
5410     Append a = new Append(row);
5411     a.setDurability(Durability.SKIP_WAL);
5412     a.add(fam1, qual1, qual1);
5413     region.append(a);
5414 
5415     Result result = region.get(new Get(row));
5416     Cell c = result.getColumnLatestCell(fam1, qual1);
5417     assertNotNull(c);
5418     assertEquals(c.getTimestamp(), 10L);
5419 
5420     edge.setValue(1); // clock goes back
5421     region.append(a);
5422     result = region.get(new Get(row));
5423     c = result.getColumnLatestCell(fam1, qual1);
5424     assertEquals(c.getTimestamp(), 10L);
5425 
5426     byte[] expected = new byte[qual1.length*2];
5427     System.arraycopy(qual1, 0, expected, 0, qual1.length);
5428     System.arraycopy(qual1, 0, expected, qual1.length, qual1.length);
5429 
5430     assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
5431       expected, 0, expected.length));
5432   }
5433 
5434   @Test
5435   public void testCheckAndMutateTimestampsAreMonotonic() throws IOException {
5436     HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5437     ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5438     EnvironmentEdgeManager.injectEdge(edge);
5439 
5440     edge.setValue(10);
5441     Put p = new Put(row);
5442     p.setDurability(Durability.SKIP_WAL);
5443     p.add(fam1, qual1, qual1);
5444     region.put(p);
5445 
5446     Result result = region.get(new Get(row));
5447     Cell c = result.getColumnLatestCell(fam1, qual1);
5448     assertNotNull(c);
5449     assertEquals(c.getTimestamp(), 10L);
5450 
5451     edge.setValue(1); // clock goes back
5452     p = new Put(row);
5453     p.setDurability(Durability.SKIP_WAL);
5454     p.add(fam1, qual1, qual2);
5455     region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(qual1), p, false);
5456     result = region.get(new Get(row));
5457     c = result.getColumnLatestCell(fam1, qual1);
5458     assertEquals(c.getTimestamp(), 10L);
5459 
5460     assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
5461       qual2, 0, qual2.length));
5462   }
5463 
5464   @Test
5465   public void testCheckAndRowMutateTimestampsAreMonotonic() throws IOException {
5466     HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5467     ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5468     EnvironmentEdgeManager.injectEdge(edge);
5469 
5470     edge.setValue(10);
5471     Put p = new Put(row);
5472     p.setDurability(Durability.SKIP_WAL);
5473     p.add(fam1, qual1, qual1);
5474     region.put(p);
5475 
5476     Result result = region.get(new Get(row));
5477     Cell c = result.getColumnLatestCell(fam1, qual1);
5478     assertNotNull(c);
5479     assertEquals(c.getTimestamp(), 10L);
5480 
5481     edge.setValue(1); // clock goes back
5482     p = new Put(row);
5483     p.setDurability(Durability.SKIP_WAL);
5484     p.add(fam1, qual1, qual2);
5485     RowMutations rm = new RowMutations(row);
5486     rm.add(p);
5487     region.checkAndRowMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(qual1),
5488       rm, false);
5489     result = region.get(new Get(row));
5490     c = result.getColumnLatestCell(fam1, qual1);
5491     assertEquals(c.getTimestamp(), 10L);
5492 
5493     assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
5494       qual2, 0, qual2.length));
5495   }
5496 
5497   private static HRegion initHRegion(byte[] tableName, String callingMethod,
5498       byte[]... families) throws IOException {
5499     return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
5500         families);
5501   }
5502 }