View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  
22  import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23  import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24  import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25  import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28  import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29  import static org.junit.Assert.assertArrayEquals;
30  import static org.junit.Assert.assertEquals;
31  import static org.junit.Assert.assertFalse;
32  import static org.junit.Assert.assertNotNull;
33  import static org.junit.Assert.assertNull;
34  import static org.junit.Assert.assertTrue;
35  import static org.junit.Assert.fail;
36  import static org.mockito.Matchers.any;
37  import static org.mockito.Matchers.anyBoolean;
38  import static org.mockito.Matchers.anyLong;
39  import static org.mockito.Matchers.eq;
40  import static org.mockito.Mockito.never;
41  import static org.mockito.Mockito.spy;
42  import static org.mockito.Mockito.times;
43  import static org.mockito.Mockito.verify;
44  
45  import java.io.IOException;
46  import java.io.InterruptedIOException;
47  import java.security.PrivilegedExceptionAction;
48  import java.util.ArrayList;
49  import java.util.Arrays;
50  import java.util.Collection;
51  import java.util.List;
52  import java.util.Map;
53  import java.util.NavigableMap;
54  import java.util.TreeMap;
55  import java.util.UUID;
56  import java.util.concurrent.atomic.AtomicBoolean;
57  import java.util.concurrent.atomic.AtomicInteger;
58  import java.util.concurrent.atomic.AtomicLong;
59  import java.util.concurrent.atomic.AtomicReference;
60  
61  import org.apache.commons.logging.Log;
62  import org.apache.commons.logging.LogFactory;
63  import org.apache.hadoop.conf.Configuration;
64  import org.apache.hadoop.fs.FSDataOutputStream;
65  import org.apache.hadoop.fs.FileStatus;
66  import org.apache.hadoop.fs.FileSystem;
67  import org.apache.hadoop.fs.Path;
68  import org.apache.hadoop.hbase.Cell;
69  import org.apache.hadoop.hbase.CellComparator;
70  import org.apache.hadoop.hbase.CellUtil;
71  import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
72  import org.apache.hadoop.hbase.DroppedSnapshotException;
73  import org.apache.hadoop.hbase.HBaseConfiguration;
74  import org.apache.hadoop.hbase.HBaseTestCase;
75  import org.apache.hadoop.hbase.HBaseTestingUtility;
76  import org.apache.hadoop.hbase.HColumnDescriptor;
77  import org.apache.hadoop.hbase.HConstants;
78  import org.apache.hadoop.hbase.RegionTooBusyException;
79  import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
80  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
81  import org.apache.hadoop.hbase.HRegionInfo;
82  import org.apache.hadoop.hbase.HTableDescriptor;
83  import org.apache.hadoop.hbase.KeyValue;
84  import org.apache.hadoop.hbase.MediumTests;
85  import org.apache.hadoop.hbase.MiniHBaseCluster;
86  import org.apache.hadoop.hbase.MultithreadedTestUtil;
87  import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
88  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
89  import org.apache.hadoop.hbase.NotServingRegionException;
90  import org.apache.hadoop.hbase.TableName;
91  import org.apache.hadoop.hbase.Waiter;
92  import org.apache.hadoop.hbase.client.Append;
93  import org.apache.hadoop.hbase.client.Delete;
94  import org.apache.hadoop.hbase.client.Durability;
95  import org.apache.hadoop.hbase.client.Get;
96  import org.apache.hadoop.hbase.client.HTable;
97  import org.apache.hadoop.hbase.client.Increment;
98  import org.apache.hadoop.hbase.client.Put;
99  import org.apache.hadoop.hbase.client.Result;
100 import org.apache.hadoop.hbase.client.Scan;
101 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
102 import org.apache.hadoop.hbase.filter.BinaryComparator;
103 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
104 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
105 import org.apache.hadoop.hbase.filter.Filter;
106 import org.apache.hadoop.hbase.filter.FilterBase;
107 import org.apache.hadoop.hbase.filter.FilterList;
108 import org.apache.hadoop.hbase.filter.NullComparator;
109 import org.apache.hadoop.hbase.filter.PrefixFilter;
110 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
111 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
112 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
113 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
114 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
115 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
116 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
117 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
118 import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
119 import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
120 import org.apache.hadoop.hbase.regionserver.wal.FaultyHLog;
121 import org.apache.hadoop.hbase.regionserver.wal.HLog;
122 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
123 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
124 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
125 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
126 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
127 import org.apache.hadoop.hbase.security.User;
128 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
129 import org.apache.hadoop.hbase.util.Bytes;
130 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
131 import org.apache.hadoop.hbase.util.FSUtils;
132 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
133 import org.apache.hadoop.hbase.util.PairOfSameType;
134 import org.apache.hadoop.hbase.util.Threads;
135 import org.junit.After;
136 import org.junit.Assert;
137 import org.junit.Before;
138 import org.junit.Rule;
139 import org.junit.Test;
140 import org.junit.experimental.categories.Category;
141 import org.junit.rules.TestName;
142 import org.mockito.Mockito;
143 
144 import com.google.common.collect.ImmutableList;
145 import com.google.common.collect.Lists;
146 import com.google.protobuf.ByteString;
147 
148 /**
149  * Basic stand-alone testing of HRegion.  No clusters!
150  *
151  * A lot of the meta information for an HRegion now lives inside other HRegions
152  * or in the HBaseMaster, so only basic testing is possible.
153  */
154 @Category(MediumTests.class)
155 @SuppressWarnings("deprecation")
156 public class TestHRegion {
157   // Do not spin up clusters in here. If you need to spin up a cluster, do it
158   // over in TestHRegionOnCluster.
159   static final Log LOG = LogFactory.getLog(TestHRegion.class);
160   @Rule public TestName name = new TestName();
161 
162   private static final String COLUMN_FAMILY = "MyCF";
163   private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
164 
165   HRegion region = null;
166   // Do not run unit tests in parallel (? Why not?  It don't work?  Why not?  St.Ack)
167   private static HBaseTestingUtility TEST_UTIL;
168   public static Configuration CONF ;
169   private String dir;
170   private static FileSystem FILESYSTEM;
171   private final int MAX_VERSIONS = 2;
172 
173   // Test names
174   protected byte[] tableName;
175   protected String method;
176   protected final byte[] qual1 = Bytes.toBytes("qual1");
177   protected final byte[] qual2 = Bytes.toBytes("qual2");
178   protected final byte[] qual3 = Bytes.toBytes("qual3");
179   protected final byte[] value1 = Bytes.toBytes("value1");
180   protected final byte[] value2 = Bytes.toBytes("value2");
181   protected final byte[] row = Bytes.toBytes("rowA");
182   protected final byte[] row2 = Bytes.toBytes("rowB");
183 
184   protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
185       .getInstance(MetricsAssertHelper.class);
186 
187   @Before
188   public void setup() throws IOException {
189     TEST_UTIL = HBaseTestingUtility.createLocalHTU();
190     FILESYSTEM = TEST_UTIL.getTestFileSystem();
191     CONF = TEST_UTIL.getConfiguration();
192     dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
193     method = name.getMethodName();
194     tableName = Bytes.toBytes(name.getMethodName());
195   }
196 
197   @After
198   public void tearDown() throws Exception {
199     EnvironmentEdgeManagerTestHelper.reset();
200     LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
201     TEST_UTIL.cleanupTestDir();
202   }
203 
204   String getName() {
205     return name.getMethodName();
206   }
207 
208   /**
209    * Test for Bug 2 of HBASE-10466.
210    * "Bug 2: Conditions for the first flush of region close (so-called pre-flush) If memstoreSize
211    * is smaller than a certain value, or when region close starts a flush is ongoing, the first
212    * flush is skipped and only the second flush takes place. However, two flushes are required in
213    * case previous flush fails and leaves some data in snapshot. The bug could cause loss of data
214    * in current memstore. The fix is removing all conditions except abort check so we ensure 2
215    * flushes for region close."
216    * @throws IOException
217    */
218   @Test (timeout=60000)
219   public void testCloseCarryingSnapshot() throws IOException {
220     HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
221     Store store = region.getStore(COLUMN_FAMILY_BYTES);
222     // Get some random bytes.
223     byte [] value = Bytes.toBytes(name.getMethodName());
224     // Make a random put against our cf.
225     Put put = new Put(value);
226     put.add(COLUMN_FAMILY_BYTES, null, value);
227     // First put something in current memstore, which will be in snapshot after flusher.prepare()
228     region.put(put);
229     StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
230     storeFlushCtx.prepare();
231     // Second put something in current memstore
232     put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
233     region.put(put);
234     // Close with something in memstore and something in the snapshot.  Make sure all is cleared.
235     region.close();
236     assertEquals(0, region.getMemstoreSize().get());
237     HRegion.closeHRegion(region);
238   }
239 
240   /*
241    * This test is for verifying memstore snapshot size is correctly updated in case of rollback
242    * See HBASE-10845
243    */
244   @Test (timeout=60000)
245   public void testMemstoreSnapshotSize() throws IOException {
246     class MyFaultyHLog extends FaultyHLog {
247       StoreFlushContext storeFlushCtx;
248       public MyFaultyHLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
249           throws IOException {
250         super(fs, rootDir, logName, conf);
251       }
252       
253       void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
254         this.storeFlushCtx = storeFlushCtx;
255       }
256 
257       @Override
258       public void sync(long txid) throws IOException {
259         storeFlushCtx.prepare();
260         super.sync(txid);
261       }
262     }
263     
264     FileSystem fs = FileSystem.get(CONF);
265     Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
266     MyFaultyHLog faultyLog = new MyFaultyHLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
267     HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
268       CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES);
269     
270     Store store = region.getStore(COLUMN_FAMILY_BYTES);
271     // Get some random bytes.
272     byte [] value = Bytes.toBytes(name.getMethodName());
273     faultyLog.setStoreFlushCtx(store.createFlushContext(12345));
274     
275     Put put = new Put(value);
276     put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
277     faultyLog.setFailureType(FaultyHLog.FailureType.SYNC);
278 
279     boolean threwIOE = false;
280     try {
281       region.put(put);
282     } catch (IOException ioe) {
283       threwIOE = true;
284     } finally {
285       assertTrue("The regionserver should have thrown an exception", threwIOE);
286     }
287     long sz = store.getFlushableSize();
288     assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
289     HRegion.closeHRegion(region);
290   }
291   
292   /**
293    * Test we do not lose data if we fail a flush and then close.
294    * Part of HBase-10466.  Tests the following from the issue description:
295    * "Bug 1: Wrong calculation of HRegion.memstoreSize: When a flush fails, data to be flushed is
296    * kept in each MemStore's snapshot and wait for next flush attempt to continue on it. But when
297    * the next flush succeeds, the counter of total memstore size in HRegion is always deduced by
298    * the sum of current memstore sizes instead of snapshots left from previous failed flush. This
299    * calculation is problematic that almost every time there is failed flush, HRegion.memstoreSize
300    * gets reduced by a wrong value. If region flush could not proceed for a couple cycles, the size
301    * in current memstore could be much larger than the snapshot. It's likely to drift memstoreSize
302    * much smaller than expected. In extreme case, if the error accumulates to even bigger than
303    * HRegion's memstore size limit, any further flush is skipped because flush does not do anything
304    * if memstoreSize is not larger than 0."
305    * @throws Exception
306    */
307   @Test (timeout=60000)
308   public void testFlushSizeAccounting() throws Exception {
309     final Configuration conf = HBaseConfiguration.create(CONF);
310     // Only retry once.
311     conf.setInt("hbase.hstore.flush.retries.number", 1);
312     final User user =
313       User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
314     // Inject our faulty LocalFileSystem
315     conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
316     user.runAs(new PrivilegedExceptionAction<Object>() {
317       @Override
318       public Object run() throws Exception {
319         // Make sure it worked (above is sensitive to caching details in hadoop core)
320         FileSystem fs = FileSystem.get(conf);
321         Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
322         FaultyFileSystem ffs = (FaultyFileSystem)fs;
323         HRegion region = null;
324         try {
325           // Initialize region
326           region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
327           long size = region.getMemstoreSize().get();
328           Assert.assertEquals(0, size);
329           // Put one item into memstore.  Measure the size of one item in memstore.
330           Put p1 = new Put(row);
331           p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
332           region.put(p1);
333           final long sizeOfOnePut = region.getMemstoreSize().get();
334           // Fail a flush which means the current memstore will hang out as memstore 'snapshot'.
335           try {
336             LOG.info("Flushing");
337             region.flushcache();
338             Assert.fail("Didn't bubble up IOE!");
339           } catch (DroppedSnapshotException dse) {
340             // What we are expecting
341           }
342           // Make it so all writes succeed from here on out
343           ffs.fault.set(false);
344           // Check sizes.  Should still be the one entry.
345           Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize().get());
346           // Now add two entries so that on this next flush that fails, we can see if we
347           // subtract the right amount, the snapshot size only.
348           Put p2 = new Put(row);
349           p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
350           p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
351           region.put(p2);
352           Assert.assertEquals(sizeOfOnePut * 3, region.getMemstoreSize().get());
353           // Do a successful flush.  It will clear the snapshot only.  Thats how flushes work.
354           // If already a snapshot, we clear it else we move the memstore to be snapshot and flush
355           // it
356           region.flushcache();
357           // Make sure our memory accounting is right.
358           Assert.assertEquals(sizeOfOnePut * 2, region.getMemstoreSize().get());
359         } finally {
360           HRegion.closeHRegion(region);
361         }
362         return null;
363       }
364     });
365     FileSystem.closeAllForUGI(user.getUGI());
366   }
367 
368   @Test
369   public void testCompactionAffectedByScanners() throws Exception {
370     byte[] family = Bytes.toBytes("family");
371     this.region = initHRegion(tableName, method, CONF, family);
372 
373     Put put = new Put(Bytes.toBytes("r1"));
374     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
375     region.put(put);
376     region.flushcache();
377 
378     Scan scan = new Scan();
379     scan.setMaxVersions(3);
380     // open the first scanner
381     RegionScanner scanner1 = region.getScanner(scan);
382 
383     Delete delete = new Delete(Bytes.toBytes("r1"));
384     region.delete(delete);
385     region.flushcache();
386 
387     // open the second scanner
388     RegionScanner scanner2 = region.getScanner(scan);
389 
390     List<Cell> results = new ArrayList<Cell>();
391 
392     System.out.println("Smallest read point:" + region.getSmallestReadPoint());
393 
394     // make a major compaction
395     region.compactStores(true);
396 
397     // open the third scanner
398     RegionScanner scanner3 = region.getScanner(scan);
399 
400     // get data from scanner 1, 2, 3 after major compaction
401     scanner1.next(results);
402     System.out.println(results);
403     assertEquals(1, results.size());
404 
405     results.clear();
406     scanner2.next(results);
407     System.out.println(results);
408     assertEquals(0, results.size());
409 
410     results.clear();
411     scanner3.next(results);
412     System.out.println(results);
413     assertEquals(0, results.size());
414   }
415 
416   @Test
417   public void testToShowNPEOnRegionScannerReseek() throws Exception {
418     byte[] family = Bytes.toBytes("family");
419     this.region = initHRegion(tableName, method, CONF, family);
420 
421     Put put = new Put(Bytes.toBytes("r1"));
422     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
423     region.put(put);
424     put = new Put(Bytes.toBytes("r2"));
425     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
426     region.put(put);
427     region.flushcache();
428 
429     Scan scan = new Scan();
430     scan.setMaxVersions(3);
431     // open the first scanner
432     RegionScanner scanner1 = region.getScanner(scan);
433 
434     System.out.println("Smallest read point:" + region.getSmallestReadPoint());
435 
436     region.compactStores(true);
437 
438     scanner1.reseek(Bytes.toBytes("r2"));
439     List<Cell> results = new ArrayList<Cell>();
440     scanner1.next(results);
441     Cell keyValue = results.get(0);
442     Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
443     scanner1.close();
444   }
445 
446   @Test
447   public void testSkipRecoveredEditsReplay() throws Exception {
448     String method = "testSkipRecoveredEditsReplay";
449     TableName tableName = TableName.valueOf(method);
450     byte[] family = Bytes.toBytes("family");
451     this.region = initHRegion(tableName, method, CONF, family);
452     try {
453       Path regiondir = region.getRegionFileSystem().getRegionDir();
454       FileSystem fs = region.getRegionFileSystem().getFileSystem();
455       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
456 
457       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
458 
459       long maxSeqId = 1050;
460       long minSeqId = 1000;
461 
462       for (long i = minSeqId; i <= maxSeqId; i += 10) {
463         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
464         fs.create(recoveredEdits);
465         HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
466 
467         long time = System.nanoTime();
468         WALEdit edit = new WALEdit();
469         edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
470             .toBytes(i)));
471         writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
472             HConstants.DEFAULT_CLUSTER_ID), edit));
473 
474         writer.close();
475       }
476       MonitoredTask status = TaskMonitor.get().createStatus(method);
477       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
478       for (Store store : region.getStores().values()) {
479         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
480       }
481       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
482       assertEquals(maxSeqId, seqId);
483       Get get = new Get(row);
484       Result result = region.get(get);
485       for (long i = minSeqId; i <= maxSeqId; i += 10) {
486         List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
487         assertEquals(1, kvs.size());
488         assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
489       }
490     } finally {
491       HRegion.closeHRegion(this.region);
492       this.region = null;
493     }
494   }
495 
496   @Test
497   public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
498     String method = "testSkipRecoveredEditsReplaySomeIgnored";
499     TableName tableName = TableName.valueOf(method);
500     byte[] family = Bytes.toBytes("family");
501     this.region = initHRegion(tableName, method, CONF, family);
502     try {
503       Path regiondir = region.getRegionFileSystem().getRegionDir();
504       FileSystem fs = region.getRegionFileSystem().getFileSystem();
505       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
506 
507       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
508 
509       long maxSeqId = 1050;
510       long minSeqId = 1000;
511 
512       for (long i = minSeqId; i <= maxSeqId; i += 10) {
513         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
514         fs.create(recoveredEdits);
515         HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
516 
517         long time = System.nanoTime();
518         WALEdit edit = new WALEdit();
519         edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
520             .toBytes(i)));
521         writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
522             HConstants.DEFAULT_CLUSTER_ID), edit));
523 
524         writer.close();
525       }
526       long recoverSeqId = 1030;
527       MonitoredTask status = TaskMonitor.get().createStatus(method);
528       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
529       for (Store store : region.getStores().values()) {
530         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
531       }
532       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
533       assertEquals(maxSeqId, seqId);
534       Get get = new Get(row);
535       Result result = region.get(get);
536       for (long i = minSeqId; i <= maxSeqId; i += 10) {
537         List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
538         if (i < recoverSeqId) {
539           assertEquals(0, kvs.size());
540         } else {
541           assertEquals(1, kvs.size());
542           assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
543         }
544       }
545     } finally {
546       HRegion.closeHRegion(this.region);
547       this.region = null;
548     }
549   }
550 
551   @Test
552   public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
553     byte[] family = Bytes.toBytes("family");
554     this.region = initHRegion(tableName, method, CONF, family);
555     try {
556       Path regiondir = region.getRegionFileSystem().getRegionDir();
557       FileSystem fs = region.getRegionFileSystem().getFileSystem();
558 
559       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
560       for (int i = 1000; i < 1050; i += 10) {
561         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
562         FSDataOutputStream dos = fs.create(recoveredEdits);
563         dos.writeInt(i);
564         dos.close();
565       }
566       long minSeqId = 2000;
567       Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
568       FSDataOutputStream dos = fs.create(recoveredEdits);
569       dos.close();
570 
571       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
572       for (Store store : region.getStores().values()) {
573         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
574       }
575       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
576       assertEquals(minSeqId, seqId);
577     } finally {
578       HRegion.closeHRegion(this.region);
579       this.region = null;
580     }
581   }
582 
583   @Test
584   public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
585     String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
586     TableName tableName = TableName.valueOf(method);
587     byte[] family = Bytes.toBytes("family");
588     this.region = initHRegion(tableName, method, CONF, family);
589     try {
590       Path regiondir = region.getRegionFileSystem().getRegionDir();
591       FileSystem fs = region.getRegionFileSystem().getFileSystem();
592       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
593 
594       assertEquals(0, region.getStoreFileList(
595         region.getStores().keySet().toArray(new byte[0][])).size());
596 
597       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
598 
599       long maxSeqId = 1050;
600       long minSeqId = 1000;
601 
602       for (long i = minSeqId; i <= maxSeqId; i += 10) {
603         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
604         fs.create(recoveredEdits);
605         HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
606 
607         long time = System.nanoTime();
608         WALEdit edit = null;
609         if (i == maxSeqId) {
610           edit = WALEdit.createCompaction(region.getRegionInfo(),
611           CompactionDescriptor.newBuilder()
612           .setTableName(ByteString.copyFrom(tableName.getName()))
613           .setFamilyName(ByteString.copyFrom(regionName))
614           .setEncodedRegionName(ByteString.copyFrom(regionName))
615           .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
616           .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName()))
617           .build());
618         } else {
619           edit = new WALEdit();
620           edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
621             .toBytes(i)));
622         }
623         writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
624             HConstants.DEFAULT_CLUSTER_ID), edit));
625         writer.close();
626       }
627 
628       long recoverSeqId = 1030;
629       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
630       MonitoredTask status = TaskMonitor.get().createStatus(method);
631       for (Store store : region.getStores().values()) {
632         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
633       }
634       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
635       assertEquals(maxSeqId, seqId);
636 
637       // assert that the files are flushed
638       assertEquals(1, region.getStoreFileList(
639         region.getStores().keySet().toArray(new byte[0][])).size());
640 
641     } finally {
642       HRegion.closeHRegion(this.region);
643       this.region = null;
644     }  }
645 
646   @Test
647   public void testRecoveredEditsReplayCompaction() throws Exception {
648     String method = name.getMethodName();
649     TableName tableName = TableName.valueOf(method);
650     byte[] family = Bytes.toBytes("family");
651     this.region = initHRegion(tableName, method, CONF, family);
652     try {
653       Path regiondir = region.getRegionFileSystem().getRegionDir();
654       FileSystem fs = region.getRegionFileSystem().getFileSystem();
655       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
656 
657       long maxSeqId = 3;
658       long minSeqId = 0;
659 
660       for (long i = minSeqId; i < maxSeqId; i++) {
661         Put put = new Put(Bytes.toBytes(i));
662         put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
663         region.put(put);
664         region.flushcache();
665       }
666 
667       // this will create a region with 3 files
668       assertEquals(3, region.getStore(family).getStorefilesCount());
669       List<Path> storeFiles = new ArrayList<Path>(3);
670       for (StoreFile sf : region.getStore(family).getStorefiles()) {
671         storeFiles.add(sf.getPath());
672       }
673 
674       // disable compaction completion
675       CONF.setBoolean("hbase.hstore.compaction.complete", false);
676       region.compactStores();
677 
678       // ensure that nothing changed
679       assertEquals(3, region.getStore(family).getStorefilesCount());
680 
681       // now find the compacted file, and manually add it to the recovered edits
682       Path tmpDir = region.getRegionFileSystem().getTempDir();
683       FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
684       String errorMsg = "Expected to find 1 file in the region temp directory "
685           + "from the compaction, could not find any";
686       assertNotNull(errorMsg, files);
687       assertEquals(errorMsg, 1, files.length);
688       // move the file inside region dir
689       Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
690           files[0].getPath());
691 
692       CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
693           .getRegionInfo(), family, storeFiles, Lists.newArrayList(newFile), region
694           .getRegionFileSystem().getStoreDir(Bytes.toString(family)));
695 
696       HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
697           this.region.getRegionInfo(), compactionDescriptor, new AtomicLong(1));
698 
699       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
700 
701       Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
702       fs.create(recoveredEdits);
703       HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
704 
705       long time = System.nanoTime();
706 
707       writer.append(new HLog.Entry(new HLogKey(regionName, tableName, 10, time,
708           HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(),
709           compactionDescriptor)));
710       writer.close();
711 
712       // close the region now, and reopen again
713       region.getTableDesc();
714       region.getRegionInfo();
715       region.close();
716       region = HRegion.openHRegion(region, null);
717 
718       // now check whether we have only one store file, the compacted one
719       Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
720       for (StoreFile sf : sfs) {
721         LOG.info(sf.getPath());
722       }
723       assertEquals(1, region.getStore(family).getStorefilesCount());
724       files = FSUtils.listStatus(fs, tmpDir);
725       assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
726 
727       for (long i = minSeqId; i < maxSeqId; i++) {
728         Get get = new Get(Bytes.toBytes(i));
729         Result result = region.get(get);
730         byte[] value = result.getValue(family, Bytes.toBytes(i));
731         assertArrayEquals(Bytes.toBytes(i), value);
732       }
733     } finally {
734       HRegion.closeHRegion(this.region);
735       this.region = null;
736     }
737   }
738 
739   @Test
740   public void testGetWhileRegionClose() throws IOException {
741     TableName tableName = TableName.valueOf(name.getMethodName());
742     Configuration hc = initSplit();
743     int numRows = 100;
744     byte[][] families = { fam1, fam2, fam3 };
745 
746     // Setting up region
747     String method = name.getMethodName();
748     this.region = initHRegion(tableName, method, hc, families);
749     try {
750       // Put data in region
751       final int startRow = 100;
752       putData(startRow, numRows, qual1, families);
753       putData(startRow, numRows, qual2, families);
754       putData(startRow, numRows, qual3, families);
755       final AtomicBoolean done = new AtomicBoolean(false);
756       final AtomicInteger gets = new AtomicInteger(0);
757       GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
758       try {
759         // Set ten threads running concurrently getting from the region.
760         for (int i = 0; i < threads.length / 2; i++) {
761           threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
762           threads[i].setDaemon(true);
763           threads[i].start();
764         }
765         // Artificially make the condition by setting closing flag explicitly.
766         // I can't make the issue happen with a call to region.close().
767         this.region.closing.set(true);
768         for (int i = threads.length / 2; i < threads.length; i++) {
769           threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
770           threads[i].setDaemon(true);
771           threads[i].start();
772         }
773       } finally {
774         if (this.region != null) {
775           HRegion.closeHRegion(this.region);
776         }
777       }
778       done.set(true);
779       for (GetTillDoneOrException t : threads) {
780         try {
781           t.join();
782         } catch (InterruptedException e) {
783           e.printStackTrace();
784         }
785         if (t.e != null) {
786           LOG.info("Exception=" + t.e);
787           assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
788         }
789       }
790     } finally {
791       HRegion.closeHRegion(this.region);
792       this.region = null;
793     }
794   }
795 
796   /*
797    * Thread that does get on single row until 'done' flag is flipped. If an
798    * exception causes us to fail, it records it.
799    */
800   class GetTillDoneOrException extends Thread {
801     private final Get g;
802     private final AtomicBoolean done;
803     private final AtomicInteger count;
804     private Exception e;
805 
806     GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
807       super("getter." + i);
808       this.g = new Get(r);
809       this.done = d;
810       this.count = c;
811     }
812 
813     @Override
814     public void run() {
815       while (!this.done.get()) {
816         try {
817           assertTrue(region.get(g).size() > 0);
818           this.count.incrementAndGet();
819         } catch (Exception e) {
820           this.e = e;
821           break;
822         }
823       }
824     }
825   }
826 
827   /*
828    * An involved filter test. Has multiple column families and deletes in mix.
829    */
830   @Test
831   public void testWeirdCacheBehaviour() throws Exception {
832     byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
833     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
834         Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
835     this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
836     try {
837       String value = "this is the value";
838       String value2 = "this is some other value";
839       String keyPrefix1 = "prefix1";
840       String keyPrefix2 = "prefix2";
841       String keyPrefix3 = "prefix3";
842       putRows(this.region, 3, value, keyPrefix1);
843       putRows(this.region, 3, value, keyPrefix2);
844       putRows(this.region, 3, value, keyPrefix3);
845       putRows(this.region, 3, value2, keyPrefix1);
846       putRows(this.region, 3, value2, keyPrefix2);
847       putRows(this.region, 3, value2, keyPrefix3);
848       System.out.println("Checking values for key: " + keyPrefix1);
849       assertEquals("Got back incorrect number of rows from scan", 3,
850           getNumberOfRows(keyPrefix1, value2, this.region));
851       System.out.println("Checking values for key: " + keyPrefix2);
852       assertEquals("Got back incorrect number of rows from scan", 3,
853           getNumberOfRows(keyPrefix2, value2, this.region));
854       System.out.println("Checking values for key: " + keyPrefix3);
855       assertEquals("Got back incorrect number of rows from scan", 3,
856           getNumberOfRows(keyPrefix3, value2, this.region));
857       deleteColumns(this.region, value2, keyPrefix1);
858       deleteColumns(this.region, value2, keyPrefix2);
859       deleteColumns(this.region, value2, keyPrefix3);
860       System.out.println("Starting important checks.....");
861       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
862           getNumberOfRows(keyPrefix1, value2, this.region));
863       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
864           getNumberOfRows(keyPrefix2, value2, this.region));
865       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
866           getNumberOfRows(keyPrefix3, value2, this.region));
867     } finally {
868       HRegion.closeHRegion(this.region);
869       this.region = null;
870     }
871   }
872 
873   @Test
874   public void testAppendWithReadOnlyTable() throws Exception {
875     byte[] TABLE = Bytes.toBytes("readOnlyTable");
876     this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
877     boolean exceptionCaught = false;
878     Append append = new Append(Bytes.toBytes("somerow"));
879     append.setDurability(Durability.SKIP_WAL);
880     append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
881         Bytes.toBytes("somevalue"));
882     try {
883       region.append(append);
884     } catch (IOException e) {
885       exceptionCaught = true;
886     } finally {
887       HRegion.closeHRegion(this.region);
888       this.region = null;
889     }
890     assertTrue(exceptionCaught == true);
891   }
892 
893   @Test
894   public void testIncrWithReadOnlyTable() throws Exception {
895     byte[] TABLE = Bytes.toBytes("readOnlyTable");
896     this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
897     boolean exceptionCaught = false;
898     Increment inc = new Increment(Bytes.toBytes("somerow"));
899     inc.setDurability(Durability.SKIP_WAL);
900     inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
901     try {
902       region.increment(inc);
903     } catch (IOException e) {
904       exceptionCaught = true;
905     } finally {
906       HRegion.closeHRegion(this.region);
907       this.region = null;
908     }
909     assertTrue(exceptionCaught == true);
910   }
911 
912   private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
913     InternalScanner scanner = buildScanner(keyPrefix, value, r);
914     int count = 0;
915     boolean more = false;
916     List<Cell> results = new ArrayList<Cell>();
917     do {
918       more = scanner.next(results);
919       if (results != null && !results.isEmpty())
920         count++;
921       else
922         break;
923       Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
924       delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
925       r.delete(delete);
926       results.clear();
927     } while (more);
928     assertEquals("Did not perform correct number of deletes", 3, count);
929   }
930 
931   private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
932     InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
933     int numberOfResults = 0;
934     List<Cell> results = new ArrayList<Cell>();
935     boolean more = false;
936     do {
937       more = resultScanner.next(results);
938       if (results != null && !results.isEmpty())
939         numberOfResults++;
940       else
941         break;
942       for (Cell kv : results) {
943         System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
944       }
945       results.clear();
946     } while (more);
947     return numberOfResults;
948   }
949 
950   private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
951       throws IOException {
952     // Defaults FilterList.Operator.MUST_PASS_ALL.
953     FilterList allFilters = new FilterList();
954     allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
955     // Only return rows where this column value exists in the row.
956     SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
957         Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
958     filter.setFilterIfMissing(true);
959     allFilters.addFilter(filter);
960     Scan scan = new Scan();
961     scan.addFamily(Bytes.toBytes("trans-blob"));
962     scan.addFamily(Bytes.toBytes("trans-type"));
963     scan.addFamily(Bytes.toBytes("trans-date"));
964     scan.addFamily(Bytes.toBytes("trans-tags"));
965     scan.addFamily(Bytes.toBytes("trans-group"));
966     scan.setFilter(allFilters);
967     return r.getScanner(scan);
968   }
969 
970   private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
971     for (int i = 0; i < numRows; i++) {
972       String row = key + "_" + i/* UUID.randomUUID().toString() */;
973       System.out.println(String.format("Saving row: %s, with value %s", row, value));
974       Put put = new Put(Bytes.toBytes(row));
975       put.setDurability(Durability.SKIP_WAL);
976       put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
977       put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
978       put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
979       put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
980       put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
981       r.put(put);
982     }
983   }
984 
985   @Test
986   public void testFamilyWithAndWithoutColon() throws Exception {
987     byte[] b = Bytes.toBytes(getName());
988     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
989     this.region = initHRegion(b, getName(), CONF, cf);
990     try {
991       Put p = new Put(b);
992       byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
993       p.add(cfwithcolon, cfwithcolon, cfwithcolon);
994       boolean exception = false;
995       try {
996         this.region.put(p);
997       } catch (NoSuchColumnFamilyException e) {
998         exception = true;
999       }
1000       assertTrue(exception);
1001     } finally {
1002       HRegion.closeHRegion(this.region);
1003       this.region = null;
1004     }
1005   }
1006 
1007   @Test
1008   public void testBatchPut() throws Exception {
1009     byte[] b = Bytes.toBytes(getName());
1010     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1011     byte[] qual = Bytes.toBytes("qual");
1012     byte[] val = Bytes.toBytes("val");
1013     this.region = initHRegion(b, getName(), CONF, cf);
1014     MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1015     try {
1016       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1017       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1018 
1019       LOG.info("First a batch put with all valid puts");
1020       final Put[] puts = new Put[10];
1021       for (int i = 0; i < 10; i++) {
1022         puts[i] = new Put(Bytes.toBytes("row_" + i));
1023         puts[i].add(cf, qual, val);
1024       }
1025 
1026       OperationStatus[] codes = this.region.batchMutate(puts);
1027       assertEquals(10, codes.length);
1028       for (int i = 0; i < 10; i++) {
1029         assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1030       }
1031       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1032 
1033       LOG.info("Next a batch put with one invalid family");
1034       puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1035       codes = this.region.batchMutate(puts);
1036       assertEquals(10, codes.length);
1037       for (int i = 0; i < 10; i++) {
1038         assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1039             codes[i].getOperationStatusCode());
1040       }
1041 
1042       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1043 
1044       LOG.info("Next a batch put that has to break into two batches to avoid a lock");
1045       RowLock rowLock = region.getRowLock(Bytes.toBytes("row_2"));
1046 
1047       MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1048       final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
1049       TestThread putter = new TestThread(ctx) {
1050         @Override
1051         public void doWork() throws IOException {
1052           retFromThread.set(region.batchMutate(puts));
1053         }
1054       };
1055       LOG.info("...starting put thread while holding lock");
1056       ctx.addThread(putter);
1057       ctx.startThreads();
1058 
1059       LOG.info("...waiting for put thread to sync first time");
1060       long startWait = System.currentTimeMillis();
1061       while (metricsAssertHelper.getCounter("syncTimeNumOps", source) == syncs + 2) {
1062         Thread.sleep(100);
1063         if (System.currentTimeMillis() - startWait > 10000) {
1064           fail("Timed out waiting for thread to sync first minibatch");
1065         }
1066       }
1067       LOG.info("...releasing row lock, which should let put thread continue");
1068       rowLock.release();
1069       LOG.info("...joining on thread");
1070       ctx.stop();
1071       LOG.info("...checking that next batch was synced");
1072       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 4, source);
1073       codes = retFromThread.get();
1074       for (int i = 0; i < 10; i++) {
1075         assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1076             codes[i].getOperationStatusCode());
1077       }
1078 
1079     } finally {
1080       HRegion.closeHRegion(this.region);
1081       this.region = null;
1082     }
1083   }
1084 
1085   @Test
1086   public void testBatchPutWithTsSlop() throws Exception {
1087     byte[] b = Bytes.toBytes(getName());
1088     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1089     byte[] qual = Bytes.toBytes("qual");
1090     byte[] val = Bytes.toBytes("val");
1091 
1092     // add data with a timestamp that is too recent for range. Ensure assert
1093     CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1094     this.region = initHRegion(b, getName(), CONF, cf);
1095 
1096     try {
1097       MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1098       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1099       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1100 
1101       final Put[] puts = new Put[10];
1102       for (int i = 0; i < 10; i++) {
1103         puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
1104         puts[i].add(cf, qual, val);
1105       }
1106 
1107       OperationStatus[] codes = this.region.batchMutate(puts);
1108       assertEquals(10, codes.length);
1109       for (int i = 0; i < 10; i++) {
1110         assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1111       }
1112       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1113 
1114     } finally {
1115       HRegion.closeHRegion(this.region);
1116       this.region = null;
1117     }
1118 
1119   }
1120 
1121   // ////////////////////////////////////////////////////////////////////////////
1122   // checkAndMutate tests
1123   // ////////////////////////////////////////////////////////////////////////////
1124   @Test
1125   public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1126     byte[] row1 = Bytes.toBytes("row1");
1127     byte[] fam1 = Bytes.toBytes("fam1");
1128     byte[] qf1 = Bytes.toBytes("qualifier");
1129     byte[] emptyVal = new byte[] {};
1130     byte[] val1 = Bytes.toBytes("value1");
1131     byte[] val2 = Bytes.toBytes("value2");
1132 
1133     // Setting up region
1134     String method = this.getName();
1135     this.region = initHRegion(tableName, method, CONF, fam1);
1136     try {
1137       // Putting empty data in key
1138       Put put = new Put(row1);
1139       put.add(fam1, qf1, emptyVal);
1140 
1141       // checkAndPut with empty value
1142       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1143           emptyVal), put, true);
1144       assertTrue(res);
1145 
1146       // Putting data in key
1147       put = new Put(row1);
1148       put.add(fam1, qf1, val1);
1149 
1150       // checkAndPut with correct value
1151       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1152           put, true);
1153       assertTrue(res);
1154 
1155       // not empty anymore
1156       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1157           put, true);
1158       assertFalse(res);
1159 
1160       Delete delete = new Delete(row1);
1161       delete.deleteColumn(fam1, qf1);
1162       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1163           delete, true);
1164       assertFalse(res);
1165 
1166       put = new Put(row1);
1167       put.add(fam1, qf1, val2);
1168       // checkAndPut with correct value
1169       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1170           put, true);
1171       assertTrue(res);
1172 
1173       // checkAndDelete with correct value
1174       delete = new Delete(row1);
1175       delete.deleteColumn(fam1, qf1);
1176       delete.deleteColumn(fam1, qf1);
1177       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1178           delete, true);
1179       assertTrue(res);
1180 
1181       delete = new Delete(row1);
1182       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1183           delete, true);
1184       assertTrue(res);
1185 
1186       // checkAndPut looking for a null value
1187       put = new Put(row1);
1188       put.add(fam1, qf1, val1);
1189 
1190       res = region
1191           .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
1192       assertTrue(res);
1193     } finally {
1194       HRegion.closeHRegion(this.region);
1195       this.region = null;
1196     }
1197   }
1198 
1199   @Test
1200   public void testCheckAndMutate_WithWrongValue() throws IOException {
1201     byte[] row1 = Bytes.toBytes("row1");
1202     byte[] fam1 = Bytes.toBytes("fam1");
1203     byte[] qf1 = Bytes.toBytes("qualifier");
1204     byte[] val1 = Bytes.toBytes("value1");
1205     byte[] val2 = Bytes.toBytes("value2");
1206 
1207     // Setting up region
1208     String method = this.getName();
1209     this.region = initHRegion(tableName, method, CONF, fam1);
1210     try {
1211       // Putting data in key
1212       Put put = new Put(row1);
1213       put.add(fam1, qf1, val1);
1214       region.put(put);
1215 
1216       // checkAndPut with wrong value
1217       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1218           val2), put, true);
1219       assertEquals(false, res);
1220 
1221       // checkAndDelete with wrong value
1222       Delete delete = new Delete(row1);
1223       delete.deleteFamily(fam1);
1224       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1225           put, true);
1226       assertEquals(false, res);
1227     } finally {
1228       HRegion.closeHRegion(this.region);
1229       this.region = null;
1230     }
1231   }
1232 
1233   @Test
1234   public void testCheckAndMutate_WithCorrectValue() throws IOException {
1235     byte[] row1 = Bytes.toBytes("row1");
1236     byte[] fam1 = Bytes.toBytes("fam1");
1237     byte[] qf1 = Bytes.toBytes("qualifier");
1238     byte[] val1 = Bytes.toBytes("value1");
1239 
1240     // Setting up region
1241     String method = this.getName();
1242     this.region = initHRegion(tableName, method, CONF, fam1);
1243     try {
1244       // Putting data in key
1245       Put put = new Put(row1);
1246       put.add(fam1, qf1, val1);
1247       region.put(put);
1248 
1249       // checkAndPut with correct value
1250       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1251           val1), put, true);
1252       assertEquals(true, res);
1253 
1254       // checkAndDelete with correct value
1255       Delete delete = new Delete(row1);
1256       delete.deleteColumn(fam1, qf1);
1257       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1258           delete, true);
1259       assertEquals(true, res);
1260     } finally {
1261       HRegion.closeHRegion(this.region);
1262       this.region = null;
1263     }
1264   }
1265 
1266   @Test
1267   public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1268     byte[] row1 = Bytes.toBytes("row1");
1269     byte[] fam1 = Bytes.toBytes("fam1");
1270     byte[] qf1 = Bytes.toBytes("qualifier");
1271     byte[] val1 = Bytes.toBytes("value1");
1272     byte[] val2 = Bytes.toBytes("value2");
1273     byte[] val3 = Bytes.toBytes("value3");
1274     byte[] val4 = Bytes.toBytes("value4");
1275 
1276     // Setting up region
1277     String method = this.getName();
1278     this.region = initHRegion(tableName, method, CONF, fam1);
1279     try {
1280       // Putting val3 in key
1281       Put put = new Put(row1);
1282       put.add(fam1, qf1, val3);
1283       region.put(put);
1284 
1285       // Test CompareOp.LESS: original = val3, compare with val3, fail
1286       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1287           new BinaryComparator(val3), put, true);
1288       assertEquals(false, res);
1289 
1290       // Test CompareOp.LESS: original = val3, compare with val4, fail
1291       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1292           new BinaryComparator(val4), put, true);
1293       assertEquals(false, res);
1294 
1295       // Test CompareOp.LESS: original = val3, compare with val2,
1296       // succeed (now value = val2)
1297       put = new Put(row1);
1298       put.add(fam1, qf1, val2);
1299       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1300           new BinaryComparator(val2), put, true);
1301       assertEquals(true, res);
1302 
1303       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail
1304       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1305           new BinaryComparator(val3), put, true);
1306       assertEquals(false, res);
1307 
1308       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2,
1309       // succeed (value still = val2)
1310       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1311           new BinaryComparator(val2), put, true);
1312       assertEquals(true, res);
1313 
1314       // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1,
1315       // succeed (now value = val3)
1316       put = new Put(row1);
1317       put.add(fam1, qf1, val3);
1318       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1319           new BinaryComparator(val1), put, true);
1320       assertEquals(true, res);
1321 
1322       // Test CompareOp.GREATER: original = val3, compare with val3, fail
1323       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1324           new BinaryComparator(val3), put, true);
1325       assertEquals(false, res);
1326 
1327       // Test CompareOp.GREATER: original = val3, compare with val2, fail
1328       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1329           new BinaryComparator(val2), put, true);
1330       assertEquals(false, res);
1331 
1332       // Test CompareOp.GREATER: original = val3, compare with val4,
1333       // succeed (now value = val2)
1334       put = new Put(row1);
1335       put.add(fam1, qf1, val2);
1336       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1337           new BinaryComparator(val4), put, true);
1338       assertEquals(true, res);
1339 
1340       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail
1341       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1342           new BinaryComparator(val1), put, true);
1343       assertEquals(false, res);
1344 
1345       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2,
1346       // succeed (value still = val2)
1347       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1348           new BinaryComparator(val2), put, true);
1349       assertEquals(true, res);
1350 
1351       // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed
1352       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1353           new BinaryComparator(val3), put, true);
1354       assertEquals(true, res);
1355     } finally {
1356       HRegion.closeHRegion(this.region);
1357       this.region = null;
1358     }
1359   }
1360 
1361   @Test
1362   public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1363     byte[] row1 = Bytes.toBytes("row1");
1364     byte[] fam1 = Bytes.toBytes("fam1");
1365     byte[] fam2 = Bytes.toBytes("fam2");
1366     byte[] qf1 = Bytes.toBytes("qualifier");
1367     byte[] val1 = Bytes.toBytes("value1");
1368     byte[] val2 = Bytes.toBytes("value2");
1369 
1370     byte[][] families = { fam1, fam2 };
1371 
1372     // Setting up region
1373     String method = this.getName();
1374     this.region = initHRegion(tableName, method, CONF, families);
1375     try {
1376       // Putting data in the key to check
1377       Put put = new Put(row1);
1378       put.add(fam1, qf1, val1);
1379       region.put(put);
1380 
1381       // Creating put to add
1382       long ts = System.currentTimeMillis();
1383       KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1384       put = new Put(row1);
1385       put.add(kv);
1386 
1387       // checkAndPut with wrong value
1388       HStore store = (HStore) region.getStore(fam1);
1389       store.memstore.kvset.size();
1390 
1391       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1392           val1), put, true);
1393       assertEquals(true, res);
1394       store.memstore.kvset.size();
1395 
1396       Get get = new Get(row1);
1397       get.addColumn(fam2, qf1);
1398       Cell[] actual = region.get(get).rawCells();
1399 
1400       Cell[] expected = { kv };
1401 
1402       assertEquals(expected.length, actual.length);
1403       for (int i = 0; i < actual.length; i++) {
1404         assertEquals(expected[i], actual[i]);
1405       }
1406     } finally {
1407       HRegion.closeHRegion(this.region);
1408       this.region = null;
1409     }
1410   }
1411 
1412   @Test
1413   public void testCheckAndPut_wrongRowInPut() throws IOException {
1414     TableName tableName = TableName.valueOf(name.getMethodName());
1415     this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
1416     try {
1417       Put put = new Put(row2);
1418       put.add(fam1, qual1, value1);
1419       try {
1420         region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1421             new BinaryComparator(value2), put, false);
1422         fail();
1423       } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1424         // expected exception.
1425       }
1426     } finally {
1427       HRegion.closeHRegion(this.region);
1428       this.region = null;
1429     }
1430   }
1431 
1432   @Test
1433   public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1434     byte[] row1 = Bytes.toBytes("row1");
1435     byte[] fam1 = Bytes.toBytes("fam1");
1436     byte[] fam2 = Bytes.toBytes("fam2");
1437     byte[] qf1 = Bytes.toBytes("qualifier1");
1438     byte[] qf2 = Bytes.toBytes("qualifier2");
1439     byte[] qf3 = Bytes.toBytes("qualifier3");
1440     byte[] val1 = Bytes.toBytes("value1");
1441     byte[] val2 = Bytes.toBytes("value2");
1442     byte[] val3 = Bytes.toBytes("value3");
1443     byte[] emptyVal = new byte[] {};
1444 
1445     byte[][] families = { fam1, fam2 };
1446 
1447     // Setting up region
1448     String method = this.getName();
1449     this.region = initHRegion(tableName, method, CONF, families);
1450     try {
1451       // Put content
1452       Put put = new Put(row1);
1453       put.add(fam1, qf1, val1);
1454       region.put(put);
1455       Threads.sleep(2);
1456 
1457       put = new Put(row1);
1458       put.add(fam1, qf1, val2);
1459       put.add(fam2, qf1, val3);
1460       put.add(fam2, qf2, val2);
1461       put.add(fam2, qf3, val1);
1462       put.add(fam1, qf3, val1);
1463       region.put(put);
1464 
1465       // Multi-column delete
1466       Delete delete = new Delete(row1);
1467       delete.deleteColumn(fam1, qf1);
1468       delete.deleteColumn(fam2, qf1);
1469       delete.deleteColumn(fam1, qf3);
1470       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1471           val2), delete, true);
1472       assertEquals(true, res);
1473 
1474       Get get = new Get(row1);
1475       get.addColumn(fam1, qf1);
1476       get.addColumn(fam1, qf3);
1477       get.addColumn(fam2, qf2);
1478       Result r = region.get(get);
1479       assertEquals(2, r.size());
1480       assertArrayEquals(val1, r.getValue(fam1, qf1));
1481       assertArrayEquals(val2, r.getValue(fam2, qf2));
1482 
1483       // Family delete
1484       delete = new Delete(row1);
1485       delete.deleteFamily(fam2);
1486       res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1487           delete, true);
1488       assertEquals(true, res);
1489 
1490       get = new Get(row1);
1491       r = region.get(get);
1492       assertEquals(1, r.size());
1493       assertArrayEquals(val1, r.getValue(fam1, qf1));
1494 
1495       // Row delete
1496       delete = new Delete(row1);
1497       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1498           delete, true);
1499       assertEquals(true, res);
1500       get = new Get(row1);
1501       r = region.get(get);
1502       assertEquals(0, r.size());
1503     } finally {
1504       HRegion.closeHRegion(this.region);
1505       this.region = null;
1506     }
1507   }
1508 
1509   // ////////////////////////////////////////////////////////////////////////////
1510   // Delete tests
1511   // ////////////////////////////////////////////////////////////////////////////
1512   @Test
1513   public void testDelete_multiDeleteColumn() throws IOException {
1514     byte[] row1 = Bytes.toBytes("row1");
1515     byte[] fam1 = Bytes.toBytes("fam1");
1516     byte[] qual = Bytes.toBytes("qualifier");
1517     byte[] value = Bytes.toBytes("value");
1518 
1519     Put put = new Put(row1);
1520     put.add(fam1, qual, 1, value);
1521     put.add(fam1, qual, 2, value);
1522 
1523     String method = this.getName();
1524     this.region = initHRegion(tableName, method, CONF, fam1);
1525     try {
1526       region.put(put);
1527 
1528       // We do support deleting more than 1 'latest' version
1529       Delete delete = new Delete(row1);
1530       delete.deleteColumn(fam1, qual);
1531       delete.deleteColumn(fam1, qual);
1532       region.delete(delete);
1533 
1534       Get get = new Get(row1);
1535       get.addFamily(fam1);
1536       Result r = region.get(get);
1537       assertEquals(0, r.size());
1538     } finally {
1539       HRegion.closeHRegion(this.region);
1540       this.region = null;
1541     }
1542   }
1543 
1544   @Test
1545   public void testDelete_CheckFamily() throws IOException {
1546     byte[] row1 = Bytes.toBytes("row1");
1547     byte[] fam1 = Bytes.toBytes("fam1");
1548     byte[] fam2 = Bytes.toBytes("fam2");
1549     byte[] fam3 = Bytes.toBytes("fam3");
1550     byte[] fam4 = Bytes.toBytes("fam4");
1551 
1552     // Setting up region
1553     String method = this.getName();
1554     this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
1555     try {
1556       List<Cell> kvs = new ArrayList<Cell>();
1557       kvs.add(new KeyValue(row1, fam4, null, null));
1558 
1559       // testing existing family
1560       byte[] family = fam2;
1561       try {
1562         NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1563             Bytes.BYTES_COMPARATOR);
1564         deleteMap.put(family, kvs);
1565         region.delete(deleteMap, Durability.SYNC_WAL);
1566       } catch (Exception e) {
1567         assertTrue("Family " + new String(family) + " does not exist", false);
1568       }
1569 
1570       // testing non existing family
1571       boolean ok = false;
1572       family = fam4;
1573       try {
1574         NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1575             Bytes.BYTES_COMPARATOR);
1576         deleteMap.put(family, kvs);
1577         region.delete(deleteMap, Durability.SYNC_WAL);
1578       } catch (Exception e) {
1579         ok = true;
1580       }
1581       assertEquals("Family " + new String(family) + " does exist", true, ok);
1582     } finally {
1583       HRegion.closeHRegion(this.region);
1584       this.region = null;
1585     }
1586   }
1587 
1588   @Test
1589   public void testDelete_mixed() throws IOException, InterruptedException {
1590     byte[] fam = Bytes.toBytes("info");
1591     byte[][] families = { fam };
1592     String method = this.getName();
1593     this.region = initHRegion(tableName, method, CONF, families);
1594     try {
1595       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1596 
1597       byte[] row = Bytes.toBytes("table_name");
1598       // column names
1599       byte[] serverinfo = Bytes.toBytes("serverinfo");
1600       byte[] splitA = Bytes.toBytes("splitA");
1601       byte[] splitB = Bytes.toBytes("splitB");
1602 
1603       // add some data:
1604       Put put = new Put(row);
1605       put.add(fam, splitA, Bytes.toBytes("reference_A"));
1606       region.put(put);
1607 
1608       put = new Put(row);
1609       put.add(fam, splitB, Bytes.toBytes("reference_B"));
1610       region.put(put);
1611 
1612       put = new Put(row);
1613       put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1614       region.put(put);
1615 
1616       // ok now delete a split:
1617       Delete delete = new Delete(row);
1618       delete.deleteColumns(fam, splitA);
1619       region.delete(delete);
1620 
1621       // assert some things:
1622       Get get = new Get(row).addColumn(fam, serverinfo);
1623       Result result = region.get(get);
1624       assertEquals(1, result.size());
1625 
1626       get = new Get(row).addColumn(fam, splitA);
1627       result = region.get(get);
1628       assertEquals(0, result.size());
1629 
1630       get = new Get(row).addColumn(fam, splitB);
1631       result = region.get(get);
1632       assertEquals(1, result.size());
1633 
1634       // Assert that after a delete, I can put.
1635       put = new Put(row);
1636       put.add(fam, splitA, Bytes.toBytes("reference_A"));
1637       region.put(put);
1638       get = new Get(row);
1639       result = region.get(get);
1640       assertEquals(3, result.size());
1641 
1642       // Now delete all... then test I can add stuff back
1643       delete = new Delete(row);
1644       region.delete(delete);
1645       assertEquals(0, region.get(get).size());
1646 
1647       region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
1648       result = region.get(get);
1649       assertEquals(1, result.size());
1650     } finally {
1651       HRegion.closeHRegion(this.region);
1652       this.region = null;
1653     }
1654   }
1655 
1656   @Test
1657   public void testDeleteRowWithFutureTs() throws IOException {
1658     byte[] fam = Bytes.toBytes("info");
1659     byte[][] families = { fam };
1660     String method = this.getName();
1661     this.region = initHRegion(tableName, method, CONF, families);
1662     try {
1663       byte[] row = Bytes.toBytes("table_name");
1664       // column names
1665       byte[] serverinfo = Bytes.toBytes("serverinfo");
1666 
1667       // add data in the far future
1668       Put put = new Put(row);
1669       put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
1670       region.put(put);
1671 
1672       // now delete something in the present
1673       Delete delete = new Delete(row);
1674       region.delete(delete);
1675 
1676       // make sure we still see our data
1677       Get get = new Get(row).addColumn(fam, serverinfo);
1678       Result result = region.get(get);
1679       assertEquals(1, result.size());
1680 
1681       // delete the future row
1682       delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
1683       region.delete(delete);
1684 
1685       // make sure it is gone
1686       get = new Get(row).addColumn(fam, serverinfo);
1687       result = region.get(get);
1688       assertEquals(0, result.size());
1689     } finally {
1690       HRegion.closeHRegion(this.region);
1691       this.region = null;
1692     }
1693   }
1694 
1695   /**
1696    * Tests that the special LATEST_TIMESTAMP option for puts gets replaced by
1697    * the actual timestamp
1698    */
1699   @Test
1700   public void testPutWithLatestTS() throws IOException {
1701     byte[] fam = Bytes.toBytes("info");
1702     byte[][] families = { fam };
1703     String method = this.getName();
1704     this.region = initHRegion(tableName, method, CONF, families);
1705     try {
1706       byte[] row = Bytes.toBytes("row1");
1707       // column names
1708       byte[] qual = Bytes.toBytes("qual");
1709 
1710       // add data with LATEST_TIMESTAMP, put without WAL
1711       Put put = new Put(row);
1712       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1713       region.put(put);
1714 
1715       // Make sure it shows up with an actual timestamp
1716       Get get = new Get(row).addColumn(fam, qual);
1717       Result result = region.get(get);
1718       assertEquals(1, result.size());
1719       Cell kv = result.rawCells()[0];
1720       LOG.info("Got: " + kv);
1721       assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1722           kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1723 
1724       // Check same with WAL enabled (historically these took different
1725       // code paths, so check both)
1726       row = Bytes.toBytes("row2");
1727       put = new Put(row);
1728       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1729       region.put(put);
1730 
1731       // Make sure it shows up with an actual timestamp
1732       get = new Get(row).addColumn(fam, qual);
1733       result = region.get(get);
1734       assertEquals(1, result.size());
1735       kv = result.rawCells()[0];
1736       LOG.info("Got: " + kv);
1737       assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1738           kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1739     } finally {
1740       HRegion.closeHRegion(this.region);
1741       this.region = null;
1742     }
1743 
1744   }
1745 
1746   /**
1747    * Tests that there is server-side filtering for invalid timestamp upper
1748    * bound. Note that the timestamp lower bound is automatically handled for us
1749    * by the TTL field.
1750    */
1751   @Test
1752   public void testPutWithTsSlop() throws IOException {
1753     byte[] fam = Bytes.toBytes("info");
1754     byte[][] families = { fam };
1755     String method = this.getName();
1756 
1757     // add data with a timestamp that is too recent for range. Ensure assert
1758     CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1759     this.region = initHRegion(tableName, method, CONF, families);
1760     boolean caughtExcep = false;
1761     try {
1762       try {
1763         // no TS specified == use latest. should not error
1764         region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
1765         // TS out of range. should error
1766         region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
1767             Bytes.toBytes("value")));
1768         fail("Expected IOE for TS out of configured timerange");
1769       } catch (FailedSanityCheckException ioe) {
1770         LOG.debug("Received expected exception", ioe);
1771         caughtExcep = true;
1772       }
1773       assertTrue("Should catch FailedSanityCheckException", caughtExcep);
1774     } finally {
1775       HRegion.closeHRegion(this.region);
1776       this.region = null;
1777     }
1778   }
1779 
1780   @Test
1781   public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
1782     byte[] fam1 = Bytes.toBytes("columnA");
1783     byte[] fam2 = Bytes.toBytes("columnB");
1784     this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
1785     try {
1786       byte[] rowA = Bytes.toBytes("rowA");
1787       byte[] rowB = Bytes.toBytes("rowB");
1788 
1789       byte[] value = Bytes.toBytes("value");
1790 
1791       Delete delete = new Delete(rowA);
1792       delete.deleteFamily(fam1);
1793 
1794       region.delete(delete);
1795 
1796       // now create data.
1797       Put put = new Put(rowA);
1798       put.add(fam2, null, value);
1799       region.put(put);
1800 
1801       put = new Put(rowB);
1802       put.add(fam1, null, value);
1803       put.add(fam2, null, value);
1804       region.put(put);
1805 
1806       Scan scan = new Scan();
1807       scan.addFamily(fam1).addFamily(fam2);
1808       InternalScanner s = region.getScanner(scan);
1809       List<Cell> results = new ArrayList<Cell>();
1810       s.next(results);
1811       assertTrue(CellUtil.matchingRow(results.get(0), rowA));
1812 
1813       results.clear();
1814       s.next(results);
1815       assertTrue(CellUtil.matchingRow(results.get(0), rowB));
1816     } finally {
1817       HRegion.closeHRegion(this.region);
1818       this.region = null;
1819     }
1820   }
1821 
1822   @Test
1823   public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
1824     Delete delete = new Delete(row);
1825     delete.deleteColumns(fam1, qual1);
1826     doTestDelete_AndPostInsert(delete);
1827   }
1828 
1829   @Test
1830   public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
1831     Delete delete = new Delete(row);
1832     delete.deleteFamily(fam1);
1833     doTestDelete_AndPostInsert(delete);
1834   }
1835 
1836   public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
1837     TableName tableName = TableName.valueOf(name.getMethodName());
1838     this.region = initHRegion(tableName, getName(), CONF, fam1);
1839     try {
1840       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1841       Put put = new Put(row);
1842       put.add(fam1, qual1, value1);
1843       region.put(put);
1844 
1845       // now delete the value:
1846       region.delete(delete);
1847 
1848       // ok put data:
1849       put = new Put(row);
1850       put.add(fam1, qual1, value2);
1851       region.put(put);
1852 
1853       // ok get:
1854       Get get = new Get(row);
1855       get.addColumn(fam1, qual1);
1856 
1857       Result r = region.get(get);
1858       assertEquals(1, r.size());
1859       assertArrayEquals(value2, r.getValue(fam1, qual1));
1860 
1861       // next:
1862       Scan scan = new Scan(row);
1863       scan.addColumn(fam1, qual1);
1864       InternalScanner s = region.getScanner(scan);
1865 
1866       List<Cell> results = new ArrayList<Cell>();
1867       assertEquals(false, s.next(results));
1868       assertEquals(1, results.size());
1869       Cell kv = results.get(0);
1870 
1871       assertArrayEquals(value2, CellUtil.cloneValue(kv));
1872       assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
1873       assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
1874       assertArrayEquals(row, CellUtil.cloneRow(kv));
1875     } finally {
1876       HRegion.closeHRegion(this.region);
1877       this.region = null;
1878     }
1879   }
1880 
1881   @Test
1882   public void testDelete_CheckTimestampUpdated() throws IOException {
1883     TableName tableName = TableName.valueOf(name.getMethodName());
1884     byte[] row1 = Bytes.toBytes("row1");
1885     byte[] col1 = Bytes.toBytes("col1");
1886     byte[] col2 = Bytes.toBytes("col2");
1887     byte[] col3 = Bytes.toBytes("col3");
1888 
1889     // Setting up region
1890     String method = this.getName();
1891     this.region = initHRegion(tableName, method, CONF, fam1);
1892     try {
1893       // Building checkerList
1894       List<Cell> kvs = new ArrayList<Cell>();
1895       kvs.add(new KeyValue(row1, fam1, col1, null));
1896       kvs.add(new KeyValue(row1, fam1, col2, null));
1897       kvs.add(new KeyValue(row1, fam1, col3, null));
1898 
1899       NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1900           Bytes.BYTES_COMPARATOR);
1901       deleteMap.put(fam1, kvs);
1902       region.delete(deleteMap, Durability.SYNC_WAL);
1903 
1904       // extract the key values out the memstore:
1905       // This is kinda hacky, but better than nothing...
1906       long now = System.currentTimeMillis();
1907       KeyValue firstKv = ((HStore) region.getStore(fam1)).memstore.kvset.first();
1908       assertTrue(firstKv.getTimestamp() <= now);
1909       now = firstKv.getTimestamp();
1910       for (KeyValue kv : ((HStore) region.getStore(fam1)).memstore.kvset) {
1911         assertTrue(kv.getTimestamp() <= now);
1912         now = kv.getTimestamp();
1913       }
1914     } finally {
1915       HRegion.closeHRegion(this.region);
1916       this.region = null;
1917     }
1918   }
1919 
1920   // ////////////////////////////////////////////////////////////////////////////
1921   // Get tests
1922   // ////////////////////////////////////////////////////////////////////////////
1923   @Test
1924   public void testGet_FamilyChecker() throws IOException {
1925     byte[] row1 = Bytes.toBytes("row1");
1926     byte[] fam1 = Bytes.toBytes("fam1");
1927     byte[] fam2 = Bytes.toBytes("False");
1928     byte[] col1 = Bytes.toBytes("col1");
1929 
1930     // Setting up region
1931     String method = this.getName();
1932     this.region = initHRegion(tableName, method, CONF, fam1);
1933     try {
1934       Get get = new Get(row1);
1935       get.addColumn(fam2, col1);
1936 
1937       // Test
1938       try {
1939         region.get(get);
1940       } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
1941         assertFalse(false);
1942         return;
1943       }
1944       assertFalse(true);
1945     } finally {
1946       HRegion.closeHRegion(this.region);
1947       this.region = null;
1948     }
1949   }
1950 
1951   @Test
1952   public void testGet_Basic() throws IOException {
1953     byte[] row1 = Bytes.toBytes("row1");
1954     byte[] fam1 = Bytes.toBytes("fam1");
1955     byte[] col1 = Bytes.toBytes("col1");
1956     byte[] col2 = Bytes.toBytes("col2");
1957     byte[] col3 = Bytes.toBytes("col3");
1958     byte[] col4 = Bytes.toBytes("col4");
1959     byte[] col5 = Bytes.toBytes("col5");
1960 
1961     // Setting up region
1962     String method = this.getName();
1963     this.region = initHRegion(tableName, method, CONF, fam1);
1964     try {
1965       // Add to memstore
1966       Put put = new Put(row1);
1967       put.add(fam1, col1, null);
1968       put.add(fam1, col2, null);
1969       put.add(fam1, col3, null);
1970       put.add(fam1, col4, null);
1971       put.add(fam1, col5, null);
1972       region.put(put);
1973 
1974       Get get = new Get(row1);
1975       get.addColumn(fam1, col2);
1976       get.addColumn(fam1, col4);
1977       // Expected result
1978       KeyValue kv1 = new KeyValue(row1, fam1, col2);
1979       KeyValue kv2 = new KeyValue(row1, fam1, col4);
1980       KeyValue[] expected = { kv1, kv2 };
1981 
1982       // Test
1983       Result res = region.get(get);
1984       assertEquals(expected.length, res.size());
1985       for (int i = 0; i < res.size(); i++) {
1986         assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
1987         assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
1988         assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
1989       }
1990 
1991       // Test using a filter on a Get
1992       Get g = new Get(row1);
1993       final int count = 2;
1994       g.setFilter(new ColumnCountGetFilter(count));
1995       res = region.get(g);
1996       assertEquals(count, res.size());
1997     } finally {
1998       HRegion.closeHRegion(this.region);
1999       this.region = null;
2000     }
2001   }
2002 
2003   @Test
2004   public void testGet_Empty() throws IOException {
2005     byte[] row = Bytes.toBytes("row");
2006     byte[] fam = Bytes.toBytes("fam");
2007 
2008     String method = this.getName();
2009     this.region = initHRegion(tableName, method, CONF, fam);
2010     try {
2011       Get get = new Get(row);
2012       get.addFamily(fam);
2013       Result r = region.get(get);
2014 
2015       assertTrue(r.isEmpty());
2016     } finally {
2017       HRegion.closeHRegion(this.region);
2018       this.region = null;
2019     }
2020   }
2021 
2022   // ////////////////////////////////////////////////////////////////////////////
2023   // Merge test
2024   // ////////////////////////////////////////////////////////////////////////////
2025   @Test
2026   public void testMerge() throws IOException {
2027     byte[][] families = { fam1, fam2, fam3 };
2028     Configuration hc = initSplit();
2029     // Setting up region
2030     String method = this.getName();
2031     this.region = initHRegion(tableName, method, hc, families);
2032     try {
2033       LOG.info("" + HBaseTestCase.addContent(region, fam3));
2034       region.flushcache();
2035       region.compactStores();
2036       byte[] splitRow = region.checkSplit();
2037       assertNotNull(splitRow);
2038       LOG.info("SplitRow: " + Bytes.toString(splitRow));
2039       HRegion[] subregions = splitRegion(region, splitRow);
2040       try {
2041         // Need to open the regions.
2042         for (int i = 0; i < subregions.length; i++) {
2043           HRegion.openHRegion(subregions[i], null);
2044           subregions[i].compactStores();
2045         }
2046         Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
2047         Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
2048         Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
2049         long startTime = System.currentTimeMillis();
2050         region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
2051         LOG.info("Merge regions elapsed time: "
2052             + ((System.currentTimeMillis() - startTime) / 1000.0));
2053         FILESYSTEM.delete(oldRegion1, true);
2054         FILESYSTEM.delete(oldRegion2, true);
2055         FILESYSTEM.delete(oldRegionPath, true);
2056         LOG.info("splitAndMerge completed.");
2057       } finally {
2058         for (int i = 0; i < subregions.length; i++) {
2059           try {
2060             HRegion.closeHRegion(subregions[i]);
2061           } catch (IOException e) {
2062             // Ignore.
2063           }
2064         }
2065       }
2066     } finally {
2067       HRegion.closeHRegion(this.region);
2068       this.region = null;
2069     }
2070   }
2071 
2072   /**
2073    * @param parent
2074    *          Region to split.
2075    * @param midkey
2076    *          Key to split around.
2077    * @return The Regions we created.
2078    * @throws IOException
2079    */
2080   HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
2081     PairOfSameType<HRegion> result = null;
2082     SplitTransaction st = new SplitTransaction(parent, midkey);
2083     // If prepare does not return true, for some reason -- logged inside in
2084     // the prepare call -- we are not ready to split just now. Just return.
2085     if (!st.prepare()) {
2086       parent.clearSplit();
2087       return null;
2088     }
2089     try {
2090       result = st.execute(null, null);
2091     } catch (IOException ioe) {
2092       try {
2093         LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; "
2094             + ioe.getMessage());
2095         st.rollback(null, null);
2096         LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString());
2097         return null;
2098       } catch (RuntimeException e) {
2099         // If failed rollback, kill this server to avoid having a hole in table.
2100         LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString()
2101             + " -- aborting server", e);
2102       }
2103     }
2104     finally {
2105       parent.clearSplit();
2106     }
2107     return new HRegion[] { result.getFirst(), result.getSecond() };
2108   }
2109 
2110   // ////////////////////////////////////////////////////////////////////////////
2111   // Scanner tests
2112   // ////////////////////////////////////////////////////////////////////////////
2113   @Test
2114   public void testGetScanner_WithOkFamilies() throws IOException {
2115     byte[] fam1 = Bytes.toBytes("fam1");
2116     byte[] fam2 = Bytes.toBytes("fam2");
2117 
2118     byte[][] families = { fam1, fam2 };
2119 
2120     // Setting up region
2121     String method = this.getName();
2122     this.region = initHRegion(tableName, method, CONF, families);
2123     try {
2124       Scan scan = new Scan();
2125       scan.addFamily(fam1);
2126       scan.addFamily(fam2);
2127       try {
2128         region.getScanner(scan);
2129       } catch (Exception e) {
2130         assertTrue("Families could not be found in Region", false);
2131       }
2132     } finally {
2133       HRegion.closeHRegion(this.region);
2134       this.region = null;
2135     }
2136   }
2137 
2138   @Test
2139   public void testGetScanner_WithNotOkFamilies() throws IOException {
2140     byte[] fam1 = Bytes.toBytes("fam1");
2141     byte[] fam2 = Bytes.toBytes("fam2");
2142 
2143     byte[][] families = { fam1 };
2144 
2145     // Setting up region
2146     String method = this.getName();
2147     this.region = initHRegion(tableName, method, CONF, families);
2148     try {
2149       Scan scan = new Scan();
2150       scan.addFamily(fam2);
2151       boolean ok = false;
2152       try {
2153         region.getScanner(scan);
2154       } catch (Exception e) {
2155         ok = true;
2156       }
2157       assertTrue("Families could not be found in Region", ok);
2158     } finally {
2159       HRegion.closeHRegion(this.region);
2160       this.region = null;
2161     }
2162   }
2163 
2164   @Test
2165   public void testGetScanner_WithNoFamilies() throws IOException {
2166     byte[] row1 = Bytes.toBytes("row1");
2167     byte[] fam1 = Bytes.toBytes("fam1");
2168     byte[] fam2 = Bytes.toBytes("fam2");
2169     byte[] fam3 = Bytes.toBytes("fam3");
2170     byte[] fam4 = Bytes.toBytes("fam4");
2171 
2172     byte[][] families = { fam1, fam2, fam3, fam4 };
2173 
2174     // Setting up region
2175     String method = this.getName();
2176     this.region = initHRegion(tableName, method, CONF, families);
2177     try {
2178 
2179       // Putting data in Region
2180       Put put = new Put(row1);
2181       put.add(fam1, null, null);
2182       put.add(fam2, null, null);
2183       put.add(fam3, null, null);
2184       put.add(fam4, null, null);
2185       region.put(put);
2186 
2187       Scan scan = null;
2188       HRegion.RegionScannerImpl is = null;
2189 
2190       // Testing to see how many scanners that is produced by getScanner,
2191       // starting
2192       // with known number, 2 - current = 1
2193       scan = new Scan();
2194       scan.addFamily(fam2);
2195       scan.addFamily(fam4);
2196       is = (RegionScannerImpl) region.getScanner(scan);
2197       assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2198 
2199       scan = new Scan();
2200       is = (RegionScannerImpl) region.getScanner(scan);
2201       assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2202     } finally {
2203       HRegion.closeHRegion(this.region);
2204       this.region = null;
2205     }
2206   }
2207 
2208   /**
2209    * This method tests https://issues.apache.org/jira/browse/HBASE-2516.
2210    * 
2211    * @throws IOException
2212    */
2213   @Test
2214   public void testGetScanner_WithRegionClosed() throws IOException {
2215     byte[] fam1 = Bytes.toBytes("fam1");
2216     byte[] fam2 = Bytes.toBytes("fam2");
2217 
2218     byte[][] families = { fam1, fam2 };
2219 
2220     // Setting up region
2221     String method = this.getName();
2222     try {
2223       this.region = initHRegion(tableName, method, CONF, families);
2224     } catch (IOException e) {
2225       e.printStackTrace();
2226       fail("Got IOException during initHRegion, " + e.getMessage());
2227     }
2228     try {
2229       region.closed.set(true);
2230       try {
2231         region.getScanner(null);
2232         fail("Expected to get an exception during getScanner on a region that is closed");
2233       } catch (NotServingRegionException e) {
2234         // this is the correct exception that is expected
2235       } catch (IOException e) {
2236         fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2237             + e.getMessage());
2238       }
2239     } finally {
2240       HRegion.closeHRegion(this.region);
2241       this.region = null;
2242     }
2243   }
2244 
2245   @Test
2246   public void testRegionScanner_Next() throws IOException {
2247     byte[] row1 = Bytes.toBytes("row1");
2248     byte[] row2 = Bytes.toBytes("row2");
2249     byte[] fam1 = Bytes.toBytes("fam1");
2250     byte[] fam2 = Bytes.toBytes("fam2");
2251     byte[] fam3 = Bytes.toBytes("fam3");
2252     byte[] fam4 = Bytes.toBytes("fam4");
2253 
2254     byte[][] families = { fam1, fam2, fam3, fam4 };
2255     long ts = System.currentTimeMillis();
2256 
2257     // Setting up region
2258     String method = this.getName();
2259     this.region = initHRegion(tableName, method, CONF, families);
2260     try {
2261       // Putting data in Region
2262       Put put = null;
2263       put = new Put(row1);
2264       put.add(fam1, (byte[]) null, ts, null);
2265       put.add(fam2, (byte[]) null, ts, null);
2266       put.add(fam3, (byte[]) null, ts, null);
2267       put.add(fam4, (byte[]) null, ts, null);
2268       region.put(put);
2269 
2270       put = new Put(row2);
2271       put.add(fam1, (byte[]) null, ts, null);
2272       put.add(fam2, (byte[]) null, ts, null);
2273       put.add(fam3, (byte[]) null, ts, null);
2274       put.add(fam4, (byte[]) null, ts, null);
2275       region.put(put);
2276 
2277       Scan scan = new Scan();
2278       scan.addFamily(fam2);
2279       scan.addFamily(fam4);
2280       InternalScanner is = region.getScanner(scan);
2281 
2282       List<Cell> res = null;
2283 
2284       // Result 1
2285       List<Cell> expected1 = new ArrayList<Cell>();
2286       expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2287       expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2288 
2289       res = new ArrayList<Cell>();
2290       is.next(res);
2291       for (int i = 0; i < res.size(); i++) {
2292         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2293       }
2294 
2295       // Result 2
2296       List<Cell> expected2 = new ArrayList<Cell>();
2297       expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2298       expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2299 
2300       res = new ArrayList<Cell>();
2301       is.next(res);
2302       for (int i = 0; i < res.size(); i++) {
2303         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2304       }
2305     } finally {
2306       HRegion.closeHRegion(this.region);
2307       this.region = null;
2308     }
2309   }
2310 
2311   @Test
2312   public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2313     byte[] row1 = Bytes.toBytes("row1");
2314     byte[] qf1 = Bytes.toBytes("qualifier1");
2315     byte[] qf2 = Bytes.toBytes("qualifier2");
2316     byte[] fam1 = Bytes.toBytes("fam1");
2317     byte[][] families = { fam1 };
2318 
2319     long ts1 = System.currentTimeMillis();
2320     long ts2 = ts1 + 1;
2321     long ts3 = ts1 + 2;
2322 
2323     // Setting up region
2324     String method = this.getName();
2325     this.region = initHRegion(tableName, method, CONF, families);
2326     try {
2327       // Putting data in Region
2328       Put put = null;
2329       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2330       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2331       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2332 
2333       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2334       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2335       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2336 
2337       put = new Put(row1);
2338       put.add(kv13);
2339       put.add(kv12);
2340       put.add(kv11);
2341       put.add(kv23);
2342       put.add(kv22);
2343       put.add(kv21);
2344       region.put(put);
2345 
2346       // Expected
2347       List<Cell> expected = new ArrayList<Cell>();
2348       expected.add(kv13);
2349       expected.add(kv12);
2350 
2351       Scan scan = new Scan(row1);
2352       scan.addColumn(fam1, qf1);
2353       scan.setMaxVersions(MAX_VERSIONS);
2354       List<Cell> actual = new ArrayList<Cell>();
2355       InternalScanner scanner = region.getScanner(scan);
2356 
2357       boolean hasNext = scanner.next(actual);
2358       assertEquals(false, hasNext);
2359 
2360       // Verify result
2361       for (int i = 0; i < expected.size(); i++) {
2362         assertEquals(expected.get(i), actual.get(i));
2363       }
2364     } finally {
2365       HRegion.closeHRegion(this.region);
2366       this.region = null;
2367     }
2368   }
2369 
2370   @Test
2371   public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2372     byte[] row1 = Bytes.toBytes("row1");
2373     byte[] qf1 = Bytes.toBytes("qualifier1");
2374     byte[] qf2 = Bytes.toBytes("qualifier2");
2375     byte[] fam1 = Bytes.toBytes("fam1");
2376     byte[][] families = { fam1 };
2377 
2378     long ts1 = 1; // System.currentTimeMillis();
2379     long ts2 = ts1 + 1;
2380     long ts3 = ts1 + 2;
2381 
2382     // Setting up region
2383     String method = this.getName();
2384     this.region = initHRegion(tableName, method, CONF, families);
2385     try {
2386       // Putting data in Region
2387       Put put = null;
2388       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2389       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2390       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2391 
2392       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2393       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2394       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2395 
2396       put = new Put(row1);
2397       put.add(kv13);
2398       put.add(kv12);
2399       put.add(kv11);
2400       put.add(kv23);
2401       put.add(kv22);
2402       put.add(kv21);
2403       region.put(put);
2404       region.flushcache();
2405 
2406       // Expected
2407       List<Cell> expected = new ArrayList<Cell>();
2408       expected.add(kv13);
2409       expected.add(kv12);
2410       expected.add(kv23);
2411       expected.add(kv22);
2412 
2413       Scan scan = new Scan(row1);
2414       scan.addColumn(fam1, qf1);
2415       scan.addColumn(fam1, qf2);
2416       scan.setMaxVersions(MAX_VERSIONS);
2417       List<Cell> actual = new ArrayList<Cell>();
2418       InternalScanner scanner = region.getScanner(scan);
2419 
2420       boolean hasNext = scanner.next(actual);
2421       assertEquals(false, hasNext);
2422 
2423       // Verify result
2424       for (int i = 0; i < expected.size(); i++) {
2425         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2426       }
2427     } finally {
2428       HRegion.closeHRegion(this.region);
2429       this.region = null;
2430     }
2431   }
2432 
2433   @Test
2434   public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2435     byte[] row1 = Bytes.toBytes("row1");
2436     byte[] fam1 = Bytes.toBytes("fam1");
2437     byte[][] families = { fam1 };
2438     byte[] qf1 = Bytes.toBytes("qualifier1");
2439     byte[] qf2 = Bytes.toBytes("qualifier2");
2440 
2441     long ts1 = 1;
2442     long ts2 = ts1 + 1;
2443     long ts3 = ts1 + 2;
2444     long ts4 = ts1 + 3;
2445 
2446     // Setting up region
2447     String method = this.getName();
2448     this.region = initHRegion(tableName, method, CONF, families);
2449     try {
2450       // Putting data in Region
2451       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2452       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2453       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2454       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2455 
2456       KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2457       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2458       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2459       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2460 
2461       Put put = null;
2462       put = new Put(row1);
2463       put.add(kv14);
2464       put.add(kv24);
2465       region.put(put);
2466       region.flushcache();
2467 
2468       put = new Put(row1);
2469       put.add(kv23);
2470       put.add(kv13);
2471       region.put(put);
2472       region.flushcache();
2473 
2474       put = new Put(row1);
2475       put.add(kv22);
2476       put.add(kv12);
2477       region.put(put);
2478       region.flushcache();
2479 
2480       put = new Put(row1);
2481       put.add(kv21);
2482       put.add(kv11);
2483       region.put(put);
2484 
2485       // Expected
2486       List<Cell> expected = new ArrayList<Cell>();
2487       expected.add(kv14);
2488       expected.add(kv13);
2489       expected.add(kv12);
2490       expected.add(kv24);
2491       expected.add(kv23);
2492       expected.add(kv22);
2493 
2494       Scan scan = new Scan(row1);
2495       scan.addColumn(fam1, qf1);
2496       scan.addColumn(fam1, qf2);
2497       int versions = 3;
2498       scan.setMaxVersions(versions);
2499       List<Cell> actual = new ArrayList<Cell>();
2500       InternalScanner scanner = region.getScanner(scan);
2501 
2502       boolean hasNext = scanner.next(actual);
2503       assertEquals(false, hasNext);
2504 
2505       // Verify result
2506       for (int i = 0; i < expected.size(); i++) {
2507         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2508       }
2509     } finally {
2510       HRegion.closeHRegion(this.region);
2511       this.region = null;
2512     }
2513   }
2514 
2515   @Test
2516   public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2517     byte[] row1 = Bytes.toBytes("row1");
2518     byte[] qf1 = Bytes.toBytes("qualifier1");
2519     byte[] qf2 = Bytes.toBytes("qualifier2");
2520     byte[] fam1 = Bytes.toBytes("fam1");
2521     byte[][] families = { fam1 };
2522 
2523     long ts1 = System.currentTimeMillis();
2524     long ts2 = ts1 + 1;
2525     long ts3 = ts1 + 2;
2526 
2527     // Setting up region
2528     String method = this.getName();
2529     this.region = initHRegion(tableName, method, CONF, families);
2530     try {
2531       // Putting data in Region
2532       Put put = null;
2533       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2534       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2535       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2536 
2537       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2538       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2539       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2540 
2541       put = new Put(row1);
2542       put.add(kv13);
2543       put.add(kv12);
2544       put.add(kv11);
2545       put.add(kv23);
2546       put.add(kv22);
2547       put.add(kv21);
2548       region.put(put);
2549 
2550       // Expected
2551       List<Cell> expected = new ArrayList<Cell>();
2552       expected.add(kv13);
2553       expected.add(kv12);
2554       expected.add(kv23);
2555       expected.add(kv22);
2556 
2557       Scan scan = new Scan(row1);
2558       scan.addFamily(fam1);
2559       scan.setMaxVersions(MAX_VERSIONS);
2560       List<Cell> actual = new ArrayList<Cell>();
2561       InternalScanner scanner = region.getScanner(scan);
2562 
2563       boolean hasNext = scanner.next(actual);
2564       assertEquals(false, hasNext);
2565 
2566       // Verify result
2567       for (int i = 0; i < expected.size(); i++) {
2568         assertEquals(expected.get(i), actual.get(i));
2569       }
2570     } finally {
2571       HRegion.closeHRegion(this.region);
2572       this.region = null;
2573     }
2574   }
2575 
2576   @Test
2577   public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
2578     byte[] row1 = Bytes.toBytes("row1");
2579     byte[] qf1 = Bytes.toBytes("qualifier1");
2580     byte[] qf2 = Bytes.toBytes("qualifier2");
2581     byte[] fam1 = Bytes.toBytes("fam1");
2582 
2583     long ts1 = 1; // System.currentTimeMillis();
2584     long ts2 = ts1 + 1;
2585     long ts3 = ts1 + 2;
2586 
2587     // Setting up region
2588     String method = this.getName();
2589     this.region = initHRegion(tableName, method, CONF, fam1);
2590     try {
2591       // Putting data in Region
2592       Put put = null;
2593       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2594       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2595       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2596 
2597       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2598       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2599       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2600 
2601       put = new Put(row1);
2602       put.add(kv13);
2603       put.add(kv12);
2604       put.add(kv11);
2605       put.add(kv23);
2606       put.add(kv22);
2607       put.add(kv21);
2608       region.put(put);
2609       region.flushcache();
2610 
2611       // Expected
2612       List<Cell> expected = new ArrayList<Cell>();
2613       expected.add(kv13);
2614       expected.add(kv12);
2615       expected.add(kv23);
2616       expected.add(kv22);
2617 
2618       Scan scan = new Scan(row1);
2619       scan.addFamily(fam1);
2620       scan.setMaxVersions(MAX_VERSIONS);
2621       List<Cell> actual = new ArrayList<Cell>();
2622       InternalScanner scanner = region.getScanner(scan);
2623 
2624       boolean hasNext = scanner.next(actual);
2625       assertEquals(false, hasNext);
2626 
2627       // Verify result
2628       for (int i = 0; i < expected.size(); i++) {
2629         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2630       }
2631     } finally {
2632       HRegion.closeHRegion(this.region);
2633       this.region = null;
2634     }
2635   }
2636 
2637   @Test
2638   public void testScanner_StopRow1542() throws IOException {
2639     byte[] family = Bytes.toBytes("testFamily");
2640     this.region = initHRegion(tableName, getName(), CONF, family);
2641     try {
2642       byte[] row1 = Bytes.toBytes("row111");
2643       byte[] row2 = Bytes.toBytes("row222");
2644       byte[] row3 = Bytes.toBytes("row333");
2645       byte[] row4 = Bytes.toBytes("row444");
2646       byte[] row5 = Bytes.toBytes("row555");
2647 
2648       byte[] col1 = Bytes.toBytes("Pub111");
2649       byte[] col2 = Bytes.toBytes("Pub222");
2650 
2651       Put put = new Put(row1);
2652       put.add(family, col1, Bytes.toBytes(10L));
2653       region.put(put);
2654 
2655       put = new Put(row2);
2656       put.add(family, col1, Bytes.toBytes(15L));
2657       region.put(put);
2658 
2659       put = new Put(row3);
2660       put.add(family, col2, Bytes.toBytes(20L));
2661       region.put(put);
2662 
2663       put = new Put(row4);
2664       put.add(family, col2, Bytes.toBytes(30L));
2665       region.put(put);
2666 
2667       put = new Put(row5);
2668       put.add(family, col1, Bytes.toBytes(40L));
2669       region.put(put);
2670 
2671       Scan scan = new Scan(row3, row4);
2672       scan.setMaxVersions();
2673       scan.addColumn(family, col1);
2674       InternalScanner s = region.getScanner(scan);
2675 
2676       List<Cell> results = new ArrayList<Cell>();
2677       assertEquals(false, s.next(results));
2678       assertEquals(0, results.size());
2679     } finally {
2680       HRegion.closeHRegion(this.region);
2681       this.region = null;
2682     }
2683   }
2684 
2685   @Test
2686   public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2687     byte[] row1 = Bytes.toBytes("row1");
2688     byte[] fam1 = Bytes.toBytes("fam1");
2689     byte[] qf1 = Bytes.toBytes("qualifier1");
2690     byte[] qf2 = Bytes.toBytes("quateslifier2");
2691 
2692     long ts1 = 1;
2693     long ts2 = ts1 + 1;
2694     long ts3 = ts1 + 2;
2695     long ts4 = ts1 + 3;
2696 
2697     // Setting up region
2698     String method = this.getName();
2699     this.region = initHRegion(tableName, method, CONF, fam1);
2700     try {
2701       // Putting data in Region
2702       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2703       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2704       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2705       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2706 
2707       KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2708       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2709       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2710       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2711 
2712       Put put = null;
2713       put = new Put(row1);
2714       put.add(kv14);
2715       put.add(kv24);
2716       region.put(put);
2717       region.flushcache();
2718 
2719       put = new Put(row1);
2720       put.add(kv23);
2721       put.add(kv13);
2722       region.put(put);
2723       region.flushcache();
2724 
2725       put = new Put(row1);
2726       put.add(kv22);
2727       put.add(kv12);
2728       region.put(put);
2729       region.flushcache();
2730 
2731       put = new Put(row1);
2732       put.add(kv21);
2733       put.add(kv11);
2734       region.put(put);
2735 
2736       // Expected
2737       List<KeyValue> expected = new ArrayList<KeyValue>();
2738       expected.add(kv14);
2739       expected.add(kv13);
2740       expected.add(kv12);
2741       expected.add(kv24);
2742       expected.add(kv23);
2743       expected.add(kv22);
2744 
2745       Scan scan = new Scan(row1);
2746       int versions = 3;
2747       scan.setMaxVersions(versions);
2748       List<Cell> actual = new ArrayList<Cell>();
2749       InternalScanner scanner = region.getScanner(scan);
2750 
2751       boolean hasNext = scanner.next(actual);
2752       assertEquals(false, hasNext);
2753 
2754       // Verify result
2755       for (int i = 0; i < expected.size(); i++) {
2756         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2757       }
2758     } finally {
2759       HRegion.closeHRegion(this.region);
2760       this.region = null;
2761     }
2762   }
2763 
2764   /**
2765    * Added for HBASE-5416
2766    * 
2767    * Here we test scan optimization when only subset of CFs are used in filter
2768    * conditions.
2769    */
2770   @Test
2771   public void testScanner_JoinedScanners() throws IOException {
2772     byte[] cf_essential = Bytes.toBytes("essential");
2773     byte[] cf_joined = Bytes.toBytes("joined");
2774     byte[] cf_alpha = Bytes.toBytes("alpha");
2775     this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
2776     try {
2777       byte[] row1 = Bytes.toBytes("row1");
2778       byte[] row2 = Bytes.toBytes("row2");
2779       byte[] row3 = Bytes.toBytes("row3");
2780 
2781       byte[] col_normal = Bytes.toBytes("d");
2782       byte[] col_alpha = Bytes.toBytes("a");
2783 
2784       byte[] filtered_val = Bytes.toBytes(3);
2785 
2786       Put put = new Put(row1);
2787       put.add(cf_essential, col_normal, Bytes.toBytes(1));
2788       put.add(cf_joined, col_alpha, Bytes.toBytes(1));
2789       region.put(put);
2790 
2791       put = new Put(row2);
2792       put.add(cf_essential, col_alpha, Bytes.toBytes(2));
2793       put.add(cf_joined, col_normal, Bytes.toBytes(2));
2794       put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
2795       region.put(put);
2796 
2797       put = new Put(row3);
2798       put.add(cf_essential, col_normal, filtered_val);
2799       put.add(cf_joined, col_normal, filtered_val);
2800       region.put(put);
2801 
2802       // Check two things:
2803       // 1. result list contains expected values
2804       // 2. result list is sorted properly
2805 
2806       Scan scan = new Scan();
2807       Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
2808           CompareOp.NOT_EQUAL, filtered_val);
2809       scan.setFilter(filter);
2810       scan.setLoadColumnFamiliesOnDemand(true);
2811       InternalScanner s = region.getScanner(scan);
2812 
2813       List<Cell> results = new ArrayList<Cell>();
2814       assertTrue(s.next(results));
2815       assertEquals(results.size(), 1);
2816       results.clear();
2817 
2818       assertTrue(s.next(results));
2819       assertEquals(results.size(), 3);
2820       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
2821       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
2822       assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
2823       results.clear();
2824 
2825       assertFalse(s.next(results));
2826       assertEquals(results.size(), 0);
2827     } finally {
2828       HRegion.closeHRegion(this.region);
2829       this.region = null;
2830     }
2831   }
2832 
2833   /**
2834    * HBASE-5416
2835    * 
2836    * Test case when scan limits amount of KVs returned on each next() call.
2837    */
2838   @Test
2839   public void testScanner_JoinedScannersWithLimits() throws IOException {
2840     final byte[] cf_first = Bytes.toBytes("first");
2841     final byte[] cf_second = Bytes.toBytes("second");
2842 
2843     this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
2844     try {
2845       final byte[] col_a = Bytes.toBytes("a");
2846       final byte[] col_b = Bytes.toBytes("b");
2847 
2848       Put put;
2849 
2850       for (int i = 0; i < 10; i++) {
2851         put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
2852         put.add(cf_first, col_a, Bytes.toBytes(i));
2853         if (i < 5) {
2854           put.add(cf_first, col_b, Bytes.toBytes(i));
2855           put.add(cf_second, col_a, Bytes.toBytes(i));
2856           put.add(cf_second, col_b, Bytes.toBytes(i));
2857         }
2858         region.put(put);
2859       }
2860 
2861       Scan scan = new Scan();
2862       scan.setLoadColumnFamiliesOnDemand(true);
2863       Filter bogusFilter = new FilterBase() {
2864         @Override
2865         public boolean isFamilyEssential(byte[] name) {
2866           return Bytes.equals(name, cf_first);
2867         }
2868       };
2869 
2870       scan.setFilter(bogusFilter);
2871       InternalScanner s = region.getScanner(scan);
2872 
2873       // Our data looks like this:
2874       // r0: first:a, first:b, second:a, second:b
2875       // r1: first:a, first:b, second:a, second:b
2876       // r2: first:a, first:b, second:a, second:b
2877       // r3: first:a, first:b, second:a, second:b
2878       // r4: first:a, first:b, second:a, second:b
2879       // r5: first:a
2880       // r6: first:a
2881       // r7: first:a
2882       // r8: first:a
2883       // r9: first:a
2884 
2885       // But due to next's limit set to 3, we should get this:
2886       // r0: first:a, first:b, second:a
2887       // r0: second:b
2888       // r1: first:a, first:b, second:a
2889       // r1: second:b
2890       // r2: first:a, first:b, second:a
2891       // r2: second:b
2892       // r3: first:a, first:b, second:a
2893       // r3: second:b
2894       // r4: first:a, first:b, second:a
2895       // r4: second:b
2896       // r5: first:a
2897       // r6: first:a
2898       // r7: first:a
2899       // r8: first:a
2900       // r9: first:a
2901 
2902       List<Cell> results = new ArrayList<Cell>();
2903       int index = 0;
2904       while (true) {
2905         boolean more = s.next(results, 3);
2906         if ((index >> 1) < 5) {
2907           if (index % 2 == 0)
2908             assertEquals(results.size(), 3);
2909           else
2910             assertEquals(results.size(), 1);
2911         } else
2912           assertEquals(results.size(), 1);
2913         results.clear();
2914         index++;
2915         if (!more)
2916           break;
2917       }
2918     } finally {
2919       HRegion.closeHRegion(this.region);
2920       this.region = null;
2921     }
2922   }
2923 
2924   // ////////////////////////////////////////////////////////////////////////////
2925   // Split test
2926   // ////////////////////////////////////////////////////////////////////////////
2927   /**
2928    * Splits twice and verifies getting from each of the split regions.
2929    * 
2930    * @throws Exception
2931    */
2932   @Test
2933   public void testBasicSplit() throws Exception {
2934     byte[][] families = { fam1, fam2, fam3 };
2935 
2936     Configuration hc = initSplit();
2937     // Setting up region
2938     String method = this.getName();
2939     this.region = initHRegion(tableName, method, hc, families);
2940 
2941     try {
2942       LOG.info("" + HBaseTestCase.addContent(region, fam3));
2943       region.flushcache();
2944       region.compactStores();
2945       byte[] splitRow = region.checkSplit();
2946       assertNotNull(splitRow);
2947       LOG.info("SplitRow: " + Bytes.toString(splitRow));
2948       HRegion[] regions = splitRegion(region, splitRow);
2949       try {
2950         // Need to open the regions.
2951         // TODO: Add an 'open' to HRegion... don't do open by constructing
2952         // instance.
2953         for (int i = 0; i < regions.length; i++) {
2954           regions[i] = HRegion.openHRegion(regions[i], null);
2955         }
2956         // Assert can get rows out of new regions. Should be able to get first
2957         // row from first region and the midkey from second region.
2958         assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
2959         assertGet(regions[1], fam3, splitRow);
2960         // Test I can get scanner and that it starts at right place.
2961         assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
2962         assertScan(regions[1], fam3, splitRow);
2963         // Now prove can't split regions that have references.
2964         for (int i = 0; i < regions.length; i++) {
2965           // Add so much data to this region, we create a store file that is >
2966           // than one of our unsplitable references. it will.
2967           for (int j = 0; j < 2; j++) {
2968             HBaseTestCase.addContent(regions[i], fam3);
2969           }
2970           HBaseTestCase.addContent(regions[i], fam2);
2971           HBaseTestCase.addContent(regions[i], fam1);
2972           regions[i].flushcache();
2973         }
2974 
2975         byte[][] midkeys = new byte[regions.length][];
2976         // To make regions splitable force compaction.
2977         for (int i = 0; i < regions.length; i++) {
2978           regions[i].compactStores();
2979           midkeys[i] = regions[i].checkSplit();
2980         }
2981 
2982         TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
2983         // Split these two daughter regions so then I'll have 4 regions. Will
2984         // split because added data above.
2985         for (int i = 0; i < regions.length; i++) {
2986           HRegion[] rs = null;
2987           if (midkeys[i] != null) {
2988             rs = splitRegion(regions[i], midkeys[i]);
2989             for (int j = 0; j < rs.length; j++) {
2990               sortedMap.put(Bytes.toString(rs[j].getRegionName()), HRegion.openHRegion(rs[j], null));
2991             }
2992           }
2993         }
2994         LOG.info("Made 4 regions");
2995         // The splits should have been even. Test I can get some arbitrary row
2996         // out of each.
2997         int interval = (LAST_CHAR - FIRST_CHAR) / 3;
2998         byte[] b = Bytes.toBytes(START_KEY);
2999         for (HRegion r : sortedMap.values()) {
3000           assertGet(r, fam3, b);
3001           b[0] += interval;
3002         }
3003       } finally {
3004         for (int i = 0; i < regions.length; i++) {
3005           try {
3006             regions[i].close();
3007           } catch (IOException e) {
3008             // Ignore.
3009           }
3010         }
3011       }
3012     } finally {
3013       HRegion.closeHRegion(this.region);
3014       this.region = null;
3015     }
3016   }
3017 
3018   @Test
3019   public void testSplitRegion() throws IOException {
3020     byte[] qualifier = Bytes.toBytes("qualifier");
3021     Configuration hc = initSplit();
3022     int numRows = 10;
3023     byte[][] families = { fam1, fam3 };
3024 
3025     // Setting up region
3026     String method = this.getName();
3027     this.region = initHRegion(tableName, method, hc, families);
3028 
3029     // Put data in region
3030     int startRow = 100;
3031     putData(startRow, numRows, qualifier, families);
3032     int splitRow = startRow + numRows;
3033     putData(splitRow, numRows, qualifier, families);
3034     region.flushcache();
3035 
3036     HRegion[] regions = null;
3037     try {
3038       regions = splitRegion(region, Bytes.toBytes("" + splitRow));
3039       // Opening the regions returned.
3040       for (int i = 0; i < regions.length; i++) {
3041         regions[i] = HRegion.openHRegion(regions[i], null);
3042       }
3043       // Verifying that the region has been split
3044       assertEquals(2, regions.length);
3045 
3046       // Verifying that all data is still there and that data is in the right
3047       // place
3048       verifyData(regions[0], startRow, numRows, qualifier, families);
3049       verifyData(regions[1], splitRow, numRows, qualifier, families);
3050 
3051     } finally {
3052       HRegion.closeHRegion(this.region);
3053       this.region = null;
3054     }
3055   }
3056 
3057   @Test
3058   public void testClearForceSplit() throws IOException {
3059     byte[] qualifier = Bytes.toBytes("qualifier");
3060     Configuration hc = initSplit();
3061     int numRows = 10;
3062     byte[][] families = { fam1, fam3 };
3063 
3064     // Setting up region
3065     String method = this.getName();
3066     this.region = initHRegion(tableName, method, hc, families);
3067 
3068     // Put data in region
3069     int startRow = 100;
3070     putData(startRow, numRows, qualifier, families);
3071     int splitRow = startRow + numRows;
3072     byte[] splitRowBytes = Bytes.toBytes("" + splitRow);
3073     putData(splitRow, numRows, qualifier, families);
3074     region.flushcache();
3075 
3076     HRegion[] regions = null;
3077     try {
3078       // Set force split
3079       region.forceSplit(splitRowBytes);
3080       assertTrue(region.shouldForceSplit());
3081       // Split point should be the force split row
3082       assertTrue(Bytes.equals(splitRowBytes, region.checkSplit()));
3083 
3084       // Add a store that has references.
3085       HStore storeMock = Mockito.mock(HStore.class);
3086       Mockito.when(storeMock.hasReferences()).thenReturn(true);
3087       Mockito.when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
3088       Mockito.when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
3089       Mockito.when(storeMock.getColumnFamilyName()).thenReturn("cf");
3090       region.stores.put(Bytes.toBytes(storeMock.getColumnFamilyName()), storeMock);
3091       assertTrue(region.hasReferences());
3092 
3093       // Will not split since the store has references.
3094       regions = splitRegion(region, splitRowBytes);
3095       assertNull(regions);
3096 
3097       // Region force split should be cleared after the split try.
3098       assertFalse(region.shouldForceSplit());
3099 
3100       // Remove the store that has references.
3101       region.stores.remove(Bytes.toBytes(storeMock.getColumnFamilyName()));
3102       assertFalse(region.hasReferences());
3103 
3104       // Now we can split.
3105       regions = splitRegion(region, splitRowBytes);
3106 
3107       // Opening the regions returned.
3108       for (int i = 0; i < regions.length; i++) {
3109         regions[i] = HRegion.openHRegion(regions[i], null);
3110       }
3111       // Verifying that the region has been split
3112       assertEquals(2, regions.length);
3113 
3114       // Verifying that all data is still there and that data is in the right
3115       // place
3116       verifyData(regions[0], startRow, numRows, qualifier, families);
3117       verifyData(regions[1], splitRow, numRows, qualifier, families);
3118 
3119     } finally {
3120       HRegion.closeHRegion(this.region);
3121       this.region = null;
3122     }
3123   }
3124 
3125   /**
3126    * Flushes the cache in a thread while scanning. The tests verify that the
3127    * scan is coherent - e.g. the returned results are always of the same or
3128    * later update as the previous results.
3129    * 
3130    * @throws IOException
3131    *           scan / compact
3132    * @throws InterruptedException
3133    *           thread join
3134    */
3135   @Test
3136   public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3137     byte[] family = Bytes.toBytes("family");
3138     int numRows = 1000;
3139     int flushAndScanInterval = 10;
3140     int compactInterval = 10 * flushAndScanInterval;
3141 
3142     String method = "testFlushCacheWhileScanning";
3143     this.region = initHRegion(tableName, method, CONF, family);
3144     try {
3145       FlushThread flushThread = new FlushThread();
3146       flushThread.start();
3147 
3148       Scan scan = new Scan();
3149       scan.addFamily(family);
3150       scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
3151           new BinaryComparator(Bytes.toBytes(5L))));
3152 
3153       int expectedCount = 0;
3154       List<Cell> res = new ArrayList<Cell>();
3155 
3156       boolean toggle = true;
3157       for (long i = 0; i < numRows; i++) {
3158         Put put = new Put(Bytes.toBytes(i));
3159         put.setDurability(Durability.SKIP_WAL);
3160         put.add(family, qual1, Bytes.toBytes(i % 10));
3161         region.put(put);
3162 
3163         if (i != 0 && i % compactInterval == 0) {
3164           // System.out.println("iteration = " + i);
3165           region.compactStores(true);
3166         }
3167 
3168         if (i % 10 == 5L) {
3169           expectedCount++;
3170         }
3171 
3172         if (i != 0 && i % flushAndScanInterval == 0) {
3173           res.clear();
3174           InternalScanner scanner = region.getScanner(scan);
3175           if (toggle) {
3176             flushThread.flush();
3177           }
3178           while (scanner.next(res))
3179             ;
3180           if (!toggle) {
3181             flushThread.flush();
3182           }
3183           assertEquals("i=" + i, expectedCount, res.size());
3184           toggle = !toggle;
3185         }
3186       }
3187 
3188       flushThread.done();
3189       flushThread.join();
3190       flushThread.checkNoError();
3191     } finally {
3192       HRegion.closeHRegion(this.region);
3193       this.region = null;
3194     }
3195   }
3196 
3197   protected class FlushThread extends Thread {
3198     private volatile boolean done;
3199     private Throwable error = null;
3200 
3201     public void done() {
3202       done = true;
3203       synchronized (this) {
3204         interrupt();
3205       }
3206     }
3207 
3208     public void checkNoError() {
3209       if (error != null) {
3210         assertNull(error);
3211       }
3212     }
3213 
3214     @Override
3215     public void run() {
3216       done = false;
3217       while (!done) {
3218         synchronized (this) {
3219           try {
3220             wait();
3221           } catch (InterruptedException ignored) {
3222             if (done) {
3223               break;
3224             }
3225           }
3226         }
3227         try {
3228           region.flushcache();
3229         } catch (IOException e) {
3230           if (!done) {
3231             LOG.error("Error while flusing cache", e);
3232             error = e;
3233           }
3234           break;
3235         }
3236       }
3237 
3238     }
3239 
3240     public void flush() {
3241       synchronized (this) {
3242         notify();
3243       }
3244 
3245     }
3246   }
3247 
3248   /**
3249    * Writes very wide records and scans for the latest every time.. Flushes and
3250    * compacts the region every now and then to keep things realistic.
3251    * 
3252    * @throws IOException
3253    *           by flush / scan / compaction
3254    * @throws InterruptedException
3255    *           when joining threads
3256    */
3257   @Test
3258   public void testWritesWhileScanning() throws IOException, InterruptedException {
3259     int testCount = 100;
3260     int numRows = 1;
3261     int numFamilies = 10;
3262     int numQualifiers = 100;
3263     int flushInterval = 7;
3264     int compactInterval = 5 * flushInterval;
3265     byte[][] families = new byte[numFamilies][];
3266     for (int i = 0; i < numFamilies; i++) {
3267       families[i] = Bytes.toBytes("family" + i);
3268     }
3269     byte[][] qualifiers = new byte[numQualifiers][];
3270     for (int i = 0; i < numQualifiers; i++) {
3271       qualifiers[i] = Bytes.toBytes("qual" + i);
3272     }
3273 
3274     String method = "testWritesWhileScanning";
3275     this.region = initHRegion(tableName, method, CONF, families);
3276     try {
3277       PutThread putThread = new PutThread(numRows, families, qualifiers);
3278       putThread.start();
3279       putThread.waitForFirstPut();
3280 
3281       FlushThread flushThread = new FlushThread();
3282       flushThread.start();
3283 
3284       Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3285 
3286       int expectedCount = numFamilies * numQualifiers;
3287       List<Cell> res = new ArrayList<Cell>();
3288 
3289       long prevTimestamp = 0L;
3290       for (int i = 0; i < testCount; i++) {
3291 
3292         if (i != 0 && i % compactInterval == 0) {
3293           region.compactStores(true);
3294         }
3295 
3296         if (i != 0 && i % flushInterval == 0) {
3297           flushThread.flush();
3298         }
3299 
3300         boolean previousEmpty = res.isEmpty();
3301         res.clear();
3302         InternalScanner scanner = region.getScanner(scan);
3303         while (scanner.next(res))
3304           ;
3305         if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3306           assertEquals("i=" + i, expectedCount, res.size());
3307           long timestamp = res.get(0).getTimestamp();
3308           assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3309               timestamp >= prevTimestamp);
3310           prevTimestamp = timestamp;
3311         }
3312       }
3313 
3314       putThread.done();
3315 
3316       region.flushcache();
3317 
3318       putThread.join();
3319       putThread.checkNoError();
3320 
3321       flushThread.done();
3322       flushThread.join();
3323       flushThread.checkNoError();
3324     } finally {
3325       try {
3326         HRegion.closeHRegion(this.region);
3327       } catch (DroppedSnapshotException dse) {
3328         // We could get this on way out because we interrupt the background flusher and it could
3329         // fail anywhere causing a DSE over in the background flusher... only it is not properly
3330         // dealt with so could still be memory hanging out when we get to here -- memory we can't
3331         // flush because the accounting is 'off' since original DSE.
3332       }
3333       this.region = null;
3334     }
3335   }
3336 
3337   protected class PutThread extends Thread {
3338     private volatile boolean done;
3339     private volatile int numPutsFinished = 0;
3340 
3341     private Throwable error = null;
3342     private int numRows;
3343     private byte[][] families;
3344     private byte[][] qualifiers;
3345 
3346     private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3347       this.numRows = numRows;
3348       this.families = families;
3349       this.qualifiers = qualifiers;
3350     }
3351 
3352     /**
3353      * Block until this thread has put at least one row.
3354      */
3355     public void waitForFirstPut() throws InterruptedException {
3356       // wait until put thread actually puts some data
3357       while (numPutsFinished == 0) {
3358         checkNoError();
3359         Thread.sleep(50);
3360       }
3361     }
3362 
3363     public void done() {
3364       done = true;
3365       synchronized (this) {
3366         interrupt();
3367       }
3368     }
3369 
3370     public void checkNoError() {
3371       if (error != null) {
3372         assertNull(error);
3373       }
3374     }
3375 
3376     @Override
3377     public void run() {
3378       done = false;
3379       while (!done) {
3380         try {
3381           for (int r = 0; r < numRows; r++) {
3382             byte[] row = Bytes.toBytes("row" + r);
3383             Put put = new Put(row);
3384             put.setDurability(Durability.SKIP_WAL);
3385             byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3386             for (byte[] family : families) {
3387               for (byte[] qualifier : qualifiers) {
3388                 put.add(family, qualifier, (long) numPutsFinished, value);
3389               }
3390             }
3391             region.put(put);
3392             numPutsFinished++;
3393             if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3394               System.out.println("put iteration = " + numPutsFinished);
3395               Delete delete = new Delete(row, (long) numPutsFinished - 30);
3396               region.delete(delete);
3397             }
3398             numPutsFinished++;
3399           }
3400         } catch (InterruptedIOException e) {
3401           // This is fine. It means we are done, or didn't get the lock on time
3402         } catch (IOException e) {
3403           LOG.error("error while putting records", e);
3404           error = e;
3405           break;
3406         }
3407       }
3408 
3409     }
3410 
3411   }
3412 
3413   /**
3414    * Writes very wide records and gets the latest row every time.. Flushes and
3415    * compacts the region aggressivly to catch issues.
3416    * 
3417    * @throws IOException
3418    *           by flush / scan / compaction
3419    * @throws InterruptedException
3420    *           when joining threads
3421    */
3422   @Test
3423   public void testWritesWhileGetting() throws Exception {
3424     int testCount = 100;
3425     int numRows = 1;
3426     int numFamilies = 10;
3427     int numQualifiers = 100;
3428     int compactInterval = 100;
3429     byte[][] families = new byte[numFamilies][];
3430     for (int i = 0; i < numFamilies; i++) {
3431       families[i] = Bytes.toBytes("family" + i);
3432     }
3433     byte[][] qualifiers = new byte[numQualifiers][];
3434     for (int i = 0; i < numQualifiers; i++) {
3435       qualifiers[i] = Bytes.toBytes("qual" + i);
3436     }
3437 
3438 
3439     String method = "testWritesWhileGetting";
3440     // This test flushes constantly and can cause many files to be created,
3441     // possibly
3442     // extending over the ulimit. Make sure compactions are aggressive in
3443     // reducing
3444     // the number of HFiles created.
3445     Configuration conf = HBaseConfiguration.create(CONF);
3446     conf.setInt("hbase.hstore.compaction.min", 1);
3447     conf.setInt("hbase.hstore.compaction.max", 1000);
3448     this.region = initHRegion(tableName, method, conf, families);
3449     PutThread putThread = null;
3450     MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3451     try {
3452       putThread = new PutThread(numRows, families, qualifiers);
3453       putThread.start();
3454       putThread.waitForFirstPut();
3455 
3456       // Add a thread that flushes as fast as possible
3457       ctx.addThread(new RepeatingTestThread(ctx) {
3458         private int flushesSinceCompact = 0;
3459         private final int maxFlushesSinceCompact = 20;
3460 
3461         @Override
3462         public void doAnAction() throws Exception {
3463           if (region.flushcache().isCompactionNeeded()) {
3464             ++flushesSinceCompact;
3465           }
3466           // Compact regularly to avoid creating too many files and exceeding
3467           // the ulimit.
3468           if (flushesSinceCompact == maxFlushesSinceCompact) {
3469             region.compactStores(false);
3470             flushesSinceCompact = 0;
3471           }
3472         }
3473       });
3474       ctx.startThreads();
3475 
3476       Get get = new Get(Bytes.toBytes("row0"));
3477       Result result = null;
3478 
3479       int expectedCount = numFamilies * numQualifiers;
3480 
3481       long prevTimestamp = 0L;
3482       for (int i = 0; i < testCount; i++) {
3483 
3484         boolean previousEmpty = result == null || result.isEmpty();
3485         result = region.get(get);
3486         if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3487           assertEquals("i=" + i, expectedCount, result.size());
3488           // TODO this was removed, now what dangit?!
3489           // search looking for the qualifier in question?
3490           long timestamp = 0;
3491           for (Cell kv : result.rawCells()) {
3492             if (CellUtil.matchingFamily(kv, families[0])
3493                 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3494               timestamp = kv.getTimestamp();
3495             }
3496           }
3497           assertTrue(timestamp >= prevTimestamp);
3498           prevTimestamp = timestamp;
3499           Cell previousKV = null;
3500 
3501           for (Cell kv : result.rawCells()) {
3502             byte[] thisValue = CellUtil.cloneValue(kv);
3503             if (previousKV != null) {
3504               if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3505                 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3506                     + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3507                     + "(memStoreTS:" + kv.getMvccVersion() + ")");
3508                 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3509               }
3510             }
3511             previousKV = kv;
3512           }
3513         }
3514       }
3515     } finally {
3516       if (putThread != null)
3517         putThread.done();
3518 
3519       region.flushcache();
3520 
3521       if (putThread != null) {
3522         putThread.join();
3523         putThread.checkNoError();
3524       }
3525 
3526       ctx.stop();
3527       HRegion.closeHRegion(this.region);
3528       this.region = null;
3529     }
3530   }
3531 
3532   @Test
3533   public void testHolesInMeta() throws Exception {
3534     byte[] family = Bytes.toBytes("family");
3535     this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF,
3536         false, family);
3537     try {
3538       byte[] rowNotServed = Bytes.toBytes("a");
3539       Get g = new Get(rowNotServed);
3540       try {
3541         region.get(g);
3542         fail();
3543       } catch (WrongRegionException x) {
3544         // OK
3545       }
3546       byte[] row = Bytes.toBytes("y");
3547       g = new Get(row);
3548       region.get(g);
3549     } finally {
3550       HRegion.closeHRegion(this.region);
3551       this.region = null;
3552     }
3553   }
3554 
3555   @Test
3556   public void testIndexesScanWithOneDeletedRow() throws IOException {
3557     byte[] family = Bytes.toBytes("family");
3558 
3559     // Setting up region
3560     String method = "testIndexesScanWithOneDeletedRow";
3561     this.region = initHRegion(tableName, method, CONF, family);
3562     try {
3563       Put put = new Put(Bytes.toBytes(1L));
3564       put.add(family, qual1, 1L, Bytes.toBytes(1L));
3565       region.put(put);
3566 
3567       region.flushcache();
3568 
3569       Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3570       region.delete(delete);
3571 
3572       put = new Put(Bytes.toBytes(2L));
3573       put.add(family, qual1, 2L, Bytes.toBytes(2L));
3574       region.put(put);
3575 
3576       Scan idxScan = new Scan();
3577       idxScan.addFamily(family);
3578       idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
3579           new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
3580               new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
3581               CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
3582       InternalScanner scanner = region.getScanner(idxScan);
3583       List<Cell> res = new ArrayList<Cell>();
3584 
3585       while (scanner.next(res))
3586         ;
3587       assertEquals(1L, res.size());
3588     } finally {
3589       HRegion.closeHRegion(this.region);
3590       this.region = null;
3591     }
3592   }
3593 
3594   // ////////////////////////////////////////////////////////////////////////////
3595   // Bloom filter test
3596   // ////////////////////////////////////////////////////////////////////////////
3597   @Test
3598   public void testBloomFilterSize() throws IOException {
3599     byte[] fam1 = Bytes.toBytes("fam1");
3600     byte[] qf1 = Bytes.toBytes("col");
3601     byte[] val1 = Bytes.toBytes("value1");
3602     // Create Table
3603     HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
3604         .setBloomFilterType(BloomType.ROWCOL);
3605 
3606     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3607     htd.addFamily(hcd);
3608     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3609     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3610     try {
3611       int num_unique_rows = 10;
3612       int duplicate_multiplier = 2;
3613       int num_storefiles = 4;
3614 
3615       int version = 0;
3616       for (int f = 0; f < num_storefiles; f++) {
3617         for (int i = 0; i < duplicate_multiplier; i++) {
3618           for (int j = 0; j < num_unique_rows; j++) {
3619             Put put = new Put(Bytes.toBytes("row" + j));
3620             put.setDurability(Durability.SKIP_WAL);
3621             put.add(fam1, qf1, version++, val1);
3622             region.put(put);
3623           }
3624         }
3625         region.flushcache();
3626       }
3627       // before compaction
3628       HStore store = (HStore) region.getStore(fam1);
3629       Collection<StoreFile> storeFiles = store.getStorefiles();
3630       for (StoreFile storefile : storeFiles) {
3631         StoreFile.Reader reader = storefile.getReader();
3632         reader.loadFileInfo();
3633         reader.loadBloomfilter();
3634         assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
3635         assertEquals(num_unique_rows, reader.getFilterEntries());
3636       }
3637 
3638       region.compactStores(true);
3639 
3640       // after compaction
3641       storeFiles = store.getStorefiles();
3642       for (StoreFile storefile : storeFiles) {
3643         StoreFile.Reader reader = storefile.getReader();
3644         reader.loadFileInfo();
3645         reader.loadBloomfilter();
3646         assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
3647         assertEquals(num_unique_rows, reader.getFilterEntries());
3648       }
3649     } finally {
3650       HRegion.closeHRegion(this.region);
3651       this.region = null;
3652     }
3653   }
3654 
3655   @Test
3656   public void testAllColumnsWithBloomFilter() throws IOException {
3657     byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
3658     byte[] FAMILY = Bytes.toBytes("family");
3659 
3660     // Create table
3661     HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
3662         .setBloomFilterType(BloomType.ROWCOL);
3663     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
3664     htd.addFamily(hcd);
3665     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3666     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3667     try {
3668       // For row:0, col:0: insert versions 1 through 5.
3669       byte row[] = Bytes.toBytes("row:" + 0);
3670       byte column[] = Bytes.toBytes("column:" + 0);
3671       Put put = new Put(row);
3672       put.setDurability(Durability.SKIP_WAL);
3673       for (long idx = 1; idx <= 4; idx++) {
3674         put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
3675       }
3676       region.put(put);
3677 
3678       // Flush
3679       region.flushcache();
3680 
3681       // Get rows
3682       Get get = new Get(row);
3683       get.setMaxVersions();
3684       Cell[] kvs = region.get(get).rawCells();
3685 
3686       // Check if rows are correct
3687       assertEquals(4, kvs.length);
3688       checkOneCell(kvs[0], FAMILY, 0, 0, 4);
3689       checkOneCell(kvs[1], FAMILY, 0, 0, 3);
3690       checkOneCell(kvs[2], FAMILY, 0, 0, 2);
3691       checkOneCell(kvs[3], FAMILY, 0, 0, 1);
3692     } finally {
3693       HRegion.closeHRegion(this.region);
3694       this.region = null;
3695     }
3696   }
3697 
3698   /**
3699    * Testcase to cover bug-fix for HBASE-2823 Ensures correct delete when
3700    * issuing delete row on columns with bloom filter set to row+col
3701    * (BloomType.ROWCOL)
3702    */
3703   @Test
3704   public void testDeleteRowWithBloomFilter() throws IOException {
3705     byte[] familyName = Bytes.toBytes("familyName");
3706 
3707     // Create Table
3708     HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
3709         .setBloomFilterType(BloomType.ROWCOL);
3710 
3711     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3712     htd.addFamily(hcd);
3713     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3714     this.region = TEST_UTIL.createLocalHRegion(info, htd);
3715     try {
3716       // Insert some data
3717       byte row[] = Bytes.toBytes("row1");
3718       byte col[] = Bytes.toBytes("col1");
3719 
3720       Put put = new Put(row);
3721       put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
3722       region.put(put);
3723       region.flushcache();
3724 
3725       Delete del = new Delete(row);
3726       region.delete(del);
3727       region.flushcache();
3728 
3729       // Get remaining rows (should have none)
3730       Get get = new Get(row);
3731       get.addColumn(familyName, col);
3732 
3733       Cell[] keyValues = region.get(get).rawCells();
3734       assertTrue(keyValues.length == 0);
3735     } finally {
3736       HRegion.closeHRegion(this.region);
3737       this.region = null;
3738     }
3739   }
3740 
3741   @Test
3742   public void testgetHDFSBlocksDistribution() throws Exception {
3743     HBaseTestingUtility htu = new HBaseTestingUtility();
3744     final int DEFAULT_BLOCK_SIZE = 1024;
3745     htu.getConfiguration().setLong("dfs.namenode.fs-limits.min-block-size", 0);
3746     htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
3747     htu.getConfiguration().setInt("dfs.replication", 2);
3748 
3749     // set up a cluster with 3 nodes
3750     MiniHBaseCluster cluster = null;
3751     String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
3752     int regionServersCount = 3;
3753 
3754     try {
3755       cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
3756       byte[][] families = { fam1, fam2 };
3757       HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);
3758 
3759       // Setting up region
3760       byte row[] = Bytes.toBytes("row1");
3761       byte col[] = Bytes.toBytes("col1");
3762 
3763       Put put = new Put(row);
3764       put.add(fam1, col, 1, Bytes.toBytes("test1"));
3765       put.add(fam2, col, 1, Bytes.toBytes("test2"));
3766       ht.put(put);
3767 
3768       HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
3769           .get(0);
3770       firstRegion.flushcache();
3771       HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
3772 
3773       // given the default replication factor is 2 and we have 2 HFiles,
3774       // we will have total of 4 replica of blocks on 3 datanodes; thus there
3775       // must be at least one host that have replica for 2 HFiles. That host's
3776       // weight will be equal to the unique block weight.
3777       long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
3778 
3779       String topHost = blocksDistribution1.getTopHosts().get(0);
3780       long topHostWeight = blocksDistribution1.getWeight(topHost);
3781       assertTrue(uniqueBlocksWeight1 == topHostWeight);
3782 
3783       // use the static method to compute the value, it should be the same.
3784       // static method is used by load balancer or other components
3785       HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
3786           htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
3787       long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
3788 
3789       assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
3790 
3791       ht.close();
3792     } finally {
3793       if (cluster != null) {
3794         htu.shutdownMiniCluster();
3795       }
3796     }
3797   }
3798 
3799   /**
3800    * Testcase to check state of region initialization task set to ABORTED or not
3801    * if any exceptions during initialization
3802    * 
3803    * @throws Exception
3804    */
3805   @Test
3806   public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
3807     TableName tableName = TableName.valueOf(name.getMethodName());
3808     HRegionInfo info = null;
3809     try {
3810       FileSystem fs = Mockito.mock(FileSystem.class);
3811       Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
3812       HTableDescriptor htd = new HTableDescriptor(tableName);
3813       htd.addFamily(new HColumnDescriptor("cf"));
3814       info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
3815           HConstants.EMPTY_BYTE_ARRAY, false);
3816       Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
3817       region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
3818       // region initialization throws IOException and set task state to ABORTED.
3819       region.initialize();
3820       fail("Region initialization should fail due to IOException");
3821     } catch (IOException io) {
3822       List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
3823       for (MonitoredTask monitoredTask : tasks) {
3824         if (!(monitoredTask instanceof MonitoredRPCHandler)
3825             && monitoredTask.getDescription().contains(region.toString())) {
3826           assertTrue("Region state should be ABORTED.",
3827               monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
3828           break;
3829         }
3830       }
3831     } finally {
3832       HRegion.closeHRegion(region);
3833     }
3834   }
3835 
3836   /**
3837    * Verifies that the .regioninfo file is written on region creation and that
3838    * is recreated if missing during region opening.
3839    */
3840   @Test
3841   public void testRegionInfoFileCreation() throws IOException {
3842     Path rootDir = new Path(dir + "testRegionInfoFileCreation");
3843 
3844     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
3845     htd.addFamily(new HColumnDescriptor("cf"));
3846 
3847     HRegionInfo hri = new HRegionInfo(htd.getTableName());
3848 
3849     // Create a region and skip the initialization (like CreateTableHandler)
3850     HRegion region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
3851 //    HRegion region = TEST_UTIL.createLocalHRegion(hri, htd);
3852     Path regionDir = region.getRegionFileSystem().getRegionDir();
3853     FileSystem fs = region.getRegionFileSystem().getFileSystem();
3854     HRegion.closeHRegion(region);
3855 
3856     Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
3857 
3858     // Verify that the .regioninfo file is present
3859     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3860         fs.exists(regionInfoFile));
3861 
3862     // Try to open the region
3863     region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3864     assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3865     HRegion.closeHRegion(region);
3866 
3867     // Verify that the .regioninfo file is still there
3868     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3869         fs.exists(regionInfoFile));
3870 
3871     // Remove the .regioninfo file and verify is recreated on region open
3872     fs.delete(regionInfoFile);
3873     assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
3874         fs.exists(regionInfoFile));
3875 
3876     region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3877 //    region = TEST_UTIL.openHRegion(hri, htd);
3878     assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3879     HRegion.closeHRegion(region);
3880 
3881     // Verify that the .regioninfo file is still there
3882     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3883         fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
3884   }
3885 
3886   /**
3887    * TestCase for increment
3888    */
3889   private static class Incrementer implements Runnable {
3890     private HRegion region;
3891     private final static byte[] incRow = Bytes.toBytes("incRow");
3892     private final static byte[] family = Bytes.toBytes("family");
3893     private final static byte[] qualifier = Bytes.toBytes("qualifier");
3894     private final static long ONE = 1l;
3895     private int incCounter;
3896 
3897     public Incrementer(HRegion region, int incCounter) {
3898       this.region = region;
3899       this.incCounter = incCounter;
3900     }
3901 
3902     @Override
3903     public void run() {
3904       int count = 0;
3905       while (count < incCounter) {
3906         Increment inc = new Increment(incRow);
3907         inc.addColumn(family, qualifier, ONE);
3908         count++;
3909         try {
3910           region.increment(inc);
3911         } catch (IOException e) {
3912           e.printStackTrace();
3913           break;
3914         }
3915       }
3916     }
3917   }
3918 
3919   /**
3920    * Test case to check increment function with memstore flushing
3921    * @throws Exception
3922    */
3923   @Test
3924   public void testParallelIncrementWithMemStoreFlush() throws Exception {
3925     byte[] family = Incrementer.family;
3926     this.region = initHRegion(tableName, method, CONF, family);
3927     final HRegion region = this.region;
3928     final AtomicBoolean incrementDone = new AtomicBoolean(false);
3929     Runnable flusher = new Runnable() {
3930       @Override
3931       public void run() {
3932         while (!incrementDone.get()) {
3933           try {
3934             region.flushcache();
3935           } catch (Exception e) {
3936             e.printStackTrace();
3937           }
3938         }
3939       }
3940     };
3941 
3942     // after all increment finished, the row will increment to 20*100 = 2000
3943     int threadNum = 20;
3944     int incCounter = 100;
3945     long expected = threadNum * incCounter;
3946     Thread[] incrementers = new Thread[threadNum];
3947     Thread flushThread = new Thread(flusher);
3948     for (int i = 0; i < threadNum; i++) {
3949       incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
3950       incrementers[i].start();
3951     }
3952     flushThread.start();
3953     for (int i = 0; i < threadNum; i++) {
3954       incrementers[i].join();
3955     }
3956 
3957     incrementDone.set(true);
3958     flushThread.join();
3959 
3960     Get get = new Get(Incrementer.incRow);
3961     get.addColumn(Incrementer.family, Incrementer.qualifier);
3962     get.setMaxVersions(1);
3963     Result res = this.region.get(get);
3964     List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
3965 
3966     // we just got the latest version
3967     assertEquals(kvs.size(), 1);
3968     Cell kv = kvs.get(0);
3969     assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
3970     this.region = null;
3971   }
3972 
3973   /**
3974    * TestCase for append
3975    */
3976   private static class Appender implements Runnable {
3977     private HRegion region;
3978     private final static byte[] appendRow = Bytes.toBytes("appendRow");
3979     private final static byte[] family = Bytes.toBytes("family");
3980     private final static byte[] qualifier = Bytes.toBytes("qualifier");
3981     private final static byte[] CHAR = Bytes.toBytes("a");
3982     private int appendCounter;
3983 
3984     public Appender(HRegion region, int appendCounter) {
3985       this.region = region;
3986       this.appendCounter = appendCounter;
3987     }
3988 
3989     @Override
3990     public void run() {
3991       int count = 0;
3992       while (count < appendCounter) {
3993         Append app = new Append(appendRow);
3994         app.add(family, qualifier, CHAR);
3995         count++;
3996         try {
3997           region.append(app);
3998         } catch (IOException e) {
3999           e.printStackTrace();
4000           break;
4001         }
4002       }
4003     }
4004   }
4005 
4006   /**
4007    * Test case to check append function with memstore flushing
4008    * @throws Exception
4009    */
4010   @Test
4011   public void testParallelAppendWithMemStoreFlush() throws Exception {
4012     byte[] family = Appender.family;
4013     this.region = initHRegion(tableName, method, CONF, family);
4014     final HRegion region = this.region;
4015     final AtomicBoolean appendDone = new AtomicBoolean(false);
4016     Runnable flusher = new Runnable() {
4017       @Override
4018       public void run() {
4019         while (!appendDone.get()) {
4020           try {
4021             region.flushcache();
4022           } catch (Exception e) {
4023             e.printStackTrace();
4024           }
4025         }
4026       }
4027     };
4028 
4029     // after all append finished, the value will append to threadNum *
4030     // appendCounter Appender.CHAR
4031     int threadNum = 20;
4032     int appendCounter = 100;
4033     byte[] expected = new byte[threadNum * appendCounter];
4034     for (int i = 0; i < threadNum * appendCounter; i++) {
4035       System.arraycopy(Appender.CHAR, 0, expected, i, 1);
4036     }
4037     Thread[] appenders = new Thread[threadNum];
4038     Thread flushThread = new Thread(flusher);
4039     for (int i = 0; i < threadNum; i++) {
4040       appenders[i] = new Thread(new Appender(this.region, appendCounter));
4041       appenders[i].start();
4042     }
4043     flushThread.start();
4044     for (int i = 0; i < threadNum; i++) {
4045       appenders[i].join();
4046     }
4047 
4048     appendDone.set(true);
4049     flushThread.join();
4050 
4051     Get get = new Get(Appender.appendRow);
4052     get.addColumn(Appender.family, Appender.qualifier);
4053     get.setMaxVersions(1);
4054     Result res = this.region.get(get);
4055     List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
4056 
4057     // we just got the latest version
4058     assertEquals(kvs.size(), 1);
4059     Cell kv = kvs.get(0);
4060     byte[] appendResult = new byte[kv.getValueLength()];
4061     System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
4062     assertArrayEquals(expected, appendResult);
4063     this.region = null;
4064   }
4065 
4066   /**
4067    * Test case to check put function with memstore flushing for same row, same ts
4068    * @throws Exception
4069    */
4070   @Test
4071   public void testPutWithMemStoreFlush() throws Exception {
4072     byte[] family = Bytes.toBytes("family");
4073     ;
4074     byte[] qualifier = Bytes.toBytes("qualifier");
4075     byte[] row = Bytes.toBytes("putRow");
4076     byte[] value = null;
4077     this.region = initHRegion(tableName, method, CONF, family);
4078     Put put = null;
4079     Get get = null;
4080     List<Cell> kvs = null;
4081     Result res = null;
4082 
4083     put = new Put(row);
4084     value = Bytes.toBytes("value0");
4085     put.add(family, qualifier, 1234567l, value);
4086     region.put(put);
4087     get = new Get(row);
4088     get.addColumn(family, qualifier);
4089     get.setMaxVersions();
4090     res = this.region.get(get);
4091     kvs = res.getColumnCells(family, qualifier);
4092     assertEquals(1, kvs.size());
4093     assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4094 
4095     region.flushcache();
4096     get = new Get(row);
4097     get.addColumn(family, qualifier);
4098     get.setMaxVersions();
4099     res = this.region.get(get);
4100     kvs = res.getColumnCells(family, qualifier);
4101     assertEquals(1, kvs.size());
4102     assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4103 
4104     put = new Put(row);
4105     value = Bytes.toBytes("value1");
4106     put.add(family, qualifier, 1234567l, value);
4107     region.put(put);
4108     get = new Get(row);
4109     get.addColumn(family, qualifier);
4110     get.setMaxVersions();
4111     res = this.region.get(get);
4112     kvs = res.getColumnCells(family, qualifier);
4113     assertEquals(1, kvs.size());
4114     assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4115 
4116     region.flushcache();
4117     get = new Get(row);
4118     get.addColumn(family, qualifier);
4119     get.setMaxVersions();
4120     res = this.region.get(get);
4121     kvs = res.getColumnCells(family, qualifier);
4122     assertEquals(1, kvs.size());
4123     assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4124   }
4125 
4126   @Test
4127   public void testDurability() throws Exception {
4128     String method = "testDurability";
4129     // there are 5 x 5 cases:
4130     // table durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) x mutation
4131     // durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT)
4132 
4133     // expected cases for append and sync wal
4134     durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4135     durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4136     durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4137 
4138     durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4139     durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4140     durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4141 
4142     durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4143     durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4144 
4145     durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
4146     durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4147 
4148     durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
4149     durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
4150     durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
4151 
4152     // expected cases for async wal
4153     durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4154     durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4155     durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4156     durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4157     durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
4158     durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4159 
4160     durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4161     durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4162     durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4163     durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4164     durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
4165     durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
4166 
4167     // expect skip wal cases
4168     durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4169     durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4170     durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4171     durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
4172     durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
4173     durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
4174 
4175   }
4176 
4177   private void durabilityTest(String method, Durability tableDurability,
4178       Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
4179       final boolean expectSyncFromLogSyncer) throws Exception {
4180     Configuration conf = HBaseConfiguration.create(CONF);
4181     method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
4182     TableName tableName = TableName.valueOf(method);
4183     byte[] family = Bytes.toBytes("family");
4184     Path logDir = new Path(new Path(dir + method), "log");
4185     HLog hlog = HLogFactory.createHLog(FILESYSTEM, logDir, UUID.randomUUID().toString(), conf);
4186     final HLog log = spy(hlog);
4187     this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
4188         HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, log,
4189         new byte[][] { family });
4190 
4191     Put put = new Put(Bytes.toBytes("r1"));
4192     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
4193     put.setDurability(mutationDurability);
4194     region.put(put);
4195 
4196     //verify append called or not
4197     verify(log, expectAppend ? times(1) : never())
4198       .appendNoSync((HRegionInfo)any(), eq(tableName), (WALEdit)any(), (List<UUID>)any(),
4199         anyLong(), (HTableDescriptor)any(), (AtomicLong)any(), anyBoolean(), anyLong(), anyLong());
4200 
4201     // verify sync called or not
4202     if (expectSync || expectSyncFromLogSyncer) {
4203       TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
4204         @Override
4205         public boolean evaluate() throws Exception {
4206           try {
4207             if (expectSync) {
4208               verify(log, times(1)).sync(anyLong()); // Hregion calls this one
4209             } else if (expectSyncFromLogSyncer) {
4210               verify(log, times(1)).sync(); // log syncer calls this one
4211             }
4212           } catch (Throwable ignore) {
4213           }
4214           return true;
4215         }
4216       });
4217     } else {
4218       verify(log, never()).sync(anyLong());
4219       verify(log, never()).sync();
4220     }
4221 
4222     HRegion.closeHRegion(this.region);
4223     this.region = null;
4224   }
4225 
4226   private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4227     for (int i = startRow; i < startRow + numRows; i++) {
4228       Put put = new Put(Bytes.toBytes("" + i));
4229       put.setDurability(Durability.SKIP_WAL);
4230       for (byte[] family : families) {
4231         put.add(family, qf, null);
4232       }
4233       region.put(put);
4234     }
4235   }
4236 
4237   private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
4238       throws IOException {
4239     for (int i = startRow; i < startRow + numRows; i++) {
4240       byte[] row = Bytes.toBytes("" + i);
4241       Get get = new Get(row);
4242       for (byte[] family : families) {
4243         get.addColumn(family, qf);
4244       }
4245       Result result = newReg.get(get);
4246       Cell[] raw = result.rawCells();
4247       assertEquals(families.length, result.size());
4248       for (int j = 0; j < families.length; j++) {
4249         assertTrue(CellUtil.matchingRow(raw[j], row));
4250         assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
4251         assertTrue(CellUtil.matchingQualifier(raw[j], qf));
4252       }
4253     }
4254   }
4255 
4256   private void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
4257     // Now I have k, get values out and assert they are as expected.
4258     Get get = new Get(k).addFamily(family).setMaxVersions();
4259     Cell[] results = r.get(get).rawCells();
4260     for (int j = 0; j < results.length; j++) {
4261       byte[] tmp = CellUtil.cloneValue(results[j]);
4262       // Row should be equal to value every time.
4263       assertTrue(Bytes.equals(k, tmp));
4264     }
4265   }
4266 
4267   /*
4268    * Assert first value in the passed region is <code>firstValue</code>.
4269    * 
4270    * @param r
4271    * 
4272    * @param fs
4273    * 
4274    * @param firstValue
4275    * 
4276    * @throws IOException
4277    */
4278   private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
4279       throws IOException {
4280     byte[][] families = { fs };
4281     Scan scan = new Scan();
4282     for (int i = 0; i < families.length; i++)
4283       scan.addFamily(families[i]);
4284     InternalScanner s = r.getScanner(scan);
4285     try {
4286       List<Cell> curVals = new ArrayList<Cell>();
4287       boolean first = true;
4288       OUTER_LOOP: while (s.next(curVals)) {
4289         for (Cell kv : curVals) {
4290           byte[] val = CellUtil.cloneValue(kv);
4291           byte[] curval = val;
4292           if (first) {
4293             first = false;
4294             assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4295           } else {
4296             // Not asserting anything. Might as well break.
4297             break OUTER_LOOP;
4298           }
4299         }
4300       }
4301     } finally {
4302       s.close();
4303     }
4304   }
4305 
4306   /**
4307    * Test that we get the expected flush results back
4308    * @throws IOException
4309    */
4310   @Test
4311   public void testFlushResult() throws IOException {
4312     String method = name.getMethodName();
4313     byte[] tableName = Bytes.toBytes(method);
4314     byte[] family = Bytes.toBytes("family");
4315 
4316     this.region = initHRegion(tableName, method, family);
4317 
4318     // empty memstore, flush doesn't run
4319     HRegion.FlushResult fr = region.flushcache();
4320     assertFalse(fr.isFlushSucceeded());
4321     assertFalse(fr.isCompactionNeeded());
4322 
4323     // Flush enough files to get up to the threshold, doesn't need compactions
4324     for (int i = 0; i < 2; i++) {
4325       Put put = new Put(tableName).add(family, family, tableName);
4326       region.put(put);
4327       fr = region.flushcache();
4328       assertTrue(fr.isFlushSucceeded());
4329       assertFalse(fr.isCompactionNeeded());
4330     }
4331 
4332     // Two flushes after the threshold, compactions are needed
4333     for (int i = 0; i < 2; i++) {
4334       Put put = new Put(tableName).add(family, family, tableName);
4335       region.put(put);
4336       fr = region.flushcache();
4337       assertTrue(fr.isFlushSucceeded());
4338       assertTrue(fr.isCompactionNeeded());
4339     }
4340   }
4341 
4342   private Configuration initSplit() {
4343     // Always compact if there is more than one store file.
4344     CONF.setInt("hbase.hstore.compactionThreshold", 2);
4345 
4346     // Make lease timeout longer, lease checks less frequent
4347     CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4348 
4349     CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4350 
4351     // Increase the amount of time between client retries
4352     CONF.setLong("hbase.client.pause", 15 * 1000);
4353 
4354     // This size should make it so we always split using the addContent
4355     // below. After adding all data, the first region is 1.3M
4356     CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4357     return CONF;
4358   }
4359 
4360   /**
4361    * @param tableName
4362    * @param callingMethod
4363    * @param conf
4364    * @param families
4365    * @throws IOException
4366    * @return A region on which you must call
4367    *         {@link HRegion#closeHRegion(HRegion)} when done.
4368    */
4369   public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
4370       byte[]... families) throws IOException {
4371     return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
4372   }
4373 
4374   /**
4375    * @param tableName
4376    * @param callingMethod
4377    * @param conf
4378    * @param families
4379    * @throws IOException
4380    * @return A region on which you must call
4381    *         {@link HRegion#closeHRegion(HRegion)} when done.
4382    */
4383   public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4384       byte[]... families) throws IOException {
4385     return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4386   }
4387 
4388   /**
4389    * @param tableName
4390    * @param callingMethod
4391    * @param conf
4392    * @param isReadOnly
4393    * @param families
4394    * @throws IOException
4395    * @return A region on which you must call
4396    *         {@link HRegion#closeHRegion(HRegion)} when done.
4397    */
4398   public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4399       boolean isReadOnly, byte[]... families) throws IOException {
4400     return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4401   }
4402 
4403   private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4404       String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4405       throws IOException {
4406     return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
4407         Durability.SYNC_WAL, null, families);
4408   }
4409 
4410   /**
4411    * @param tableName
4412    * @param startKey
4413    * @param stopKey
4414    * @param callingMethod
4415    * @param conf
4416    * @param isReadOnly
4417    * @param families
4418    * @throws IOException
4419    * @return A region on which you must call
4420    *         {@link HRegion#closeHRegion(HRegion)} when done.
4421    */
4422   private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4423       String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4424       HLog hlog, byte[]... families) throws IOException {
4425     return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly, durability, hlog, families);
4426   }
4427 
4428   /**
4429    * Assert that the passed in Cell has expected contents for the specified row,
4430    * column & timestamp.
4431    */
4432   private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
4433     String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4434     assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
4435         Bytes.toString(CellUtil.cloneRow(kv)));
4436     assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
4437         Bytes.toString(CellUtil.cloneFamily(kv)));
4438     assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
4439         Bytes.toString(CellUtil.cloneQualifier(kv)));
4440     assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
4441     assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
4442         Bytes.toString(CellUtil.cloneValue(kv)));
4443   }
4444 
4445   public void testReverseScanner_FromMemStore_SingleCF_Normal()
4446       throws IOException {
4447     byte[] rowC = Bytes.toBytes("rowC");
4448     byte[] rowA = Bytes.toBytes("rowA");
4449     byte[] rowB = Bytes.toBytes("rowB");
4450     byte[] cf = Bytes.toBytes("CF");
4451     byte[][] families = { cf };
4452     byte[] col = Bytes.toBytes("C");
4453     long ts = 1;
4454     String method = this.getName();
4455     this.region = initHRegion(tableName, method, families);
4456     try {
4457       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4458       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4459           null);
4460       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4461       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4462       Put put = null;
4463       put = new Put(rowC);
4464       put.add(kv1);
4465       put.add(kv11);
4466       region.put(put);
4467       put = new Put(rowA);
4468       put.add(kv2);
4469       region.put(put);
4470       put = new Put(rowB);
4471       put.add(kv3);
4472       region.put(put);
4473 
4474       Scan scan = new Scan(rowC);
4475       scan.setMaxVersions(5);
4476       scan.setReversed(true);
4477       InternalScanner scanner = region.getScanner(scan);
4478       List<Cell> currRow = new ArrayList<Cell>();
4479       boolean hasNext = scanner.next(currRow);
4480       assertEquals(2, currRow.size());
4481       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4482       assertTrue(hasNext);
4483       currRow.clear();
4484       hasNext = scanner.next(currRow);
4485       assertEquals(1, currRow.size());
4486       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4487       assertTrue(hasNext);
4488       currRow.clear();
4489       hasNext = scanner.next(currRow);
4490       assertEquals(1, currRow.size());
4491       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4492       assertFalse(hasNext);
4493       scanner.close();
4494     } finally {
4495       HRegion.closeHRegion(this.region);
4496       this.region = null;
4497     }
4498   }
4499 
4500   public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
4501       throws IOException {
4502     byte[] rowC = Bytes.toBytes("rowC");
4503     byte[] rowA = Bytes.toBytes("rowA");
4504     byte[] rowB = Bytes.toBytes("rowB");
4505     byte[] rowD = Bytes.toBytes("rowD");
4506     byte[] cf = Bytes.toBytes("CF");
4507     byte[][] families = { cf };
4508     byte[] col = Bytes.toBytes("C");
4509     long ts = 1;
4510     String method = this.getName();
4511     this.region = initHRegion(tableName, method, families);
4512     try {
4513       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4514       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4515           null);
4516       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4517       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4518       Put put = null;
4519       put = new Put(rowC);
4520       put.add(kv1);
4521       put.add(kv11);
4522       region.put(put);
4523       put = new Put(rowA);
4524       put.add(kv2);
4525       region.put(put);
4526       put = new Put(rowB);
4527       put.add(kv3);
4528       region.put(put);
4529 
4530       Scan scan = new Scan(rowD);
4531       List<Cell> currRow = new ArrayList<Cell>();
4532       scan.setReversed(true);
4533       scan.setMaxVersions(5);
4534       InternalScanner scanner = region.getScanner(scan);
4535       boolean hasNext = scanner.next(currRow);
4536       assertEquals(2, currRow.size());
4537       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4538       assertTrue(hasNext);
4539       currRow.clear();
4540       hasNext = scanner.next(currRow);
4541       assertEquals(1, currRow.size());
4542       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4543       assertTrue(hasNext);
4544       currRow.clear();
4545       hasNext = scanner.next(currRow);
4546       assertEquals(1, currRow.size());
4547       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4548       assertFalse(hasNext);
4549       scanner.close();
4550     } finally {
4551       HRegion.closeHRegion(this.region);
4552       this.region = null;
4553     }
4554   }
4555 
4556   public void testReverseScanner_FromMemStore_SingleCF_FullScan()
4557       throws IOException {
4558     byte[] rowC = Bytes.toBytes("rowC");
4559     byte[] rowA = Bytes.toBytes("rowA");
4560     byte[] rowB = Bytes.toBytes("rowB");
4561     byte[] cf = Bytes.toBytes("CF");
4562     byte[][] families = { cf };
4563     byte[] col = Bytes.toBytes("C");
4564     long ts = 1;
4565     String method = this.getName();
4566     this.region = initHRegion(tableName, method, families);
4567     try {
4568       KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4569       KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4570           null);
4571       KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4572       KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4573       Put put = null;
4574       put = new Put(rowC);
4575       put.add(kv1);
4576       put.add(kv11);
4577       region.put(put);
4578       put = new Put(rowA);
4579       put.add(kv2);
4580       region.put(put);
4581       put = new Put(rowB);
4582       put.add(kv3);
4583       region.put(put);
4584       Scan scan = new Scan();
4585       List<Cell> currRow = new ArrayList<Cell>();
4586       scan.setReversed(true);
4587       InternalScanner scanner = region.getScanner(scan);
4588       boolean hasNext = scanner.next(currRow);
4589       assertEquals(1, currRow.size());
4590       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4591       assertTrue(hasNext);
4592       currRow.clear();
4593       hasNext = scanner.next(currRow);
4594       assertEquals(1, currRow.size());
4595       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4596       assertTrue(hasNext);
4597       currRow.clear();
4598       hasNext = scanner.next(currRow);
4599       assertEquals(1, currRow.size());
4600       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4601       assertFalse(hasNext);
4602       scanner.close();
4603     } finally {
4604       HRegion.closeHRegion(this.region);
4605       this.region = null;
4606     }
4607   }
4608 
4609   public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
4610     // case for "INCLUDE_AND_SEEK_NEXT_ROW & SEEK_NEXT_ROW" endless loop
4611     byte[] rowA = Bytes.toBytes("rowA");
4612     byte[] rowB = Bytes.toBytes("rowB");
4613     byte[] rowC = Bytes.toBytes("rowC");
4614     byte[] rowD = Bytes.toBytes("rowD");
4615     byte[] rowE = Bytes.toBytes("rowE");
4616     byte[] cf = Bytes.toBytes("CF");
4617     byte[][] families = { cf };
4618     byte[] col1 = Bytes.toBytes("col1");
4619     byte[] col2 = Bytes.toBytes("col2");
4620     long ts = 1;
4621     String method = this.getName();
4622     this.region = initHRegion(tableName, method, families);
4623     try {
4624       KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4625       KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4626       KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4627       KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4628       KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4629       KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4630       Put put = null;
4631       put = new Put(rowA);
4632       put.add(kv1);
4633       region.put(put);
4634       put = new Put(rowB);
4635       put.add(kv2);
4636       region.put(put);
4637       put = new Put(rowC);
4638       put.add(kv3);
4639       region.put(put);
4640       put = new Put(rowD);
4641       put.add(kv4_1);
4642       region.put(put);
4643       put = new Put(rowD);
4644       put.add(kv4_2);
4645       region.put(put);
4646       put = new Put(rowE);
4647       put.add(kv5);
4648       region.put(put);
4649       region.flushcache();
4650       Scan scan = new Scan(rowD, rowA);
4651       scan.addColumn(families[0], col1);
4652       scan.setReversed(true);
4653       List<Cell> currRow = new ArrayList<Cell>();
4654       InternalScanner scanner = region.getScanner(scan);
4655       boolean hasNext = scanner.next(currRow);
4656       assertEquals(1, currRow.size());
4657       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4658       assertTrue(hasNext);
4659       currRow.clear();
4660       hasNext = scanner.next(currRow);
4661       assertEquals(1, currRow.size());
4662       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4663       assertTrue(hasNext);
4664       currRow.clear();
4665       hasNext = scanner.next(currRow);
4666       assertEquals(1, currRow.size());
4667       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4668       assertFalse(hasNext);
4669       scanner.close();
4670 
4671       scan = new Scan(rowD, rowA);
4672       scan.addColumn(families[0], col2);
4673       scan.setReversed(true);
4674       currRow.clear();
4675       scanner = region.getScanner(scan);
4676       hasNext = scanner.next(currRow);
4677       assertEquals(1, currRow.size());
4678       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4679       scanner.close();
4680     } finally {
4681       HRegion.closeHRegion(this.region);
4682       this.region = null;
4683     }
4684   }
4685 
4686   public void testReverseScanner_smaller_blocksize() throws IOException {
4687     // case to ensure no conflict with HFile index optimization
4688     byte[] rowA = Bytes.toBytes("rowA");
4689     byte[] rowB = Bytes.toBytes("rowB");
4690     byte[] rowC = Bytes.toBytes("rowC");
4691     byte[] rowD = Bytes.toBytes("rowD");
4692     byte[] rowE = Bytes.toBytes("rowE");
4693     byte[] cf = Bytes.toBytes("CF");
4694     byte[][] families = { cf };
4695     byte[] col1 = Bytes.toBytes("col1");
4696     byte[] col2 = Bytes.toBytes("col2");
4697     long ts = 1;
4698     String method = this.getName();
4699     HBaseConfiguration config = new HBaseConfiguration();
4700     config.setInt("test.block.size", 1);
4701     this.region = initHRegion(tableName, method, config, families);
4702     try {
4703       KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4704       KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4705       KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4706       KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4707       KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4708       KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4709       Put put = null;
4710       put = new Put(rowA);
4711       put.add(kv1);
4712       region.put(put);
4713       put = new Put(rowB);
4714       put.add(kv2);
4715       region.put(put);
4716       put = new Put(rowC);
4717       put.add(kv3);
4718       region.put(put);
4719       put = new Put(rowD);
4720       put.add(kv4_1);
4721       region.put(put);
4722       put = new Put(rowD);
4723       put.add(kv4_2);
4724       region.put(put);
4725       put = new Put(rowE);
4726       put.add(kv5);
4727       region.put(put);
4728       region.flushcache();
4729       Scan scan = new Scan(rowD, rowA);
4730       scan.addColumn(families[0], col1);
4731       scan.setReversed(true);
4732       List<Cell> currRow = new ArrayList<Cell>();
4733       InternalScanner scanner = region.getScanner(scan);
4734       boolean hasNext = scanner.next(currRow);
4735       assertEquals(1, currRow.size());
4736       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4737       assertTrue(hasNext);
4738       currRow.clear();
4739       hasNext = scanner.next(currRow);
4740       assertEquals(1, currRow.size());
4741       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4742       assertTrue(hasNext);
4743       currRow.clear();
4744       hasNext = scanner.next(currRow);
4745       assertEquals(1, currRow.size());
4746       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4747       assertFalse(hasNext);
4748       scanner.close();
4749 
4750       scan = new Scan(rowD, rowA);
4751       scan.addColumn(families[0], col2);
4752       scan.setReversed(true);
4753       currRow.clear();
4754       scanner = region.getScanner(scan);
4755       hasNext = scanner.next(currRow);
4756       assertEquals(1, currRow.size());
4757       assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4758       scanner.close();
4759     } finally {
4760       HRegion.closeHRegion(this.region);
4761       this.region = null;
4762     }
4763   }
4764 
4765   public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
4766       throws IOException {
4767     byte[] row0 = Bytes.toBytes("row0"); // 1 kv
4768     byte[] row1 = Bytes.toBytes("row1"); // 2 kv
4769     byte[] row2 = Bytes.toBytes("row2"); // 4 kv
4770     byte[] row3 = Bytes.toBytes("row3"); // 2 kv
4771     byte[] row4 = Bytes.toBytes("row4"); // 5 kv
4772     byte[] row5 = Bytes.toBytes("row5"); // 2 kv
4773     byte[] cf1 = Bytes.toBytes("CF1");
4774     byte[] cf2 = Bytes.toBytes("CF2");
4775     byte[] cf3 = Bytes.toBytes("CF3");
4776     byte[][] families = { cf1, cf2, cf3 };
4777     byte[] col = Bytes.toBytes("C");
4778     long ts = 1;
4779     String method = this.getName();
4780     HBaseConfiguration conf = new HBaseConfiguration();
4781     // disable compactions in this test.
4782     conf.setInt("hbase.hstore.compactionThreshold", 10000);
4783     this.region = initHRegion(tableName, method, conf, families);
4784     try {
4785       // kv naming style: kv(row number) totalKvCountInThisRow seq no
4786       KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
4787           null);
4788       KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
4789           null);
4790       KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
4791           KeyValue.Type.Put, null);
4792       KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
4793           null);
4794       KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
4795           null);
4796       KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
4797           null);
4798       KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
4799           KeyValue.Type.Put, null);
4800       KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
4801           null);
4802       KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
4803           KeyValue.Type.Put, null);
4804       KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
4805           null);
4806       KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
4807           null);
4808       KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
4809           KeyValue.Type.Put, null);
4810       KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
4811           null);
4812       KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
4813           KeyValue.Type.Put, null);
4814       KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
4815           null);
4816       KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
4817           null);
4818       // hfiles(cf1/cf2) :"row1"(1 kv) / "row2"(1 kv) / "row4"(2 kv)
4819       Put put = null;
4820       put = new Put(row1);
4821       put.add(kv1_2_1);
4822       region.put(put);
4823       put = new Put(row2);
4824       put.add(kv2_4_1);
4825       region.put(put);
4826       put = new Put(row4);
4827       put.add(kv4_5_4);
4828       put.add(kv4_5_5);
4829       region.put(put);
4830       region.flushcache();
4831       // hfiles(cf1/cf3) : "row1" (1 kvs) / "row2" (1 kv) / "row4" (2 kv)
4832       put = new Put(row4);
4833       put.add(kv4_5_1);
4834       put.add(kv4_5_3);
4835       region.put(put);
4836       put = new Put(row1);
4837       put.add(kv1_2_2);
4838       region.put(put);
4839       put = new Put(row2);
4840       put.add(kv2_4_4);
4841       region.put(put);
4842       region.flushcache();
4843       // hfiles(cf1/cf3) : "row2"(2 kv) / "row3"(1 kvs) / "row4" (1 kv)
4844       put = new Put(row4);
4845       put.add(kv4_5_2);
4846       region.put(put);
4847       put = new Put(row2);
4848       put.add(kv2_4_2);
4849       put.add(kv2_4_3);
4850       region.put(put);
4851       put = new Put(row3);
4852       put.add(kv3_2_2);
4853       region.put(put);
4854       region.flushcache();
4855       // memstore(cf1/cf2/cf3) : "row0" (1 kvs) / "row3" ( 1 kv) / "row5" (max)
4856       // ( 2 kv)
4857       put = new Put(row0);
4858       put.add(kv0_1_1);
4859       region.put(put);
4860       put = new Put(row3);
4861       put.add(kv3_2_1);
4862       region.put(put);
4863       put = new Put(row5);
4864       put.add(kv5_2_1);
4865       put.add(kv5_2_2);
4866       region.put(put);
4867       // scan range = ["row4", min), skip the max "row5"
4868       Scan scan = new Scan(row4);
4869       scan.setMaxVersions(5);
4870       scan.setBatch(3);
4871       scan.setReversed(true);
4872       InternalScanner scanner = region.getScanner(scan);
4873       List<Cell> currRow = new ArrayList<Cell>();
4874       boolean hasNext = false;
4875       // 1. scan out "row4" (5 kvs), "row5" can't be scanned out since not
4876       // included in scan range
4877       // "row4" takes 2 next() calls since batch=3
4878       hasNext = scanner.next(currRow);
4879       assertEquals(3, currRow.size());
4880       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4881       assertTrue(hasNext);
4882       currRow.clear();
4883       hasNext = scanner.next(currRow);
4884       assertEquals(2, currRow.size());
4885       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4886       assertTrue(hasNext);
4887       // 2. scan out "row3" (2 kv)
4888       currRow.clear();
4889       hasNext = scanner.next(currRow);
4890       assertEquals(2, currRow.size());
4891       assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
4892       assertTrue(hasNext);
4893       // 3. scan out "row2" (4 kvs)
4894       // "row2" takes 2 next() calls since batch=3
4895       currRow.clear();
4896       hasNext = scanner.next(currRow);
4897       assertEquals(3, currRow.size());
4898       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4899       assertTrue(hasNext);
4900       currRow.clear();
4901       hasNext = scanner.next(currRow);
4902       assertEquals(1, currRow.size());
4903       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4904       assertTrue(hasNext);
4905       // 4. scan out "row1" (2 kv)
4906       currRow.clear();
4907       hasNext = scanner.next(currRow);
4908       assertEquals(2, currRow.size());
4909       assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
4910       assertTrue(hasNext);
4911       // 5. scan out "row0" (1 kv)
4912       currRow.clear();
4913       hasNext = scanner.next(currRow);
4914       assertEquals(1, currRow.size());
4915       assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
4916       assertFalse(hasNext);
4917 
4918       scanner.close();
4919     } finally {
4920       HRegion.closeHRegion(this.region);
4921       this.region = null;
4922     }
4923   }
4924 
4925   public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
4926       throws IOException {
4927     byte[] row1 = Bytes.toBytes("row1");
4928     byte[] row2 = Bytes.toBytes("row2");
4929     byte[] row3 = Bytes.toBytes("row3");
4930     byte[] row4 = Bytes.toBytes("row4");
4931     byte[] cf1 = Bytes.toBytes("CF1");
4932     byte[] cf2 = Bytes.toBytes("CF2");
4933     byte[] cf3 = Bytes.toBytes("CF3");
4934     byte[] cf4 = Bytes.toBytes("CF4");
4935     byte[][] families = { cf1, cf2, cf3, cf4 };
4936     byte[] col = Bytes.toBytes("C");
4937     long ts = 1;
4938     String method = this.getName();
4939     HBaseConfiguration conf = new HBaseConfiguration();
4940     // disable compactions in this test.
4941     conf.setInt("hbase.hstore.compactionThreshold", 10000);
4942     this.region = initHRegion(tableName, method, conf, families);
4943     try {
4944       KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
4945       KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
4946       KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
4947       KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
4948       // storefile1
4949       Put put = new Put(row1);
4950       put.add(kv1);
4951       region.put(put);
4952       region.flushcache();
4953       // storefile2
4954       put = new Put(row2);
4955       put.add(kv2);
4956       region.put(put);
4957       region.flushcache();
4958       // storefile3
4959       put = new Put(row3);
4960       put.add(kv3);
4961       region.put(put);
4962       region.flushcache();
4963       // memstore
4964       put = new Put(row4);
4965       put.add(kv4);
4966       region.put(put);
4967       // scan range = ["row4", min)
4968       Scan scan = new Scan(row4);
4969       scan.setReversed(true);
4970       scan.setBatch(10);
4971       InternalScanner scanner = region.getScanner(scan);
4972       List<Cell> currRow = new ArrayList<Cell>();
4973       boolean hasNext = scanner.next(currRow);
4974       assertEquals(1, currRow.size());
4975       assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4976       assertTrue(hasNext);
4977       currRow.clear();
4978       hasNext = scanner.next(currRow);
4979       assertEquals(1, currRow.size());
4980       assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
4981       assertTrue(hasNext);
4982       currRow.clear();
4983       hasNext = scanner.next(currRow);
4984       assertEquals(1, currRow.size());
4985       assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4986       assertTrue(hasNext);
4987       currRow.clear();
4988       hasNext = scanner.next(currRow);
4989       assertEquals(1, currRow.size());
4990       assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
4991       assertFalse(hasNext);
4992     } finally {
4993       HRegion.closeHRegion(this.region);
4994       this.region = null;
4995     }
4996   }
4997 
4998   @Test
4999   public void testWriteRequestsCounter() throws IOException {
5000     byte[] fam = Bytes.toBytes("info");
5001     byte[][] families = { fam };
5002     this.region = initHRegion(tableName, method, CONF, families);
5003 
5004     Assert.assertEquals(0L, region.getWriteRequestsCount());
5005 
5006     Put put = new Put(row);
5007     put.add(fam, fam, fam);
5008 
5009     Assert.assertEquals(0L, region.getWriteRequestsCount());
5010     region.put(put);
5011     Assert.assertEquals(1L, region.getWriteRequestsCount());
5012     region.put(put);
5013     Assert.assertEquals(2L, region.getWriteRequestsCount());
5014     region.put(put);
5015     Assert.assertEquals(3L, region.getWriteRequestsCount());
5016 
5017     region.delete(new Delete(row));
5018     Assert.assertEquals(4L, region.getWriteRequestsCount());
5019 
5020     HRegion.closeHRegion(this.region);
5021     this.region = null;
5022   }
5023 
5024   /**
5025    * Test RegionTooBusyException thrown when region is busy
5026    */
5027   @Test (timeout=24000)
5028   public void testRegionTooBusy() throws IOException {
5029     String method = "testRegionTooBusy";
5030     byte[] tableName = Bytes.toBytes(method);
5031     byte[] family = Bytes.toBytes("family");
5032     long defaultBusyWaitDuration = CONF.getLong("hbase.busy.wait.duration",
5033       HRegion.DEFAULT_BUSY_WAIT_DURATION);
5034     CONF.setLong("hbase.busy.wait.duration", 1000);
5035     region = initHRegion(tableName, method, CONF, family);
5036     final AtomicBoolean stopped = new AtomicBoolean(true);
5037     Thread t = new Thread(new Runnable() {
5038       @Override
5039       public void run() {
5040         try {
5041           region.lock.writeLock().lock();
5042           stopped.set(false);
5043           while (!stopped.get()) {
5044             Thread.sleep(100);
5045           }
5046         } catch (InterruptedException ie) {
5047         } finally {
5048           region.lock.writeLock().unlock();
5049         }
5050       }
5051     });
5052     t.start();
5053     Get get = new Get(row);
5054     try {
5055       while (stopped.get()) {
5056         Thread.sleep(100);
5057       }
5058       region.get(get);
5059       fail("Should throw RegionTooBusyException");
5060     } catch (InterruptedException ie) {
5061       fail("test interrupted");
5062     } catch (RegionTooBusyException e) {
5063       // Good, expected
5064     } finally {
5065       stopped.set(true);
5066       try {
5067         t.join();
5068       } catch (Throwable e) {
5069       }
5070 
5071       HRegion.closeHRegion(region);
5072       region = null;
5073       CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration);
5074     }
5075   }
5076 
5077   private static HRegion initHRegion(byte[] tableName, String callingMethod,
5078       byte[]... families) throws IOException {
5079     return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
5080         families);
5081   }
5082 }