View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import static org.mockito.Matchers.any;
22  import static org.mockito.Matchers.anyLong;
23  import static org.mockito.Matchers.eq;
24  import static org.mockito.Mockito.never;
25  import static org.mockito.Mockito.spy;
26  import static org.mockito.Mockito.times;
27  import static org.mockito.Mockito.verify;
28  
29  import java.io.IOException;
30  import java.io.InterruptedIOException;
31  import java.util.ArrayList;
32  import java.util.Arrays;
33  import java.util.Collection;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.NavigableMap;
37  import java.util.TreeMap;
38  import java.util.UUID;
39  import java.util.concurrent.atomic.AtomicBoolean;
40  import java.util.concurrent.atomic.AtomicInteger;
41  import java.util.concurrent.atomic.AtomicReference;
42  
43  import org.apache.commons.logging.Log;
44  import org.apache.commons.logging.LogFactory;
45  import org.apache.hadoop.conf.Configuration;
46  import org.apache.hadoop.fs.FSDataOutputStream;
47  import org.apache.hadoop.fs.FileStatus;
48  import org.apache.hadoop.fs.FileSystem;
49  import org.apache.hadoop.fs.Path;
50  import org.apache.hadoop.hbase.Cell;
51  import org.apache.hadoop.hbase.CellComparator;
52  import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
53  import org.apache.hadoop.hbase.TableName;
54  import org.apache.hadoop.hbase.HBaseConfiguration;
55  import org.apache.hadoop.hbase.HBaseTestCase;
56  import org.apache.hadoop.hbase.HBaseTestingUtility;
57  import org.apache.hadoop.hbase.HColumnDescriptor;
58  import org.apache.hadoop.hbase.HConstants;
59  import org.apache.hadoop.hbase.NotServingRegionException;
60  import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
61  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
62  import org.apache.hadoop.hbase.HRegionInfo;
63  import org.apache.hadoop.hbase.HTableDescriptor;
64  import org.apache.hadoop.hbase.KeyValue;
65  import org.apache.hadoop.hbase.MediumTests;
66  import org.apache.hadoop.hbase.MiniHBaseCluster;
67  import org.apache.hadoop.hbase.MultithreadedTestUtil;
68  import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
69  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
70  import org.apache.hadoop.hbase.Waiter;
71  import org.apache.hadoop.hbase.client.Append;
72  import org.apache.hadoop.hbase.client.Delete;
73  import org.apache.hadoop.hbase.client.Durability;
74  import org.apache.hadoop.hbase.client.Get;
75  import org.apache.hadoop.hbase.client.HTable;
76  import org.apache.hadoop.hbase.client.Increment;
77  import org.apache.hadoop.hbase.client.Put;
78  import org.apache.hadoop.hbase.client.Result;
79  import org.apache.hadoop.hbase.client.Scan;
80  import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
81  import org.apache.hadoop.hbase.filter.BinaryComparator;
82  import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
83  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
84  import org.apache.hadoop.hbase.filter.Filter;
85  import org.apache.hadoop.hbase.filter.FilterBase;
86  import org.apache.hadoop.hbase.filter.FilterList;
87  import org.apache.hadoop.hbase.filter.NullComparator;
88  import org.apache.hadoop.hbase.filter.PrefixFilter;
89  import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
90  import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
91  import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
92  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
93  import org.apache.hadoop.hbase.monitoring.TaskMonitor;
94  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
95  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
96  import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
97  import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
98  import org.apache.hadoop.hbase.regionserver.wal.HLog;
99  import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
100 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
101 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
102 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
103 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
104 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
105 import org.apache.hadoop.hbase.util.Bytes;
106 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
107 import org.apache.hadoop.hbase.util.FSUtils;
108 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
109 import org.apache.hadoop.hbase.util.PairOfSameType;
110 import org.apache.hadoop.hbase.util.Threads;
111 import org.junit.Assert;
112 import org.junit.Test;
113 import org.junit.experimental.categories.Category;
114 import org.mockito.Mockito;
115 
116 import com.google.common.collect.Lists;
117 
118 /**
119  * Basic stand-alone testing of HRegion.
120  *
121  * A lot of the meta information for an HRegion now lives inside other
122  * HRegions or in the HBaseMaster, so only basic testing is possible.
123  */
124 @Category(MediumTests.class)
125 @SuppressWarnings("deprecation")
126 public class TestHRegion extends HBaseTestCase {
127   // Do not spin up clusters in here.  If you need to spin up a cluster, do it
128   // over in TestHRegionOnCluster.
129   static final Log LOG = LogFactory.getLog(TestHRegion.class);
130 
131   private static final String COLUMN_FAMILY = "MyCF";
132 
133   HRegion region = null;
134   private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
135   private static final String DIR = TEST_UTIL.getDataTestDir("TestHRegion").toString();
136 
137   private final int MAX_VERSIONS = 2;
138 
139   // Test names
140   protected final byte[] tableName = Bytes.toBytes("testtable");;
141   protected final byte[] qual1 = Bytes.toBytes("qual1");
142   protected final byte[] qual2 = Bytes.toBytes("qual2");
143   protected final byte[] qual3 = Bytes.toBytes("qual3");
144   protected final byte[] value1 = Bytes.toBytes("value1");
145   protected final byte[] value2 = Bytes.toBytes("value2");
146   protected final byte [] row = Bytes.toBytes("rowA");
147   protected final byte [] row2 = Bytes.toBytes("rowB");
148 
149   protected final MetricsAssertHelper metricsAssertHelper =
150     CompatibilitySingletonFactory.getInstance(MetricsAssertHelper.class);
151 
152   /**
153    * @see org.apache.hadoop.hbase.HBaseTestCase#setUp()
154    */
155   @Override
156   protected void setUp() throws Exception {
157     super.setUp();
158   }
159 
160   @Override
161   protected void tearDown() throws Exception {
162     super.tearDown();
163     EnvironmentEdgeManagerTestHelper.reset();
164   }
165 
166   //////////////////////////////////////////////////////////////////////////////
167   // New tests that doesn't spin up a mini cluster but rather just test the
168   // individual code pieces in the HRegion. Putting files locally in
169   // /tmp/testtable
170   //////////////////////////////////////////////////////////////////////////////
171 
172   public void testCompactionAffectedByScanners() throws Exception {
173     String method = "testCompactionAffectedByScanners";
174     byte[] tableName = Bytes.toBytes(method);
175     byte[] family = Bytes.toBytes("family");
176     this.region = initHRegion(tableName, method, conf, family);
177 
178     Put put = new Put(Bytes.toBytes("r1"));
179     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
180     region.put(put);
181     region.flushcache();
182 
183 
184     Scan scan = new Scan();
185     scan.setMaxVersions(3);
186     // open the first scanner
187     RegionScanner scanner1 = region.getScanner(scan);
188 
189     Delete delete = new Delete(Bytes.toBytes("r1"));
190     region.delete(delete);
191     region.flushcache();
192 
193     // open the second scanner
194     RegionScanner scanner2 = region.getScanner(scan);
195 
196     List<KeyValue> results = new ArrayList<KeyValue>();
197 
198     System.out.println("Smallest read point:" + region.getSmallestReadPoint());
199 
200     // make a major compaction
201     region.compactStores(true);
202 
203     // open the third scanner
204     RegionScanner scanner3 = region.getScanner(scan);
205 
206     // get data from scanner 1, 2, 3 after major compaction
207     scanner1.next(results);
208     System.out.println(results);
209     assertEquals(1, results.size());
210 
211     results.clear();
212     scanner2.next(results);
213     System.out.println(results);
214     assertEquals(0, results.size());
215 
216     results.clear();
217     scanner3.next(results);
218     System.out.println(results);
219     assertEquals(0, results.size());
220   }
221 
222   @Test
223   public void testToShowNPEOnRegionScannerReseek() throws Exception{
224     String method = "testToShowNPEOnRegionScannerReseek";
225     byte[] tableName = Bytes.toBytes(method);
226     byte[] family = Bytes.toBytes("family");
227     this.region = initHRegion(tableName, method, conf, family);
228 
229     Put put = new Put(Bytes.toBytes("r1"));
230     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
231     region.put(put);
232     put = new Put(Bytes.toBytes("r2"));
233     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
234     region.put(put);
235     region.flushcache();
236 
237 
238     Scan scan = new Scan();
239     scan.setMaxVersions(3);
240     // open the first scanner
241     RegionScanner scanner1 = region.getScanner(scan);
242 
243     System.out.println("Smallest read point:" + region.getSmallestReadPoint());
244 
245     region.compactStores(true);
246 
247     scanner1.reseek(Bytes.toBytes("r2"));
248     List<KeyValue> results = new ArrayList<KeyValue>();
249     scanner1.next(results);
250     KeyValue keyValue = results.get(0);
251     Assert.assertTrue(Bytes.compareTo(keyValue.getRow(), Bytes.toBytes("r2")) == 0);
252     scanner1.close();
253   }
254 
255   public void testSkipRecoveredEditsReplay() throws Exception {
256     String method = "testSkipRecoveredEditsReplay";
257     TableName tableName =
258         TableName.valueOf(method);
259     byte[] family = Bytes.toBytes("family");
260     this.region = initHRegion(tableName, method, conf, family);
261     try {
262       Path regiondir = region.getRegionFileSystem().getRegionDir();
263       FileSystem fs = region.getRegionFileSystem().getFileSystem();
264       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
265 
266       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
267 
268       long maxSeqId = 1050;
269       long minSeqId = 1000;
270 
271       for (long i = minSeqId; i <= maxSeqId; i += 10) {
272         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
273         fs.create(recoveredEdits);
274         HLog.Writer writer = HLogFactory.createWriter(fs, recoveredEdits, conf);
275 
276         long time = System.nanoTime();
277         WALEdit edit = new WALEdit();
278         edit.add(new KeyValue(row, family, Bytes.toBytes(i),
279             time, KeyValue.Type.Put, Bytes.toBytes(i)));
280         writer.append(new HLog.Entry(new HLogKey(regionName, tableName,
281             i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
282 
283         writer.close();
284       }
285       MonitoredTask status = TaskMonitor.get().createStatus(method);
286       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(
287           Bytes.BYTES_COMPARATOR);
288       for (Store store : region.getStores().values()) {
289         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
290       }
291       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
292       assertEquals(maxSeqId, seqId);
293       Get get = new Get(row);
294       Result result = region.get(get);
295       for (long i = minSeqId; i <= maxSeqId; i += 10) {
296         List<KeyValue> kvs = result.getColumn(family, Bytes.toBytes(i));
297         assertEquals(1, kvs.size());
298         assertEquals(Bytes.toBytes(i), kvs.get(0).getValue());
299       }
300     } finally {
301       HRegion.closeHRegion(this.region);
302       this.region = null;
303     }
304   }
305 
306   public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
307     String method = "testSkipRecoveredEditsReplaySomeIgnored";
308     TableName tableName =
309         TableName.valueOf(method);
310     byte[] family = Bytes.toBytes("family");
311     this.region = initHRegion(tableName, method, conf, family);
312     try {
313       Path regiondir = region.getRegionFileSystem().getRegionDir();
314       FileSystem fs = region.getRegionFileSystem().getFileSystem();
315       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
316 
317       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
318 
319       long maxSeqId = 1050;
320       long minSeqId = 1000;
321 
322       for (long i = minSeqId; i <= maxSeqId; i += 10) {
323         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
324         fs.create(recoveredEdits);
325         HLog.Writer writer = HLogFactory.createWriter(fs, recoveredEdits, conf);
326 
327         long time = System.nanoTime();
328         WALEdit edit = new WALEdit();
329         edit.add(new KeyValue(row, family, Bytes.toBytes(i),
330             time, KeyValue.Type.Put, Bytes.toBytes(i)));
331         writer.append(new HLog.Entry(new HLogKey(regionName, tableName,
332             i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
333 
334         writer.close();
335       }
336       long recoverSeqId = 1030;
337       MonitoredTask status = TaskMonitor.get().createStatus(method);
338       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(
339           Bytes.BYTES_COMPARATOR);
340       for (Store store : region.getStores().values()) {
341         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(),
342             recoverSeqId - 1);
343       }
344       long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
345       assertEquals(maxSeqId, seqId);
346       Get get = new Get(row);
347       Result result = region.get(get);
348       for (long i = minSeqId; i <= maxSeqId; i += 10) {
349         List<KeyValue> kvs = result.getColumn(family, Bytes.toBytes(i));
350         if (i < recoverSeqId) {
351           assertEquals(0, kvs.size());
352         } else {
353           assertEquals(1, kvs.size());
354           assertEquals(Bytes.toBytes(i), kvs.get(0).getValue());
355         }
356       }
357     } finally {
358       HRegion.closeHRegion(this.region);
359       this.region = null;
360     }
361   }
362 
363   public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
364     String method = "testSkipRecoveredEditsReplayAllIgnored";
365     byte[] tableName = Bytes.toBytes(method);
366     byte[] family = Bytes.toBytes("family");
367     this.region = initHRegion(tableName, method, conf, family);
368     try {
369       Path regiondir = region.getRegionFileSystem().getRegionDir();
370       FileSystem fs = region.getRegionFileSystem().getFileSystem();
371 
372       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
373       for (int i = 1000; i < 1050; i += 10) {
374         Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
375         FSDataOutputStream dos=  fs.create(recoveredEdits);
376         dos.writeInt(i);
377         dos.close();
378       }
379       long minSeqId = 2000;
380       Path recoveredEdits = new Path(
381           recoveredEditsDir, String.format("%019d", minSeqId-1));
382       FSDataOutputStream dos=  fs.create(recoveredEdits);
383       dos.close();
384 
385       Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(
386         Bytes.BYTES_COMPARATOR);
387       for (Store store : region.getStores().values()) {
388         maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
389       }
390       long seqId = region.replayRecoveredEditsIfAny(regiondir,
391           maxSeqIdInStores, null, null);
392       assertEquals(minSeqId, seqId);
393     } finally {
394       HRegion.closeHRegion(this.region);
395       this.region = null;
396     }
397   }
398 
399   @Test
400   public void testRecoveredEditsReplayCompaction() throws Exception {
401     String method = "testRecoveredEditsReplayCompaction";
402     TableName tableName =
403         TableName.valueOf(method);
404     byte[] family = Bytes.toBytes("family");
405     this.region = initHRegion(tableName, method, conf, family);
406     try {
407       Path regiondir = region.getRegionFileSystem().getRegionDir();
408       FileSystem fs = region.getRegionFileSystem().getFileSystem();
409       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
410 
411       long maxSeqId = 3;
412       long minSeqId = 0;
413 
414       for (long i = minSeqId; i < maxSeqId; i++) {
415         Put put = new Put(Bytes.toBytes(i));
416         put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
417         region.put(put);
418         region.flushcache();
419       }
420 
421       //this will create a region with 3 files
422       assertEquals(3, region.getStore(family).getStorefilesCount());
423       List<Path> storeFiles = new ArrayList<Path>(3);
424       for (StoreFile sf : region.getStore(family).getStorefiles()) {
425         storeFiles.add(sf.getPath());
426       }
427 
428       //disable compaction completion
429       conf.setBoolean("hbase.hstore.compaction.complete",false);
430       region.compactStores();
431 
432       //ensure that nothing changed
433       assertEquals(3, region.getStore(family).getStorefilesCount());
434 
435       //now find the compacted file, and manually add it to the recovered edits
436       Path tmpDir = region.getRegionFileSystem().getTempDir();
437       FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
438       String errorMsg = "Expected to find 1 file in the region temp directory " +
439           "from the compaction, could not find any";
440       assertNotNull(errorMsg, files);
441       assertEquals(errorMsg, 1, files.length);
442       //move the file inside region dir
443       Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family), files[0].getPath());
444 
445       CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(
446           this.region.getRegionInfo(), family,
447           storeFiles, Lists.newArrayList(newFile),
448           region.getRegionFileSystem().getStoreDir(Bytes.toString(family)));
449 
450       HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
451           this.region.getRegionInfo(), compactionDescriptor);
452 
453       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
454 
455       Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
456       fs.create(recoveredEdits);
457       HLog.Writer writer = HLogFactory.createWriter(fs, recoveredEdits, conf);
458 
459       long time = System.nanoTime();
460 
461       writer.append(new HLog.Entry(new HLogKey(regionName, tableName, 10, time, HConstants.DEFAULT_CLUSTER_ID),
462           WALEdit.createCompaction(compactionDescriptor)));
463       writer.close();
464 
465       //close the region now, and reopen again
466       HTableDescriptor htd = region.getTableDesc();
467       HRegionInfo info = region.getRegionInfo();
468       region.close();
469       region = HRegion.openHRegion(conf, fs, new Path(DIR+method),info, htd, null);
470 
471       //now check whether we have only one store file, the compacted one
472       Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
473       for (StoreFile sf : sfs) {
474         LOG.info(sf.getPath());
475       }
476       assertEquals(1, region.getStore(family).getStorefilesCount());
477       files = FSUtils.listStatus(fs, tmpDir);
478       assertTrue("Expected to find 0 files inside " + tmpDir,
479         files == null || files.length == 0);
480 
481       for (long i = minSeqId; i < maxSeqId; i++) {
482         Get get = new Get(Bytes.toBytes(i));
483         Result result = region.get(get);
484         byte[] value = result.getValue(family, Bytes.toBytes(i));
485         assertEquals(Bytes.toBytes(i), value);
486       }
487     } finally {
488       HRegion.closeHRegion(this.region);
489       this.region = null;
490     }
491   }
492 
493   public void testGetWhileRegionClose() throws IOException {
494     Configuration hc = initSplit();
495     int numRows = 100;
496     byte [][] families = {fam1, fam2, fam3};
497 
498     //Setting up region
499     String method = this.getName();
500     this.region = initHRegion(tableName, method, hc, families);
501     try {
502       // Put data in region
503       final int startRow = 100;
504       putData(startRow, numRows, qual1, families);
505       putData(startRow, numRows, qual2, families);
506       putData(startRow, numRows, qual3, families);
507       // this.region.flushcache();
508       final AtomicBoolean done = new AtomicBoolean(false);
509       final AtomicInteger gets = new AtomicInteger(0);
510       GetTillDoneOrException [] threads = new GetTillDoneOrException[10];
511       try {
512         // Set ten threads running concurrently getting from the region.
513         for (int i = 0; i < threads.length / 2; i++) {
514           threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow),
515               done, gets);
516           threads[i].setDaemon(true);
517           threads[i].start();
518         }
519         // Artificially make the condition by setting closing flag explicitly.
520         // I can't make the issue happen with a call to region.close().
521         this.region.closing.set(true);
522         for (int i = threads.length / 2; i < threads.length; i++) {
523           threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow),
524               done, gets);
525           threads[i].setDaemon(true);
526           threads[i].start();
527         }
528       } finally {
529         if (this.region != null) {
530           HRegion.closeHRegion(this.region);
531         }
532       }
533       done.set(true);
534       for (GetTillDoneOrException t: threads) {
535         try {
536           t.join();
537         } catch (InterruptedException e) {
538           e.printStackTrace();
539         }
540         if (t.e != null) {
541           LOG.info("Exception=" + t.e);
542           assertFalse("Found a NPE in " + t.getName(),
543               t.e instanceof NullPointerException);
544         }
545       }
546     } finally {
547       HRegion.closeHRegion(this.region);
548       this.region = null;
549     }
550   }
551 
552   /*
553    * Thread that does get on single row until 'done' flag is flipped.  If an
554    * exception causes us to fail, it records it.
555    */
556   class GetTillDoneOrException extends Thread {
557     private final Get g;
558     private final AtomicBoolean done;
559     private final AtomicInteger count;
560     private Exception e;
561 
562     GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d,
563         final AtomicInteger c) {
564       super("getter." + i);
565       this.g = new Get(r);
566       this.done = d;
567       this.count = c;
568     }
569 
570     @Override
571     public void run() {
572       while (!this.done.get()) {
573         try {
574           assertTrue(region.get(g).size() > 0);
575           this.count.incrementAndGet();
576         } catch (Exception e) {
577           this.e = e;
578           break;
579         }
580       }
581     }
582   }
583 
584   /*
585    * An involved filter test.  Has multiple column families and deletes in mix.
586    */
587   public void testWeirdCacheBehaviour() throws Exception {
588     byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
589     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
590         Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
591         Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
592     this.region = initHRegion(TABLE, getName(), conf, FAMILIES);
593     try {
594       String value = "this is the value";
595       String value2 = "this is some other value";
596       String keyPrefix1 = "prefix1"; // UUID.randomUUID().toString();
597       String keyPrefix2 = "prefix2"; // UUID.randomUUID().toString();
598       String keyPrefix3 = "prefix3"; // UUID.randomUUID().toString();
599       putRows(this.region, 3, value, keyPrefix1);
600       putRows(this.region, 3, value, keyPrefix2);
601       putRows(this.region, 3, value, keyPrefix3);
602       // this.region.flushCommits();
603       putRows(this.region, 3, value2, keyPrefix1);
604       putRows(this.region, 3, value2, keyPrefix2);
605       putRows(this.region, 3, value2, keyPrefix3);
606       System.out.println("Checking values for key: " + keyPrefix1);
607       assertEquals("Got back incorrect number of rows from scan", 3,
608           getNumberOfRows(keyPrefix1, value2, this.region));
609       System.out.println("Checking values for key: " + keyPrefix2);
610       assertEquals("Got back incorrect number of rows from scan", 3,
611           getNumberOfRows(keyPrefix2, value2, this.region));
612       System.out.println("Checking values for key: " + keyPrefix3);
613       assertEquals("Got back incorrect number of rows from scan", 3,
614           getNumberOfRows(keyPrefix3, value2, this.region));
615       deleteColumns(this.region, value2, keyPrefix1);
616       deleteColumns(this.region, value2, keyPrefix2);
617       deleteColumns(this.region, value2, keyPrefix3);
618       System.out.println("Starting important checks.....");
619       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,
620           0, getNumberOfRows(keyPrefix1, value2, this.region));
621       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,
622           0, getNumberOfRows(keyPrefix2, value2, this.region));
623       assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,
624           0, getNumberOfRows(keyPrefix3, value2, this.region));
625     } finally {
626       HRegion.closeHRegion(this.region);
627       this.region = null;
628     }
629   }
630 
631   public void testAppendWithReadOnlyTable() throws Exception {
632     byte[] TABLE = Bytes.toBytes("readOnlyTable");
633     this.region = initHRegion(TABLE, getName(), conf, true, Bytes.toBytes("somefamily"));
634     boolean exceptionCaught = false;
635     Append append = new Append(Bytes.toBytes("somerow"));
636     append.setDurability(Durability.SKIP_WAL);
637     append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
638         Bytes.toBytes("somevalue"));
639     try {
640       region.append(append);
641     } catch (IOException e) {
642       exceptionCaught = true;
643     } finally {
644       HRegion.closeHRegion(this.region);
645       this.region = null;
646     }
647     assertTrue(exceptionCaught == true);
648   }
649 
650   public void testIncrWithReadOnlyTable() throws Exception {
651     byte[] TABLE = Bytes.toBytes("readOnlyTable");
652     this.region = initHRegion(TABLE, getName(), conf, true, Bytes.toBytes("somefamily"));
653     boolean exceptionCaught = false;
654     Increment inc = new Increment(Bytes.toBytes("somerow"));
655     inc.setDurability(Durability.SKIP_WAL);
656     inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
657     try {
658       region.increment(inc);
659     } catch (IOException e) {
660       exceptionCaught = true;
661     } finally {
662       HRegion.closeHRegion(this.region);
663       this.region = null;
664     }
665     assertTrue(exceptionCaught == true);
666   }
667 
668   private void deleteColumns(HRegion r, String value, String keyPrefix)
669   throws IOException {
670     InternalScanner scanner = buildScanner(keyPrefix, value, r);
671     int count = 0;
672     boolean more = false;
673     List<KeyValue> results = new ArrayList<KeyValue>();
674     do {
675       more = scanner.next(results);
676       if (results != null && !results.isEmpty())
677         count++;
678       else
679         break;
680       Delete delete = new Delete(results.get(0).getRow());
681       delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
682       r.delete(delete);
683       results.clear();
684     } while (more);
685     assertEquals("Did not perform correct number of deletes", 3, count);
686   }
687 
688   private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
689     InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
690     int numberOfResults = 0;
691     List<KeyValue> results = new ArrayList<KeyValue>();
692     boolean more = false;
693     do {
694       more = resultScanner.next(results);
695       if (results != null && !results.isEmpty()) numberOfResults++;
696       else break;
697       for (KeyValue kv: results) {
698         System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(kv.getValue()));
699       }
700       results.clear();
701     } while(more);
702     return numberOfResults;
703   }
704 
705   private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
706   throws IOException {
707     // Defaults FilterList.Operator.MUST_PASS_ALL.
708     FilterList allFilters = new FilterList();
709     allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
710     // Only return rows where this column value exists in the row.
711     SingleColumnValueFilter filter =
712       new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
713         Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
714     filter.setFilterIfMissing(true);
715     allFilters.addFilter(filter);
716     Scan scan = new Scan();
717     scan.addFamily(Bytes.toBytes("trans-blob"));
718     scan.addFamily(Bytes.toBytes("trans-type"));
719     scan.addFamily(Bytes.toBytes("trans-date"));
720     scan.addFamily(Bytes.toBytes("trans-tags"));
721     scan.addFamily(Bytes.toBytes("trans-group"));
722     scan.setFilter(allFilters);
723     return r.getScanner(scan);
724   }
725 
726   private void putRows(HRegion r, int numRows, String value, String key)
727   throws IOException {
728     for (int i = 0; i < numRows; i++) {
729       String row = key + "_" + i/* UUID.randomUUID().toString() */;
730       System.out.println(String.format("Saving row: %s, with value %s", row,
731         value));
732       Put put = new Put(Bytes.toBytes(row));
733       put.setDurability(Durability.SKIP_WAL);
734       put.add(Bytes.toBytes("trans-blob"), null,
735         Bytes.toBytes("value for blob"));
736       put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
737       put.add(Bytes.toBytes("trans-date"), null,
738         Bytes.toBytes("20090921010101999"));
739       put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"),
740         Bytes.toBytes(value));
741       put.add(Bytes.toBytes("trans-group"), null,
742         Bytes.toBytes("adhocTransactionGroupId"));
743       r.put(put);
744     }
745   }
746 
747   public void testFamilyWithAndWithoutColon() throws Exception {
748     byte [] b = Bytes.toBytes(getName());
749     byte [] cf = Bytes.toBytes(COLUMN_FAMILY);
750     this.region = initHRegion(b, getName(), conf, cf);
751     try {
752       Put p = new Put(b);
753       byte [] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
754       p.add(cfwithcolon, cfwithcolon, cfwithcolon);
755       boolean exception = false;
756       try {
757         this.region.put(p);
758       } catch (NoSuchColumnFamilyException e) {
759         exception = true;
760       }
761       assertTrue(exception);
762     } finally {
763        HRegion.closeHRegion(this.region);
764       this.region = null;
765     }
766   }
767 
768   public void testBatchPut() throws Exception {
769     byte[] b = Bytes.toBytes(getName());
770     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
771     byte[] qual = Bytes.toBytes("qual");
772     byte[] val = Bytes.toBytes("val");
773     this.region = initHRegion(b, getName(), conf, cf);
774     MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
775     try {
776       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
777       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
778 
779       LOG.info("First a batch put with all valid puts");
780       final Put[] puts = new Put[10];
781       for (int i = 0; i < 10; i++) {
782         puts[i] = new Put(Bytes.toBytes("row_" + i));
783         puts[i].add(cf, qual, val);
784       }
785 
786       OperationStatus[] codes = this.region.batchMutate(puts);
787       assertEquals(10, codes.length);
788       for (int i = 0; i < 10; i++) {
789         assertEquals(OperationStatusCode.SUCCESS, codes[i]
790             .getOperationStatusCode());
791       }
792       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
793 
794 
795       LOG.info("Next a batch put with one invalid family");
796       puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
797       codes = this.region.batchMutate(puts);
798       assertEquals(10, codes.length);
799       for (int i = 0; i < 10; i++) {
800         assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY :
801           OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
802       }
803 
804       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
805 
806       LOG.info("Next a batch put that has to break into two batches to avoid a lock");
807       RowLock rowLock = region.getRowLock(Bytes.toBytes("row_2"));
808 
809       MultithreadedTestUtil.TestContext ctx =
810         new MultithreadedTestUtil.TestContext(conf);
811       final AtomicReference<OperationStatus[]> retFromThread =
812         new AtomicReference<OperationStatus[]>();
813       TestThread putter = new TestThread(ctx) {
814         @Override
815         public void doWork() throws IOException {
816           retFromThread.set(region.batchMutate(puts));
817         }
818       };
819       LOG.info("...starting put thread while holding lock");
820       ctx.addThread(putter);
821       ctx.startThreads();
822 
823       LOG.info("...waiting for put thread to sync first time");
824       long startWait = System.currentTimeMillis();
825       while (metricsAssertHelper.getCounter("syncTimeNumOps", source) == syncs +2 ) {
826         Thread.sleep(100);
827         if (System.currentTimeMillis() - startWait > 10000) {
828           fail("Timed out waiting for thread to sync first minibatch");
829         }
830       }
831       LOG.info("...releasing row lock, which should let put thread continue");
832       rowLock.release();
833       LOG.info("...joining on thread");
834       ctx.stop();
835       LOG.info("...checking that next batch was synced");
836       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 4, source);
837       codes = retFromThread.get();
838       for (int i = 0; i < 10; i++) {
839         assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY :
840           OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
841       }
842 
843     } finally {
844       HRegion.closeHRegion(this.region);
845        this.region = null;
846     }
847   }
848 
849   public void testBatchPutWithTsSlop() throws Exception {
850     byte[] b = Bytes.toBytes(getName());
851     byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
852     byte[] qual = Bytes.toBytes("qual");
853     byte[] val = Bytes.toBytes("val");
854     Configuration conf = HBaseConfiguration.create(this.conf);
855 
856     // add data with a timestamp that is too recent for range. Ensure assert
857     conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
858     this.region = initHRegion(b, getName(), conf, cf);
859 
860     try{
861       MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
862       long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
863       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
864 
865       final Put[] puts = new Put[10];
866       for (int i = 0; i < 10; i++) {
867         puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
868         puts[i].add(cf, qual, val);
869       }
870 
871       OperationStatus[] codes = this.region.batchMutate(puts);
872       assertEquals(10, codes.length);
873       for (int i = 0; i < 10; i++) {
874         assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i]
875             .getOperationStatusCode());
876       }
877       metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
878 
879     } finally {
880       HRegion.closeHRegion(this.region);
881       this.region = null;
882     }
883 
884   }
885 
886   //////////////////////////////////////////////////////////////////////////////
887   // checkAndMutate tests
888   //////////////////////////////////////////////////////////////////////////////
889   public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
890     byte [] tableName = Bytes.toBytes("testtable");
891     byte [] row1 = Bytes.toBytes("row1");
892     byte [] fam1 = Bytes.toBytes("fam1");
893     byte [] qf1  = Bytes.toBytes("qualifier");
894     byte [] emptyVal  = new byte[] {};
895     byte [] val1  = Bytes.toBytes("value1");
896     byte [] val2  = Bytes.toBytes("value2");
897 
898     //Setting up region
899     String method = this.getName();
900     this.region = initHRegion(tableName, method, conf, fam1);
901     try {
902       //Putting empty data in key
903       Put put = new Put(row1);
904       put.add(fam1, qf1, emptyVal);
905 
906       //checkAndPut with empty value
907       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
908           new BinaryComparator(emptyVal), put, true);
909       assertTrue(res);
910 
911       //Putting data in key
912       put = new Put(row1);
913       put.add(fam1, qf1, val1);
914 
915       //checkAndPut with correct value
916       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
917           new BinaryComparator(emptyVal), put, true);
918       assertTrue(res);
919 
920       // not empty anymore
921       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
922           new BinaryComparator(emptyVal), put, true);
923       assertFalse(res);
924 
925       Delete delete = new Delete(row1);
926       delete.deleteColumn(fam1, qf1);
927       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
928           new BinaryComparator(emptyVal), delete, true);
929       assertFalse(res);
930 
931       put = new Put(row1);
932       put.add(fam1, qf1, val2);
933       //checkAndPut with correct value
934       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
935           new BinaryComparator(val1), put, true);
936       assertTrue(res);
937 
938       //checkAndDelete with correct value
939       delete = new Delete(row1);
940       delete.deleteColumn(fam1, qf1);
941       delete.deleteColumn(fam1, qf1);
942       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
943           new BinaryComparator(val2), delete, true);
944       assertTrue(res);
945 
946       delete = new Delete(row1);
947       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
948           new BinaryComparator(emptyVal), delete, true);
949       assertTrue(res);
950 
951       //checkAndPut looking for a null value
952       put = new Put(row1);
953       put.add(fam1, qf1, val1);
954 
955       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
956           new NullComparator(), put, true);
957       assertTrue(res);
958     } finally {
959       HRegion.closeHRegion(this.region);
960       this.region = null;
961     }
962   }
963 
964   public void testCheckAndMutate_WithWrongValue() throws IOException{
965     byte [] tableName = Bytes.toBytes("testtable");
966     byte [] row1 = Bytes.toBytes("row1");
967     byte [] fam1 = Bytes.toBytes("fam1");
968     byte [] qf1  = Bytes.toBytes("qualifier");
969     byte [] val1  = Bytes.toBytes("value1");
970     byte [] val2  = Bytes.toBytes("value2");
971 
972     //Setting up region
973     String method = this.getName();
974     this.region = initHRegion(tableName, method, conf, fam1);
975     try {
976       //Putting data in key
977       Put put = new Put(row1);
978       put.add(fam1, qf1, val1);
979       region.put(put);
980 
981       //checkAndPut with wrong value
982       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
983           new BinaryComparator(val2), put, true);
984       assertEquals(false, res);
985 
986       //checkAndDelete with wrong value
987       Delete delete = new Delete(row1);
988       delete.deleteFamily(fam1);
989       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
990           new BinaryComparator(val2), delete, true);
991       assertEquals(false, res);
992     } finally {
993       HRegion.closeHRegion(this.region);
994       this.region = null;
995     }
996   }
997 
998   public void testCheckAndMutate_WithCorrectValue() throws IOException{
999     byte [] tableName = Bytes.toBytes("testtable");
1000     byte [] row1 = Bytes.toBytes("row1");
1001     byte [] fam1 = Bytes.toBytes("fam1");
1002     byte [] qf1  = Bytes.toBytes("qualifier");
1003     byte [] val1  = Bytes.toBytes("value1");
1004 
1005     //Setting up region
1006     String method = this.getName();
1007     this.region = initHRegion(tableName, method, conf, fam1);
1008     try {
1009       //Putting data in key
1010       Put put = new Put(row1);
1011       put.add(fam1, qf1, val1);
1012       region.put(put);
1013 
1014       //checkAndPut with correct value
1015       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
1016           new BinaryComparator(val1), put, true);
1017       assertEquals(true, res);
1018 
1019       //checkAndDelete with correct value
1020       Delete delete = new Delete(row1);
1021       delete.deleteColumn(fam1, qf1);
1022       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
1023           new BinaryComparator(val1), put, true);
1024       assertEquals(true, res);
1025     } finally {
1026       HRegion.closeHRegion(this.region);
1027       this.region = null;
1028     }
1029   }
1030 
1031   public void testCheckAndPut_ThatPutWasWritten() throws IOException{
1032     byte [] tableName = Bytes.toBytes("testtable");
1033     byte [] row1 = Bytes.toBytes("row1");
1034     byte [] fam1 = Bytes.toBytes("fam1");
1035     byte [] fam2 = Bytes.toBytes("fam2");
1036     byte [] qf1  = Bytes.toBytes("qualifier");
1037     byte [] val1  = Bytes.toBytes("value1");
1038     byte [] val2  = Bytes.toBytes("value2");
1039 
1040     byte [][] families = {fam1, fam2};
1041 
1042     //Setting up region
1043     String method = this.getName();
1044     this.region = initHRegion(tableName, method, conf, families);
1045     try {
1046       //Putting data in the key to check
1047       Put put = new Put(row1);
1048       put.add(fam1, qf1, val1);
1049       region.put(put);
1050 
1051       //Creating put to add
1052       long ts = System.currentTimeMillis();
1053       KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1054       put = new Put(row1);
1055       put.add(kv);
1056 
1057       //checkAndPut with wrong value
1058       HStore store = (HStore) region.getStore(fam1);
1059       store.memstore.kvset.size();
1060 
1061       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
1062           new BinaryComparator(val1), put, true);
1063       assertEquals(true, res);
1064       store.memstore.kvset.size();
1065 
1066       Get get = new Get(row1);
1067       get.addColumn(fam2, qf1);
1068       KeyValue [] actual = region.get(get).raw();
1069 
1070       KeyValue [] expected = {kv};
1071 
1072       assertEquals(expected.length, actual.length);
1073       for(int i=0; i<actual.length; i++) {
1074         assertEquals(expected[i], actual[i]);
1075       }
1076     } finally {
1077       HRegion.closeHRegion(this.region);
1078       this.region = null;
1079     }
1080   }
1081 
1082   public void testCheckAndPut_wrongRowInPut() throws IOException {
1083     this.region = initHRegion(tableName, this.getName(), conf, COLUMNS);
1084     try {
1085       Put put = new Put(row2);
1086       put.add(fam1, qual1, value1);
1087       try {
1088         boolean res = region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1089             new BinaryComparator(value2), put, false);
1090         fail();
1091       } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1092         // expected exception.
1093       }
1094     } finally {
1095       HRegion.closeHRegion(this.region);
1096       this.region = null;
1097     }
1098   }
1099 
1100   public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException{
1101     byte [] tableName = Bytes.toBytes("testtable");
1102     byte [] row1 = Bytes.toBytes("row1");
1103     byte [] fam1 = Bytes.toBytes("fam1");
1104     byte [] fam2 = Bytes.toBytes("fam2");
1105     byte [] qf1  = Bytes.toBytes("qualifier1");
1106     byte [] qf2  = Bytes.toBytes("qualifier2");
1107     byte [] qf3  = Bytes.toBytes("qualifier3");
1108     byte [] val1  = Bytes.toBytes("value1");
1109     byte [] val2  = Bytes.toBytes("value2");
1110     byte [] val3  = Bytes.toBytes("value3");
1111     byte[] emptyVal = new byte[] { };
1112 
1113     byte [][] families = {fam1, fam2};
1114 
1115     //Setting up region
1116     String method = this.getName();
1117     this.region = initHRegion(tableName, method, conf, families);
1118     try {
1119       //Put content
1120       Put put = new Put(row1);
1121       put.add(fam1, qf1, val1);
1122       region.put(put);
1123       Threads.sleep(2);
1124 
1125       put = new Put(row1);
1126       put.add(fam1, qf1, val2);
1127       put.add(fam2, qf1, val3);
1128       put.add(fam2, qf2, val2);
1129       put.add(fam2, qf3, val1);
1130       put.add(fam1, qf3, val1);
1131       region.put(put);
1132 
1133       //Multi-column delete
1134       Delete delete = new Delete(row1);
1135       delete.deleteColumn(fam1, qf1);
1136       delete.deleteColumn(fam2, qf1);
1137       delete.deleteColumn(fam1, qf3);
1138       boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
1139           new BinaryComparator(val2), delete, true);
1140       assertEquals(true, res);
1141 
1142       Get get = new Get(row1);
1143       get.addColumn(fam1, qf1);
1144       get.addColumn(fam1, qf3);
1145       get.addColumn(fam2, qf2);
1146       Result r = region.get(get);
1147       assertEquals(2, r.size());
1148       assertEquals(val1, r.getValue(fam1, qf1));
1149       assertEquals(val2, r.getValue(fam2, qf2));
1150 
1151       //Family delete
1152       delete = new Delete(row1);
1153       delete.deleteFamily(fam2);
1154       res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL,
1155           new BinaryComparator(emptyVal), delete, true);
1156       assertEquals(true, res);
1157 
1158       get = new Get(row1);
1159       r = region.get(get);
1160       assertEquals(1, r.size());
1161       assertEquals(val1, r.getValue(fam1, qf1));
1162 
1163       //Row delete
1164       delete = new Delete(row1);
1165       res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
1166           new BinaryComparator(val1), delete, true);
1167       assertEquals(true, res);
1168       get = new Get(row1);
1169       r = region.get(get);
1170       assertEquals(0, r.size());
1171     } finally {
1172       HRegion.closeHRegion(this.region);
1173       this.region = null;
1174     }
1175   }
1176 
1177   //////////////////////////////////////////////////////////////////////////////
1178   // Delete tests
1179   //////////////////////////////////////////////////////////////////////////////
1180   public void testDelete_multiDeleteColumn() throws IOException {
1181     byte [] tableName = Bytes.toBytes("testtable");
1182     byte [] row1 = Bytes.toBytes("row1");
1183     byte [] fam1 = Bytes.toBytes("fam1");
1184     byte [] qual = Bytes.toBytes("qualifier");
1185     byte [] value = Bytes.toBytes("value");
1186 
1187     Put put = new Put(row1);
1188     put.add(fam1, qual, 1, value);
1189     put.add(fam1, qual, 2, value);
1190 
1191     String method = this.getName();
1192     this.region = initHRegion(tableName, method, conf, fam1);
1193     try {
1194       region.put(put);
1195 
1196       // We do support deleting more than 1 'latest' version
1197       Delete delete = new Delete(row1);
1198       delete.deleteColumn(fam1, qual);
1199       delete.deleteColumn(fam1, qual);
1200       region.delete(delete);
1201 
1202       Get get = new Get(row1);
1203       get.addFamily(fam1);
1204       Result r = region.get(get);
1205       assertEquals(0, r.size());
1206     } finally {
1207       HRegion.closeHRegion(this.region);
1208       this.region = null;
1209     }
1210   }
1211 
1212   public void testDelete_CheckFamily() throws IOException {
1213     byte [] tableName = Bytes.toBytes("testtable");
1214     byte [] row1 = Bytes.toBytes("row1");
1215     byte [] fam1 = Bytes.toBytes("fam1");
1216     byte [] fam2 = Bytes.toBytes("fam2");
1217     byte [] fam3 = Bytes.toBytes("fam3");
1218     byte [] fam4 = Bytes.toBytes("fam4");
1219 
1220     //Setting up region
1221     String method = this.getName();
1222     this.region = initHRegion(tableName, method, conf, fam1, fam2, fam3);
1223     try {
1224       List<Cell> kvs  = new ArrayList<Cell>();
1225       kvs.add(new KeyValue(row1, fam4, null, null));
1226 
1227 
1228       //testing existing family
1229       byte [] family = fam2;
1230       try {
1231         NavigableMap<byte[], List<Cell>> deleteMap =
1232           new TreeMap<byte[], List<Cell>>(Bytes.BYTES_COMPARATOR);
1233         deleteMap.put(family, kvs);
1234         region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, Durability.SYNC_WAL);
1235       } catch (Exception e) {
1236         assertTrue("Family " +new String(family)+ " does not exist", false);
1237       }
1238 
1239       //testing non existing family
1240       boolean ok = false;
1241       family = fam4;
1242       try {
1243         NavigableMap<byte[], List<Cell>> deleteMap =
1244           new TreeMap<byte[], List<Cell>>(Bytes.BYTES_COMPARATOR);
1245         deleteMap.put(family, kvs);
1246         region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, Durability.SYNC_WAL);
1247       } catch (Exception e) {
1248         ok = true;
1249       }
1250       assertEquals("Family " +new String(family)+ " does exist", true, ok);
1251     } finally {
1252       HRegion.closeHRegion(this.region);
1253       this.region = null;
1254     }
1255   }
1256 
1257   public void testDelete_mixed() throws IOException, InterruptedException {
1258     byte [] tableName = Bytes.toBytes("testtable");
1259     byte [] fam = Bytes.toBytes("info");
1260     byte [][] families = {fam};
1261     String method = this.getName();
1262     this.region = initHRegion(tableName, method, conf, families);
1263     try {
1264       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1265 
1266       byte [] row = Bytes.toBytes("table_name");
1267       // column names
1268       byte [] serverinfo = Bytes.toBytes("serverinfo");
1269       byte [] splitA = Bytes.toBytes("splitA");
1270       byte [] splitB = Bytes.toBytes("splitB");
1271 
1272       // add some data:
1273       Put put = new Put(row);
1274       put.add(fam, splitA, Bytes.toBytes("reference_A"));
1275       region.put(put);
1276 
1277       put = new Put(row);
1278       put.add(fam, splitB, Bytes.toBytes("reference_B"));
1279       region.put(put);
1280 
1281       put = new Put(row);
1282       put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1283       region.put(put);
1284 
1285       // ok now delete a split:
1286       Delete delete = new Delete(row);
1287       delete.deleteColumns(fam, splitA);
1288       region.delete(delete);
1289 
1290       // assert some things:
1291       Get get = new Get(row).addColumn(fam, serverinfo);
1292       Result result = region.get(get);
1293       assertEquals(1, result.size());
1294 
1295       get = new Get(row).addColumn(fam, splitA);
1296       result = region.get(get);
1297       assertEquals(0, result.size());
1298 
1299       get = new Get(row).addColumn(fam, splitB);
1300       result = region.get(get);
1301       assertEquals(1, result.size());
1302 
1303       // Assert that after a delete, I can put.
1304       put = new Put(row);
1305       put.add(fam, splitA, Bytes.toBytes("reference_A"));
1306       region.put(put);
1307       get = new Get(row);
1308       result = region.get(get);
1309       assertEquals(3, result.size());
1310 
1311       // Now delete all... then test I can add stuff back
1312       delete = new Delete(row);
1313       region.delete(delete);
1314       assertEquals(0, region.get(get).size());
1315 
1316       region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
1317       result = region.get(get);
1318       assertEquals(1, result.size());
1319     } finally {
1320       HRegion.closeHRegion(this.region);
1321       this.region = null;
1322     }
1323   }
1324 
1325   public void testDeleteRowWithFutureTs() throws IOException {
1326     byte [] tableName = Bytes.toBytes("testtable");
1327     byte [] fam = Bytes.toBytes("info");
1328     byte [][] families = {fam};
1329     String method = this.getName();
1330     this.region = initHRegion(tableName, method, conf, families);
1331     try {
1332       byte [] row = Bytes.toBytes("table_name");
1333       // column names
1334       byte [] serverinfo = Bytes.toBytes("serverinfo");
1335 
1336       // add data in the far future
1337       Put put = new Put(row);
1338       put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP-5,Bytes.toBytes("value"));
1339       region.put(put);
1340 
1341       // now delete something in the present
1342       Delete delete = new Delete(row);
1343       region.delete(delete);
1344 
1345       // make sure we still see our data
1346       Get get = new Get(row).addColumn(fam, serverinfo);
1347       Result result = region.get(get);
1348       assertEquals(1, result.size());
1349 
1350       // delete the future row
1351       delete = new Delete(row,HConstants.LATEST_TIMESTAMP-3);
1352       region.delete(delete);
1353 
1354       // make sure it is gone
1355       get = new Get(row).addColumn(fam, serverinfo);
1356       result = region.get(get);
1357       assertEquals(0, result.size());
1358     } finally {
1359       HRegion.closeHRegion(this.region);
1360       this.region = null;
1361     }
1362   }
1363 
1364   /**
1365    * Tests that the special LATEST_TIMESTAMP option for puts gets
1366    * replaced by the actual timestamp
1367    */
1368   public void testPutWithLatestTS() throws IOException {
1369     byte [] tableName = Bytes.toBytes("testtable");
1370     byte [] fam = Bytes.toBytes("info");
1371     byte [][] families = {fam};
1372     String method = this.getName();
1373     this.region = initHRegion(tableName, method, conf, families);
1374     try {
1375       byte [] row = Bytes.toBytes("row1");
1376       // column names
1377       byte [] qual = Bytes.toBytes("qual");
1378 
1379       // add data with LATEST_TIMESTAMP, put without WAL
1380       Put put = new Put(row);
1381       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1382       region.put(put);
1383 
1384       // Make sure it shows up with an actual timestamp
1385       Get get = new Get(row).addColumn(fam, qual);
1386       Result result = region.get(get);
1387       assertEquals(1, result.size());
1388       KeyValue kv = result.raw()[0];
1389       LOG.info("Got: " + kv);
1390       assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1391           kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1392 
1393       // Check same with WAL enabled (historically these took different
1394       // code paths, so check both)
1395       row = Bytes.toBytes("row2");
1396       put = new Put(row);
1397       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1398       region.put(put);
1399 
1400       // Make sure it shows up with an actual timestamp
1401       get = new Get(row).addColumn(fam, qual);
1402       result = region.get(get);
1403       assertEquals(1, result.size());
1404       kv = result.raw()[0];
1405       LOG.info("Got: " + kv);
1406       assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1407           kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1408     } finally {
1409       HRegion.closeHRegion(this.region);
1410       this.region = null;
1411     }
1412 
1413   }
1414 
1415 
1416   /**
1417    * Tests that there is server-side filtering for invalid timestamp upper
1418    * bound. Note that the timestamp lower bound is automatically handled for us
1419    * by the TTL field.
1420    */
1421   public void testPutWithTsSlop() throws IOException {
1422     byte[] tableName = Bytes.toBytes("testtable");
1423     byte[] fam = Bytes.toBytes("info");
1424     byte[][] families = { fam };
1425     String method = this.getName();
1426     Configuration conf = HBaseConfiguration.create(this.conf);
1427 
1428     // add data with a timestamp that is too recent for range. Ensure assert
1429     conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1430     this.region = initHRegion(tableName, method, conf, families);
1431     boolean caughtExcep = false;
1432     try {
1433       try {
1434         // no TS specified == use latest. should not error
1435         region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes
1436             .toBytes("value")));
1437         // TS out of range. should error
1438         region.put(new Put(row).add(fam, Bytes.toBytes("qual"),
1439             System.currentTimeMillis() + 2000,
1440             Bytes.toBytes("value")));
1441         fail("Expected IOE for TS out of configured timerange");
1442       } catch (FailedSanityCheckException ioe) {
1443         LOG.debug("Received expected exception", ioe);
1444         caughtExcep = true;
1445       }
1446       assertTrue("Should catch FailedSanityCheckException", caughtExcep);
1447     } finally {
1448       HRegion.closeHRegion(this.region);
1449       this.region = null;
1450     }
1451   }
1452 
1453   public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
1454     byte [] tableName = Bytes.toBytes("test_table");
1455     byte [] fam1 = Bytes.toBytes("columnA");
1456     byte [] fam2 = Bytes.toBytes("columnB");
1457     this.region = initHRegion(tableName, getName(), conf, fam1, fam2);
1458     try {
1459       byte [] rowA = Bytes.toBytes("rowA");
1460       byte [] rowB = Bytes.toBytes("rowB");
1461 
1462       byte [] value = Bytes.toBytes("value");
1463 
1464       Delete delete = new Delete(rowA);
1465       delete.deleteFamily(fam1);
1466 
1467       region.delete(delete);
1468 
1469       // now create data.
1470       Put put = new Put(rowA);
1471       put.add(fam2, null, value);
1472       region.put(put);
1473 
1474       put = new Put(rowB);
1475       put.add(fam1, null, value);
1476       put.add(fam2, null, value);
1477       region.put(put);
1478 
1479       Scan scan = new Scan();
1480       scan.addFamily(fam1).addFamily(fam2);
1481       InternalScanner s = region.getScanner(scan);
1482       List<KeyValue> results = new ArrayList<KeyValue>();
1483       s.next(results);
1484       assertTrue(Bytes.equals(rowA, results.get(0).getRow()));
1485 
1486       results.clear();
1487       s.next(results);
1488       assertTrue(Bytes.equals(rowB, results.get(0).getRow()));
1489     } finally {
1490       HRegion.closeHRegion(this.region);
1491       this.region = null;
1492     }
1493   }
1494 
1495   public void testDeleteColumns_PostInsert() throws IOException,
1496       InterruptedException {
1497     Delete delete = new Delete(row);
1498     delete.deleteColumns(fam1, qual1);
1499     doTestDelete_AndPostInsert(delete);
1500   }
1501 
1502   public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
1503     Delete delete = new Delete(row);
1504     delete.deleteFamily(fam1);
1505     doTestDelete_AndPostInsert(delete);
1506   }
1507 
1508   public void doTestDelete_AndPostInsert(Delete delete)
1509       throws IOException, InterruptedException {
1510     this.region = initHRegion(tableName, getName(), conf, fam1);
1511     try {
1512       EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1513       Put put = new Put(row);
1514       put.add(fam1, qual1, value1);
1515       region.put(put);
1516 
1517       // now delete the value:
1518       region.delete(delete);
1519 
1520 
1521       // ok put data:
1522       put = new Put(row);
1523       put.add(fam1, qual1, value2);
1524       region.put(put);
1525 
1526       // ok get:
1527       Get get = new Get(row);
1528       get.addColumn(fam1, qual1);
1529 
1530       Result r = region.get(get);
1531       assertEquals(1, r.size());
1532       assertByteEquals(value2, r.getValue(fam1, qual1));
1533 
1534       // next:
1535       Scan scan = new Scan(row);
1536       scan.addColumn(fam1, qual1);
1537       InternalScanner s = region.getScanner(scan);
1538 
1539       List<KeyValue> results = new ArrayList<KeyValue>();
1540       assertEquals(false, s.next(results));
1541       assertEquals(1, results.size());
1542       KeyValue kv = results.get(0);
1543 
1544       assertByteEquals(value2, kv.getValue());
1545       assertByteEquals(fam1, kv.getFamily());
1546       assertByteEquals(qual1, kv.getQualifier());
1547       assertByteEquals(row, kv.getRow());
1548     } finally {
1549       HRegion.closeHRegion(this.region);
1550       this.region = null;
1551     }
1552   }
1553 
1554   public void testDelete_CheckTimestampUpdated()
1555   throws IOException {
1556     byte [] row1 = Bytes.toBytes("row1");
1557     byte [] col1 = Bytes.toBytes("col1");
1558     byte [] col2 = Bytes.toBytes("col2");
1559     byte [] col3 = Bytes.toBytes("col3");
1560 
1561     //Setting up region
1562     String method = this.getName();
1563     this.region = initHRegion(tableName, method, conf, fam1);
1564     try {
1565       //Building checkerList
1566       List<Cell> kvs  = new ArrayList<Cell>();
1567       kvs.add(new KeyValue(row1, fam1, col1, null));
1568       kvs.add(new KeyValue(row1, fam1, col2, null));
1569       kvs.add(new KeyValue(row1, fam1, col3, null));
1570 
1571       NavigableMap<byte[], List<Cell>> deleteMap =
1572         new TreeMap<byte[], List<Cell>>(Bytes.BYTES_COMPARATOR);
1573       deleteMap.put(fam1, kvs);
1574       region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, Durability.SYNC_WAL);
1575 
1576       // extract the key values out the memstore:
1577       // This is kinda hacky, but better than nothing...
1578       long now = System.currentTimeMillis();
1579       KeyValue firstKv = ((HStore) region.getStore(fam1)).memstore.kvset.first();
1580       assertTrue(firstKv.getTimestamp() <= now);
1581       now = firstKv.getTimestamp();
1582       for (KeyValue kv : ((HStore) region.getStore(fam1)).memstore.kvset) {
1583         assertTrue(kv.getTimestamp() <= now);
1584         now = kv.getTimestamp();
1585       }
1586     } finally {
1587       HRegion.closeHRegion(this.region);
1588       this.region = null;
1589     }
1590   }
1591 
1592   //////////////////////////////////////////////////////////////////////////////
1593   // Get tests
1594   //////////////////////////////////////////////////////////////////////////////
1595   public void testGet_FamilyChecker() throws IOException {
1596     byte [] tableName = Bytes.toBytes("testtable");
1597     byte [] row1 = Bytes.toBytes("row1");
1598     byte [] fam1 = Bytes.toBytes("fam1");
1599     byte [] fam2 = Bytes.toBytes("False");
1600     byte [] col1 = Bytes.toBytes("col1");
1601 
1602     //Setting up region
1603     String method = this.getName();
1604     this.region = initHRegion(tableName, method, conf, fam1);
1605     try {
1606       Get get = new Get(row1);
1607       get.addColumn(fam2, col1);
1608 
1609       //Test
1610       try {
1611         region.get(get);
1612       } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
1613         assertFalse(false);
1614         return;
1615       }
1616       assertFalse(true);
1617     } finally {
1618       HRegion.closeHRegion(this.region);
1619       this.region = null;
1620     }
1621   }
1622 
1623   public void testGet_Basic() throws IOException {
1624     byte [] tableName = Bytes.toBytes("testtable");
1625     byte [] row1 = Bytes.toBytes("row1");
1626     byte [] fam1 = Bytes.toBytes("fam1");
1627     byte [] col1 = Bytes.toBytes("col1");
1628     byte [] col2 = Bytes.toBytes("col2");
1629     byte [] col3 = Bytes.toBytes("col3");
1630     byte [] col4 = Bytes.toBytes("col4");
1631     byte [] col5 = Bytes.toBytes("col5");
1632 
1633     //Setting up region
1634     String method = this.getName();
1635     this.region = initHRegion(tableName, method, conf, fam1);
1636     try {
1637       //Add to memstore
1638       Put put = new Put(row1);
1639       put.add(fam1, col1, null);
1640       put.add(fam1, col2, null);
1641       put.add(fam1, col3, null);
1642       put.add(fam1, col4, null);
1643       put.add(fam1, col5, null);
1644       region.put(put);
1645 
1646       Get get = new Get(row1);
1647       get.addColumn(fam1, col2);
1648       get.addColumn(fam1, col4);
1649       //Expected result
1650       KeyValue kv1 = new KeyValue(row1, fam1, col2);
1651       KeyValue kv2 = new KeyValue(row1, fam1, col4);
1652       KeyValue [] expected = {kv1, kv2};
1653 
1654       //Test
1655       Result res = region.get(get);
1656       assertEquals(expected.length, res.size());
1657       for(int i=0; i<res.size(); i++){
1658         assertEquals(0,
1659             Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
1660         assertEquals(0,
1661             Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
1662         assertEquals(0,
1663             Bytes.compareTo(
1664                 expected[i].getQualifier(), res.raw()[i].getQualifier()));
1665       }
1666 
1667       // Test using a filter on a Get
1668       Get g = new Get(row1);
1669       final int count = 2;
1670       g.setFilter(new ColumnCountGetFilter(count));
1671       res = region.get(g);
1672       assertEquals(count, res.size());
1673     } finally {
1674       HRegion.closeHRegion(this.region);
1675       this.region = null;
1676     }
1677   }
1678 
1679   public void testGet_Empty() throws IOException {
1680     byte [] tableName = Bytes.toBytes("emptytable");
1681     byte [] row = Bytes.toBytes("row");
1682     byte [] fam = Bytes.toBytes("fam");
1683 
1684     String method = this.getName();
1685     this.region = initHRegion(tableName, method, conf, fam);
1686     try {
1687       Get get = new Get(row);
1688       get.addFamily(fam);
1689       Result r = region.get(get);
1690 
1691       assertTrue(r.isEmpty());
1692     } finally {
1693       HRegion.closeHRegion(this.region);
1694       this.region = null;
1695     }
1696   }
1697 
1698   //Test that checked if there was anything special when reading from the ROOT
1699   //table. To be able to use this test you need to comment the part in
1700   //HTableDescriptor that checks for '-' and '.'. You also need to remove the
1701   //s in the beginning of the name.
1702   public void stestGet_Root() throws IOException {
1703     //Setting up region
1704     String method = this.getName();
1705     this.region = initHRegion(TableName.ROOT_TABLE_NAME,
1706       method, conf, HConstants.CATALOG_FAMILY);
1707     try {
1708       //Add to memstore
1709       Put put = new Put(HConstants.EMPTY_START_ROW);
1710       put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, null);
1711       region.put(put);
1712 
1713       Get get = new Get(HConstants.EMPTY_START_ROW);
1714       get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
1715 
1716       //Expected result
1717       KeyValue kv1 = new KeyValue(HConstants.EMPTY_START_ROW,
1718           HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
1719       KeyValue [] expected = {kv1};
1720 
1721       //Test from memstore
1722       Result res = region.get(get);
1723 
1724       assertEquals(expected.length, res.size());
1725       for(int i=0; i<res.size(); i++){
1726         assertEquals(0,
1727             Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
1728         assertEquals(0,
1729             Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
1730         assertEquals(0,
1731             Bytes.compareTo(
1732                 expected[i].getQualifier(), res.raw()[i].getQualifier()));
1733       }
1734 
1735       //flush
1736       region.flushcache();
1737 
1738       //test2
1739       res = region.get(get);
1740 
1741       assertEquals(expected.length, res.size());
1742       for(int i=0; i<res.size(); i++){
1743         assertEquals(0,
1744             Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
1745         assertEquals(0,
1746             Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
1747         assertEquals(0,
1748             Bytes.compareTo(
1749                 expected[i].getQualifier(), res.raw()[i].getQualifier()));
1750       }
1751 
1752       //Scan
1753       Scan scan = new Scan();
1754       scan.addFamily(HConstants.CATALOG_FAMILY);
1755       InternalScanner s = region.getScanner(scan);
1756       List<KeyValue> result = new ArrayList<KeyValue>();
1757       s.next(result);
1758 
1759       assertEquals(expected.length, result.size());
1760       for(int i=0; i<res.size(); i++){
1761         assertEquals(0,
1762             Bytes.compareTo(expected[i].getRow(), result.get(i).getRow()));
1763         assertEquals(0,
1764             Bytes.compareTo(expected[i].getFamily(), result.get(i).getFamily()));
1765         assertEquals(0,
1766             Bytes.compareTo(
1767                 expected[i].getQualifier(), result.get(i).getQualifier()));
1768       }
1769     } finally {
1770       HRegion.closeHRegion(this.region);
1771       this.region = null;
1772     }
1773   }
1774 
1775   //////////////////////////////////////////////////////////////////////////////
1776   // Merge test
1777   //////////////////////////////////////////////////////////////////////////////
1778   public void testMerge() throws IOException {
1779     byte [] tableName = Bytes.toBytes("testtable");
1780     byte [][] families = {fam1, fam2, fam3};
1781     Configuration hc = initSplit();
1782     //Setting up region
1783     String method = this.getName();
1784     this.region = initHRegion(tableName, method, hc, families);
1785     try {
1786       LOG.info("" + addContent(region, fam3));
1787       region.flushcache();
1788       region.compactStores();
1789       byte [] splitRow = region.checkSplit();
1790       assertNotNull(splitRow);
1791       LOG.info("SplitRow: " + Bytes.toString(splitRow));
1792       HRegion [] subregions = splitRegion(region, splitRow);
1793       try {
1794         // Need to open the regions.
1795         for (int i = 0; i < subregions.length; i++) {
1796           openClosedRegion(subregions[i]);
1797           subregions[i].compactStores();
1798         }
1799         Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
1800         Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
1801         Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
1802         long startTime = System.currentTimeMillis();
1803         region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
1804         LOG.info("Merge regions elapsed time: " +
1805             ((System.currentTimeMillis() - startTime) / 1000.0));
1806         fs.delete(oldRegion1, true);
1807         fs.delete(oldRegion2, true);
1808         fs.delete(oldRegionPath, true);
1809         LOG.info("splitAndMerge completed.");
1810       } finally {
1811         for (int i = 0; i < subregions.length; i++) {
1812           try {
1813             HRegion.closeHRegion(subregions[i]);
1814           } catch (IOException e) {
1815             // Ignore.
1816           }
1817         }
1818       }
1819     } finally {
1820       HRegion.closeHRegion(this.region);
1821       this.region = null;
1822     }
1823   }
1824 
1825   /**
1826    * @param parent Region to split.
1827    * @param midkey Key to split around.
1828    * @return The Regions we created.
1829    * @throws IOException
1830    */
1831   HRegion [] splitRegion(final HRegion parent, final byte [] midkey)
1832   throws IOException {
1833     PairOfSameType<HRegion> result = null;
1834     SplitTransaction st = new SplitTransaction(parent, midkey);
1835     // If prepare does not return true, for some reason -- logged inside in
1836     // the prepare call -- we are not ready to split just now.  Just return.
1837     if (!st.prepare()) return null;
1838     try {
1839       result = st.execute(null, null);
1840     } catch (IOException ioe) {
1841       try {
1842         LOG.info("Running rollback of failed split of " +
1843           parent.getRegionNameAsString() + "; " + ioe.getMessage());
1844         st.rollback(null, null);
1845         LOG.info("Successful rollback of failed split of " +
1846           parent.getRegionNameAsString());
1847         return null;
1848       } catch (RuntimeException e) {
1849         // If failed rollback, kill this server to avoid having a hole in table.
1850         LOG.info("Failed rollback of failed split of " +
1851           parent.getRegionNameAsString() + " -- aborting server", e);
1852       }
1853     }
1854     return new HRegion [] {result.getFirst(), result.getSecond()};
1855   }
1856 
1857   //////////////////////////////////////////////////////////////////////////////
1858   // Scanner tests
1859   //////////////////////////////////////////////////////////////////////////////
1860   public void testGetScanner_WithOkFamilies() throws IOException {
1861     byte [] tableName = Bytes.toBytes("testtable");
1862     byte [] fam1 = Bytes.toBytes("fam1");
1863     byte [] fam2 = Bytes.toBytes("fam2");
1864 
1865     byte [][] families = {fam1, fam2};
1866 
1867     //Setting up region
1868     String method = this.getName();
1869     this.region = initHRegion(tableName, method, conf, families);
1870     try {
1871       Scan scan = new Scan();
1872       scan.addFamily(fam1);
1873       scan.addFamily(fam2);
1874       try {
1875         region.getScanner(scan);
1876       } catch (Exception e) {
1877         assertTrue("Families could not be found in Region", false);
1878       }
1879     } finally {
1880       HRegion.closeHRegion(this.region);
1881       this.region = null;
1882     }
1883   }
1884 
1885   public void testGetScanner_WithNotOkFamilies() throws IOException {
1886     byte [] tableName = Bytes.toBytes("testtable");
1887     byte [] fam1 = Bytes.toBytes("fam1");
1888     byte [] fam2 = Bytes.toBytes("fam2");
1889 
1890     byte [][] families = {fam1};
1891 
1892     //Setting up region
1893     String method = this.getName();
1894     this.region = initHRegion(tableName, method, conf, families);
1895     try {
1896       Scan scan = new Scan();
1897       scan.addFamily(fam2);
1898       boolean ok = false;
1899       try {
1900         region.getScanner(scan);
1901       } catch (Exception e) {
1902         ok = true;
1903       }
1904       assertTrue("Families could not be found in Region", ok);
1905     } finally {
1906       HRegion.closeHRegion(this.region);
1907       this.region = null;
1908     }
1909   }
1910 
1911   public void testGetScanner_WithNoFamilies() throws IOException {
1912     byte [] tableName = Bytes.toBytes("testtable");
1913     byte [] row1 = Bytes.toBytes("row1");
1914     byte [] fam1 = Bytes.toBytes("fam1");
1915     byte [] fam2 = Bytes.toBytes("fam2");
1916     byte [] fam3 = Bytes.toBytes("fam3");
1917     byte [] fam4 = Bytes.toBytes("fam4");
1918 
1919     byte [][] families = {fam1, fam2, fam3, fam4};
1920 
1921     //Setting up region
1922     String method = this.getName();
1923     this.region = initHRegion(tableName, method, conf, families);
1924     try {
1925 
1926       //Putting data in Region
1927       Put put = new Put(row1);
1928       put.add(fam1, null, null);
1929       put.add(fam2, null, null);
1930       put.add(fam3, null, null);
1931       put.add(fam4, null, null);
1932       region.put(put);
1933 
1934       Scan scan = null;
1935       HRegion.RegionScannerImpl is = null;
1936 
1937       //Testing to see how many scanners that is produced by getScanner, starting
1938       //with known number, 2 - current = 1
1939       scan = new Scan();
1940       scan.addFamily(fam2);
1941       scan.addFamily(fam4);
1942       is = (RegionScannerImpl) region.getScanner(scan);
1943       MultiVersionConsistencyControl.resetThreadReadPoint(region.getMVCC());
1944       assertEquals(1, ((RegionScannerImpl)is).storeHeap.getHeap().size());
1945 
1946       scan = new Scan();
1947       is = (RegionScannerImpl) region.getScanner(scan);
1948       MultiVersionConsistencyControl.resetThreadReadPoint(region.getMVCC());
1949       assertEquals(families.length -1,
1950           ((RegionScannerImpl)is).storeHeap.getHeap().size());
1951     } finally {
1952       HRegion.closeHRegion(this.region);
1953       this.region = null;
1954     }
1955   }
1956 
1957   /**
1958    * This method tests https://issues.apache.org/jira/browse/HBASE-2516.
1959    * @throws IOException
1960    */
1961   public void testGetScanner_WithRegionClosed() throws IOException {
1962     byte[] tableName = Bytes.toBytes("testtable");
1963     byte[] fam1 = Bytes.toBytes("fam1");
1964     byte[] fam2 = Bytes.toBytes("fam2");
1965 
1966     byte[][] families = {fam1, fam2};
1967 
1968     //Setting up region
1969     String method = this.getName();
1970     try {
1971       this.region = initHRegion(tableName, method, conf, families);
1972     } catch (IOException e) {
1973       e.printStackTrace();
1974       fail("Got IOException during initHRegion, " + e.getMessage());
1975     }
1976     try {
1977       region.closed.set(true);
1978       try {
1979         region.getScanner(null);
1980         fail("Expected to get an exception during getScanner on a region that is closed");
1981       } catch (NotServingRegionException e) {
1982         //this is the correct exception that is expected
1983       } catch (IOException e) {
1984         fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
1985             + e.getMessage());
1986       }
1987     } finally {
1988       HRegion.closeHRegion(this.region);
1989       this.region = null;
1990     }
1991   }
1992 
1993   public void testRegionScanner_Next() throws IOException {
1994     byte [] tableName = Bytes.toBytes("testtable");
1995     byte [] row1 = Bytes.toBytes("row1");
1996     byte [] row2 = Bytes.toBytes("row2");
1997     byte [] fam1 = Bytes.toBytes("fam1");
1998     byte [] fam2 = Bytes.toBytes("fam2");
1999     byte [] fam3 = Bytes.toBytes("fam3");
2000     byte [] fam4 = Bytes.toBytes("fam4");
2001 
2002     byte [][] families = {fam1, fam2, fam3, fam4};
2003     long ts = System.currentTimeMillis();
2004 
2005     //Setting up region
2006     String method = this.getName();
2007     this.region = initHRegion(tableName, method, conf, families);
2008     try {
2009       //Putting data in Region
2010       Put put = null;
2011       put = new Put(row1);
2012       put.add(fam1, null, ts, null);
2013       put.add(fam2, null, ts, null);
2014       put.add(fam3, null, ts, null);
2015       put.add(fam4, null, ts, null);
2016       region.put(put);
2017 
2018       put = new Put(row2);
2019       put.add(fam1, null, ts, null);
2020       put.add(fam2, null, ts, null);
2021       put.add(fam3, null, ts, null);
2022       put.add(fam4, null, ts, null);
2023       region.put(put);
2024 
2025       Scan scan = new Scan();
2026       scan.addFamily(fam2);
2027       scan.addFamily(fam4);
2028       InternalScanner is = region.getScanner(scan);
2029 
2030       List<KeyValue> res = null;
2031 
2032       //Result 1
2033       List<KeyValue> expected1 = new ArrayList<KeyValue>();
2034       expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2035       expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2036 
2037       res = new ArrayList<KeyValue>();
2038       is.next(res);
2039       for (int i = 0; i < res.size(); i++) {
2040         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2041       }
2042 
2043       //Result 2
2044       List<KeyValue> expected2 = new ArrayList<KeyValue>();
2045       expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2046       expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2047 
2048       res = new ArrayList<KeyValue>();
2049       is.next(res);
2050       for(int i=0; i<res.size(); i++) {
2051         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2052       }
2053     } finally {
2054       HRegion.closeHRegion(this.region);
2055       this.region = null;
2056     }
2057   }
2058 
2059   public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions()
2060   throws IOException {
2061     byte [] tableName = Bytes.toBytes("testtable");
2062     byte [] row1 = Bytes.toBytes("row1");
2063     byte [] qf1 = Bytes.toBytes("qualifier1");
2064     byte [] qf2 = Bytes.toBytes("qualifier2");
2065     byte [] fam1 = Bytes.toBytes("fam1");
2066     byte [][] families = {fam1};
2067 
2068     long ts1 = System.currentTimeMillis();
2069     long ts2 = ts1 + 1;
2070     long ts3 = ts1 + 2;
2071 
2072     //Setting up region
2073     String method = this.getName();
2074     this.region = initHRegion(tableName, method, conf, families);
2075     try {
2076       //Putting data in Region
2077       Put put = null;
2078       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2079       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2080       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2081 
2082       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2083       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2084       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2085 
2086       put = new Put(row1);
2087       put.add(kv13);
2088       put.add(kv12);
2089       put.add(kv11);
2090       put.add(kv23);
2091       put.add(kv22);
2092       put.add(kv21);
2093       region.put(put);
2094 
2095       //Expected
2096       List<KeyValue> expected = new ArrayList<KeyValue>();
2097       expected.add(kv13);
2098       expected.add(kv12);
2099 
2100       Scan scan = new Scan(row1);
2101       scan.addColumn(fam1, qf1);
2102       scan.setMaxVersions(MAX_VERSIONS);
2103       List<KeyValue> actual = new ArrayList<KeyValue>();
2104       InternalScanner scanner = region.getScanner(scan);
2105 
2106       boolean hasNext = scanner.next(actual);
2107       assertEquals(false, hasNext);
2108 
2109       //Verify result
2110       for(int i=0; i<expected.size(); i++) {
2111         assertEquals(expected.get(i), actual.get(i));
2112       }
2113     } finally {
2114       HRegion.closeHRegion(this.region);
2115       this.region = null;
2116     }
2117   }
2118 
2119   public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions()
2120   throws IOException{
2121     byte [] tableName = Bytes.toBytes("testtable");
2122     byte [] row1 = Bytes.toBytes("row1");
2123     byte [] qf1 = Bytes.toBytes("qualifier1");
2124     byte [] qf2 = Bytes.toBytes("qualifier2");
2125     byte [] fam1 = Bytes.toBytes("fam1");
2126     byte [][] families = {fam1};
2127 
2128     long ts1 = 1; //System.currentTimeMillis();
2129     long ts2 = ts1 + 1;
2130     long ts3 = ts1 + 2;
2131 
2132     //Setting up region
2133     String method = this.getName();
2134     this.region = initHRegion(tableName, method, conf, families);
2135     try {
2136       //Putting data in Region
2137       Put put = null;
2138       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2139       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2140       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2141 
2142       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2143       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2144       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2145 
2146       put = new Put(row1);
2147       put.add(kv13);
2148       put.add(kv12);
2149       put.add(kv11);
2150       put.add(kv23);
2151       put.add(kv22);
2152       put.add(kv21);
2153       region.put(put);
2154       region.flushcache();
2155 
2156       //Expected
2157       List<KeyValue> expected = new ArrayList<KeyValue>();
2158       expected.add(kv13);
2159       expected.add(kv12);
2160       expected.add(kv23);
2161       expected.add(kv22);
2162 
2163       Scan scan = new Scan(row1);
2164       scan.addColumn(fam1, qf1);
2165       scan.addColumn(fam1, qf2);
2166       scan.setMaxVersions(MAX_VERSIONS);
2167       List<KeyValue> actual = new ArrayList<KeyValue>();
2168       InternalScanner scanner = region.getScanner(scan);
2169 
2170       boolean hasNext = scanner.next(actual);
2171       assertEquals(false, hasNext);
2172 
2173       //Verify result
2174       for(int i=0; i<expected.size(); i++) {
2175         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2176       }
2177     } finally {
2178       HRegion.closeHRegion(this.region);
2179       this.region = null;
2180     }
2181   }
2182 
2183   public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions()
2184   throws IOException {
2185     byte [] tableName = Bytes.toBytes("testtable");
2186     byte [] row1 = Bytes.toBytes("row1");
2187     byte [] fam1 = Bytes.toBytes("fam1");
2188     byte [][] families = {fam1};
2189     byte [] qf1 = Bytes.toBytes("qualifier1");
2190     byte [] qf2 = Bytes.toBytes("qualifier2");
2191 
2192     long ts1 = 1;
2193     long ts2 = ts1 + 1;
2194     long ts3 = ts1 + 2;
2195     long ts4 = ts1 + 3;
2196 
2197     //Setting up region
2198     String method = this.getName();
2199     this.region = initHRegion(tableName, method, conf, families);
2200     try {
2201       //Putting data in Region
2202       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2203       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2204       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2205       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2206 
2207       KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2208       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2209       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2210       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2211 
2212       Put put = null;
2213       put = new Put(row1);
2214       put.add(kv14);
2215       put.add(kv24);
2216       region.put(put);
2217       region.flushcache();
2218 
2219       put = new Put(row1);
2220       put.add(kv23);
2221       put.add(kv13);
2222       region.put(put);
2223       region.flushcache();
2224 
2225       put = new Put(row1);
2226       put.add(kv22);
2227       put.add(kv12);
2228       region.put(put);
2229       region.flushcache();
2230 
2231       put = new Put(row1);
2232       put.add(kv21);
2233       put.add(kv11);
2234       region.put(put);
2235 
2236       //Expected
2237       List<KeyValue> expected = new ArrayList<KeyValue>();
2238       expected.add(kv14);
2239       expected.add(kv13);
2240       expected.add(kv12);
2241       expected.add(kv24);
2242       expected.add(kv23);
2243       expected.add(kv22);
2244 
2245       Scan scan = new Scan(row1);
2246       scan.addColumn(fam1, qf1);
2247       scan.addColumn(fam1, qf2);
2248       int versions = 3;
2249       scan.setMaxVersions(versions);
2250       List<KeyValue> actual = new ArrayList<KeyValue>();
2251       InternalScanner scanner = region.getScanner(scan);
2252 
2253       boolean hasNext = scanner.next(actual);
2254       assertEquals(false, hasNext);
2255 
2256       //Verify result
2257       for(int i=0; i<expected.size(); i++) {
2258         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2259       }
2260     } finally {
2261       HRegion.closeHRegion(this.region);
2262       this.region = null;
2263     }
2264   }
2265 
2266   public void testScanner_Wildcard_FromMemStore_EnforceVersions()
2267   throws IOException {
2268     byte [] tableName = Bytes.toBytes("testtable");
2269     byte [] row1 = Bytes.toBytes("row1");
2270     byte [] qf1 = Bytes.toBytes("qualifier1");
2271     byte [] qf2 = Bytes.toBytes("qualifier2");
2272     byte [] fam1 = Bytes.toBytes("fam1");
2273     byte [][] families = {fam1};
2274 
2275     long ts1 = System.currentTimeMillis();
2276     long ts2 = ts1 + 1;
2277     long ts3 = ts1 + 2;
2278 
2279     //Setting up region
2280     String method = this.getName();
2281     this.region = initHRegion(tableName, method, conf, families);
2282     try {
2283       //Putting data in Region
2284       Put put = null;
2285       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2286       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2287       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2288 
2289       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2290       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2291       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2292 
2293       put = new Put(row1);
2294       put.add(kv13);
2295       put.add(kv12);
2296       put.add(kv11);
2297       put.add(kv23);
2298       put.add(kv22);
2299       put.add(kv21);
2300       region.put(put);
2301 
2302       //Expected
2303       List<KeyValue> expected = new ArrayList<KeyValue>();
2304       expected.add(kv13);
2305       expected.add(kv12);
2306       expected.add(kv23);
2307       expected.add(kv22);
2308 
2309       Scan scan = new Scan(row1);
2310       scan.addFamily(fam1);
2311       scan.setMaxVersions(MAX_VERSIONS);
2312       List<KeyValue> actual = new ArrayList<KeyValue>();
2313       InternalScanner scanner = region.getScanner(scan);
2314 
2315       boolean hasNext = scanner.next(actual);
2316       assertEquals(false, hasNext);
2317 
2318       //Verify result
2319       for(int i=0; i<expected.size(); i++) {
2320         assertEquals(expected.get(i), actual.get(i));
2321       }
2322     } finally {
2323       HRegion.closeHRegion(this.region);
2324       this.region = null;
2325     }
2326   }
2327 
2328   public void testScanner_Wildcard_FromFilesOnly_EnforceVersions()
2329   throws IOException{
2330     byte [] tableName = Bytes.toBytes("testtable");
2331     byte [] row1 = Bytes.toBytes("row1");
2332     byte [] qf1 = Bytes.toBytes("qualifier1");
2333     byte [] qf2 = Bytes.toBytes("qualifier2");
2334     byte [] fam1 = Bytes.toBytes("fam1");
2335 
2336     long ts1 = 1; //System.currentTimeMillis();
2337     long ts2 = ts1 + 1;
2338     long ts3 = ts1 + 2;
2339 
2340     //Setting up region
2341     String method = this.getName();
2342     this.region = initHRegion(tableName, method, conf, fam1);
2343     try {
2344       //Putting data in Region
2345       Put put = null;
2346       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2347       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2348       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2349 
2350       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2351       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2352       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2353 
2354       put = new Put(row1);
2355       put.add(kv13);
2356       put.add(kv12);
2357       put.add(kv11);
2358       put.add(kv23);
2359       put.add(kv22);
2360       put.add(kv21);
2361       region.put(put);
2362       region.flushcache();
2363 
2364       //Expected
2365       List<KeyValue> expected = new ArrayList<KeyValue>();
2366       expected.add(kv13);
2367       expected.add(kv12);
2368       expected.add(kv23);
2369       expected.add(kv22);
2370 
2371       Scan scan = new Scan(row1);
2372       scan.addFamily(fam1);
2373       scan.setMaxVersions(MAX_VERSIONS);
2374       List<KeyValue> actual = new ArrayList<KeyValue>();
2375       InternalScanner scanner = region.getScanner(scan);
2376 
2377       boolean hasNext = scanner.next(actual);
2378       assertEquals(false, hasNext);
2379 
2380       //Verify result
2381       for(int i=0; i<expected.size(); i++) {
2382         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2383       }
2384     } finally {
2385       HRegion.closeHRegion(this.region);
2386       this.region = null;
2387     }
2388   }
2389 
2390   public void testScanner_StopRow1542() throws IOException {
2391     byte [] tableName = Bytes.toBytes("test_table");
2392     byte [] family = Bytes.toBytes("testFamily");
2393     this.region = initHRegion(tableName, getName(), conf, family);
2394     try {
2395       byte [] row1 = Bytes.toBytes("row111");
2396       byte [] row2 = Bytes.toBytes("row222");
2397       byte [] row3 = Bytes.toBytes("row333");
2398       byte [] row4 = Bytes.toBytes("row444");
2399       byte [] row5 = Bytes.toBytes("row555");
2400 
2401       byte [] col1 = Bytes.toBytes("Pub111");
2402       byte [] col2 = Bytes.toBytes("Pub222");
2403 
2404 
2405       Put put = new Put(row1);
2406       put.add(family, col1, Bytes.toBytes(10L));
2407       region.put(put);
2408 
2409       put = new Put(row2);
2410       put.add(family, col1, Bytes.toBytes(15L));
2411       region.put(put);
2412 
2413       put = new Put(row3);
2414       put.add(family, col2, Bytes.toBytes(20L));
2415       region.put(put);
2416 
2417       put = new Put(row4);
2418       put.add(family, col2, Bytes.toBytes(30L));
2419       region.put(put);
2420 
2421       put = new Put(row5);
2422       put.add(family, col1, Bytes.toBytes(40L));
2423       region.put(put);
2424 
2425       Scan scan = new Scan(row3, row4);
2426       scan.setMaxVersions();
2427       scan.addColumn(family, col1);
2428       InternalScanner s = region.getScanner(scan);
2429 
2430       List<KeyValue> results = new ArrayList<KeyValue>();
2431       assertEquals(false, s.next(results));
2432       assertEquals(0, results.size());
2433     } finally {
2434       HRegion.closeHRegion(this.region);
2435       this.region = null;
2436     }
2437   }
2438 
2439   private void assertICV(byte [] row,
2440                          byte [] familiy,
2441                          byte[] qualifier,
2442                          long amount) throws IOException {
2443     // run a get and see?
2444     Get get = new Get(row);
2445     get.addColumn(familiy, qualifier);
2446     Result result = region.get(get);
2447     assertEquals(1, result.size());
2448 
2449     KeyValue kv = result.raw()[0];
2450     long r = Bytes.toLong(kv.getValue());
2451     assertEquals(amount, r);
2452   }
2453 
2454   private void assertICV(byte [] row,
2455                          byte [] familiy,
2456                          byte[] qualifier,
2457                          int amount) throws IOException {
2458     // run a get and see?
2459     Get get = new Get(row);
2460     get.addColumn(familiy, qualifier);
2461     Result result = region.get(get);
2462     assertEquals(1, result.size());
2463 
2464     KeyValue kv = result.raw()[0];
2465     int r = Bytes.toInt(kv.getValue());
2466     assertEquals(amount, r);
2467   }
2468 
2469   public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions()
2470   throws IOException {
2471     byte [] tableName = Bytes.toBytes("testtable");
2472     byte [] row1 = Bytes.toBytes("row1");
2473     byte [] fam1 = Bytes.toBytes("fam1");
2474     byte [] qf1 = Bytes.toBytes("qualifier1");
2475     byte [] qf2 = Bytes.toBytes("quateslifier2");
2476 
2477     long ts1 = 1;
2478     long ts2 = ts1 + 1;
2479     long ts3 = ts1 + 2;
2480     long ts4 = ts1 + 3;
2481 
2482     //Setting up region
2483     String method = this.getName();
2484     this.region = initHRegion(tableName, method, conf, fam1);
2485     try {
2486       //Putting data in Region
2487       KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2488       KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2489       KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2490       KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2491 
2492       KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2493       KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2494       KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2495       KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2496 
2497       Put put = null;
2498       put = new Put(row1);
2499       put.add(kv14);
2500       put.add(kv24);
2501       region.put(put);
2502       region.flushcache();
2503 
2504       put = new Put(row1);
2505       put.add(kv23);
2506       put.add(kv13);
2507       region.put(put);
2508       region.flushcache();
2509 
2510       put = new Put(row1);
2511       put.add(kv22);
2512       put.add(kv12);
2513       region.put(put);
2514       region.flushcache();
2515 
2516       put = new Put(row1);
2517       put.add(kv21);
2518       put.add(kv11);
2519       region.put(put);
2520 
2521       //Expected
2522       List<KeyValue> expected = new ArrayList<KeyValue>();
2523       expected.add(kv14);
2524       expected.add(kv13);
2525       expected.add(kv12);
2526       expected.add(kv24);
2527       expected.add(kv23);
2528       expected.add(kv22);
2529 
2530       Scan scan = new Scan(row1);
2531       int versions = 3;
2532       scan.setMaxVersions(versions);
2533       List<KeyValue> actual = new ArrayList<KeyValue>();
2534       InternalScanner scanner = region.getScanner(scan);
2535 
2536       boolean hasNext = scanner.next(actual);
2537       assertEquals(false, hasNext);
2538 
2539       //Verify result
2540       for(int i=0; i<expected.size(); i++) {
2541         assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2542       }
2543     } finally {
2544       HRegion.closeHRegion(this.region);
2545       this.region = null;
2546     }
2547   }
2548 
2549   /**
2550    * Added for HBASE-5416
2551    *
2552    * Here we test scan optimization when only subset of CFs are used in filter
2553    * conditions.
2554    */
2555   public void testScanner_JoinedScanners() throws IOException {
2556     byte [] tableName = Bytes.toBytes("testTable");
2557     byte [] cf_essential = Bytes.toBytes("essential");
2558     byte [] cf_joined = Bytes.toBytes("joined");
2559     byte [] cf_alpha = Bytes.toBytes("alpha");
2560     this.region = initHRegion(tableName, getName(), conf, cf_essential, cf_joined, cf_alpha);
2561     try {
2562       byte [] row1 = Bytes.toBytes("row1");
2563       byte [] row2 = Bytes.toBytes("row2");
2564       byte [] row3 = Bytes.toBytes("row3");
2565 
2566       byte [] col_normal = Bytes.toBytes("d");
2567       byte [] col_alpha = Bytes.toBytes("a");
2568 
2569       byte [] filtered_val = Bytes.toBytes(3);
2570 
2571       Put put = new Put(row1);
2572       put.add(cf_essential, col_normal, Bytes.toBytes(1));
2573       put.add(cf_joined, col_alpha, Bytes.toBytes(1));
2574       region.put(put);
2575 
2576       put = new Put(row2);
2577       put.add(cf_essential, col_alpha, Bytes.toBytes(2));
2578       put.add(cf_joined, col_normal, Bytes.toBytes(2));
2579       put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
2580       region.put(put);
2581 
2582       put = new Put(row3);
2583       put.add(cf_essential, col_normal, filtered_val);
2584       put.add(cf_joined, col_normal, filtered_val);
2585       region.put(put);
2586 
2587       // Check two things:
2588       // 1. result list contains expected values
2589       // 2. result list is sorted properly
2590 
2591       Scan scan = new Scan();
2592       Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
2593                                                          CompareOp.NOT_EQUAL, filtered_val);
2594       scan.setFilter(filter);
2595       scan.setLoadColumnFamiliesOnDemand(true);
2596       InternalScanner s = region.getScanner(scan);
2597 
2598       List<KeyValue> results = new ArrayList<KeyValue>();
2599       assertTrue(s.next(results));
2600       assertEquals(results.size(), 1);
2601       results.clear();
2602 
2603       assertTrue(s.next(results));
2604       assertEquals(results.size(), 3);
2605       assertTrue("orderCheck", results.get(0).matchingFamily(cf_alpha));
2606       assertTrue("orderCheck", results.get(1).matchingFamily(cf_essential));
2607       assertTrue("orderCheck", results.get(2).matchingFamily(cf_joined));
2608       results.clear();
2609 
2610       assertFalse(s.next(results));
2611       assertEquals(results.size(), 0);
2612     } finally {
2613       HRegion.closeHRegion(this.region);
2614       this.region = null;
2615     }
2616   }
2617 
2618   /**
2619    * HBASE-5416
2620    *
2621    * Test case when scan limits amount of KVs returned on each next() call.
2622    */
2623   public void testScanner_JoinedScannersWithLimits() throws IOException {
2624     final byte [] tableName = Bytes.toBytes("testTable");
2625     final byte [] cf_first = Bytes.toBytes("first");
2626     final byte [] cf_second = Bytes.toBytes("second");
2627 
2628     this.region = initHRegion(tableName, getName(), conf, cf_first, cf_second);
2629     try {
2630       final byte [] col_a = Bytes.toBytes("a");
2631       final byte [] col_b = Bytes.toBytes("b");
2632 
2633       Put put;
2634 
2635       for (int i = 0; i < 10; i++) {
2636         put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
2637         put.add(cf_first, col_a, Bytes.toBytes(i));
2638         if (i < 5) {
2639           put.add(cf_first, col_b, Bytes.toBytes(i));
2640           put.add(cf_second, col_a, Bytes.toBytes(i));
2641           put.add(cf_second, col_b, Bytes.toBytes(i));
2642         }
2643         region.put(put);
2644       }
2645 
2646       Scan scan = new Scan();
2647       scan.setLoadColumnFamiliesOnDemand(true);
2648       Filter bogusFilter = new FilterBase() {
2649         @Override
2650         public boolean isFamilyEssential(byte[] name) {
2651           return Bytes.equals(name, cf_first);
2652         }
2653       };
2654 
2655       scan.setFilter(bogusFilter);
2656       InternalScanner s = region.getScanner(scan);
2657 
2658       // Our data looks like this:
2659       // r0: first:a, first:b, second:a, second:b
2660       // r1: first:a, first:b, second:a, second:b
2661       // r2: first:a, first:b, second:a, second:b
2662       // r3: first:a, first:b, second:a, second:b
2663       // r4: first:a, first:b, second:a, second:b
2664       // r5: first:a
2665       // r6: first:a
2666       // r7: first:a
2667       // r8: first:a
2668       // r9: first:a
2669 
2670       // But due to next's limit set to 3, we should get this:
2671       // r0: first:a, first:b, second:a
2672       // r0: second:b
2673       // r1: first:a, first:b, second:a
2674       // r1: second:b
2675       // r2: first:a, first:b, second:a
2676       // r2: second:b
2677       // r3: first:a, first:b, second:a
2678       // r3: second:b
2679       // r4: first:a, first:b, second:a
2680       // r4: second:b
2681       // r5: first:a
2682       // r6: first:a
2683       // r7: first:a
2684       // r8: first:a
2685       // r9: first:a
2686 
2687       List<KeyValue> results = new ArrayList<KeyValue>();
2688       int index = 0;
2689       while (true) {
2690         boolean more = s.next(results, 3);
2691         if ((index >> 1) < 5) {
2692           if (index % 2 == 0)
2693             assertEquals(results.size(), 3);
2694           else
2695             assertEquals(results.size(), 1);
2696         }
2697         else
2698           assertEquals(results.size(), 1);
2699         results.clear();
2700         index++;
2701         if (!more) break;
2702       }
2703     } finally {
2704       HRegion.closeHRegion(this.region);
2705       this.region = null;
2706     }
2707   }
2708 
2709   //////////////////////////////////////////////////////////////////////////////
2710   // Split test
2711   //////////////////////////////////////////////////////////////////////////////
2712   /**
2713    * Splits twice and verifies getting from each of the split regions.
2714    * @throws Exception
2715    */
2716   public void testBasicSplit() throws Exception {
2717     byte [] tableName = Bytes.toBytes("testtable");
2718     byte [][] families = {fam1, fam2, fam3};
2719 
2720     Configuration hc = initSplit();
2721     //Setting up region
2722     String method = this.getName();
2723     this.region = initHRegion(tableName, method, hc, families);
2724 
2725     try {
2726       LOG.info("" + addContent(region, fam3));
2727       region.flushcache();
2728       region.compactStores();
2729       byte [] splitRow = region.checkSplit();
2730       assertNotNull(splitRow);
2731       LOG.info("SplitRow: " + Bytes.toString(splitRow));
2732       HRegion [] regions = splitRegion(region, splitRow);
2733       try {
2734         // Need to open the regions.
2735         // TODO: Add an 'open' to HRegion... don't do open by constructing
2736         // instance.
2737         for (int i = 0; i < regions.length; i++) {
2738           regions[i] = openClosedRegion(regions[i]);
2739         }
2740         // Assert can get rows out of new regions. Should be able to get first
2741         // row from first region and the midkey from second region.
2742         assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
2743         assertGet(regions[1], fam3, splitRow);
2744         // Test I can get scanner and that it starts at right place.
2745         assertScan(regions[0], fam3,
2746             Bytes.toBytes(START_KEY));
2747         assertScan(regions[1], fam3, splitRow);
2748         // Now prove can't split regions that have references.
2749         for (int i = 0; i < regions.length; i++) {
2750           // Add so much data to this region, we create a store file that is >
2751           // than one of our unsplitable references. it will.
2752           for (int j = 0; j < 2; j++) {
2753             addContent(regions[i], fam3);
2754           }
2755           addContent(regions[i], fam2);
2756           addContent(regions[i], fam1);
2757           regions[i].flushcache();
2758         }
2759 
2760         byte [][] midkeys = new byte [regions.length][];
2761         // To make regions splitable force compaction.
2762         for (int i = 0; i < regions.length; i++) {
2763           regions[i].compactStores();
2764           midkeys[i] = regions[i].checkSplit();
2765         }
2766 
2767         TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
2768         // Split these two daughter regions so then I'll have 4 regions. Will
2769         // split because added data above.
2770         for (int i = 0; i < regions.length; i++) {
2771           HRegion[] rs = null;
2772           if (midkeys[i] != null) {
2773             rs = splitRegion(regions[i], midkeys[i]);
2774             for (int j = 0; j < rs.length; j++) {
2775               sortedMap.put(Bytes.toString(rs[j].getRegionName()),
2776                 openClosedRegion(rs[j]));
2777             }
2778           }
2779         }
2780         LOG.info("Made 4 regions");
2781         // The splits should have been even. Test I can get some arbitrary row
2782         // out of each.
2783         int interval = (LAST_CHAR - FIRST_CHAR) / 3;
2784         byte[] b = Bytes.toBytes(START_KEY);
2785         for (HRegion r : sortedMap.values()) {
2786           assertGet(r, fam3, b);
2787           b[0] += interval;
2788         }
2789       } finally {
2790         for (int i = 0; i < regions.length; i++) {
2791           try {
2792             regions[i].close();
2793           } catch (IOException e) {
2794             // Ignore.
2795           }
2796         }
2797       }
2798     } finally {
2799       HRegion.closeHRegion(this.region);
2800       this.region = null;
2801     }
2802   }
2803 
2804   public void testSplitRegion() throws IOException {
2805     byte [] tableName = Bytes.toBytes("testtable");
2806     byte [] qualifier = Bytes.toBytes("qualifier");
2807     Configuration hc = initSplit();
2808     int numRows = 10;
2809     byte [][] families = {fam1, fam3};
2810 
2811     //Setting up region
2812     String method = this.getName();
2813     this.region = initHRegion(tableName, method, hc, families);
2814 
2815     //Put data in region
2816     int startRow = 100;
2817     putData(startRow, numRows, qualifier, families);
2818     int splitRow = startRow + numRows;
2819     putData(splitRow, numRows, qualifier, families);
2820     region.flushcache();
2821 
2822     HRegion [] regions = null;
2823     try {
2824       regions = splitRegion(region, Bytes.toBytes("" + splitRow));
2825       //Opening the regions returned.
2826       for (int i = 0; i < regions.length; i++) {
2827         regions[i] = openClosedRegion(regions[i]);
2828       }
2829       //Verifying that the region has been split
2830       assertEquals(2, regions.length);
2831 
2832       //Verifying that all data is still there and that data is in the right
2833       //place
2834       verifyData(regions[0], startRow, numRows, qualifier, families);
2835       verifyData(regions[1], splitRow, numRows, qualifier, families);
2836 
2837     } finally {
2838       HRegion.closeHRegion(this.region);
2839       this.region = null;
2840     }
2841   }
2842 
2843 
2844   /**
2845    * Flushes the cache in a thread while scanning. The tests verify that the
2846    * scan is coherent - e.g. the returned results are always of the same or
2847    * later update as the previous results.
2848    * @throws IOException scan / compact
2849    * @throws InterruptedException thread join
2850    */
2851   public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
2852     byte[] tableName = Bytes.toBytes("testFlushCacheWhileScanning");
2853     byte[] family = Bytes.toBytes("family");
2854     int numRows = 1000;
2855     int flushAndScanInterval = 10;
2856     int compactInterval = 10 * flushAndScanInterval;
2857 
2858     String method = "testFlushCacheWhileScanning";
2859     this.region = initHRegion(tableName,method, conf, family);
2860     try {
2861       FlushThread flushThread = new FlushThread();
2862       flushThread.start();
2863 
2864       Scan scan = new Scan();
2865       scan.addFamily(family);
2866       scan.setFilter(new SingleColumnValueFilter(family, qual1,
2867           CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(5L))));
2868 
2869       int expectedCount = 0;
2870       List<KeyValue> res = new ArrayList<KeyValue>();
2871 
2872       boolean toggle=true;
2873       for (long i = 0; i < numRows; i++) {
2874         Put put = new Put(Bytes.toBytes(i));
2875         put.setDurability(Durability.SKIP_WAL);
2876         put.add(family, qual1, Bytes.toBytes(i % 10));
2877         region.put(put);
2878 
2879         if (i != 0 && i % compactInterval == 0) {
2880           //System.out.println("iteration = " + i);
2881           region.compactStores(true);
2882         }
2883 
2884         if (i % 10 == 5L) {
2885           expectedCount++;
2886         }
2887 
2888         if (i != 0 && i % flushAndScanInterval == 0) {
2889           res.clear();
2890           InternalScanner scanner = region.getScanner(scan);
2891           if (toggle) {
2892             flushThread.flush();
2893           }
2894           while (scanner.next(res)) ;
2895           if (!toggle) {
2896             flushThread.flush();
2897           }
2898           assertEquals("i=" + i, expectedCount, res.size());
2899           toggle = !toggle;
2900         }
2901       }
2902 
2903       flushThread.done();
2904       flushThread.join();
2905       flushThread.checkNoError();
2906     } finally {
2907       HRegion.closeHRegion(this.region);
2908       this.region = null;
2909     }
2910   }
2911 
2912   protected class FlushThread extends Thread {
2913     private volatile boolean done;
2914     private Throwable error = null;
2915 
2916     public void done() {
2917       done = true;
2918       synchronized (this) {
2919         interrupt();
2920       }
2921     }
2922 
2923     public void checkNoError() {
2924       if (error != null) {
2925         assertNull(error);
2926       }
2927     }
2928 
2929     @Override
2930     public void run() {
2931       done = false;
2932       while (!done) {
2933         synchronized (this) {
2934           try {
2935             wait();
2936           } catch (InterruptedException ignored) {
2937             if (done) {
2938               break;
2939             }
2940           }
2941         }
2942         try {
2943           region.flushcache();
2944         } catch (IOException e) {
2945           if (!done) {
2946             LOG.error("Error while flusing cache", e);
2947             error = e;
2948           }
2949           break;
2950         }
2951       }
2952 
2953     }
2954 
2955     public void flush() {
2956       synchronized (this) {
2957         notify();
2958       }
2959 
2960     }
2961   }
2962 
2963   /**
2964    * Writes very wide records and scans for the latest every time..
2965    * Flushes and compacts the region every now and then to keep things
2966    * realistic.
2967    *
2968    * @throws IOException          by flush / scan / compaction
2969    * @throws InterruptedException when joining threads
2970    */
2971   public void testWritesWhileScanning()
2972     throws IOException, InterruptedException {
2973     byte[] tableName = Bytes.toBytes("testWritesWhileScanning");
2974     int testCount = 100;
2975     int numRows = 1;
2976     int numFamilies = 10;
2977     int numQualifiers = 100;
2978     int flushInterval = 7;
2979     int compactInterval = 5 * flushInterval;
2980     byte[][] families = new byte[numFamilies][];
2981     for (int i = 0; i < numFamilies; i++) {
2982       families[i] = Bytes.toBytes("family" + i);
2983     }
2984     byte[][] qualifiers = new byte[numQualifiers][];
2985     for (int i = 0; i < numQualifiers; i++) {
2986       qualifiers[i] = Bytes.toBytes("qual" + i);
2987     }
2988 
2989     String method = "testWritesWhileScanning";
2990     this.region = initHRegion(tableName, method, conf, families);
2991     try {
2992       PutThread putThread = new PutThread(numRows, families, qualifiers);
2993       putThread.start();
2994       putThread.waitForFirstPut();
2995 
2996       FlushThread flushThread = new FlushThread();
2997       flushThread.start();
2998 
2999       Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3000       //    scan.setFilter(new RowFilter(CompareFilter.CompareOp.EQUAL,
3001       //      new BinaryComparator(Bytes.toBytes("row0"))));
3002 
3003       int expectedCount = numFamilies * numQualifiers;
3004       List<KeyValue> res = new ArrayList<KeyValue>();
3005 
3006       long prevTimestamp = 0L;
3007       for (int i = 0; i < testCount; i++) {
3008 
3009         if (i != 0 && i % compactInterval == 0) {
3010           region.compactStores(true);
3011         }
3012 
3013         if (i != 0 && i % flushInterval == 0) {
3014           //System.out.println("flush scan iteration = " + i);
3015           flushThread.flush();
3016         }
3017 
3018         boolean previousEmpty = res.isEmpty();
3019         res.clear();
3020         InternalScanner scanner = region.getScanner(scan);
3021         while (scanner.next(res)) ;
3022         if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3023           assertEquals("i=" + i, expectedCount, res.size());
3024           long timestamp = res.get(0).getTimestamp();
3025           assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3026               timestamp >= prevTimestamp);
3027           prevTimestamp = timestamp;
3028         }
3029       }
3030 
3031       putThread.done();
3032 
3033       region.flushcache();
3034 
3035       putThread.join();
3036       putThread.checkNoError();
3037 
3038       flushThread.done();
3039       flushThread.join();
3040       flushThread.checkNoError();
3041     } finally {
3042       HRegion.closeHRegion(this.region);
3043       this.region = null;
3044     }
3045   }
3046 
3047   protected class PutThread extends Thread {
3048     private volatile boolean done;
3049     private volatile int numPutsFinished = 0;
3050 
3051     private Throwable error = null;
3052     private int numRows;
3053     private byte[][] families;
3054     private byte[][] qualifiers;
3055 
3056     private PutThread(int numRows, byte[][] families,
3057       byte[][] qualifiers) {
3058       this.numRows = numRows;
3059       this.families = families;
3060       this.qualifiers = qualifiers;
3061     }
3062 
3063     /**
3064      * Block until this thread has put at least one row.
3065      */
3066     public void waitForFirstPut() throws InterruptedException {
3067       // wait until put thread actually puts some data
3068       while (numPutsFinished == 0) {
3069         checkNoError();
3070         Thread.sleep(50);
3071       }
3072     }
3073 
3074     public void done() {
3075       done = true;
3076       synchronized (this) {
3077         interrupt();
3078       }
3079     }
3080 
3081     public void checkNoError() {
3082       if (error != null) {
3083         assertNull(error);
3084       }
3085     }
3086 
3087     @Override
3088     public void run() {
3089       done = false;
3090       while (!done) {
3091         try {
3092           for (int r = 0; r < numRows; r++) {
3093             byte[] row = Bytes.toBytes("row" + r);
3094             Put put = new Put(row);
3095             put.setDurability(Durability.SKIP_WAL);
3096             byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3097             for (byte[] family : families) {
3098               for (byte[] qualifier : qualifiers) {
3099                 put.add(family, qualifier, (long) numPutsFinished, value);
3100               }
3101             }
3102 //            System.out.println("Putting of kvsetsize=" + put.size());
3103             region.put(put);
3104             numPutsFinished++;
3105             if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3106               System.out.println("put iteration = " + numPutsFinished);
3107               Delete delete = new Delete(row, (long)numPutsFinished-30);
3108               region.delete(delete);
3109             }
3110             numPutsFinished++;
3111           }
3112         } catch (InterruptedIOException e) {
3113           // This is fine. It means we are done, or didn't get the lock on time
3114         } catch (IOException e) {
3115           LOG.error("error while putting records", e);
3116           error = e;
3117           break;
3118         }
3119       }
3120 
3121     }
3122 
3123   }
3124 
3125 
3126   /**
3127    * Writes very wide records and gets the latest row every time..
3128    * Flushes and compacts the region aggressivly to catch issues.
3129    *
3130    * @throws IOException          by flush / scan / compaction
3131    * @throws InterruptedException when joining threads
3132    */
3133   public void testWritesWhileGetting()
3134     throws Exception {
3135     byte[] tableName = Bytes.toBytes("testWritesWhileGetting");
3136     int testCount = 100;
3137     int numRows = 1;
3138     int numFamilies = 10;
3139     int numQualifiers = 100;
3140     int compactInterval = 100;
3141     byte[][] families = new byte[numFamilies][];
3142     for (int i = 0; i < numFamilies; i++) {
3143       families[i] = Bytes.toBytes("family" + i);
3144     }
3145     byte[][] qualifiers = new byte[numQualifiers][];
3146     for (int i = 0; i < numQualifiers; i++) {
3147       qualifiers[i] = Bytes.toBytes("qual" + i);
3148     }
3149 
3150     Configuration conf = HBaseConfiguration.create(this.conf);
3151 
3152     String method = "testWritesWhileGetting";
3153     // This test flushes constantly and can cause many files to be created, possibly
3154     // extending over the ulimit.  Make sure compactions are aggressive in reducing
3155     // the number of HFiles created.
3156     conf.setInt("hbase.hstore.compaction.min", 1);
3157     conf.setInt("hbase.hstore.compaction.max", 1000);
3158     this.region = initHRegion(tableName, method, conf, families);
3159     PutThread putThread = null;
3160     MultithreadedTestUtil.TestContext ctx =
3161       new MultithreadedTestUtil.TestContext(conf);
3162     try {
3163       putThread = new PutThread(numRows, families, qualifiers);
3164       putThread.start();
3165       putThread.waitForFirstPut();
3166 
3167       // Add a thread that flushes as fast as possible
3168       ctx.addThread(new RepeatingTestThread(ctx) {
3169     	private int flushesSinceCompact = 0;
3170     	private final int maxFlushesSinceCompact = 20;
3171         @Override
3172         public void doAnAction() throws Exception {
3173           if (region.flushcache()) {
3174             ++flushesSinceCompact;
3175           }
3176           // Compact regularly to avoid creating too many files and exceeding the ulimit.
3177           if (flushesSinceCompact == maxFlushesSinceCompact) {
3178             region.compactStores(false);
3179             flushesSinceCompact = 0;
3180           }
3181         }
3182       });
3183       ctx.startThreads();
3184 
3185       Get get = new Get(Bytes.toBytes("row0"));
3186       Result result = null;
3187 
3188       int expectedCount = numFamilies * numQualifiers;
3189 
3190       long prevTimestamp = 0L;
3191       for (int i = 0; i < testCount; i++) {
3192 
3193         boolean previousEmpty = result == null || result.isEmpty();
3194         result = region.get(get);
3195         if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3196           assertEquals("i=" + i, expectedCount, result.size());
3197           // TODO this was removed, now what dangit?!
3198           // search looking for the qualifier in question?
3199           long timestamp = 0;
3200           for (KeyValue kv : result.raw()) {
3201             if (Bytes.equals(kv.getFamily(), families[0])
3202                 && Bytes.equals(kv.getQualifier(), qualifiers[0])) {
3203               timestamp = kv.getTimestamp();
3204             }
3205           }
3206           assertTrue(timestamp >= prevTimestamp);
3207           prevTimestamp = timestamp;
3208           KeyValue previousKV = null;
3209 
3210           for (KeyValue kv : result.raw()) {
3211             byte[] thisValue = kv.getValue();
3212             if (previousKV != null) {
3213               if (Bytes.compareTo(previousKV.getValue(), thisValue) != 0) {
3214                 LOG.warn("These two KV should have the same value." +
3215                     " Previous KV:" +
3216                     previousKV + "(memStoreTS:" + previousKV.getMvccVersion() + ")" +
3217                     ", New KV: " +
3218                     kv + "(memStoreTS:" + kv.getMvccVersion() + ")"
3219                     );
3220                 assertEquals(0, Bytes.compareTo(previousKV.getValue(), thisValue));
3221               }
3222             }
3223             previousKV = kv;
3224           }
3225         }
3226       }
3227     } finally {
3228       if (putThread != null) putThread.done();
3229 
3230       region.flushcache();
3231 
3232       if (putThread != null) {
3233         putThread.join();
3234         putThread.checkNoError();
3235       }
3236 
3237       ctx.stop();
3238       HRegion.closeHRegion(this.region);
3239       this.region = null;
3240     }
3241   }
3242 
3243   public void testHolesInMeta() throws Exception {
3244     String method = "testHolesInMeta";
3245     byte[] tableName = Bytes.toBytes(method);
3246     byte[] family = Bytes.toBytes("family");
3247     this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method,
3248         conf, false, family);
3249     try {
3250       byte[] rowNotServed = Bytes.toBytes("a");
3251       Get g = new Get(rowNotServed);
3252       try {
3253         region.get(g);
3254         fail();
3255       } catch (WrongRegionException x) {
3256         // OK
3257       }
3258       byte[] row = Bytes.toBytes("y");
3259       g = new Get(row);
3260       region.get(g);
3261     } finally {
3262       HRegion.closeHRegion(this.region);
3263       this.region = null;
3264     }
3265   }
3266 
3267   public void testIndexesScanWithOneDeletedRow() throws IOException {
3268     byte[] tableName = Bytes.toBytes("testIndexesScanWithOneDeletedRow");
3269     byte[] family = Bytes.toBytes("family");
3270 
3271     //Setting up region
3272     String method = "testIndexesScanWithOneDeletedRow";
3273     this.region = initHRegion(tableName, method, conf, family);
3274     try {
3275       Put put = new Put(Bytes.toBytes(1L));
3276       put.add(family, qual1, 1L, Bytes.toBytes(1L));
3277       region.put(put);
3278 
3279       region.flushcache();
3280 
3281       Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3282       //delete.deleteColumn(family, qual1);
3283       region.delete(delete);
3284 
3285       put = new Put(Bytes.toBytes(2L));
3286       put.add(family, qual1, 2L, Bytes.toBytes(2L));
3287       region.put(put);
3288 
3289       Scan idxScan = new Scan();
3290       idxScan.addFamily(family);
3291       idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,
3292           Arrays.<Filter>asList(new SingleColumnValueFilter(family, qual1,
3293               CompareOp.GREATER_OR_EQUAL,
3294               new BinaryComparator(Bytes.toBytes(0L))),
3295               new SingleColumnValueFilter(family, qual1, CompareOp.LESS_OR_EQUAL,
3296                   new BinaryComparator(Bytes.toBytes(3L)))
3297               )));
3298       InternalScanner scanner = region.getScanner(idxScan);
3299       List<KeyValue> res = new ArrayList<KeyValue>();
3300 
3301       //long start = System.nanoTime();
3302       while (scanner.next(res)) ;
3303       //long end = System.nanoTime();
3304       //System.out.println("memStoreEmpty=" + memStoreEmpty + ", time=" + (end - start)/1000000D);
3305       assertEquals(1L, res.size());
3306     } finally {
3307       HRegion.closeHRegion(this.region);
3308       this.region = null;
3309     }
3310   }
3311 
3312   //////////////////////////////////////////////////////////////////////////////
3313   // Bloom filter test
3314   //////////////////////////////////////////////////////////////////////////////
3315   public void testBloomFilterSize() throws IOException {
3316     byte [] tableName = Bytes.toBytes("testBloomFilterSize");
3317     byte [] row1 = Bytes.toBytes("row1");
3318     byte [] fam1 = Bytes.toBytes("fam1");
3319     byte [] qf1  = Bytes.toBytes("col");
3320     byte [] val1  = Bytes.toBytes("value1");
3321     // Create Table
3322     HColumnDescriptor hcd = new HColumnDescriptor(fam1)
3323         .setMaxVersions(Integer.MAX_VALUE)
3324         .setBloomFilterType(BloomType.ROWCOL);
3325 
3326     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3327     htd.addFamily(hcd);
3328     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3329     Path path = new Path(DIR + "testBloomFilterSize");
3330     this.region = HRegion.createHRegion(info, path, conf, htd);
3331     try {
3332       int num_unique_rows = 10;
3333       int duplicate_multiplier =2;
3334       int num_storefiles = 4;
3335 
3336       int version = 0;
3337       for (int f =0 ; f < num_storefiles; f++) {
3338         for (int i = 0; i < duplicate_multiplier; i ++) {
3339           for (int j = 0; j < num_unique_rows; j++) {
3340             Put put = new Put(Bytes.toBytes("row" + j));
3341             put.setDurability(Durability.SKIP_WAL);
3342             put.add(fam1, qf1, version++, val1);
3343             region.put(put);
3344           }
3345         }
3346         region.flushcache();
3347       }
3348       //before compaction
3349       HStore store = (HStore) region.getStore(fam1);
3350       Collection<StoreFile> storeFiles = store.getStorefiles();
3351       for (StoreFile storefile : storeFiles) {
3352         StoreFile.Reader reader = storefile.getReader();
3353         reader.loadFileInfo();
3354         reader.loadBloomfilter();
3355         assertEquals(num_unique_rows*duplicate_multiplier, reader.getEntries());
3356         assertEquals(num_unique_rows, reader.getFilterEntries());
3357       }
3358 
3359       region.compactStores(true);
3360 
3361       //after compaction
3362       storeFiles = store.getStorefiles();
3363       for (StoreFile storefile : storeFiles) {
3364         StoreFile.Reader reader = storefile.getReader();
3365         reader.loadFileInfo();
3366         reader.loadBloomfilter();
3367         assertEquals(num_unique_rows*duplicate_multiplier*num_storefiles,
3368             reader.getEntries());
3369         assertEquals(num_unique_rows, reader.getFilterEntries());
3370       }
3371     } finally {
3372       HRegion.closeHRegion(this.region);
3373       this.region = null;
3374     }
3375   }
3376 
3377   public void testAllColumnsWithBloomFilter() throws IOException {
3378     byte [] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
3379     byte [] FAMILY = Bytes.toBytes("family");
3380 
3381     //Create table
3382     HColumnDescriptor hcd = new HColumnDescriptor(FAMILY)
3383         .setMaxVersions(Integer.MAX_VALUE)
3384         .setBloomFilterType(BloomType.ROWCOL);
3385     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
3386     htd.addFamily(hcd);
3387     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3388     Path path = new Path(DIR + "testAllColumnsWithBloomFilter");
3389     this.region = HRegion.createHRegion(info, path, conf, htd);
3390     try {
3391       // For row:0, col:0: insert versions 1 through 5.
3392       byte row[] = Bytes.toBytes("row:" + 0);
3393       byte column[] = Bytes.toBytes("column:" + 0);
3394       Put put = new Put(row);
3395       put.setDurability(Durability.SKIP_WAL);
3396       for (long idx = 1; idx <= 4; idx++) {
3397         put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
3398       }
3399       region.put(put);
3400 
3401       //Flush
3402       region.flushcache();
3403 
3404       //Get rows
3405       Get get = new Get(row);
3406       get.setMaxVersions();
3407       KeyValue[] kvs = region.get(get).raw();
3408 
3409       //Check if rows are correct
3410       assertEquals(4, kvs.length);
3411       checkOneCell(kvs[0], FAMILY, 0, 0, 4);
3412       checkOneCell(kvs[1], FAMILY, 0, 0, 3);
3413       checkOneCell(kvs[2], FAMILY, 0, 0, 2);
3414       checkOneCell(kvs[3], FAMILY, 0, 0, 1);
3415     } finally {
3416       HRegion.closeHRegion(this.region);
3417       this.region = null;
3418     }
3419   }
3420 
3421   /**
3422     * Testcase to cover bug-fix for HBASE-2823
3423     * Ensures correct delete when issuing delete row
3424     * on columns with bloom filter set to row+col (BloomType.ROWCOL)
3425    */
3426   public void testDeleteRowWithBloomFilter() throws IOException {
3427     byte [] tableName = Bytes.toBytes("testDeleteRowWithBloomFilter");
3428     byte [] familyName = Bytes.toBytes("familyName");
3429 
3430     // Create Table
3431     HColumnDescriptor hcd = new HColumnDescriptor(familyName)
3432         .setMaxVersions(Integer.MAX_VALUE)
3433         .setBloomFilterType(BloomType.ROWCOL);
3434 
3435     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3436     htd.addFamily(hcd);
3437     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3438     Path path = new Path(DIR + "TestDeleteRowWithBloomFilter");
3439     this.region = HRegion.createHRegion(info, path, conf, htd);
3440     try {
3441       // Insert some data
3442       byte row[] = Bytes.toBytes("row1");
3443       byte col[] = Bytes.toBytes("col1");
3444 
3445       Put put = new Put(row);
3446       put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
3447       region.put(put);
3448       region.flushcache();
3449 
3450       Delete del = new Delete(row);
3451       region.delete(del);
3452       region.flushcache();
3453 
3454       // Get remaining rows (should have none)
3455       Get get = new Get(row);
3456       get.addColumn(familyName, col);
3457 
3458       KeyValue[] keyValues = region.get(get).raw();
3459       assertTrue(keyValues.length == 0);
3460     } finally {
3461       HRegion.closeHRegion(this.region);
3462       this.region = null;
3463     }
3464   }
3465 
3466   @Test public void testgetHDFSBlocksDistribution() throws Exception {
3467     HBaseTestingUtility htu = new HBaseTestingUtility();
3468     final int DEFAULT_BLOCK_SIZE = 1024;
3469     htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
3470     htu.getConfiguration().setInt("dfs.replication", 2);
3471 
3472 
3473     // set up a cluster with 3 nodes
3474     MiniHBaseCluster cluster = null;
3475     String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
3476     int regionServersCount = 3;
3477 
3478     try {
3479       cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
3480       byte [][] families = {fam1, fam2};
3481       HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);
3482 
3483       //Setting up region
3484       byte row[] = Bytes.toBytes("row1");
3485       byte col[] = Bytes.toBytes("col1");
3486 
3487       Put put = new Put(row);
3488       put.add(fam1, col, 1, Bytes.toBytes("test1"));
3489       put.add(fam2, col, 1, Bytes.toBytes("test2"));
3490       ht.put(put);
3491 
3492       HRegion firstRegion = htu.getHBaseCluster().
3493           getRegions(TableName.valueOf(this.getName())).get(0);
3494       firstRegion.flushcache();
3495       HDFSBlocksDistribution blocksDistribution1 =
3496           firstRegion.getHDFSBlocksDistribution();
3497 
3498       // given the default replication factor is 2 and we have 2 HFiles,
3499       // we will have total of 4 replica of blocks on 3 datanodes; thus there
3500       // must be at least one host that have replica for 2 HFiles. That host's
3501       // weight will be equal to the unique block weight.
3502       long uniqueBlocksWeight1 =
3503           blocksDistribution1.getUniqueBlocksTotalWeight();
3504 
3505       String topHost = blocksDistribution1.getTopHosts().get(0);
3506       long topHostWeight = blocksDistribution1.getWeight(topHost);
3507       assertTrue(uniqueBlocksWeight1 == topHostWeight);
3508 
3509       // use the static method to compute the value, it should be the same.
3510       // static method is used by load balancer or other components
3511       HDFSBlocksDistribution blocksDistribution2 =
3512         HRegion.computeHDFSBlocksDistribution(htu.getConfiguration(),
3513         firstRegion.getTableDesc(), firstRegion.getRegionInfo());
3514       long uniqueBlocksWeight2 =
3515         blocksDistribution2.getUniqueBlocksTotalWeight();
3516 
3517       assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
3518 
3519       ht.close();
3520       } finally {
3521         if (cluster != null) {
3522           htu.shutdownMiniCluster();
3523         }
3524       }
3525   }
3526 
3527   /**
3528    * Testcase to check state of region initialization task set to ABORTED or not if any exceptions
3529    * during initialization
3530    *
3531    * @throws Exception
3532    */
3533   @Test
3534   public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
3535     HRegionInfo info = null;
3536     try {
3537       FileSystem fs = Mockito.mock(FileSystem.class);
3538       Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
3539       HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3540       htd.addFamily(new HColumnDescriptor("cf"));
3541       info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
3542           HConstants.EMPTY_BYTE_ARRAY, false);
3543       Path path = new Path(DIR + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
3544       region = HRegion.newHRegion(path, null, fs, conf, info, htd, null);
3545       // region initialization throws IOException and set task state to ABORTED.
3546       region.initialize();
3547       fail("Region initialization should fail due to IOException");
3548     } catch (IOException io) {
3549       List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
3550       for (MonitoredTask monitoredTask : tasks) {
3551         if (!(monitoredTask instanceof MonitoredRPCHandler)
3552             && monitoredTask.getDescription().contains(region.toString())) {
3553           assertTrue("Region state should be ABORTED.",
3554               monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
3555           break;
3556         }
3557       }
3558     } finally {
3559       HRegion.closeHRegion(region);
3560     }
3561   }
3562 
3563   /**
3564    * Verifies that the .regioninfo file is written on region creation
3565    * and that is recreated if missing during region opening.
3566    */
3567   public void testRegionInfoFileCreation() throws IOException {
3568     Path rootDir = new Path(DIR + "testRegionInfoFileCreation");
3569     Configuration conf = HBaseConfiguration.create(this.conf);
3570 
3571     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
3572     htd.addFamily(new HColumnDescriptor("cf"));
3573 
3574     HRegionInfo hri = new HRegionInfo(htd.getTableName());
3575 
3576     // Create a region and skip the initialization (like CreateTableHandler)
3577     HRegion region = HRegion.createHRegion(hri, rootDir, conf, htd, null, false, true);
3578     Path regionDir = region.getRegionFileSystem().getRegionDir();
3579     FileSystem fs = region.getRegionFileSystem().getFileSystem();
3580     HRegion.closeHRegion(region);
3581 
3582     Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
3583 
3584     // Verify that the .regioninfo file is present
3585     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3586       fs.exists(regionInfoFile));
3587 
3588     // Try to open the region
3589     region = HRegion.openHRegion(rootDir, hri, htd, null, conf);
3590     assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3591     HRegion.closeHRegion(region);
3592 
3593     // Verify that the .regioninfo file is still there
3594     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3595       fs.exists(regionInfoFile));
3596 
3597     // Remove the .regioninfo file and verify is recreated on region open
3598     fs.delete(regionInfoFile);
3599     assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
3600       fs.exists(regionInfoFile));
3601 
3602     region = HRegion.openHRegion(rootDir, hri, htd, null, conf);
3603     assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3604     HRegion.closeHRegion(region);
3605 
3606     // Verify that the .regioninfo file is still there
3607     assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3608       fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
3609   }
3610 
3611   /**
3612    * TestCase for increment
3613    *
3614    */
3615   private static class Incrementer implements Runnable {
3616     private HRegion region;
3617     private final static byte[] incRow = Bytes.toBytes("incRow");
3618     private final static byte[] family = Bytes.toBytes("family");
3619     private final static byte[] qualifier = Bytes.toBytes("qualifier");
3620     private final static long ONE = 1l;
3621     private int incCounter;
3622 
3623     public Incrementer(HRegion region, int incCounter) {
3624       this.region = region;
3625       this.incCounter = incCounter;
3626     }
3627 
3628     @Override
3629     public void run() {
3630       int count = 0;
3631       while (count < incCounter) {
3632         Increment inc = new Increment(incRow);
3633         inc.addColumn(family, qualifier, ONE);
3634         count++;
3635         try {
3636           region.increment(inc);
3637         } catch (IOException e) {
3638           e.printStackTrace();
3639           break;
3640         }
3641       }
3642     }
3643   }
3644 
3645   /**
3646    * Test case to check increment function with memstore flushing
3647    * @throws Exception
3648    */
3649   @Test
3650   public void testParallelIncrementWithMemStoreFlush() throws Exception {
3651     String method = "testParallelIncrementWithMemStoreFlush";
3652     byte[] tableName = Bytes.toBytes(method);
3653     byte[] family = Incrementer.family;
3654     this.region = initHRegion(tableName, method, conf, family);
3655     final HRegion region = this.region;
3656     final AtomicBoolean incrementDone = new AtomicBoolean(false);
3657     Runnable flusher = new Runnable() {
3658       @Override
3659       public void run() {
3660         while (!incrementDone.get()) {
3661           try {
3662             region.flushcache();
3663           } catch (Exception e) {
3664             e.printStackTrace();
3665           }
3666         }
3667       }
3668     };
3669 
3670     //after all increment finished, the row will increment to 20*100 = 2000
3671     int threadNum = 20;
3672     int incCounter = 100;
3673     long expected = threadNum * incCounter;
3674     Thread[] incrementers = new Thread[threadNum];
3675     Thread flushThread = new Thread(flusher);
3676     for (int i = 0; i < threadNum; i++) {
3677       incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
3678       incrementers[i].start();
3679     }
3680     flushThread.start();
3681     for (int i = 0; i < threadNum; i++) {
3682       incrementers[i].join();
3683     }
3684 
3685     incrementDone.set(true);
3686     flushThread.join();
3687 
3688     Get get = new Get(Incrementer.incRow);
3689     get.addColumn(Incrementer.family, Incrementer.qualifier);
3690     get.setMaxVersions(1);
3691     Result res = this.region.get(get);
3692     List<KeyValue> kvs = res.getColumn(Incrementer.family,
3693         Incrementer.qualifier);
3694 
3695     //we just got the latest version
3696     assertEquals(kvs.size(), 1);
3697     KeyValue kv = kvs.get(0);
3698     assertEquals(expected, Bytes.toLong(kv.getBuffer(), kv.getValueOffset()));
3699     this.region = null;
3700   }
3701 
3702   /**
3703    * TestCase for append
3704    *
3705    */
3706   private static class Appender implements Runnable {
3707     private HRegion region;
3708     private final static byte[] appendRow = Bytes.toBytes("appendRow");
3709     private final static byte[] family = Bytes.toBytes("family");
3710     private final static byte[] qualifier = Bytes.toBytes("qualifier");
3711     private final static byte[] CHAR = Bytes.toBytes("a");
3712     private int appendCounter;
3713 
3714     public Appender(HRegion region, int appendCounter) {
3715       this.region = region;
3716       this.appendCounter = appendCounter;
3717     }
3718 
3719     @Override
3720     public void run() {
3721       int count = 0;
3722       while (count < appendCounter) {
3723         Append app = new Append(appendRow);
3724         app.add(family, qualifier, CHAR);
3725         count++;
3726         try {
3727           region.append(app);
3728         } catch (IOException e) {
3729           e.printStackTrace();
3730           break;
3731         }
3732       }
3733     }
3734   }
3735 
3736   /**
3737    * Test case to check append function with memstore flushing
3738    * @throws Exception
3739    */
3740   @Test
3741   public void testParallelAppendWithMemStoreFlush() throws Exception {
3742     String method = "testParallelAppendWithMemStoreFlush";
3743     byte[] tableName = Bytes.toBytes(method);
3744     byte[] family = Appender.family;
3745     this.region = initHRegion(tableName, method, conf, family);
3746     final HRegion region = this.region;
3747     final AtomicBoolean appendDone = new AtomicBoolean(false);
3748     Runnable flusher = new Runnable() {
3749       @Override
3750       public void run() {
3751         while (!appendDone.get()) {
3752           try {
3753             region.flushcache();
3754           } catch (Exception e) {
3755             e.printStackTrace();
3756           }
3757         }
3758       }
3759     };
3760 
3761     //after all append finished, the value will append to threadNum * appendCounter Appender.CHAR
3762     int threadNum = 20;
3763     int appendCounter = 100;
3764     byte[] expected = new byte[threadNum * appendCounter];
3765     for (int i = 0; i < threadNum * appendCounter; i++) {
3766       System.arraycopy(Appender.CHAR, 0, expected, i, 1);
3767     }
3768     Thread[] appenders = new Thread[threadNum];
3769     Thread flushThread = new Thread(flusher);
3770     for (int i = 0; i < threadNum; i++) {
3771       appenders[i] = new Thread(new Appender(this.region, appendCounter));
3772       appenders[i].start();
3773     }
3774     flushThread.start();
3775     for (int i = 0; i < threadNum; i++) {
3776       appenders[i].join();
3777     }
3778 
3779     appendDone.set(true);
3780     flushThread.join();
3781 
3782     Get get = new Get(Appender.appendRow);
3783     get.addColumn(Appender.family, Appender.qualifier);
3784     get.setMaxVersions(1);
3785     Result res = this.region.get(get);
3786     List<KeyValue> kvs = res.getColumn(Appender.family,
3787         Appender.qualifier);
3788 
3789     //we just got the latest version
3790     assertEquals(kvs.size(), 1);
3791     KeyValue kv = kvs.get(0);
3792     byte[] appendResult = new byte[kv.getValueLength()];
3793     System.arraycopy(kv.getBuffer(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
3794     assertEquals(expected, appendResult);
3795     this.region = null;
3796   }
3797 
3798   /**
3799    * Test case to check put function with memstore flushing for same row, same ts
3800    * @throws Exception
3801    */
3802   public void testPutWithMemStoreFlush() throws Exception {
3803     String method = "testPutWithMemStoreFlush";
3804     byte[] tableName = Bytes.toBytes(method);
3805     byte[] family = Bytes.toBytes("family");;
3806     byte[] qualifier = Bytes.toBytes("qualifier");
3807     byte[] row = Bytes.toBytes("putRow");
3808     byte[] value = null;
3809     this.region = initHRegion(tableName, method, conf, family);
3810     Put put = null;
3811     Get get = null;
3812     List<KeyValue> kvs = null;
3813     Result res = null;
3814 
3815     put = new Put(row);
3816     value = Bytes.toBytes("value0");
3817     put.add(family, qualifier, 1234567l, value);
3818     region.put(put);
3819     get = new Get(row);
3820     get.addColumn(family, qualifier);
3821     get.setMaxVersions();
3822     res = this.region.get(get);
3823     kvs = res.getColumn(family, qualifier);
3824     assertEquals(1, kvs.size());
3825     assertEquals(Bytes.toBytes("value0"), kvs.get(0).getValue());
3826 
3827     region.flushcache();
3828     get = new Get(row);
3829     get.addColumn(family, qualifier);
3830     get.setMaxVersions();
3831     res = this.region.get(get);
3832     kvs = res.getColumn(family, qualifier);
3833     assertEquals(1, kvs.size());
3834     assertEquals(Bytes.toBytes("value0"), kvs.get(0).getValue());
3835 
3836     put = new Put(row);
3837     value = Bytes.toBytes("value1");
3838     put.add(family, qualifier, 1234567l, value);
3839     region.put(put);
3840     get = new Get(row);
3841     get.addColumn(family, qualifier);
3842     get.setMaxVersions();
3843     res = this.region.get(get);
3844     kvs = res.getColumn(family, qualifier);
3845     assertEquals(1, kvs.size());
3846     assertEquals(Bytes.toBytes("value1"), kvs.get(0).getValue());
3847 
3848     region.flushcache();
3849     get = new Get(row);
3850     get.addColumn(family, qualifier);
3851     get.setMaxVersions();
3852     res = this.region.get(get);
3853     kvs = res.getColumn(family, qualifier);
3854     assertEquals(1, kvs.size());
3855     assertEquals(Bytes.toBytes("value1"), kvs.get(0).getValue());
3856   }
3857 
3858   @Test
3859   public void testDurability() throws Exception {
3860     String method = "testDurability";
3861     // there are 5 x 5 cases:
3862     // table durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) x mutation durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT)
3863 
3864     // expected cases for append and sync wal
3865     durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
3866     durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
3867     durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
3868 
3869     durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
3870     durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
3871     durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
3872 
3873     durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
3874     durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
3875 
3876     durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
3877     durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
3878 
3879     durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
3880     durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
3881     durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
3882 
3883     // expected cases for async wal
3884     // do not sync for deferred flush with large optionallogflushinterval
3885     conf.setLong("hbase.regionserver.optionallogflushinterval", Integer.MAX_VALUE);
3886     durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
3887     durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
3888     durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
3889     durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
3890     durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
3891     durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
3892 
3893     // now small deferred log flush optionallogflushinterval, expect sync
3894     conf.setLong("hbase.regionserver.optionallogflushinterval", 5);
3895     durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
3896     durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
3897     durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
3898     durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
3899     durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
3900     durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
3901 
3902     // expect skip wal cases
3903     durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
3904     durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
3905     durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
3906     durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
3907     durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
3908     durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
3909 
3910   }
3911 
3912   private void durabilityTest(String method, Durability tableDurability,
3913       Durability mutationDurability, long timeout, boolean expectAppend,
3914       final boolean expectSync, final boolean expectSyncFromLogSyncer) throws Exception {
3915     method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
3916     TableName tableName = TableName.valueOf(method);
3917     byte[] family = Bytes.toBytes("family");
3918     Path logDir = new Path(new Path(DIR + method), "log");
3919     HLog hlog = HLogFactory.createHLog(fs, logDir, UUID.randomUUID().toString(), conf);
3920     final HLog log = spy(hlog);
3921     this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
3922       HConstants.EMPTY_END_ROW, method, conf, false,
3923       tableDurability, log, new byte[][] {family});
3924 
3925     Put put = new Put(Bytes.toBytes("r1"));
3926     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
3927     put.setDurability(mutationDurability);
3928     region.put(put);
3929 
3930     //verify append called or not
3931     verify(log, expectAppend ? times(1) : never())
3932       .appendNoSync((HRegionInfo)any(), eq(tableName),
3933         (WALEdit)any(), (UUID)any(), anyLong(), (HTableDescriptor)any());
3934 
3935     //verify sync called or not
3936     if (expectSync || expectSyncFromLogSyncer) {
3937       TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
3938         @Override
3939         public boolean evaluate() throws Exception {
3940           try {
3941             if (expectSync) {
3942               verify(log, times(1)).sync(anyLong()); //Hregion calls this one
3943             } else if (expectSyncFromLogSyncer) {
3944               verify(log, times(1)).sync(); //log syncer calls this one
3945             }
3946           } catch (Throwable ignore) {}
3947           return true;
3948         }
3949       });
3950     } else {
3951       verify(log, never()).sync(anyLong());
3952       verify(log, never()).sync();
3953     }
3954 
3955     hlog.close();
3956     region.close();
3957   }
3958 
3959   private void putData(int startRow, int numRows, byte [] qf,
3960       byte [] ...families)
3961   throws IOException {
3962     for(int i=startRow; i<startRow+numRows; i++) {
3963       Put put = new Put(Bytes.toBytes("" + i));
3964       put.setDurability(Durability.SKIP_WAL);
3965       for(byte [] family : families) {
3966         put.add(family, qf, null);
3967       }
3968       region.put(put);
3969     }
3970   }
3971 
3972   private void verifyData(HRegion newReg, int startRow, int numRows, byte [] qf,
3973       byte [] ... families)
3974   throws IOException {
3975     for(int i=startRow; i<startRow + numRows; i++) {
3976       byte [] row = Bytes.toBytes("" + i);
3977       Get get = new Get(row);
3978       for(byte [] family : families) {
3979         get.addColumn(family, qf);
3980       }
3981       Result result = newReg.get(get);
3982       KeyValue [] raw = result.raw();
3983       assertEquals(families.length, result.size());
3984       for(int j=0; j<families.length; j++) {
3985         assertEquals(0, Bytes.compareTo(row, raw[j].getRow()));
3986         assertEquals(0, Bytes.compareTo(families[j], raw[j].getFamily()));
3987         assertEquals(0, Bytes.compareTo(qf, raw[j].getQualifier()));
3988       }
3989     }
3990   }
3991 
3992   private void assertGet(final HRegion r, final byte [] family, final byte [] k)
3993   throws IOException {
3994     // Now I have k, get values out and assert they are as expected.
3995     Get get = new Get(k).addFamily(family).setMaxVersions();
3996     KeyValue [] results = r.get(get).raw();
3997     for (int j = 0; j < results.length; j++) {
3998       byte [] tmp = results[j].getValue();
3999       // Row should be equal to value every time.
4000       assertTrue(Bytes.equals(k, tmp));
4001     }
4002   }
4003 
4004   /*
4005    * Assert first value in the passed region is <code>firstValue</code>.
4006    * @param r
4007    * @param fs
4008    * @param firstValue
4009    * @throws IOException
4010    */
4011   private void assertScan(final HRegion r, final byte [] fs,
4012       final byte [] firstValue)
4013   throws IOException {
4014     byte [][] families = {fs};
4015     Scan scan = new Scan();
4016     for (int i = 0; i < families.length; i++) scan.addFamily(families[i]);
4017     InternalScanner s = r.getScanner(scan);
4018     try {
4019       List<KeyValue> curVals = new ArrayList<KeyValue>();
4020       boolean first = true;
4021       OUTER_LOOP: while(s.next(curVals)) {
4022         for (KeyValue kv: curVals) {
4023           byte [] val = kv.getValue();
4024           byte [] curval = val;
4025           if (first) {
4026             first = false;
4027             assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4028           } else {
4029             // Not asserting anything.  Might as well break.
4030             break OUTER_LOOP;
4031           }
4032         }
4033       }
4034     } finally {
4035       s.close();
4036     }
4037   }
4038 
4039   private Configuration initSplit() {
4040     Configuration conf = HBaseConfiguration.create(this.conf);
4041 
4042     // Always compact if there is more than one store file.
4043     conf.setInt("hbase.hstore.compactionThreshold", 2);
4044 
4045     // Make lease timeout longer, lease checks less frequent
4046     conf.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4047 
4048     conf.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4049 
4050     // Increase the amount of time between client retries
4051     conf.setLong("hbase.client.pause", 15 * 1000);
4052 
4053     // This size should make it so we always split using the addContent
4054     // below.  After adding all data, the first region is 1.3M
4055     conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4056     return conf;
4057   }
4058 
4059   /**
4060    * @param tableName
4061    * @param callingMethod
4062    * @param conf
4063    * @param families
4064    * @throws IOException
4065    * @return A region on which you must call {@link HRegion#closeHRegion(HRegion)} when done.
4066    */
4067   public static HRegion initHRegion (TableName tableName, String callingMethod,
4068       Configuration conf, byte [] ... families)
4069     throws IOException{
4070     return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
4071   }
4072 
4073   /**
4074    * @param tableName
4075    * @param callingMethod
4076    * @param conf
4077    * @param families
4078    * @throws IOException
4079    * @return A region on which you must call {@link HRegion#closeHRegion(HRegion)} when done.
4080    */
4081   public static HRegion initHRegion (byte [] tableName, String callingMethod,
4082       Configuration conf, byte [] ... families)
4083     throws IOException{
4084     return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4085   }
4086 
4087   /**
4088    * @param tableName
4089    * @param callingMethod
4090    * @param conf
4091    * @param isReadOnly
4092    * @param families
4093    * @throws IOException
4094    * @return A region on which you must call {@link HRegion#closeHRegion(HRegion)} when done.
4095    */
4096   public static HRegion initHRegion (byte [] tableName, String callingMethod,
4097       Configuration conf, boolean isReadOnly, byte [] ... families)
4098     throws IOException{
4099     return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4100   }
4101 
4102   private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4103       String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4104       throws IOException {
4105     return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
4106       Durability.SYNC_WAL, null, families);
4107   }
4108 
4109   /**
4110    * @param tableName
4111    * @param startKey
4112    * @param stopKey
4113    * @param callingMethod
4114    * @param conf
4115    * @param isReadOnly
4116    * @param families
4117    * @throws IOException
4118    * @return A region on which you must call {@link HRegion#closeHRegion(HRegion)} when done.
4119    */
4120   private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4121       String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4122       HLog hlog, byte[]... families)
4123       throws IOException {
4124     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
4125     htd.setReadOnly(isReadOnly);
4126     for(byte [] family : families) {
4127       HColumnDescriptor hcd = new HColumnDescriptor(family);
4128       // Set default to be three versions.
4129       hcd.setMaxVersions(Integer.MAX_VALUE);
4130       htd.addFamily(hcd);
4131     }
4132     htd.setDurability(durability);
4133     HRegionInfo info = new HRegionInfo(htd.getTableName(), startKey, stopKey, false);
4134     Path path = new Path(DIR + callingMethod);
4135     FileSystem fs = FileSystem.get(conf);
4136     if (fs.exists(path)) {
4137       if (!fs.delete(path, true)) {
4138         throw new IOException("Failed delete of " + path);
4139       }
4140     }
4141     return HRegion.createHRegion(info, path, conf, htd, hlog);
4142   }
4143 
4144   /**
4145    * Assert that the passed in KeyValue has expected contents for the
4146    * specified row, column & timestamp.
4147    */
4148   private void checkOneCell(KeyValue kv, byte[] cf,
4149                              int rowIdx, int colIdx, long ts) {
4150     String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4151     assertEquals("Row mismatch which checking: " + ctx,
4152                  "row:"+ rowIdx, Bytes.toString(kv.getRow()));
4153     assertEquals("ColumnFamily mismatch while checking: " + ctx,
4154                  Bytes.toString(cf), Bytes.toString(kv.getFamily()));
4155     assertEquals("Column qualifier mismatch while checking: " + ctx,
4156                  "column:" + colIdx, Bytes.toString(kv.getQualifier()));
4157     assertEquals("Timestamp mismatch while checking: " + ctx,
4158                  ts, kv.getTimestamp());
4159     assertEquals("Value mismatch while checking: " + ctx,
4160                  "value-version-" + ts, Bytes.toString(kv.getValue()));
4161   }
4162 }
4163