1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21
22 import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23 import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24 import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25 import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29 import static org.junit.Assert.assertArrayEquals;
30 import static org.junit.Assert.assertEquals;
31 import static org.junit.Assert.assertFalse;
32 import static org.junit.Assert.assertNotNull;
33 import static org.junit.Assert.assertNull;
34 import static org.junit.Assert.assertTrue;
35 import static org.junit.Assert.fail;
36 import static org.mockito.Matchers.any;
37 import static org.mockito.Matchers.anyLong;
38 import static org.mockito.Matchers.eq;
39 import static org.mockito.Mockito.never;
40 import static org.mockito.Mockito.spy;
41 import static org.mockito.Mockito.times;
42 import static org.mockito.Mockito.verify;
43
44 import java.io.IOException;
45 import java.io.InterruptedIOException;
46 import java.util.ArrayList;
47 import java.util.Arrays;
48 import java.util.Collection;
49 import java.util.List;
50 import java.util.Map;
51 import java.util.NavigableMap;
52 import java.util.TreeMap;
53 import java.util.UUID;
54 import java.util.concurrent.atomic.AtomicBoolean;
55 import java.util.concurrent.atomic.AtomicInteger;
56 import java.util.concurrent.atomic.AtomicReference;
57
58 import org.apache.commons.logging.Log;
59 import org.apache.commons.logging.LogFactory;
60 import org.apache.hadoop.conf.Configuration;
61 import org.apache.hadoop.fs.FSDataOutputStream;
62 import org.apache.hadoop.fs.FileStatus;
63 import org.apache.hadoop.fs.FileSystem;
64 import org.apache.hadoop.fs.Path;
65 import org.apache.hadoop.hbase.Cell;
66 import org.apache.hadoop.hbase.CellComparator;
67 import org.apache.hadoop.hbase.CellUtil;
68 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
69 import org.apache.hadoop.hbase.HBaseTestCase;
70 import org.apache.hadoop.hbase.HBaseTestingUtility;
71 import org.apache.hadoop.hbase.HColumnDescriptor;
72 import org.apache.hadoop.hbase.HConstants;
73 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
74 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
75 import org.apache.hadoop.hbase.HRegionInfo;
76 import org.apache.hadoop.hbase.HTableDescriptor;
77 import org.apache.hadoop.hbase.KeyValue;
78 import org.apache.hadoop.hbase.MediumTests;
79 import org.apache.hadoop.hbase.MiniHBaseCluster;
80 import org.apache.hadoop.hbase.MultithreadedTestUtil;
81 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
82 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
83 import org.apache.hadoop.hbase.NotServingRegionException;
84 import org.apache.hadoop.hbase.TableName;
85 import org.apache.hadoop.hbase.Waiter;
86 import org.apache.hadoop.hbase.client.Append;
87 import org.apache.hadoop.hbase.client.Delete;
88 import org.apache.hadoop.hbase.client.Durability;
89 import org.apache.hadoop.hbase.client.Get;
90 import org.apache.hadoop.hbase.client.HTable;
91 import org.apache.hadoop.hbase.client.Increment;
92 import org.apache.hadoop.hbase.client.Put;
93 import org.apache.hadoop.hbase.client.Result;
94 import org.apache.hadoop.hbase.client.Scan;
95 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
96 import org.apache.hadoop.hbase.filter.BinaryComparator;
97 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
98 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
99 import org.apache.hadoop.hbase.filter.Filter;
100 import org.apache.hadoop.hbase.filter.FilterBase;
101 import org.apache.hadoop.hbase.filter.FilterList;
102 import org.apache.hadoop.hbase.filter.NullComparator;
103 import org.apache.hadoop.hbase.filter.PrefixFilter;
104 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
105 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
106 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
107 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
108 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
109 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
110 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
111 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
112 import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
113 import org.apache.hadoop.hbase.regionserver.wal.HLog;
114 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
115 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
116 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
117 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
118 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
119 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
120 import org.apache.hadoop.hbase.util.Bytes;
121 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
122 import org.apache.hadoop.hbase.util.FSUtils;
123 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
124 import org.apache.hadoop.hbase.util.PairOfSameType;
125 import org.apache.hadoop.hbase.util.Threads;
126 import org.junit.After;
127 import org.junit.Assert;
128 import org.junit.Before;
129 import org.junit.Rule;
130 import org.junit.Test;
131 import org.junit.experimental.categories.Category;
132 import org.junit.rules.TestName;
133 import org.mockito.Mockito;
134
135 import com.google.common.collect.Lists;
136
137
138
139
140
141
142
143 @Category(MediumTests.class)
144 @SuppressWarnings("deprecation")
145 public class TestHRegion {
146
147
148 static final Log LOG = LogFactory.getLog(TestHRegion.class);
149 @Rule public TestName name = new TestName();
150
151 private static final String COLUMN_FAMILY = "MyCF";
152
153 HRegion region = null;
154 private static HBaseTestingUtility TEST_UTIL;
155 public static Configuration conf ;
156 private String DIR;
157 private static FileSystem fs;
158 private final int MAX_VERSIONS = 2;
159
160
161 protected byte[] tableName;
162 protected String method;
163 protected final byte[] qual1 = Bytes.toBytes("qual1");
164 protected final byte[] qual2 = Bytes.toBytes("qual2");
165 protected final byte[] qual3 = Bytes.toBytes("qual3");
166 protected final byte[] value1 = Bytes.toBytes("value1");
167 protected final byte[] value2 = Bytes.toBytes("value2");
168 protected final byte[] row = Bytes.toBytes("rowA");
169 protected final byte[] row2 = Bytes.toBytes("rowB");
170
171 protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
172 .getInstance(MetricsAssertHelper.class);
173
174 @Before
175 public void setup() throws IOException {
176 this.TEST_UTIL = HBaseTestingUtility.createLocalHTU();
177 this.fs = TEST_UTIL.getTestFileSystem();
178 this.conf = TEST_UTIL.getConfiguration();
179 this.DIR = TEST_UTIL.getDataTestDir("TestHRegion").toString();
180 method = name.getMethodName();
181 tableName = Bytes.toBytes(name.getMethodName());
182 }
183
184 @After
185 public void tearDown() throws Exception {
186 EnvironmentEdgeManagerTestHelper.reset();
187 }
188
189 String getName() {
190 return name.getMethodName();
191 }
192
193
194
195
196
197
198
199 @Test
200 public void testCompactionAffectedByScanners() throws Exception {
201 byte[] family = Bytes.toBytes("family");
202 this.region = initHRegion(tableName, method, conf, family);
203
204 Put put = new Put(Bytes.toBytes("r1"));
205 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
206 region.put(put);
207 region.flushcache();
208
209 Scan scan = new Scan();
210 scan.setMaxVersions(3);
211
212 RegionScanner scanner1 = region.getScanner(scan);
213
214 Delete delete = new Delete(Bytes.toBytes("r1"));
215 region.delete(delete);
216 region.flushcache();
217
218
219 RegionScanner scanner2 = region.getScanner(scan);
220
221 List<Cell> results = new ArrayList<Cell>();
222
223 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
224
225
226 region.compactStores(true);
227
228
229 RegionScanner scanner3 = region.getScanner(scan);
230
231
232 scanner1.next(results);
233 System.out.println(results);
234 assertEquals(1, results.size());
235
236 results.clear();
237 scanner2.next(results);
238 System.out.println(results);
239 assertEquals(0, results.size());
240
241 results.clear();
242 scanner3.next(results);
243 System.out.println(results);
244 assertEquals(0, results.size());
245 }
246
247 @Test
248 public void testToShowNPEOnRegionScannerReseek() throws Exception {
249 byte[] family = Bytes.toBytes("family");
250 this.region = initHRegion(tableName, method, conf, family);
251
252 Put put = new Put(Bytes.toBytes("r1"));
253 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
254 region.put(put);
255 put = new Put(Bytes.toBytes("r2"));
256 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
257 region.put(put);
258 region.flushcache();
259
260 Scan scan = new Scan();
261 scan.setMaxVersions(3);
262
263 RegionScanner scanner1 = region.getScanner(scan);
264
265 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
266
267 region.compactStores(true);
268
269 scanner1.reseek(Bytes.toBytes("r2"));
270 List<Cell> results = new ArrayList<Cell>();
271 scanner1.next(results);
272 Cell keyValue = results.get(0);
273 Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
274 scanner1.close();
275 }
276
277 @Test
278 public void testSkipRecoveredEditsReplay() throws Exception {
279 String method = "testSkipRecoveredEditsReplay";
280 TableName tableName = TableName.valueOf(method);
281 byte[] family = Bytes.toBytes("family");
282 this.region = initHRegion(tableName, method, conf, family);
283 try {
284 Path regiondir = region.getRegionFileSystem().getRegionDir();
285 FileSystem fs = region.getRegionFileSystem().getFileSystem();
286 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
287
288 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
289
290 long maxSeqId = 1050;
291 long minSeqId = 1000;
292
293 for (long i = minSeqId; i <= maxSeqId; i += 10) {
294 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
295 fs.create(recoveredEdits);
296 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, conf);
297
298 long time = System.nanoTime();
299 WALEdit edit = new WALEdit();
300 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
301 .toBytes(i)));
302 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
303 HConstants.DEFAULT_CLUSTER_ID), edit));
304
305 writer.close();
306 }
307 MonitoredTask status = TaskMonitor.get().createStatus(method);
308 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
309 for (Store store : region.getStores().values()) {
310 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
311 }
312 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
313 assertEquals(maxSeqId, seqId);
314 Get get = new Get(row);
315 Result result = region.get(get);
316 for (long i = minSeqId; i <= maxSeqId; i += 10) {
317 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
318 assertEquals(1, kvs.size());
319 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
320 }
321 } finally {
322 HRegion.closeHRegion(this.region);
323 this.region = null;
324 }
325 }
326
327 @Test
328 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
329 String method = "testSkipRecoveredEditsReplaySomeIgnored";
330 TableName tableName = TableName.valueOf(method);
331 byte[] family = Bytes.toBytes("family");
332 this.region = initHRegion(tableName, method, conf, family);
333 try {
334 Path regiondir = region.getRegionFileSystem().getRegionDir();
335 FileSystem fs = region.getRegionFileSystem().getFileSystem();
336 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
337
338 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
339
340 long maxSeqId = 1050;
341 long minSeqId = 1000;
342
343 for (long i = minSeqId; i <= maxSeqId; i += 10) {
344 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
345 fs.create(recoveredEdits);
346 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, conf);
347
348 long time = System.nanoTime();
349 WALEdit edit = new WALEdit();
350 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
351 .toBytes(i)));
352 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
353 HConstants.DEFAULT_CLUSTER_ID), edit));
354
355 writer.close();
356 }
357 long recoverSeqId = 1030;
358 MonitoredTask status = TaskMonitor.get().createStatus(method);
359 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
360 for (Store store : region.getStores().values()) {
361 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
362 }
363 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
364 assertEquals(maxSeqId, seqId);
365 Get get = new Get(row);
366 Result result = region.get(get);
367 for (long i = minSeqId; i <= maxSeqId; i += 10) {
368 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
369 if (i < recoverSeqId) {
370 assertEquals(0, kvs.size());
371 } else {
372 assertEquals(1, kvs.size());
373 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
374 }
375 }
376 } finally {
377 HRegion.closeHRegion(this.region);
378 this.region = null;
379 }
380 }
381
382 @Test
383 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
384 byte[] family = Bytes.toBytes("family");
385 this.region = initHRegion(tableName, method, conf, family);
386 try {
387 Path regiondir = region.getRegionFileSystem().getRegionDir();
388 FileSystem fs = region.getRegionFileSystem().getFileSystem();
389
390 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
391 for (int i = 1000; i < 1050; i += 10) {
392 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
393 FSDataOutputStream dos = fs.create(recoveredEdits);
394 dos.writeInt(i);
395 dos.close();
396 }
397 long minSeqId = 2000;
398 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
399 FSDataOutputStream dos = fs.create(recoveredEdits);
400 dos.close();
401
402 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
403 for (Store store : region.getStores().values()) {
404 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
405 }
406 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
407 assertEquals(minSeqId, seqId);
408 } finally {
409 HRegion.closeHRegion(this.region);
410 this.region = null;
411 }
412 }
413
414 @Test
415 public void testRecoveredEditsReplayCompaction() throws Exception {
416 String method = name.getMethodName();
417 TableName tableName = TableName.valueOf(method);
418 byte[] family = Bytes.toBytes("family");
419 this.region = initHRegion(tableName, method, conf, family);
420 try {
421 Path regiondir = region.getRegionFileSystem().getRegionDir();
422 FileSystem fs = region.getRegionFileSystem().getFileSystem();
423 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
424
425 long maxSeqId = 3;
426 long minSeqId = 0;
427
428 for (long i = minSeqId; i < maxSeqId; i++) {
429 Put put = new Put(Bytes.toBytes(i));
430 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
431 region.put(put);
432 region.flushcache();
433 }
434
435
436 assertEquals(3, region.getStore(family).getStorefilesCount());
437 List<Path> storeFiles = new ArrayList<Path>(3);
438 for (StoreFile sf : region.getStore(family).getStorefiles()) {
439 storeFiles.add(sf.getPath());
440 }
441
442
443 conf.setBoolean("hbase.hstore.compaction.complete", false);
444 region.compactStores();
445
446
447 assertEquals(3, region.getStore(family).getStorefilesCount());
448
449
450 Path tmpDir = region.getRegionFileSystem().getTempDir();
451 FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
452 String errorMsg = "Expected to find 1 file in the region temp directory "
453 + "from the compaction, could not find any";
454 assertNotNull(errorMsg, files);
455 assertEquals(errorMsg, 1, files.length);
456
457 Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
458 files[0].getPath());
459
460 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
461 .getRegionInfo(), family, storeFiles, Lists.newArrayList(newFile), region
462 .getRegionFileSystem().getStoreDir(Bytes.toString(family)));
463
464 HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
465 this.region.getRegionInfo(), compactionDescriptor);
466
467 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
468
469 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
470 fs.create(recoveredEdits);
471 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, conf);
472
473 long time = System.nanoTime();
474
475 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, 10, time,
476 HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(compactionDescriptor)));
477 writer.close();
478
479
480 HTableDescriptor htd = region.getTableDesc();
481 HRegionInfo info = region.getRegionInfo();
482 region.close();
483 region = HRegion.openHRegion(region, null);
484
485
486 Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
487 for (StoreFile sf : sfs) {
488 LOG.info(sf.getPath());
489 }
490 assertEquals(1, region.getStore(family).getStorefilesCount());
491 files = FSUtils.listStatus(fs, tmpDir);
492 assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
493
494 for (long i = minSeqId; i < maxSeqId; i++) {
495 Get get = new Get(Bytes.toBytes(i));
496 Result result = region.get(get);
497 byte[] value = result.getValue(family, Bytes.toBytes(i));
498 assertArrayEquals(Bytes.toBytes(i), value);
499 }
500 } finally {
501 HRegion.closeHRegion(this.region);
502 this.region = null;
503 }
504 }
505
506 @Test
507 public void testGetWhileRegionClose() throws IOException {
508 TableName tableName = TableName.valueOf(name.getMethodName());
509 Configuration hc = initSplit();
510 int numRows = 100;
511 byte[][] families = { fam1, fam2, fam3 };
512
513
514 String method = name.getMethodName();
515 this.region = initHRegion(tableName, method, hc, families);
516 try {
517
518 final int startRow = 100;
519 putData(startRow, numRows, qual1, families);
520 putData(startRow, numRows, qual2, families);
521 putData(startRow, numRows, qual3, families);
522 final AtomicBoolean done = new AtomicBoolean(false);
523 final AtomicInteger gets = new AtomicInteger(0);
524 GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
525 try {
526
527 for (int i = 0; i < threads.length / 2; i++) {
528 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
529 threads[i].setDaemon(true);
530 threads[i].start();
531 }
532
533
534 this.region.closing.set(true);
535 for (int i = threads.length / 2; i < threads.length; i++) {
536 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
537 threads[i].setDaemon(true);
538 threads[i].start();
539 }
540 } finally {
541 if (this.region != null) {
542 HRegion.closeHRegion(this.region);
543 }
544 }
545 done.set(true);
546 for (GetTillDoneOrException t : threads) {
547 try {
548 t.join();
549 } catch (InterruptedException e) {
550 e.printStackTrace();
551 }
552 if (t.e != null) {
553 LOG.info("Exception=" + t.e);
554 assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
555 }
556 }
557 } finally {
558 HRegion.closeHRegion(this.region);
559 this.region = null;
560 }
561 }
562
563
564
565
566
567 class GetTillDoneOrException extends Thread {
568 private final Get g;
569 private final AtomicBoolean done;
570 private final AtomicInteger count;
571 private Exception e;
572
573 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
574 super("getter." + i);
575 this.g = new Get(r);
576 this.done = d;
577 this.count = c;
578 }
579
580 @Override
581 public void run() {
582 while (!this.done.get()) {
583 try {
584 assertTrue(region.get(g).size() > 0);
585 this.count.incrementAndGet();
586 } catch (Exception e) {
587 this.e = e;
588 break;
589 }
590 }
591 }
592 }
593
594
595
596
597 @Test
598 public void testWeirdCacheBehaviour() throws Exception {
599 byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
600 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
601 Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
602 this.region = initHRegion(TABLE, getName(), conf, FAMILIES);
603 try {
604 String value = "this is the value";
605 String value2 = "this is some other value";
606 String keyPrefix1 = "prefix1";
607 String keyPrefix2 = "prefix2";
608 String keyPrefix3 = "prefix3";
609 putRows(this.region, 3, value, keyPrefix1);
610 putRows(this.region, 3, value, keyPrefix2);
611 putRows(this.region, 3, value, keyPrefix3);
612 putRows(this.region, 3, value2, keyPrefix1);
613 putRows(this.region, 3, value2, keyPrefix2);
614 putRows(this.region, 3, value2, keyPrefix3);
615 System.out.println("Checking values for key: " + keyPrefix1);
616 assertEquals("Got back incorrect number of rows from scan", 3,
617 getNumberOfRows(keyPrefix1, value2, this.region));
618 System.out.println("Checking values for key: " + keyPrefix2);
619 assertEquals("Got back incorrect number of rows from scan", 3,
620 getNumberOfRows(keyPrefix2, value2, this.region));
621 System.out.println("Checking values for key: " + keyPrefix3);
622 assertEquals("Got back incorrect number of rows from scan", 3,
623 getNumberOfRows(keyPrefix3, value2, this.region));
624 deleteColumns(this.region, value2, keyPrefix1);
625 deleteColumns(this.region, value2, keyPrefix2);
626 deleteColumns(this.region, value2, keyPrefix3);
627 System.out.println("Starting important checks.....");
628 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
629 getNumberOfRows(keyPrefix1, value2, this.region));
630 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
631 getNumberOfRows(keyPrefix2, value2, this.region));
632 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
633 getNumberOfRows(keyPrefix3, value2, this.region));
634 } finally {
635 HRegion.closeHRegion(this.region);
636 this.region = null;
637 }
638 }
639
640 @Test
641 public void testAppendWithReadOnlyTable() throws Exception {
642 byte[] TABLE = Bytes.toBytes("readOnlyTable");
643 this.region = initHRegion(TABLE, getName(), conf, true, Bytes.toBytes("somefamily"));
644 boolean exceptionCaught = false;
645 Append append = new Append(Bytes.toBytes("somerow"));
646 append.setDurability(Durability.SKIP_WAL);
647 append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
648 Bytes.toBytes("somevalue"));
649 try {
650 region.append(append);
651 } catch (IOException e) {
652 exceptionCaught = true;
653 } finally {
654 HRegion.closeHRegion(this.region);
655 this.region = null;
656 }
657 assertTrue(exceptionCaught == true);
658 }
659
660 @Test
661 public void testIncrWithReadOnlyTable() throws Exception {
662 byte[] TABLE = Bytes.toBytes("readOnlyTable");
663 this.region = initHRegion(TABLE, getName(), conf, true, Bytes.toBytes("somefamily"));
664 boolean exceptionCaught = false;
665 Increment inc = new Increment(Bytes.toBytes("somerow"));
666 inc.setDurability(Durability.SKIP_WAL);
667 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
668 try {
669 region.increment(inc);
670 } catch (IOException e) {
671 exceptionCaught = true;
672 } finally {
673 HRegion.closeHRegion(this.region);
674 this.region = null;
675 }
676 assertTrue(exceptionCaught == true);
677 }
678
679 private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
680 InternalScanner scanner = buildScanner(keyPrefix, value, r);
681 int count = 0;
682 boolean more = false;
683 List<Cell> results = new ArrayList<Cell>();
684 do {
685 more = scanner.next(results);
686 if (results != null && !results.isEmpty())
687 count++;
688 else
689 break;
690 Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
691 delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
692 r.delete(delete);
693 results.clear();
694 } while (more);
695 assertEquals("Did not perform correct number of deletes", 3, count);
696 }
697
698 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
699 InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
700 int numberOfResults = 0;
701 List<Cell> results = new ArrayList<Cell>();
702 boolean more = false;
703 do {
704 more = resultScanner.next(results);
705 if (results != null && !results.isEmpty())
706 numberOfResults++;
707 else
708 break;
709 for (Cell kv : results) {
710 System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
711 }
712 results.clear();
713 } while (more);
714 return numberOfResults;
715 }
716
717 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
718 throws IOException {
719
720 FilterList allFilters = new FilterList();
721 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
722
723 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
724 Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
725 filter.setFilterIfMissing(true);
726 allFilters.addFilter(filter);
727 Scan scan = new Scan();
728 scan.addFamily(Bytes.toBytes("trans-blob"));
729 scan.addFamily(Bytes.toBytes("trans-type"));
730 scan.addFamily(Bytes.toBytes("trans-date"));
731 scan.addFamily(Bytes.toBytes("trans-tags"));
732 scan.addFamily(Bytes.toBytes("trans-group"));
733 scan.setFilter(allFilters);
734 return r.getScanner(scan);
735 }
736
737 private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
738 for (int i = 0; i < numRows; i++) {
739 String row = key + "_" + i
740 System.out.println(String.format("Saving row: %s, with value %s", row, value));
741 Put put = new Put(Bytes.toBytes(row));
742 put.setDurability(Durability.SKIP_WAL);
743 put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
744 put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
745 put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
746 put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
747 put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
748 r.put(put);
749 }
750 }
751
752 @Test
753 public void testFamilyWithAndWithoutColon() throws Exception {
754 byte[] b = Bytes.toBytes(getName());
755 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
756 this.region = initHRegion(b, getName(), conf, cf);
757 try {
758 Put p = new Put(b);
759 byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
760 p.add(cfwithcolon, cfwithcolon, cfwithcolon);
761 boolean exception = false;
762 try {
763 this.region.put(p);
764 } catch (NoSuchColumnFamilyException e) {
765 exception = true;
766 }
767 assertTrue(exception);
768 } finally {
769 HRegion.closeHRegion(this.region);
770 this.region = null;
771 }
772 }
773
774 @Test
775 public void testBatchPut() throws Exception {
776 byte[] b = Bytes.toBytes(getName());
777 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
778 byte[] qual = Bytes.toBytes("qual");
779 byte[] val = Bytes.toBytes("val");
780 this.region = initHRegion(b, getName(), conf, cf);
781 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
782 try {
783 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
784 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
785
786 LOG.info("First a batch put with all valid puts");
787 final Put[] puts = new Put[10];
788 for (int i = 0; i < 10; i++) {
789 puts[i] = new Put(Bytes.toBytes("row_" + i));
790 puts[i].add(cf, qual, val);
791 }
792
793 OperationStatus[] codes = this.region.batchMutate(puts);
794 assertEquals(10, codes.length);
795 for (int i = 0; i < 10; i++) {
796 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
797 }
798 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
799
800 LOG.info("Next a batch put with one invalid family");
801 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
802 codes = this.region.batchMutate(puts);
803 assertEquals(10, codes.length);
804 for (int i = 0; i < 10; i++) {
805 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
806 codes[i].getOperationStatusCode());
807 }
808
809 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
810
811 LOG.info("Next a batch put that has to break into two batches to avoid a lock");
812 RowLock rowLock = region.getRowLock(Bytes.toBytes("row_2"));
813
814 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
815 final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
816 TestThread putter = new TestThread(ctx) {
817 @Override
818 public void doWork() throws IOException {
819 retFromThread.set(region.batchMutate(puts));
820 }
821 };
822 LOG.info("...starting put thread while holding lock");
823 ctx.addThread(putter);
824 ctx.startThreads();
825
826 LOG.info("...waiting for put thread to sync first time");
827 long startWait = System.currentTimeMillis();
828 while (metricsAssertHelper.getCounter("syncTimeNumOps", source) == syncs + 2) {
829 Thread.sleep(100);
830 if (System.currentTimeMillis() - startWait > 10000) {
831 fail("Timed out waiting for thread to sync first minibatch");
832 }
833 }
834 LOG.info("...releasing row lock, which should let put thread continue");
835 rowLock.release();
836 LOG.info("...joining on thread");
837 ctx.stop();
838 LOG.info("...checking that next batch was synced");
839 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 4, source);
840 codes = retFromThread.get();
841 for (int i = 0; i < 10; i++) {
842 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
843 codes[i].getOperationStatusCode());
844 }
845
846 } finally {
847 HRegion.closeHRegion(this.region);
848 this.region = null;
849 }
850 }
851
852 @Test
853 public void testBatchPutWithTsSlop() throws Exception {
854 byte[] b = Bytes.toBytes(getName());
855 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
856 byte[] qual = Bytes.toBytes("qual");
857 byte[] val = Bytes.toBytes("val");
858
859
860 conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
861 this.region = initHRegion(b, getName(), conf, cf);
862
863 try {
864 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
865 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
866 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
867
868 final Put[] puts = new Put[10];
869 for (int i = 0; i < 10; i++) {
870 puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
871 puts[i].add(cf, qual, val);
872 }
873
874 OperationStatus[] codes = this.region.batchMutate(puts);
875 assertEquals(10, codes.length);
876 for (int i = 0; i < 10; i++) {
877 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
878 }
879 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
880
881 } finally {
882 HRegion.closeHRegion(this.region);
883 this.region = null;
884 }
885
886 }
887
888
889
890
891 @Test
892 public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
893 byte[] row1 = Bytes.toBytes("row1");
894 byte[] fam1 = Bytes.toBytes("fam1");
895 byte[] qf1 = Bytes.toBytes("qualifier");
896 byte[] emptyVal = new byte[] {};
897 byte[] val1 = Bytes.toBytes("value1");
898 byte[] val2 = Bytes.toBytes("value2");
899
900
901 String method = this.getName();
902 this.region = initHRegion(tableName, method, conf, fam1);
903 try {
904
905 Put put = new Put(row1);
906 put.add(fam1, qf1, emptyVal);
907
908
909 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
910 emptyVal), put, true);
911 assertTrue(res);
912
913
914 put = new Put(row1);
915 put.add(fam1, qf1, val1);
916
917
918 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
919 put, true);
920 assertTrue(res);
921
922
923 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
924 put, true);
925 assertFalse(res);
926
927 Delete delete = new Delete(row1);
928 delete.deleteColumn(fam1, qf1);
929 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
930 delete, true);
931 assertFalse(res);
932
933 put = new Put(row1);
934 put.add(fam1, qf1, val2);
935
936 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
937 put, true);
938 assertTrue(res);
939
940
941 delete = new Delete(row1);
942 delete.deleteColumn(fam1, qf1);
943 delete.deleteColumn(fam1, qf1);
944 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
945 delete, true);
946 assertTrue(res);
947
948 delete = new Delete(row1);
949 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
950 delete, true);
951 assertTrue(res);
952
953
954 put = new Put(row1);
955 put.add(fam1, qf1, val1);
956
957 res = region
958 .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
959 assertTrue(res);
960 } finally {
961 HRegion.closeHRegion(this.region);
962 this.region = null;
963 }
964 }
965
966 @Test
967 public void testCheckAndMutate_WithWrongValue() throws IOException {
968 byte[] row1 = Bytes.toBytes("row1");
969 byte[] fam1 = Bytes.toBytes("fam1");
970 byte[] qf1 = Bytes.toBytes("qualifier");
971 byte[] val1 = Bytes.toBytes("value1");
972 byte[] val2 = Bytes.toBytes("value2");
973
974
975 String method = this.getName();
976 this.region = initHRegion(tableName, method, conf, fam1);
977 try {
978
979 Put put = new Put(row1);
980 put.add(fam1, qf1, val1);
981 region.put(put);
982
983
984 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
985 val2), put, true);
986 assertEquals(false, res);
987
988
989 Delete delete = new Delete(row1);
990 delete.deleteFamily(fam1);
991 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
992 delete, true);
993 assertEquals(false, res);
994 } finally {
995 HRegion.closeHRegion(this.region);
996 this.region = null;
997 }
998 }
999
1000 @Test
1001 public void testCheckAndMutate_WithCorrectValue() throws IOException {
1002 byte[] row1 = Bytes.toBytes("row1");
1003 byte[] fam1 = Bytes.toBytes("fam1");
1004 byte[] qf1 = Bytes.toBytes("qualifier");
1005 byte[] val1 = Bytes.toBytes("value1");
1006
1007
1008 String method = this.getName();
1009 this.region = initHRegion(tableName, method, conf, fam1);
1010 try {
1011
1012 Put put = new Put(row1);
1013 put.add(fam1, qf1, val1);
1014 region.put(put);
1015
1016
1017 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1018 val1), put, true);
1019 assertEquals(true, res);
1020
1021
1022 Delete delete = new Delete(row1);
1023 delete.deleteColumn(fam1, qf1);
1024 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1025 put, true);
1026 assertEquals(true, res);
1027 } finally {
1028 HRegion.closeHRegion(this.region);
1029 this.region = null;
1030 }
1031 }
1032
1033 @Test
1034 public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1035 byte[] row1 = Bytes.toBytes("row1");
1036 byte[] fam1 = Bytes.toBytes("fam1");
1037 byte[] fam2 = Bytes.toBytes("fam2");
1038 byte[] qf1 = Bytes.toBytes("qualifier");
1039 byte[] val1 = Bytes.toBytes("value1");
1040 byte[] val2 = Bytes.toBytes("value2");
1041
1042 byte[][] families = { fam1, fam2 };
1043
1044
1045 String method = this.getName();
1046 this.region = initHRegion(tableName, method, conf, families);
1047 try {
1048
1049 Put put = new Put(row1);
1050 put.add(fam1, qf1, val1);
1051 region.put(put);
1052
1053
1054 long ts = System.currentTimeMillis();
1055 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1056 put = new Put(row1);
1057 put.add(kv);
1058
1059
1060 HStore store = (HStore) region.getStore(fam1);
1061 store.memstore.kvset.size();
1062
1063 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1064 val1), put, true);
1065 assertEquals(true, res);
1066 store.memstore.kvset.size();
1067
1068 Get get = new Get(row1);
1069 get.addColumn(fam2, qf1);
1070 Cell[] actual = region.get(get).rawCells();
1071
1072 Cell[] expected = { kv };
1073
1074 assertEquals(expected.length, actual.length);
1075 for (int i = 0; i < actual.length; i++) {
1076 assertEquals(expected[i], actual[i]);
1077 }
1078 } finally {
1079 HRegion.closeHRegion(this.region);
1080 this.region = null;
1081 }
1082 }
1083
1084 @Test
1085 public void testCheckAndPut_wrongRowInPut() throws IOException {
1086 TableName tableName = TableName.valueOf(name.getMethodName());
1087 this.region = initHRegion(tableName, this.getName(), conf, COLUMNS);
1088 try {
1089 Put put = new Put(row2);
1090 put.add(fam1, qual1, value1);
1091 try {
1092 boolean res = region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1093 new BinaryComparator(value2), put, false);
1094 fail();
1095 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1096
1097 }
1098 } finally {
1099 HRegion.closeHRegion(this.region);
1100 this.region = null;
1101 }
1102 }
1103
1104 @Test
1105 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1106 byte[] row1 = Bytes.toBytes("row1");
1107 byte[] fam1 = Bytes.toBytes("fam1");
1108 byte[] fam2 = Bytes.toBytes("fam2");
1109 byte[] qf1 = Bytes.toBytes("qualifier1");
1110 byte[] qf2 = Bytes.toBytes("qualifier2");
1111 byte[] qf3 = Bytes.toBytes("qualifier3");
1112 byte[] val1 = Bytes.toBytes("value1");
1113 byte[] val2 = Bytes.toBytes("value2");
1114 byte[] val3 = Bytes.toBytes("value3");
1115 byte[] emptyVal = new byte[] {};
1116
1117 byte[][] families = { fam1, fam2 };
1118
1119
1120 String method = this.getName();
1121 this.region = initHRegion(tableName, method, conf, families);
1122 try {
1123
1124 Put put = new Put(row1);
1125 put.add(fam1, qf1, val1);
1126 region.put(put);
1127 Threads.sleep(2);
1128
1129 put = new Put(row1);
1130 put.add(fam1, qf1, val2);
1131 put.add(fam2, qf1, val3);
1132 put.add(fam2, qf2, val2);
1133 put.add(fam2, qf3, val1);
1134 put.add(fam1, qf3, val1);
1135 region.put(put);
1136
1137
1138 Delete delete = new Delete(row1);
1139 delete.deleteColumn(fam1, qf1);
1140 delete.deleteColumn(fam2, qf1);
1141 delete.deleteColumn(fam1, qf3);
1142 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1143 val2), delete, true);
1144 assertEquals(true, res);
1145
1146 Get get = new Get(row1);
1147 get.addColumn(fam1, qf1);
1148 get.addColumn(fam1, qf3);
1149 get.addColumn(fam2, qf2);
1150 Result r = region.get(get);
1151 assertEquals(2, r.size());
1152 assertArrayEquals(val1, r.getValue(fam1, qf1));
1153 assertArrayEquals(val2, r.getValue(fam2, qf2));
1154
1155
1156 delete = new Delete(row1);
1157 delete.deleteFamily(fam2);
1158 res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1159 delete, true);
1160 assertEquals(true, res);
1161
1162 get = new Get(row1);
1163 r = region.get(get);
1164 assertEquals(1, r.size());
1165 assertArrayEquals(val1, r.getValue(fam1, qf1));
1166
1167
1168 delete = new Delete(row1);
1169 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1170 delete, true);
1171 assertEquals(true, res);
1172 get = new Get(row1);
1173 r = region.get(get);
1174 assertEquals(0, r.size());
1175 } finally {
1176 HRegion.closeHRegion(this.region);
1177 this.region = null;
1178 }
1179 }
1180
1181
1182
1183
1184 @Test
1185 public void testDelete_multiDeleteColumn() throws IOException {
1186 byte[] row1 = Bytes.toBytes("row1");
1187 byte[] fam1 = Bytes.toBytes("fam1");
1188 byte[] qual = Bytes.toBytes("qualifier");
1189 byte[] value = Bytes.toBytes("value");
1190
1191 Put put = new Put(row1);
1192 put.add(fam1, qual, 1, value);
1193 put.add(fam1, qual, 2, value);
1194
1195 String method = this.getName();
1196 this.region = initHRegion(tableName, method, conf, fam1);
1197 try {
1198 region.put(put);
1199
1200
1201 Delete delete = new Delete(row1);
1202 delete.deleteColumn(fam1, qual);
1203 delete.deleteColumn(fam1, qual);
1204 region.delete(delete);
1205
1206 Get get = new Get(row1);
1207 get.addFamily(fam1);
1208 Result r = region.get(get);
1209 assertEquals(0, r.size());
1210 } finally {
1211 HRegion.closeHRegion(this.region);
1212 this.region = null;
1213 }
1214 }
1215
1216 @Test
1217 public void testDelete_CheckFamily() throws IOException {
1218 byte[] row1 = Bytes.toBytes("row1");
1219 byte[] fam1 = Bytes.toBytes("fam1");
1220 byte[] fam2 = Bytes.toBytes("fam2");
1221 byte[] fam3 = Bytes.toBytes("fam3");
1222 byte[] fam4 = Bytes.toBytes("fam4");
1223
1224
1225 String method = this.getName();
1226 this.region = initHRegion(tableName, method, conf, fam1, fam2, fam3);
1227 try {
1228 List<Cell> kvs = new ArrayList<Cell>();
1229 kvs.add(new KeyValue(row1, fam4, null, null));
1230
1231
1232 byte[] family = fam2;
1233 try {
1234 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1235 Bytes.BYTES_COMPARATOR);
1236 deleteMap.put(family, kvs);
1237 region.delete(deleteMap, Durability.SYNC_WAL);
1238 } catch (Exception e) {
1239 assertTrue("Family " + new String(family) + " does not exist", false);
1240 }
1241
1242
1243 boolean ok = false;
1244 family = fam4;
1245 try {
1246 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1247 Bytes.BYTES_COMPARATOR);
1248 deleteMap.put(family, kvs);
1249 region.delete(deleteMap, Durability.SYNC_WAL);
1250 } catch (Exception e) {
1251 ok = true;
1252 }
1253 assertEquals("Family " + new String(family) + " does exist", true, ok);
1254 } finally {
1255 HRegion.closeHRegion(this.region);
1256 this.region = null;
1257 }
1258 }
1259
1260 @Test
1261 public void testDelete_mixed() throws IOException, InterruptedException {
1262 byte[] fam = Bytes.toBytes("info");
1263 byte[][] families = { fam };
1264 String method = this.getName();
1265 this.region = initHRegion(tableName, method, conf, families);
1266 try {
1267 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1268
1269 byte[] row = Bytes.toBytes("table_name");
1270
1271 byte[] serverinfo = Bytes.toBytes("serverinfo");
1272 byte[] splitA = Bytes.toBytes("splitA");
1273 byte[] splitB = Bytes.toBytes("splitB");
1274
1275
1276 Put put = new Put(row);
1277 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1278 region.put(put);
1279
1280 put = new Put(row);
1281 put.add(fam, splitB, Bytes.toBytes("reference_B"));
1282 region.put(put);
1283
1284 put = new Put(row);
1285 put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1286 region.put(put);
1287
1288
1289 Delete delete = new Delete(row);
1290 delete.deleteColumns(fam, splitA);
1291 region.delete(delete);
1292
1293
1294 Get get = new Get(row).addColumn(fam, serverinfo);
1295 Result result = region.get(get);
1296 assertEquals(1, result.size());
1297
1298 get = new Get(row).addColumn(fam, splitA);
1299 result = region.get(get);
1300 assertEquals(0, result.size());
1301
1302 get = new Get(row).addColumn(fam, splitB);
1303 result = region.get(get);
1304 assertEquals(1, result.size());
1305
1306
1307 put = new Put(row);
1308 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1309 region.put(put);
1310 get = new Get(row);
1311 result = region.get(get);
1312 assertEquals(3, result.size());
1313
1314
1315 delete = new Delete(row);
1316 region.delete(delete);
1317 assertEquals(0, region.get(get).size());
1318
1319 region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
1320 result = region.get(get);
1321 assertEquals(1, result.size());
1322 } finally {
1323 HRegion.closeHRegion(this.region);
1324 this.region = null;
1325 }
1326 }
1327
1328 @Test
1329 public void testDeleteRowWithFutureTs() throws IOException {
1330 byte[] fam = Bytes.toBytes("info");
1331 byte[][] families = { fam };
1332 String method = this.getName();
1333 this.region = initHRegion(tableName, method, conf, families);
1334 try {
1335 byte[] row = Bytes.toBytes("table_name");
1336
1337 byte[] serverinfo = Bytes.toBytes("serverinfo");
1338
1339
1340 Put put = new Put(row);
1341 put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
1342 region.put(put);
1343
1344
1345 Delete delete = new Delete(row);
1346 region.delete(delete);
1347
1348
1349 Get get = new Get(row).addColumn(fam, serverinfo);
1350 Result result = region.get(get);
1351 assertEquals(1, result.size());
1352
1353
1354 delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
1355 region.delete(delete);
1356
1357
1358 get = new Get(row).addColumn(fam, serverinfo);
1359 result = region.get(get);
1360 assertEquals(0, result.size());
1361 } finally {
1362 HRegion.closeHRegion(this.region);
1363 this.region = null;
1364 }
1365 }
1366
1367
1368
1369
1370
1371 @Test
1372 public void testPutWithLatestTS() throws IOException {
1373 byte[] fam = Bytes.toBytes("info");
1374 byte[][] families = { fam };
1375 String method = this.getName();
1376 this.region = initHRegion(tableName, method, conf, families);
1377 try {
1378 byte[] row = Bytes.toBytes("row1");
1379
1380 byte[] qual = Bytes.toBytes("qual");
1381
1382
1383 Put put = new Put(row);
1384 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1385 region.put(put);
1386
1387
1388 Get get = new Get(row).addColumn(fam, qual);
1389 Result result = region.get(get);
1390 assertEquals(1, result.size());
1391 Cell kv = result.rawCells()[0];
1392 LOG.info("Got: " + kv);
1393 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1394 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1395
1396
1397
1398 row = Bytes.toBytes("row2");
1399 put = new Put(row);
1400 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1401 region.put(put);
1402
1403
1404 get = new Get(row).addColumn(fam, qual);
1405 result = region.get(get);
1406 assertEquals(1, result.size());
1407 kv = result.rawCells()[0];
1408 LOG.info("Got: " + kv);
1409 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1410 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1411 } finally {
1412 HRegion.closeHRegion(this.region);
1413 this.region = null;
1414 }
1415
1416 }
1417
1418
1419
1420
1421
1422
1423 @Test
1424 public void testPutWithTsSlop() throws IOException {
1425 byte[] fam = Bytes.toBytes("info");
1426 byte[][] families = { fam };
1427 String method = this.getName();
1428
1429
1430 conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1431 this.region = initHRegion(tableName, method, conf, families);
1432 boolean caughtExcep = false;
1433 try {
1434 try {
1435
1436 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
1437
1438 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
1439 Bytes.toBytes("value")));
1440 fail("Expected IOE for TS out of configured timerange");
1441 } catch (FailedSanityCheckException ioe) {
1442 LOG.debug("Received expected exception", ioe);
1443 caughtExcep = true;
1444 }
1445 assertTrue("Should catch FailedSanityCheckException", caughtExcep);
1446 } finally {
1447 HRegion.closeHRegion(this.region);
1448 this.region = null;
1449 }
1450 }
1451
1452 @Test
1453 public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
1454 byte[] fam1 = Bytes.toBytes("columnA");
1455 byte[] fam2 = Bytes.toBytes("columnB");
1456 this.region = initHRegion(tableName, getName(), conf, fam1, fam2);
1457 try {
1458 byte[] rowA = Bytes.toBytes("rowA");
1459 byte[] rowB = Bytes.toBytes("rowB");
1460
1461 byte[] value = Bytes.toBytes("value");
1462
1463 Delete delete = new Delete(rowA);
1464 delete.deleteFamily(fam1);
1465
1466 region.delete(delete);
1467
1468
1469 Put put = new Put(rowA);
1470 put.add(fam2, null, value);
1471 region.put(put);
1472
1473 put = new Put(rowB);
1474 put.add(fam1, null, value);
1475 put.add(fam2, null, value);
1476 region.put(put);
1477
1478 Scan scan = new Scan();
1479 scan.addFamily(fam1).addFamily(fam2);
1480 InternalScanner s = region.getScanner(scan);
1481 List<Cell> results = new ArrayList<Cell>();
1482 s.next(results);
1483 assertTrue(CellUtil.matchingRow(results.get(0), rowA));
1484
1485 results.clear();
1486 s.next(results);
1487 assertTrue(CellUtil.matchingRow(results.get(0), rowB));
1488 } finally {
1489 HRegion.closeHRegion(this.region);
1490 this.region = null;
1491 }
1492 }
1493
1494 @Test
1495 public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
1496 Delete delete = new Delete(row);
1497 delete.deleteColumns(fam1, qual1);
1498 doTestDelete_AndPostInsert(delete);
1499 }
1500
1501 @Test
1502 public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
1503 Delete delete = new Delete(row);
1504 delete.deleteFamily(fam1);
1505 doTestDelete_AndPostInsert(delete);
1506 }
1507
1508 public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
1509 TableName tableName = TableName.valueOf(name.getMethodName());
1510 this.region = initHRegion(tableName, getName(), conf, fam1);
1511 try {
1512 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1513 Put put = new Put(row);
1514 put.add(fam1, qual1, value1);
1515 region.put(put);
1516
1517
1518 region.delete(delete);
1519
1520
1521 put = new Put(row);
1522 put.add(fam1, qual1, value2);
1523 region.put(put);
1524
1525
1526 Get get = new Get(row);
1527 get.addColumn(fam1, qual1);
1528
1529 Result r = region.get(get);
1530 assertEquals(1, r.size());
1531 assertArrayEquals(value2, r.getValue(fam1, qual1));
1532
1533
1534 Scan scan = new Scan(row);
1535 scan.addColumn(fam1, qual1);
1536 InternalScanner s = region.getScanner(scan);
1537
1538 List<Cell> results = new ArrayList<Cell>();
1539 assertEquals(false, s.next(results));
1540 assertEquals(1, results.size());
1541 Cell kv = results.get(0);
1542
1543 assertArrayEquals(value2, CellUtil.cloneValue(kv));
1544 assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
1545 assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
1546 assertArrayEquals(row, CellUtil.cloneRow(kv));
1547 } finally {
1548 HRegion.closeHRegion(this.region);
1549 this.region = null;
1550 }
1551 }
1552
1553 @Test
1554 public void testDelete_CheckTimestampUpdated() throws IOException {
1555 TableName tableName = TableName.valueOf(name.getMethodName());
1556 byte[] row1 = Bytes.toBytes("row1");
1557 byte[] col1 = Bytes.toBytes("col1");
1558 byte[] col2 = Bytes.toBytes("col2");
1559 byte[] col3 = Bytes.toBytes("col3");
1560
1561
1562 String method = this.getName();
1563 this.region = initHRegion(tableName, method, conf, fam1);
1564 try {
1565
1566 List<Cell> kvs = new ArrayList<Cell>();
1567 kvs.add(new KeyValue(row1, fam1, col1, null));
1568 kvs.add(new KeyValue(row1, fam1, col2, null));
1569 kvs.add(new KeyValue(row1, fam1, col3, null));
1570
1571 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1572 Bytes.BYTES_COMPARATOR);
1573 deleteMap.put(fam1, kvs);
1574 region.delete(deleteMap, Durability.SYNC_WAL);
1575
1576
1577
1578 long now = System.currentTimeMillis();
1579 KeyValue firstKv = ((HStore) region.getStore(fam1)).memstore.kvset.first();
1580 assertTrue(firstKv.getTimestamp() <= now);
1581 now = firstKv.getTimestamp();
1582 for (KeyValue kv : ((HStore) region.getStore(fam1)).memstore.kvset) {
1583 assertTrue(kv.getTimestamp() <= now);
1584 now = kv.getTimestamp();
1585 }
1586 } finally {
1587 HRegion.closeHRegion(this.region);
1588 this.region = null;
1589 }
1590 }
1591
1592
1593
1594
1595 @Test
1596 public void testGet_FamilyChecker() throws IOException {
1597 byte[] row1 = Bytes.toBytes("row1");
1598 byte[] fam1 = Bytes.toBytes("fam1");
1599 byte[] fam2 = Bytes.toBytes("False");
1600 byte[] col1 = Bytes.toBytes("col1");
1601
1602
1603 String method = this.getName();
1604 this.region = initHRegion(tableName, method, conf, fam1);
1605 try {
1606 Get get = new Get(row1);
1607 get.addColumn(fam2, col1);
1608
1609
1610 try {
1611 region.get(get);
1612 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
1613 assertFalse(false);
1614 return;
1615 }
1616 assertFalse(true);
1617 } finally {
1618 HRegion.closeHRegion(this.region);
1619 this.region = null;
1620 }
1621 }
1622
1623 @Test
1624 public void testGet_Basic() throws IOException {
1625 byte[] row1 = Bytes.toBytes("row1");
1626 byte[] fam1 = Bytes.toBytes("fam1");
1627 byte[] col1 = Bytes.toBytes("col1");
1628 byte[] col2 = Bytes.toBytes("col2");
1629 byte[] col3 = Bytes.toBytes("col3");
1630 byte[] col4 = Bytes.toBytes("col4");
1631 byte[] col5 = Bytes.toBytes("col5");
1632
1633
1634 String method = this.getName();
1635 this.region = initHRegion(tableName, method, conf, fam1);
1636 try {
1637
1638 Put put = new Put(row1);
1639 put.add(fam1, col1, null);
1640 put.add(fam1, col2, null);
1641 put.add(fam1, col3, null);
1642 put.add(fam1, col4, null);
1643 put.add(fam1, col5, null);
1644 region.put(put);
1645
1646 Get get = new Get(row1);
1647 get.addColumn(fam1, col2);
1648 get.addColumn(fam1, col4);
1649
1650 KeyValue kv1 = new KeyValue(row1, fam1, col2);
1651 KeyValue kv2 = new KeyValue(row1, fam1, col4);
1652 KeyValue[] expected = { kv1, kv2 };
1653
1654
1655 Result res = region.get(get);
1656 assertEquals(expected.length, res.size());
1657 for (int i = 0; i < res.size(); i++) {
1658 assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
1659 assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
1660 assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
1661 }
1662
1663
1664 Get g = new Get(row1);
1665 final int count = 2;
1666 g.setFilter(new ColumnCountGetFilter(count));
1667 res = region.get(g);
1668 assertEquals(count, res.size());
1669 } finally {
1670 HRegion.closeHRegion(this.region);
1671 this.region = null;
1672 }
1673 }
1674
1675 @Test
1676 public void testGet_Empty() throws IOException {
1677 byte[] row = Bytes.toBytes("row");
1678 byte[] fam = Bytes.toBytes("fam");
1679
1680 String method = this.getName();
1681 this.region = initHRegion(tableName, method, conf, fam);
1682 try {
1683 Get get = new Get(row);
1684 get.addFamily(fam);
1685 Result r = region.get(get);
1686
1687 assertTrue(r.isEmpty());
1688 } finally {
1689 HRegion.closeHRegion(this.region);
1690 this.region = null;
1691 }
1692 }
1693
1694
1695
1696
1697 @Test
1698 public void testMerge() throws IOException {
1699 byte[][] families = { fam1, fam2, fam3 };
1700 Configuration hc = initSplit();
1701
1702 String method = this.getName();
1703 this.region = initHRegion(tableName, method, hc, families);
1704 try {
1705 LOG.info("" + HBaseTestCase.addContent(region, fam3));
1706 region.flushcache();
1707 region.compactStores();
1708 byte[] splitRow = region.checkSplit();
1709 assertNotNull(splitRow);
1710 LOG.info("SplitRow: " + Bytes.toString(splitRow));
1711 HRegion[] subregions = splitRegion(region, splitRow);
1712 try {
1713
1714 for (int i = 0; i < subregions.length; i++) {
1715 HRegion.openHRegion(subregions[i], null);
1716 subregions[i].compactStores();
1717 }
1718 Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
1719 Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
1720 Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
1721 long startTime = System.currentTimeMillis();
1722 region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
1723 LOG.info("Merge regions elapsed time: "
1724 + ((System.currentTimeMillis() - startTime) / 1000.0));
1725 fs.delete(oldRegion1, true);
1726 fs.delete(oldRegion2, true);
1727 fs.delete(oldRegionPath, true);
1728 LOG.info("splitAndMerge completed.");
1729 } finally {
1730 for (int i = 0; i < subregions.length; i++) {
1731 try {
1732 HRegion.closeHRegion(subregions[i]);
1733 } catch (IOException e) {
1734
1735 }
1736 }
1737 }
1738 } finally {
1739 HRegion.closeHRegion(this.region);
1740 this.region = null;
1741 }
1742 }
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752 HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
1753 PairOfSameType<HRegion> result = null;
1754 SplitTransaction st = new SplitTransaction(parent, midkey);
1755
1756
1757 if (!st.prepare())
1758 return null;
1759 try {
1760 result = st.execute(null, null);
1761 } catch (IOException ioe) {
1762 try {
1763 LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; "
1764 + ioe.getMessage());
1765 st.rollback(null, null);
1766 LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString());
1767 return null;
1768 } catch (RuntimeException e) {
1769
1770 LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString()
1771 + " -- aborting server", e);
1772 }
1773 }
1774 return new HRegion[] { result.getFirst(), result.getSecond() };
1775 }
1776
1777
1778
1779
1780 @Test
1781 public void testGetScanner_WithOkFamilies() throws IOException {
1782 byte[] fam1 = Bytes.toBytes("fam1");
1783 byte[] fam2 = Bytes.toBytes("fam2");
1784
1785 byte[][] families = { fam1, fam2 };
1786
1787
1788 String method = this.getName();
1789 this.region = initHRegion(tableName, method, conf, families);
1790 try {
1791 Scan scan = new Scan();
1792 scan.addFamily(fam1);
1793 scan.addFamily(fam2);
1794 try {
1795 region.getScanner(scan);
1796 } catch (Exception e) {
1797 assertTrue("Families could not be found in Region", false);
1798 }
1799 } finally {
1800 HRegion.closeHRegion(this.region);
1801 this.region = null;
1802 }
1803 }
1804
1805 @Test
1806 public void testGetScanner_WithNotOkFamilies() throws IOException {
1807 byte[] fam1 = Bytes.toBytes("fam1");
1808 byte[] fam2 = Bytes.toBytes("fam2");
1809
1810 byte[][] families = { fam1 };
1811
1812
1813 String method = this.getName();
1814 this.region = initHRegion(tableName, method, conf, families);
1815 try {
1816 Scan scan = new Scan();
1817 scan.addFamily(fam2);
1818 boolean ok = false;
1819 try {
1820 region.getScanner(scan);
1821 } catch (Exception e) {
1822 ok = true;
1823 }
1824 assertTrue("Families could not be found in Region", ok);
1825 } finally {
1826 HRegion.closeHRegion(this.region);
1827 this.region = null;
1828 }
1829 }
1830
1831 @Test
1832 public void testGetScanner_WithNoFamilies() throws IOException {
1833 byte[] row1 = Bytes.toBytes("row1");
1834 byte[] fam1 = Bytes.toBytes("fam1");
1835 byte[] fam2 = Bytes.toBytes("fam2");
1836 byte[] fam3 = Bytes.toBytes("fam3");
1837 byte[] fam4 = Bytes.toBytes("fam4");
1838
1839 byte[][] families = { fam1, fam2, fam3, fam4 };
1840
1841
1842 String method = this.getName();
1843 this.region = initHRegion(tableName, method, conf, families);
1844 try {
1845
1846
1847 Put put = new Put(row1);
1848 put.add(fam1, null, null);
1849 put.add(fam2, null, null);
1850 put.add(fam3, null, null);
1851 put.add(fam4, null, null);
1852 region.put(put);
1853
1854 Scan scan = null;
1855 HRegion.RegionScannerImpl is = null;
1856
1857
1858
1859
1860 scan = new Scan();
1861 scan.addFamily(fam2);
1862 scan.addFamily(fam4);
1863 is = (RegionScannerImpl) region.getScanner(scan);
1864 MultiVersionConsistencyControl.resetThreadReadPoint(region.getMVCC());
1865 assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
1866
1867 scan = new Scan();
1868 is = (RegionScannerImpl) region.getScanner(scan);
1869 MultiVersionConsistencyControl.resetThreadReadPoint(region.getMVCC());
1870 assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
1871 } finally {
1872 HRegion.closeHRegion(this.region);
1873 this.region = null;
1874 }
1875 }
1876
1877
1878
1879
1880
1881
1882 @Test
1883 public void testGetScanner_WithRegionClosed() throws IOException {
1884 byte[] fam1 = Bytes.toBytes("fam1");
1885 byte[] fam2 = Bytes.toBytes("fam2");
1886
1887 byte[][] families = { fam1, fam2 };
1888
1889
1890 String method = this.getName();
1891 try {
1892 this.region = initHRegion(tableName, method, conf, families);
1893 } catch (IOException e) {
1894 e.printStackTrace();
1895 fail("Got IOException during initHRegion, " + e.getMessage());
1896 }
1897 try {
1898 region.closed.set(true);
1899 try {
1900 region.getScanner(null);
1901 fail("Expected to get an exception during getScanner on a region that is closed");
1902 } catch (NotServingRegionException e) {
1903
1904 } catch (IOException e) {
1905 fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
1906 + e.getMessage());
1907 }
1908 } finally {
1909 HRegion.closeHRegion(this.region);
1910 this.region = null;
1911 }
1912 }
1913
1914 @Test
1915 public void testRegionScanner_Next() throws IOException {
1916 byte[] row1 = Bytes.toBytes("row1");
1917 byte[] row2 = Bytes.toBytes("row2");
1918 byte[] fam1 = Bytes.toBytes("fam1");
1919 byte[] fam2 = Bytes.toBytes("fam2");
1920 byte[] fam3 = Bytes.toBytes("fam3");
1921 byte[] fam4 = Bytes.toBytes("fam4");
1922
1923 byte[][] families = { fam1, fam2, fam3, fam4 };
1924 long ts = System.currentTimeMillis();
1925
1926
1927 String method = this.getName();
1928 this.region = initHRegion(tableName, method, conf, families);
1929 try {
1930
1931 Put put = null;
1932 put = new Put(row1);
1933 put.add(fam1, (byte[]) null, ts, null);
1934 put.add(fam2, (byte[]) null, ts, null);
1935 put.add(fam3, (byte[]) null, ts, null);
1936 put.add(fam4, (byte[]) null, ts, null);
1937 region.put(put);
1938
1939 put = new Put(row2);
1940 put.add(fam1, (byte[]) null, ts, null);
1941 put.add(fam2, (byte[]) null, ts, null);
1942 put.add(fam3, (byte[]) null, ts, null);
1943 put.add(fam4, (byte[]) null, ts, null);
1944 region.put(put);
1945
1946 Scan scan = new Scan();
1947 scan.addFamily(fam2);
1948 scan.addFamily(fam4);
1949 InternalScanner is = region.getScanner(scan);
1950
1951 List<Cell> res = null;
1952
1953
1954 List<Cell> expected1 = new ArrayList<Cell>();
1955 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
1956 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
1957
1958 res = new ArrayList<Cell>();
1959 is.next(res);
1960 for (int i = 0; i < res.size(); i++) {
1961 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
1962 }
1963
1964
1965 List<Cell> expected2 = new ArrayList<Cell>();
1966 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
1967 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
1968
1969 res = new ArrayList<Cell>();
1970 is.next(res);
1971 for (int i = 0; i < res.size(); i++) {
1972 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
1973 }
1974 } finally {
1975 HRegion.closeHRegion(this.region);
1976 this.region = null;
1977 }
1978 }
1979
1980 @Test
1981 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
1982 byte[] row1 = Bytes.toBytes("row1");
1983 byte[] qf1 = Bytes.toBytes("qualifier1");
1984 byte[] qf2 = Bytes.toBytes("qualifier2");
1985 byte[] fam1 = Bytes.toBytes("fam1");
1986 byte[][] families = { fam1 };
1987
1988 long ts1 = System.currentTimeMillis();
1989 long ts2 = ts1 + 1;
1990 long ts3 = ts1 + 2;
1991
1992
1993 String method = this.getName();
1994 this.region = initHRegion(tableName, method, conf, families);
1995 try {
1996
1997 Put put = null;
1998 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
1999 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2000 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2001
2002 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2003 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2004 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2005
2006 put = new Put(row1);
2007 put.add(kv13);
2008 put.add(kv12);
2009 put.add(kv11);
2010 put.add(kv23);
2011 put.add(kv22);
2012 put.add(kv21);
2013 region.put(put);
2014
2015
2016 List<Cell> expected = new ArrayList<Cell>();
2017 expected.add(kv13);
2018 expected.add(kv12);
2019
2020 Scan scan = new Scan(row1);
2021 scan.addColumn(fam1, qf1);
2022 scan.setMaxVersions(MAX_VERSIONS);
2023 List<Cell> actual = new ArrayList<Cell>();
2024 InternalScanner scanner = region.getScanner(scan);
2025
2026 boolean hasNext = scanner.next(actual);
2027 assertEquals(false, hasNext);
2028
2029
2030 for (int i = 0; i < expected.size(); i++) {
2031 assertEquals(expected.get(i), actual.get(i));
2032 }
2033 } finally {
2034 HRegion.closeHRegion(this.region);
2035 this.region = null;
2036 }
2037 }
2038
2039 @Test
2040 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2041 byte[] row1 = Bytes.toBytes("row1");
2042 byte[] qf1 = Bytes.toBytes("qualifier1");
2043 byte[] qf2 = Bytes.toBytes("qualifier2");
2044 byte[] fam1 = Bytes.toBytes("fam1");
2045 byte[][] families = { fam1 };
2046
2047 long ts1 = 1;
2048 long ts2 = ts1 + 1;
2049 long ts3 = ts1 + 2;
2050
2051
2052 String method = this.getName();
2053 this.region = initHRegion(tableName, method, conf, families);
2054 try {
2055
2056 Put put = null;
2057 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2058 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2059 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2060
2061 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2062 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2063 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2064
2065 put = new Put(row1);
2066 put.add(kv13);
2067 put.add(kv12);
2068 put.add(kv11);
2069 put.add(kv23);
2070 put.add(kv22);
2071 put.add(kv21);
2072 region.put(put);
2073 region.flushcache();
2074
2075
2076 List<Cell> expected = new ArrayList<Cell>();
2077 expected.add(kv13);
2078 expected.add(kv12);
2079 expected.add(kv23);
2080 expected.add(kv22);
2081
2082 Scan scan = new Scan(row1);
2083 scan.addColumn(fam1, qf1);
2084 scan.addColumn(fam1, qf2);
2085 scan.setMaxVersions(MAX_VERSIONS);
2086 List<Cell> actual = new ArrayList<Cell>();
2087 InternalScanner scanner = region.getScanner(scan);
2088
2089 boolean hasNext = scanner.next(actual);
2090 assertEquals(false, hasNext);
2091
2092
2093 for (int i = 0; i < expected.size(); i++) {
2094 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2095 }
2096 } finally {
2097 HRegion.closeHRegion(this.region);
2098 this.region = null;
2099 }
2100 }
2101
2102 @Test
2103 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2104 byte[] row1 = Bytes.toBytes("row1");
2105 byte[] fam1 = Bytes.toBytes("fam1");
2106 byte[][] families = { fam1 };
2107 byte[] qf1 = Bytes.toBytes("qualifier1");
2108 byte[] qf2 = Bytes.toBytes("qualifier2");
2109
2110 long ts1 = 1;
2111 long ts2 = ts1 + 1;
2112 long ts3 = ts1 + 2;
2113 long ts4 = ts1 + 3;
2114
2115
2116 String method = this.getName();
2117 this.region = initHRegion(tableName, method, conf, families);
2118 try {
2119
2120 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2121 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2122 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2123 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2124
2125 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2126 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2127 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2128 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2129
2130 Put put = null;
2131 put = new Put(row1);
2132 put.add(kv14);
2133 put.add(kv24);
2134 region.put(put);
2135 region.flushcache();
2136
2137 put = new Put(row1);
2138 put.add(kv23);
2139 put.add(kv13);
2140 region.put(put);
2141 region.flushcache();
2142
2143 put = new Put(row1);
2144 put.add(kv22);
2145 put.add(kv12);
2146 region.put(put);
2147 region.flushcache();
2148
2149 put = new Put(row1);
2150 put.add(kv21);
2151 put.add(kv11);
2152 region.put(put);
2153
2154
2155 List<Cell> expected = new ArrayList<Cell>();
2156 expected.add(kv14);
2157 expected.add(kv13);
2158 expected.add(kv12);
2159 expected.add(kv24);
2160 expected.add(kv23);
2161 expected.add(kv22);
2162
2163 Scan scan = new Scan(row1);
2164 scan.addColumn(fam1, qf1);
2165 scan.addColumn(fam1, qf2);
2166 int versions = 3;
2167 scan.setMaxVersions(versions);
2168 List<Cell> actual = new ArrayList<Cell>();
2169 InternalScanner scanner = region.getScanner(scan);
2170
2171 boolean hasNext = scanner.next(actual);
2172 assertEquals(false, hasNext);
2173
2174
2175 for (int i = 0; i < expected.size(); i++) {
2176 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2177 }
2178 } finally {
2179 HRegion.closeHRegion(this.region);
2180 this.region = null;
2181 }
2182 }
2183
2184 @Test
2185 public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2186 byte[] row1 = Bytes.toBytes("row1");
2187 byte[] qf1 = Bytes.toBytes("qualifier1");
2188 byte[] qf2 = Bytes.toBytes("qualifier2");
2189 byte[] fam1 = Bytes.toBytes("fam1");
2190 byte[][] families = { fam1 };
2191
2192 long ts1 = System.currentTimeMillis();
2193 long ts2 = ts1 + 1;
2194 long ts3 = ts1 + 2;
2195
2196
2197 String method = this.getName();
2198 this.region = initHRegion(tableName, method, conf, families);
2199 try {
2200
2201 Put put = null;
2202 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2203 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2204 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2205
2206 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2207 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2208 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2209
2210 put = new Put(row1);
2211 put.add(kv13);
2212 put.add(kv12);
2213 put.add(kv11);
2214 put.add(kv23);
2215 put.add(kv22);
2216 put.add(kv21);
2217 region.put(put);
2218
2219
2220 List<Cell> expected = new ArrayList<Cell>();
2221 expected.add(kv13);
2222 expected.add(kv12);
2223 expected.add(kv23);
2224 expected.add(kv22);
2225
2226 Scan scan = new Scan(row1);
2227 scan.addFamily(fam1);
2228 scan.setMaxVersions(MAX_VERSIONS);
2229 List<Cell> actual = new ArrayList<Cell>();
2230 InternalScanner scanner = region.getScanner(scan);
2231
2232 boolean hasNext = scanner.next(actual);
2233 assertEquals(false, hasNext);
2234
2235
2236 for (int i = 0; i < expected.size(); i++) {
2237 assertEquals(expected.get(i), actual.get(i));
2238 }
2239 } finally {
2240 HRegion.closeHRegion(this.region);
2241 this.region = null;
2242 }
2243 }
2244
2245 @Test
2246 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
2247 byte[] row1 = Bytes.toBytes("row1");
2248 byte[] qf1 = Bytes.toBytes("qualifier1");
2249 byte[] qf2 = Bytes.toBytes("qualifier2");
2250 byte[] fam1 = Bytes.toBytes("fam1");
2251
2252 long ts1 = 1;
2253 long ts2 = ts1 + 1;
2254 long ts3 = ts1 + 2;
2255
2256
2257 String method = this.getName();
2258 this.region = initHRegion(tableName, method, conf, fam1);
2259 try {
2260
2261 Put put = null;
2262 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2263 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2264 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2265
2266 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2267 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2268 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2269
2270 put = new Put(row1);
2271 put.add(kv13);
2272 put.add(kv12);
2273 put.add(kv11);
2274 put.add(kv23);
2275 put.add(kv22);
2276 put.add(kv21);
2277 region.put(put);
2278 region.flushcache();
2279
2280
2281 List<Cell> expected = new ArrayList<Cell>();
2282 expected.add(kv13);
2283 expected.add(kv12);
2284 expected.add(kv23);
2285 expected.add(kv22);
2286
2287 Scan scan = new Scan(row1);
2288 scan.addFamily(fam1);
2289 scan.setMaxVersions(MAX_VERSIONS);
2290 List<Cell> actual = new ArrayList<Cell>();
2291 InternalScanner scanner = region.getScanner(scan);
2292
2293 boolean hasNext = scanner.next(actual);
2294 assertEquals(false, hasNext);
2295
2296
2297 for (int i = 0; i < expected.size(); i++) {
2298 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2299 }
2300 } finally {
2301 HRegion.closeHRegion(this.region);
2302 this.region = null;
2303 }
2304 }
2305
2306 @Test
2307 public void testScanner_StopRow1542() throws IOException {
2308 byte[] family = Bytes.toBytes("testFamily");
2309 this.region = initHRegion(tableName, getName(), conf, family);
2310 try {
2311 byte[] row1 = Bytes.toBytes("row111");
2312 byte[] row2 = Bytes.toBytes("row222");
2313 byte[] row3 = Bytes.toBytes("row333");
2314 byte[] row4 = Bytes.toBytes("row444");
2315 byte[] row5 = Bytes.toBytes("row555");
2316
2317 byte[] col1 = Bytes.toBytes("Pub111");
2318 byte[] col2 = Bytes.toBytes("Pub222");
2319
2320 Put put = new Put(row1);
2321 put.add(family, col1, Bytes.toBytes(10L));
2322 region.put(put);
2323
2324 put = new Put(row2);
2325 put.add(family, col1, Bytes.toBytes(15L));
2326 region.put(put);
2327
2328 put = new Put(row3);
2329 put.add(family, col2, Bytes.toBytes(20L));
2330 region.put(put);
2331
2332 put = new Put(row4);
2333 put.add(family, col2, Bytes.toBytes(30L));
2334 region.put(put);
2335
2336 put = new Put(row5);
2337 put.add(family, col1, Bytes.toBytes(40L));
2338 region.put(put);
2339
2340 Scan scan = new Scan(row3, row4);
2341 scan.setMaxVersions();
2342 scan.addColumn(family, col1);
2343 InternalScanner s = region.getScanner(scan);
2344
2345 List<Cell> results = new ArrayList<Cell>();
2346 assertEquals(false, s.next(results));
2347 assertEquals(0, results.size());
2348 } finally {
2349 HRegion.closeHRegion(this.region);
2350 this.region = null;
2351 }
2352 }
2353
2354 private void assertICV(byte [] row,
2355 byte [] familiy,
2356 byte[] qualifier,
2357 long amount) throws IOException {
2358
2359 Get get = new Get(row);
2360 get.addColumn(familiy, qualifier);
2361 Result result = region.get(get);
2362 assertEquals(1, result.size());
2363
2364 Cell kv = result.rawCells()[0];
2365 long r = Bytes.toLong(CellUtil.cloneValue(kv));
2366 assertEquals(amount, r);
2367 }
2368
2369 private void assertICV(byte[] row, byte[] familiy, byte[] qualifier, int amount)
2370 throws IOException {
2371
2372 Get get = new Get(row);
2373 get.addColumn(familiy, qualifier);
2374 Result result = region.get(get);
2375 assertEquals(1, result.size());
2376
2377 Cell kv = result.rawCells()[0];
2378 int r = Bytes.toInt(CellUtil.cloneValue(kv));
2379 assertEquals(amount, r);
2380 }
2381
2382 @Test
2383 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2384 byte[] row1 = Bytes.toBytes("row1");
2385 byte[] fam1 = Bytes.toBytes("fam1");
2386 byte[] qf1 = Bytes.toBytes("qualifier1");
2387 byte[] qf2 = Bytes.toBytes("quateslifier2");
2388
2389 long ts1 = 1;
2390 long ts2 = ts1 + 1;
2391 long ts3 = ts1 + 2;
2392 long ts4 = ts1 + 3;
2393
2394
2395 String method = this.getName();
2396 this.region = initHRegion(tableName, method, conf, fam1);
2397 try {
2398
2399 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2400 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2401 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2402 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2403
2404 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2405 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2406 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2407 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2408
2409 Put put = null;
2410 put = new Put(row1);
2411 put.add(kv14);
2412 put.add(kv24);
2413 region.put(put);
2414 region.flushcache();
2415
2416 put = new Put(row1);
2417 put.add(kv23);
2418 put.add(kv13);
2419 region.put(put);
2420 region.flushcache();
2421
2422 put = new Put(row1);
2423 put.add(kv22);
2424 put.add(kv12);
2425 region.put(put);
2426 region.flushcache();
2427
2428 put = new Put(row1);
2429 put.add(kv21);
2430 put.add(kv11);
2431 region.put(put);
2432
2433
2434 List<KeyValue> expected = new ArrayList<KeyValue>();
2435 expected.add(kv14);
2436 expected.add(kv13);
2437 expected.add(kv12);
2438 expected.add(kv24);
2439 expected.add(kv23);
2440 expected.add(kv22);
2441
2442 Scan scan = new Scan(row1);
2443 int versions = 3;
2444 scan.setMaxVersions(versions);
2445 List<Cell> actual = new ArrayList<Cell>();
2446 InternalScanner scanner = region.getScanner(scan);
2447
2448 boolean hasNext = scanner.next(actual);
2449 assertEquals(false, hasNext);
2450
2451
2452 for (int i = 0; i < expected.size(); i++) {
2453 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2454 }
2455 } finally {
2456 HRegion.closeHRegion(this.region);
2457 this.region = null;
2458 }
2459 }
2460
2461
2462
2463
2464
2465
2466
2467 @Test
2468 public void testScanner_JoinedScanners() throws IOException {
2469 byte[] cf_essential = Bytes.toBytes("essential");
2470 byte[] cf_joined = Bytes.toBytes("joined");
2471 byte[] cf_alpha = Bytes.toBytes("alpha");
2472 this.region = initHRegion(tableName, getName(), conf, cf_essential, cf_joined, cf_alpha);
2473 try {
2474 byte[] row1 = Bytes.toBytes("row1");
2475 byte[] row2 = Bytes.toBytes("row2");
2476 byte[] row3 = Bytes.toBytes("row3");
2477
2478 byte[] col_normal = Bytes.toBytes("d");
2479 byte[] col_alpha = Bytes.toBytes("a");
2480
2481 byte[] filtered_val = Bytes.toBytes(3);
2482
2483 Put put = new Put(row1);
2484 put.add(cf_essential, col_normal, Bytes.toBytes(1));
2485 put.add(cf_joined, col_alpha, Bytes.toBytes(1));
2486 region.put(put);
2487
2488 put = new Put(row2);
2489 put.add(cf_essential, col_alpha, Bytes.toBytes(2));
2490 put.add(cf_joined, col_normal, Bytes.toBytes(2));
2491 put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
2492 region.put(put);
2493
2494 put = new Put(row3);
2495 put.add(cf_essential, col_normal, filtered_val);
2496 put.add(cf_joined, col_normal, filtered_val);
2497 region.put(put);
2498
2499
2500
2501
2502
2503 Scan scan = new Scan();
2504 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
2505 CompareOp.NOT_EQUAL, filtered_val);
2506 scan.setFilter(filter);
2507 scan.setLoadColumnFamiliesOnDemand(true);
2508 InternalScanner s = region.getScanner(scan);
2509
2510 List<Cell> results = new ArrayList<Cell>();
2511 assertTrue(s.next(results));
2512 assertEquals(results.size(), 1);
2513 results.clear();
2514
2515 assertTrue(s.next(results));
2516 assertEquals(results.size(), 3);
2517 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
2518 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
2519 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
2520 results.clear();
2521
2522 assertFalse(s.next(results));
2523 assertEquals(results.size(), 0);
2524 } finally {
2525 HRegion.closeHRegion(this.region);
2526 this.region = null;
2527 }
2528 }
2529
2530
2531
2532
2533
2534
2535 @Test
2536 public void testScanner_JoinedScannersWithLimits() throws IOException {
2537 final byte[] cf_first = Bytes.toBytes("first");
2538 final byte[] cf_second = Bytes.toBytes("second");
2539
2540 this.region = initHRegion(tableName, getName(), conf, cf_first, cf_second);
2541 try {
2542 final byte[] col_a = Bytes.toBytes("a");
2543 final byte[] col_b = Bytes.toBytes("b");
2544
2545 Put put;
2546
2547 for (int i = 0; i < 10; i++) {
2548 put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
2549 put.add(cf_first, col_a, Bytes.toBytes(i));
2550 if (i < 5) {
2551 put.add(cf_first, col_b, Bytes.toBytes(i));
2552 put.add(cf_second, col_a, Bytes.toBytes(i));
2553 put.add(cf_second, col_b, Bytes.toBytes(i));
2554 }
2555 region.put(put);
2556 }
2557
2558 Scan scan = new Scan();
2559 scan.setLoadColumnFamiliesOnDemand(true);
2560 Filter bogusFilter = new FilterBase() {
2561 @Override
2562 public boolean isFamilyEssential(byte[] name) {
2563 return Bytes.equals(name, cf_first);
2564 }
2565 };
2566
2567 scan.setFilter(bogusFilter);
2568 InternalScanner s = region.getScanner(scan);
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599 List<Cell> results = new ArrayList<Cell>();
2600 int index = 0;
2601 while (true) {
2602 boolean more = s.next(results, 3);
2603 if ((index >> 1) < 5) {
2604 if (index % 2 == 0)
2605 assertEquals(results.size(), 3);
2606 else
2607 assertEquals(results.size(), 1);
2608 } else
2609 assertEquals(results.size(), 1);
2610 results.clear();
2611 index++;
2612 if (!more)
2613 break;
2614 }
2615 } finally {
2616 HRegion.closeHRegion(this.region);
2617 this.region = null;
2618 }
2619 }
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629 @Test
2630 public void testBasicSplit() throws Exception {
2631 byte[][] families = { fam1, fam2, fam3 };
2632
2633 Configuration hc = initSplit();
2634
2635 String method = this.getName();
2636 this.region = initHRegion(tableName, method, hc, families);
2637
2638 try {
2639 LOG.info("" + HBaseTestCase.addContent(region, fam3));
2640 region.flushcache();
2641 region.compactStores();
2642 byte[] splitRow = region.checkSplit();
2643 assertNotNull(splitRow);
2644 LOG.info("SplitRow: " + Bytes.toString(splitRow));
2645 HRegion[] regions = splitRegion(region, splitRow);
2646 try {
2647
2648
2649
2650 for (int i = 0; i < regions.length; i++) {
2651 regions[i] = HRegion.openHRegion(regions[i], null);
2652 }
2653
2654
2655 assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
2656 assertGet(regions[1], fam3, splitRow);
2657
2658 assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
2659 assertScan(regions[1], fam3, splitRow);
2660
2661 for (int i = 0; i < regions.length; i++) {
2662
2663
2664 for (int j = 0; j < 2; j++) {
2665 HBaseTestCase.addContent(regions[i], fam3);
2666 }
2667 HBaseTestCase.addContent(regions[i], fam2);
2668 HBaseTestCase.addContent(regions[i], fam1);
2669 regions[i].flushcache();
2670 }
2671
2672 byte[][] midkeys = new byte[regions.length][];
2673
2674 for (int i = 0; i < regions.length; i++) {
2675 regions[i].compactStores();
2676 midkeys[i] = regions[i].checkSplit();
2677 }
2678
2679 TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
2680
2681
2682 for (int i = 0; i < regions.length; i++) {
2683 HRegion[] rs = null;
2684 if (midkeys[i] != null) {
2685 rs = splitRegion(regions[i], midkeys[i]);
2686 for (int j = 0; j < rs.length; j++) {
2687 sortedMap.put(Bytes.toString(rs[j].getRegionName()), HRegion.openHRegion(rs[j], null));
2688 }
2689 }
2690 }
2691 LOG.info("Made 4 regions");
2692
2693
2694 int interval = (LAST_CHAR - FIRST_CHAR) / 3;
2695 byte[] b = Bytes.toBytes(START_KEY);
2696 for (HRegion r : sortedMap.values()) {
2697 assertGet(r, fam3, b);
2698 b[0] += interval;
2699 }
2700 } finally {
2701 for (int i = 0; i < regions.length; i++) {
2702 try {
2703 regions[i].close();
2704 } catch (IOException e) {
2705
2706 }
2707 }
2708 }
2709 } finally {
2710 HRegion.closeHRegion(this.region);
2711 this.region = null;
2712 }
2713 }
2714
2715 @Test
2716 public void testSplitRegion() throws IOException {
2717 byte[] qualifier = Bytes.toBytes("qualifier");
2718 Configuration hc = initSplit();
2719 int numRows = 10;
2720 byte[][] families = { fam1, fam3 };
2721
2722
2723 String method = this.getName();
2724 this.region = initHRegion(tableName, method, hc, families);
2725
2726
2727 int startRow = 100;
2728 putData(startRow, numRows, qualifier, families);
2729 int splitRow = startRow + numRows;
2730 putData(splitRow, numRows, qualifier, families);
2731 region.flushcache();
2732
2733 HRegion[] regions = null;
2734 try {
2735 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
2736
2737 for (int i = 0; i < regions.length; i++) {
2738 regions[i] = HRegion.openHRegion(regions[i], null);
2739 }
2740
2741 assertEquals(2, regions.length);
2742
2743
2744
2745 verifyData(regions[0], startRow, numRows, qualifier, families);
2746 verifyData(regions[1], splitRow, numRows, qualifier, families);
2747
2748 } finally {
2749 HRegion.closeHRegion(this.region);
2750 this.region = null;
2751 }
2752 }
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764 @Test
2765 public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
2766 byte[] family = Bytes.toBytes("family");
2767 int numRows = 1000;
2768 int flushAndScanInterval = 10;
2769 int compactInterval = 10 * flushAndScanInterval;
2770
2771 String method = "testFlushCacheWhileScanning";
2772 this.region = initHRegion(tableName, method, conf, family);
2773 try {
2774 FlushThread flushThread = new FlushThread();
2775 flushThread.start();
2776
2777 Scan scan = new Scan();
2778 scan.addFamily(family);
2779 scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
2780 new BinaryComparator(Bytes.toBytes(5L))));
2781
2782 int expectedCount = 0;
2783 List<Cell> res = new ArrayList<Cell>();
2784
2785 boolean toggle = true;
2786 for (long i = 0; i < numRows; i++) {
2787 Put put = new Put(Bytes.toBytes(i));
2788 put.setDurability(Durability.SKIP_WAL);
2789 put.add(family, qual1, Bytes.toBytes(i % 10));
2790 region.put(put);
2791
2792 if (i != 0 && i % compactInterval == 0) {
2793
2794 region.compactStores(true);
2795 }
2796
2797 if (i % 10 == 5L) {
2798 expectedCount++;
2799 }
2800
2801 if (i != 0 && i % flushAndScanInterval == 0) {
2802 res.clear();
2803 InternalScanner scanner = region.getScanner(scan);
2804 if (toggle) {
2805 flushThread.flush();
2806 }
2807 while (scanner.next(res))
2808 ;
2809 if (!toggle) {
2810 flushThread.flush();
2811 }
2812 assertEquals("i=" + i, expectedCount, res.size());
2813 toggle = !toggle;
2814 }
2815 }
2816
2817 flushThread.done();
2818 flushThread.join();
2819 flushThread.checkNoError();
2820 } finally {
2821 HRegion.closeHRegion(this.region);
2822 this.region = null;
2823 }
2824 }
2825
2826 protected class FlushThread extends Thread {
2827 private volatile boolean done;
2828 private Throwable error = null;
2829
2830 public void done() {
2831 done = true;
2832 synchronized (this) {
2833 interrupt();
2834 }
2835 }
2836
2837 public void checkNoError() {
2838 if (error != null) {
2839 assertNull(error);
2840 }
2841 }
2842
2843 @Override
2844 public void run() {
2845 done = false;
2846 while (!done) {
2847 synchronized (this) {
2848 try {
2849 wait();
2850 } catch (InterruptedException ignored) {
2851 if (done) {
2852 break;
2853 }
2854 }
2855 }
2856 try {
2857 region.flushcache();
2858 } catch (IOException e) {
2859 if (!done) {
2860 LOG.error("Error while flusing cache", e);
2861 error = e;
2862 }
2863 break;
2864 }
2865 }
2866
2867 }
2868
2869 public void flush() {
2870 synchronized (this) {
2871 notify();
2872 }
2873
2874 }
2875 }
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886 @Test
2887 public void testWritesWhileScanning() throws IOException, InterruptedException {
2888 int testCount = 100;
2889 int numRows = 1;
2890 int numFamilies = 10;
2891 int numQualifiers = 100;
2892 int flushInterval = 7;
2893 int compactInterval = 5 * flushInterval;
2894 byte[][] families = new byte[numFamilies][];
2895 for (int i = 0; i < numFamilies; i++) {
2896 families[i] = Bytes.toBytes("family" + i);
2897 }
2898 byte[][] qualifiers = new byte[numQualifiers][];
2899 for (int i = 0; i < numQualifiers; i++) {
2900 qualifiers[i] = Bytes.toBytes("qual" + i);
2901 }
2902
2903 String method = "testWritesWhileScanning";
2904 this.region = initHRegion(tableName, method, conf, families);
2905 try {
2906 PutThread putThread = new PutThread(numRows, families, qualifiers);
2907 putThread.start();
2908 putThread.waitForFirstPut();
2909
2910 FlushThread flushThread = new FlushThread();
2911 flushThread.start();
2912
2913 Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
2914
2915 int expectedCount = numFamilies * numQualifiers;
2916 List<Cell> res = new ArrayList<Cell>();
2917
2918 long prevTimestamp = 0L;
2919 for (int i = 0; i < testCount; i++) {
2920
2921 if (i != 0 && i % compactInterval == 0) {
2922 region.compactStores(true);
2923 }
2924
2925 if (i != 0 && i % flushInterval == 0) {
2926 flushThread.flush();
2927 }
2928
2929 boolean previousEmpty = res.isEmpty();
2930 res.clear();
2931 InternalScanner scanner = region.getScanner(scan);
2932 while (scanner.next(res))
2933 ;
2934 if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
2935 assertEquals("i=" + i, expectedCount, res.size());
2936 long timestamp = res.get(0).getTimestamp();
2937 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
2938 timestamp >= prevTimestamp);
2939 prevTimestamp = timestamp;
2940 }
2941 }
2942
2943 putThread.done();
2944
2945 region.flushcache();
2946
2947 putThread.join();
2948 putThread.checkNoError();
2949
2950 flushThread.done();
2951 flushThread.join();
2952 flushThread.checkNoError();
2953 } finally {
2954 HRegion.closeHRegion(this.region);
2955 this.region = null;
2956 }
2957 }
2958
2959 protected class PutThread extends Thread {
2960 private volatile boolean done;
2961 private volatile int numPutsFinished = 0;
2962
2963 private Throwable error = null;
2964 private int numRows;
2965 private byte[][] families;
2966 private byte[][] qualifiers;
2967
2968 private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
2969 this.numRows = numRows;
2970 this.families = families;
2971 this.qualifiers = qualifiers;
2972 }
2973
2974
2975
2976
2977 public void waitForFirstPut() throws InterruptedException {
2978
2979 while (numPutsFinished == 0) {
2980 checkNoError();
2981 Thread.sleep(50);
2982 }
2983 }
2984
2985 public void done() {
2986 done = true;
2987 synchronized (this) {
2988 interrupt();
2989 }
2990 }
2991
2992 public void checkNoError() {
2993 if (error != null) {
2994 assertNull(error);
2995 }
2996 }
2997
2998 @Override
2999 public void run() {
3000 done = false;
3001 while (!done) {
3002 try {
3003 for (int r = 0; r < numRows; r++) {
3004 byte[] row = Bytes.toBytes("row" + r);
3005 Put put = new Put(row);
3006 put.setDurability(Durability.SKIP_WAL);
3007 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3008 for (byte[] family : families) {
3009 for (byte[] qualifier : qualifiers) {
3010 put.add(family, qualifier, (long) numPutsFinished, value);
3011 }
3012 }
3013 region.put(put);
3014 numPutsFinished++;
3015 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3016 System.out.println("put iteration = " + numPutsFinished);
3017 Delete delete = new Delete(row, (long) numPutsFinished - 30);
3018 region.delete(delete);
3019 }
3020 numPutsFinished++;
3021 }
3022 } catch (InterruptedIOException e) {
3023
3024 } catch (IOException e) {
3025 LOG.error("error while putting records", e);
3026 error = e;
3027 break;
3028 }
3029 }
3030
3031 }
3032
3033 }
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044 @Test
3045 public void testWritesWhileGetting() throws Exception {
3046 int testCount = 100;
3047 int numRows = 1;
3048 int numFamilies = 10;
3049 int numQualifiers = 100;
3050 int compactInterval = 100;
3051 byte[][] families = new byte[numFamilies][];
3052 for (int i = 0; i < numFamilies; i++) {
3053 families[i] = Bytes.toBytes("family" + i);
3054 }
3055 byte[][] qualifiers = new byte[numQualifiers][];
3056 for (int i = 0; i < numQualifiers; i++) {
3057 qualifiers[i] = Bytes.toBytes("qual" + i);
3058 }
3059
3060
3061 String method = "testWritesWhileGetting";
3062
3063
3064
3065
3066
3067 conf.setInt("hbase.hstore.compaction.min", 1);
3068 conf.setInt("hbase.hstore.compaction.max", 1000);
3069 this.region = initHRegion(tableName, method, conf, families);
3070 PutThread putThread = null;
3071 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3072 try {
3073 putThread = new PutThread(numRows, families, qualifiers);
3074 putThread.start();
3075 putThread.waitForFirstPut();
3076
3077
3078 ctx.addThread(new RepeatingTestThread(ctx) {
3079 private int flushesSinceCompact = 0;
3080 private final int maxFlushesSinceCompact = 20;
3081
3082 @Override
3083 public void doAnAction() throws Exception {
3084 if (region.flushcache()) {
3085 ++flushesSinceCompact;
3086 }
3087
3088
3089 if (flushesSinceCompact == maxFlushesSinceCompact) {
3090 region.compactStores(false);
3091 flushesSinceCompact = 0;
3092 }
3093 }
3094 });
3095 ctx.startThreads();
3096
3097 Get get = new Get(Bytes.toBytes("row0"));
3098 Result result = null;
3099
3100 int expectedCount = numFamilies * numQualifiers;
3101
3102 long prevTimestamp = 0L;
3103 for (int i = 0; i < testCount; i++) {
3104
3105 boolean previousEmpty = result == null || result.isEmpty();
3106 result = region.get(get);
3107 if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3108 assertEquals("i=" + i, expectedCount, result.size());
3109
3110
3111 long timestamp = 0;
3112 for (Cell kv : result.rawCells()) {
3113 if (CellUtil.matchingFamily(kv, families[0])
3114 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3115 timestamp = kv.getTimestamp();
3116 }
3117 }
3118 assertTrue(timestamp >= prevTimestamp);
3119 prevTimestamp = timestamp;
3120 Cell previousKV = null;
3121
3122 for (Cell kv : result.rawCells()) {
3123 byte[] thisValue = CellUtil.cloneValue(kv);
3124 if (previousKV != null) {
3125 if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3126 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3127 + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3128 + "(memStoreTS:" + kv.getMvccVersion() + ")");
3129 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3130 }
3131 }
3132 previousKV = kv;
3133 }
3134 }
3135 }
3136 } finally {
3137 if (putThread != null)
3138 putThread.done();
3139
3140 region.flushcache();
3141
3142 if (putThread != null) {
3143 putThread.join();
3144 putThread.checkNoError();
3145 }
3146
3147 ctx.stop();
3148 HRegion.closeHRegion(this.region);
3149 this.region = null;
3150 }
3151 }
3152
3153 @Test
3154 public void testHolesInMeta() throws Exception {
3155 byte[] family = Bytes.toBytes("family");
3156 this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, conf,
3157 false, family);
3158 try {
3159 byte[] rowNotServed = Bytes.toBytes("a");
3160 Get g = new Get(rowNotServed);
3161 try {
3162 region.get(g);
3163 fail();
3164 } catch (WrongRegionException x) {
3165
3166 }
3167 byte[] row = Bytes.toBytes("y");
3168 g = new Get(row);
3169 region.get(g);
3170 } finally {
3171 HRegion.closeHRegion(this.region);
3172 this.region = null;
3173 }
3174 }
3175
3176 @Test
3177 public void testIndexesScanWithOneDeletedRow() throws IOException {
3178 byte[] family = Bytes.toBytes("family");
3179
3180
3181 String method = "testIndexesScanWithOneDeletedRow";
3182 this.region = initHRegion(tableName, method, conf, family);
3183 try {
3184 Put put = new Put(Bytes.toBytes(1L));
3185 put.add(family, qual1, 1L, Bytes.toBytes(1L));
3186 region.put(put);
3187
3188 region.flushcache();
3189
3190 Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3191 region.delete(delete);
3192
3193 put = new Put(Bytes.toBytes(2L));
3194 put.add(family, qual1, 2L, Bytes.toBytes(2L));
3195 region.put(put);
3196
3197 Scan idxScan = new Scan();
3198 idxScan.addFamily(family);
3199 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
3200 new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
3201 new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
3202 CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
3203 InternalScanner scanner = region.getScanner(idxScan);
3204 List<Cell> res = new ArrayList<Cell>();
3205
3206 while (scanner.next(res))
3207 ;
3208 assertEquals(1L, res.size());
3209 } finally {
3210 HRegion.closeHRegion(this.region);
3211 this.region = null;
3212 }
3213 }
3214
3215
3216
3217
3218 @Test
3219 public void testBloomFilterSize() throws IOException {
3220 byte[] row1 = Bytes.toBytes("row1");
3221 byte[] fam1 = Bytes.toBytes("fam1");
3222 byte[] qf1 = Bytes.toBytes("col");
3223 byte[] val1 = Bytes.toBytes("value1");
3224
3225 HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
3226 .setBloomFilterType(BloomType.ROWCOL);
3227
3228 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3229 htd.addFamily(hcd);
3230 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3231 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3232 try {
3233 int num_unique_rows = 10;
3234 int duplicate_multiplier = 2;
3235 int num_storefiles = 4;
3236
3237 int version = 0;
3238 for (int f = 0; f < num_storefiles; f++) {
3239 for (int i = 0; i < duplicate_multiplier; i++) {
3240 for (int j = 0; j < num_unique_rows; j++) {
3241 Put put = new Put(Bytes.toBytes("row" + j));
3242 put.setDurability(Durability.SKIP_WAL);
3243 put.add(fam1, qf1, version++, val1);
3244 region.put(put);
3245 }
3246 }
3247 region.flushcache();
3248 }
3249
3250 HStore store = (HStore) region.getStore(fam1);
3251 Collection<StoreFile> storeFiles = store.getStorefiles();
3252 for (StoreFile storefile : storeFiles) {
3253 StoreFile.Reader reader = storefile.getReader();
3254 reader.loadFileInfo();
3255 reader.loadBloomfilter();
3256 assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
3257 assertEquals(num_unique_rows, reader.getFilterEntries());
3258 }
3259
3260 region.compactStores(true);
3261
3262
3263 storeFiles = store.getStorefiles();
3264 for (StoreFile storefile : storeFiles) {
3265 StoreFile.Reader reader = storefile.getReader();
3266 reader.loadFileInfo();
3267 reader.loadBloomfilter();
3268 assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
3269 assertEquals(num_unique_rows, reader.getFilterEntries());
3270 }
3271 } finally {
3272 HRegion.closeHRegion(this.region);
3273 this.region = null;
3274 }
3275 }
3276
3277 @Test
3278 public void testAllColumnsWithBloomFilter() throws IOException {
3279 byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
3280 byte[] FAMILY = Bytes.toBytes("family");
3281
3282
3283 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
3284 .setBloomFilterType(BloomType.ROWCOL);
3285 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
3286 htd.addFamily(hcd);
3287 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3288 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3289 try {
3290
3291 byte row[] = Bytes.toBytes("row:" + 0);
3292 byte column[] = Bytes.toBytes("column:" + 0);
3293 Put put = new Put(row);
3294 put.setDurability(Durability.SKIP_WAL);
3295 for (long idx = 1; idx <= 4; idx++) {
3296 put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
3297 }
3298 region.put(put);
3299
3300
3301 region.flushcache();
3302
3303
3304 Get get = new Get(row);
3305 get.setMaxVersions();
3306 Cell[] kvs = region.get(get).rawCells();
3307
3308
3309 assertEquals(4, kvs.length);
3310 checkOneCell(kvs[0], FAMILY, 0, 0, 4);
3311 checkOneCell(kvs[1], FAMILY, 0, 0, 3);
3312 checkOneCell(kvs[2], FAMILY, 0, 0, 2);
3313 checkOneCell(kvs[3], FAMILY, 0, 0, 1);
3314 } finally {
3315 HRegion.closeHRegion(this.region);
3316 this.region = null;
3317 }
3318 }
3319
3320
3321
3322
3323
3324
3325 @Test
3326 public void testDeleteRowWithBloomFilter() throws IOException {
3327 byte[] familyName = Bytes.toBytes("familyName");
3328
3329
3330 HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
3331 .setBloomFilterType(BloomType.ROWCOL);
3332
3333 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3334 htd.addFamily(hcd);
3335 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3336 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3337 try {
3338
3339 byte row[] = Bytes.toBytes("row1");
3340 byte col[] = Bytes.toBytes("col1");
3341
3342 Put put = new Put(row);
3343 put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
3344 region.put(put);
3345 region.flushcache();
3346
3347 Delete del = new Delete(row);
3348 region.delete(del);
3349 region.flushcache();
3350
3351
3352 Get get = new Get(row);
3353 get.addColumn(familyName, col);
3354
3355 Cell[] keyValues = region.get(get).rawCells();
3356 assertTrue(keyValues.length == 0);
3357 } finally {
3358 HRegion.closeHRegion(this.region);
3359 this.region = null;
3360 }
3361 }
3362
3363 @Test
3364 public void testgetHDFSBlocksDistribution() throws Exception {
3365 HBaseTestingUtility htu = new HBaseTestingUtility();
3366 final int DEFAULT_BLOCK_SIZE = 1024;
3367 htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
3368 htu.getConfiguration().setInt("dfs.replication", 2);
3369
3370
3371 MiniHBaseCluster cluster = null;
3372 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
3373 int regionServersCount = 3;
3374
3375 try {
3376 cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
3377 byte[][] families = { fam1, fam2 };
3378 HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);
3379
3380
3381 byte row[] = Bytes.toBytes("row1");
3382 byte col[] = Bytes.toBytes("col1");
3383
3384 Put put = new Put(row);
3385 put.add(fam1, col, 1, Bytes.toBytes("test1"));
3386 put.add(fam2, col, 1, Bytes.toBytes("test2"));
3387 ht.put(put);
3388
3389 HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
3390 .get(0);
3391 firstRegion.flushcache();
3392 HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
3393
3394
3395
3396
3397
3398 long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
3399
3400 String topHost = blocksDistribution1.getTopHosts().get(0);
3401 long topHostWeight = blocksDistribution1.getWeight(topHost);
3402 assertTrue(uniqueBlocksWeight1 == topHostWeight);
3403
3404
3405
3406 HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
3407 htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
3408 long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
3409
3410 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
3411
3412 ht.close();
3413 } finally {
3414 if (cluster != null) {
3415 htu.shutdownMiniCluster();
3416 }
3417 }
3418 }
3419
3420
3421
3422
3423
3424
3425
3426 @Test
3427 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
3428 TableName tableName = TableName.valueOf(name.getMethodName());
3429 HRegionInfo info = null;
3430 try {
3431 FileSystem fs = Mockito.mock(FileSystem.class);
3432 Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
3433 HTableDescriptor htd = new HTableDescriptor(tableName);
3434 htd.addFamily(new HColumnDescriptor("cf"));
3435 info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
3436 HConstants.EMPTY_BYTE_ARRAY, false);
3437 Path path = new Path(DIR + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
3438 region = HRegion.newHRegion(path, null, fs, conf, info, htd, null);
3439
3440 region.initialize();
3441 fail("Region initialization should fail due to IOException");
3442 } catch (IOException io) {
3443 List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
3444 for (MonitoredTask monitoredTask : tasks) {
3445 if (!(monitoredTask instanceof MonitoredRPCHandler)
3446 && monitoredTask.getDescription().contains(region.toString())) {
3447 assertTrue("Region state should be ABORTED.",
3448 monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
3449 break;
3450 }
3451 }
3452 } finally {
3453 HRegion.closeHRegion(region);
3454 }
3455 }
3456
3457
3458
3459
3460
3461 @Test
3462 public void testRegionInfoFileCreation() throws IOException {
3463 Path rootDir = new Path(DIR + "testRegionInfoFileCreation");
3464
3465 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
3466 htd.addFamily(new HColumnDescriptor("cf"));
3467
3468 HRegionInfo hri = new HRegionInfo(htd.getTableName());
3469
3470
3471 HRegion region = HRegion.createHRegion(hri, rootDir, conf, htd, null, false, true);
3472
3473 Path regionDir = region.getRegionFileSystem().getRegionDir();
3474 FileSystem fs = region.getRegionFileSystem().getFileSystem();
3475 HRegion.closeHRegion(region);
3476
3477 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
3478
3479
3480 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3481 fs.exists(regionInfoFile));
3482
3483
3484 region = HRegion.openHRegion(rootDir, hri, htd, null, conf);
3485 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3486 HRegion.closeHRegion(region);
3487
3488
3489 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3490 fs.exists(regionInfoFile));
3491
3492
3493 fs.delete(regionInfoFile);
3494 assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
3495 fs.exists(regionInfoFile));
3496
3497 region = HRegion.openHRegion(rootDir, hri, htd, null, conf);
3498
3499 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3500 HRegion.closeHRegion(region);
3501
3502
3503 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3504 fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
3505 }
3506
3507
3508
3509
3510 private static class Incrementer implements Runnable {
3511 private HRegion region;
3512 private final static byte[] incRow = Bytes.toBytes("incRow");
3513 private final static byte[] family = Bytes.toBytes("family");
3514 private final static byte[] qualifier = Bytes.toBytes("qualifier");
3515 private final static long ONE = 1l;
3516 private int incCounter;
3517
3518 public Incrementer(HRegion region, int incCounter) {
3519 this.region = region;
3520 this.incCounter = incCounter;
3521 }
3522
3523 @Override
3524 public void run() {
3525 int count = 0;
3526 while (count < incCounter) {
3527 Increment inc = new Increment(incRow);
3528 inc.addColumn(family, qualifier, ONE);
3529 count++;
3530 try {
3531 region.increment(inc);
3532 } catch (IOException e) {
3533 e.printStackTrace();
3534 break;
3535 }
3536 }
3537 }
3538 }
3539
3540
3541
3542
3543
3544 @Test
3545 public void testParallelIncrementWithMemStoreFlush() throws Exception {
3546 byte[] family = Incrementer.family;
3547 this.region = initHRegion(tableName, method, conf, family);
3548 final HRegion region = this.region;
3549 final AtomicBoolean incrementDone = new AtomicBoolean(false);
3550 Runnable flusher = new Runnable() {
3551 @Override
3552 public void run() {
3553 while (!incrementDone.get()) {
3554 try {
3555 region.flushcache();
3556 } catch (Exception e) {
3557 e.printStackTrace();
3558 }
3559 }
3560 }
3561 };
3562
3563
3564 int threadNum = 20;
3565 int incCounter = 100;
3566 long expected = threadNum * incCounter;
3567 Thread[] incrementers = new Thread[threadNum];
3568 Thread flushThread = new Thread(flusher);
3569 for (int i = 0; i < threadNum; i++) {
3570 incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
3571 incrementers[i].start();
3572 }
3573 flushThread.start();
3574 for (int i = 0; i < threadNum; i++) {
3575 incrementers[i].join();
3576 }
3577
3578 incrementDone.set(true);
3579 flushThread.join();
3580
3581 Get get = new Get(Incrementer.incRow);
3582 get.addColumn(Incrementer.family, Incrementer.qualifier);
3583 get.setMaxVersions(1);
3584 Result res = this.region.get(get);
3585 List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
3586
3587
3588 assertEquals(kvs.size(), 1);
3589 Cell kv = kvs.get(0);
3590 assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
3591 this.region = null;
3592 }
3593
3594
3595
3596
3597 private static class Appender implements Runnable {
3598 private HRegion region;
3599 private final static byte[] appendRow = Bytes.toBytes("appendRow");
3600 private final static byte[] family = Bytes.toBytes("family");
3601 private final static byte[] qualifier = Bytes.toBytes("qualifier");
3602 private final static byte[] CHAR = Bytes.toBytes("a");
3603 private int appendCounter;
3604
3605 public Appender(HRegion region, int appendCounter) {
3606 this.region = region;
3607 this.appendCounter = appendCounter;
3608 }
3609
3610 @Override
3611 public void run() {
3612 int count = 0;
3613 while (count < appendCounter) {
3614 Append app = new Append(appendRow);
3615 app.add(family, qualifier, CHAR);
3616 count++;
3617 try {
3618 region.append(app);
3619 } catch (IOException e) {
3620 e.printStackTrace();
3621 break;
3622 }
3623 }
3624 }
3625 }
3626
3627
3628
3629
3630
3631 @Test
3632 public void testParallelAppendWithMemStoreFlush() throws Exception {
3633 byte[] family = Appender.family;
3634 this.region = initHRegion(tableName, method, conf, family);
3635 final HRegion region = this.region;
3636 final AtomicBoolean appendDone = new AtomicBoolean(false);
3637 Runnable flusher = new Runnable() {
3638 @Override
3639 public void run() {
3640 while (!appendDone.get()) {
3641 try {
3642 region.flushcache();
3643 } catch (Exception e) {
3644 e.printStackTrace();
3645 }
3646 }
3647 }
3648 };
3649
3650
3651
3652 int threadNum = 20;
3653 int appendCounter = 100;
3654 byte[] expected = new byte[threadNum * appendCounter];
3655 for (int i = 0; i < threadNum * appendCounter; i++) {
3656 System.arraycopy(Appender.CHAR, 0, expected, i, 1);
3657 }
3658 Thread[] appenders = new Thread[threadNum];
3659 Thread flushThread = new Thread(flusher);
3660 for (int i = 0; i < threadNum; i++) {
3661 appenders[i] = new Thread(new Appender(this.region, appendCounter));
3662 appenders[i].start();
3663 }
3664 flushThread.start();
3665 for (int i = 0; i < threadNum; i++) {
3666 appenders[i].join();
3667 }
3668
3669 appendDone.set(true);
3670 flushThread.join();
3671
3672 Get get = new Get(Appender.appendRow);
3673 get.addColumn(Appender.family, Appender.qualifier);
3674 get.setMaxVersions(1);
3675 Result res = this.region.get(get);
3676 List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
3677
3678
3679 assertEquals(kvs.size(), 1);
3680 Cell kv = kvs.get(0);
3681 byte[] appendResult = new byte[kv.getValueLength()];
3682 System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
3683 assertArrayEquals(expected, appendResult);
3684 this.region = null;
3685 }
3686
3687
3688
3689
3690
3691 @Test
3692 public void testPutWithMemStoreFlush() throws Exception {
3693 byte[] family = Bytes.toBytes("family");
3694 ;
3695 byte[] qualifier = Bytes.toBytes("qualifier");
3696 byte[] row = Bytes.toBytes("putRow");
3697 byte[] value = null;
3698 this.region = initHRegion(tableName, method, conf, family);
3699 Put put = null;
3700 Get get = null;
3701 List<Cell> kvs = null;
3702 Result res = null;
3703
3704 put = new Put(row);
3705 value = Bytes.toBytes("value0");
3706 put.add(family, qualifier, 1234567l, value);
3707 region.put(put);
3708 get = new Get(row);
3709 get.addColumn(family, qualifier);
3710 get.setMaxVersions();
3711 res = this.region.get(get);
3712 kvs = res.getColumnCells(family, qualifier);
3713 assertEquals(1, kvs.size());
3714 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
3715
3716 region.flushcache();
3717 get = new Get(row);
3718 get.addColumn(family, qualifier);
3719 get.setMaxVersions();
3720 res = this.region.get(get);
3721 kvs = res.getColumnCells(family, qualifier);
3722 assertEquals(1, kvs.size());
3723 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
3724
3725 put = new Put(row);
3726 value = Bytes.toBytes("value1");
3727 put.add(family, qualifier, 1234567l, value);
3728 region.put(put);
3729 get = new Get(row);
3730 get.addColumn(family, qualifier);
3731 get.setMaxVersions();
3732 res = this.region.get(get);
3733 kvs = res.getColumnCells(family, qualifier);
3734 assertEquals(1, kvs.size());
3735 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
3736
3737 region.flushcache();
3738 get = new Get(row);
3739 get.addColumn(family, qualifier);
3740 get.setMaxVersions();
3741 res = this.region.get(get);
3742 kvs = res.getColumnCells(family, qualifier);
3743 assertEquals(1, kvs.size());
3744 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
3745 }
3746
3747 @Test
3748 public void testDurability() throws Exception {
3749 String method = "testDurability";
3750
3751
3752
3753
3754
3755 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
3756 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
3757 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
3758
3759 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
3760 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
3761 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
3762
3763 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
3764 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
3765
3766 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
3767 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
3768
3769 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
3770 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
3771 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
3772
3773
3774
3775 conf.setLong("hbase.regionserver.optionallogflushinterval", Integer.MAX_VALUE);
3776 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
3777 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
3778 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
3779 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
3780 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
3781 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
3782
3783
3784 conf.setLong("hbase.regionserver.optionallogflushinterval", 5);
3785 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
3786 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
3787 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
3788 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
3789 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
3790 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
3791
3792
3793 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
3794 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
3795 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
3796 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
3797 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
3798 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
3799
3800 }
3801
3802 private void durabilityTest(String method, Durability tableDurability,
3803 Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
3804 final boolean expectSyncFromLogSyncer) throws Exception {
3805 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
3806 TableName tableName = TableName.valueOf(method);
3807 byte[] family = Bytes.toBytes("family");
3808 Path logDir = new Path(new Path(DIR + method), "log");
3809 HLog hlog = HLogFactory.createHLog(fs, logDir, UUID.randomUUID().toString(), conf);
3810 final HLog log = spy(hlog);
3811 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
3812 HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, log,
3813 new byte[][] { family });
3814
3815 Put put = new Put(Bytes.toBytes("r1"));
3816 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
3817 put.setDurability(mutationDurability);
3818 region.put(put);
3819
3820
3821 verify(log, expectAppend ? times(1) : never()).appendNoSync((HRegionInfo) any(), eq(tableName),
3822 (WALEdit) any(), (List<UUID>) any(), anyLong(), (HTableDescriptor) any());
3823
3824
3825 if (expectSync || expectSyncFromLogSyncer) {
3826 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
3827 @Override
3828 public boolean evaluate() throws Exception {
3829 try {
3830 if (expectSync) {
3831 verify(log, times(1)).sync(anyLong());
3832 } else if (expectSyncFromLogSyncer) {
3833 verify(log, times(1)).sync();
3834 }
3835 } catch (Throwable ignore) {
3836 }
3837 return true;
3838 }
3839 });
3840 } else {
3841 verify(log, never()).sync(anyLong());
3842 verify(log, never()).sync();
3843 }
3844
3845 hlog.close();
3846 region.close();
3847 }
3848
3849 private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
3850 for (int i = startRow; i < startRow + numRows; i++) {
3851 Put put = new Put(Bytes.toBytes("" + i));
3852 put.setDurability(Durability.SKIP_WAL);
3853 for (byte[] family : families) {
3854 put.add(family, qf, null);
3855 }
3856 region.put(put);
3857 }
3858 }
3859
3860 private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
3861 throws IOException {
3862 for (int i = startRow; i < startRow + numRows; i++) {
3863 byte[] row = Bytes.toBytes("" + i);
3864 Get get = new Get(row);
3865 for (byte[] family : families) {
3866 get.addColumn(family, qf);
3867 }
3868 Result result = newReg.get(get);
3869 Cell[] raw = result.rawCells();
3870 assertEquals(families.length, result.size());
3871 for (int j = 0; j < families.length; j++) {
3872 assertTrue(CellUtil.matchingRow(raw[j], row));
3873 assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
3874 assertTrue(CellUtil.matchingQualifier(raw[j], qf));
3875 }
3876 }
3877 }
3878
3879 private void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
3880
3881 Get get = new Get(k).addFamily(family).setMaxVersions();
3882 Cell[] results = r.get(get).rawCells();
3883 for (int j = 0; j < results.length; j++) {
3884 byte[] tmp = CellUtil.cloneValue(results[j]);
3885
3886 assertTrue(Bytes.equals(k, tmp));
3887 }
3888 }
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901 private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
3902 throws IOException {
3903 byte[][] families = { fs };
3904 Scan scan = new Scan();
3905 for (int i = 0; i < families.length; i++)
3906 scan.addFamily(families[i]);
3907 InternalScanner s = r.getScanner(scan);
3908 try {
3909 List<Cell> curVals = new ArrayList<Cell>();
3910 boolean first = true;
3911 OUTER_LOOP: while (s.next(curVals)) {
3912 for (Cell kv : curVals) {
3913 byte[] val = CellUtil.cloneValue(kv);
3914 byte[] curval = val;
3915 if (first) {
3916 first = false;
3917 assertTrue(Bytes.compareTo(curval, firstValue) == 0);
3918 } else {
3919
3920 break OUTER_LOOP;
3921 }
3922 }
3923 }
3924 } finally {
3925 s.close();
3926 }
3927 }
3928
3929 private Configuration initSplit() {
3930
3931 conf.setInt("hbase.hstore.compactionThreshold", 2);
3932
3933
3934 conf.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
3935
3936 conf.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
3937
3938
3939 conf.setLong("hbase.client.pause", 15 * 1000);
3940
3941
3942
3943 conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
3944 return conf;
3945 }
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956 public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
3957 byte[]... families) throws IOException {
3958 return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
3959 }
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
3971 byte[]... families) throws IOException {
3972 return initHRegion(tableName, null, null, callingMethod, conf, false, families);
3973 }
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
3986 boolean isReadOnly, byte[]... families) throws IOException {
3987 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
3988 }
3989
3990 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
3991 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
3992 throws IOException {
3993 return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
3994 Durability.SYNC_WAL, null, families);
3995 }
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4010 String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4011 HLog hlog, byte[]... families) throws IOException {
4012 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly, durability, hlog, families);
4013 }
4014
4015
4016
4017
4018
4019 private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
4020 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4021 assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
4022 Bytes.toString(CellUtil.cloneRow(kv)));
4023 assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
4024 Bytes.toString(CellUtil.cloneFamily(kv)));
4025 assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
4026 Bytes.toString(CellUtil.cloneQualifier(kv)));
4027 assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
4028 assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
4029 Bytes.toString(CellUtil.cloneValue(kv)));
4030 }
4031 }