1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.mockito.Matchers.any;
22 import static org.mockito.Matchers.anyLong;
23 import static org.mockito.Matchers.eq;
24 import static org.mockito.Mockito.never;
25 import static org.mockito.Mockito.spy;
26 import static org.mockito.Mockito.times;
27 import static org.mockito.Mockito.verify;
28
29 import java.io.IOException;
30 import java.io.InterruptedIOException;
31 import java.util.ArrayList;
32 import java.util.Arrays;
33 import java.util.Collection;
34 import java.util.List;
35 import java.util.Map;
36 import java.util.NavigableMap;
37 import java.util.TreeMap;
38 import java.util.UUID;
39 import java.util.concurrent.atomic.AtomicBoolean;
40 import java.util.concurrent.atomic.AtomicInteger;
41 import java.util.concurrent.atomic.AtomicReference;
42
43 import org.apache.commons.logging.Log;
44 import org.apache.commons.logging.LogFactory;
45 import org.apache.hadoop.conf.Configuration;
46 import org.apache.hadoop.fs.FSDataOutputStream;
47 import org.apache.hadoop.fs.FileStatus;
48 import org.apache.hadoop.fs.FileSystem;
49 import org.apache.hadoop.fs.Path;
50 import org.apache.hadoop.hbase.Cell;
51 import org.apache.hadoop.hbase.CellComparator;
52 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
53 import org.apache.hadoop.hbase.TableName;
54 import org.apache.hadoop.hbase.HBaseConfiguration;
55 import org.apache.hadoop.hbase.HBaseTestCase;
56 import org.apache.hadoop.hbase.HBaseTestingUtility;
57 import org.apache.hadoop.hbase.HColumnDescriptor;
58 import org.apache.hadoop.hbase.HConstants;
59 import org.apache.hadoop.hbase.NotServingRegionException;
60 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
61 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
62 import org.apache.hadoop.hbase.HRegionInfo;
63 import org.apache.hadoop.hbase.HTableDescriptor;
64 import org.apache.hadoop.hbase.KeyValue;
65 import org.apache.hadoop.hbase.MediumTests;
66 import org.apache.hadoop.hbase.MiniHBaseCluster;
67 import org.apache.hadoop.hbase.MultithreadedTestUtil;
68 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
69 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
70 import org.apache.hadoop.hbase.Waiter;
71 import org.apache.hadoop.hbase.client.Append;
72 import org.apache.hadoop.hbase.client.Delete;
73 import org.apache.hadoop.hbase.client.Durability;
74 import org.apache.hadoop.hbase.client.Get;
75 import org.apache.hadoop.hbase.client.HTable;
76 import org.apache.hadoop.hbase.client.Increment;
77 import org.apache.hadoop.hbase.client.Put;
78 import org.apache.hadoop.hbase.client.Result;
79 import org.apache.hadoop.hbase.client.Scan;
80 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
81 import org.apache.hadoop.hbase.filter.BinaryComparator;
82 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
83 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
84 import org.apache.hadoop.hbase.filter.Filter;
85 import org.apache.hadoop.hbase.filter.FilterBase;
86 import org.apache.hadoop.hbase.filter.FilterList;
87 import org.apache.hadoop.hbase.filter.NullComparator;
88 import org.apache.hadoop.hbase.filter.PrefixFilter;
89 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
90 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
91 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
92 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
93 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
94 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
95 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
96 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
97 import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
98 import org.apache.hadoop.hbase.regionserver.wal.HLog;
99 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
100 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
101 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
102 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
103 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
104 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
105 import org.apache.hadoop.hbase.util.Bytes;
106 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
107 import org.apache.hadoop.hbase.util.FSUtils;
108 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
109 import org.apache.hadoop.hbase.util.PairOfSameType;
110 import org.apache.hadoop.hbase.util.Threads;
111 import org.junit.Assert;
112 import org.junit.Test;
113 import org.junit.experimental.categories.Category;
114 import org.mockito.Mockito;
115
116 import com.google.common.collect.Lists;
117
118
119
120
121
122
123
124 @Category(MediumTests.class)
125 @SuppressWarnings("deprecation")
126 public class TestHRegion extends HBaseTestCase {
127
128
129 static final Log LOG = LogFactory.getLog(TestHRegion.class);
130
131 private static final String COLUMN_FAMILY = "MyCF";
132
133 HRegion region = null;
134 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
135 private static final String DIR = TEST_UTIL.getDataTestDir("TestHRegion").toString();
136
137 private final int MAX_VERSIONS = 2;
138
139
140 protected final byte[] tableName = Bytes.toBytes("testtable");;
141 protected final byte[] qual1 = Bytes.toBytes("qual1");
142 protected final byte[] qual2 = Bytes.toBytes("qual2");
143 protected final byte[] qual3 = Bytes.toBytes("qual3");
144 protected final byte[] value1 = Bytes.toBytes("value1");
145 protected final byte[] value2 = Bytes.toBytes("value2");
146 protected final byte [] row = Bytes.toBytes("rowA");
147 protected final byte [] row2 = Bytes.toBytes("rowB");
148
149 protected final MetricsAssertHelper metricsAssertHelper =
150 CompatibilitySingletonFactory.getInstance(MetricsAssertHelper.class);
151
152
153
154
155 @Override
156 protected void setUp() throws Exception {
157 super.setUp();
158 }
159
160 @Override
161 protected void tearDown() throws Exception {
162 super.tearDown();
163 EnvironmentEdgeManagerTestHelper.reset();
164 }
165
166
167
168
169
170
171
172 public void testCompactionAffectedByScanners() throws Exception {
173 String method = "testCompactionAffectedByScanners";
174 byte[] tableName = Bytes.toBytes(method);
175 byte[] family = Bytes.toBytes("family");
176 this.region = initHRegion(tableName, method, conf, family);
177
178 Put put = new Put(Bytes.toBytes("r1"));
179 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
180 region.put(put);
181 region.flushcache();
182
183
184 Scan scan = new Scan();
185 scan.setMaxVersions(3);
186
187 RegionScanner scanner1 = region.getScanner(scan);
188
189 Delete delete = new Delete(Bytes.toBytes("r1"));
190 region.delete(delete);
191 region.flushcache();
192
193
194 RegionScanner scanner2 = region.getScanner(scan);
195
196 List<KeyValue> results = new ArrayList<KeyValue>();
197
198 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
199
200
201 region.compactStores(true);
202
203
204 RegionScanner scanner3 = region.getScanner(scan);
205
206
207 scanner1.next(results);
208 System.out.println(results);
209 assertEquals(1, results.size());
210
211 results.clear();
212 scanner2.next(results);
213 System.out.println(results);
214 assertEquals(0, results.size());
215
216 results.clear();
217 scanner3.next(results);
218 System.out.println(results);
219 assertEquals(0, results.size());
220 }
221
222 @Test
223 public void testToShowNPEOnRegionScannerReseek() throws Exception{
224 String method = "testToShowNPEOnRegionScannerReseek";
225 byte[] tableName = Bytes.toBytes(method);
226 byte[] family = Bytes.toBytes("family");
227 this.region = initHRegion(tableName, method, conf, family);
228
229 Put put = new Put(Bytes.toBytes("r1"));
230 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
231 region.put(put);
232 put = new Put(Bytes.toBytes("r2"));
233 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
234 region.put(put);
235 region.flushcache();
236
237
238 Scan scan = new Scan();
239 scan.setMaxVersions(3);
240
241 RegionScanner scanner1 = region.getScanner(scan);
242
243 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
244
245 region.compactStores(true);
246
247 scanner1.reseek(Bytes.toBytes("r2"));
248 List<KeyValue> results = new ArrayList<KeyValue>();
249 scanner1.next(results);
250 KeyValue keyValue = results.get(0);
251 Assert.assertTrue(Bytes.compareTo(keyValue.getRow(), Bytes.toBytes("r2")) == 0);
252 scanner1.close();
253 }
254
255 public void testSkipRecoveredEditsReplay() throws Exception {
256 String method = "testSkipRecoveredEditsReplay";
257 TableName tableName =
258 TableName.valueOf(method);
259 byte[] family = Bytes.toBytes("family");
260 this.region = initHRegion(tableName, method, conf, family);
261 try {
262 Path regiondir = region.getRegionFileSystem().getRegionDir();
263 FileSystem fs = region.getRegionFileSystem().getFileSystem();
264 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
265
266 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
267
268 long maxSeqId = 1050;
269 long minSeqId = 1000;
270
271 for (long i = minSeqId; i <= maxSeqId; i += 10) {
272 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
273 fs.create(recoveredEdits);
274 HLog.Writer writer = HLogFactory.createWriter(fs, recoveredEdits, conf);
275
276 long time = System.nanoTime();
277 WALEdit edit = new WALEdit();
278 edit.add(new KeyValue(row, family, Bytes.toBytes(i),
279 time, KeyValue.Type.Put, Bytes.toBytes(i)));
280 writer.append(new HLog.Entry(new HLogKey(regionName, tableName,
281 i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
282
283 writer.close();
284 }
285 MonitoredTask status = TaskMonitor.get().createStatus(method);
286 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(
287 Bytes.BYTES_COMPARATOR);
288 for (Store store : region.getStores().values()) {
289 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
290 }
291 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
292 assertEquals(maxSeqId, seqId);
293 Get get = new Get(row);
294 Result result = region.get(get);
295 for (long i = minSeqId; i <= maxSeqId; i += 10) {
296 List<KeyValue> kvs = result.getColumn(family, Bytes.toBytes(i));
297 assertEquals(1, kvs.size());
298 assertEquals(Bytes.toBytes(i), kvs.get(0).getValue());
299 }
300 } finally {
301 HRegion.closeHRegion(this.region);
302 this.region = null;
303 }
304 }
305
306 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
307 String method = "testSkipRecoveredEditsReplaySomeIgnored";
308 TableName tableName =
309 TableName.valueOf(method);
310 byte[] family = Bytes.toBytes("family");
311 this.region = initHRegion(tableName, method, conf, family);
312 try {
313 Path regiondir = region.getRegionFileSystem().getRegionDir();
314 FileSystem fs = region.getRegionFileSystem().getFileSystem();
315 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
316
317 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
318
319 long maxSeqId = 1050;
320 long minSeqId = 1000;
321
322 for (long i = minSeqId; i <= maxSeqId; i += 10) {
323 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
324 fs.create(recoveredEdits);
325 HLog.Writer writer = HLogFactory.createWriter(fs, recoveredEdits, conf);
326
327 long time = System.nanoTime();
328 WALEdit edit = new WALEdit();
329 edit.add(new KeyValue(row, family, Bytes.toBytes(i),
330 time, KeyValue.Type.Put, Bytes.toBytes(i)));
331 writer.append(new HLog.Entry(new HLogKey(regionName, tableName,
332 i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
333
334 writer.close();
335 }
336 long recoverSeqId = 1030;
337 MonitoredTask status = TaskMonitor.get().createStatus(method);
338 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(
339 Bytes.BYTES_COMPARATOR);
340 for (Store store : region.getStores().values()) {
341 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(),
342 recoverSeqId - 1);
343 }
344 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
345 assertEquals(maxSeqId, seqId);
346 Get get = new Get(row);
347 Result result = region.get(get);
348 for (long i = minSeqId; i <= maxSeqId; i += 10) {
349 List<KeyValue> kvs = result.getColumn(family, Bytes.toBytes(i));
350 if (i < recoverSeqId) {
351 assertEquals(0, kvs.size());
352 } else {
353 assertEquals(1, kvs.size());
354 assertEquals(Bytes.toBytes(i), kvs.get(0).getValue());
355 }
356 }
357 } finally {
358 HRegion.closeHRegion(this.region);
359 this.region = null;
360 }
361 }
362
363 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
364 String method = "testSkipRecoveredEditsReplayAllIgnored";
365 byte[] tableName = Bytes.toBytes(method);
366 byte[] family = Bytes.toBytes("family");
367 this.region = initHRegion(tableName, method, conf, family);
368 try {
369 Path regiondir = region.getRegionFileSystem().getRegionDir();
370 FileSystem fs = region.getRegionFileSystem().getFileSystem();
371
372 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
373 for (int i = 1000; i < 1050; i += 10) {
374 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
375 FSDataOutputStream dos= fs.create(recoveredEdits);
376 dos.writeInt(i);
377 dos.close();
378 }
379 long minSeqId = 2000;
380 Path recoveredEdits = new Path(
381 recoveredEditsDir, String.format("%019d", minSeqId-1));
382 FSDataOutputStream dos= fs.create(recoveredEdits);
383 dos.close();
384
385 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(
386 Bytes.BYTES_COMPARATOR);
387 for (Store store : region.getStores().values()) {
388 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
389 }
390 long seqId = region.replayRecoveredEditsIfAny(regiondir,
391 maxSeqIdInStores, null, null);
392 assertEquals(minSeqId, seqId);
393 } finally {
394 HRegion.closeHRegion(this.region);
395 this.region = null;
396 }
397 }
398
399 @Test
400 public void testRecoveredEditsReplayCompaction() throws Exception {
401 String method = "testRecoveredEditsReplayCompaction";
402 TableName tableName =
403 TableName.valueOf(method);
404 byte[] family = Bytes.toBytes("family");
405 this.region = initHRegion(tableName, method, conf, family);
406 try {
407 Path regiondir = region.getRegionFileSystem().getRegionDir();
408 FileSystem fs = region.getRegionFileSystem().getFileSystem();
409 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
410
411 long maxSeqId = 3;
412 long minSeqId = 0;
413
414 for (long i = minSeqId; i < maxSeqId; i++) {
415 Put put = new Put(Bytes.toBytes(i));
416 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
417 region.put(put);
418 region.flushcache();
419 }
420
421
422 assertEquals(3, region.getStore(family).getStorefilesCount());
423 List<Path> storeFiles = new ArrayList<Path>(3);
424 for (StoreFile sf : region.getStore(family).getStorefiles()) {
425 storeFiles.add(sf.getPath());
426 }
427
428
429 conf.setBoolean("hbase.hstore.compaction.complete",false);
430 region.compactStores();
431
432
433 assertEquals(3, region.getStore(family).getStorefilesCount());
434
435
436 Path tmpDir = region.getRegionFileSystem().getTempDir();
437 FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
438 String errorMsg = "Expected to find 1 file in the region temp directory " +
439 "from the compaction, could not find any";
440 assertNotNull(errorMsg, files);
441 assertEquals(errorMsg, 1, files.length);
442
443 Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family), files[0].getPath());
444
445 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(
446 this.region.getRegionInfo(), family,
447 storeFiles, Lists.newArrayList(newFile),
448 region.getRegionFileSystem().getStoreDir(Bytes.toString(family)));
449
450 HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
451 this.region.getRegionInfo(), compactionDescriptor);
452
453 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
454
455 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
456 fs.create(recoveredEdits);
457 HLog.Writer writer = HLogFactory.createWriter(fs, recoveredEdits, conf);
458
459 long time = System.nanoTime();
460
461 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, 10, time, HConstants.DEFAULT_CLUSTER_ID),
462 WALEdit.createCompaction(compactionDescriptor)));
463 writer.close();
464
465
466 HTableDescriptor htd = region.getTableDesc();
467 HRegionInfo info = region.getRegionInfo();
468 region.close();
469 region = HRegion.openHRegion(conf, fs, new Path(DIR+method),info, htd, null);
470
471
472 Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
473 for (StoreFile sf : sfs) {
474 LOG.info(sf.getPath());
475 }
476 assertEquals(1, region.getStore(family).getStorefilesCount());
477 files = FSUtils.listStatus(fs, tmpDir);
478 assertTrue("Expected to find 0 files inside " + tmpDir,
479 files == null || files.length == 0);
480
481 for (long i = minSeqId; i < maxSeqId; i++) {
482 Get get = new Get(Bytes.toBytes(i));
483 Result result = region.get(get);
484 byte[] value = result.getValue(family, Bytes.toBytes(i));
485 assertEquals(Bytes.toBytes(i), value);
486 }
487 } finally {
488 HRegion.closeHRegion(this.region);
489 this.region = null;
490 }
491 }
492
493 public void testGetWhileRegionClose() throws IOException {
494 Configuration hc = initSplit();
495 int numRows = 100;
496 byte [][] families = {fam1, fam2, fam3};
497
498
499 String method = this.getName();
500 this.region = initHRegion(tableName, method, hc, families);
501 try {
502
503 final int startRow = 100;
504 putData(startRow, numRows, qual1, families);
505 putData(startRow, numRows, qual2, families);
506 putData(startRow, numRows, qual3, families);
507
508 final AtomicBoolean done = new AtomicBoolean(false);
509 final AtomicInteger gets = new AtomicInteger(0);
510 GetTillDoneOrException [] threads = new GetTillDoneOrException[10];
511 try {
512
513 for (int i = 0; i < threads.length / 2; i++) {
514 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow),
515 done, gets);
516 threads[i].setDaemon(true);
517 threads[i].start();
518 }
519
520
521 this.region.closing.set(true);
522 for (int i = threads.length / 2; i < threads.length; i++) {
523 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow),
524 done, gets);
525 threads[i].setDaemon(true);
526 threads[i].start();
527 }
528 } finally {
529 if (this.region != null) {
530 HRegion.closeHRegion(this.region);
531 }
532 }
533 done.set(true);
534 for (GetTillDoneOrException t: threads) {
535 try {
536 t.join();
537 } catch (InterruptedException e) {
538 e.printStackTrace();
539 }
540 if (t.e != null) {
541 LOG.info("Exception=" + t.e);
542 assertFalse("Found a NPE in " + t.getName(),
543 t.e instanceof NullPointerException);
544 }
545 }
546 } finally {
547 HRegion.closeHRegion(this.region);
548 this.region = null;
549 }
550 }
551
552
553
554
555
556 class GetTillDoneOrException extends Thread {
557 private final Get g;
558 private final AtomicBoolean done;
559 private final AtomicInteger count;
560 private Exception e;
561
562 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d,
563 final AtomicInteger c) {
564 super("getter." + i);
565 this.g = new Get(r);
566 this.done = d;
567 this.count = c;
568 }
569
570 @Override
571 public void run() {
572 while (!this.done.get()) {
573 try {
574 assertTrue(region.get(g).size() > 0);
575 this.count.incrementAndGet();
576 } catch (Exception e) {
577 this.e = e;
578 break;
579 }
580 }
581 }
582 }
583
584
585
586
587 public void testWeirdCacheBehaviour() throws Exception {
588 byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
589 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
590 Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
591 Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
592 this.region = initHRegion(TABLE, getName(), conf, FAMILIES);
593 try {
594 String value = "this is the value";
595 String value2 = "this is some other value";
596 String keyPrefix1 = "prefix1";
597 String keyPrefix2 = "prefix2";
598 String keyPrefix3 = "prefix3";
599 putRows(this.region, 3, value, keyPrefix1);
600 putRows(this.region, 3, value, keyPrefix2);
601 putRows(this.region, 3, value, keyPrefix3);
602
603 putRows(this.region, 3, value2, keyPrefix1);
604 putRows(this.region, 3, value2, keyPrefix2);
605 putRows(this.region, 3, value2, keyPrefix3);
606 System.out.println("Checking values for key: " + keyPrefix1);
607 assertEquals("Got back incorrect number of rows from scan", 3,
608 getNumberOfRows(keyPrefix1, value2, this.region));
609 System.out.println("Checking values for key: " + keyPrefix2);
610 assertEquals("Got back incorrect number of rows from scan", 3,
611 getNumberOfRows(keyPrefix2, value2, this.region));
612 System.out.println("Checking values for key: " + keyPrefix3);
613 assertEquals("Got back incorrect number of rows from scan", 3,
614 getNumberOfRows(keyPrefix3, value2, this.region));
615 deleteColumns(this.region, value2, keyPrefix1);
616 deleteColumns(this.region, value2, keyPrefix2);
617 deleteColumns(this.region, value2, keyPrefix3);
618 System.out.println("Starting important checks.....");
619 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,
620 0, getNumberOfRows(keyPrefix1, value2, this.region));
621 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,
622 0, getNumberOfRows(keyPrefix2, value2, this.region));
623 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,
624 0, getNumberOfRows(keyPrefix3, value2, this.region));
625 } finally {
626 HRegion.closeHRegion(this.region);
627 this.region = null;
628 }
629 }
630
631 public void testAppendWithReadOnlyTable() throws Exception {
632 byte[] TABLE = Bytes.toBytes("readOnlyTable");
633 this.region = initHRegion(TABLE, getName(), conf, true, Bytes.toBytes("somefamily"));
634 boolean exceptionCaught = false;
635 Append append = new Append(Bytes.toBytes("somerow"));
636 append.setDurability(Durability.SKIP_WAL);
637 append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
638 Bytes.toBytes("somevalue"));
639 try {
640 region.append(append);
641 } catch (IOException e) {
642 exceptionCaught = true;
643 } finally {
644 HRegion.closeHRegion(this.region);
645 this.region = null;
646 }
647 assertTrue(exceptionCaught == true);
648 }
649
650 public void testIncrWithReadOnlyTable() throws Exception {
651 byte[] TABLE = Bytes.toBytes("readOnlyTable");
652 this.region = initHRegion(TABLE, getName(), conf, true, Bytes.toBytes("somefamily"));
653 boolean exceptionCaught = false;
654 Increment inc = new Increment(Bytes.toBytes("somerow"));
655 inc.setDurability(Durability.SKIP_WAL);
656 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
657 try {
658 region.increment(inc);
659 } catch (IOException e) {
660 exceptionCaught = true;
661 } finally {
662 HRegion.closeHRegion(this.region);
663 this.region = null;
664 }
665 assertTrue(exceptionCaught == true);
666 }
667
668 private void deleteColumns(HRegion r, String value, String keyPrefix)
669 throws IOException {
670 InternalScanner scanner = buildScanner(keyPrefix, value, r);
671 int count = 0;
672 boolean more = false;
673 List<KeyValue> results = new ArrayList<KeyValue>();
674 do {
675 more = scanner.next(results);
676 if (results != null && !results.isEmpty())
677 count++;
678 else
679 break;
680 Delete delete = new Delete(results.get(0).getRow());
681 delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
682 r.delete(delete);
683 results.clear();
684 } while (more);
685 assertEquals("Did not perform correct number of deletes", 3, count);
686 }
687
688 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
689 InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
690 int numberOfResults = 0;
691 List<KeyValue> results = new ArrayList<KeyValue>();
692 boolean more = false;
693 do {
694 more = resultScanner.next(results);
695 if (results != null && !results.isEmpty()) numberOfResults++;
696 else break;
697 for (KeyValue kv: results) {
698 System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(kv.getValue()));
699 }
700 results.clear();
701 } while(more);
702 return numberOfResults;
703 }
704
705 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
706 throws IOException {
707
708 FilterList allFilters = new FilterList();
709 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
710
711 SingleColumnValueFilter filter =
712 new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
713 Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
714 filter.setFilterIfMissing(true);
715 allFilters.addFilter(filter);
716 Scan scan = new Scan();
717 scan.addFamily(Bytes.toBytes("trans-blob"));
718 scan.addFamily(Bytes.toBytes("trans-type"));
719 scan.addFamily(Bytes.toBytes("trans-date"));
720 scan.addFamily(Bytes.toBytes("trans-tags"));
721 scan.addFamily(Bytes.toBytes("trans-group"));
722 scan.setFilter(allFilters);
723 return r.getScanner(scan);
724 }
725
726 private void putRows(HRegion r, int numRows, String value, String key)
727 throws IOException {
728 for (int i = 0; i < numRows; i++) {
729 String row = key + "_" + i
730 System.out.println(String.format("Saving row: %s, with value %s", row,
731 value));
732 Put put = new Put(Bytes.toBytes(row));
733 put.setDurability(Durability.SKIP_WAL);
734 put.add(Bytes.toBytes("trans-blob"), null,
735 Bytes.toBytes("value for blob"));
736 put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
737 put.add(Bytes.toBytes("trans-date"), null,
738 Bytes.toBytes("20090921010101999"));
739 put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"),
740 Bytes.toBytes(value));
741 put.add(Bytes.toBytes("trans-group"), null,
742 Bytes.toBytes("adhocTransactionGroupId"));
743 r.put(put);
744 }
745 }
746
747 public void testFamilyWithAndWithoutColon() throws Exception {
748 byte [] b = Bytes.toBytes(getName());
749 byte [] cf = Bytes.toBytes(COLUMN_FAMILY);
750 this.region = initHRegion(b, getName(), conf, cf);
751 try {
752 Put p = new Put(b);
753 byte [] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
754 p.add(cfwithcolon, cfwithcolon, cfwithcolon);
755 boolean exception = false;
756 try {
757 this.region.put(p);
758 } catch (NoSuchColumnFamilyException e) {
759 exception = true;
760 }
761 assertTrue(exception);
762 } finally {
763 HRegion.closeHRegion(this.region);
764 this.region = null;
765 }
766 }
767
768 public void testBatchPut() throws Exception {
769 byte[] b = Bytes.toBytes(getName());
770 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
771 byte[] qual = Bytes.toBytes("qual");
772 byte[] val = Bytes.toBytes("val");
773 this.region = initHRegion(b, getName(), conf, cf);
774 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
775 try {
776 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
777 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
778
779 LOG.info("First a batch put with all valid puts");
780 final Put[] puts = new Put[10];
781 for (int i = 0; i < 10; i++) {
782 puts[i] = new Put(Bytes.toBytes("row_" + i));
783 puts[i].add(cf, qual, val);
784 }
785
786 OperationStatus[] codes = this.region.batchMutate(puts);
787 assertEquals(10, codes.length);
788 for (int i = 0; i < 10; i++) {
789 assertEquals(OperationStatusCode.SUCCESS, codes[i]
790 .getOperationStatusCode());
791 }
792 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
793
794
795 LOG.info("Next a batch put with one invalid family");
796 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
797 codes = this.region.batchMutate(puts);
798 assertEquals(10, codes.length);
799 for (int i = 0; i < 10; i++) {
800 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY :
801 OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
802 }
803
804 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
805
806 LOG.info("Next a batch put that has to break into two batches to avoid a lock");
807 RowLock rowLock = region.getRowLock(Bytes.toBytes("row_2"));
808
809 MultithreadedTestUtil.TestContext ctx =
810 new MultithreadedTestUtil.TestContext(conf);
811 final AtomicReference<OperationStatus[]> retFromThread =
812 new AtomicReference<OperationStatus[]>();
813 TestThread putter = new TestThread(ctx) {
814 @Override
815 public void doWork() throws IOException {
816 retFromThread.set(region.batchMutate(puts));
817 }
818 };
819 LOG.info("...starting put thread while holding lock");
820 ctx.addThread(putter);
821 ctx.startThreads();
822
823 LOG.info("...waiting for put thread to sync first time");
824 long startWait = System.currentTimeMillis();
825 while (metricsAssertHelper.getCounter("syncTimeNumOps", source) == syncs +2 ) {
826 Thread.sleep(100);
827 if (System.currentTimeMillis() - startWait > 10000) {
828 fail("Timed out waiting for thread to sync first minibatch");
829 }
830 }
831 LOG.info("...releasing row lock, which should let put thread continue");
832 rowLock.release();
833 LOG.info("...joining on thread");
834 ctx.stop();
835 LOG.info("...checking that next batch was synced");
836 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 4, source);
837 codes = retFromThread.get();
838 for (int i = 0; i < 10; i++) {
839 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY :
840 OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
841 }
842
843 } finally {
844 HRegion.closeHRegion(this.region);
845 this.region = null;
846 }
847 }
848
849 public void testBatchPutWithTsSlop() throws Exception {
850 byte[] b = Bytes.toBytes(getName());
851 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
852 byte[] qual = Bytes.toBytes("qual");
853 byte[] val = Bytes.toBytes("val");
854 Configuration conf = HBaseConfiguration.create(this.conf);
855
856
857 conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
858 this.region = initHRegion(b, getName(), conf, cf);
859
860 try{
861 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
862 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
863 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
864
865 final Put[] puts = new Put[10];
866 for (int i = 0; i < 10; i++) {
867 puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
868 puts[i].add(cf, qual, val);
869 }
870
871 OperationStatus[] codes = this.region.batchMutate(puts);
872 assertEquals(10, codes.length);
873 for (int i = 0; i < 10; i++) {
874 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i]
875 .getOperationStatusCode());
876 }
877 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
878
879 } finally {
880 HRegion.closeHRegion(this.region);
881 this.region = null;
882 }
883
884 }
885
886
887
888
889 public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
890 byte [] tableName = Bytes.toBytes("testtable");
891 byte [] row1 = Bytes.toBytes("row1");
892 byte [] fam1 = Bytes.toBytes("fam1");
893 byte [] qf1 = Bytes.toBytes("qualifier");
894 byte [] emptyVal = new byte[] {};
895 byte [] val1 = Bytes.toBytes("value1");
896 byte [] val2 = Bytes.toBytes("value2");
897
898
899 String method = this.getName();
900 this.region = initHRegion(tableName, method, conf, fam1);
901 try {
902
903 Put put = new Put(row1);
904 put.add(fam1, qf1, emptyVal);
905
906
907 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
908 new BinaryComparator(emptyVal), put, true);
909 assertTrue(res);
910
911
912 put = new Put(row1);
913 put.add(fam1, qf1, val1);
914
915
916 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
917 new BinaryComparator(emptyVal), put, true);
918 assertTrue(res);
919
920
921 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
922 new BinaryComparator(emptyVal), put, true);
923 assertFalse(res);
924
925 Delete delete = new Delete(row1);
926 delete.deleteColumn(fam1, qf1);
927 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
928 new BinaryComparator(emptyVal), delete, true);
929 assertFalse(res);
930
931 put = new Put(row1);
932 put.add(fam1, qf1, val2);
933
934 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
935 new BinaryComparator(val1), put, true);
936 assertTrue(res);
937
938
939 delete = new Delete(row1);
940 delete.deleteColumn(fam1, qf1);
941 delete.deleteColumn(fam1, qf1);
942 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
943 new BinaryComparator(val2), delete, true);
944 assertTrue(res);
945
946 delete = new Delete(row1);
947 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
948 new BinaryComparator(emptyVal), delete, true);
949 assertTrue(res);
950
951
952 put = new Put(row1);
953 put.add(fam1, qf1, val1);
954
955 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
956 new NullComparator(), put, true);
957 assertTrue(res);
958 } finally {
959 HRegion.closeHRegion(this.region);
960 this.region = null;
961 }
962 }
963
964 public void testCheckAndMutate_WithWrongValue() throws IOException{
965 byte [] tableName = Bytes.toBytes("testtable");
966 byte [] row1 = Bytes.toBytes("row1");
967 byte [] fam1 = Bytes.toBytes("fam1");
968 byte [] qf1 = Bytes.toBytes("qualifier");
969 byte [] val1 = Bytes.toBytes("value1");
970 byte [] val2 = Bytes.toBytes("value2");
971
972
973 String method = this.getName();
974 this.region = initHRegion(tableName, method, conf, fam1);
975 try {
976
977 Put put = new Put(row1);
978 put.add(fam1, qf1, val1);
979 region.put(put);
980
981
982 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
983 new BinaryComparator(val2), put, true);
984 assertEquals(false, res);
985
986
987 Delete delete = new Delete(row1);
988 delete.deleteFamily(fam1);
989 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
990 new BinaryComparator(val2), delete, true);
991 assertEquals(false, res);
992 } finally {
993 HRegion.closeHRegion(this.region);
994 this.region = null;
995 }
996 }
997
998 public void testCheckAndMutate_WithCorrectValue() throws IOException{
999 byte [] tableName = Bytes.toBytes("testtable");
1000 byte [] row1 = Bytes.toBytes("row1");
1001 byte [] fam1 = Bytes.toBytes("fam1");
1002 byte [] qf1 = Bytes.toBytes("qualifier");
1003 byte [] val1 = Bytes.toBytes("value1");
1004
1005
1006 String method = this.getName();
1007 this.region = initHRegion(tableName, method, conf, fam1);
1008 try {
1009
1010 Put put = new Put(row1);
1011 put.add(fam1, qf1, val1);
1012 region.put(put);
1013
1014
1015 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
1016 new BinaryComparator(val1), put, true);
1017 assertEquals(true, res);
1018
1019
1020 Delete delete = new Delete(row1);
1021 delete.deleteColumn(fam1, qf1);
1022 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
1023 new BinaryComparator(val1), put, true);
1024 assertEquals(true, res);
1025 } finally {
1026 HRegion.closeHRegion(this.region);
1027 this.region = null;
1028 }
1029 }
1030
1031 public void testCheckAndPut_ThatPutWasWritten() throws IOException{
1032 byte [] tableName = Bytes.toBytes("testtable");
1033 byte [] row1 = Bytes.toBytes("row1");
1034 byte [] fam1 = Bytes.toBytes("fam1");
1035 byte [] fam2 = Bytes.toBytes("fam2");
1036 byte [] qf1 = Bytes.toBytes("qualifier");
1037 byte [] val1 = Bytes.toBytes("value1");
1038 byte [] val2 = Bytes.toBytes("value2");
1039
1040 byte [][] families = {fam1, fam2};
1041
1042
1043 String method = this.getName();
1044 this.region = initHRegion(tableName, method, conf, families);
1045 try {
1046
1047 Put put = new Put(row1);
1048 put.add(fam1, qf1, val1);
1049 region.put(put);
1050
1051
1052 long ts = System.currentTimeMillis();
1053 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1054 put = new Put(row1);
1055 put.add(kv);
1056
1057
1058 HStore store = (HStore) region.getStore(fam1);
1059 store.memstore.kvset.size();
1060
1061 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
1062 new BinaryComparator(val1), put, true);
1063 assertEquals(true, res);
1064 store.memstore.kvset.size();
1065
1066 Get get = new Get(row1);
1067 get.addColumn(fam2, qf1);
1068 KeyValue [] actual = region.get(get).raw();
1069
1070 KeyValue [] expected = {kv};
1071
1072 assertEquals(expected.length, actual.length);
1073 for(int i=0; i<actual.length; i++) {
1074 assertEquals(expected[i], actual[i]);
1075 }
1076 } finally {
1077 HRegion.closeHRegion(this.region);
1078 this.region = null;
1079 }
1080 }
1081
1082 public void testCheckAndPut_wrongRowInPut() throws IOException {
1083 this.region = initHRegion(tableName, this.getName(), conf, COLUMNS);
1084 try {
1085 Put put = new Put(row2);
1086 put.add(fam1, qual1, value1);
1087 try {
1088 boolean res = region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1089 new BinaryComparator(value2), put, false);
1090 fail();
1091 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1092
1093 }
1094 } finally {
1095 HRegion.closeHRegion(this.region);
1096 this.region = null;
1097 }
1098 }
1099
1100 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException{
1101 byte [] tableName = Bytes.toBytes("testtable");
1102 byte [] row1 = Bytes.toBytes("row1");
1103 byte [] fam1 = Bytes.toBytes("fam1");
1104 byte [] fam2 = Bytes.toBytes("fam2");
1105 byte [] qf1 = Bytes.toBytes("qualifier1");
1106 byte [] qf2 = Bytes.toBytes("qualifier2");
1107 byte [] qf3 = Bytes.toBytes("qualifier3");
1108 byte [] val1 = Bytes.toBytes("value1");
1109 byte [] val2 = Bytes.toBytes("value2");
1110 byte [] val3 = Bytes.toBytes("value3");
1111 byte[] emptyVal = new byte[] { };
1112
1113 byte [][] families = {fam1, fam2};
1114
1115
1116 String method = this.getName();
1117 this.region = initHRegion(tableName, method, conf, families);
1118 try {
1119
1120 Put put = new Put(row1);
1121 put.add(fam1, qf1, val1);
1122 region.put(put);
1123 Threads.sleep(2);
1124
1125 put = new Put(row1);
1126 put.add(fam1, qf1, val2);
1127 put.add(fam2, qf1, val3);
1128 put.add(fam2, qf2, val2);
1129 put.add(fam2, qf3, val1);
1130 put.add(fam1, qf3, val1);
1131 region.put(put);
1132
1133
1134 Delete delete = new Delete(row1);
1135 delete.deleteColumn(fam1, qf1);
1136 delete.deleteColumn(fam2, qf1);
1137 delete.deleteColumn(fam1, qf3);
1138 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
1139 new BinaryComparator(val2), delete, true);
1140 assertEquals(true, res);
1141
1142 Get get = new Get(row1);
1143 get.addColumn(fam1, qf1);
1144 get.addColumn(fam1, qf3);
1145 get.addColumn(fam2, qf2);
1146 Result r = region.get(get);
1147 assertEquals(2, r.size());
1148 assertEquals(val1, r.getValue(fam1, qf1));
1149 assertEquals(val2, r.getValue(fam2, qf2));
1150
1151
1152 delete = new Delete(row1);
1153 delete.deleteFamily(fam2);
1154 res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL,
1155 new BinaryComparator(emptyVal), delete, true);
1156 assertEquals(true, res);
1157
1158 get = new Get(row1);
1159 r = region.get(get);
1160 assertEquals(1, r.size());
1161 assertEquals(val1, r.getValue(fam1, qf1));
1162
1163
1164 delete = new Delete(row1);
1165 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
1166 new BinaryComparator(val1), delete, true);
1167 assertEquals(true, res);
1168 get = new Get(row1);
1169 r = region.get(get);
1170 assertEquals(0, r.size());
1171 } finally {
1172 HRegion.closeHRegion(this.region);
1173 this.region = null;
1174 }
1175 }
1176
1177
1178
1179
1180 public void testDelete_multiDeleteColumn() throws IOException {
1181 byte [] tableName = Bytes.toBytes("testtable");
1182 byte [] row1 = Bytes.toBytes("row1");
1183 byte [] fam1 = Bytes.toBytes("fam1");
1184 byte [] qual = Bytes.toBytes("qualifier");
1185 byte [] value = Bytes.toBytes("value");
1186
1187 Put put = new Put(row1);
1188 put.add(fam1, qual, 1, value);
1189 put.add(fam1, qual, 2, value);
1190
1191 String method = this.getName();
1192 this.region = initHRegion(tableName, method, conf, fam1);
1193 try {
1194 region.put(put);
1195
1196
1197 Delete delete = new Delete(row1);
1198 delete.deleteColumn(fam1, qual);
1199 delete.deleteColumn(fam1, qual);
1200 region.delete(delete);
1201
1202 Get get = new Get(row1);
1203 get.addFamily(fam1);
1204 Result r = region.get(get);
1205 assertEquals(0, r.size());
1206 } finally {
1207 HRegion.closeHRegion(this.region);
1208 this.region = null;
1209 }
1210 }
1211
1212 public void testDelete_CheckFamily() throws IOException {
1213 byte [] tableName = Bytes.toBytes("testtable");
1214 byte [] row1 = Bytes.toBytes("row1");
1215 byte [] fam1 = Bytes.toBytes("fam1");
1216 byte [] fam2 = Bytes.toBytes("fam2");
1217 byte [] fam3 = Bytes.toBytes("fam3");
1218 byte [] fam4 = Bytes.toBytes("fam4");
1219
1220
1221 String method = this.getName();
1222 this.region = initHRegion(tableName, method, conf, fam1, fam2, fam3);
1223 try {
1224 List<Cell> kvs = new ArrayList<Cell>();
1225 kvs.add(new KeyValue(row1, fam4, null, null));
1226
1227
1228
1229 byte [] family = fam2;
1230 try {
1231 NavigableMap<byte[], List<Cell>> deleteMap =
1232 new TreeMap<byte[], List<Cell>>(Bytes.BYTES_COMPARATOR);
1233 deleteMap.put(family, kvs);
1234 region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, Durability.SYNC_WAL);
1235 } catch (Exception e) {
1236 assertTrue("Family " +new String(family)+ " does not exist", false);
1237 }
1238
1239
1240 boolean ok = false;
1241 family = fam4;
1242 try {
1243 NavigableMap<byte[], List<Cell>> deleteMap =
1244 new TreeMap<byte[], List<Cell>>(Bytes.BYTES_COMPARATOR);
1245 deleteMap.put(family, kvs);
1246 region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, Durability.SYNC_WAL);
1247 } catch (Exception e) {
1248 ok = true;
1249 }
1250 assertEquals("Family " +new String(family)+ " does exist", true, ok);
1251 } finally {
1252 HRegion.closeHRegion(this.region);
1253 this.region = null;
1254 }
1255 }
1256
1257 public void testDelete_mixed() throws IOException, InterruptedException {
1258 byte [] tableName = Bytes.toBytes("testtable");
1259 byte [] fam = Bytes.toBytes("info");
1260 byte [][] families = {fam};
1261 String method = this.getName();
1262 this.region = initHRegion(tableName, method, conf, families);
1263 try {
1264 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1265
1266 byte [] row = Bytes.toBytes("table_name");
1267
1268 byte [] serverinfo = Bytes.toBytes("serverinfo");
1269 byte [] splitA = Bytes.toBytes("splitA");
1270 byte [] splitB = Bytes.toBytes("splitB");
1271
1272
1273 Put put = new Put(row);
1274 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1275 region.put(put);
1276
1277 put = new Put(row);
1278 put.add(fam, splitB, Bytes.toBytes("reference_B"));
1279 region.put(put);
1280
1281 put = new Put(row);
1282 put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1283 region.put(put);
1284
1285
1286 Delete delete = new Delete(row);
1287 delete.deleteColumns(fam, splitA);
1288 region.delete(delete);
1289
1290
1291 Get get = new Get(row).addColumn(fam, serverinfo);
1292 Result result = region.get(get);
1293 assertEquals(1, result.size());
1294
1295 get = new Get(row).addColumn(fam, splitA);
1296 result = region.get(get);
1297 assertEquals(0, result.size());
1298
1299 get = new Get(row).addColumn(fam, splitB);
1300 result = region.get(get);
1301 assertEquals(1, result.size());
1302
1303
1304 put = new Put(row);
1305 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1306 region.put(put);
1307 get = new Get(row);
1308 result = region.get(get);
1309 assertEquals(3, result.size());
1310
1311
1312 delete = new Delete(row);
1313 region.delete(delete);
1314 assertEquals(0, region.get(get).size());
1315
1316 region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
1317 result = region.get(get);
1318 assertEquals(1, result.size());
1319 } finally {
1320 HRegion.closeHRegion(this.region);
1321 this.region = null;
1322 }
1323 }
1324
1325 public void testDeleteRowWithFutureTs() throws IOException {
1326 byte [] tableName = Bytes.toBytes("testtable");
1327 byte [] fam = Bytes.toBytes("info");
1328 byte [][] families = {fam};
1329 String method = this.getName();
1330 this.region = initHRegion(tableName, method, conf, families);
1331 try {
1332 byte [] row = Bytes.toBytes("table_name");
1333
1334 byte [] serverinfo = Bytes.toBytes("serverinfo");
1335
1336
1337 Put put = new Put(row);
1338 put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP-5,Bytes.toBytes("value"));
1339 region.put(put);
1340
1341
1342 Delete delete = new Delete(row);
1343 region.delete(delete);
1344
1345
1346 Get get = new Get(row).addColumn(fam, serverinfo);
1347 Result result = region.get(get);
1348 assertEquals(1, result.size());
1349
1350
1351 delete = new Delete(row,HConstants.LATEST_TIMESTAMP-3);
1352 region.delete(delete);
1353
1354
1355 get = new Get(row).addColumn(fam, serverinfo);
1356 result = region.get(get);
1357 assertEquals(0, result.size());
1358 } finally {
1359 HRegion.closeHRegion(this.region);
1360 this.region = null;
1361 }
1362 }
1363
1364
1365
1366
1367
1368 public void testPutWithLatestTS() throws IOException {
1369 byte [] tableName = Bytes.toBytes("testtable");
1370 byte [] fam = Bytes.toBytes("info");
1371 byte [][] families = {fam};
1372 String method = this.getName();
1373 this.region = initHRegion(tableName, method, conf, families);
1374 try {
1375 byte [] row = Bytes.toBytes("row1");
1376
1377 byte [] qual = Bytes.toBytes("qual");
1378
1379
1380 Put put = new Put(row);
1381 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1382 region.put(put);
1383
1384
1385 Get get = new Get(row).addColumn(fam, qual);
1386 Result result = region.get(get);
1387 assertEquals(1, result.size());
1388 KeyValue kv = result.raw()[0];
1389 LOG.info("Got: " + kv);
1390 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1391 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1392
1393
1394
1395 row = Bytes.toBytes("row2");
1396 put = new Put(row);
1397 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1398 region.put(put);
1399
1400
1401 get = new Get(row).addColumn(fam, qual);
1402 result = region.get(get);
1403 assertEquals(1, result.size());
1404 kv = result.raw()[0];
1405 LOG.info("Got: " + kv);
1406 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1407 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1408 } finally {
1409 HRegion.closeHRegion(this.region);
1410 this.region = null;
1411 }
1412
1413 }
1414
1415
1416
1417
1418
1419
1420
1421 public void testPutWithTsSlop() throws IOException {
1422 byte[] tableName = Bytes.toBytes("testtable");
1423 byte[] fam = Bytes.toBytes("info");
1424 byte[][] families = { fam };
1425 String method = this.getName();
1426 Configuration conf = HBaseConfiguration.create(this.conf);
1427
1428
1429 conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1430 this.region = initHRegion(tableName, method, conf, families);
1431 boolean caughtExcep = false;
1432 try {
1433 try {
1434
1435 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes
1436 .toBytes("value")));
1437
1438 region.put(new Put(row).add(fam, Bytes.toBytes("qual"),
1439 System.currentTimeMillis() + 2000,
1440 Bytes.toBytes("value")));
1441 fail("Expected IOE for TS out of configured timerange");
1442 } catch (FailedSanityCheckException ioe) {
1443 LOG.debug("Received expected exception", ioe);
1444 caughtExcep = true;
1445 }
1446 assertTrue("Should catch FailedSanityCheckException", caughtExcep);
1447 } finally {
1448 HRegion.closeHRegion(this.region);
1449 this.region = null;
1450 }
1451 }
1452
1453 public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
1454 byte [] tableName = Bytes.toBytes("test_table");
1455 byte [] fam1 = Bytes.toBytes("columnA");
1456 byte [] fam2 = Bytes.toBytes("columnB");
1457 this.region = initHRegion(tableName, getName(), conf, fam1, fam2);
1458 try {
1459 byte [] rowA = Bytes.toBytes("rowA");
1460 byte [] rowB = Bytes.toBytes("rowB");
1461
1462 byte [] value = Bytes.toBytes("value");
1463
1464 Delete delete = new Delete(rowA);
1465 delete.deleteFamily(fam1);
1466
1467 region.delete(delete);
1468
1469
1470 Put put = new Put(rowA);
1471 put.add(fam2, null, value);
1472 region.put(put);
1473
1474 put = new Put(rowB);
1475 put.add(fam1, null, value);
1476 put.add(fam2, null, value);
1477 region.put(put);
1478
1479 Scan scan = new Scan();
1480 scan.addFamily(fam1).addFamily(fam2);
1481 InternalScanner s = region.getScanner(scan);
1482 List<KeyValue> results = new ArrayList<KeyValue>();
1483 s.next(results);
1484 assertTrue(Bytes.equals(rowA, results.get(0).getRow()));
1485
1486 results.clear();
1487 s.next(results);
1488 assertTrue(Bytes.equals(rowB, results.get(0).getRow()));
1489 } finally {
1490 HRegion.closeHRegion(this.region);
1491 this.region = null;
1492 }
1493 }
1494
1495 public void testDeleteColumns_PostInsert() throws IOException,
1496 InterruptedException {
1497 Delete delete = new Delete(row);
1498 delete.deleteColumns(fam1, qual1);
1499 doTestDelete_AndPostInsert(delete);
1500 }
1501
1502 public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
1503 Delete delete = new Delete(row);
1504 delete.deleteFamily(fam1);
1505 doTestDelete_AndPostInsert(delete);
1506 }
1507
1508 public void doTestDelete_AndPostInsert(Delete delete)
1509 throws IOException, InterruptedException {
1510 this.region = initHRegion(tableName, getName(), conf, fam1);
1511 try {
1512 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1513 Put put = new Put(row);
1514 put.add(fam1, qual1, value1);
1515 region.put(put);
1516
1517
1518 region.delete(delete);
1519
1520
1521
1522 put = new Put(row);
1523 put.add(fam1, qual1, value2);
1524 region.put(put);
1525
1526
1527 Get get = new Get(row);
1528 get.addColumn(fam1, qual1);
1529
1530 Result r = region.get(get);
1531 assertEquals(1, r.size());
1532 assertByteEquals(value2, r.getValue(fam1, qual1));
1533
1534
1535 Scan scan = new Scan(row);
1536 scan.addColumn(fam1, qual1);
1537 InternalScanner s = region.getScanner(scan);
1538
1539 List<KeyValue> results = new ArrayList<KeyValue>();
1540 assertEquals(false, s.next(results));
1541 assertEquals(1, results.size());
1542 KeyValue kv = results.get(0);
1543
1544 assertByteEquals(value2, kv.getValue());
1545 assertByteEquals(fam1, kv.getFamily());
1546 assertByteEquals(qual1, kv.getQualifier());
1547 assertByteEquals(row, kv.getRow());
1548 } finally {
1549 HRegion.closeHRegion(this.region);
1550 this.region = null;
1551 }
1552 }
1553
1554 public void testDelete_CheckTimestampUpdated()
1555 throws IOException {
1556 byte [] row1 = Bytes.toBytes("row1");
1557 byte [] col1 = Bytes.toBytes("col1");
1558 byte [] col2 = Bytes.toBytes("col2");
1559 byte [] col3 = Bytes.toBytes("col3");
1560
1561
1562 String method = this.getName();
1563 this.region = initHRegion(tableName, method, conf, fam1);
1564 try {
1565
1566 List<Cell> kvs = new ArrayList<Cell>();
1567 kvs.add(new KeyValue(row1, fam1, col1, null));
1568 kvs.add(new KeyValue(row1, fam1, col2, null));
1569 kvs.add(new KeyValue(row1, fam1, col3, null));
1570
1571 NavigableMap<byte[], List<Cell>> deleteMap =
1572 new TreeMap<byte[], List<Cell>>(Bytes.BYTES_COMPARATOR);
1573 deleteMap.put(fam1, kvs);
1574 region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, Durability.SYNC_WAL);
1575
1576
1577
1578 long now = System.currentTimeMillis();
1579 KeyValue firstKv = ((HStore) region.getStore(fam1)).memstore.kvset.first();
1580 assertTrue(firstKv.getTimestamp() <= now);
1581 now = firstKv.getTimestamp();
1582 for (KeyValue kv : ((HStore) region.getStore(fam1)).memstore.kvset) {
1583 assertTrue(kv.getTimestamp() <= now);
1584 now = kv.getTimestamp();
1585 }
1586 } finally {
1587 HRegion.closeHRegion(this.region);
1588 this.region = null;
1589 }
1590 }
1591
1592
1593
1594
1595 public void testGet_FamilyChecker() throws IOException {
1596 byte [] tableName = Bytes.toBytes("testtable");
1597 byte [] row1 = Bytes.toBytes("row1");
1598 byte [] fam1 = Bytes.toBytes("fam1");
1599 byte [] fam2 = Bytes.toBytes("False");
1600 byte [] col1 = Bytes.toBytes("col1");
1601
1602
1603 String method = this.getName();
1604 this.region = initHRegion(tableName, method, conf, fam1);
1605 try {
1606 Get get = new Get(row1);
1607 get.addColumn(fam2, col1);
1608
1609
1610 try {
1611 region.get(get);
1612 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
1613 assertFalse(false);
1614 return;
1615 }
1616 assertFalse(true);
1617 } finally {
1618 HRegion.closeHRegion(this.region);
1619 this.region = null;
1620 }
1621 }
1622
1623 public void testGet_Basic() throws IOException {
1624 byte [] tableName = Bytes.toBytes("testtable");
1625 byte [] row1 = Bytes.toBytes("row1");
1626 byte [] fam1 = Bytes.toBytes("fam1");
1627 byte [] col1 = Bytes.toBytes("col1");
1628 byte [] col2 = Bytes.toBytes("col2");
1629 byte [] col3 = Bytes.toBytes("col3");
1630 byte [] col4 = Bytes.toBytes("col4");
1631 byte [] col5 = Bytes.toBytes("col5");
1632
1633
1634 String method = this.getName();
1635 this.region = initHRegion(tableName, method, conf, fam1);
1636 try {
1637
1638 Put put = new Put(row1);
1639 put.add(fam1, col1, null);
1640 put.add(fam1, col2, null);
1641 put.add(fam1, col3, null);
1642 put.add(fam1, col4, null);
1643 put.add(fam1, col5, null);
1644 region.put(put);
1645
1646 Get get = new Get(row1);
1647 get.addColumn(fam1, col2);
1648 get.addColumn(fam1, col4);
1649
1650 KeyValue kv1 = new KeyValue(row1, fam1, col2);
1651 KeyValue kv2 = new KeyValue(row1, fam1, col4);
1652 KeyValue [] expected = {kv1, kv2};
1653
1654
1655 Result res = region.get(get);
1656 assertEquals(expected.length, res.size());
1657 for(int i=0; i<res.size(); i++){
1658 assertEquals(0,
1659 Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
1660 assertEquals(0,
1661 Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
1662 assertEquals(0,
1663 Bytes.compareTo(
1664 expected[i].getQualifier(), res.raw()[i].getQualifier()));
1665 }
1666
1667
1668 Get g = new Get(row1);
1669 final int count = 2;
1670 g.setFilter(new ColumnCountGetFilter(count));
1671 res = region.get(g);
1672 assertEquals(count, res.size());
1673 } finally {
1674 HRegion.closeHRegion(this.region);
1675 this.region = null;
1676 }
1677 }
1678
1679 public void testGet_Empty() throws IOException {
1680 byte [] tableName = Bytes.toBytes("emptytable");
1681 byte [] row = Bytes.toBytes("row");
1682 byte [] fam = Bytes.toBytes("fam");
1683
1684 String method = this.getName();
1685 this.region = initHRegion(tableName, method, conf, fam);
1686 try {
1687 Get get = new Get(row);
1688 get.addFamily(fam);
1689 Result r = region.get(get);
1690
1691 assertTrue(r.isEmpty());
1692 } finally {
1693 HRegion.closeHRegion(this.region);
1694 this.region = null;
1695 }
1696 }
1697
1698
1699
1700
1701
1702 public void stestGet_Root() throws IOException {
1703
1704 String method = this.getName();
1705 this.region = initHRegion(TableName.ROOT_TABLE_NAME,
1706 method, conf, HConstants.CATALOG_FAMILY);
1707 try {
1708
1709 Put put = new Put(HConstants.EMPTY_START_ROW);
1710 put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, null);
1711 region.put(put);
1712
1713 Get get = new Get(HConstants.EMPTY_START_ROW);
1714 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
1715
1716
1717 KeyValue kv1 = new KeyValue(HConstants.EMPTY_START_ROW,
1718 HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
1719 KeyValue [] expected = {kv1};
1720
1721
1722 Result res = region.get(get);
1723
1724 assertEquals(expected.length, res.size());
1725 for(int i=0; i<res.size(); i++){
1726 assertEquals(0,
1727 Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
1728 assertEquals(0,
1729 Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
1730 assertEquals(0,
1731 Bytes.compareTo(
1732 expected[i].getQualifier(), res.raw()[i].getQualifier()));
1733 }
1734
1735
1736 region.flushcache();
1737
1738
1739 res = region.get(get);
1740
1741 assertEquals(expected.length, res.size());
1742 for(int i=0; i<res.size(); i++){
1743 assertEquals(0,
1744 Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
1745 assertEquals(0,
1746 Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
1747 assertEquals(0,
1748 Bytes.compareTo(
1749 expected[i].getQualifier(), res.raw()[i].getQualifier()));
1750 }
1751
1752
1753 Scan scan = new Scan();
1754 scan.addFamily(HConstants.CATALOG_FAMILY);
1755 InternalScanner s = region.getScanner(scan);
1756 List<KeyValue> result = new ArrayList<KeyValue>();
1757 s.next(result);
1758
1759 assertEquals(expected.length, result.size());
1760 for(int i=0; i<res.size(); i++){
1761 assertEquals(0,
1762 Bytes.compareTo(expected[i].getRow(), result.get(i).getRow()));
1763 assertEquals(0,
1764 Bytes.compareTo(expected[i].getFamily(), result.get(i).getFamily()));
1765 assertEquals(0,
1766 Bytes.compareTo(
1767 expected[i].getQualifier(), result.get(i).getQualifier()));
1768 }
1769 } finally {
1770 HRegion.closeHRegion(this.region);
1771 this.region = null;
1772 }
1773 }
1774
1775
1776
1777
1778 public void testMerge() throws IOException {
1779 byte [] tableName = Bytes.toBytes("testtable");
1780 byte [][] families = {fam1, fam2, fam3};
1781 Configuration hc = initSplit();
1782
1783 String method = this.getName();
1784 this.region = initHRegion(tableName, method, hc, families);
1785 try {
1786 LOG.info("" + addContent(region, fam3));
1787 region.flushcache();
1788 region.compactStores();
1789 byte [] splitRow = region.checkSplit();
1790 assertNotNull(splitRow);
1791 LOG.info("SplitRow: " + Bytes.toString(splitRow));
1792 HRegion [] subregions = splitRegion(region, splitRow);
1793 try {
1794
1795 for (int i = 0; i < subregions.length; i++) {
1796 openClosedRegion(subregions[i]);
1797 subregions[i].compactStores();
1798 }
1799 Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
1800 Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
1801 Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
1802 long startTime = System.currentTimeMillis();
1803 region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
1804 LOG.info("Merge regions elapsed time: " +
1805 ((System.currentTimeMillis() - startTime) / 1000.0));
1806 fs.delete(oldRegion1, true);
1807 fs.delete(oldRegion2, true);
1808 fs.delete(oldRegionPath, true);
1809 LOG.info("splitAndMerge completed.");
1810 } finally {
1811 for (int i = 0; i < subregions.length; i++) {
1812 try {
1813 HRegion.closeHRegion(subregions[i]);
1814 } catch (IOException e) {
1815
1816 }
1817 }
1818 }
1819 } finally {
1820 HRegion.closeHRegion(this.region);
1821 this.region = null;
1822 }
1823 }
1824
1825
1826
1827
1828
1829
1830
1831 HRegion [] splitRegion(final HRegion parent, final byte [] midkey)
1832 throws IOException {
1833 PairOfSameType<HRegion> result = null;
1834 SplitTransaction st = new SplitTransaction(parent, midkey);
1835
1836
1837 if (!st.prepare()) return null;
1838 try {
1839 result = st.execute(null, null);
1840 } catch (IOException ioe) {
1841 try {
1842 LOG.info("Running rollback of failed split of " +
1843 parent.getRegionNameAsString() + "; " + ioe.getMessage());
1844 st.rollback(null, null);
1845 LOG.info("Successful rollback of failed split of " +
1846 parent.getRegionNameAsString());
1847 return null;
1848 } catch (RuntimeException e) {
1849
1850 LOG.info("Failed rollback of failed split of " +
1851 parent.getRegionNameAsString() + " -- aborting server", e);
1852 }
1853 }
1854 return new HRegion [] {result.getFirst(), result.getSecond()};
1855 }
1856
1857
1858
1859
1860 public void testGetScanner_WithOkFamilies() throws IOException {
1861 byte [] tableName = Bytes.toBytes("testtable");
1862 byte [] fam1 = Bytes.toBytes("fam1");
1863 byte [] fam2 = Bytes.toBytes("fam2");
1864
1865 byte [][] families = {fam1, fam2};
1866
1867
1868 String method = this.getName();
1869 this.region = initHRegion(tableName, method, conf, families);
1870 try {
1871 Scan scan = new Scan();
1872 scan.addFamily(fam1);
1873 scan.addFamily(fam2);
1874 try {
1875 region.getScanner(scan);
1876 } catch (Exception e) {
1877 assertTrue("Families could not be found in Region", false);
1878 }
1879 } finally {
1880 HRegion.closeHRegion(this.region);
1881 this.region = null;
1882 }
1883 }
1884
1885 public void testGetScanner_WithNotOkFamilies() throws IOException {
1886 byte [] tableName = Bytes.toBytes("testtable");
1887 byte [] fam1 = Bytes.toBytes("fam1");
1888 byte [] fam2 = Bytes.toBytes("fam2");
1889
1890 byte [][] families = {fam1};
1891
1892
1893 String method = this.getName();
1894 this.region = initHRegion(tableName, method, conf, families);
1895 try {
1896 Scan scan = new Scan();
1897 scan.addFamily(fam2);
1898 boolean ok = false;
1899 try {
1900 region.getScanner(scan);
1901 } catch (Exception e) {
1902 ok = true;
1903 }
1904 assertTrue("Families could not be found in Region", ok);
1905 } finally {
1906 HRegion.closeHRegion(this.region);
1907 this.region = null;
1908 }
1909 }
1910
1911 public void testGetScanner_WithNoFamilies() throws IOException {
1912 byte [] tableName = Bytes.toBytes("testtable");
1913 byte [] row1 = Bytes.toBytes("row1");
1914 byte [] fam1 = Bytes.toBytes("fam1");
1915 byte [] fam2 = Bytes.toBytes("fam2");
1916 byte [] fam3 = Bytes.toBytes("fam3");
1917 byte [] fam4 = Bytes.toBytes("fam4");
1918
1919 byte [][] families = {fam1, fam2, fam3, fam4};
1920
1921
1922 String method = this.getName();
1923 this.region = initHRegion(tableName, method, conf, families);
1924 try {
1925
1926
1927 Put put = new Put(row1);
1928 put.add(fam1, null, null);
1929 put.add(fam2, null, null);
1930 put.add(fam3, null, null);
1931 put.add(fam4, null, null);
1932 region.put(put);
1933
1934 Scan scan = null;
1935 HRegion.RegionScannerImpl is = null;
1936
1937
1938
1939 scan = new Scan();
1940 scan.addFamily(fam2);
1941 scan.addFamily(fam4);
1942 is = (RegionScannerImpl) region.getScanner(scan);
1943 MultiVersionConsistencyControl.resetThreadReadPoint(region.getMVCC());
1944 assertEquals(1, ((RegionScannerImpl)is).storeHeap.getHeap().size());
1945
1946 scan = new Scan();
1947 is = (RegionScannerImpl) region.getScanner(scan);
1948 MultiVersionConsistencyControl.resetThreadReadPoint(region.getMVCC());
1949 assertEquals(families.length -1,
1950 ((RegionScannerImpl)is).storeHeap.getHeap().size());
1951 } finally {
1952 HRegion.closeHRegion(this.region);
1953 this.region = null;
1954 }
1955 }
1956
1957
1958
1959
1960
1961 public void testGetScanner_WithRegionClosed() throws IOException {
1962 byte[] tableName = Bytes.toBytes("testtable");
1963 byte[] fam1 = Bytes.toBytes("fam1");
1964 byte[] fam2 = Bytes.toBytes("fam2");
1965
1966 byte[][] families = {fam1, fam2};
1967
1968
1969 String method = this.getName();
1970 try {
1971 this.region = initHRegion(tableName, method, conf, families);
1972 } catch (IOException e) {
1973 e.printStackTrace();
1974 fail("Got IOException during initHRegion, " + e.getMessage());
1975 }
1976 try {
1977 region.closed.set(true);
1978 try {
1979 region.getScanner(null);
1980 fail("Expected to get an exception during getScanner on a region that is closed");
1981 } catch (NotServingRegionException e) {
1982
1983 } catch (IOException e) {
1984 fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
1985 + e.getMessage());
1986 }
1987 } finally {
1988 HRegion.closeHRegion(this.region);
1989 this.region = null;
1990 }
1991 }
1992
1993 public void testRegionScanner_Next() throws IOException {
1994 byte [] tableName = Bytes.toBytes("testtable");
1995 byte [] row1 = Bytes.toBytes("row1");
1996 byte [] row2 = Bytes.toBytes("row2");
1997 byte [] fam1 = Bytes.toBytes("fam1");
1998 byte [] fam2 = Bytes.toBytes("fam2");
1999 byte [] fam3 = Bytes.toBytes("fam3");
2000 byte [] fam4 = Bytes.toBytes("fam4");
2001
2002 byte [][] families = {fam1, fam2, fam3, fam4};
2003 long ts = System.currentTimeMillis();
2004
2005
2006 String method = this.getName();
2007 this.region = initHRegion(tableName, method, conf, families);
2008 try {
2009
2010 Put put = null;
2011 put = new Put(row1);
2012 put.add(fam1, null, ts, null);
2013 put.add(fam2, null, ts, null);
2014 put.add(fam3, null, ts, null);
2015 put.add(fam4, null, ts, null);
2016 region.put(put);
2017
2018 put = new Put(row2);
2019 put.add(fam1, null, ts, null);
2020 put.add(fam2, null, ts, null);
2021 put.add(fam3, null, ts, null);
2022 put.add(fam4, null, ts, null);
2023 region.put(put);
2024
2025 Scan scan = new Scan();
2026 scan.addFamily(fam2);
2027 scan.addFamily(fam4);
2028 InternalScanner is = region.getScanner(scan);
2029
2030 List<KeyValue> res = null;
2031
2032
2033 List<KeyValue> expected1 = new ArrayList<KeyValue>();
2034 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2035 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2036
2037 res = new ArrayList<KeyValue>();
2038 is.next(res);
2039 for (int i = 0; i < res.size(); i++) {
2040 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2041 }
2042
2043
2044 List<KeyValue> expected2 = new ArrayList<KeyValue>();
2045 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2046 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2047
2048 res = new ArrayList<KeyValue>();
2049 is.next(res);
2050 for(int i=0; i<res.size(); i++) {
2051 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2052 }
2053 } finally {
2054 HRegion.closeHRegion(this.region);
2055 this.region = null;
2056 }
2057 }
2058
2059 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions()
2060 throws IOException {
2061 byte [] tableName = Bytes.toBytes("testtable");
2062 byte [] row1 = Bytes.toBytes("row1");
2063 byte [] qf1 = Bytes.toBytes("qualifier1");
2064 byte [] qf2 = Bytes.toBytes("qualifier2");
2065 byte [] fam1 = Bytes.toBytes("fam1");
2066 byte [][] families = {fam1};
2067
2068 long ts1 = System.currentTimeMillis();
2069 long ts2 = ts1 + 1;
2070 long ts3 = ts1 + 2;
2071
2072
2073 String method = this.getName();
2074 this.region = initHRegion(tableName, method, conf, families);
2075 try {
2076
2077 Put put = null;
2078 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2079 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2080 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2081
2082 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2083 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2084 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2085
2086 put = new Put(row1);
2087 put.add(kv13);
2088 put.add(kv12);
2089 put.add(kv11);
2090 put.add(kv23);
2091 put.add(kv22);
2092 put.add(kv21);
2093 region.put(put);
2094
2095
2096 List<KeyValue> expected = new ArrayList<KeyValue>();
2097 expected.add(kv13);
2098 expected.add(kv12);
2099
2100 Scan scan = new Scan(row1);
2101 scan.addColumn(fam1, qf1);
2102 scan.setMaxVersions(MAX_VERSIONS);
2103 List<KeyValue> actual = new ArrayList<KeyValue>();
2104 InternalScanner scanner = region.getScanner(scan);
2105
2106 boolean hasNext = scanner.next(actual);
2107 assertEquals(false, hasNext);
2108
2109
2110 for(int i=0; i<expected.size(); i++) {
2111 assertEquals(expected.get(i), actual.get(i));
2112 }
2113 } finally {
2114 HRegion.closeHRegion(this.region);
2115 this.region = null;
2116 }
2117 }
2118
2119 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions()
2120 throws IOException{
2121 byte [] tableName = Bytes.toBytes("testtable");
2122 byte [] row1 = Bytes.toBytes("row1");
2123 byte [] qf1 = Bytes.toBytes("qualifier1");
2124 byte [] qf2 = Bytes.toBytes("qualifier2");
2125 byte [] fam1 = Bytes.toBytes("fam1");
2126 byte [][] families = {fam1};
2127
2128 long ts1 = 1;
2129 long ts2 = ts1 + 1;
2130 long ts3 = ts1 + 2;
2131
2132
2133 String method = this.getName();
2134 this.region = initHRegion(tableName, method, conf, families);
2135 try {
2136
2137 Put put = null;
2138 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2139 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2140 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2141
2142 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2143 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2144 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2145
2146 put = new Put(row1);
2147 put.add(kv13);
2148 put.add(kv12);
2149 put.add(kv11);
2150 put.add(kv23);
2151 put.add(kv22);
2152 put.add(kv21);
2153 region.put(put);
2154 region.flushcache();
2155
2156
2157 List<KeyValue> expected = new ArrayList<KeyValue>();
2158 expected.add(kv13);
2159 expected.add(kv12);
2160 expected.add(kv23);
2161 expected.add(kv22);
2162
2163 Scan scan = new Scan(row1);
2164 scan.addColumn(fam1, qf1);
2165 scan.addColumn(fam1, qf2);
2166 scan.setMaxVersions(MAX_VERSIONS);
2167 List<KeyValue> actual = new ArrayList<KeyValue>();
2168 InternalScanner scanner = region.getScanner(scan);
2169
2170 boolean hasNext = scanner.next(actual);
2171 assertEquals(false, hasNext);
2172
2173
2174 for(int i=0; i<expected.size(); i++) {
2175 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2176 }
2177 } finally {
2178 HRegion.closeHRegion(this.region);
2179 this.region = null;
2180 }
2181 }
2182
2183 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions()
2184 throws IOException {
2185 byte [] tableName = Bytes.toBytes("testtable");
2186 byte [] row1 = Bytes.toBytes("row1");
2187 byte [] fam1 = Bytes.toBytes("fam1");
2188 byte [][] families = {fam1};
2189 byte [] qf1 = Bytes.toBytes("qualifier1");
2190 byte [] qf2 = Bytes.toBytes("qualifier2");
2191
2192 long ts1 = 1;
2193 long ts2 = ts1 + 1;
2194 long ts3 = ts1 + 2;
2195 long ts4 = ts1 + 3;
2196
2197
2198 String method = this.getName();
2199 this.region = initHRegion(tableName, method, conf, families);
2200 try {
2201
2202 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2203 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2204 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2205 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2206
2207 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2208 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2209 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2210 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2211
2212 Put put = null;
2213 put = new Put(row1);
2214 put.add(kv14);
2215 put.add(kv24);
2216 region.put(put);
2217 region.flushcache();
2218
2219 put = new Put(row1);
2220 put.add(kv23);
2221 put.add(kv13);
2222 region.put(put);
2223 region.flushcache();
2224
2225 put = new Put(row1);
2226 put.add(kv22);
2227 put.add(kv12);
2228 region.put(put);
2229 region.flushcache();
2230
2231 put = new Put(row1);
2232 put.add(kv21);
2233 put.add(kv11);
2234 region.put(put);
2235
2236
2237 List<KeyValue> expected = new ArrayList<KeyValue>();
2238 expected.add(kv14);
2239 expected.add(kv13);
2240 expected.add(kv12);
2241 expected.add(kv24);
2242 expected.add(kv23);
2243 expected.add(kv22);
2244
2245 Scan scan = new Scan(row1);
2246 scan.addColumn(fam1, qf1);
2247 scan.addColumn(fam1, qf2);
2248 int versions = 3;
2249 scan.setMaxVersions(versions);
2250 List<KeyValue> actual = new ArrayList<KeyValue>();
2251 InternalScanner scanner = region.getScanner(scan);
2252
2253 boolean hasNext = scanner.next(actual);
2254 assertEquals(false, hasNext);
2255
2256
2257 for(int i=0; i<expected.size(); i++) {
2258 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2259 }
2260 } finally {
2261 HRegion.closeHRegion(this.region);
2262 this.region = null;
2263 }
2264 }
2265
2266 public void testScanner_Wildcard_FromMemStore_EnforceVersions()
2267 throws IOException {
2268 byte [] tableName = Bytes.toBytes("testtable");
2269 byte [] row1 = Bytes.toBytes("row1");
2270 byte [] qf1 = Bytes.toBytes("qualifier1");
2271 byte [] qf2 = Bytes.toBytes("qualifier2");
2272 byte [] fam1 = Bytes.toBytes("fam1");
2273 byte [][] families = {fam1};
2274
2275 long ts1 = System.currentTimeMillis();
2276 long ts2 = ts1 + 1;
2277 long ts3 = ts1 + 2;
2278
2279
2280 String method = this.getName();
2281 this.region = initHRegion(tableName, method, conf, families);
2282 try {
2283
2284 Put put = null;
2285 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2286 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2287 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2288
2289 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2290 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2291 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2292
2293 put = new Put(row1);
2294 put.add(kv13);
2295 put.add(kv12);
2296 put.add(kv11);
2297 put.add(kv23);
2298 put.add(kv22);
2299 put.add(kv21);
2300 region.put(put);
2301
2302
2303 List<KeyValue> expected = new ArrayList<KeyValue>();
2304 expected.add(kv13);
2305 expected.add(kv12);
2306 expected.add(kv23);
2307 expected.add(kv22);
2308
2309 Scan scan = new Scan(row1);
2310 scan.addFamily(fam1);
2311 scan.setMaxVersions(MAX_VERSIONS);
2312 List<KeyValue> actual = new ArrayList<KeyValue>();
2313 InternalScanner scanner = region.getScanner(scan);
2314
2315 boolean hasNext = scanner.next(actual);
2316 assertEquals(false, hasNext);
2317
2318
2319 for(int i=0; i<expected.size(); i++) {
2320 assertEquals(expected.get(i), actual.get(i));
2321 }
2322 } finally {
2323 HRegion.closeHRegion(this.region);
2324 this.region = null;
2325 }
2326 }
2327
2328 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions()
2329 throws IOException{
2330 byte [] tableName = Bytes.toBytes("testtable");
2331 byte [] row1 = Bytes.toBytes("row1");
2332 byte [] qf1 = Bytes.toBytes("qualifier1");
2333 byte [] qf2 = Bytes.toBytes("qualifier2");
2334 byte [] fam1 = Bytes.toBytes("fam1");
2335
2336 long ts1 = 1;
2337 long ts2 = ts1 + 1;
2338 long ts3 = ts1 + 2;
2339
2340
2341 String method = this.getName();
2342 this.region = initHRegion(tableName, method, conf, fam1);
2343 try {
2344
2345 Put put = null;
2346 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2347 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2348 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2349
2350 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2351 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2352 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2353
2354 put = new Put(row1);
2355 put.add(kv13);
2356 put.add(kv12);
2357 put.add(kv11);
2358 put.add(kv23);
2359 put.add(kv22);
2360 put.add(kv21);
2361 region.put(put);
2362 region.flushcache();
2363
2364
2365 List<KeyValue> expected = new ArrayList<KeyValue>();
2366 expected.add(kv13);
2367 expected.add(kv12);
2368 expected.add(kv23);
2369 expected.add(kv22);
2370
2371 Scan scan = new Scan(row1);
2372 scan.addFamily(fam1);
2373 scan.setMaxVersions(MAX_VERSIONS);
2374 List<KeyValue> actual = new ArrayList<KeyValue>();
2375 InternalScanner scanner = region.getScanner(scan);
2376
2377 boolean hasNext = scanner.next(actual);
2378 assertEquals(false, hasNext);
2379
2380
2381 for(int i=0; i<expected.size(); i++) {
2382 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2383 }
2384 } finally {
2385 HRegion.closeHRegion(this.region);
2386 this.region = null;
2387 }
2388 }
2389
2390 public void testScanner_StopRow1542() throws IOException {
2391 byte [] tableName = Bytes.toBytes("test_table");
2392 byte [] family = Bytes.toBytes("testFamily");
2393 this.region = initHRegion(tableName, getName(), conf, family);
2394 try {
2395 byte [] row1 = Bytes.toBytes("row111");
2396 byte [] row2 = Bytes.toBytes("row222");
2397 byte [] row3 = Bytes.toBytes("row333");
2398 byte [] row4 = Bytes.toBytes("row444");
2399 byte [] row5 = Bytes.toBytes("row555");
2400
2401 byte [] col1 = Bytes.toBytes("Pub111");
2402 byte [] col2 = Bytes.toBytes("Pub222");
2403
2404
2405 Put put = new Put(row1);
2406 put.add(family, col1, Bytes.toBytes(10L));
2407 region.put(put);
2408
2409 put = new Put(row2);
2410 put.add(family, col1, Bytes.toBytes(15L));
2411 region.put(put);
2412
2413 put = new Put(row3);
2414 put.add(family, col2, Bytes.toBytes(20L));
2415 region.put(put);
2416
2417 put = new Put(row4);
2418 put.add(family, col2, Bytes.toBytes(30L));
2419 region.put(put);
2420
2421 put = new Put(row5);
2422 put.add(family, col1, Bytes.toBytes(40L));
2423 region.put(put);
2424
2425 Scan scan = new Scan(row3, row4);
2426 scan.setMaxVersions();
2427 scan.addColumn(family, col1);
2428 InternalScanner s = region.getScanner(scan);
2429
2430 List<KeyValue> results = new ArrayList<KeyValue>();
2431 assertEquals(false, s.next(results));
2432 assertEquals(0, results.size());
2433 } finally {
2434 HRegion.closeHRegion(this.region);
2435 this.region = null;
2436 }
2437 }
2438
2439 private void assertICV(byte [] row,
2440 byte [] familiy,
2441 byte[] qualifier,
2442 long amount) throws IOException {
2443
2444 Get get = new Get(row);
2445 get.addColumn(familiy, qualifier);
2446 Result result = region.get(get);
2447 assertEquals(1, result.size());
2448
2449 KeyValue kv = result.raw()[0];
2450 long r = Bytes.toLong(kv.getValue());
2451 assertEquals(amount, r);
2452 }
2453
2454 private void assertICV(byte [] row,
2455 byte [] familiy,
2456 byte[] qualifier,
2457 int amount) throws IOException {
2458
2459 Get get = new Get(row);
2460 get.addColumn(familiy, qualifier);
2461 Result result = region.get(get);
2462 assertEquals(1, result.size());
2463
2464 KeyValue kv = result.raw()[0];
2465 int r = Bytes.toInt(kv.getValue());
2466 assertEquals(amount, r);
2467 }
2468
2469 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions()
2470 throws IOException {
2471 byte [] tableName = Bytes.toBytes("testtable");
2472 byte [] row1 = Bytes.toBytes("row1");
2473 byte [] fam1 = Bytes.toBytes("fam1");
2474 byte [] qf1 = Bytes.toBytes("qualifier1");
2475 byte [] qf2 = Bytes.toBytes("quateslifier2");
2476
2477 long ts1 = 1;
2478 long ts2 = ts1 + 1;
2479 long ts3 = ts1 + 2;
2480 long ts4 = ts1 + 3;
2481
2482
2483 String method = this.getName();
2484 this.region = initHRegion(tableName, method, conf, fam1);
2485 try {
2486
2487 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2488 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2489 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2490 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2491
2492 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2493 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2494 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2495 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2496
2497 Put put = null;
2498 put = new Put(row1);
2499 put.add(kv14);
2500 put.add(kv24);
2501 region.put(put);
2502 region.flushcache();
2503
2504 put = new Put(row1);
2505 put.add(kv23);
2506 put.add(kv13);
2507 region.put(put);
2508 region.flushcache();
2509
2510 put = new Put(row1);
2511 put.add(kv22);
2512 put.add(kv12);
2513 region.put(put);
2514 region.flushcache();
2515
2516 put = new Put(row1);
2517 put.add(kv21);
2518 put.add(kv11);
2519 region.put(put);
2520
2521
2522 List<KeyValue> expected = new ArrayList<KeyValue>();
2523 expected.add(kv14);
2524 expected.add(kv13);
2525 expected.add(kv12);
2526 expected.add(kv24);
2527 expected.add(kv23);
2528 expected.add(kv22);
2529
2530 Scan scan = new Scan(row1);
2531 int versions = 3;
2532 scan.setMaxVersions(versions);
2533 List<KeyValue> actual = new ArrayList<KeyValue>();
2534 InternalScanner scanner = region.getScanner(scan);
2535
2536 boolean hasNext = scanner.next(actual);
2537 assertEquals(false, hasNext);
2538
2539
2540 for(int i=0; i<expected.size(); i++) {
2541 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2542 }
2543 } finally {
2544 HRegion.closeHRegion(this.region);
2545 this.region = null;
2546 }
2547 }
2548
2549
2550
2551
2552
2553
2554
2555 public void testScanner_JoinedScanners() throws IOException {
2556 byte [] tableName = Bytes.toBytes("testTable");
2557 byte [] cf_essential = Bytes.toBytes("essential");
2558 byte [] cf_joined = Bytes.toBytes("joined");
2559 byte [] cf_alpha = Bytes.toBytes("alpha");
2560 this.region = initHRegion(tableName, getName(), conf, cf_essential, cf_joined, cf_alpha);
2561 try {
2562 byte [] row1 = Bytes.toBytes("row1");
2563 byte [] row2 = Bytes.toBytes("row2");
2564 byte [] row3 = Bytes.toBytes("row3");
2565
2566 byte [] col_normal = Bytes.toBytes("d");
2567 byte [] col_alpha = Bytes.toBytes("a");
2568
2569 byte [] filtered_val = Bytes.toBytes(3);
2570
2571 Put put = new Put(row1);
2572 put.add(cf_essential, col_normal, Bytes.toBytes(1));
2573 put.add(cf_joined, col_alpha, Bytes.toBytes(1));
2574 region.put(put);
2575
2576 put = new Put(row2);
2577 put.add(cf_essential, col_alpha, Bytes.toBytes(2));
2578 put.add(cf_joined, col_normal, Bytes.toBytes(2));
2579 put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
2580 region.put(put);
2581
2582 put = new Put(row3);
2583 put.add(cf_essential, col_normal, filtered_val);
2584 put.add(cf_joined, col_normal, filtered_val);
2585 region.put(put);
2586
2587
2588
2589
2590
2591 Scan scan = new Scan();
2592 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
2593 CompareOp.NOT_EQUAL, filtered_val);
2594 scan.setFilter(filter);
2595 scan.setLoadColumnFamiliesOnDemand(true);
2596 InternalScanner s = region.getScanner(scan);
2597
2598 List<KeyValue> results = new ArrayList<KeyValue>();
2599 assertTrue(s.next(results));
2600 assertEquals(results.size(), 1);
2601 results.clear();
2602
2603 assertTrue(s.next(results));
2604 assertEquals(results.size(), 3);
2605 assertTrue("orderCheck", results.get(0).matchingFamily(cf_alpha));
2606 assertTrue("orderCheck", results.get(1).matchingFamily(cf_essential));
2607 assertTrue("orderCheck", results.get(2).matchingFamily(cf_joined));
2608 results.clear();
2609
2610 assertFalse(s.next(results));
2611 assertEquals(results.size(), 0);
2612 } finally {
2613 HRegion.closeHRegion(this.region);
2614 this.region = null;
2615 }
2616 }
2617
2618
2619
2620
2621
2622
2623 public void testScanner_JoinedScannersWithLimits() throws IOException {
2624 final byte [] tableName = Bytes.toBytes("testTable");
2625 final byte [] cf_first = Bytes.toBytes("first");
2626 final byte [] cf_second = Bytes.toBytes("second");
2627
2628 this.region = initHRegion(tableName, getName(), conf, cf_first, cf_second);
2629 try {
2630 final byte [] col_a = Bytes.toBytes("a");
2631 final byte [] col_b = Bytes.toBytes("b");
2632
2633 Put put;
2634
2635 for (int i = 0; i < 10; i++) {
2636 put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
2637 put.add(cf_first, col_a, Bytes.toBytes(i));
2638 if (i < 5) {
2639 put.add(cf_first, col_b, Bytes.toBytes(i));
2640 put.add(cf_second, col_a, Bytes.toBytes(i));
2641 put.add(cf_second, col_b, Bytes.toBytes(i));
2642 }
2643 region.put(put);
2644 }
2645
2646 Scan scan = new Scan();
2647 scan.setLoadColumnFamiliesOnDemand(true);
2648 Filter bogusFilter = new FilterBase() {
2649 @Override
2650 public boolean isFamilyEssential(byte[] name) {
2651 return Bytes.equals(name, cf_first);
2652 }
2653 };
2654
2655 scan.setFilter(bogusFilter);
2656 InternalScanner s = region.getScanner(scan);
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687 List<KeyValue> results = new ArrayList<KeyValue>();
2688 int index = 0;
2689 while (true) {
2690 boolean more = s.next(results, 3);
2691 if ((index >> 1) < 5) {
2692 if (index % 2 == 0)
2693 assertEquals(results.size(), 3);
2694 else
2695 assertEquals(results.size(), 1);
2696 }
2697 else
2698 assertEquals(results.size(), 1);
2699 results.clear();
2700 index++;
2701 if (!more) break;
2702 }
2703 } finally {
2704 HRegion.closeHRegion(this.region);
2705 this.region = null;
2706 }
2707 }
2708
2709
2710
2711
2712
2713
2714
2715
2716 public void testBasicSplit() throws Exception {
2717 byte [] tableName = Bytes.toBytes("testtable");
2718 byte [][] families = {fam1, fam2, fam3};
2719
2720 Configuration hc = initSplit();
2721
2722 String method = this.getName();
2723 this.region = initHRegion(tableName, method, hc, families);
2724
2725 try {
2726 LOG.info("" + addContent(region, fam3));
2727 region.flushcache();
2728 region.compactStores();
2729 byte [] splitRow = region.checkSplit();
2730 assertNotNull(splitRow);
2731 LOG.info("SplitRow: " + Bytes.toString(splitRow));
2732 HRegion [] regions = splitRegion(region, splitRow);
2733 try {
2734
2735
2736
2737 for (int i = 0; i < regions.length; i++) {
2738 regions[i] = openClosedRegion(regions[i]);
2739 }
2740
2741
2742 assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
2743 assertGet(regions[1], fam3, splitRow);
2744
2745 assertScan(regions[0], fam3,
2746 Bytes.toBytes(START_KEY));
2747 assertScan(regions[1], fam3, splitRow);
2748
2749 for (int i = 0; i < regions.length; i++) {
2750
2751
2752 for (int j = 0; j < 2; j++) {
2753 addContent(regions[i], fam3);
2754 }
2755 addContent(regions[i], fam2);
2756 addContent(regions[i], fam1);
2757 regions[i].flushcache();
2758 }
2759
2760 byte [][] midkeys = new byte [regions.length][];
2761
2762 for (int i = 0; i < regions.length; i++) {
2763 regions[i].compactStores();
2764 midkeys[i] = regions[i].checkSplit();
2765 }
2766
2767 TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
2768
2769
2770 for (int i = 0; i < regions.length; i++) {
2771 HRegion[] rs = null;
2772 if (midkeys[i] != null) {
2773 rs = splitRegion(regions[i], midkeys[i]);
2774 for (int j = 0; j < rs.length; j++) {
2775 sortedMap.put(Bytes.toString(rs[j].getRegionName()),
2776 openClosedRegion(rs[j]));
2777 }
2778 }
2779 }
2780 LOG.info("Made 4 regions");
2781
2782
2783 int interval = (LAST_CHAR - FIRST_CHAR) / 3;
2784 byte[] b = Bytes.toBytes(START_KEY);
2785 for (HRegion r : sortedMap.values()) {
2786 assertGet(r, fam3, b);
2787 b[0] += interval;
2788 }
2789 } finally {
2790 for (int i = 0; i < regions.length; i++) {
2791 try {
2792 regions[i].close();
2793 } catch (IOException e) {
2794
2795 }
2796 }
2797 }
2798 } finally {
2799 HRegion.closeHRegion(this.region);
2800 this.region = null;
2801 }
2802 }
2803
2804 public void testSplitRegion() throws IOException {
2805 byte [] tableName = Bytes.toBytes("testtable");
2806 byte [] qualifier = Bytes.toBytes("qualifier");
2807 Configuration hc = initSplit();
2808 int numRows = 10;
2809 byte [][] families = {fam1, fam3};
2810
2811
2812 String method = this.getName();
2813 this.region = initHRegion(tableName, method, hc, families);
2814
2815
2816 int startRow = 100;
2817 putData(startRow, numRows, qualifier, families);
2818 int splitRow = startRow + numRows;
2819 putData(splitRow, numRows, qualifier, families);
2820 region.flushcache();
2821
2822 HRegion [] regions = null;
2823 try {
2824 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
2825
2826 for (int i = 0; i < regions.length; i++) {
2827 regions[i] = openClosedRegion(regions[i]);
2828 }
2829
2830 assertEquals(2, regions.length);
2831
2832
2833
2834 verifyData(regions[0], startRow, numRows, qualifier, families);
2835 verifyData(regions[1], splitRow, numRows, qualifier, families);
2836
2837 } finally {
2838 HRegion.closeHRegion(this.region);
2839 this.region = null;
2840 }
2841 }
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851 public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
2852 byte[] tableName = Bytes.toBytes("testFlushCacheWhileScanning");
2853 byte[] family = Bytes.toBytes("family");
2854 int numRows = 1000;
2855 int flushAndScanInterval = 10;
2856 int compactInterval = 10 * flushAndScanInterval;
2857
2858 String method = "testFlushCacheWhileScanning";
2859 this.region = initHRegion(tableName,method, conf, family);
2860 try {
2861 FlushThread flushThread = new FlushThread();
2862 flushThread.start();
2863
2864 Scan scan = new Scan();
2865 scan.addFamily(family);
2866 scan.setFilter(new SingleColumnValueFilter(family, qual1,
2867 CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(5L))));
2868
2869 int expectedCount = 0;
2870 List<KeyValue> res = new ArrayList<KeyValue>();
2871
2872 boolean toggle=true;
2873 for (long i = 0; i < numRows; i++) {
2874 Put put = new Put(Bytes.toBytes(i));
2875 put.setDurability(Durability.SKIP_WAL);
2876 put.add(family, qual1, Bytes.toBytes(i % 10));
2877 region.put(put);
2878
2879 if (i != 0 && i % compactInterval == 0) {
2880
2881 region.compactStores(true);
2882 }
2883
2884 if (i % 10 == 5L) {
2885 expectedCount++;
2886 }
2887
2888 if (i != 0 && i % flushAndScanInterval == 0) {
2889 res.clear();
2890 InternalScanner scanner = region.getScanner(scan);
2891 if (toggle) {
2892 flushThread.flush();
2893 }
2894 while (scanner.next(res)) ;
2895 if (!toggle) {
2896 flushThread.flush();
2897 }
2898 assertEquals("i=" + i, expectedCount, res.size());
2899 toggle = !toggle;
2900 }
2901 }
2902
2903 flushThread.done();
2904 flushThread.join();
2905 flushThread.checkNoError();
2906 } finally {
2907 HRegion.closeHRegion(this.region);
2908 this.region = null;
2909 }
2910 }
2911
2912 protected class FlushThread extends Thread {
2913 private volatile boolean done;
2914 private Throwable error = null;
2915
2916 public void done() {
2917 done = true;
2918 synchronized (this) {
2919 interrupt();
2920 }
2921 }
2922
2923 public void checkNoError() {
2924 if (error != null) {
2925 assertNull(error);
2926 }
2927 }
2928
2929 @Override
2930 public void run() {
2931 done = false;
2932 while (!done) {
2933 synchronized (this) {
2934 try {
2935 wait();
2936 } catch (InterruptedException ignored) {
2937 if (done) {
2938 break;
2939 }
2940 }
2941 }
2942 try {
2943 region.flushcache();
2944 } catch (IOException e) {
2945 if (!done) {
2946 LOG.error("Error while flusing cache", e);
2947 error = e;
2948 }
2949 break;
2950 }
2951 }
2952
2953 }
2954
2955 public void flush() {
2956 synchronized (this) {
2957 notify();
2958 }
2959
2960 }
2961 }
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971 public void testWritesWhileScanning()
2972 throws IOException, InterruptedException {
2973 byte[] tableName = Bytes.toBytes("testWritesWhileScanning");
2974 int testCount = 100;
2975 int numRows = 1;
2976 int numFamilies = 10;
2977 int numQualifiers = 100;
2978 int flushInterval = 7;
2979 int compactInterval = 5 * flushInterval;
2980 byte[][] families = new byte[numFamilies][];
2981 for (int i = 0; i < numFamilies; i++) {
2982 families[i] = Bytes.toBytes("family" + i);
2983 }
2984 byte[][] qualifiers = new byte[numQualifiers][];
2985 for (int i = 0; i < numQualifiers; i++) {
2986 qualifiers[i] = Bytes.toBytes("qual" + i);
2987 }
2988
2989 String method = "testWritesWhileScanning";
2990 this.region = initHRegion(tableName, method, conf, families);
2991 try {
2992 PutThread putThread = new PutThread(numRows, families, qualifiers);
2993 putThread.start();
2994 putThread.waitForFirstPut();
2995
2996 FlushThread flushThread = new FlushThread();
2997 flushThread.start();
2998
2999 Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3000
3001
3002
3003 int expectedCount = numFamilies * numQualifiers;
3004 List<KeyValue> res = new ArrayList<KeyValue>();
3005
3006 long prevTimestamp = 0L;
3007 for (int i = 0; i < testCount; i++) {
3008
3009 if (i != 0 && i % compactInterval == 0) {
3010 region.compactStores(true);
3011 }
3012
3013 if (i != 0 && i % flushInterval == 0) {
3014
3015 flushThread.flush();
3016 }
3017
3018 boolean previousEmpty = res.isEmpty();
3019 res.clear();
3020 InternalScanner scanner = region.getScanner(scan);
3021 while (scanner.next(res)) ;
3022 if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3023 assertEquals("i=" + i, expectedCount, res.size());
3024 long timestamp = res.get(0).getTimestamp();
3025 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3026 timestamp >= prevTimestamp);
3027 prevTimestamp = timestamp;
3028 }
3029 }
3030
3031 putThread.done();
3032
3033 region.flushcache();
3034
3035 putThread.join();
3036 putThread.checkNoError();
3037
3038 flushThread.done();
3039 flushThread.join();
3040 flushThread.checkNoError();
3041 } finally {
3042 HRegion.closeHRegion(this.region);
3043 this.region = null;
3044 }
3045 }
3046
3047 protected class PutThread extends Thread {
3048 private volatile boolean done;
3049 private volatile int numPutsFinished = 0;
3050
3051 private Throwable error = null;
3052 private int numRows;
3053 private byte[][] families;
3054 private byte[][] qualifiers;
3055
3056 private PutThread(int numRows, byte[][] families,
3057 byte[][] qualifiers) {
3058 this.numRows = numRows;
3059 this.families = families;
3060 this.qualifiers = qualifiers;
3061 }
3062
3063
3064
3065
3066 public void waitForFirstPut() throws InterruptedException {
3067
3068 while (numPutsFinished == 0) {
3069 checkNoError();
3070 Thread.sleep(50);
3071 }
3072 }
3073
3074 public void done() {
3075 done = true;
3076 synchronized (this) {
3077 interrupt();
3078 }
3079 }
3080
3081 public void checkNoError() {
3082 if (error != null) {
3083 assertNull(error);
3084 }
3085 }
3086
3087 @Override
3088 public void run() {
3089 done = false;
3090 while (!done) {
3091 try {
3092 for (int r = 0; r < numRows; r++) {
3093 byte[] row = Bytes.toBytes("row" + r);
3094 Put put = new Put(row);
3095 put.setDurability(Durability.SKIP_WAL);
3096 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3097 for (byte[] family : families) {
3098 for (byte[] qualifier : qualifiers) {
3099 put.add(family, qualifier, (long) numPutsFinished, value);
3100 }
3101 }
3102
3103 region.put(put);
3104 numPutsFinished++;
3105 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3106 System.out.println("put iteration = " + numPutsFinished);
3107 Delete delete = new Delete(row, (long)numPutsFinished-30);
3108 region.delete(delete);
3109 }
3110 numPutsFinished++;
3111 }
3112 } catch (InterruptedIOException e) {
3113
3114 } catch (IOException e) {
3115 LOG.error("error while putting records", e);
3116 error = e;
3117 break;
3118 }
3119 }
3120
3121 }
3122
3123 }
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133 public void testWritesWhileGetting()
3134 throws Exception {
3135 byte[] tableName = Bytes.toBytes("testWritesWhileGetting");
3136 int testCount = 100;
3137 int numRows = 1;
3138 int numFamilies = 10;
3139 int numQualifiers = 100;
3140 int compactInterval = 100;
3141 byte[][] families = new byte[numFamilies][];
3142 for (int i = 0; i < numFamilies; i++) {
3143 families[i] = Bytes.toBytes("family" + i);
3144 }
3145 byte[][] qualifiers = new byte[numQualifiers][];
3146 for (int i = 0; i < numQualifiers; i++) {
3147 qualifiers[i] = Bytes.toBytes("qual" + i);
3148 }
3149
3150 Configuration conf = HBaseConfiguration.create(this.conf);
3151
3152 String method = "testWritesWhileGetting";
3153
3154
3155
3156 conf.setInt("hbase.hstore.compaction.min", 1);
3157 conf.setInt("hbase.hstore.compaction.max", 1000);
3158 this.region = initHRegion(tableName, method, conf, families);
3159 PutThread putThread = null;
3160 MultithreadedTestUtil.TestContext ctx =
3161 new MultithreadedTestUtil.TestContext(conf);
3162 try {
3163 putThread = new PutThread(numRows, families, qualifiers);
3164 putThread.start();
3165 putThread.waitForFirstPut();
3166
3167
3168 ctx.addThread(new RepeatingTestThread(ctx) {
3169 private int flushesSinceCompact = 0;
3170 private final int maxFlushesSinceCompact = 20;
3171 @Override
3172 public void doAnAction() throws Exception {
3173 if (region.flushcache()) {
3174 ++flushesSinceCompact;
3175 }
3176
3177 if (flushesSinceCompact == maxFlushesSinceCompact) {
3178 region.compactStores(false);
3179 flushesSinceCompact = 0;
3180 }
3181 }
3182 });
3183 ctx.startThreads();
3184
3185 Get get = new Get(Bytes.toBytes("row0"));
3186 Result result = null;
3187
3188 int expectedCount = numFamilies * numQualifiers;
3189
3190 long prevTimestamp = 0L;
3191 for (int i = 0; i < testCount; i++) {
3192
3193 boolean previousEmpty = result == null || result.isEmpty();
3194 result = region.get(get);
3195 if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3196 assertEquals("i=" + i, expectedCount, result.size());
3197
3198
3199 long timestamp = 0;
3200 for (KeyValue kv : result.raw()) {
3201 if (Bytes.equals(kv.getFamily(), families[0])
3202 && Bytes.equals(kv.getQualifier(), qualifiers[0])) {
3203 timestamp = kv.getTimestamp();
3204 }
3205 }
3206 assertTrue(timestamp >= prevTimestamp);
3207 prevTimestamp = timestamp;
3208 KeyValue previousKV = null;
3209
3210 for (KeyValue kv : result.raw()) {
3211 byte[] thisValue = kv.getValue();
3212 if (previousKV != null) {
3213 if (Bytes.compareTo(previousKV.getValue(), thisValue) != 0) {
3214 LOG.warn("These two KV should have the same value." +
3215 " Previous KV:" +
3216 previousKV + "(memStoreTS:" + previousKV.getMvccVersion() + ")" +
3217 ", New KV: " +
3218 kv + "(memStoreTS:" + kv.getMvccVersion() + ")"
3219 );
3220 assertEquals(0, Bytes.compareTo(previousKV.getValue(), thisValue));
3221 }
3222 }
3223 previousKV = kv;
3224 }
3225 }
3226 }
3227 } finally {
3228 if (putThread != null) putThread.done();
3229
3230 region.flushcache();
3231
3232 if (putThread != null) {
3233 putThread.join();
3234 putThread.checkNoError();
3235 }
3236
3237 ctx.stop();
3238 HRegion.closeHRegion(this.region);
3239 this.region = null;
3240 }
3241 }
3242
3243 public void testHolesInMeta() throws Exception {
3244 String method = "testHolesInMeta";
3245 byte[] tableName = Bytes.toBytes(method);
3246 byte[] family = Bytes.toBytes("family");
3247 this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method,
3248 conf, false, family);
3249 try {
3250 byte[] rowNotServed = Bytes.toBytes("a");
3251 Get g = new Get(rowNotServed);
3252 try {
3253 region.get(g);
3254 fail();
3255 } catch (WrongRegionException x) {
3256
3257 }
3258 byte[] row = Bytes.toBytes("y");
3259 g = new Get(row);
3260 region.get(g);
3261 } finally {
3262 HRegion.closeHRegion(this.region);
3263 this.region = null;
3264 }
3265 }
3266
3267 public void testIndexesScanWithOneDeletedRow() throws IOException {
3268 byte[] tableName = Bytes.toBytes("testIndexesScanWithOneDeletedRow");
3269 byte[] family = Bytes.toBytes("family");
3270
3271
3272 String method = "testIndexesScanWithOneDeletedRow";
3273 this.region = initHRegion(tableName, method, conf, family);
3274 try {
3275 Put put = new Put(Bytes.toBytes(1L));
3276 put.add(family, qual1, 1L, Bytes.toBytes(1L));
3277 region.put(put);
3278
3279 region.flushcache();
3280
3281 Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3282
3283 region.delete(delete);
3284
3285 put = new Put(Bytes.toBytes(2L));
3286 put.add(family, qual1, 2L, Bytes.toBytes(2L));
3287 region.put(put);
3288
3289 Scan idxScan = new Scan();
3290 idxScan.addFamily(family);
3291 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,
3292 Arrays.<Filter>asList(new SingleColumnValueFilter(family, qual1,
3293 CompareOp.GREATER_OR_EQUAL,
3294 new BinaryComparator(Bytes.toBytes(0L))),
3295 new SingleColumnValueFilter(family, qual1, CompareOp.LESS_OR_EQUAL,
3296 new BinaryComparator(Bytes.toBytes(3L)))
3297 )));
3298 InternalScanner scanner = region.getScanner(idxScan);
3299 List<KeyValue> res = new ArrayList<KeyValue>();
3300
3301
3302 while (scanner.next(res)) ;
3303
3304
3305 assertEquals(1L, res.size());
3306 } finally {
3307 HRegion.closeHRegion(this.region);
3308 this.region = null;
3309 }
3310 }
3311
3312
3313
3314
3315 public void testBloomFilterSize() throws IOException {
3316 byte [] tableName = Bytes.toBytes("testBloomFilterSize");
3317 byte [] row1 = Bytes.toBytes("row1");
3318 byte [] fam1 = Bytes.toBytes("fam1");
3319 byte [] qf1 = Bytes.toBytes("col");
3320 byte [] val1 = Bytes.toBytes("value1");
3321
3322 HColumnDescriptor hcd = new HColumnDescriptor(fam1)
3323 .setMaxVersions(Integer.MAX_VALUE)
3324 .setBloomFilterType(BloomType.ROWCOL);
3325
3326 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3327 htd.addFamily(hcd);
3328 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3329 Path path = new Path(DIR + "testBloomFilterSize");
3330 this.region = HRegion.createHRegion(info, path, conf, htd);
3331 try {
3332 int num_unique_rows = 10;
3333 int duplicate_multiplier =2;
3334 int num_storefiles = 4;
3335
3336 int version = 0;
3337 for (int f =0 ; f < num_storefiles; f++) {
3338 for (int i = 0; i < duplicate_multiplier; i ++) {
3339 for (int j = 0; j < num_unique_rows; j++) {
3340 Put put = new Put(Bytes.toBytes("row" + j));
3341 put.setDurability(Durability.SKIP_WAL);
3342 put.add(fam1, qf1, version++, val1);
3343 region.put(put);
3344 }
3345 }
3346 region.flushcache();
3347 }
3348
3349 HStore store = (HStore) region.getStore(fam1);
3350 Collection<StoreFile> storeFiles = store.getStorefiles();
3351 for (StoreFile storefile : storeFiles) {
3352 StoreFile.Reader reader = storefile.getReader();
3353 reader.loadFileInfo();
3354 reader.loadBloomfilter();
3355 assertEquals(num_unique_rows*duplicate_multiplier, reader.getEntries());
3356 assertEquals(num_unique_rows, reader.getFilterEntries());
3357 }
3358
3359 region.compactStores(true);
3360
3361
3362 storeFiles = store.getStorefiles();
3363 for (StoreFile storefile : storeFiles) {
3364 StoreFile.Reader reader = storefile.getReader();
3365 reader.loadFileInfo();
3366 reader.loadBloomfilter();
3367 assertEquals(num_unique_rows*duplicate_multiplier*num_storefiles,
3368 reader.getEntries());
3369 assertEquals(num_unique_rows, reader.getFilterEntries());
3370 }
3371 } finally {
3372 HRegion.closeHRegion(this.region);
3373 this.region = null;
3374 }
3375 }
3376
3377 public void testAllColumnsWithBloomFilter() throws IOException {
3378 byte [] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
3379 byte [] FAMILY = Bytes.toBytes("family");
3380
3381
3382 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY)
3383 .setMaxVersions(Integer.MAX_VALUE)
3384 .setBloomFilterType(BloomType.ROWCOL);
3385 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
3386 htd.addFamily(hcd);
3387 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3388 Path path = new Path(DIR + "testAllColumnsWithBloomFilter");
3389 this.region = HRegion.createHRegion(info, path, conf, htd);
3390 try {
3391
3392 byte row[] = Bytes.toBytes("row:" + 0);
3393 byte column[] = Bytes.toBytes("column:" + 0);
3394 Put put = new Put(row);
3395 put.setDurability(Durability.SKIP_WAL);
3396 for (long idx = 1; idx <= 4; idx++) {
3397 put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
3398 }
3399 region.put(put);
3400
3401
3402 region.flushcache();
3403
3404
3405 Get get = new Get(row);
3406 get.setMaxVersions();
3407 KeyValue[] kvs = region.get(get).raw();
3408
3409
3410 assertEquals(4, kvs.length);
3411 checkOneCell(kvs[0], FAMILY, 0, 0, 4);
3412 checkOneCell(kvs[1], FAMILY, 0, 0, 3);
3413 checkOneCell(kvs[2], FAMILY, 0, 0, 2);
3414 checkOneCell(kvs[3], FAMILY, 0, 0, 1);
3415 } finally {
3416 HRegion.closeHRegion(this.region);
3417 this.region = null;
3418 }
3419 }
3420
3421
3422
3423
3424
3425
3426 public void testDeleteRowWithBloomFilter() throws IOException {
3427 byte [] tableName = Bytes.toBytes("testDeleteRowWithBloomFilter");
3428 byte [] familyName = Bytes.toBytes("familyName");
3429
3430
3431 HColumnDescriptor hcd = new HColumnDescriptor(familyName)
3432 .setMaxVersions(Integer.MAX_VALUE)
3433 .setBloomFilterType(BloomType.ROWCOL);
3434
3435 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3436 htd.addFamily(hcd);
3437 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3438 Path path = new Path(DIR + "TestDeleteRowWithBloomFilter");
3439 this.region = HRegion.createHRegion(info, path, conf, htd);
3440 try {
3441
3442 byte row[] = Bytes.toBytes("row1");
3443 byte col[] = Bytes.toBytes("col1");
3444
3445 Put put = new Put(row);
3446 put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
3447 region.put(put);
3448 region.flushcache();
3449
3450 Delete del = new Delete(row);
3451 region.delete(del);
3452 region.flushcache();
3453
3454
3455 Get get = new Get(row);
3456 get.addColumn(familyName, col);
3457
3458 KeyValue[] keyValues = region.get(get).raw();
3459 assertTrue(keyValues.length == 0);
3460 } finally {
3461 HRegion.closeHRegion(this.region);
3462 this.region = null;
3463 }
3464 }
3465
3466 @Test public void testgetHDFSBlocksDistribution() throws Exception {
3467 HBaseTestingUtility htu = new HBaseTestingUtility();
3468 final int DEFAULT_BLOCK_SIZE = 1024;
3469 htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
3470 htu.getConfiguration().setInt("dfs.replication", 2);
3471
3472
3473
3474 MiniHBaseCluster cluster = null;
3475 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
3476 int regionServersCount = 3;
3477
3478 try {
3479 cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
3480 byte [][] families = {fam1, fam2};
3481 HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);
3482
3483
3484 byte row[] = Bytes.toBytes("row1");
3485 byte col[] = Bytes.toBytes("col1");
3486
3487 Put put = new Put(row);
3488 put.add(fam1, col, 1, Bytes.toBytes("test1"));
3489 put.add(fam2, col, 1, Bytes.toBytes("test2"));
3490 ht.put(put);
3491
3492 HRegion firstRegion = htu.getHBaseCluster().
3493 getRegions(TableName.valueOf(this.getName())).get(0);
3494 firstRegion.flushcache();
3495 HDFSBlocksDistribution blocksDistribution1 =
3496 firstRegion.getHDFSBlocksDistribution();
3497
3498
3499
3500
3501
3502 long uniqueBlocksWeight1 =
3503 blocksDistribution1.getUniqueBlocksTotalWeight();
3504
3505 String topHost = blocksDistribution1.getTopHosts().get(0);
3506 long topHostWeight = blocksDistribution1.getWeight(topHost);
3507 assertTrue(uniqueBlocksWeight1 == topHostWeight);
3508
3509
3510
3511 HDFSBlocksDistribution blocksDistribution2 =
3512 HRegion.computeHDFSBlocksDistribution(htu.getConfiguration(),
3513 firstRegion.getTableDesc(), firstRegion.getRegionInfo());
3514 long uniqueBlocksWeight2 =
3515 blocksDistribution2.getUniqueBlocksTotalWeight();
3516
3517 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
3518
3519 ht.close();
3520 } finally {
3521 if (cluster != null) {
3522 htu.shutdownMiniCluster();
3523 }
3524 }
3525 }
3526
3527
3528
3529
3530
3531
3532
3533 @Test
3534 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
3535 HRegionInfo info = null;
3536 try {
3537 FileSystem fs = Mockito.mock(FileSystem.class);
3538 Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
3539 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3540 htd.addFamily(new HColumnDescriptor("cf"));
3541 info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
3542 HConstants.EMPTY_BYTE_ARRAY, false);
3543 Path path = new Path(DIR + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
3544 region = HRegion.newHRegion(path, null, fs, conf, info, htd, null);
3545
3546 region.initialize();
3547 fail("Region initialization should fail due to IOException");
3548 } catch (IOException io) {
3549 List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
3550 for (MonitoredTask monitoredTask : tasks) {
3551 if (!(monitoredTask instanceof MonitoredRPCHandler)
3552 && monitoredTask.getDescription().contains(region.toString())) {
3553 assertTrue("Region state should be ABORTED.",
3554 monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
3555 break;
3556 }
3557 }
3558 } finally {
3559 HRegion.closeHRegion(region);
3560 }
3561 }
3562
3563
3564
3565
3566
3567 public void testRegionInfoFileCreation() throws IOException {
3568 Path rootDir = new Path(DIR + "testRegionInfoFileCreation");
3569 Configuration conf = HBaseConfiguration.create(this.conf);
3570
3571 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
3572 htd.addFamily(new HColumnDescriptor("cf"));
3573
3574 HRegionInfo hri = new HRegionInfo(htd.getTableName());
3575
3576
3577 HRegion region = HRegion.createHRegion(hri, rootDir, conf, htd, null, false, true);
3578 Path regionDir = region.getRegionFileSystem().getRegionDir();
3579 FileSystem fs = region.getRegionFileSystem().getFileSystem();
3580 HRegion.closeHRegion(region);
3581
3582 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
3583
3584
3585 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3586 fs.exists(regionInfoFile));
3587
3588
3589 region = HRegion.openHRegion(rootDir, hri, htd, null, conf);
3590 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3591 HRegion.closeHRegion(region);
3592
3593
3594 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3595 fs.exists(regionInfoFile));
3596
3597
3598 fs.delete(regionInfoFile);
3599 assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
3600 fs.exists(regionInfoFile));
3601
3602 region = HRegion.openHRegion(rootDir, hri, htd, null, conf);
3603 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3604 HRegion.closeHRegion(region);
3605
3606
3607 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3608 fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
3609 }
3610
3611
3612
3613
3614
3615 private static class Incrementer implements Runnable {
3616 private HRegion region;
3617 private final static byte[] incRow = Bytes.toBytes("incRow");
3618 private final static byte[] family = Bytes.toBytes("family");
3619 private final static byte[] qualifier = Bytes.toBytes("qualifier");
3620 private final static long ONE = 1l;
3621 private int incCounter;
3622
3623 public Incrementer(HRegion region, int incCounter) {
3624 this.region = region;
3625 this.incCounter = incCounter;
3626 }
3627
3628 @Override
3629 public void run() {
3630 int count = 0;
3631 while (count < incCounter) {
3632 Increment inc = new Increment(incRow);
3633 inc.addColumn(family, qualifier, ONE);
3634 count++;
3635 try {
3636 region.increment(inc);
3637 } catch (IOException e) {
3638 e.printStackTrace();
3639 break;
3640 }
3641 }
3642 }
3643 }
3644
3645
3646
3647
3648
3649 @Test
3650 public void testParallelIncrementWithMemStoreFlush() throws Exception {
3651 String method = "testParallelIncrementWithMemStoreFlush";
3652 byte[] tableName = Bytes.toBytes(method);
3653 byte[] family = Incrementer.family;
3654 this.region = initHRegion(tableName, method, conf, family);
3655 final HRegion region = this.region;
3656 final AtomicBoolean incrementDone = new AtomicBoolean(false);
3657 Runnable flusher = new Runnable() {
3658 @Override
3659 public void run() {
3660 while (!incrementDone.get()) {
3661 try {
3662 region.flushcache();
3663 } catch (Exception e) {
3664 e.printStackTrace();
3665 }
3666 }
3667 }
3668 };
3669
3670
3671 int threadNum = 20;
3672 int incCounter = 100;
3673 long expected = threadNum * incCounter;
3674 Thread[] incrementers = new Thread[threadNum];
3675 Thread flushThread = new Thread(flusher);
3676 for (int i = 0; i < threadNum; i++) {
3677 incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
3678 incrementers[i].start();
3679 }
3680 flushThread.start();
3681 for (int i = 0; i < threadNum; i++) {
3682 incrementers[i].join();
3683 }
3684
3685 incrementDone.set(true);
3686 flushThread.join();
3687
3688 Get get = new Get(Incrementer.incRow);
3689 get.addColumn(Incrementer.family, Incrementer.qualifier);
3690 get.setMaxVersions(1);
3691 Result res = this.region.get(get);
3692 List<KeyValue> kvs = res.getColumn(Incrementer.family,
3693 Incrementer.qualifier);
3694
3695
3696 assertEquals(kvs.size(), 1);
3697 KeyValue kv = kvs.get(0);
3698 assertEquals(expected, Bytes.toLong(kv.getBuffer(), kv.getValueOffset()));
3699 this.region = null;
3700 }
3701
3702
3703
3704
3705
3706 private static class Appender implements Runnable {
3707 private HRegion region;
3708 private final static byte[] appendRow = Bytes.toBytes("appendRow");
3709 private final static byte[] family = Bytes.toBytes("family");
3710 private final static byte[] qualifier = Bytes.toBytes("qualifier");
3711 private final static byte[] CHAR = Bytes.toBytes("a");
3712 private int appendCounter;
3713
3714 public Appender(HRegion region, int appendCounter) {
3715 this.region = region;
3716 this.appendCounter = appendCounter;
3717 }
3718
3719 @Override
3720 public void run() {
3721 int count = 0;
3722 while (count < appendCounter) {
3723 Append app = new Append(appendRow);
3724 app.add(family, qualifier, CHAR);
3725 count++;
3726 try {
3727 region.append(app);
3728 } catch (IOException e) {
3729 e.printStackTrace();
3730 break;
3731 }
3732 }
3733 }
3734 }
3735
3736
3737
3738
3739
3740 @Test
3741 public void testParallelAppendWithMemStoreFlush() throws Exception {
3742 String method = "testParallelAppendWithMemStoreFlush";
3743 byte[] tableName = Bytes.toBytes(method);
3744 byte[] family = Appender.family;
3745 this.region = initHRegion(tableName, method, conf, family);
3746 final HRegion region = this.region;
3747 final AtomicBoolean appendDone = new AtomicBoolean(false);
3748 Runnable flusher = new Runnable() {
3749 @Override
3750 public void run() {
3751 while (!appendDone.get()) {
3752 try {
3753 region.flushcache();
3754 } catch (Exception e) {
3755 e.printStackTrace();
3756 }
3757 }
3758 }
3759 };
3760
3761
3762 int threadNum = 20;
3763 int appendCounter = 100;
3764 byte[] expected = new byte[threadNum * appendCounter];
3765 for (int i = 0; i < threadNum * appendCounter; i++) {
3766 System.arraycopy(Appender.CHAR, 0, expected, i, 1);
3767 }
3768 Thread[] appenders = new Thread[threadNum];
3769 Thread flushThread = new Thread(flusher);
3770 for (int i = 0; i < threadNum; i++) {
3771 appenders[i] = new Thread(new Appender(this.region, appendCounter));
3772 appenders[i].start();
3773 }
3774 flushThread.start();
3775 for (int i = 0; i < threadNum; i++) {
3776 appenders[i].join();
3777 }
3778
3779 appendDone.set(true);
3780 flushThread.join();
3781
3782 Get get = new Get(Appender.appendRow);
3783 get.addColumn(Appender.family, Appender.qualifier);
3784 get.setMaxVersions(1);
3785 Result res = this.region.get(get);
3786 List<KeyValue> kvs = res.getColumn(Appender.family,
3787 Appender.qualifier);
3788
3789
3790 assertEquals(kvs.size(), 1);
3791 KeyValue kv = kvs.get(0);
3792 byte[] appendResult = new byte[kv.getValueLength()];
3793 System.arraycopy(kv.getBuffer(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
3794 assertEquals(expected, appendResult);
3795 this.region = null;
3796 }
3797
3798
3799
3800
3801
3802 public void testPutWithMemStoreFlush() throws Exception {
3803 String method = "testPutWithMemStoreFlush";
3804 byte[] tableName = Bytes.toBytes(method);
3805 byte[] family = Bytes.toBytes("family");;
3806 byte[] qualifier = Bytes.toBytes("qualifier");
3807 byte[] row = Bytes.toBytes("putRow");
3808 byte[] value = null;
3809 this.region = initHRegion(tableName, method, conf, family);
3810 Put put = null;
3811 Get get = null;
3812 List<KeyValue> kvs = null;
3813 Result res = null;
3814
3815 put = new Put(row);
3816 value = Bytes.toBytes("value0");
3817 put.add(family, qualifier, 1234567l, value);
3818 region.put(put);
3819 get = new Get(row);
3820 get.addColumn(family, qualifier);
3821 get.setMaxVersions();
3822 res = this.region.get(get);
3823 kvs = res.getColumn(family, qualifier);
3824 assertEquals(1, kvs.size());
3825 assertEquals(Bytes.toBytes("value0"), kvs.get(0).getValue());
3826
3827 region.flushcache();
3828 get = new Get(row);
3829 get.addColumn(family, qualifier);
3830 get.setMaxVersions();
3831 res = this.region.get(get);
3832 kvs = res.getColumn(family, qualifier);
3833 assertEquals(1, kvs.size());
3834 assertEquals(Bytes.toBytes("value0"), kvs.get(0).getValue());
3835
3836 put = new Put(row);
3837 value = Bytes.toBytes("value1");
3838 put.add(family, qualifier, 1234567l, value);
3839 region.put(put);
3840 get = new Get(row);
3841 get.addColumn(family, qualifier);
3842 get.setMaxVersions();
3843 res = this.region.get(get);
3844 kvs = res.getColumn(family, qualifier);
3845 assertEquals(1, kvs.size());
3846 assertEquals(Bytes.toBytes("value1"), kvs.get(0).getValue());
3847
3848 region.flushcache();
3849 get = new Get(row);
3850 get.addColumn(family, qualifier);
3851 get.setMaxVersions();
3852 res = this.region.get(get);
3853 kvs = res.getColumn(family, qualifier);
3854 assertEquals(1, kvs.size());
3855 assertEquals(Bytes.toBytes("value1"), kvs.get(0).getValue());
3856 }
3857
3858 @Test
3859 public void testDurability() throws Exception {
3860 String method = "testDurability";
3861
3862
3863
3864
3865 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
3866 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
3867 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
3868
3869 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
3870 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
3871 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
3872
3873 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
3874 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
3875
3876 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
3877 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
3878
3879 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
3880 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
3881 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
3882
3883
3884
3885 conf.setLong("hbase.regionserver.optionallogflushinterval", Integer.MAX_VALUE);
3886 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
3887 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
3888 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
3889 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
3890 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
3891 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
3892
3893
3894 conf.setLong("hbase.regionserver.optionallogflushinterval", 5);
3895 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
3896 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
3897 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
3898 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
3899 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
3900 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
3901
3902
3903 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
3904 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
3905 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
3906 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
3907 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
3908 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
3909
3910 }
3911
3912 private void durabilityTest(String method, Durability tableDurability,
3913 Durability mutationDurability, long timeout, boolean expectAppend,
3914 final boolean expectSync, final boolean expectSyncFromLogSyncer) throws Exception {
3915 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
3916 TableName tableName = TableName.valueOf(method);
3917 byte[] family = Bytes.toBytes("family");
3918 Path logDir = new Path(new Path(DIR + method), "log");
3919 HLog hlog = HLogFactory.createHLog(fs, logDir, UUID.randomUUID().toString(), conf);
3920 final HLog log = spy(hlog);
3921 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
3922 HConstants.EMPTY_END_ROW, method, conf, false,
3923 tableDurability, log, new byte[][] {family});
3924
3925 Put put = new Put(Bytes.toBytes("r1"));
3926 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
3927 put.setDurability(mutationDurability);
3928 region.put(put);
3929
3930
3931 verify(log, expectAppend ? times(1) : never())
3932 .appendNoSync((HRegionInfo)any(), eq(tableName),
3933 (WALEdit)any(), (UUID)any(), anyLong(), (HTableDescriptor)any());
3934
3935
3936 if (expectSync || expectSyncFromLogSyncer) {
3937 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
3938 @Override
3939 public boolean evaluate() throws Exception {
3940 try {
3941 if (expectSync) {
3942 verify(log, times(1)).sync(anyLong());
3943 } else if (expectSyncFromLogSyncer) {
3944 verify(log, times(1)).sync();
3945 }
3946 } catch (Throwable ignore) {}
3947 return true;
3948 }
3949 });
3950 } else {
3951 verify(log, never()).sync(anyLong());
3952 verify(log, never()).sync();
3953 }
3954
3955 hlog.close();
3956 region.close();
3957 }
3958
3959 private void putData(int startRow, int numRows, byte [] qf,
3960 byte [] ...families)
3961 throws IOException {
3962 for(int i=startRow; i<startRow+numRows; i++) {
3963 Put put = new Put(Bytes.toBytes("" + i));
3964 put.setDurability(Durability.SKIP_WAL);
3965 for(byte [] family : families) {
3966 put.add(family, qf, null);
3967 }
3968 region.put(put);
3969 }
3970 }
3971
3972 private void verifyData(HRegion newReg, int startRow, int numRows, byte [] qf,
3973 byte [] ... families)
3974 throws IOException {
3975 for(int i=startRow; i<startRow + numRows; i++) {
3976 byte [] row = Bytes.toBytes("" + i);
3977 Get get = new Get(row);
3978 for(byte [] family : families) {
3979 get.addColumn(family, qf);
3980 }
3981 Result result = newReg.get(get);
3982 KeyValue [] raw = result.raw();
3983 assertEquals(families.length, result.size());
3984 for(int j=0; j<families.length; j++) {
3985 assertEquals(0, Bytes.compareTo(row, raw[j].getRow()));
3986 assertEquals(0, Bytes.compareTo(families[j], raw[j].getFamily()));
3987 assertEquals(0, Bytes.compareTo(qf, raw[j].getQualifier()));
3988 }
3989 }
3990 }
3991
3992 private void assertGet(final HRegion r, final byte [] family, final byte [] k)
3993 throws IOException {
3994
3995 Get get = new Get(k).addFamily(family).setMaxVersions();
3996 KeyValue [] results = r.get(get).raw();
3997 for (int j = 0; j < results.length; j++) {
3998 byte [] tmp = results[j].getValue();
3999
4000 assertTrue(Bytes.equals(k, tmp));
4001 }
4002 }
4003
4004
4005
4006
4007
4008
4009
4010
4011 private void assertScan(final HRegion r, final byte [] fs,
4012 final byte [] firstValue)
4013 throws IOException {
4014 byte [][] families = {fs};
4015 Scan scan = new Scan();
4016 for (int i = 0; i < families.length; i++) scan.addFamily(families[i]);
4017 InternalScanner s = r.getScanner(scan);
4018 try {
4019 List<KeyValue> curVals = new ArrayList<KeyValue>();
4020 boolean first = true;
4021 OUTER_LOOP: while(s.next(curVals)) {
4022 for (KeyValue kv: curVals) {
4023 byte [] val = kv.getValue();
4024 byte [] curval = val;
4025 if (first) {
4026 first = false;
4027 assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4028 } else {
4029
4030 break OUTER_LOOP;
4031 }
4032 }
4033 }
4034 } finally {
4035 s.close();
4036 }
4037 }
4038
4039 private Configuration initSplit() {
4040 Configuration conf = HBaseConfiguration.create(this.conf);
4041
4042
4043 conf.setInt("hbase.hstore.compactionThreshold", 2);
4044
4045
4046 conf.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4047
4048 conf.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4049
4050
4051 conf.setLong("hbase.client.pause", 15 * 1000);
4052
4053
4054
4055 conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4056 return conf;
4057 }
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067 public static HRegion initHRegion (TableName tableName, String callingMethod,
4068 Configuration conf, byte [] ... families)
4069 throws IOException{
4070 return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
4071 }
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081 public static HRegion initHRegion (byte [] tableName, String callingMethod,
4082 Configuration conf, byte [] ... families)
4083 throws IOException{
4084 return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4085 }
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096 public static HRegion initHRegion (byte [] tableName, String callingMethod,
4097 Configuration conf, boolean isReadOnly, byte [] ... families)
4098 throws IOException{
4099 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4100 }
4101
4102 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4103 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4104 throws IOException {
4105 return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
4106 Durability.SYNC_WAL, null, families);
4107 }
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4121 String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4122 HLog hlog, byte[]... families)
4123 throws IOException {
4124 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
4125 htd.setReadOnly(isReadOnly);
4126 for(byte [] family : families) {
4127 HColumnDescriptor hcd = new HColumnDescriptor(family);
4128
4129 hcd.setMaxVersions(Integer.MAX_VALUE);
4130 htd.addFamily(hcd);
4131 }
4132 htd.setDurability(durability);
4133 HRegionInfo info = new HRegionInfo(htd.getTableName(), startKey, stopKey, false);
4134 Path path = new Path(DIR + callingMethod);
4135 FileSystem fs = FileSystem.get(conf);
4136 if (fs.exists(path)) {
4137 if (!fs.delete(path, true)) {
4138 throw new IOException("Failed delete of " + path);
4139 }
4140 }
4141 return HRegion.createHRegion(info, path, conf, htd, hlog);
4142 }
4143
4144
4145
4146
4147
4148 private void checkOneCell(KeyValue kv, byte[] cf,
4149 int rowIdx, int colIdx, long ts) {
4150 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4151 assertEquals("Row mismatch which checking: " + ctx,
4152 "row:"+ rowIdx, Bytes.toString(kv.getRow()));
4153 assertEquals("ColumnFamily mismatch while checking: " + ctx,
4154 Bytes.toString(cf), Bytes.toString(kv.getFamily()));
4155 assertEquals("Column qualifier mismatch while checking: " + ctx,
4156 "column:" + colIdx, Bytes.toString(kv.getQualifier()));
4157 assertEquals("Timestamp mismatch while checking: " + ctx,
4158 ts, kv.getTimestamp());
4159 assertEquals("Value mismatch while checking: " + ctx,
4160 "value-version-" + ts, Bytes.toString(kv.getValue()));
4161 }
4162 }
4163