1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21
22 import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23 import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24 import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25 import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29 import static org.junit.Assert.assertArrayEquals;
30 import static org.junit.Assert.assertEquals;
31 import static org.junit.Assert.assertFalse;
32 import static org.junit.Assert.assertNotNull;
33 import static org.junit.Assert.assertNull;
34 import static org.junit.Assert.assertTrue;
35 import static org.junit.Assert.fail;
36 import static org.mockito.Matchers.any;
37 import static org.mockito.Matchers.anyBoolean;
38 import static org.mockito.Matchers.anyLong;
39 import static org.mockito.Matchers.argThat;
40 import static org.mockito.Mockito.mock;
41 import static org.mockito.Mockito.never;
42 import static org.mockito.Mockito.spy;
43 import static org.mockito.Mockito.times;
44 import static org.mockito.Mockito.verify;
45 import static org.mockito.Mockito.when;
46
47 import java.io.IOException;
48 import java.io.InterruptedIOException;
49 import java.security.PrivilegedExceptionAction;
50 import java.util.ArrayList;
51 import java.util.Arrays;
52 import java.util.Collection;
53 import java.util.List;
54 import java.util.Map;
55 import java.util.NavigableMap;
56 import java.util.TreeMap;
57 import java.util.UUID;
58 import java.util.concurrent.atomic.AtomicBoolean;
59 import java.util.concurrent.atomic.AtomicInteger;
60 import java.util.concurrent.atomic.AtomicLong;
61 import java.util.concurrent.atomic.AtomicReference;
62
63 import org.apache.commons.logging.Log;
64 import org.apache.commons.logging.LogFactory;
65 import org.apache.hadoop.conf.Configuration;
66 import org.apache.hadoop.fs.FSDataOutputStream;
67 import org.apache.hadoop.fs.FileStatus;
68 import org.apache.hadoop.fs.FileSystem;
69 import org.apache.hadoop.fs.Path;
70 import org.apache.hadoop.hbase.Cell;
71 import org.apache.hadoop.hbase.CellComparator;
72 import org.apache.hadoop.hbase.CellUtil;
73 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
74 import org.apache.hadoop.hbase.DroppedSnapshotException;
75 import org.apache.hadoop.hbase.HBaseConfiguration;
76 import org.apache.hadoop.hbase.HBaseTestCase;
77 import org.apache.hadoop.hbase.HBaseTestingUtility;
78 import org.apache.hadoop.hbase.HColumnDescriptor;
79 import org.apache.hadoop.hbase.HConstants;
80 import org.apache.hadoop.hbase.RegionTooBusyException;
81 import org.apache.hadoop.hbase.Tag;
82 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
83 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
84 import org.apache.hadoop.hbase.HRegionInfo;
85 import org.apache.hadoop.hbase.HTableDescriptor;
86 import org.apache.hadoop.hbase.KeyValue;
87 import org.apache.hadoop.hbase.testclassification.MediumTests;
88 import org.apache.hadoop.hbase.MiniHBaseCluster;
89 import org.apache.hadoop.hbase.MultithreadedTestUtil;
90 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
91 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
92 import org.apache.hadoop.hbase.NotServingRegionException;
93 import org.apache.hadoop.hbase.ServerName;
94 import org.apache.hadoop.hbase.TableName;
95 import org.apache.hadoop.hbase.TagType;
96 import org.apache.hadoop.hbase.Waiter;
97 import org.apache.hadoop.hbase.client.Append;
98 import org.apache.hadoop.hbase.client.Delete;
99 import org.apache.hadoop.hbase.client.Durability;
100 import org.apache.hadoop.hbase.client.Get;
101 import org.apache.hadoop.hbase.client.Increment;
102 import org.apache.hadoop.hbase.client.Put;
103 import org.apache.hadoop.hbase.client.Result;
104 import org.apache.hadoop.hbase.client.Scan;
105 import org.apache.hadoop.hbase.client.Table;
106 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
107 import org.apache.hadoop.hbase.filter.BinaryComparator;
108 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
109 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
110 import org.apache.hadoop.hbase.filter.Filter;
111 import org.apache.hadoop.hbase.filter.FilterBase;
112 import org.apache.hadoop.hbase.filter.FilterList;
113 import org.apache.hadoop.hbase.filter.NullComparator;
114 import org.apache.hadoop.hbase.filter.PrefixFilter;
115 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
116 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
117 import org.apache.hadoop.hbase.io.hfile.HFile;
118 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
119 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
120 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
121 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
122 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
123 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
124 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
125 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
126 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor;
127 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
128 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
129 import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
130 import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
131 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
132 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
133 import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
134 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
135 import org.apache.hadoop.hbase.wal.DefaultWALProvider;
136 import org.apache.hadoop.hbase.wal.FaultyFSLog;
137 import org.apache.hadoop.hbase.wal.WAL;
138 import org.apache.hadoop.hbase.wal.WALFactory;
139 import org.apache.hadoop.hbase.wal.WALKey;
140 import org.apache.hadoop.hbase.wal.WALProvider;
141 import org.apache.hadoop.hbase.wal.WALSplitter;
142 import org.apache.hadoop.hbase.security.User;
143 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
144 import org.apache.hadoop.hbase.util.Bytes;
145 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
146 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
147 import org.apache.hadoop.hbase.util.FSUtils;
148 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
149 import org.apache.hadoop.hbase.util.PairOfSameType;
150 import org.apache.hadoop.hbase.util.Threads;
151 import org.junit.After;
152 import org.junit.Assert;
153 import org.junit.Before;
154 import org.junit.Rule;
155 import org.junit.Test;
156 import org.junit.experimental.categories.Category;
157 import org.junit.rules.TestName;
158 import org.mockito.ArgumentCaptor;
159 import org.mockito.ArgumentMatcher;
160 import org.mockito.Mockito;
161
162 import com.google.common.collect.ImmutableList;
163 import com.google.common.collect.Lists;
164 import com.google.protobuf.ByteString;
165
166
167
168
169
170
171
172 @Category(MediumTests.class)
173 @SuppressWarnings("deprecation")
174 public class TestHRegion {
175
176
177 static final Log LOG = LogFactory.getLog(TestHRegion.class);
178 @Rule public TestName name = new TestName();
179
180 private static final String COLUMN_FAMILY = "MyCF";
181 private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
182
183 HRegion region = null;
184
185 private static HBaseTestingUtility TEST_UTIL;
186 public static Configuration CONF ;
187 private String dir;
188 private static FileSystem FILESYSTEM;
189 private final int MAX_VERSIONS = 2;
190
191
192 protected byte[] tableName;
193 protected String method;
194 protected final byte[] qual1 = Bytes.toBytes("qual1");
195 protected final byte[] qual2 = Bytes.toBytes("qual2");
196 protected final byte[] qual3 = Bytes.toBytes("qual3");
197 protected final byte[] value1 = Bytes.toBytes("value1");
198 protected final byte[] value2 = Bytes.toBytes("value2");
199 protected final byte[] row = Bytes.toBytes("rowA");
200 protected final byte[] row2 = Bytes.toBytes("rowB");
201
202 protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
203 .getInstance(MetricsAssertHelper.class);
204
205 @Before
206 public void setup() throws IOException {
207 TEST_UTIL = HBaseTestingUtility.createLocalHTU();
208 FILESYSTEM = TEST_UTIL.getTestFileSystem();
209 CONF = TEST_UTIL.getConfiguration();
210 dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
211 method = name.getMethodName();
212 tableName = Bytes.toBytes(name.getMethodName());
213 }
214
215 @After
216 public void tearDown() throws Exception {
217 EnvironmentEdgeManagerTestHelper.reset();
218 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
219 TEST_UTIL.cleanupTestDir();
220 }
221
222 String getName() {
223 return name.getMethodName();
224 }
225
226
227
228
229
230
231
232
233
234
235
236 @Test (timeout=60000)
237 public void testCloseCarryingSnapshot() throws IOException {
238 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
239 Store store = region.getStore(COLUMN_FAMILY_BYTES);
240
241 byte [] value = Bytes.toBytes(name.getMethodName());
242
243 Put put = new Put(value);
244 put.add(COLUMN_FAMILY_BYTES, null, value);
245
246 region.put(put);
247 StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
248 storeFlushCtx.prepare();
249
250 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
251 region.put(put);
252
253 region.close();
254 assertEquals(0, region.getMemstoreSize().get());
255 HRegion.closeHRegion(region);
256 }
257
258
259
260
261
262 @Test (timeout=60000)
263 public void testMemstoreSnapshotSize() throws IOException {
264 class MyFaultyFSLog extends FaultyFSLog {
265 StoreFlushContext storeFlushCtx;
266 public MyFaultyFSLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
267 throws IOException {
268 super(fs, rootDir, logName, conf);
269 }
270
271 void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
272 this.storeFlushCtx = storeFlushCtx;
273 }
274
275 @Override
276 public void sync(long txid) throws IOException {
277 storeFlushCtx.prepare();
278 super.sync(txid);
279 }
280 }
281
282 FileSystem fs = FileSystem.get(CONF);
283 Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
284 MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
285 HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
286 CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES);
287
288 Store store = region.getStore(COLUMN_FAMILY_BYTES);
289
290 byte [] value = Bytes.toBytes(name.getMethodName());
291 faultyLog.setStoreFlushCtx(store.createFlushContext(12345));
292
293 Put put = new Put(value);
294 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
295 faultyLog.setFailureType(FaultyFSLog.FailureType.SYNC);
296
297 boolean threwIOE = false;
298 try {
299 region.put(put);
300 } catch (IOException ioe) {
301 threwIOE = true;
302 } finally {
303 assertTrue("The regionserver should have thrown an exception", threwIOE);
304 }
305 long sz = store.getFlushableSize();
306 assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
307 HRegion.closeHRegion(region);
308 }
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325 @Test (timeout=60000)
326 public void testFlushSizeAccounting() throws Exception {
327 final Configuration conf = HBaseConfiguration.create(CONF);
328
329 conf.setInt("hbase.hstore.flush.retries.number", 1);
330 final User user =
331 User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
332
333 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
334 user.runAs(new PrivilegedExceptionAction<Object>() {
335 @Override
336 public Object run() throws Exception {
337
338 FileSystem fs = FileSystem.get(conf);
339 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
340 FaultyFileSystem ffs = (FaultyFileSystem)fs;
341 HRegion region = null;
342 try {
343
344 region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
345 long size = region.getMemstoreSize().get();
346 Assert.assertEquals(0, size);
347
348 Put p1 = new Put(row);
349 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
350 region.put(p1);
351 final long sizeOfOnePut = region.getMemstoreSize().get();
352
353 try {
354 LOG.info("Flushing");
355 region.flushcache();
356 Assert.fail("Didn't bubble up IOE!");
357 } catch (DroppedSnapshotException dse) {
358
359 }
360
361 ffs.fault.set(false);
362
363 Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize().get());
364
365
366 Put p2 = new Put(row);
367 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
368 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
369 region.put(p2);
370 Assert.assertEquals(sizeOfOnePut * 3, region.getMemstoreSize().get());
371
372
373
374 region.flushcache();
375
376 Assert.assertEquals(sizeOfOnePut * 2, region.getMemstoreSize().get());
377 } finally {
378 HRegion.closeHRegion(region);
379 }
380 return null;
381 }
382 });
383 FileSystem.closeAllForUGI(user.getUGI());
384 }
385
386 @Test (timeout=60000)
387 public void testCloseWithFailingFlush() throws Exception {
388 final Configuration conf = HBaseConfiguration.create(CONF);
389
390 conf.setInt("hbase.hstore.flush.retries.number", 1);
391 final User user =
392 User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
393
394 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
395 user.runAs(new PrivilegedExceptionAction<Object>() {
396 @Override
397 public Object run() throws Exception {
398
399 FileSystem fs = FileSystem.get(conf);
400 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
401 FaultyFileSystem ffs = (FaultyFileSystem)fs;
402 HRegion region = null;
403 try {
404
405 region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
406 long size = region.getMemstoreSize().get();
407 Assert.assertEquals(0, size);
408
409 Put p1 = new Put(row);
410 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
411 region.put(p1);
412
413 Store store = region.getStore(COLUMN_FAMILY_BYTES);
414 StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
415 storeFlushCtx.prepare();
416
417 Put p2 = new Put(row);
418 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
419 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
420 region.put(p2);
421
422 region.close();
423 fail();
424 } catch (DroppedSnapshotException dse) {
425
426 LOG.info("Expected DroppedSnapshotException");
427 } finally {
428
429 ffs.fault.set(false);
430 HRegion.closeHRegion(region);
431 }
432 return null;
433 }
434 });
435 FileSystem.closeAllForUGI(user.getUGI());
436 }
437
438 @Test
439 public void testCompactionAffectedByScanners() throws Exception {
440 byte[] family = Bytes.toBytes("family");
441 this.region = initHRegion(tableName, method, CONF, family);
442
443 Put put = new Put(Bytes.toBytes("r1"));
444 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
445 region.put(put);
446 region.flushcache();
447
448 Scan scan = new Scan();
449 scan.setMaxVersions(3);
450
451 RegionScanner scanner1 = region.getScanner(scan);
452
453 Delete delete = new Delete(Bytes.toBytes("r1"));
454 region.delete(delete);
455 region.flushcache();
456
457
458 RegionScanner scanner2 = region.getScanner(scan);
459
460 List<Cell> results = new ArrayList<Cell>();
461
462 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
463
464
465 region.compactStores(true);
466
467
468 RegionScanner scanner3 = region.getScanner(scan);
469
470
471 scanner1.next(results);
472 System.out.println(results);
473 assertEquals(1, results.size());
474
475 results.clear();
476 scanner2.next(results);
477 System.out.println(results);
478 assertEquals(0, results.size());
479
480 results.clear();
481 scanner3.next(results);
482 System.out.println(results);
483 assertEquals(0, results.size());
484 }
485
486 @Test
487 public void testToShowNPEOnRegionScannerReseek() throws Exception {
488 byte[] family = Bytes.toBytes("family");
489 this.region = initHRegion(tableName, method, CONF, family);
490
491 Put put = new Put(Bytes.toBytes("r1"));
492 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
493 region.put(put);
494 put = new Put(Bytes.toBytes("r2"));
495 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
496 region.put(put);
497 region.flushcache();
498
499 Scan scan = new Scan();
500 scan.setMaxVersions(3);
501
502 RegionScanner scanner1 = region.getScanner(scan);
503
504 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
505
506 region.compactStores(true);
507
508 scanner1.reseek(Bytes.toBytes("r2"));
509 List<Cell> results = new ArrayList<Cell>();
510 scanner1.next(results);
511 Cell keyValue = results.get(0);
512 Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
513 scanner1.close();
514 }
515
516 @Test
517 public void testSkipRecoveredEditsReplay() throws Exception {
518 String method = "testSkipRecoveredEditsReplay";
519 TableName tableName = TableName.valueOf(method);
520 byte[] family = Bytes.toBytes("family");
521 this.region = initHRegion(tableName, method, CONF, family);
522 final WALFactory wals = new WALFactory(CONF, null, method);
523 try {
524 Path regiondir = region.getRegionFileSystem().getRegionDir();
525 FileSystem fs = region.getRegionFileSystem().getFileSystem();
526 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
527
528 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
529
530 long maxSeqId = 1050;
531 long minSeqId = 1000;
532
533 for (long i = minSeqId; i <= maxSeqId; i += 10) {
534 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
535 fs.create(recoveredEdits);
536 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
537
538 long time = System.nanoTime();
539 WALEdit edit = new WALEdit();
540 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
541 .toBytes(i)));
542 writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time,
543 HConstants.DEFAULT_CLUSTER_ID), edit));
544
545 writer.close();
546 }
547 MonitoredTask status = TaskMonitor.get().createStatus(method);
548 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
549 for (Store store : region.getStores().values()) {
550 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
551 }
552 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
553 assertEquals(maxSeqId, seqId);
554 region.getMVCC().initialize(seqId);
555 Get get = new Get(row);
556 Result result = region.get(get);
557 for (long i = minSeqId; i <= maxSeqId; i += 10) {
558 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
559 assertEquals(1, kvs.size());
560 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
561 }
562 } finally {
563 HRegion.closeHRegion(this.region);
564 this.region = null;
565 wals.close();
566 }
567 }
568
569 @Test
570 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
571 String method = "testSkipRecoveredEditsReplaySomeIgnored";
572 TableName tableName = TableName.valueOf(method);
573 byte[] family = Bytes.toBytes("family");
574 this.region = initHRegion(tableName, method, CONF, family);
575 final WALFactory wals = new WALFactory(CONF, null, method);
576 try {
577 Path regiondir = region.getRegionFileSystem().getRegionDir();
578 FileSystem fs = region.getRegionFileSystem().getFileSystem();
579 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
580
581 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
582
583 long maxSeqId = 1050;
584 long minSeqId = 1000;
585
586 for (long i = minSeqId; i <= maxSeqId; i += 10) {
587 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
588 fs.create(recoveredEdits);
589 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
590
591 long time = System.nanoTime();
592 WALEdit edit = new WALEdit();
593 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
594 .toBytes(i)));
595 writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time,
596 HConstants.DEFAULT_CLUSTER_ID), edit));
597
598 writer.close();
599 }
600 long recoverSeqId = 1030;
601 MonitoredTask status = TaskMonitor.get().createStatus(method);
602 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
603 for (Store store : region.getStores().values()) {
604 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
605 }
606 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
607 assertEquals(maxSeqId, seqId);
608 region.getMVCC().initialize(seqId);
609 Get get = new Get(row);
610 Result result = region.get(get);
611 for (long i = minSeqId; i <= maxSeqId; i += 10) {
612 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
613 if (i < recoverSeqId) {
614 assertEquals(0, kvs.size());
615 } else {
616 assertEquals(1, kvs.size());
617 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
618 }
619 }
620 } finally {
621 HRegion.closeHRegion(this.region);
622 this.region = null;
623 wals.close();
624 }
625 }
626
627 @Test
628 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
629 byte[] family = Bytes.toBytes("family");
630 this.region = initHRegion(tableName, method, CONF, family);
631 try {
632 Path regiondir = region.getRegionFileSystem().getRegionDir();
633 FileSystem fs = region.getRegionFileSystem().getFileSystem();
634
635 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
636 for (int i = 1000; i < 1050; i += 10) {
637 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
638 FSDataOutputStream dos = fs.create(recoveredEdits);
639 dos.writeInt(i);
640 dos.close();
641 }
642 long minSeqId = 2000;
643 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
644 FSDataOutputStream dos = fs.create(recoveredEdits);
645 dos.close();
646
647 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
648 for (Store store : region.getStores().values()) {
649 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
650 }
651 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
652 assertEquals(minSeqId, seqId);
653 } finally {
654 HRegion.closeHRegion(this.region);
655 this.region = null;
656 }
657 }
658
659 @Test
660 public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
661 String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
662 TableName tableName = TableName.valueOf(method);
663 byte[] family = Bytes.toBytes("family");
664 this.region = initHRegion(tableName, method, CONF, family);
665 final WALFactory wals = new WALFactory(CONF, null, method);
666 try {
667 Path regiondir = region.getRegionFileSystem().getRegionDir();
668 FileSystem fs = region.getRegionFileSystem().getFileSystem();
669 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
670
671 assertEquals(0, region.getStoreFileList(
672 region.getStores().keySet().toArray(new byte[0][])).size());
673
674 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
675
676 long maxSeqId = 1050;
677 long minSeqId = 1000;
678
679 for (long i = minSeqId; i <= maxSeqId; i += 10) {
680 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
681 fs.create(recoveredEdits);
682 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
683
684 long time = System.nanoTime();
685 WALEdit edit = null;
686 if (i == maxSeqId) {
687 edit = WALEdit.createCompaction(region.getRegionInfo(),
688 CompactionDescriptor.newBuilder()
689 .setTableName(ByteString.copyFrom(tableName.getName()))
690 .setFamilyName(ByteString.copyFrom(regionName))
691 .setEncodedRegionName(ByteString.copyFrom(regionName))
692 .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
693 .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName()))
694 .build());
695 } else {
696 edit = new WALEdit();
697 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
698 .toBytes(i)));
699 }
700 writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time,
701 HConstants.DEFAULT_CLUSTER_ID), edit));
702 writer.close();
703 }
704
705 long recoverSeqId = 1030;
706 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
707 MonitoredTask status = TaskMonitor.get().createStatus(method);
708 for (Store store : region.getStores().values()) {
709 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
710 }
711 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
712 assertEquals(maxSeqId, seqId);
713
714
715 assertEquals(1, region.getStoreFileList(
716 region.getStores().keySet().toArray(new byte[0][])).size());
717
718 } finally {
719 HRegion.closeHRegion(this.region);
720 this.region = null;
721 wals.close();
722 }
723 }
724
725 @Test
726 public void testRecoveredEditsReplayCompaction() throws Exception {
727 String method = name.getMethodName();
728 TableName tableName = TableName.valueOf(method);
729 byte[] family = Bytes.toBytes("family");
730 this.region = initHRegion(tableName, method, CONF, family);
731 final WALFactory wals = new WALFactory(CONF, null, method);
732 try {
733 Path regiondir = region.getRegionFileSystem().getRegionDir();
734 FileSystem fs = region.getRegionFileSystem().getFileSystem();
735 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
736
737 long maxSeqId = 3;
738 long minSeqId = 0;
739
740 for (long i = minSeqId; i < maxSeqId; i++) {
741 Put put = new Put(Bytes.toBytes(i));
742 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
743 region.put(put);
744 region.flushcache();
745 }
746
747
748 assertEquals(3, region.getStore(family).getStorefilesCount());
749 List<Path> storeFiles = new ArrayList<Path>(3);
750 for (StoreFile sf : region.getStore(family).getStorefiles()) {
751 storeFiles.add(sf.getPath());
752 }
753
754
755 CONF.setBoolean("hbase.hstore.compaction.complete", false);
756 region.compactStores();
757
758
759 assertEquals(3, region.getStore(family).getStorefilesCount());
760
761
762 Path tmpDir = region.getRegionFileSystem().getTempDir();
763 FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
764 String errorMsg = "Expected to find 1 file in the region temp directory "
765 + "from the compaction, could not find any";
766 assertNotNull(errorMsg, files);
767 assertEquals(errorMsg, 1, files.length);
768
769 Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
770 files[0].getPath());
771
772 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
773 .getRegionInfo(), family, storeFiles, Lists.newArrayList(newFile), region
774 .getRegionFileSystem().getStoreDir(Bytes.toString(family)));
775
776 WALUtil.writeCompactionMarker(region.getWAL(), this.region.getTableDesc(),
777 this.region.getRegionInfo(), compactionDescriptor, new AtomicLong(1));
778
779 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
780
781 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
782 fs.create(recoveredEdits);
783 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
784
785 long time = System.nanoTime();
786
787 writer.append(new WAL.Entry(new HLogKey(regionName, tableName, 10, time,
788 HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(),
789 compactionDescriptor)));
790 writer.close();
791
792
793 region.getTableDesc();
794 region.getRegionInfo();
795 region.close();
796 region = HRegion.openHRegion(region, null);
797
798
799 Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
800 for (StoreFile sf : sfs) {
801 LOG.info(sf.getPath());
802 }
803 assertEquals(1, region.getStore(family).getStorefilesCount());
804 files = FSUtils.listStatus(fs, tmpDir);
805 assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
806
807 for (long i = minSeqId; i < maxSeqId; i++) {
808 Get get = new Get(Bytes.toBytes(i));
809 Result result = region.get(get);
810 byte[] value = result.getValue(family, Bytes.toBytes(i));
811 assertArrayEquals(Bytes.toBytes(i), value);
812 }
813 } finally {
814 HRegion.closeHRegion(this.region);
815 this.region = null;
816 wals.close();
817 }
818 }
819
820 @Test
821 public void testFlushMarkers() throws Exception {
822
823 String method = name.getMethodName();
824 TableName tableName = TableName.valueOf(method);
825 byte[] family = Bytes.toBytes("family");
826 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log");
827 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
828 FSUtils.setRootDir(walConf, logDir);
829 final WALFactory wals = new WALFactory(walConf, null, method);
830 final WAL wal = wals.getWAL(tableName.getName());
831
832 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
833 HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family);
834 try {
835 Path regiondir = region.getRegionFileSystem().getRegionDir();
836 FileSystem fs = region.getRegionFileSystem().getFileSystem();
837 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
838
839 long maxSeqId = 3;
840 long minSeqId = 0;
841
842 for (long i = minSeqId; i < maxSeqId; i++) {
843 Put put = new Put(Bytes.toBytes(i));
844 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
845 region.put(put);
846 region.flushcache();
847 }
848
849
850 assertEquals(3, region.getStore(family).getStorefilesCount());
851 List<String> storeFiles = new ArrayList<String>(3);
852 for (StoreFile sf : region.getStore(family).getStorefiles()) {
853 storeFiles.add(sf.getPath().getName());
854 }
855
856
857 wal.shutdown();
858 WAL.Reader reader = wals.createReader(fs, DefaultWALProvider.getCurrentFileName(wal),
859 TEST_UTIL.getConfiguration());
860 try {
861 List<WAL.Entry> flushDescriptors = new ArrayList<WAL.Entry>();
862 long lastFlushSeqId = -1;
863 while (true) {
864 WAL.Entry entry = reader.next();
865 if (entry == null) {
866 break;
867 }
868 Cell cell = entry.getEdit().getCells().get(0);
869 if (WALEdit.isMetaEditFamily(cell)) {
870 FlushDescriptor flushDesc = WALEdit.getFlushDescriptor(cell);
871 assertNotNull(flushDesc);
872 assertArrayEquals(tableName.getName(), flushDesc.getTableName().toByteArray());
873 if (flushDesc.getAction() == FlushAction.START_FLUSH) {
874 assertTrue(flushDesc.getFlushSequenceNumber() > lastFlushSeqId);
875 } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) {
876 assertTrue(flushDesc.getFlushSequenceNumber() == lastFlushSeqId);
877 }
878 lastFlushSeqId = flushDesc.getFlushSequenceNumber();
879 assertArrayEquals(regionName, flushDesc.getEncodedRegionName().toByteArray());
880 assertEquals(1, flushDesc.getStoreFlushesCount());
881 StoreFlushDescriptor storeFlushDesc = flushDesc.getStoreFlushes(0);
882 assertArrayEquals(family, storeFlushDesc.getFamilyName().toByteArray());
883 assertEquals("family", storeFlushDesc.getStoreHomeDir());
884 if (flushDesc.getAction() == FlushAction.START_FLUSH) {
885 assertEquals(0, storeFlushDesc.getFlushOutputCount());
886 } else {
887 assertEquals(1, storeFlushDesc.getFlushOutputCount());
888 assertTrue(storeFiles.contains(storeFlushDesc.getFlushOutput(0)));
889 }
890
891 flushDescriptors.add(entry);
892 }
893 }
894
895 assertEquals(3 * 2, flushDescriptors.size());
896
897
898
899 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
900
901 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
902 fs.create(recoveredEdits);
903 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
904
905 for (WAL.Entry entry : flushDescriptors) {
906 writer.append(entry);
907 }
908 writer.close();
909 } finally {
910 if (null != reader) {
911 try {
912 reader.close();
913 } catch (IOException exception) {
914 LOG.warn("Problem closing wal: " + exception.getMessage());
915 LOG.debug("exception details", exception);
916 }
917 }
918 }
919
920
921
922 region.close();
923 region = HRegion.openHRegion(region, null);
924
925
926 for (long i = minSeqId; i < maxSeqId; i++) {
927 Get get = new Get(Bytes.toBytes(i));
928 Result result = region.get(get);
929 byte[] value = result.getValue(family, Bytes.toBytes(i));
930 assertArrayEquals(Bytes.toBytes(i), value);
931 }
932 } finally {
933 HRegion.closeHRegion(this.region);
934 this.region = null;
935 wals.close();
936 }
937 }
938
939 class IsFlushWALMarker extends ArgumentMatcher<WALEdit> {
940 volatile FlushAction[] actions;
941 public IsFlushWALMarker(FlushAction... actions) {
942 this.actions = actions;
943 }
944 @Override
945 public boolean matches(Object edit) {
946 List<Cell> cells = ((WALEdit)edit).getCells();
947 if (cells.isEmpty()) {
948 return false;
949 }
950 if (WALEdit.isMetaEditFamily(cells.get(0))) {
951 FlushDescriptor desc = null;
952 try {
953 desc = WALEdit.getFlushDescriptor(cells.get(0));
954 } catch (IOException e) {
955 LOG.warn(e);
956 return false;
957 }
958 if (desc != null) {
959 for (FlushAction action : actions) {
960 if (desc.getAction() == action) {
961 return true;
962 }
963 }
964 }
965 }
966 return false;
967 }
968 public IsFlushWALMarker set(FlushAction... actions) {
969 this.actions = actions;
970 return this;
971 }
972 }
973
974 @Test
975 @SuppressWarnings("unchecked")
976 public void testFlushMarkersWALFail() throws Exception {
977
978 String method = name.getMethodName();
979 TableName tableName = TableName.valueOf(method);
980 byte[] family = Bytes.toBytes("family");
981
982
983 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + "log");
984
985 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
986 FSUtils.setRootDir(walConf, logDir);
987 final WALFactory wals = new WALFactory(walConf, null, method);
988 WAL wal = spy(wals.getWAL(tableName.getName()));
989
990 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
991 HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family);
992 try {
993 int i = 0;
994 Put put = new Put(Bytes.toBytes(i));
995 put.setDurability(Durability.SKIP_WAL);
996 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
997 region.put(put);
998
999
1000 IsFlushWALMarker isFlushWALMarker = new IsFlushWALMarker(FlushAction.START_FLUSH);
1001
1002
1003 when(wal.append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any(),
1004 (WALEdit)argThat(isFlushWALMarker), (AtomicLong)any(), Mockito.anyBoolean(),
1005 (List<Cell>)any()))
1006 .thenThrow(new IOException("Fail to append flush marker"));
1007
1008
1009 try {
1010 region.flushcache();
1011 fail("This should have thrown exception");
1012 } catch (DroppedSnapshotException unexpected) {
1013
1014 throw unexpected;
1015 } catch (IOException expected) {
1016
1017 }
1018
1019
1020 isFlushWALMarker.set(FlushAction.COMMIT_FLUSH);
1021
1022 try {
1023 region.flushcache();
1024 fail("This should have thrown exception");
1025 } catch (DroppedSnapshotException expected) {
1026
1027
1028 } catch (IOException unexpected) {
1029 throw unexpected;
1030 }
1031
1032 region.close();
1033 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
1034 HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family);
1035 region.put(put);
1036
1037
1038
1039
1040 isFlushWALMarker.set(FlushAction.COMMIT_FLUSH, FlushAction.ABORT_FLUSH);
1041
1042 try {
1043 region.flushcache();
1044 fail("This should have thrown exception");
1045 } catch (DroppedSnapshotException expected) {
1046
1047
1048 } catch (IOException unexpected) {
1049 throw unexpected;
1050 }
1051
1052 } finally {
1053 HRegion.closeHRegion(this.region);
1054 this.region = null;
1055 }
1056 }
1057
1058 @Test
1059 public void testGetWhileRegionClose() throws IOException {
1060 TableName tableName = TableName.valueOf(name.getMethodName());
1061 Configuration hc = initSplit();
1062 int numRows = 100;
1063 byte[][] families = { fam1, fam2, fam3 };
1064
1065
1066 String method = name.getMethodName();
1067 this.region = initHRegion(tableName, method, hc, families);
1068 try {
1069
1070 final int startRow = 100;
1071 putData(startRow, numRows, qual1, families);
1072 putData(startRow, numRows, qual2, families);
1073 putData(startRow, numRows, qual3, families);
1074 final AtomicBoolean done = new AtomicBoolean(false);
1075 final AtomicInteger gets = new AtomicInteger(0);
1076 GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
1077 try {
1078
1079 for (int i = 0; i < threads.length / 2; i++) {
1080 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
1081 threads[i].setDaemon(true);
1082 threads[i].start();
1083 }
1084
1085
1086 this.region.closing.set(true);
1087 for (int i = threads.length / 2; i < threads.length; i++) {
1088 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
1089 threads[i].setDaemon(true);
1090 threads[i].start();
1091 }
1092 } finally {
1093 if (this.region != null) {
1094 HRegion.closeHRegion(this.region);
1095 }
1096 }
1097 done.set(true);
1098 for (GetTillDoneOrException t : threads) {
1099 try {
1100 t.join();
1101 } catch (InterruptedException e) {
1102 e.printStackTrace();
1103 }
1104 if (t.e != null) {
1105 LOG.info("Exception=" + t.e);
1106 assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
1107 }
1108 }
1109 } finally {
1110 HRegion.closeHRegion(this.region);
1111 this.region = null;
1112 }
1113 }
1114
1115
1116
1117
1118
1119 class GetTillDoneOrException extends Thread {
1120 private final Get g;
1121 private final AtomicBoolean done;
1122 private final AtomicInteger count;
1123 private Exception e;
1124
1125 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
1126 super("getter." + i);
1127 this.g = new Get(r);
1128 this.done = d;
1129 this.count = c;
1130 }
1131
1132 @Override
1133 public void run() {
1134 while (!this.done.get()) {
1135 try {
1136 assertTrue(region.get(g).size() > 0);
1137 this.count.incrementAndGet();
1138 } catch (Exception e) {
1139 this.e = e;
1140 break;
1141 }
1142 }
1143 }
1144 }
1145
1146
1147
1148
1149 @Test
1150 public void testWeirdCacheBehaviour() throws Exception {
1151 byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
1152 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
1153 Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
1154 this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
1155 try {
1156 String value = "this is the value";
1157 String value2 = "this is some other value";
1158 String keyPrefix1 = "prefix1";
1159 String keyPrefix2 = "prefix2";
1160 String keyPrefix3 = "prefix3";
1161 putRows(this.region, 3, value, keyPrefix1);
1162 putRows(this.region, 3, value, keyPrefix2);
1163 putRows(this.region, 3, value, keyPrefix3);
1164 putRows(this.region, 3, value2, keyPrefix1);
1165 putRows(this.region, 3, value2, keyPrefix2);
1166 putRows(this.region, 3, value2, keyPrefix3);
1167 System.out.println("Checking values for key: " + keyPrefix1);
1168 assertEquals("Got back incorrect number of rows from scan", 3,
1169 getNumberOfRows(keyPrefix1, value2, this.region));
1170 System.out.println("Checking values for key: " + keyPrefix2);
1171 assertEquals("Got back incorrect number of rows from scan", 3,
1172 getNumberOfRows(keyPrefix2, value2, this.region));
1173 System.out.println("Checking values for key: " + keyPrefix3);
1174 assertEquals("Got back incorrect number of rows from scan", 3,
1175 getNumberOfRows(keyPrefix3, value2, this.region));
1176 deleteColumns(this.region, value2, keyPrefix1);
1177 deleteColumns(this.region, value2, keyPrefix2);
1178 deleteColumns(this.region, value2, keyPrefix3);
1179 System.out.println("Starting important checks.....");
1180 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
1181 getNumberOfRows(keyPrefix1, value2, this.region));
1182 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
1183 getNumberOfRows(keyPrefix2, value2, this.region));
1184 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
1185 getNumberOfRows(keyPrefix3, value2, this.region));
1186 } finally {
1187 HRegion.closeHRegion(this.region);
1188 this.region = null;
1189 }
1190 }
1191
1192 @Test
1193 public void testAppendWithReadOnlyTable() throws Exception {
1194 byte[] TABLE = Bytes.toBytes("readOnlyTable");
1195 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
1196 boolean exceptionCaught = false;
1197 Append append = new Append(Bytes.toBytes("somerow"));
1198 append.setDurability(Durability.SKIP_WAL);
1199 append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
1200 Bytes.toBytes("somevalue"));
1201 try {
1202 region.append(append);
1203 } catch (IOException e) {
1204 exceptionCaught = true;
1205 } finally {
1206 HRegion.closeHRegion(this.region);
1207 this.region = null;
1208 }
1209 assertTrue(exceptionCaught == true);
1210 }
1211
1212 @Test
1213 public void testIncrWithReadOnlyTable() throws Exception {
1214 byte[] TABLE = Bytes.toBytes("readOnlyTable");
1215 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
1216 boolean exceptionCaught = false;
1217 Increment inc = new Increment(Bytes.toBytes("somerow"));
1218 inc.setDurability(Durability.SKIP_WAL);
1219 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
1220 try {
1221 region.increment(inc);
1222 } catch (IOException e) {
1223 exceptionCaught = true;
1224 } finally {
1225 HRegion.closeHRegion(this.region);
1226 this.region = null;
1227 }
1228 assertTrue(exceptionCaught == true);
1229 }
1230
1231 private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
1232 InternalScanner scanner = buildScanner(keyPrefix, value, r);
1233 int count = 0;
1234 boolean more = false;
1235 List<Cell> results = new ArrayList<Cell>();
1236 do {
1237 more = scanner.next(results);
1238 if (results != null && !results.isEmpty())
1239 count++;
1240 else
1241 break;
1242 Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
1243 delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
1244 r.delete(delete);
1245 results.clear();
1246 } while (more);
1247 assertEquals("Did not perform correct number of deletes", 3, count);
1248 }
1249
1250 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
1251 InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
1252 int numberOfResults = 0;
1253 List<Cell> results = new ArrayList<Cell>();
1254 boolean more = false;
1255 do {
1256 more = resultScanner.next(results);
1257 if (results != null && !results.isEmpty())
1258 numberOfResults++;
1259 else
1260 break;
1261 for (Cell kv : results) {
1262 System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
1263 }
1264 results.clear();
1265 } while (more);
1266 return numberOfResults;
1267 }
1268
1269 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
1270 throws IOException {
1271
1272 FilterList allFilters = new FilterList();
1273 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
1274
1275 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
1276 Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
1277 filter.setFilterIfMissing(true);
1278 allFilters.addFilter(filter);
1279 Scan scan = new Scan();
1280 scan.addFamily(Bytes.toBytes("trans-blob"));
1281 scan.addFamily(Bytes.toBytes("trans-type"));
1282 scan.addFamily(Bytes.toBytes("trans-date"));
1283 scan.addFamily(Bytes.toBytes("trans-tags"));
1284 scan.addFamily(Bytes.toBytes("trans-group"));
1285 scan.setFilter(allFilters);
1286 return r.getScanner(scan);
1287 }
1288
1289 private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
1290 for (int i = 0; i < numRows; i++) {
1291 String row = key + "_" + i
1292 System.out.println(String.format("Saving row: %s, with value %s", row, value));
1293 Put put = new Put(Bytes.toBytes(row));
1294 put.setDurability(Durability.SKIP_WAL);
1295 put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
1296 put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
1297 put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
1298 put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
1299 put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
1300 r.put(put);
1301 }
1302 }
1303
1304 @Test
1305 public void testFamilyWithAndWithoutColon() throws Exception {
1306 byte[] b = Bytes.toBytes(getName());
1307 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1308 this.region = initHRegion(b, getName(), CONF, cf);
1309 try {
1310 Put p = new Put(b);
1311 byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
1312 p.add(cfwithcolon, cfwithcolon, cfwithcolon);
1313 boolean exception = false;
1314 try {
1315 this.region.put(p);
1316 } catch (NoSuchColumnFamilyException e) {
1317 exception = true;
1318 }
1319 assertTrue(exception);
1320 } finally {
1321 HRegion.closeHRegion(this.region);
1322 this.region = null;
1323 }
1324 }
1325
1326 @Test
1327 public void testBatchPut_whileNoRowLocksHeld() throws IOException {
1328 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1329 byte[] qual = Bytes.toBytes("qual");
1330 byte[] val = Bytes.toBytes("val");
1331 this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1332 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1333 try {
1334 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1335 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1336
1337 LOG.info("First a batch put with all valid puts");
1338 final Put[] puts = new Put[10];
1339 for (int i = 0; i < 10; i++) {
1340 puts[i] = new Put(Bytes.toBytes("row_" + i));
1341 puts[i].add(cf, qual, val);
1342 }
1343
1344 OperationStatus[] codes = this.region.batchMutate(puts);
1345 assertEquals(10, codes.length);
1346 for (int i = 0; i < 10; i++) {
1347 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1348 }
1349 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1350
1351 LOG.info("Next a batch put with one invalid family");
1352 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1353 codes = this.region.batchMutate(puts);
1354 assertEquals(10, codes.length);
1355 for (int i = 0; i < 10; i++) {
1356 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1357 codes[i].getOperationStatusCode());
1358 }
1359
1360 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1361 } finally {
1362 HRegion.closeHRegion(this.region);
1363 this.region = null;
1364 }
1365 }
1366
1367 @Test
1368 public void testBatchPut_whileMultipleRowLocksHeld() throws Exception {
1369 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1370 byte[] qual = Bytes.toBytes("qual");
1371 byte[] val = Bytes.toBytes("val");
1372 this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1373 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1374 try {
1375 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1376 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1377
1378 final Put[] puts = new Put[10];
1379 for (int i = 0; i < 10; i++) {
1380 puts[i] = new Put(Bytes.toBytes("row_" + i));
1381 puts[i].add(cf, qual, val);
1382 }
1383 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1384
1385 LOG.info("batchPut will have to break into four batches to avoid row locks");
1386 RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2"));
1387 RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_4"));
1388 RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_6"));
1389
1390 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1391 final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
1392 TestThread putter = new TestThread(ctx) {
1393 @Override
1394 public void doWork() throws IOException {
1395 retFromThread.set(region.batchMutate(puts));
1396 }
1397 };
1398 LOG.info("...starting put thread while holding locks");
1399 ctx.addThread(putter);
1400 ctx.startThreads();
1401
1402 LOG.info("...waiting for put thread to sync 1st time");
1403 waitForCounter(source, "syncTimeNumOps", syncs + 1);
1404
1405
1406
1407
1408
1409
1410 Thread regionCloseThread = new Thread() {
1411 @Override
1412 public void run() {
1413 try {
1414 HRegion.closeHRegion(region);
1415 } catch (IOException e) {
1416 throw new RuntimeException(e);
1417 }
1418 }
1419 };
1420 regionCloseThread.start();
1421
1422 LOG.info("...releasing row lock 1, which should let put thread continue");
1423 rowLock1.release();
1424
1425 LOG.info("...waiting for put thread to sync 2nd time");
1426 waitForCounter(source, "syncTimeNumOps", syncs + 2);
1427
1428 LOG.info("...releasing row lock 2, which should let put thread continue");
1429 rowLock2.release();
1430
1431 LOG.info("...waiting for put thread to sync 3rd time");
1432 waitForCounter(source, "syncTimeNumOps", syncs + 3);
1433
1434 LOG.info("...releasing row lock 3, which should let put thread continue");
1435 rowLock3.release();
1436
1437 LOG.info("...waiting for put thread to sync 4th time");
1438 waitForCounter(source, "syncTimeNumOps", syncs + 4);
1439
1440 LOG.info("...joining on put thread");
1441 ctx.stop();
1442 regionCloseThread.join();
1443
1444 OperationStatus[] codes = retFromThread.get();
1445 for (int i = 0; i < codes.length; i++) {
1446 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1447 codes[i].getOperationStatusCode());
1448 }
1449 } finally {
1450 HRegion.closeHRegion(this.region);
1451 this.region = null;
1452 }
1453 }
1454
1455 private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount)
1456 throws InterruptedException {
1457 long startWait = System.currentTimeMillis();
1458 long currentCount;
1459 while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) {
1460 Thread.sleep(100);
1461 if (System.currentTimeMillis() - startWait > 10000) {
1462 fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName,
1463 expectedCount, currentCount));
1464 }
1465 }
1466 }
1467
1468 @Test
1469 public void testBatchPutWithTsSlop() throws Exception {
1470 byte[] b = Bytes.toBytes(getName());
1471 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1472 byte[] qual = Bytes.toBytes("qual");
1473 byte[] val = Bytes.toBytes("val");
1474
1475
1476 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1477 this.region = initHRegion(b, getName(), CONF, cf);
1478
1479 try {
1480 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1481 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1482 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1483
1484 final Put[] puts = new Put[10];
1485 for (int i = 0; i < 10; i++) {
1486 puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
1487 puts[i].add(cf, qual, val);
1488 }
1489
1490 OperationStatus[] codes = this.region.batchMutate(puts);
1491 assertEquals(10, codes.length);
1492 for (int i = 0; i < 10; i++) {
1493 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1494 }
1495 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1496
1497 } finally {
1498 HRegion.closeHRegion(this.region);
1499 this.region = null;
1500 }
1501
1502 }
1503
1504
1505
1506
1507 @Test
1508 public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1509 byte[] row1 = Bytes.toBytes("row1");
1510 byte[] fam1 = Bytes.toBytes("fam1");
1511 byte[] qf1 = Bytes.toBytes("qualifier");
1512 byte[] emptyVal = new byte[] {};
1513 byte[] val1 = Bytes.toBytes("value1");
1514 byte[] val2 = Bytes.toBytes("value2");
1515
1516
1517 String method = this.getName();
1518 this.region = initHRegion(tableName, method, CONF, fam1);
1519 try {
1520
1521 Put put = new Put(row1);
1522 put.add(fam1, qf1, emptyVal);
1523
1524
1525 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1526 emptyVal), put, true);
1527 assertTrue(res);
1528
1529
1530 put = new Put(row1);
1531 put.add(fam1, qf1, val1);
1532
1533
1534 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1535 put, true);
1536 assertTrue(res);
1537
1538
1539 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1540 put, true);
1541 assertFalse(res);
1542
1543 Delete delete = new Delete(row1);
1544 delete.deleteColumn(fam1, qf1);
1545 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1546 delete, true);
1547 assertFalse(res);
1548
1549 put = new Put(row1);
1550 put.add(fam1, qf1, val2);
1551
1552 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1553 put, true);
1554 assertTrue(res);
1555
1556
1557 delete = new Delete(row1);
1558 delete.deleteColumn(fam1, qf1);
1559 delete.deleteColumn(fam1, qf1);
1560 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1561 delete, true);
1562 assertTrue(res);
1563
1564 delete = new Delete(row1);
1565 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1566 delete, true);
1567 assertTrue(res);
1568
1569
1570 put = new Put(row1);
1571 put.add(fam1, qf1, val1);
1572
1573 res = region
1574 .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
1575 assertTrue(res);
1576 } finally {
1577 HRegion.closeHRegion(this.region);
1578 this.region = null;
1579 }
1580 }
1581
1582 @Test
1583 public void testCheckAndMutate_WithWrongValue() throws IOException {
1584 byte[] row1 = Bytes.toBytes("row1");
1585 byte[] fam1 = Bytes.toBytes("fam1");
1586 byte[] qf1 = Bytes.toBytes("qualifier");
1587 byte[] val1 = Bytes.toBytes("value1");
1588 byte[] val2 = Bytes.toBytes("value2");
1589
1590
1591 String method = this.getName();
1592 this.region = initHRegion(tableName, method, CONF, fam1);
1593 try {
1594
1595 Put put = new Put(row1);
1596 put.add(fam1, qf1, val1);
1597 region.put(put);
1598
1599
1600 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1601 val2), put, true);
1602 assertEquals(false, res);
1603
1604
1605 Delete delete = new Delete(row1);
1606 delete.deleteFamily(fam1);
1607 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1608 put, true);
1609 assertEquals(false, res);
1610 } finally {
1611 HRegion.closeHRegion(this.region);
1612 this.region = null;
1613 }
1614 }
1615
1616 @Test
1617 public void testCheckAndMutate_WithCorrectValue() throws IOException {
1618 byte[] row1 = Bytes.toBytes("row1");
1619 byte[] fam1 = Bytes.toBytes("fam1");
1620 byte[] qf1 = Bytes.toBytes("qualifier");
1621 byte[] val1 = Bytes.toBytes("value1");
1622
1623
1624 String method = this.getName();
1625 this.region = initHRegion(tableName, method, CONF, fam1);
1626 try {
1627
1628 Put put = new Put(row1);
1629 put.add(fam1, qf1, val1);
1630 region.put(put);
1631
1632
1633 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1634 val1), put, true);
1635 assertEquals(true, res);
1636
1637
1638 Delete delete = new Delete(row1);
1639 delete.deleteColumn(fam1, qf1);
1640 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1641 delete, true);
1642 assertEquals(true, res);
1643 } finally {
1644 HRegion.closeHRegion(this.region);
1645 this.region = null;
1646 }
1647 }
1648
1649 @Test
1650 public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1651 byte[] row1 = Bytes.toBytes("row1");
1652 byte[] fam1 = Bytes.toBytes("fam1");
1653 byte[] qf1 = Bytes.toBytes("qualifier");
1654 byte[] val1 = Bytes.toBytes("value1");
1655 byte[] val2 = Bytes.toBytes("value2");
1656 byte[] val3 = Bytes.toBytes("value3");
1657 byte[] val4 = Bytes.toBytes("value4");
1658
1659
1660 String method = this.getName();
1661 this.region = initHRegion(tableName, method, CONF, fam1);
1662 try {
1663
1664 Put put = new Put(row1);
1665 put.add(fam1, qf1, val3);
1666 region.put(put);
1667
1668
1669 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1670 new BinaryComparator(val3), put, true);
1671 assertEquals(false, res);
1672
1673
1674 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1675 new BinaryComparator(val4), put, true);
1676 assertEquals(false, res);
1677
1678
1679
1680 put = new Put(row1);
1681 put.add(fam1, qf1, val2);
1682 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1683 new BinaryComparator(val2), put, true);
1684 assertEquals(true, res);
1685
1686
1687 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1688 new BinaryComparator(val3), put, true);
1689 assertEquals(false, res);
1690
1691
1692
1693 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1694 new BinaryComparator(val2), put, true);
1695 assertEquals(true, res);
1696
1697
1698
1699 put = new Put(row1);
1700 put.add(fam1, qf1, val3);
1701 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1702 new BinaryComparator(val1), put, true);
1703 assertEquals(true, res);
1704
1705
1706 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1707 new BinaryComparator(val3), put, true);
1708 assertEquals(false, res);
1709
1710
1711 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1712 new BinaryComparator(val2), put, true);
1713 assertEquals(false, res);
1714
1715
1716
1717 put = new Put(row1);
1718 put.add(fam1, qf1, val2);
1719 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1720 new BinaryComparator(val4), put, true);
1721 assertEquals(true, res);
1722
1723
1724 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1725 new BinaryComparator(val1), put, true);
1726 assertEquals(false, res);
1727
1728
1729
1730 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1731 new BinaryComparator(val2), put, true);
1732 assertEquals(true, res);
1733
1734
1735 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1736 new BinaryComparator(val3), put, true);
1737 assertEquals(true, res);
1738 } finally {
1739 HRegion.closeHRegion(this.region);
1740 this.region = null;
1741 }
1742 }
1743
1744 @Test
1745 public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1746 byte[] row1 = Bytes.toBytes("row1");
1747 byte[] fam1 = Bytes.toBytes("fam1");
1748 byte[] fam2 = Bytes.toBytes("fam2");
1749 byte[] qf1 = Bytes.toBytes("qualifier");
1750 byte[] val1 = Bytes.toBytes("value1");
1751 byte[] val2 = Bytes.toBytes("value2");
1752
1753 byte[][] families = { fam1, fam2 };
1754
1755
1756 String method = this.getName();
1757 this.region = initHRegion(tableName, method, CONF, families);
1758 try {
1759
1760 Put put = new Put(row1);
1761 put.add(fam1, qf1, val1);
1762 region.put(put);
1763
1764
1765 long ts = System.currentTimeMillis();
1766 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1767 put = new Put(row1);
1768 put.add(kv);
1769
1770
1771 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1772 val1), put, true);
1773 assertEquals(true, res);
1774
1775 Get get = new Get(row1);
1776 get.addColumn(fam2, qf1);
1777 Cell[] actual = region.get(get).rawCells();
1778
1779 Cell[] expected = { kv };
1780
1781 assertEquals(expected.length, actual.length);
1782 for (int i = 0; i < actual.length; i++) {
1783 assertEquals(expected[i], actual[i]);
1784 }
1785 } finally {
1786 HRegion.closeHRegion(this.region);
1787 this.region = null;
1788 }
1789 }
1790
1791 @Test
1792 public void testCheckAndPut_wrongRowInPut() throws IOException {
1793 TableName tableName = TableName.valueOf(name.getMethodName());
1794 this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
1795 try {
1796 Put put = new Put(row2);
1797 put.add(fam1, qual1, value1);
1798 try {
1799 region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1800 new BinaryComparator(value2), put, false);
1801 fail();
1802 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1803
1804 }
1805 } finally {
1806 HRegion.closeHRegion(this.region);
1807 this.region = null;
1808 }
1809 }
1810
1811 @Test
1812 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1813 byte[] row1 = Bytes.toBytes("row1");
1814 byte[] fam1 = Bytes.toBytes("fam1");
1815 byte[] fam2 = Bytes.toBytes("fam2");
1816 byte[] qf1 = Bytes.toBytes("qualifier1");
1817 byte[] qf2 = Bytes.toBytes("qualifier2");
1818 byte[] qf3 = Bytes.toBytes("qualifier3");
1819 byte[] val1 = Bytes.toBytes("value1");
1820 byte[] val2 = Bytes.toBytes("value2");
1821 byte[] val3 = Bytes.toBytes("value3");
1822 byte[] emptyVal = new byte[] {};
1823
1824 byte[][] families = { fam1, fam2 };
1825
1826
1827 String method = this.getName();
1828 this.region = initHRegion(tableName, method, CONF, families);
1829 try {
1830
1831 Put put = new Put(row1);
1832 put.add(fam1, qf1, val1);
1833 region.put(put);
1834 Threads.sleep(2);
1835
1836 put = new Put(row1);
1837 put.add(fam1, qf1, val2);
1838 put.add(fam2, qf1, val3);
1839 put.add(fam2, qf2, val2);
1840 put.add(fam2, qf3, val1);
1841 put.add(fam1, qf3, val1);
1842 region.put(put);
1843
1844
1845 Delete delete = new Delete(row1);
1846 delete.deleteColumn(fam1, qf1);
1847 delete.deleteColumn(fam2, qf1);
1848 delete.deleteColumn(fam1, qf3);
1849 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1850 val2), delete, true);
1851 assertEquals(true, res);
1852
1853 Get get = new Get(row1);
1854 get.addColumn(fam1, qf1);
1855 get.addColumn(fam1, qf3);
1856 get.addColumn(fam2, qf2);
1857 Result r = region.get(get);
1858 assertEquals(2, r.size());
1859 assertArrayEquals(val1, r.getValue(fam1, qf1));
1860 assertArrayEquals(val2, r.getValue(fam2, qf2));
1861
1862
1863 delete = new Delete(row1);
1864 delete.deleteFamily(fam2);
1865 res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1866 delete, true);
1867 assertEquals(true, res);
1868
1869 get = new Get(row1);
1870 r = region.get(get);
1871 assertEquals(1, r.size());
1872 assertArrayEquals(val1, r.getValue(fam1, qf1));
1873
1874
1875 delete = new Delete(row1);
1876 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1877 delete, true);
1878 assertEquals(true, res);
1879 get = new Get(row1);
1880 r = region.get(get);
1881 assertEquals(0, r.size());
1882 } finally {
1883 HRegion.closeHRegion(this.region);
1884 this.region = null;
1885 }
1886 }
1887
1888
1889
1890
1891 @Test
1892 public void testDelete_multiDeleteColumn() throws IOException {
1893 byte[] row1 = Bytes.toBytes("row1");
1894 byte[] fam1 = Bytes.toBytes("fam1");
1895 byte[] qual = Bytes.toBytes("qualifier");
1896 byte[] value = Bytes.toBytes("value");
1897
1898 Put put = new Put(row1);
1899 put.add(fam1, qual, 1, value);
1900 put.add(fam1, qual, 2, value);
1901
1902 String method = this.getName();
1903 this.region = initHRegion(tableName, method, CONF, fam1);
1904 try {
1905 region.put(put);
1906
1907
1908 Delete delete = new Delete(row1);
1909 delete.deleteColumn(fam1, qual);
1910 delete.deleteColumn(fam1, qual);
1911 region.delete(delete);
1912
1913 Get get = new Get(row1);
1914 get.addFamily(fam1);
1915 Result r = region.get(get);
1916 assertEquals(0, r.size());
1917 } finally {
1918 HRegion.closeHRegion(this.region);
1919 this.region = null;
1920 }
1921 }
1922
1923 @Test
1924 public void testDelete_CheckFamily() throws IOException {
1925 byte[] row1 = Bytes.toBytes("row1");
1926 byte[] fam1 = Bytes.toBytes("fam1");
1927 byte[] fam2 = Bytes.toBytes("fam2");
1928 byte[] fam3 = Bytes.toBytes("fam3");
1929 byte[] fam4 = Bytes.toBytes("fam4");
1930
1931
1932 String method = this.getName();
1933 this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
1934 try {
1935 List<Cell> kvs = new ArrayList<Cell>();
1936 kvs.add(new KeyValue(row1, fam4, null, null));
1937
1938
1939 byte[] family = fam2;
1940 try {
1941 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1942 Bytes.BYTES_COMPARATOR);
1943 deleteMap.put(family, kvs);
1944 region.delete(deleteMap, Durability.SYNC_WAL);
1945 } catch (Exception e) {
1946 assertTrue("Family " + new String(family) + " does not exist", false);
1947 }
1948
1949
1950 boolean ok = false;
1951 family = fam4;
1952 try {
1953 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1954 Bytes.BYTES_COMPARATOR);
1955 deleteMap.put(family, kvs);
1956 region.delete(deleteMap, Durability.SYNC_WAL);
1957 } catch (Exception e) {
1958 ok = true;
1959 }
1960 assertEquals("Family " + new String(family) + " does exist", true, ok);
1961 } finally {
1962 HRegion.closeHRegion(this.region);
1963 this.region = null;
1964 }
1965 }
1966
1967 @Test
1968 public void testDelete_mixed() throws IOException, InterruptedException {
1969 byte[] fam = Bytes.toBytes("info");
1970 byte[][] families = { fam };
1971 String method = this.getName();
1972 this.region = initHRegion(tableName, method, CONF, families);
1973 try {
1974 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1975
1976 byte[] row = Bytes.toBytes("table_name");
1977
1978 byte[] serverinfo = Bytes.toBytes("serverinfo");
1979 byte[] splitA = Bytes.toBytes("splitA");
1980 byte[] splitB = Bytes.toBytes("splitB");
1981
1982
1983 Put put = new Put(row);
1984 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1985 region.put(put);
1986
1987 put = new Put(row);
1988 put.add(fam, splitB, Bytes.toBytes("reference_B"));
1989 region.put(put);
1990
1991 put = new Put(row);
1992 put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1993 region.put(put);
1994
1995
1996 Delete delete = new Delete(row);
1997 delete.deleteColumns(fam, splitA);
1998 region.delete(delete);
1999
2000
2001 Get get = new Get(row).addColumn(fam, serverinfo);
2002 Result result = region.get(get);
2003 assertEquals(1, result.size());
2004
2005 get = new Get(row).addColumn(fam, splitA);
2006 result = region.get(get);
2007 assertEquals(0, result.size());
2008
2009 get = new Get(row).addColumn(fam, splitB);
2010 result = region.get(get);
2011 assertEquals(1, result.size());
2012
2013
2014 put = new Put(row);
2015 put.add(fam, splitA, Bytes.toBytes("reference_A"));
2016 region.put(put);
2017 get = new Get(row);
2018 result = region.get(get);
2019 assertEquals(3, result.size());
2020
2021
2022 delete = new Delete(row);
2023 region.delete(delete);
2024 assertEquals(0, region.get(get).size());
2025
2026 region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
2027 result = region.get(get);
2028 assertEquals(1, result.size());
2029 } finally {
2030 HRegion.closeHRegion(this.region);
2031 this.region = null;
2032 }
2033 }
2034
2035 @Test
2036 public void testDeleteRowWithFutureTs() throws IOException {
2037 byte[] fam = Bytes.toBytes("info");
2038 byte[][] families = { fam };
2039 String method = this.getName();
2040 this.region = initHRegion(tableName, method, CONF, families);
2041 try {
2042 byte[] row = Bytes.toBytes("table_name");
2043
2044 byte[] serverinfo = Bytes.toBytes("serverinfo");
2045
2046
2047 Put put = new Put(row);
2048 put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
2049 region.put(put);
2050
2051
2052 Delete delete = new Delete(row);
2053 region.delete(delete);
2054
2055
2056 Get get = new Get(row).addColumn(fam, serverinfo);
2057 Result result = region.get(get);
2058 assertEquals(1, result.size());
2059
2060
2061 delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
2062 region.delete(delete);
2063
2064
2065 get = new Get(row).addColumn(fam, serverinfo);
2066 result = region.get(get);
2067 assertEquals(0, result.size());
2068 } finally {
2069 HRegion.closeHRegion(this.region);
2070 this.region = null;
2071 }
2072 }
2073
2074
2075
2076
2077
2078 @Test
2079 public void testPutWithLatestTS() throws IOException {
2080 byte[] fam = Bytes.toBytes("info");
2081 byte[][] families = { fam };
2082 String method = this.getName();
2083 this.region = initHRegion(tableName, method, CONF, families);
2084 try {
2085 byte[] row = Bytes.toBytes("row1");
2086
2087 byte[] qual = Bytes.toBytes("qual");
2088
2089
2090 Put put = new Put(row);
2091 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
2092 region.put(put);
2093
2094
2095 Get get = new Get(row).addColumn(fam, qual);
2096 Result result = region.get(get);
2097 assertEquals(1, result.size());
2098 Cell kv = result.rawCells()[0];
2099 LOG.info("Got: " + kv);
2100 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
2101 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
2102
2103
2104
2105 row = Bytes.toBytes("row2");
2106 put = new Put(row);
2107 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
2108 region.put(put);
2109
2110
2111 get = new Get(row).addColumn(fam, qual);
2112 result = region.get(get);
2113 assertEquals(1, result.size());
2114 kv = result.rawCells()[0];
2115 LOG.info("Got: " + kv);
2116 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
2117 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
2118 } finally {
2119 HRegion.closeHRegion(this.region);
2120 this.region = null;
2121 }
2122
2123 }
2124
2125
2126
2127
2128
2129
2130 @Test
2131 public void testPutWithTsSlop() throws IOException {
2132 byte[] fam = Bytes.toBytes("info");
2133 byte[][] families = { fam };
2134 String method = this.getName();
2135
2136
2137 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
2138 this.region = initHRegion(tableName, method, CONF, families);
2139 boolean caughtExcep = false;
2140 try {
2141 try {
2142
2143 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
2144
2145 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
2146 Bytes.toBytes("value")));
2147 fail("Expected IOE for TS out of configured timerange");
2148 } catch (FailedSanityCheckException ioe) {
2149 LOG.debug("Received expected exception", ioe);
2150 caughtExcep = true;
2151 }
2152 assertTrue("Should catch FailedSanityCheckException", caughtExcep);
2153 } finally {
2154 HRegion.closeHRegion(this.region);
2155 this.region = null;
2156 }
2157 }
2158
2159 @Test
2160 public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
2161 byte[] fam1 = Bytes.toBytes("columnA");
2162 byte[] fam2 = Bytes.toBytes("columnB");
2163 this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
2164 try {
2165 byte[] rowA = Bytes.toBytes("rowA");
2166 byte[] rowB = Bytes.toBytes("rowB");
2167
2168 byte[] value = Bytes.toBytes("value");
2169
2170 Delete delete = new Delete(rowA);
2171 delete.deleteFamily(fam1);
2172
2173 region.delete(delete);
2174
2175
2176 Put put = new Put(rowA);
2177 put.add(fam2, null, value);
2178 region.put(put);
2179
2180 put = new Put(rowB);
2181 put.add(fam1, null, value);
2182 put.add(fam2, null, value);
2183 region.put(put);
2184
2185 Scan scan = new Scan();
2186 scan.addFamily(fam1).addFamily(fam2);
2187 InternalScanner s = region.getScanner(scan);
2188 List<Cell> results = new ArrayList<Cell>();
2189 s.next(results);
2190 assertTrue(CellUtil.matchingRow(results.get(0), rowA));
2191
2192 results.clear();
2193 s.next(results);
2194 assertTrue(CellUtil.matchingRow(results.get(0), rowB));
2195 } finally {
2196 HRegion.closeHRegion(this.region);
2197 this.region = null;
2198 }
2199 }
2200
2201 @Test
2202 public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
2203 Delete delete = new Delete(row);
2204 delete.deleteColumns(fam1, qual1);
2205 doTestDelete_AndPostInsert(delete);
2206 }
2207
2208 @Test
2209 public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
2210 Delete delete = new Delete(row);
2211 delete.deleteFamily(fam1);
2212 doTestDelete_AndPostInsert(delete);
2213 }
2214
2215 public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
2216 TableName tableName = TableName.valueOf(name.getMethodName());
2217 this.region = initHRegion(tableName, getName(), CONF, fam1);
2218 try {
2219 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
2220 Put put = new Put(row);
2221 put.add(fam1, qual1, value1);
2222 region.put(put);
2223
2224
2225 region.delete(delete);
2226
2227
2228 put = new Put(row);
2229 put.add(fam1, qual1, value2);
2230 region.put(put);
2231
2232
2233 Get get = new Get(row);
2234 get.addColumn(fam1, qual1);
2235
2236 Result r = region.get(get);
2237 assertEquals(1, r.size());
2238 assertArrayEquals(value2, r.getValue(fam1, qual1));
2239
2240
2241 Scan scan = new Scan(row);
2242 scan.addColumn(fam1, qual1);
2243 InternalScanner s = region.getScanner(scan);
2244
2245 List<Cell> results = new ArrayList<Cell>();
2246 assertEquals(false, s.next(results));
2247 assertEquals(1, results.size());
2248 Cell kv = results.get(0);
2249
2250 assertArrayEquals(value2, CellUtil.cloneValue(kv));
2251 assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
2252 assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
2253 assertArrayEquals(row, CellUtil.cloneRow(kv));
2254 } finally {
2255 HRegion.closeHRegion(this.region);
2256 this.region = null;
2257 }
2258 }
2259
2260 @Test
2261 public void testDelete_CheckTimestampUpdated() throws IOException {
2262 TableName tableName = TableName.valueOf(name.getMethodName());
2263 byte[] row1 = Bytes.toBytes("row1");
2264 byte[] col1 = Bytes.toBytes("col1");
2265 byte[] col2 = Bytes.toBytes("col2");
2266 byte[] col3 = Bytes.toBytes("col3");
2267
2268
2269 String method = this.getName();
2270 this.region = initHRegion(tableName, method, CONF, fam1);
2271 try {
2272
2273 List<Cell> kvs = new ArrayList<Cell>();
2274 kvs.add(new KeyValue(row1, fam1, col1, null));
2275 kvs.add(new KeyValue(row1, fam1, col2, null));
2276 kvs.add(new KeyValue(row1, fam1, col3, null));
2277
2278 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
2279 Bytes.BYTES_COMPARATOR);
2280 deleteMap.put(fam1, kvs);
2281 region.delete(deleteMap, Durability.SYNC_WAL);
2282
2283
2284
2285 long now = System.currentTimeMillis();
2286 DefaultMemStore memstore = (DefaultMemStore) ((HStore) region.getStore(fam1)).memstore;
2287 Cell firstCell = memstore.cellSet.first();
2288 assertTrue(firstCell.getTimestamp() <= now);
2289 now = firstCell.getTimestamp();
2290 for (Cell cell : memstore.cellSet) {
2291 assertTrue(cell.getTimestamp() <= now);
2292 now = cell.getTimestamp();
2293 }
2294 } finally {
2295 HRegion.closeHRegion(this.region);
2296 this.region = null;
2297 }
2298 }
2299
2300
2301
2302
2303 @Test
2304 public void testGet_FamilyChecker() throws IOException {
2305 byte[] row1 = Bytes.toBytes("row1");
2306 byte[] fam1 = Bytes.toBytes("fam1");
2307 byte[] fam2 = Bytes.toBytes("False");
2308 byte[] col1 = Bytes.toBytes("col1");
2309
2310
2311 String method = this.getName();
2312 this.region = initHRegion(tableName, method, CONF, fam1);
2313 try {
2314 Get get = new Get(row1);
2315 get.addColumn(fam2, col1);
2316
2317
2318 try {
2319 region.get(get);
2320 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
2321 assertFalse(false);
2322 return;
2323 }
2324 assertFalse(true);
2325 } finally {
2326 HRegion.closeHRegion(this.region);
2327 this.region = null;
2328 }
2329 }
2330
2331 @Test
2332 public void testGet_Basic() throws IOException {
2333 byte[] row1 = Bytes.toBytes("row1");
2334 byte[] fam1 = Bytes.toBytes("fam1");
2335 byte[] col1 = Bytes.toBytes("col1");
2336 byte[] col2 = Bytes.toBytes("col2");
2337 byte[] col3 = Bytes.toBytes("col3");
2338 byte[] col4 = Bytes.toBytes("col4");
2339 byte[] col5 = Bytes.toBytes("col5");
2340
2341
2342 String method = this.getName();
2343 this.region = initHRegion(tableName, method, CONF, fam1);
2344 try {
2345
2346 Put put = new Put(row1);
2347 put.add(fam1, col1, null);
2348 put.add(fam1, col2, null);
2349 put.add(fam1, col3, null);
2350 put.add(fam1, col4, null);
2351 put.add(fam1, col5, null);
2352 region.put(put);
2353
2354 Get get = new Get(row1);
2355 get.addColumn(fam1, col2);
2356 get.addColumn(fam1, col4);
2357
2358 KeyValue kv1 = new KeyValue(row1, fam1, col2);
2359 KeyValue kv2 = new KeyValue(row1, fam1, col4);
2360 KeyValue[] expected = { kv1, kv2 };
2361
2362
2363 Result res = region.get(get);
2364 assertEquals(expected.length, res.size());
2365 for (int i = 0; i < res.size(); i++) {
2366 assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
2367 assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
2368 assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
2369 }
2370
2371
2372 Get g = new Get(row1);
2373 final int count = 2;
2374 g.setFilter(new ColumnCountGetFilter(count));
2375 res = region.get(g);
2376 assertEquals(count, res.size());
2377 } finally {
2378 HRegion.closeHRegion(this.region);
2379 this.region = null;
2380 }
2381 }
2382
2383 @Test
2384 public void testGet_Empty() throws IOException {
2385 byte[] row = Bytes.toBytes("row");
2386 byte[] fam = Bytes.toBytes("fam");
2387
2388 String method = this.getName();
2389 this.region = initHRegion(tableName, method, CONF, fam);
2390 try {
2391 Get get = new Get(row);
2392 get.addFamily(fam);
2393 Result r = region.get(get);
2394
2395 assertTrue(r.isEmpty());
2396 } finally {
2397 HRegion.closeHRegion(this.region);
2398 this.region = null;
2399 }
2400 }
2401
2402
2403
2404
2405 @Test
2406 public void testMerge() throws IOException {
2407 byte[][] families = { fam1, fam2, fam3 };
2408 Configuration hc = initSplit();
2409
2410 String method = this.getName();
2411 this.region = initHRegion(tableName, method, hc, families);
2412 try {
2413 LOG.info("" + HBaseTestCase.addContent(region, fam3));
2414 region.flushcache();
2415 region.compactStores();
2416 byte[] splitRow = region.checkSplit();
2417 assertNotNull(splitRow);
2418 LOG.info("SplitRow: " + Bytes.toString(splitRow));
2419 HRegion[] subregions = splitRegion(region, splitRow);
2420 try {
2421
2422 for (int i = 0; i < subregions.length; i++) {
2423 HRegion.openHRegion(subregions[i], null);
2424 subregions[i].compactStores();
2425 }
2426 Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
2427 Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
2428 Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
2429 long startTime = System.currentTimeMillis();
2430 region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
2431 LOG.info("Merge regions elapsed time: "
2432 + ((System.currentTimeMillis() - startTime) / 1000.0));
2433 FILESYSTEM.delete(oldRegion1, true);
2434 FILESYSTEM.delete(oldRegion2, true);
2435 FILESYSTEM.delete(oldRegionPath, true);
2436 LOG.info("splitAndMerge completed.");
2437 } finally {
2438 for (int i = 0; i < subregions.length; i++) {
2439 try {
2440 HRegion.closeHRegion(subregions[i]);
2441 } catch (IOException e) {
2442
2443 }
2444 }
2445 }
2446 } finally {
2447 HRegion.closeHRegion(this.region);
2448 this.region = null;
2449 }
2450 }
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460 HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
2461 PairOfSameType<HRegion> result = null;
2462 SplitTransaction st = new SplitTransaction(parent, midkey);
2463
2464
2465 if (!st.prepare()) {
2466 parent.clearSplit();
2467 return null;
2468 }
2469 try {
2470 result = st.execute(null, null);
2471 } catch (IOException ioe) {
2472 try {
2473 LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; "
2474 + ioe.getMessage());
2475 st.rollback(null, null);
2476 LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString());
2477 return null;
2478 } catch (RuntimeException e) {
2479
2480 LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString()
2481 + " -- aborting server", e);
2482 }
2483 }
2484 finally {
2485 parent.clearSplit();
2486 }
2487 return new HRegion[] { result.getFirst(), result.getSecond() };
2488 }
2489
2490
2491
2492
2493 @Test
2494 public void testGetScanner_WithOkFamilies() throws IOException {
2495 byte[] fam1 = Bytes.toBytes("fam1");
2496 byte[] fam2 = Bytes.toBytes("fam2");
2497
2498 byte[][] families = { fam1, fam2 };
2499
2500
2501 String method = this.getName();
2502 this.region = initHRegion(tableName, method, CONF, families);
2503 try {
2504 Scan scan = new Scan();
2505 scan.addFamily(fam1);
2506 scan.addFamily(fam2);
2507 try {
2508 region.getScanner(scan);
2509 } catch (Exception e) {
2510 assertTrue("Families could not be found in Region", false);
2511 }
2512 } finally {
2513 HRegion.closeHRegion(this.region);
2514 this.region = null;
2515 }
2516 }
2517
2518 @Test
2519 public void testGetScanner_WithNotOkFamilies() throws IOException {
2520 byte[] fam1 = Bytes.toBytes("fam1");
2521 byte[] fam2 = Bytes.toBytes("fam2");
2522
2523 byte[][] families = { fam1 };
2524
2525
2526 String method = this.getName();
2527 this.region = initHRegion(tableName, method, CONF, families);
2528 try {
2529 Scan scan = new Scan();
2530 scan.addFamily(fam2);
2531 boolean ok = false;
2532 try {
2533 region.getScanner(scan);
2534 } catch (Exception e) {
2535 ok = true;
2536 }
2537 assertTrue("Families could not be found in Region", ok);
2538 } finally {
2539 HRegion.closeHRegion(this.region);
2540 this.region = null;
2541 }
2542 }
2543
2544 @Test
2545 public void testGetScanner_WithNoFamilies() throws IOException {
2546 byte[] row1 = Bytes.toBytes("row1");
2547 byte[] fam1 = Bytes.toBytes("fam1");
2548 byte[] fam2 = Bytes.toBytes("fam2");
2549 byte[] fam3 = Bytes.toBytes("fam3");
2550 byte[] fam4 = Bytes.toBytes("fam4");
2551
2552 byte[][] families = { fam1, fam2, fam3, fam4 };
2553
2554
2555 String method = this.getName();
2556 this.region = initHRegion(tableName, method, CONF, families);
2557 try {
2558
2559
2560 Put put = new Put(row1);
2561 put.add(fam1, null, null);
2562 put.add(fam2, null, null);
2563 put.add(fam3, null, null);
2564 put.add(fam4, null, null);
2565 region.put(put);
2566
2567 Scan scan = null;
2568 HRegion.RegionScannerImpl is = null;
2569
2570
2571
2572
2573 scan = new Scan();
2574 scan.addFamily(fam2);
2575 scan.addFamily(fam4);
2576 is = (RegionScannerImpl) region.getScanner(scan);
2577 assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2578
2579 scan = new Scan();
2580 is = (RegionScannerImpl) region.getScanner(scan);
2581 assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2582 } finally {
2583 HRegion.closeHRegion(this.region);
2584 this.region = null;
2585 }
2586 }
2587
2588
2589
2590
2591
2592
2593 @Test
2594 public void testGetScanner_WithRegionClosed() throws IOException {
2595 byte[] fam1 = Bytes.toBytes("fam1");
2596 byte[] fam2 = Bytes.toBytes("fam2");
2597
2598 byte[][] families = { fam1, fam2 };
2599
2600
2601 String method = this.getName();
2602 try {
2603 this.region = initHRegion(tableName, method, CONF, families);
2604 } catch (IOException e) {
2605 e.printStackTrace();
2606 fail("Got IOException during initHRegion, " + e.getMessage());
2607 }
2608 try {
2609 region.closed.set(true);
2610 try {
2611 region.getScanner(null);
2612 fail("Expected to get an exception during getScanner on a region that is closed");
2613 } catch (NotServingRegionException e) {
2614
2615 } catch (IOException e) {
2616 fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2617 + e.getMessage());
2618 }
2619 } finally {
2620 HRegion.closeHRegion(this.region);
2621 this.region = null;
2622 }
2623 }
2624
2625 @Test
2626 public void testRegionScanner_Next() throws IOException {
2627 byte[] row1 = Bytes.toBytes("row1");
2628 byte[] row2 = Bytes.toBytes("row2");
2629 byte[] fam1 = Bytes.toBytes("fam1");
2630 byte[] fam2 = Bytes.toBytes("fam2");
2631 byte[] fam3 = Bytes.toBytes("fam3");
2632 byte[] fam4 = Bytes.toBytes("fam4");
2633
2634 byte[][] families = { fam1, fam2, fam3, fam4 };
2635 long ts = System.currentTimeMillis();
2636
2637
2638 String method = this.getName();
2639 this.region = initHRegion(tableName, method, CONF, families);
2640 try {
2641
2642 Put put = null;
2643 put = new Put(row1);
2644 put.add(fam1, (byte[]) null, ts, null);
2645 put.add(fam2, (byte[]) null, ts, null);
2646 put.add(fam3, (byte[]) null, ts, null);
2647 put.add(fam4, (byte[]) null, ts, null);
2648 region.put(put);
2649
2650 put = new Put(row2);
2651 put.add(fam1, (byte[]) null, ts, null);
2652 put.add(fam2, (byte[]) null, ts, null);
2653 put.add(fam3, (byte[]) null, ts, null);
2654 put.add(fam4, (byte[]) null, ts, null);
2655 region.put(put);
2656
2657 Scan scan = new Scan();
2658 scan.addFamily(fam2);
2659 scan.addFamily(fam4);
2660 InternalScanner is = region.getScanner(scan);
2661
2662 List<Cell> res = null;
2663
2664
2665 List<Cell> expected1 = new ArrayList<Cell>();
2666 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2667 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2668
2669 res = new ArrayList<Cell>();
2670 is.next(res);
2671 for (int i = 0; i < res.size(); i++) {
2672 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2673 }
2674
2675
2676 List<Cell> expected2 = new ArrayList<Cell>();
2677 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2678 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2679
2680 res = new ArrayList<Cell>();
2681 is.next(res);
2682 for (int i = 0; i < res.size(); i++) {
2683 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2684 }
2685 } finally {
2686 HRegion.closeHRegion(this.region);
2687 this.region = null;
2688 }
2689 }
2690
2691 @Test
2692 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2693 byte[] row1 = Bytes.toBytes("row1");
2694 byte[] qf1 = Bytes.toBytes("qualifier1");
2695 byte[] qf2 = Bytes.toBytes("qualifier2");
2696 byte[] fam1 = Bytes.toBytes("fam1");
2697 byte[][] families = { fam1 };
2698
2699 long ts1 = System.currentTimeMillis();
2700 long ts2 = ts1 + 1;
2701 long ts3 = ts1 + 2;
2702
2703
2704 String method = this.getName();
2705 this.region = initHRegion(tableName, method, CONF, families);
2706 try {
2707
2708 Put put = null;
2709 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2710 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2711 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2712
2713 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2714 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2715 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2716
2717 put = new Put(row1);
2718 put.add(kv13);
2719 put.add(kv12);
2720 put.add(kv11);
2721 put.add(kv23);
2722 put.add(kv22);
2723 put.add(kv21);
2724 region.put(put);
2725
2726
2727 List<Cell> expected = new ArrayList<Cell>();
2728 expected.add(kv13);
2729 expected.add(kv12);
2730
2731 Scan scan = new Scan(row1);
2732 scan.addColumn(fam1, qf1);
2733 scan.setMaxVersions(MAX_VERSIONS);
2734 List<Cell> actual = new ArrayList<Cell>();
2735 InternalScanner scanner = region.getScanner(scan);
2736
2737 boolean hasNext = scanner.next(actual);
2738 assertEquals(false, hasNext);
2739
2740
2741 for (int i = 0; i < expected.size(); i++) {
2742 assertEquals(expected.get(i), actual.get(i));
2743 }
2744 } finally {
2745 HRegion.closeHRegion(this.region);
2746 this.region = null;
2747 }
2748 }
2749
2750 @Test
2751 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2752 byte[] row1 = Bytes.toBytes("row1");
2753 byte[] qf1 = Bytes.toBytes("qualifier1");
2754 byte[] qf2 = Bytes.toBytes("qualifier2");
2755 byte[] fam1 = Bytes.toBytes("fam1");
2756 byte[][] families = { fam1 };
2757
2758 long ts1 = 1;
2759 long ts2 = ts1 + 1;
2760 long ts3 = ts1 + 2;
2761
2762
2763 String method = this.getName();
2764 this.region = initHRegion(tableName, method, CONF, families);
2765 try {
2766
2767 Put put = null;
2768 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2769 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2770 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2771
2772 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2773 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2774 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2775
2776 put = new Put(row1);
2777 put.add(kv13);
2778 put.add(kv12);
2779 put.add(kv11);
2780 put.add(kv23);
2781 put.add(kv22);
2782 put.add(kv21);
2783 region.put(put);
2784 region.flushcache();
2785
2786
2787 List<Cell> expected = new ArrayList<Cell>();
2788 expected.add(kv13);
2789 expected.add(kv12);
2790 expected.add(kv23);
2791 expected.add(kv22);
2792
2793 Scan scan = new Scan(row1);
2794 scan.addColumn(fam1, qf1);
2795 scan.addColumn(fam1, qf2);
2796 scan.setMaxVersions(MAX_VERSIONS);
2797 List<Cell> actual = new ArrayList<Cell>();
2798 InternalScanner scanner = region.getScanner(scan);
2799
2800 boolean hasNext = scanner.next(actual);
2801 assertEquals(false, hasNext);
2802
2803
2804 for (int i = 0; i < expected.size(); i++) {
2805 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2806 }
2807 } finally {
2808 HRegion.closeHRegion(this.region);
2809 this.region = null;
2810 }
2811 }
2812
2813 @Test
2814 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2815 byte[] row1 = Bytes.toBytes("row1");
2816 byte[] fam1 = Bytes.toBytes("fam1");
2817 byte[][] families = { fam1 };
2818 byte[] qf1 = Bytes.toBytes("qualifier1");
2819 byte[] qf2 = Bytes.toBytes("qualifier2");
2820
2821 long ts1 = 1;
2822 long ts2 = ts1 + 1;
2823 long ts3 = ts1 + 2;
2824 long ts4 = ts1 + 3;
2825
2826
2827 String method = this.getName();
2828 this.region = initHRegion(tableName, method, CONF, families);
2829 try {
2830
2831 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2832 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2833 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2834 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2835
2836 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2837 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2838 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2839 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2840
2841 Put put = null;
2842 put = new Put(row1);
2843 put.add(kv14);
2844 put.add(kv24);
2845 region.put(put);
2846 region.flushcache();
2847
2848 put = new Put(row1);
2849 put.add(kv23);
2850 put.add(kv13);
2851 region.put(put);
2852 region.flushcache();
2853
2854 put = new Put(row1);
2855 put.add(kv22);
2856 put.add(kv12);
2857 region.put(put);
2858 region.flushcache();
2859
2860 put = new Put(row1);
2861 put.add(kv21);
2862 put.add(kv11);
2863 region.put(put);
2864
2865
2866 List<Cell> expected = new ArrayList<Cell>();
2867 expected.add(kv14);
2868 expected.add(kv13);
2869 expected.add(kv12);
2870 expected.add(kv24);
2871 expected.add(kv23);
2872 expected.add(kv22);
2873
2874 Scan scan = new Scan(row1);
2875 scan.addColumn(fam1, qf1);
2876 scan.addColumn(fam1, qf2);
2877 int versions = 3;
2878 scan.setMaxVersions(versions);
2879 List<Cell> actual = new ArrayList<Cell>();
2880 InternalScanner scanner = region.getScanner(scan);
2881
2882 boolean hasNext = scanner.next(actual);
2883 assertEquals(false, hasNext);
2884
2885
2886 for (int i = 0; i < expected.size(); i++) {
2887 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2888 }
2889 } finally {
2890 HRegion.closeHRegion(this.region);
2891 this.region = null;
2892 }
2893 }
2894
2895 @Test
2896 public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2897 byte[] row1 = Bytes.toBytes("row1");
2898 byte[] qf1 = Bytes.toBytes("qualifier1");
2899 byte[] qf2 = Bytes.toBytes("qualifier2");
2900 byte[] fam1 = Bytes.toBytes("fam1");
2901 byte[][] families = { fam1 };
2902
2903 long ts1 = System.currentTimeMillis();
2904 long ts2 = ts1 + 1;
2905 long ts3 = ts1 + 2;
2906
2907
2908 String method = this.getName();
2909 this.region = initHRegion(tableName, method, CONF, families);
2910 try {
2911
2912 Put put = null;
2913 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2914 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2915 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2916
2917 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2918 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2919 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2920
2921 put = new Put(row1);
2922 put.add(kv13);
2923 put.add(kv12);
2924 put.add(kv11);
2925 put.add(kv23);
2926 put.add(kv22);
2927 put.add(kv21);
2928 region.put(put);
2929
2930
2931 List<Cell> expected = new ArrayList<Cell>();
2932 expected.add(kv13);
2933 expected.add(kv12);
2934 expected.add(kv23);
2935 expected.add(kv22);
2936
2937 Scan scan = new Scan(row1);
2938 scan.addFamily(fam1);
2939 scan.setMaxVersions(MAX_VERSIONS);
2940 List<Cell> actual = new ArrayList<Cell>();
2941 InternalScanner scanner = region.getScanner(scan);
2942
2943 boolean hasNext = scanner.next(actual);
2944 assertEquals(false, hasNext);
2945
2946
2947 for (int i = 0; i < expected.size(); i++) {
2948 assertEquals(expected.get(i), actual.get(i));
2949 }
2950 } finally {
2951 HRegion.closeHRegion(this.region);
2952 this.region = null;
2953 }
2954 }
2955
2956 @Test
2957 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
2958 byte[] row1 = Bytes.toBytes("row1");
2959 byte[] qf1 = Bytes.toBytes("qualifier1");
2960 byte[] qf2 = Bytes.toBytes("qualifier2");
2961 byte[] fam1 = Bytes.toBytes("fam1");
2962
2963 long ts1 = 1;
2964 long ts2 = ts1 + 1;
2965 long ts3 = ts1 + 2;
2966
2967
2968 String method = this.getName();
2969 this.region = initHRegion(tableName, method, CONF, fam1);
2970 try {
2971
2972 Put put = null;
2973 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2974 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2975 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2976
2977 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2978 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2979 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2980
2981 put = new Put(row1);
2982 put.add(kv13);
2983 put.add(kv12);
2984 put.add(kv11);
2985 put.add(kv23);
2986 put.add(kv22);
2987 put.add(kv21);
2988 region.put(put);
2989 region.flushcache();
2990
2991
2992 List<Cell> expected = new ArrayList<Cell>();
2993 expected.add(kv13);
2994 expected.add(kv12);
2995 expected.add(kv23);
2996 expected.add(kv22);
2997
2998 Scan scan = new Scan(row1);
2999 scan.addFamily(fam1);
3000 scan.setMaxVersions(MAX_VERSIONS);
3001 List<Cell> actual = new ArrayList<Cell>();
3002 InternalScanner scanner = region.getScanner(scan);
3003
3004 boolean hasNext = scanner.next(actual);
3005 assertEquals(false, hasNext);
3006
3007
3008 for (int i = 0; i < expected.size(); i++) {
3009 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
3010 }
3011 } finally {
3012 HRegion.closeHRegion(this.region);
3013 this.region = null;
3014 }
3015 }
3016
3017 @Test
3018 public void testScanner_StopRow1542() throws IOException {
3019 byte[] family = Bytes.toBytes("testFamily");
3020 this.region = initHRegion(tableName, getName(), CONF, family);
3021 try {
3022 byte[] row1 = Bytes.toBytes("row111");
3023 byte[] row2 = Bytes.toBytes("row222");
3024 byte[] row3 = Bytes.toBytes("row333");
3025 byte[] row4 = Bytes.toBytes("row444");
3026 byte[] row5 = Bytes.toBytes("row555");
3027
3028 byte[] col1 = Bytes.toBytes("Pub111");
3029 byte[] col2 = Bytes.toBytes("Pub222");
3030
3031 Put put = new Put(row1);
3032 put.add(family, col1, Bytes.toBytes(10L));
3033 region.put(put);
3034
3035 put = new Put(row2);
3036 put.add(family, col1, Bytes.toBytes(15L));
3037 region.put(put);
3038
3039 put = new Put(row3);
3040 put.add(family, col2, Bytes.toBytes(20L));
3041 region.put(put);
3042
3043 put = new Put(row4);
3044 put.add(family, col2, Bytes.toBytes(30L));
3045 region.put(put);
3046
3047 put = new Put(row5);
3048 put.add(family, col1, Bytes.toBytes(40L));
3049 region.put(put);
3050
3051 Scan scan = new Scan(row3, row4);
3052 scan.setMaxVersions();
3053 scan.addColumn(family, col1);
3054 InternalScanner s = region.getScanner(scan);
3055
3056 List<Cell> results = new ArrayList<Cell>();
3057 assertEquals(false, s.next(results));
3058 assertEquals(0, results.size());
3059 } finally {
3060 HRegion.closeHRegion(this.region);
3061 this.region = null;
3062 }
3063 }
3064
3065 @Test
3066 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
3067 byte[] row1 = Bytes.toBytes("row1");
3068 byte[] fam1 = Bytes.toBytes("fam1");
3069 byte[] qf1 = Bytes.toBytes("qualifier1");
3070 byte[] qf2 = Bytes.toBytes("quateslifier2");
3071
3072 long ts1 = 1;
3073 long ts2 = ts1 + 1;
3074 long ts3 = ts1 + 2;
3075 long ts4 = ts1 + 3;
3076
3077
3078 String method = this.getName();
3079 this.region = initHRegion(tableName, method, CONF, fam1);
3080 try {
3081
3082 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
3083 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
3084 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
3085 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
3086
3087 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
3088 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
3089 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
3090 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
3091
3092 Put put = null;
3093 put = new Put(row1);
3094 put.add(kv14);
3095 put.add(kv24);
3096 region.put(put);
3097 region.flushcache();
3098
3099 put = new Put(row1);
3100 put.add(kv23);
3101 put.add(kv13);
3102 region.put(put);
3103 region.flushcache();
3104
3105 put = new Put(row1);
3106 put.add(kv22);
3107 put.add(kv12);
3108 region.put(put);
3109 region.flushcache();
3110
3111 put = new Put(row1);
3112 put.add(kv21);
3113 put.add(kv11);
3114 region.put(put);
3115
3116
3117 List<KeyValue> expected = new ArrayList<KeyValue>();
3118 expected.add(kv14);
3119 expected.add(kv13);
3120 expected.add(kv12);
3121 expected.add(kv24);
3122 expected.add(kv23);
3123 expected.add(kv22);
3124
3125 Scan scan = new Scan(row1);
3126 int versions = 3;
3127 scan.setMaxVersions(versions);
3128 List<Cell> actual = new ArrayList<Cell>();
3129 InternalScanner scanner = region.getScanner(scan);
3130
3131 boolean hasNext = scanner.next(actual);
3132 assertEquals(false, hasNext);
3133
3134
3135 for (int i = 0; i < expected.size(); i++) {
3136 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
3137 }
3138 } finally {
3139 HRegion.closeHRegion(this.region);
3140 this.region = null;
3141 }
3142 }
3143
3144
3145
3146
3147
3148
3149
3150 @Test
3151 public void testScanner_JoinedScanners() throws IOException {
3152 byte[] cf_essential = Bytes.toBytes("essential");
3153 byte[] cf_joined = Bytes.toBytes("joined");
3154 byte[] cf_alpha = Bytes.toBytes("alpha");
3155 this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
3156 try {
3157 byte[] row1 = Bytes.toBytes("row1");
3158 byte[] row2 = Bytes.toBytes("row2");
3159 byte[] row3 = Bytes.toBytes("row3");
3160
3161 byte[] col_normal = Bytes.toBytes("d");
3162 byte[] col_alpha = Bytes.toBytes("a");
3163
3164 byte[] filtered_val = Bytes.toBytes(3);
3165
3166 Put put = new Put(row1);
3167 put.add(cf_essential, col_normal, Bytes.toBytes(1));
3168 put.add(cf_joined, col_alpha, Bytes.toBytes(1));
3169 region.put(put);
3170
3171 put = new Put(row2);
3172 put.add(cf_essential, col_alpha, Bytes.toBytes(2));
3173 put.add(cf_joined, col_normal, Bytes.toBytes(2));
3174 put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
3175 region.put(put);
3176
3177 put = new Put(row3);
3178 put.add(cf_essential, col_normal, filtered_val);
3179 put.add(cf_joined, col_normal, filtered_val);
3180 region.put(put);
3181
3182
3183
3184
3185
3186 Scan scan = new Scan();
3187 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
3188 CompareOp.NOT_EQUAL, filtered_val);
3189 scan.setFilter(filter);
3190 scan.setLoadColumnFamiliesOnDemand(true);
3191 InternalScanner s = region.getScanner(scan);
3192
3193 List<Cell> results = new ArrayList<Cell>();
3194 assertTrue(s.next(results));
3195 assertEquals(results.size(), 1);
3196 results.clear();
3197
3198 assertTrue(s.next(results));
3199 assertEquals(results.size(), 3);
3200 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
3201 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
3202 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
3203 results.clear();
3204
3205 assertFalse(s.next(results));
3206 assertEquals(results.size(), 0);
3207 } finally {
3208 HRegion.closeHRegion(this.region);
3209 this.region = null;
3210 }
3211 }
3212
3213
3214
3215
3216
3217
3218 @Test
3219 public void testScanner_JoinedScannersWithLimits() throws IOException {
3220 final byte[] cf_first = Bytes.toBytes("first");
3221 final byte[] cf_second = Bytes.toBytes("second");
3222
3223 this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
3224 try {
3225 final byte[] col_a = Bytes.toBytes("a");
3226 final byte[] col_b = Bytes.toBytes("b");
3227
3228 Put put;
3229
3230 for (int i = 0; i < 10; i++) {
3231 put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
3232 put.add(cf_first, col_a, Bytes.toBytes(i));
3233 if (i < 5) {
3234 put.add(cf_first, col_b, Bytes.toBytes(i));
3235 put.add(cf_second, col_a, Bytes.toBytes(i));
3236 put.add(cf_second, col_b, Bytes.toBytes(i));
3237 }
3238 region.put(put);
3239 }
3240
3241 Scan scan = new Scan();
3242 scan.setLoadColumnFamiliesOnDemand(true);
3243 Filter bogusFilter = new FilterBase() {
3244 @Override
3245 public ReturnCode filterKeyValue(Cell ignored) throws IOException {
3246 return ReturnCode.INCLUDE;
3247 }
3248 @Override
3249 public boolean isFamilyEssential(byte[] name) {
3250 return Bytes.equals(name, cf_first);
3251 }
3252 };
3253
3254 scan.setFilter(bogusFilter);
3255 InternalScanner s = region.getScanner(scan);
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286 List<Cell> results = new ArrayList<Cell>();
3287 int index = 0;
3288 while (true) {
3289 boolean more = s.next(results, 3);
3290 if ((index >> 1) < 5) {
3291 if (index % 2 == 0)
3292 assertEquals(results.size(), 3);
3293 else
3294 assertEquals(results.size(), 1);
3295 } else
3296 assertEquals(results.size(), 1);
3297 results.clear();
3298 index++;
3299 if (!more)
3300 break;
3301 }
3302 } finally {
3303 HRegion.closeHRegion(this.region);
3304 this.region = null;
3305 }
3306 }
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316 @Test
3317 public void testBasicSplit() throws Exception {
3318 byte[][] families = { fam1, fam2, fam3 };
3319
3320 Configuration hc = initSplit();
3321
3322 String method = this.getName();
3323 this.region = initHRegion(tableName, method, hc, families);
3324
3325 try {
3326 LOG.info("" + HBaseTestCase.addContent(region, fam3));
3327 region.flushcache();
3328 region.compactStores();
3329 byte[] splitRow = region.checkSplit();
3330 assertNotNull(splitRow);
3331 LOG.info("SplitRow: " + Bytes.toString(splitRow));
3332 HRegion[] regions = splitRegion(region, splitRow);
3333 try {
3334
3335
3336
3337 for (int i = 0; i < regions.length; i++) {
3338 regions[i] = HRegion.openHRegion(regions[i], null);
3339 }
3340
3341
3342 assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
3343 assertGet(regions[1], fam3, splitRow);
3344
3345 assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
3346 assertScan(regions[1], fam3, splitRow);
3347
3348 for (int i = 0; i < regions.length; i++) {
3349
3350
3351 for (int j = 0; j < 2; j++) {
3352 HBaseTestCase.addContent(regions[i], fam3);
3353 }
3354 HBaseTestCase.addContent(regions[i], fam2);
3355 HBaseTestCase.addContent(regions[i], fam1);
3356 regions[i].flushcache();
3357 }
3358
3359 byte[][] midkeys = new byte[regions.length][];
3360
3361 for (int i = 0; i < regions.length; i++) {
3362 regions[i].compactStores();
3363 midkeys[i] = regions[i].checkSplit();
3364 }
3365
3366 TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
3367
3368
3369 for (int i = 0; i < regions.length; i++) {
3370 HRegion[] rs = null;
3371 if (midkeys[i] != null) {
3372 rs = splitRegion(regions[i], midkeys[i]);
3373 for (int j = 0; j < rs.length; j++) {
3374 sortedMap.put(Bytes.toString(rs[j].getRegionName()), HRegion.openHRegion(rs[j], null));
3375 }
3376 }
3377 }
3378 LOG.info("Made 4 regions");
3379
3380
3381 int interval = (LAST_CHAR - FIRST_CHAR) / 3;
3382 byte[] b = Bytes.toBytes(START_KEY);
3383 for (HRegion r : sortedMap.values()) {
3384 assertGet(r, fam3, b);
3385 b[0] += interval;
3386 }
3387 } finally {
3388 for (int i = 0; i < regions.length; i++) {
3389 try {
3390 regions[i].close();
3391 } catch (IOException e) {
3392
3393 }
3394 }
3395 }
3396 } finally {
3397 HRegion.closeHRegion(this.region);
3398 this.region = null;
3399 }
3400 }
3401
3402 @Test
3403 public void testSplitRegion() throws IOException {
3404 byte[] qualifier = Bytes.toBytes("qualifier");
3405 Configuration hc = initSplit();
3406 int numRows = 10;
3407 byte[][] families = { fam1, fam3 };
3408
3409
3410 String method = this.getName();
3411 this.region = initHRegion(tableName, method, hc, families);
3412
3413
3414 int startRow = 100;
3415 putData(startRow, numRows, qualifier, families);
3416 int splitRow = startRow + numRows;
3417 putData(splitRow, numRows, qualifier, families);
3418 region.flushcache();
3419
3420 HRegion[] regions = null;
3421 try {
3422 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
3423
3424 for (int i = 0; i < regions.length; i++) {
3425 regions[i] = HRegion.openHRegion(regions[i], null);
3426 }
3427
3428 assertEquals(2, regions.length);
3429
3430
3431
3432 verifyData(regions[0], startRow, numRows, qualifier, families);
3433 verifyData(regions[1], splitRow, numRows, qualifier, families);
3434
3435 } finally {
3436 HRegion.closeHRegion(this.region);
3437 this.region = null;
3438 }
3439 }
3440
3441 @Test
3442 public void testClearForceSplit() throws IOException {
3443 byte[] qualifier = Bytes.toBytes("qualifier");
3444 Configuration hc = initSplit();
3445 int numRows = 10;
3446 byte[][] families = { fam1, fam3 };
3447
3448
3449 String method = this.getName();
3450 this.region = initHRegion(tableName, method, hc, families);
3451
3452
3453 int startRow = 100;
3454 putData(startRow, numRows, qualifier, families);
3455 int splitRow = startRow + numRows;
3456 byte[] splitRowBytes = Bytes.toBytes("" + splitRow);
3457 putData(splitRow, numRows, qualifier, families);
3458 region.flushcache();
3459
3460 HRegion[] regions = null;
3461 try {
3462
3463 region.forceSplit(splitRowBytes);
3464 assertTrue(region.shouldForceSplit());
3465
3466 assertTrue(Bytes.equals(splitRowBytes, region.checkSplit()));
3467
3468
3469 HStore storeMock = Mockito.mock(HStore.class);
3470 when(storeMock.hasReferences()).thenReturn(true);
3471 when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
3472 when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
3473 when(storeMock.getColumnFamilyName()).thenReturn("cf");
3474 region.stores.put(Bytes.toBytes(storeMock.getColumnFamilyName()), storeMock);
3475 assertTrue(region.hasReferences());
3476
3477
3478 regions = splitRegion(region, splitRowBytes);
3479 assertNull(regions);
3480
3481
3482 assertFalse(region.shouldForceSplit());
3483
3484
3485 region.stores.remove(Bytes.toBytes(storeMock.getColumnFamilyName()));
3486 assertFalse(region.hasReferences());
3487
3488
3489 regions = splitRegion(region, splitRowBytes);
3490
3491
3492 for (int i = 0; i < regions.length; i++) {
3493 regions[i] = HRegion.openHRegion(regions[i], null);
3494 }
3495
3496 assertEquals(2, regions.length);
3497
3498
3499
3500 verifyData(regions[0], startRow, numRows, qualifier, families);
3501 verifyData(regions[1], splitRow, numRows, qualifier, families);
3502
3503 } finally {
3504 HRegion.closeHRegion(this.region);
3505 this.region = null;
3506 }
3507 }
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519 @Test
3520 public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3521 byte[] family = Bytes.toBytes("family");
3522 int numRows = 1000;
3523 int flushAndScanInterval = 10;
3524 int compactInterval = 10 * flushAndScanInterval;
3525
3526 String method = "testFlushCacheWhileScanning";
3527 this.region = initHRegion(tableName, method, CONF, family);
3528 try {
3529 FlushThread flushThread = new FlushThread();
3530 flushThread.start();
3531
3532 Scan scan = new Scan();
3533 scan.addFamily(family);
3534 scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
3535 new BinaryComparator(Bytes.toBytes(5L))));
3536
3537 int expectedCount = 0;
3538 List<Cell> res = new ArrayList<Cell>();
3539
3540 boolean toggle = true;
3541 for (long i = 0; i < numRows; i++) {
3542 Put put = new Put(Bytes.toBytes(i));
3543 put.setDurability(Durability.SKIP_WAL);
3544 put.add(family, qual1, Bytes.toBytes(i % 10));
3545 region.put(put);
3546
3547 if (i != 0 && i % compactInterval == 0) {
3548
3549 region.compactStores(true);
3550 }
3551
3552 if (i % 10 == 5L) {
3553 expectedCount++;
3554 }
3555
3556 if (i != 0 && i % flushAndScanInterval == 0) {
3557 res.clear();
3558 InternalScanner scanner = region.getScanner(scan);
3559 if (toggle) {
3560 flushThread.flush();
3561 }
3562 while (scanner.next(res))
3563 ;
3564 if (!toggle) {
3565 flushThread.flush();
3566 }
3567 assertEquals("i=" + i, expectedCount, res.size());
3568 toggle = !toggle;
3569 }
3570 }
3571
3572 flushThread.done();
3573 flushThread.join();
3574 flushThread.checkNoError();
3575 } finally {
3576 HRegion.closeHRegion(this.region);
3577 this.region = null;
3578 }
3579 }
3580
3581 protected class FlushThread extends Thread {
3582 private volatile boolean done;
3583 private Throwable error = null;
3584
3585 public void done() {
3586 done = true;
3587 synchronized (this) {
3588 interrupt();
3589 }
3590 }
3591
3592 public void checkNoError() {
3593 if (error != null) {
3594 assertNull(error);
3595 }
3596 }
3597
3598 @Override
3599 public void run() {
3600 done = false;
3601 while (!done) {
3602 synchronized (this) {
3603 try {
3604 wait();
3605 } catch (InterruptedException ignored) {
3606 if (done) {
3607 break;
3608 }
3609 }
3610 }
3611 try {
3612 region.flushcache();
3613 } catch (IOException e) {
3614 if (!done) {
3615 LOG.error("Error while flusing cache", e);
3616 error = e;
3617 }
3618 break;
3619 }
3620 }
3621
3622 }
3623
3624 public void flush() {
3625 synchronized (this) {
3626 notify();
3627 }
3628
3629 }
3630 }
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641 @Test
3642 public void testWritesWhileScanning() throws IOException, InterruptedException {
3643 int testCount = 100;
3644 int numRows = 1;
3645 int numFamilies = 10;
3646 int numQualifiers = 100;
3647 int flushInterval = 7;
3648 int compactInterval = 5 * flushInterval;
3649 byte[][] families = new byte[numFamilies][];
3650 for (int i = 0; i < numFamilies; i++) {
3651 families[i] = Bytes.toBytes("family" + i);
3652 }
3653 byte[][] qualifiers = new byte[numQualifiers][];
3654 for (int i = 0; i < numQualifiers; i++) {
3655 qualifiers[i] = Bytes.toBytes("qual" + i);
3656 }
3657
3658 String method = "testWritesWhileScanning";
3659 this.region = initHRegion(tableName, method, CONF, families);
3660 try {
3661 PutThread putThread = new PutThread(numRows, families, qualifiers);
3662 putThread.start();
3663 putThread.waitForFirstPut();
3664
3665 FlushThread flushThread = new FlushThread();
3666 flushThread.start();
3667
3668 Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3669
3670 int expectedCount = numFamilies * numQualifiers;
3671 List<Cell> res = new ArrayList<Cell>();
3672
3673 long prevTimestamp = 0L;
3674 for (int i = 0; i < testCount; i++) {
3675
3676 if (i != 0 && i % compactInterval == 0) {
3677 region.compactStores(true);
3678 }
3679
3680 if (i != 0 && i % flushInterval == 0) {
3681 flushThread.flush();
3682 }
3683
3684 boolean previousEmpty = res.isEmpty();
3685 res.clear();
3686 InternalScanner scanner = region.getScanner(scan);
3687 while (scanner.next(res))
3688 ;
3689 if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3690 assertEquals("i=" + i, expectedCount, res.size());
3691 long timestamp = res.get(0).getTimestamp();
3692 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3693 timestamp >= prevTimestamp);
3694 prevTimestamp = timestamp;
3695 }
3696 }
3697
3698 putThread.done();
3699
3700 region.flushcache();
3701
3702 putThread.join();
3703 putThread.checkNoError();
3704
3705 flushThread.done();
3706 flushThread.join();
3707 flushThread.checkNoError();
3708 } finally {
3709 try {
3710 HRegion.closeHRegion(this.region);
3711 } catch (DroppedSnapshotException dse) {
3712
3713
3714
3715
3716 }
3717 this.region = null;
3718 }
3719 }
3720
3721 protected class PutThread extends Thread {
3722 private volatile boolean done;
3723 private volatile int numPutsFinished = 0;
3724
3725 private Throwable error = null;
3726 private int numRows;
3727 private byte[][] families;
3728 private byte[][] qualifiers;
3729
3730 private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3731 this.numRows = numRows;
3732 this.families = families;
3733 this.qualifiers = qualifiers;
3734 }
3735
3736
3737
3738
3739 public void waitForFirstPut() throws InterruptedException {
3740
3741 while (numPutsFinished == 0) {
3742 checkNoError();
3743 Thread.sleep(50);
3744 }
3745 }
3746
3747 public void done() {
3748 done = true;
3749 synchronized (this) {
3750 interrupt();
3751 }
3752 }
3753
3754 public void checkNoError() {
3755 if (error != null) {
3756 assertNull(error);
3757 }
3758 }
3759
3760 @Override
3761 public void run() {
3762 done = false;
3763 while (!done) {
3764 try {
3765 for (int r = 0; r < numRows; r++) {
3766 byte[] row = Bytes.toBytes("row" + r);
3767 Put put = new Put(row);
3768 put.setDurability(Durability.SKIP_WAL);
3769 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3770 for (byte[] family : families) {
3771 for (byte[] qualifier : qualifiers) {
3772 put.add(family, qualifier, (long) numPutsFinished, value);
3773 }
3774 }
3775 region.put(put);
3776 numPutsFinished++;
3777 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3778 System.out.println("put iteration = " + numPutsFinished);
3779 Delete delete = new Delete(row, (long) numPutsFinished - 30);
3780 region.delete(delete);
3781 }
3782 numPutsFinished++;
3783 }
3784 } catch (InterruptedIOException e) {
3785
3786 } catch (IOException e) {
3787 LOG.error("error while putting records", e);
3788 error = e;
3789 break;
3790 }
3791 }
3792
3793 }
3794
3795 }
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806 @Test
3807 public void testWritesWhileGetting() throws Exception {
3808 int testCount = 100;
3809 int numRows = 1;
3810 int numFamilies = 10;
3811 int numQualifiers = 100;
3812 int compactInterval = 100;
3813 byte[][] families = new byte[numFamilies][];
3814 for (int i = 0; i < numFamilies; i++) {
3815 families[i] = Bytes.toBytes("family" + i);
3816 }
3817 byte[][] qualifiers = new byte[numQualifiers][];
3818 for (int i = 0; i < numQualifiers; i++) {
3819 qualifiers[i] = Bytes.toBytes("qual" + i);
3820 }
3821
3822
3823 String method = "testWritesWhileGetting";
3824
3825
3826
3827
3828
3829 Configuration conf = HBaseConfiguration.create(CONF);
3830 conf.setInt("hbase.hstore.compaction.min", 1);
3831 conf.setInt("hbase.hstore.compaction.max", 1000);
3832 this.region = initHRegion(tableName, method, conf, families);
3833 PutThread putThread = null;
3834 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3835 try {
3836 putThread = new PutThread(numRows, families, qualifiers);
3837 putThread.start();
3838 putThread.waitForFirstPut();
3839
3840
3841 ctx.addThread(new RepeatingTestThread(ctx) {
3842 private int flushesSinceCompact = 0;
3843 private final int maxFlushesSinceCompact = 20;
3844
3845 @Override
3846 public void doAnAction() throws Exception {
3847 if (region.flushcache().isCompactionNeeded()) {
3848 ++flushesSinceCompact;
3849 }
3850
3851
3852 if (flushesSinceCompact == maxFlushesSinceCompact) {
3853 region.compactStores(false);
3854 flushesSinceCompact = 0;
3855 }
3856 }
3857 });
3858 ctx.startThreads();
3859
3860 Get get = new Get(Bytes.toBytes("row0"));
3861 Result result = null;
3862
3863 int expectedCount = numFamilies * numQualifiers;
3864
3865 long prevTimestamp = 0L;
3866 for (int i = 0; i < testCount; i++) {
3867
3868 boolean previousEmpty = result == null || result.isEmpty();
3869 result = region.get(get);
3870 if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3871 assertEquals("i=" + i, expectedCount, result.size());
3872
3873
3874 long timestamp = 0;
3875 for (Cell kv : result.rawCells()) {
3876 if (CellUtil.matchingFamily(kv, families[0])
3877 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3878 timestamp = kv.getTimestamp();
3879 }
3880 }
3881 assertTrue(timestamp >= prevTimestamp);
3882 prevTimestamp = timestamp;
3883 Cell previousKV = null;
3884
3885 for (Cell kv : result.rawCells()) {
3886 byte[] thisValue = CellUtil.cloneValue(kv);
3887 if (previousKV != null) {
3888 if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3889 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3890 + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3891 + "(memStoreTS:" + kv.getMvccVersion() + ")");
3892 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3893 }
3894 }
3895 previousKV = kv;
3896 }
3897 }
3898 }
3899 } finally {
3900 if (putThread != null)
3901 putThread.done();
3902
3903 region.flushcache();
3904
3905 if (putThread != null) {
3906 putThread.join();
3907 putThread.checkNoError();
3908 }
3909
3910 ctx.stop();
3911 HRegion.closeHRegion(this.region);
3912 this.region = null;
3913 }
3914 }
3915
3916 @Test
3917 public void testHolesInMeta() throws Exception {
3918 byte[] family = Bytes.toBytes("family");
3919 this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF,
3920 false, family);
3921 try {
3922 byte[] rowNotServed = Bytes.toBytes("a");
3923 Get g = new Get(rowNotServed);
3924 try {
3925 region.get(g);
3926 fail();
3927 } catch (WrongRegionException x) {
3928
3929 }
3930 byte[] row = Bytes.toBytes("y");
3931 g = new Get(row);
3932 region.get(g);
3933 } finally {
3934 HRegion.closeHRegion(this.region);
3935 this.region = null;
3936 }
3937 }
3938
3939 @Test
3940 public void testIndexesScanWithOneDeletedRow() throws IOException {
3941 byte[] family = Bytes.toBytes("family");
3942
3943
3944 String method = "testIndexesScanWithOneDeletedRow";
3945 this.region = initHRegion(tableName, method, CONF, family);
3946 try {
3947 Put put = new Put(Bytes.toBytes(1L));
3948 put.add(family, qual1, 1L, Bytes.toBytes(1L));
3949 region.put(put);
3950
3951 region.flushcache();
3952
3953 Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3954 region.delete(delete);
3955
3956 put = new Put(Bytes.toBytes(2L));
3957 put.add(family, qual1, 2L, Bytes.toBytes(2L));
3958 region.put(put);
3959
3960 Scan idxScan = new Scan();
3961 idxScan.addFamily(family);
3962 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
3963 new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
3964 new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
3965 CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
3966 InternalScanner scanner = region.getScanner(idxScan);
3967 List<Cell> res = new ArrayList<Cell>();
3968
3969 while (scanner.next(res))
3970 ;
3971 assertEquals(1L, res.size());
3972 } finally {
3973 HRegion.closeHRegion(this.region);
3974 this.region = null;
3975 }
3976 }
3977
3978
3979
3980
3981 @Test
3982 public void testBloomFilterSize() throws IOException {
3983 byte[] fam1 = Bytes.toBytes("fam1");
3984 byte[] qf1 = Bytes.toBytes("col");
3985 byte[] val1 = Bytes.toBytes("value1");
3986
3987 HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
3988 .setBloomFilterType(BloomType.ROWCOL);
3989
3990 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3991 htd.addFamily(hcd);
3992 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3993 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3994 try {
3995 int num_unique_rows = 10;
3996 int duplicate_multiplier = 2;
3997 int num_storefiles = 4;
3998
3999 int version = 0;
4000 for (int f = 0; f < num_storefiles; f++) {
4001 for (int i = 0; i < duplicate_multiplier; i++) {
4002 for (int j = 0; j < num_unique_rows; j++) {
4003 Put put = new Put(Bytes.toBytes("row" + j));
4004 put.setDurability(Durability.SKIP_WAL);
4005 put.add(fam1, qf1, version++, val1);
4006 region.put(put);
4007 }
4008 }
4009 region.flushcache();
4010 }
4011
4012 HStore store = (HStore) region.getStore(fam1);
4013 Collection<StoreFile> storeFiles = store.getStorefiles();
4014 for (StoreFile storefile : storeFiles) {
4015 StoreFile.Reader reader = storefile.getReader();
4016 reader.loadFileInfo();
4017 reader.loadBloomfilter();
4018 assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
4019 assertEquals(num_unique_rows, reader.getFilterEntries());
4020 }
4021
4022 region.compactStores(true);
4023
4024
4025 storeFiles = store.getStorefiles();
4026 for (StoreFile storefile : storeFiles) {
4027 StoreFile.Reader reader = storefile.getReader();
4028 reader.loadFileInfo();
4029 reader.loadBloomfilter();
4030 assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
4031 assertEquals(num_unique_rows, reader.getFilterEntries());
4032 }
4033 } finally {
4034 HRegion.closeHRegion(this.region);
4035 this.region = null;
4036 }
4037 }
4038
4039 @Test
4040 public void testAllColumnsWithBloomFilter() throws IOException {
4041 byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
4042 byte[] FAMILY = Bytes.toBytes("family");
4043
4044
4045 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
4046 .setBloomFilterType(BloomType.ROWCOL);
4047 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
4048 htd.addFamily(hcd);
4049 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
4050 this.region = TEST_UTIL.createLocalHRegion(info, htd);
4051 try {
4052
4053 byte row[] = Bytes.toBytes("row:" + 0);
4054 byte column[] = Bytes.toBytes("column:" + 0);
4055 Put put = new Put(row);
4056 put.setDurability(Durability.SKIP_WAL);
4057 for (long idx = 1; idx <= 4; idx++) {
4058 put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
4059 }
4060 region.put(put);
4061
4062
4063 region.flushcache();
4064
4065
4066 Get get = new Get(row);
4067 get.setMaxVersions();
4068 Cell[] kvs = region.get(get).rawCells();
4069
4070
4071 assertEquals(4, kvs.length);
4072 checkOneCell(kvs[0], FAMILY, 0, 0, 4);
4073 checkOneCell(kvs[1], FAMILY, 0, 0, 3);
4074 checkOneCell(kvs[2], FAMILY, 0, 0, 2);
4075 checkOneCell(kvs[3], FAMILY, 0, 0, 1);
4076 } finally {
4077 HRegion.closeHRegion(this.region);
4078 this.region = null;
4079 }
4080 }
4081
4082
4083
4084
4085
4086
4087 @Test
4088 public void testDeleteRowWithBloomFilter() throws IOException {
4089 byte[] familyName = Bytes.toBytes("familyName");
4090
4091
4092 HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
4093 .setBloomFilterType(BloomType.ROWCOL);
4094
4095 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
4096 htd.addFamily(hcd);
4097 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
4098 this.region = TEST_UTIL.createLocalHRegion(info, htd);
4099 try {
4100
4101 byte row[] = Bytes.toBytes("row1");
4102 byte col[] = Bytes.toBytes("col1");
4103
4104 Put put = new Put(row);
4105 put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
4106 region.put(put);
4107 region.flushcache();
4108
4109 Delete del = new Delete(row);
4110 region.delete(del);
4111 region.flushcache();
4112
4113
4114 Get get = new Get(row);
4115 get.addColumn(familyName, col);
4116
4117 Cell[] keyValues = region.get(get).rawCells();
4118 assertTrue(keyValues.length == 0);
4119 } finally {
4120 HRegion.closeHRegion(this.region);
4121 this.region = null;
4122 }
4123 }
4124
4125 @Test
4126 public void testgetHDFSBlocksDistribution() throws Exception {
4127 HBaseTestingUtility htu = new HBaseTestingUtility();
4128
4129
4130
4131
4132
4133
4134
4135 htu.getConfiguration().setInt("dfs.replication", 2);
4136
4137
4138 MiniHBaseCluster cluster = null;
4139 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
4140 int regionServersCount = 3;
4141
4142 try {
4143 cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
4144 byte[][] families = { fam1, fam2 };
4145 Table ht = htu.createTable(Bytes.toBytes(this.getName()), families);
4146
4147
4148 byte row[] = Bytes.toBytes("row1");
4149 byte col[] = Bytes.toBytes("col1");
4150
4151 Put put = new Put(row);
4152 put.add(fam1, col, 1, Bytes.toBytes("test1"));
4153 put.add(fam2, col, 1, Bytes.toBytes("test2"));
4154 ht.put(put);
4155
4156 HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
4157 .get(0);
4158 firstRegion.flushcache();
4159 HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
4160
4161
4162
4163
4164
4165 long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
4166 StringBuilder sb = new StringBuilder();
4167 for (String host: blocksDistribution1.getTopHosts()) {
4168 if (sb.length() > 0) sb.append(", ");
4169 sb.append(host);
4170 sb.append("=");
4171 sb.append(blocksDistribution1.getWeight(host));
4172 }
4173
4174 String topHost = blocksDistribution1.getTopHosts().get(0);
4175 long topHostWeight = blocksDistribution1.getWeight(topHost);
4176 String msg = "uniqueBlocksWeight=" + uniqueBlocksWeight1 + ", topHostWeight=" +
4177 topHostWeight + ", topHost=" + topHost + "; " + sb.toString();
4178 LOG.info(msg);
4179 assertTrue(msg, uniqueBlocksWeight1 == topHostWeight);
4180
4181
4182
4183 HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
4184 htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
4185 long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
4186
4187 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
4188
4189 ht.close();
4190 } finally {
4191 if (cluster != null) {
4192 htu.shutdownMiniCluster();
4193 }
4194 }
4195 }
4196
4197
4198
4199
4200
4201
4202
4203 @Test
4204 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
4205 TableName tableName = TableName.valueOf(name.getMethodName());
4206 HRegionInfo info = null;
4207 try {
4208 FileSystem fs = Mockito.mock(FileSystem.class);
4209 Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
4210 HTableDescriptor htd = new HTableDescriptor(tableName);
4211 htd.addFamily(new HColumnDescriptor("cf"));
4212 info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
4213 HConstants.EMPTY_BYTE_ARRAY, false);
4214 Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
4215 region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
4216
4217 region.initialize();
4218 fail("Region initialization should fail due to IOException");
4219 } catch (IOException io) {
4220 List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
4221 for (MonitoredTask monitoredTask : tasks) {
4222 if (!(monitoredTask instanceof MonitoredRPCHandler)
4223 && monitoredTask.getDescription().contains(region.toString())) {
4224 assertTrue("Region state should be ABORTED.",
4225 monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
4226 break;
4227 }
4228 }
4229 } finally {
4230 HRegion.closeHRegion(region);
4231 }
4232 }
4233
4234
4235
4236
4237
4238 @Test
4239 public void testRegionInfoFileCreation() throws IOException {
4240 Path rootDir = new Path(dir + "testRegionInfoFileCreation");
4241
4242 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
4243 htd.addFamily(new HColumnDescriptor("cf"));
4244
4245 HRegionInfo hri = new HRegionInfo(htd.getTableName());
4246
4247
4248 HRegion region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
4249
4250 Path regionDir = region.getRegionFileSystem().getRegionDir();
4251 FileSystem fs = region.getRegionFileSystem().getFileSystem();
4252 HRegion.closeHRegion(region);
4253
4254 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
4255
4256
4257 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4258 fs.exists(regionInfoFile));
4259
4260
4261 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
4262 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
4263 HRegion.closeHRegion(region);
4264
4265
4266 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4267 fs.exists(regionInfoFile));
4268
4269
4270 fs.delete(regionInfoFile, true);
4271 assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
4272 fs.exists(regionInfoFile));
4273
4274 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
4275
4276 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
4277 HRegion.closeHRegion(region);
4278
4279
4280 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4281 fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
4282 }
4283
4284
4285
4286
4287 private static class Incrementer implements Runnable {
4288 private HRegion region;
4289 private final static byte[] incRow = Bytes.toBytes("incRow");
4290 private final static byte[] family = Bytes.toBytes("family");
4291 private final static byte[] qualifier = Bytes.toBytes("qualifier");
4292 private final static long ONE = 1l;
4293 private int incCounter;
4294
4295 public Incrementer(HRegion region, int incCounter) {
4296 this.region = region;
4297 this.incCounter = incCounter;
4298 }
4299
4300 @Override
4301 public void run() {
4302 int count = 0;
4303 while (count < incCounter) {
4304 Increment inc = new Increment(incRow);
4305 inc.addColumn(family, qualifier, ONE);
4306 count++;
4307 try {
4308 region.increment(inc);
4309 } catch (IOException e) {
4310 e.printStackTrace();
4311 break;
4312 }
4313 }
4314 }
4315 }
4316
4317
4318
4319
4320
4321 @Test
4322 public void testParallelIncrementWithMemStoreFlush() throws Exception {
4323 byte[] family = Incrementer.family;
4324 this.region = initHRegion(tableName, method, CONF, family);
4325 final HRegion region = this.region;
4326 final AtomicBoolean incrementDone = new AtomicBoolean(false);
4327 Runnable flusher = new Runnable() {
4328 @Override
4329 public void run() {
4330 while (!incrementDone.get()) {
4331 try {
4332 region.flushcache();
4333 } catch (Exception e) {
4334 e.printStackTrace();
4335 }
4336 }
4337 }
4338 };
4339
4340
4341 int threadNum = 20;
4342 int incCounter = 100;
4343 long expected = threadNum * incCounter;
4344 Thread[] incrementers = new Thread[threadNum];
4345 Thread flushThread = new Thread(flusher);
4346 for (int i = 0; i < threadNum; i++) {
4347 incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
4348 incrementers[i].start();
4349 }
4350 flushThread.start();
4351 for (int i = 0; i < threadNum; i++) {
4352 incrementers[i].join();
4353 }
4354
4355 incrementDone.set(true);
4356 flushThread.join();
4357
4358 Get get = new Get(Incrementer.incRow);
4359 get.addColumn(Incrementer.family, Incrementer.qualifier);
4360 get.setMaxVersions(1);
4361 Result res = this.region.get(get);
4362 List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
4363
4364
4365 assertEquals(kvs.size(), 1);
4366 Cell kv = kvs.get(0);
4367 assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
4368 this.region = null;
4369 }
4370
4371
4372
4373
4374 private static class Appender implements Runnable {
4375 private HRegion region;
4376 private final static byte[] appendRow = Bytes.toBytes("appendRow");
4377 private final static byte[] family = Bytes.toBytes("family");
4378 private final static byte[] qualifier = Bytes.toBytes("qualifier");
4379 private final static byte[] CHAR = Bytes.toBytes("a");
4380 private int appendCounter;
4381
4382 public Appender(HRegion region, int appendCounter) {
4383 this.region = region;
4384 this.appendCounter = appendCounter;
4385 }
4386
4387 @Override
4388 public void run() {
4389 int count = 0;
4390 while (count < appendCounter) {
4391 Append app = new Append(appendRow);
4392 app.add(family, qualifier, CHAR);
4393 count++;
4394 try {
4395 region.append(app);
4396 } catch (IOException e) {
4397 e.printStackTrace();
4398 break;
4399 }
4400 }
4401 }
4402 }
4403
4404
4405
4406
4407
4408 @Test
4409 public void testParallelAppendWithMemStoreFlush() throws Exception {
4410 byte[] family = Appender.family;
4411 this.region = initHRegion(tableName, method, CONF, family);
4412 final HRegion region = this.region;
4413 final AtomicBoolean appendDone = new AtomicBoolean(false);
4414 Runnable flusher = new Runnable() {
4415 @Override
4416 public void run() {
4417 while (!appendDone.get()) {
4418 try {
4419 region.flushcache();
4420 } catch (Exception e) {
4421 e.printStackTrace();
4422 }
4423 }
4424 }
4425 };
4426
4427
4428
4429 int threadNum = 20;
4430 int appendCounter = 100;
4431 byte[] expected = new byte[threadNum * appendCounter];
4432 for (int i = 0; i < threadNum * appendCounter; i++) {
4433 System.arraycopy(Appender.CHAR, 0, expected, i, 1);
4434 }
4435 Thread[] appenders = new Thread[threadNum];
4436 Thread flushThread = new Thread(flusher);
4437 for (int i = 0; i < threadNum; i++) {
4438 appenders[i] = new Thread(new Appender(this.region, appendCounter));
4439 appenders[i].start();
4440 }
4441 flushThread.start();
4442 for (int i = 0; i < threadNum; i++) {
4443 appenders[i].join();
4444 }
4445
4446 appendDone.set(true);
4447 flushThread.join();
4448
4449 Get get = new Get(Appender.appendRow);
4450 get.addColumn(Appender.family, Appender.qualifier);
4451 get.setMaxVersions(1);
4452 Result res = this.region.get(get);
4453 List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
4454
4455
4456 assertEquals(kvs.size(), 1);
4457 Cell kv = kvs.get(0);
4458 byte[] appendResult = new byte[kv.getValueLength()];
4459 System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
4460 assertArrayEquals(expected, appendResult);
4461 this.region = null;
4462 }
4463
4464
4465
4466
4467
4468 @Test
4469 public void testPutWithMemStoreFlush() throws Exception {
4470 byte[] family = Bytes.toBytes("family");
4471 ;
4472 byte[] qualifier = Bytes.toBytes("qualifier");
4473 byte[] row = Bytes.toBytes("putRow");
4474 byte[] value = null;
4475 this.region = initHRegion(tableName, method, CONF, family);
4476 Put put = null;
4477 Get get = null;
4478 List<Cell> kvs = null;
4479 Result res = null;
4480
4481 put = new Put(row);
4482 value = Bytes.toBytes("value0");
4483 put.add(family, qualifier, 1234567l, value);
4484 region.put(put);
4485 get = new Get(row);
4486 get.addColumn(family, qualifier);
4487 get.setMaxVersions();
4488 res = this.region.get(get);
4489 kvs = res.getColumnCells(family, qualifier);
4490 assertEquals(1, kvs.size());
4491 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4492
4493 region.flushcache();
4494 get = new Get(row);
4495 get.addColumn(family, qualifier);
4496 get.setMaxVersions();
4497 res = this.region.get(get);
4498 kvs = res.getColumnCells(family, qualifier);
4499 assertEquals(1, kvs.size());
4500 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4501
4502 put = new Put(row);
4503 value = Bytes.toBytes("value1");
4504 put.add(family, qualifier, 1234567l, value);
4505 region.put(put);
4506 get = new Get(row);
4507 get.addColumn(family, qualifier);
4508 get.setMaxVersions();
4509 res = this.region.get(get);
4510 kvs = res.getColumnCells(family, qualifier);
4511 assertEquals(1, kvs.size());
4512 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4513
4514 region.flushcache();
4515 get = new Get(row);
4516 get.addColumn(family, qualifier);
4517 get.setMaxVersions();
4518 res = this.region.get(get);
4519 kvs = res.getColumnCells(family, qualifier);
4520 assertEquals(1, kvs.size());
4521 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4522 }
4523
4524 @Test
4525 public void testDurability() throws Exception {
4526 String method = "testDurability";
4527
4528
4529
4530
4531
4532 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4533 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4534 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4535
4536 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4537 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4538 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4539
4540 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4541 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4542
4543 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
4544 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4545
4546 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
4547 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
4548 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
4549
4550
4551 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4552 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4553 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4554 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4555 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
4556 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4557
4558 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4559 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4560 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4561 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4562 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
4563 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
4564
4565
4566 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, true, false, false);
4567 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, true, false, false);
4568 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, true, false, false);
4569 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, true, false, false);
4570 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, true, false, false);
4571 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4572
4573 }
4574
4575 @SuppressWarnings("unchecked")
4576 private void durabilityTest(String method, Durability tableDurability,
4577 Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
4578 final boolean expectSyncFromLogSyncer) throws Exception {
4579 Configuration conf = HBaseConfiguration.create(CONF);
4580 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
4581 TableName tableName = TableName.valueOf(method);
4582 byte[] family = Bytes.toBytes("family");
4583 Path logDir = new Path(new Path(dir + method), "log");
4584 final Configuration walConf = new Configuration(conf);
4585 FSUtils.setRootDir(walConf, logDir);
4586 final WALFactory wals = new WALFactory(walConf, null, UUID.randomUUID().toString());
4587 final WAL wal = spy(wals.getWAL(tableName.getName()));
4588 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
4589 HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, wal,
4590 new byte[][] { family });
4591
4592 Put put = new Put(Bytes.toBytes("r1"));
4593 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
4594 put.setDurability(mutationDurability);
4595 region.put(put);
4596
4597
4598 verify(wal, expectAppend ? times(1) : never())
4599 .append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any(),
4600 (WALEdit)any(), (AtomicLong)any(), Mockito.anyBoolean(), (List<Cell>)any());
4601
4602
4603 if (expectSync || expectSyncFromLogSyncer) {
4604 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
4605 @Override
4606 public boolean evaluate() throws Exception {
4607 try {
4608 if (expectSync) {
4609 verify(wal, times(1)).sync(anyLong());
4610 } else if (expectSyncFromLogSyncer) {
4611 verify(wal, times(1)).sync();
4612 }
4613 } catch (Throwable ignore) {
4614 }
4615 return true;
4616 }
4617 });
4618 } else {
4619
4620 verify(wal, never()).sync();
4621 }
4622
4623 HRegion.closeHRegion(this.region);
4624 this.region = null;
4625 }
4626
4627 @Test
4628 public void testRegionReplicaSecondary() throws IOException {
4629
4630
4631 Path rootDir = new Path(dir + "testRegionReplicaSecondary");
4632 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
4633
4634 byte[][] families = new byte[][] {
4635 Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")
4636 };
4637 byte[] cq = Bytes.toBytes("cq");
4638 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionReplicaSecondary"));
4639 for (byte[] family : families) {
4640 htd.addFamily(new HColumnDescriptor(family));
4641 }
4642
4643 long time = System.currentTimeMillis();
4644 HRegionInfo primaryHri = new HRegionInfo(htd.getTableName(),
4645 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4646 false, time, 0);
4647 HRegionInfo secondaryHri = new HRegionInfo(htd.getTableName(),
4648 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4649 false, time, 1);
4650
4651 HRegion primaryRegion = null, secondaryRegion = null;
4652
4653 try {
4654 primaryRegion = HRegion.createHRegion(primaryHri,
4655 rootDir, TEST_UTIL.getConfiguration(), htd);
4656
4657
4658 putData(primaryRegion, 0, 1000, cq, families);
4659
4660
4661 primaryRegion.flushcache();
4662
4663
4664 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, htd, null, CONF);
4665
4666 verifyData(secondaryRegion, 0, 1000, cq, families);
4667 } finally {
4668 if (primaryRegion != null) {
4669 HRegion.closeHRegion(primaryRegion);
4670 }
4671 if (secondaryRegion != null) {
4672 HRegion.closeHRegion(secondaryRegion);
4673 }
4674 }
4675 }
4676
4677 @Test
4678 public void testRegionReplicaSecondaryIsReadOnly() throws IOException {
4679
4680
4681 Path rootDir = new Path(dir + "testRegionReplicaSecondary");
4682 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
4683
4684 byte[][] families = new byte[][] {
4685 Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")
4686 };
4687 byte[] cq = Bytes.toBytes("cq");
4688 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionReplicaSecondary"));
4689 for (byte[] family : families) {
4690 htd.addFamily(new HColumnDescriptor(family));
4691 }
4692
4693 long time = System.currentTimeMillis();
4694 HRegionInfo primaryHri = new HRegionInfo(htd.getTableName(),
4695 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4696 false, time, 0);
4697 HRegionInfo secondaryHri = new HRegionInfo(htd.getTableName(),
4698 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4699 false, time, 1);
4700
4701 HRegion primaryRegion = null, secondaryRegion = null;
4702
4703 try {
4704 primaryRegion = HRegion.createHRegion(primaryHri,
4705 rootDir, TEST_UTIL.getConfiguration(), htd);
4706
4707
4708 putData(primaryRegion, 0, 1000, cq, families);
4709
4710
4711 primaryRegion.flushcache();
4712
4713
4714 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, htd, null, CONF);
4715
4716 try {
4717 putData(secondaryRegion, 0, 1000, cq, families);
4718 fail("Should have thrown exception");
4719 } catch (IOException ex) {
4720
4721 }
4722 } finally {
4723 if (primaryRegion != null) {
4724 HRegion.closeHRegion(primaryRegion);
4725 }
4726 if (secondaryRegion != null) {
4727 HRegion.closeHRegion(secondaryRegion);
4728 }
4729 }
4730 }
4731
4732 @Test
4733 public void testCompactionFromPrimary() throws IOException {
4734 Path rootDir = new Path(dir + "testRegionReplicaSecondary");
4735 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
4736
4737 byte[][] families = new byte[][] {
4738 Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")
4739 };
4740 byte[] cq = Bytes.toBytes("cq");
4741 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionReplicaSecondary"));
4742 for (byte[] family : families) {
4743 htd.addFamily(new HColumnDescriptor(family));
4744 }
4745
4746 long time = System.currentTimeMillis();
4747 HRegionInfo primaryHri = new HRegionInfo(htd.getTableName(),
4748 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4749 false, time, 0);
4750 HRegionInfo secondaryHri = new HRegionInfo(htd.getTableName(),
4751 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4752 false, time, 1);
4753
4754 HRegion primaryRegion = null, secondaryRegion = null;
4755
4756 try {
4757 primaryRegion = HRegion.createHRegion(primaryHri,
4758 rootDir, TEST_UTIL.getConfiguration(), htd);
4759
4760
4761 putData(primaryRegion, 0, 1000, cq, families);
4762
4763
4764 primaryRegion.flushcache();
4765
4766
4767 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, htd, null, CONF);
4768
4769
4770 Collection<StoreFile> storeFiles = primaryRegion.getStore(families[0]).getStorefiles();
4771 primaryRegion.getRegionFileSystem().removeStoreFiles(Bytes.toString(families[0]), storeFiles);
4772 Collection<StoreFileInfo> storeFileInfos = primaryRegion.getRegionFileSystem().getStoreFiles(families[0]);
4773 Assert.assertTrue(storeFileInfos == null || storeFileInfos.size() == 0);
4774
4775 verifyData(secondaryRegion, 0, 1000, cq, families);
4776 } finally {
4777 if (primaryRegion != null) {
4778 HRegion.closeHRegion(primaryRegion);
4779 }
4780 if (secondaryRegion != null) {
4781 HRegion.closeHRegion(secondaryRegion);
4782 }
4783 }
4784 }
4785
4786 private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4787 putData(this.region, startRow, numRows, qf, families);
4788 }
4789
4790 private void putData(HRegion region,
4791 int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4792 for (int i = startRow; i < startRow + numRows; i++) {
4793 Put put = new Put(Bytes.toBytes("" + i));
4794 put.setDurability(Durability.SKIP_WAL);
4795 for (byte[] family : families) {
4796 put.add(family, qf, null);
4797 }
4798 region.put(put);
4799 }
4800 }
4801
4802 private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
4803 throws IOException {
4804 for (int i = startRow; i < startRow + numRows; i++) {
4805 byte[] row = Bytes.toBytes("" + i);
4806 Get get = new Get(row);
4807 for (byte[] family : families) {
4808 get.addColumn(family, qf);
4809 }
4810 Result result = newReg.get(get);
4811 Cell[] raw = result.rawCells();
4812 assertEquals(families.length, result.size());
4813 for (int j = 0; j < families.length; j++) {
4814 assertTrue(CellUtil.matchingRow(raw[j], row));
4815 assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
4816 assertTrue(CellUtil.matchingQualifier(raw[j], qf));
4817 }
4818 }
4819 }
4820
4821 private void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
4822
4823 Get get = new Get(k).addFamily(family).setMaxVersions();
4824 Cell[] results = r.get(get).rawCells();
4825 for (int j = 0; j < results.length; j++) {
4826 byte[] tmp = CellUtil.cloneValue(results[j]);
4827
4828 assertTrue(Bytes.equals(k, tmp));
4829 }
4830 }
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843 private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
4844 throws IOException {
4845 byte[][] families = { fs };
4846 Scan scan = new Scan();
4847 for (int i = 0; i < families.length; i++)
4848 scan.addFamily(families[i]);
4849 InternalScanner s = r.getScanner(scan);
4850 try {
4851 List<Cell> curVals = new ArrayList<Cell>();
4852 boolean first = true;
4853 OUTER_LOOP: while (s.next(curVals)) {
4854 for (Cell kv : curVals) {
4855 byte[] val = CellUtil.cloneValue(kv);
4856 byte[] curval = val;
4857 if (first) {
4858 first = false;
4859 assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4860 } else {
4861
4862 break OUTER_LOOP;
4863 }
4864 }
4865 }
4866 } finally {
4867 s.close();
4868 }
4869 }
4870
4871
4872
4873
4874
4875 @Test
4876 public void testFlushResult() throws IOException {
4877 String method = name.getMethodName();
4878 byte[] tableName = Bytes.toBytes(method);
4879 byte[] family = Bytes.toBytes("family");
4880
4881 this.region = initHRegion(tableName, method, family);
4882
4883
4884 HRegion.FlushResult fr = region.flushcache();
4885 assertFalse(fr.isFlushSucceeded());
4886 assertFalse(fr.isCompactionNeeded());
4887
4888
4889 for (int i = 0; i < 2; i++) {
4890 Put put = new Put(tableName).add(family, family, tableName);
4891 region.put(put);
4892 fr = region.flushcache();
4893 assertTrue(fr.isFlushSucceeded());
4894 assertFalse(fr.isCompactionNeeded());
4895 }
4896
4897
4898 for (int i = 0; i < 2; i++) {
4899 Put put = new Put(tableName).add(family, family, tableName);
4900 region.put(put);
4901 fr = region.flushcache();
4902 assertTrue(fr.isFlushSucceeded());
4903 assertTrue(fr.isCompactionNeeded());
4904 }
4905 }
4906
4907 private Configuration initSplit() {
4908
4909 CONF.setInt("hbase.hstore.compactionThreshold", 2);
4910
4911
4912 CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4913
4914 CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4915
4916
4917 CONF.setLong("hbase.client.pause", 15 * 1000);
4918
4919
4920
4921 CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4922 return CONF;
4923 }
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934 public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
4935 byte[]... families) throws IOException {
4936 return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
4937 }
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4949 byte[]... families) throws IOException {
4950 return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4951 }
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4964 boolean isReadOnly, byte[]... families) throws IOException {
4965 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4966 }
4967
4968 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4969 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4970 throws IOException {
4971 return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
4972 Durability.SYNC_WAL, null, families);
4973 }
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4988 String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4989 WAL wal, byte[]... families) throws IOException {
4990 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf,
4991 isReadOnly, durability, wal, families);
4992 }
4993
4994
4995
4996
4997
4998 private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
4999 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
5000 assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
5001 Bytes.toString(CellUtil.cloneRow(kv)));
5002 assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
5003 Bytes.toString(CellUtil.cloneFamily(kv)));
5004 assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
5005 Bytes.toString(CellUtil.cloneQualifier(kv)));
5006 assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
5007 assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
5008 Bytes.toString(CellUtil.cloneValue(kv)));
5009 }
5010
5011 @Test (timeout=60000)
5012 public void testReverseScanner_FromMemStore_SingleCF_Normal()
5013 throws IOException {
5014 byte[] rowC = Bytes.toBytes("rowC");
5015 byte[] rowA = Bytes.toBytes("rowA");
5016 byte[] rowB = Bytes.toBytes("rowB");
5017 byte[] cf = Bytes.toBytes("CF");
5018 byte[][] families = { cf };
5019 byte[] col = Bytes.toBytes("C");
5020 long ts = 1;
5021 String method = this.getName();
5022 this.region = initHRegion(tableName, method, families);
5023 try {
5024 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5025 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
5026 null);
5027 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5028 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5029 Put put = null;
5030 put = new Put(rowC);
5031 put.add(kv1);
5032 put.add(kv11);
5033 region.put(put);
5034 put = new Put(rowA);
5035 put.add(kv2);
5036 region.put(put);
5037 put = new Put(rowB);
5038 put.add(kv3);
5039 region.put(put);
5040
5041 Scan scan = new Scan(rowC);
5042 scan.setMaxVersions(5);
5043 scan.setReversed(true);
5044 InternalScanner scanner = region.getScanner(scan);
5045 List<Cell> currRow = new ArrayList<Cell>();
5046 boolean hasNext = scanner.next(currRow);
5047 assertEquals(2, currRow.size());
5048 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5049 assertTrue(hasNext);
5050 currRow.clear();
5051 hasNext = scanner.next(currRow);
5052 assertEquals(1, currRow.size());
5053 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5054 assertTrue(hasNext);
5055 currRow.clear();
5056 hasNext = scanner.next(currRow);
5057 assertEquals(1, currRow.size());
5058 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
5059 assertFalse(hasNext);
5060 scanner.close();
5061 } finally {
5062 HRegion.closeHRegion(this.region);
5063 this.region = null;
5064 }
5065 }
5066
5067 @Test (timeout=60000)
5068 public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
5069 throws IOException {
5070 byte[] rowC = Bytes.toBytes("rowC");
5071 byte[] rowA = Bytes.toBytes("rowA");
5072 byte[] rowB = Bytes.toBytes("rowB");
5073 byte[] rowD = Bytes.toBytes("rowD");
5074 byte[] cf = Bytes.toBytes("CF");
5075 byte[][] families = { cf };
5076 byte[] col = Bytes.toBytes("C");
5077 long ts = 1;
5078 String method = this.getName();
5079 this.region = initHRegion(tableName, method, families);
5080 try {
5081 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5082 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
5083 null);
5084 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5085 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5086 Put put = null;
5087 put = new Put(rowC);
5088 put.add(kv1);
5089 put.add(kv11);
5090 region.put(put);
5091 put = new Put(rowA);
5092 put.add(kv2);
5093 region.put(put);
5094 put = new Put(rowB);
5095 put.add(kv3);
5096 region.put(put);
5097
5098 Scan scan = new Scan(rowD);
5099 List<Cell> currRow = new ArrayList<Cell>();
5100 scan.setReversed(true);
5101 scan.setMaxVersions(5);
5102 InternalScanner scanner = region.getScanner(scan);
5103 boolean hasNext = scanner.next(currRow);
5104 assertEquals(2, currRow.size());
5105 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5106 assertTrue(hasNext);
5107 currRow.clear();
5108 hasNext = scanner.next(currRow);
5109 assertEquals(1, currRow.size());
5110 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5111 assertTrue(hasNext);
5112 currRow.clear();
5113 hasNext = scanner.next(currRow);
5114 assertEquals(1, currRow.size());
5115 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
5116 assertFalse(hasNext);
5117 scanner.close();
5118 } finally {
5119 HRegion.closeHRegion(this.region);
5120 this.region = null;
5121 }
5122 }
5123
5124 @Test (timeout=60000)
5125 public void testReverseScanner_FromMemStore_SingleCF_FullScan()
5126 throws IOException {
5127 byte[] rowC = Bytes.toBytes("rowC");
5128 byte[] rowA = Bytes.toBytes("rowA");
5129 byte[] rowB = Bytes.toBytes("rowB");
5130 byte[] cf = Bytes.toBytes("CF");
5131 byte[][] families = { cf };
5132 byte[] col = Bytes.toBytes("C");
5133 long ts = 1;
5134 String method = this.getName();
5135 this.region = initHRegion(tableName, method, families);
5136 try {
5137 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5138 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
5139 null);
5140 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5141 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5142 Put put = null;
5143 put = new Put(rowC);
5144 put.add(kv1);
5145 put.add(kv11);
5146 region.put(put);
5147 put = new Put(rowA);
5148 put.add(kv2);
5149 region.put(put);
5150 put = new Put(rowB);
5151 put.add(kv3);
5152 region.put(put);
5153 Scan scan = new Scan();
5154 List<Cell> currRow = new ArrayList<Cell>();
5155 scan.setReversed(true);
5156 InternalScanner scanner = region.getScanner(scan);
5157 boolean hasNext = scanner.next(currRow);
5158 assertEquals(1, currRow.size());
5159 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5160 assertTrue(hasNext);
5161 currRow.clear();
5162 hasNext = scanner.next(currRow);
5163 assertEquals(1, currRow.size());
5164 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5165 assertTrue(hasNext);
5166 currRow.clear();
5167 hasNext = scanner.next(currRow);
5168 assertEquals(1, currRow.size());
5169 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
5170 assertFalse(hasNext);
5171 scanner.close();
5172 } finally {
5173 HRegion.closeHRegion(this.region);
5174 this.region = null;
5175 }
5176 }
5177
5178 @Test (timeout=60000)
5179 public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
5180
5181 byte[] rowA = Bytes.toBytes("rowA");
5182 byte[] rowB = Bytes.toBytes("rowB");
5183 byte[] rowC = Bytes.toBytes("rowC");
5184 byte[] rowD = Bytes.toBytes("rowD");
5185 byte[] rowE = Bytes.toBytes("rowE");
5186 byte[] cf = Bytes.toBytes("CF");
5187 byte[][] families = { cf };
5188 byte[] col1 = Bytes.toBytes("col1");
5189 byte[] col2 = Bytes.toBytes("col2");
5190 long ts = 1;
5191 String method = this.getName();
5192 this.region = initHRegion(tableName, method, families);
5193 try {
5194 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
5195 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
5196 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
5197 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
5198 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
5199 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
5200 Put put = null;
5201 put = new Put(rowA);
5202 put.add(kv1);
5203 region.put(put);
5204 put = new Put(rowB);
5205 put.add(kv2);
5206 region.put(put);
5207 put = new Put(rowC);
5208 put.add(kv3);
5209 region.put(put);
5210 put = new Put(rowD);
5211 put.add(kv4_1);
5212 region.put(put);
5213 put = new Put(rowD);
5214 put.add(kv4_2);
5215 region.put(put);
5216 put = new Put(rowE);
5217 put.add(kv5);
5218 region.put(put);
5219 region.flushcache();
5220 Scan scan = new Scan(rowD, rowA);
5221 scan.addColumn(families[0], col1);
5222 scan.setReversed(true);
5223 List<Cell> currRow = new ArrayList<Cell>();
5224 InternalScanner scanner = region.getScanner(scan);
5225 boolean hasNext = scanner.next(currRow);
5226 assertEquals(1, currRow.size());
5227 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5228 assertTrue(hasNext);
5229 currRow.clear();
5230 hasNext = scanner.next(currRow);
5231 assertEquals(1, currRow.size());
5232 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5233 assertTrue(hasNext);
5234 currRow.clear();
5235 hasNext = scanner.next(currRow);
5236 assertEquals(1, currRow.size());
5237 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5238 assertFalse(hasNext);
5239 scanner.close();
5240
5241 scan = new Scan(rowD, rowA);
5242 scan.addColumn(families[0], col2);
5243 scan.setReversed(true);
5244 currRow.clear();
5245 scanner = region.getScanner(scan);
5246 hasNext = scanner.next(currRow);
5247 assertEquals(1, currRow.size());
5248 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5249 scanner.close();
5250 } finally {
5251 HRegion.closeHRegion(this.region);
5252 this.region = null;
5253 }
5254 }
5255
5256 @Test (timeout=60000)
5257 public void testReverseScanner_smaller_blocksize() throws IOException {
5258
5259 byte[] rowA = Bytes.toBytes("rowA");
5260 byte[] rowB = Bytes.toBytes("rowB");
5261 byte[] rowC = Bytes.toBytes("rowC");
5262 byte[] rowD = Bytes.toBytes("rowD");
5263 byte[] rowE = Bytes.toBytes("rowE");
5264 byte[] cf = Bytes.toBytes("CF");
5265 byte[][] families = { cf };
5266 byte[] col1 = Bytes.toBytes("col1");
5267 byte[] col2 = Bytes.toBytes("col2");
5268 long ts = 1;
5269 String method = this.getName();
5270 HBaseConfiguration config = new HBaseConfiguration();
5271 config.setInt("test.block.size", 1);
5272 this.region = initHRegion(tableName, method, config, families);
5273 try {
5274 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
5275 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
5276 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
5277 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
5278 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
5279 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
5280 Put put = null;
5281 put = new Put(rowA);
5282 put.add(kv1);
5283 region.put(put);
5284 put = new Put(rowB);
5285 put.add(kv2);
5286 region.put(put);
5287 put = new Put(rowC);
5288 put.add(kv3);
5289 region.put(put);
5290 put = new Put(rowD);
5291 put.add(kv4_1);
5292 region.put(put);
5293 put = new Put(rowD);
5294 put.add(kv4_2);
5295 region.put(put);
5296 put = new Put(rowE);
5297 put.add(kv5);
5298 region.put(put);
5299 region.flushcache();
5300 Scan scan = new Scan(rowD, rowA);
5301 scan.addColumn(families[0], col1);
5302 scan.setReversed(true);
5303 List<Cell> currRow = new ArrayList<Cell>();
5304 InternalScanner scanner = region.getScanner(scan);
5305 boolean hasNext = scanner.next(currRow);
5306 assertEquals(1, currRow.size());
5307 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5308 assertTrue(hasNext);
5309 currRow.clear();
5310 hasNext = scanner.next(currRow);
5311 assertEquals(1, currRow.size());
5312 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5313 assertTrue(hasNext);
5314 currRow.clear();
5315 hasNext = scanner.next(currRow);
5316 assertEquals(1, currRow.size());
5317 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5318 assertFalse(hasNext);
5319 scanner.close();
5320
5321 scan = new Scan(rowD, rowA);
5322 scan.addColumn(families[0], col2);
5323 scan.setReversed(true);
5324 currRow.clear();
5325 scanner = region.getScanner(scan);
5326 hasNext = scanner.next(currRow);
5327 assertEquals(1, currRow.size());
5328 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5329 scanner.close();
5330 } finally {
5331 HRegion.closeHRegion(this.region);
5332 this.region = null;
5333 }
5334 }
5335
5336 @Test (timeout=60000)
5337 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
5338 throws IOException {
5339 byte[] row0 = Bytes.toBytes("row0");
5340 byte[] row1 = Bytes.toBytes("row1");
5341 byte[] row2 = Bytes.toBytes("row2");
5342 byte[] row3 = Bytes.toBytes("row3");
5343 byte[] row4 = Bytes.toBytes("row4");
5344 byte[] row5 = Bytes.toBytes("row5");
5345 byte[] cf1 = Bytes.toBytes("CF1");
5346 byte[] cf2 = Bytes.toBytes("CF2");
5347 byte[] cf3 = Bytes.toBytes("CF3");
5348 byte[][] families = { cf1, cf2, cf3 };
5349 byte[] col = Bytes.toBytes("C");
5350 long ts = 1;
5351 String method = this.getName();
5352 HBaseConfiguration conf = new HBaseConfiguration();
5353
5354 conf.setInt("hbase.hstore.compactionThreshold", 10000);
5355 this.region = initHRegion(tableName, method, conf, families);
5356 try {
5357
5358 KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
5359 null);
5360 KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
5361 null);
5362 KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
5363 KeyValue.Type.Put, null);
5364 KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
5365 null);
5366 KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
5367 null);
5368 KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
5369 null);
5370 KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
5371 KeyValue.Type.Put, null);
5372 KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
5373 null);
5374 KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
5375 KeyValue.Type.Put, null);
5376 KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
5377 null);
5378 KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
5379 null);
5380 KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
5381 KeyValue.Type.Put, null);
5382 KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
5383 null);
5384 KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
5385 KeyValue.Type.Put, null);
5386 KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
5387 null);
5388 KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
5389 null);
5390
5391 Put put = null;
5392 put = new Put(row1);
5393 put.add(kv1_2_1);
5394 region.put(put);
5395 put = new Put(row2);
5396 put.add(kv2_4_1);
5397 region.put(put);
5398 put = new Put(row4);
5399 put.add(kv4_5_4);
5400 put.add(kv4_5_5);
5401 region.put(put);
5402 region.flushcache();
5403
5404 put = new Put(row4);
5405 put.add(kv4_5_1);
5406 put.add(kv4_5_3);
5407 region.put(put);
5408 put = new Put(row1);
5409 put.add(kv1_2_2);
5410 region.put(put);
5411 put = new Put(row2);
5412 put.add(kv2_4_4);
5413 region.put(put);
5414 region.flushcache();
5415
5416 put = new Put(row4);
5417 put.add(kv4_5_2);
5418 region.put(put);
5419 put = new Put(row2);
5420 put.add(kv2_4_2);
5421 put.add(kv2_4_3);
5422 region.put(put);
5423 put = new Put(row3);
5424 put.add(kv3_2_2);
5425 region.put(put);
5426 region.flushcache();
5427
5428
5429 put = new Put(row0);
5430 put.add(kv0_1_1);
5431 region.put(put);
5432 put = new Put(row3);
5433 put.add(kv3_2_1);
5434 region.put(put);
5435 put = new Put(row5);
5436 put.add(kv5_2_1);
5437 put.add(kv5_2_2);
5438 region.put(put);
5439
5440 Scan scan = new Scan(row4);
5441 scan.setMaxVersions(5);
5442 scan.setBatch(3);
5443 scan.setReversed(true);
5444 InternalScanner scanner = region.getScanner(scan);
5445 List<Cell> currRow = new ArrayList<Cell>();
5446 boolean hasNext = false;
5447
5448
5449
5450 hasNext = scanner.next(currRow);
5451 assertEquals(3, currRow.size());
5452 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5453 assertTrue(hasNext);
5454 currRow.clear();
5455 hasNext = scanner.next(currRow);
5456 assertEquals(2, currRow.size());
5457 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5458 assertTrue(hasNext);
5459
5460 currRow.clear();
5461 hasNext = scanner.next(currRow);
5462 assertEquals(2, currRow.size());
5463 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5464 assertTrue(hasNext);
5465
5466
5467 currRow.clear();
5468 hasNext = scanner.next(currRow);
5469 assertEquals(3, currRow.size());
5470 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5471 assertTrue(hasNext);
5472 currRow.clear();
5473 hasNext = scanner.next(currRow);
5474 assertEquals(1, currRow.size());
5475 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5476 assertTrue(hasNext);
5477
5478 currRow.clear();
5479 hasNext = scanner.next(currRow);
5480 assertEquals(2, currRow.size());
5481 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5482 assertTrue(hasNext);
5483
5484 currRow.clear();
5485 hasNext = scanner.next(currRow);
5486 assertEquals(1, currRow.size());
5487 assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
5488 assertFalse(hasNext);
5489
5490 scanner.close();
5491 } finally {
5492 HRegion.closeHRegion(this.region);
5493 this.region = null;
5494 }
5495 }
5496
5497 @Test (timeout=60000)
5498 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
5499 throws IOException {
5500 byte[] row1 = Bytes.toBytes("row1");
5501 byte[] row2 = Bytes.toBytes("row2");
5502 byte[] row3 = Bytes.toBytes("row3");
5503 byte[] row4 = Bytes.toBytes("row4");
5504 byte[] cf1 = Bytes.toBytes("CF1");
5505 byte[] cf2 = Bytes.toBytes("CF2");
5506 byte[] cf3 = Bytes.toBytes("CF3");
5507 byte[] cf4 = Bytes.toBytes("CF4");
5508 byte[][] families = { cf1, cf2, cf3, cf4 };
5509 byte[] col = Bytes.toBytes("C");
5510 long ts = 1;
5511 String method = this.getName();
5512 HBaseConfiguration conf = new HBaseConfiguration();
5513
5514 conf.setInt("hbase.hstore.compactionThreshold", 10000);
5515 this.region = initHRegion(tableName, method, conf, families);
5516 try {
5517 KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
5518 KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
5519 KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
5520 KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
5521
5522 Put put = new Put(row1);
5523 put.add(kv1);
5524 region.put(put);
5525 region.flushcache();
5526
5527 put = new Put(row2);
5528 put.add(kv2);
5529 region.put(put);
5530 region.flushcache();
5531
5532 put = new Put(row3);
5533 put.add(kv3);
5534 region.put(put);
5535 region.flushcache();
5536
5537 put = new Put(row4);
5538 put.add(kv4);
5539 region.put(put);
5540
5541 Scan scan = new Scan(row4);
5542 scan.setReversed(true);
5543 scan.setBatch(10);
5544 InternalScanner scanner = region.getScanner(scan);
5545 List<Cell> currRow = new ArrayList<Cell>();
5546 boolean hasNext = scanner.next(currRow);
5547 assertEquals(1, currRow.size());
5548 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5549 assertTrue(hasNext);
5550 currRow.clear();
5551 hasNext = scanner.next(currRow);
5552 assertEquals(1, currRow.size());
5553 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5554 assertTrue(hasNext);
5555 currRow.clear();
5556 hasNext = scanner.next(currRow);
5557 assertEquals(1, currRow.size());
5558 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5559 assertTrue(hasNext);
5560 currRow.clear();
5561 hasNext = scanner.next(currRow);
5562 assertEquals(1, currRow.size());
5563 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5564 assertFalse(hasNext);
5565 } finally {
5566 HRegion.closeHRegion(this.region);
5567 this.region = null;
5568 }
5569 }
5570
5571 @Test (timeout=60000)
5572 public void testSplitRegionWithReverseScan() throws IOException {
5573 byte [] tableName = Bytes.toBytes("testSplitRegionWithReverseScan");
5574 byte [] qualifier = Bytes.toBytes("qualifier");
5575 Configuration hc = initSplit();
5576 int numRows = 3;
5577 byte [][] families = {fam1};
5578
5579
5580 String method = this.getName();
5581 this.region = initHRegion(tableName, method, hc, families);
5582
5583
5584 int startRow = 100;
5585 putData(startRow, numRows, qualifier, families);
5586 int splitRow = startRow + numRows;
5587 putData(splitRow, numRows, qualifier, families);
5588 int endRow = splitRow + numRows;
5589 region.flushcache();
5590
5591 HRegion [] regions = null;
5592 try {
5593 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
5594
5595 for (int i = 0; i < regions.length; i++) {
5596 regions[i] = HRegion.openHRegion(regions[i], null);
5597 }
5598
5599 assertEquals(2, regions.length);
5600
5601
5602
5603 verifyData(regions[0], startRow, numRows, qualifier, families);
5604 verifyData(regions[1], splitRow, numRows, qualifier, families);
5605
5606
5607 Scan scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 10 * numRows)));
5608 scan.setReversed(true);
5609 InternalScanner scanner = regions[1].getScanner(scan);
5610 List<Cell> currRow = new ArrayList<Cell>();
5611 boolean more = false;
5612 int verify = startRow + 2 * numRows - 1;
5613 do {
5614 more = scanner.next(currRow);
5615 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5616 verify--;
5617 currRow.clear();
5618 } while(more);
5619 assertEquals(verify, startRow + numRows - 1);
5620 scanner.close();
5621
5622 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 2 * numRows - 1)));
5623 scan.setReversed(true);
5624 scanner = regions[1].getScanner(scan);
5625 verify = startRow + 2 * numRows - 1;
5626 do {
5627 more = scanner.next(currRow);
5628 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5629 verify--;
5630 currRow.clear();
5631 } while(more);
5632 assertEquals(verify, startRow + numRows - 1);
5633 scanner.close();
5634
5635 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows)));
5636 scan.setReversed(true);
5637 scanner = regions[0].getScanner(scan);
5638 verify = startRow + numRows - 1;
5639 do {
5640 more = scanner.next(currRow);
5641 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5642 verify--;
5643 currRow.clear();
5644 } while(more);
5645 assertEquals(verify, 99);
5646 scanner.close();
5647
5648 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows - 1)));
5649 scan.setReversed(true);
5650 scanner = regions[0].getScanner(scan);
5651 verify = startRow + numRows - 1;
5652 do {
5653 more = scanner.next(currRow);
5654 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5655 verify--;
5656 currRow.clear();
5657 } while(more);
5658 assertEquals(verify, startRow - 1);
5659 scanner.close();
5660 } finally {
5661 HRegion.closeHRegion(this.region);
5662 this.region = null;
5663 }
5664 }
5665
5666 @Test
5667 public void testWriteRequestsCounter() throws IOException {
5668 byte[] fam = Bytes.toBytes("info");
5669 byte[][] families = { fam };
5670 this.region = initHRegion(tableName, method, CONF, families);
5671
5672 Assert.assertEquals(0L, region.getWriteRequestsCount());
5673
5674 Put put = new Put(row);
5675 put.add(fam, fam, fam);
5676
5677 Assert.assertEquals(0L, region.getWriteRequestsCount());
5678 region.put(put);
5679 Assert.assertEquals(1L, region.getWriteRequestsCount());
5680 region.put(put);
5681 Assert.assertEquals(2L, region.getWriteRequestsCount());
5682 region.put(put);
5683 Assert.assertEquals(3L, region.getWriteRequestsCount());
5684
5685 region.delete(new Delete(row));
5686 Assert.assertEquals(4L, region.getWriteRequestsCount());
5687
5688 HRegion.closeHRegion(this.region);
5689 this.region = null;
5690 }
5691
5692 @Test
5693 @SuppressWarnings("unchecked")
5694 public void testOpenRegionWrittenToWAL() throws Exception {
5695 final ServerName serverName = ServerName.valueOf("testOpenRegionWrittenToWAL", 100, 42);
5696 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
5697
5698 HTableDescriptor htd
5699 = new HTableDescriptor(TableName.valueOf("testOpenRegionWrittenToWAL"));
5700 htd.addFamily(new HColumnDescriptor(fam1));
5701 htd.addFamily(new HColumnDescriptor(fam2));
5702
5703 HRegionInfo hri = new HRegionInfo(htd.getTableName(),
5704 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY);
5705
5706
5707 HRegion region =
5708 HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL
5709 .getConfiguration(), htd);
5710 assertNotNull(region);
5711
5712
5713 region.put(new Put(Bytes.toBytes("a")).add(fam1, fam1, fam1));
5714 region.flushcache();
5715 region.close();
5716
5717 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
5718
5719
5720 WAL wal = mock(WAL.class);
5721 when(rss.getWAL((HRegionInfo) any())).thenReturn(wal);
5722
5723 try {
5724 region = HRegion.openHRegion(hri, htd, rss.getWAL(hri),
5725 TEST_UTIL.getConfiguration(), rss, null);
5726
5727 verify(wal, times(1)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any()
5728 , editCaptor.capture(), (AtomicLong)any(), anyBoolean(), (List<Cell>)any());
5729
5730 WALEdit edit = editCaptor.getValue();
5731 assertNotNull(edit);
5732 assertNotNull(edit.getCells());
5733 assertEquals(1, edit.getCells().size());
5734 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
5735 assertNotNull(desc);
5736
5737 LOG.info("RegionEventDescriptor from WAL: " + desc);
5738
5739 assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType());
5740 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getName()));
5741 assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(),
5742 hri.getEncodedNameAsBytes()));
5743 assertTrue(desc.getLogSequenceNumber() > 0);
5744 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
5745 assertEquals(2, desc.getStoresCount());
5746
5747 StoreDescriptor store = desc.getStores(0);
5748 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
5749 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
5750 assertEquals(1, store.getStoreFileCount());
5751 assertFalse(store.getStoreFile(0).contains("/"));
5752
5753 store = desc.getStores(1);
5754 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
5755 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
5756 assertEquals(0, store.getStoreFileCount());
5757
5758 } finally {
5759 HRegion.closeHRegion(region);
5760 }
5761 }
5762
5763 @Test
5764 @SuppressWarnings("unchecked")
5765 public void testCloseRegionWrittenToWAL() throws Exception {
5766 final ServerName serverName = ServerName.valueOf("testCloseRegionWrittenToWAL", 100, 42);
5767 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
5768
5769 HTableDescriptor htd
5770 = new HTableDescriptor(TableName.valueOf("testOpenRegionWrittenToWAL"));
5771 htd.addFamily(new HColumnDescriptor(fam1));
5772 htd.addFamily(new HColumnDescriptor(fam2));
5773
5774 HRegionInfo hri = new HRegionInfo(htd.getTableName(),
5775 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY);
5776
5777 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
5778
5779
5780 WAL wal = mock(WAL.class);
5781 when(rss.getWAL((HRegionInfo) any())).thenReturn(wal);
5782
5783
5784 region = HRegion.openHRegion(hri, htd, rss.getWAL(hri),
5785 TEST_UTIL.getConfiguration(), rss, null);
5786
5787
5788 region.close(false);
5789
5790
5791 verify(wal, times(2)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any(),
5792 editCaptor.capture(), (AtomicLong)any(), anyBoolean(), (List<Cell>)any());
5793
5794 WALEdit edit = editCaptor.getAllValues().get(1);
5795 assertNotNull(edit);
5796 assertNotNull(edit.getCells());
5797 assertEquals(1, edit.getCells().size());
5798 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
5799 assertNotNull(desc);
5800
5801 LOG.info("RegionEventDescriptor from WAL: " + desc);
5802
5803 assertEquals(RegionEventDescriptor.EventType.REGION_CLOSE, desc.getEventType());
5804 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getName()));
5805 assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(),
5806 hri.getEncodedNameAsBytes()));
5807 assertTrue(desc.getLogSequenceNumber() > 0);
5808 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
5809 assertEquals(2, desc.getStoresCount());
5810
5811 StoreDescriptor store = desc.getStores(0);
5812 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
5813 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
5814 assertEquals(0, store.getStoreFileCount());
5815
5816 store = desc.getStores(1);
5817 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
5818 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
5819 assertEquals(0, store.getStoreFileCount());
5820 }
5821
5822
5823
5824
5825 @Test (timeout=24000)
5826 public void testRegionTooBusy() throws IOException {
5827 String method = "testRegionTooBusy";
5828 byte[] tableName = Bytes.toBytes(method);
5829 byte[] family = Bytes.toBytes("family");
5830 long defaultBusyWaitDuration = CONF.getLong("hbase.busy.wait.duration",
5831 HRegion.DEFAULT_BUSY_WAIT_DURATION);
5832 CONF.setLong("hbase.busy.wait.duration", 1000);
5833 region = initHRegion(tableName, method, CONF, family);
5834 final AtomicBoolean stopped = new AtomicBoolean(true);
5835 Thread t = new Thread(new Runnable() {
5836 @Override
5837 public void run() {
5838 try {
5839 region.lock.writeLock().lock();
5840 stopped.set(false);
5841 while (!stopped.get()) {
5842 Thread.sleep(100);
5843 }
5844 } catch (InterruptedException ie) {
5845 } finally {
5846 region.lock.writeLock().unlock();
5847 }
5848 }
5849 });
5850 t.start();
5851 Get get = new Get(row);
5852 try {
5853 while (stopped.get()) {
5854 Thread.sleep(100);
5855 }
5856 region.get(get);
5857 fail("Should throw RegionTooBusyException");
5858 } catch (InterruptedException ie) {
5859 fail("test interrupted");
5860 } catch (RegionTooBusyException e) {
5861
5862 } finally {
5863 stopped.set(true);
5864 try {
5865 t.join();
5866 } catch (Throwable e) {
5867 }
5868
5869 HRegion.closeHRegion(region);
5870 region = null;
5871 CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration);
5872 }
5873 }
5874
5875 @Test
5876 public void testCellTTLs() throws IOException {
5877 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
5878 EnvironmentEdgeManager.injectEdge(edge);
5879
5880 final byte[] row = Bytes.toBytes("testRow");
5881 final byte[] q1 = Bytes.toBytes("q1");
5882 final byte[] q2 = Bytes.toBytes("q2");
5883 final byte[] q3 = Bytes.toBytes("q3");
5884 final byte[] q4 = Bytes.toBytes("q4");
5885
5886 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCellTTLs"));
5887 HColumnDescriptor hcd = new HColumnDescriptor(fam1);
5888 hcd.setTimeToLive(10);
5889 htd.addFamily(hcd);
5890
5891 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
5892 conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS);
5893
5894 HRegion region = HRegion.createHRegion(new HRegionInfo(htd.getTableName(),
5895 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY),
5896 TEST_UTIL.getDataTestDir(), conf, htd);
5897 assertNotNull(region);
5898 try {
5899 long now = EnvironmentEdgeManager.currentTime();
5900
5901 region.put(new Put(row).add(new KeyValue(row, fam1, q1, now,
5902 HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
5903
5904 new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
5905
5906 region.put(new Put(row).add(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY));
5907
5908 region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1,
5909 HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
5910
5911 new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
5912
5913 region.put(new Put(row).add(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY));
5914
5915
5916 region.flushcache();
5917
5918
5919 Result r = region.get(new Get(row));
5920 assertNotNull(r.getValue(fam1, q1));
5921 assertNotNull(r.getValue(fam1, q2));
5922 assertNotNull(r.getValue(fam1, q3));
5923 assertNotNull(r.getValue(fam1, q4));
5924
5925
5926 edge.incrementTime(5000);
5927
5928 r = region.get(new Get(row));
5929 assertNull(r.getValue(fam1, q1));
5930 assertNotNull(r.getValue(fam1, q2));
5931 assertNotNull(r.getValue(fam1, q3));
5932 assertNotNull(r.getValue(fam1, q4));
5933
5934
5935 edge.incrementTime(5000);
5936
5937 r = region.get(new Get(row));
5938 assertNull(r.getValue(fam1, q1));
5939 assertNull(r.getValue(fam1, q2));
5940 assertNotNull(r.getValue(fam1, q3));
5941 assertNotNull(r.getValue(fam1, q4));
5942
5943
5944 edge.incrementTime(5000);
5945
5946 r = region.get(new Get(row));
5947 assertNull(r.getValue(fam1, q1));
5948 assertNull(r.getValue(fam1, q2));
5949 assertNull(r.getValue(fam1, q3));
5950 assertNotNull(r.getValue(fam1, q4));
5951
5952
5953 edge.incrementTime(10000);
5954
5955 r = region.get(new Get(row));
5956 assertNull(r.getValue(fam1, q1));
5957 assertNull(r.getValue(fam1, q2));
5958 assertNull(r.getValue(fam1, q3));
5959 assertNull(r.getValue(fam1, q4));
5960
5961
5962
5963
5964 region.put(new Put(row).add(fam1, q1, Bytes.toBytes(1L)));
5965 r = region.get(new Get(row));
5966 byte[] val = r.getValue(fam1, q1);
5967 assertNotNull(val);
5968 assertEquals(Bytes.toLong(val), 1L);
5969
5970
5971 Increment incr = new Increment(row).addColumn(fam1, q1, 1L);
5972 incr.setTTL(5000);
5973 region.increment(incr);
5974
5975
5976 r = region.get(new Get(row));
5977 val = r.getValue(fam1, q1);
5978 assertNotNull(val);
5979 assertEquals(Bytes.toLong(val), 2L);
5980
5981
5982 edge.incrementTime(5000);
5983
5984
5985 r = region.get(new Get(row));
5986 val = r.getValue(fam1, q1);
5987 assertNotNull(val);
5988 assertEquals(Bytes.toLong(val), 1L);
5989
5990
5991 edge.incrementTime(5000);
5992
5993
5994 r = region.get(new Get(row));
5995 assertNull(r.getValue(fam1, q1));
5996
5997 } finally {
5998 HRegion.closeHRegion(region);
5999 }
6000 }
6001
6002 private static HRegion initHRegion(byte[] tableName, String callingMethod,
6003 byte[]... families) throws IOException {
6004 return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
6005 families);
6006 }
6007 }