1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21
22 import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23 import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24 import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25 import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29 import static org.junit.Assert.assertArrayEquals;
30 import static org.junit.Assert.assertEquals;
31 import static org.junit.Assert.assertFalse;
32 import static org.junit.Assert.assertNotNull;
33 import static org.junit.Assert.assertNull;
34 import static org.junit.Assert.assertTrue;
35 import static org.junit.Assert.fail;
36 import static org.mockito.Matchers.any;
37 import static org.mockito.Matchers.anyBoolean;
38 import static org.mockito.Matchers.anyLong;
39 import static org.mockito.Matchers.eq;
40 import static org.mockito.Mockito.never;
41 import static org.mockito.Mockito.spy;
42 import static org.mockito.Mockito.times;
43 import static org.mockito.Mockito.verify;
44 import static org.mockito.Mockito.when;
45
46 import java.io.IOException;
47 import java.io.InterruptedIOException;
48 import java.security.PrivilegedExceptionAction;
49 import java.util.ArrayList;
50 import java.util.Arrays;
51 import java.util.Collection;
52 import java.util.List;
53 import java.util.Map;
54 import java.util.NavigableMap;
55 import java.util.TreeMap;
56 import java.util.UUID;
57 import java.util.concurrent.atomic.AtomicBoolean;
58 import java.util.concurrent.atomic.AtomicInteger;
59 import java.util.concurrent.atomic.AtomicLong;
60 import java.util.concurrent.atomic.AtomicReference;
61
62 import org.apache.commons.logging.Log;
63 import org.apache.commons.logging.LogFactory;
64 import org.apache.hadoop.conf.Configuration;
65 import org.apache.hadoop.fs.FSDataOutputStream;
66 import org.apache.hadoop.fs.FileStatus;
67 import org.apache.hadoop.fs.FileSystem;
68 import org.apache.hadoop.fs.Path;
69 import org.apache.hadoop.hbase.Cell;
70 import org.apache.hadoop.hbase.CellComparator;
71 import org.apache.hadoop.hbase.CellUtil;
72 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
73 import org.apache.hadoop.hbase.DroppedSnapshotException;
74 import org.apache.hadoop.hbase.HBaseConfiguration;
75 import org.apache.hadoop.hbase.HBaseTestCase;
76 import org.apache.hadoop.hbase.HBaseTestingUtility;
77 import org.apache.hadoop.hbase.HColumnDescriptor;
78 import org.apache.hadoop.hbase.HConstants;
79 import org.apache.hadoop.hbase.RegionTooBusyException;
80 import org.apache.hadoop.hbase.Tag;
81 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
82 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
83 import org.apache.hadoop.hbase.HRegionInfo;
84 import org.apache.hadoop.hbase.HTableDescriptor;
85 import org.apache.hadoop.hbase.KeyValue;
86 import org.apache.hadoop.hbase.testclassification.MediumTests;
87 import org.apache.hadoop.hbase.MiniHBaseCluster;
88 import org.apache.hadoop.hbase.MultithreadedTestUtil;
89 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
90 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
91 import org.apache.hadoop.hbase.NotServingRegionException;
92 import org.apache.hadoop.hbase.TableName;
93 import org.apache.hadoop.hbase.TagType;
94 import org.apache.hadoop.hbase.Waiter;
95 import org.apache.hadoop.hbase.client.Append;
96 import org.apache.hadoop.hbase.client.Delete;
97 import org.apache.hadoop.hbase.client.Durability;
98 import org.apache.hadoop.hbase.client.Get;
99 import org.apache.hadoop.hbase.client.HTable;
100 import org.apache.hadoop.hbase.client.Increment;
101 import org.apache.hadoop.hbase.client.Put;
102 import org.apache.hadoop.hbase.client.Result;
103 import org.apache.hadoop.hbase.client.RowMutations;
104 import org.apache.hadoop.hbase.client.Scan;
105 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
106 import org.apache.hadoop.hbase.filter.BinaryComparator;
107 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
108 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
109 import org.apache.hadoop.hbase.filter.Filter;
110 import org.apache.hadoop.hbase.filter.FilterBase;
111 import org.apache.hadoop.hbase.filter.FilterList;
112 import org.apache.hadoop.hbase.filter.NullComparator;
113 import org.apache.hadoop.hbase.filter.PrefixFilter;
114 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
115 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
116 import org.apache.hadoop.hbase.io.hfile.HFile;
117 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
118 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
119 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
120 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
121 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
122 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
123 import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
124 import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
125 import org.apache.hadoop.hbase.regionserver.wal.FaultyHLog;
126 import org.apache.hadoop.hbase.regionserver.wal.HLog;
127 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
128 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
129 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
130 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
131 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
132 import org.apache.hadoop.hbase.security.User;
133 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
134 import org.apache.hadoop.hbase.util.Bytes;
135 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
136 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
137 import org.apache.hadoop.hbase.util.FSUtils;
138 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
139 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
140 import org.apache.hadoop.hbase.util.PairOfSameType;
141 import org.apache.hadoop.hbase.util.Threads;
142 import org.junit.After;
143 import org.junit.Assert;
144 import org.junit.Before;
145 import org.junit.Rule;
146 import org.junit.Test;
147 import org.junit.experimental.categories.Category;
148 import org.junit.rules.TestName;
149 import org.mockito.Mockito;
150
151 import com.google.common.collect.ImmutableList;
152 import com.google.common.collect.Lists;
153 import com.google.protobuf.ByteString;
154
155
156
157
158
159
160
161 @Category(MediumTests.class)
162 @SuppressWarnings("deprecation")
163 public class TestHRegion {
164
165
166 static final Log LOG = LogFactory.getLog(TestHRegion.class);
167 @Rule public TestName name = new TestName();
168
169 private static final String COLUMN_FAMILY = "MyCF";
170 private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
171
172 HRegion region = null;
173
174 private static HBaseTestingUtility TEST_UTIL;
175 public static Configuration CONF ;
176 private String dir;
177 private static FileSystem FILESYSTEM;
178 private final int MAX_VERSIONS = 2;
179
180
181 protected byte[] tableName;
182 protected String method;
183 protected final byte[] qual1 = Bytes.toBytes("qual1");
184 protected final byte[] qual2 = Bytes.toBytes("qual2");
185 protected final byte[] qual3 = Bytes.toBytes("qual3");
186 protected final byte[] value1 = Bytes.toBytes("value1");
187 protected final byte[] value2 = Bytes.toBytes("value2");
188 protected final byte[] row = Bytes.toBytes("rowA");
189 protected final byte[] row2 = Bytes.toBytes("rowB");
190
191 protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
192 .getInstance(MetricsAssertHelper.class);
193
194 @Before
195 public void setup() throws IOException {
196 TEST_UTIL = HBaseTestingUtility.createLocalHTU();
197 FILESYSTEM = TEST_UTIL.getTestFileSystem();
198 CONF = TEST_UTIL.getConfiguration();
199 dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
200 method = name.getMethodName();
201 tableName = Bytes.toBytes(name.getMethodName());
202 }
203
204 @After
205 public void tearDown() throws Exception {
206 EnvironmentEdgeManagerTestHelper.reset();
207 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
208 TEST_UTIL.cleanupTestDir();
209 }
210
211 String getName() {
212 return name.getMethodName();
213 }
214
215
216
217
218
219
220
221
222
223
224
225 @Test (timeout=60000)
226 public void testCloseCarryingSnapshot() throws IOException {
227 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
228 Store store = region.getStore(COLUMN_FAMILY_BYTES);
229
230 byte [] value = Bytes.toBytes(name.getMethodName());
231
232 Put put = new Put(value);
233 put.add(COLUMN_FAMILY_BYTES, null, value);
234
235 region.put(put);
236 StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
237 storeFlushCtx.prepare();
238
239 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
240 region.put(put);
241
242 region.close();
243 assertEquals(0, region.getMemstoreSize().get());
244 HRegion.closeHRegion(region);
245 }
246
247
248
249
250
251 @Test (timeout=60000)
252 public void testMemstoreSnapshotSize() throws IOException {
253 class MyFaultyHLog extends FaultyHLog {
254 StoreFlushContext storeFlushCtx;
255 public MyFaultyHLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
256 throws IOException {
257 super(fs, rootDir, logName, conf);
258 }
259
260 void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
261 this.storeFlushCtx = storeFlushCtx;
262 }
263
264 @Override
265 public void sync(long txid) throws IOException {
266 storeFlushCtx.prepare();
267 super.sync(txid);
268 }
269 }
270
271 FileSystem fs = FileSystem.get(CONF);
272 Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
273 MyFaultyHLog faultyLog = new MyFaultyHLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
274 HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
275 CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES);
276
277 Store store = region.getStore(COLUMN_FAMILY_BYTES);
278
279 byte [] value = Bytes.toBytes(name.getMethodName());
280 faultyLog.setStoreFlushCtx(store.createFlushContext(12345));
281
282 Put put = new Put(value);
283 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
284 faultyLog.setFailureType(FaultyHLog.FailureType.SYNC);
285
286 boolean threwIOE = false;
287 try {
288 region.put(put);
289 } catch (IOException ioe) {
290 threwIOE = true;
291 } finally {
292 assertTrue("The regionserver should have thrown an exception", threwIOE);
293 }
294 long sz = store.getFlushableSize();
295 assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
296 HRegion.closeHRegion(region);
297 }
298
299
300
301
302 @Test
303 public void testMemstoreSizeWithFlushCanceling() throws IOException {
304 FileSystem fs = FileSystem.get(CONF);
305 Path rootDir = new Path(dir + "testMemstoreSizeWithFlushCanceling");
306 FaultyHLog hLog = new FaultyHLog(fs, rootDir, "testMemstoreSizeWithFlushCanceling", CONF);
307 HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
308 CONF, false, Durability.SYNC_WAL, hLog, COLUMN_FAMILY_BYTES);
309 Store store = region.getStore(COLUMN_FAMILY_BYTES);
310 assertEquals(0, region.getMemstoreSize().get());
311
312
313 byte [] value = Bytes.toBytes(name.getMethodName());
314 Put put = new Put(value);
315 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
316 region.put(put);
317 long onePutSize = region.getMemstoreSize().get();
318 assertTrue(onePutSize > 0);
319 region.flushcache();
320 assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize().get());
321 assertEquals("flushable size should be zero", 0, store.getFlushableSize());
322
323
324 RegionCoprocessorHost normalCPHost = region.getCoprocessorHost();
325 RegionCoprocessorHost mockedCPHost = Mockito.mock(RegionCoprocessorHost.class);
326 when(mockedCPHost.preFlush(any(HStore.class), any(InternalScanner.class))).thenReturn(null);
327 region.setCoprocessorHost(mockedCPHost);
328 region.put(put);
329 region.flushcache();
330 assertEquals("memstoreSize should NOT be zero", onePutSize, region.getMemstoreSize().get());
331 assertEquals("flushable size should NOT be zero", onePutSize, store.getFlushableSize());
332
333
334 region.setCoprocessorHost(normalCPHost);
335 region.flushcache();
336 assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize().get());
337 assertEquals("flushable size should be zero", 0, store.getFlushableSize());
338 HRegion.closeHRegion(region);
339 }
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356 @Test (timeout=60000)
357 public void testFlushSizeAccounting() throws Exception {
358 final Configuration conf = HBaseConfiguration.create(CONF);
359
360 conf.setInt("hbase.hstore.flush.retries.number", 1);
361 final User user =
362 User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
363
364 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
365 user.runAs(new PrivilegedExceptionAction<Object>() {
366 @Override
367 public Object run() throws Exception {
368
369 FileSystem fs = FileSystem.get(conf);
370 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
371 FaultyFileSystem ffs = (FaultyFileSystem)fs;
372 HRegion region = null;
373 try {
374
375 region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
376 long size = region.getMemstoreSize().get();
377 Assert.assertEquals(0, size);
378
379 Put p1 = new Put(row);
380 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
381 region.put(p1);
382 final long sizeOfOnePut = region.getMemstoreSize().get();
383
384 try {
385 LOG.info("Flushing");
386 region.flushcache();
387 Assert.fail("Didn't bubble up IOE!");
388 } catch (DroppedSnapshotException dse) {
389
390 region.closing.set(false);
391 }
392
393 ffs.fault.set(false);
394
395 Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize().get());
396
397
398 Put p2 = new Put(row);
399 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
400 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
401 region.put(p2);
402 Assert.assertEquals(sizeOfOnePut * 3, region.getMemstoreSize().get());
403
404
405
406 region.flushcache();
407
408 Assert.assertEquals(sizeOfOnePut * 2, region.getMemstoreSize().get());
409 } finally {
410 HRegion.closeHRegion(region);
411 }
412 return null;
413 }
414 });
415 FileSystem.closeAllForUGI(user.getUGI());
416 }
417
418 @Test
419 public void testCompactionAffectedByScanners() throws Exception {
420 byte[] family = Bytes.toBytes("family");
421 this.region = initHRegion(tableName, method, CONF, family);
422
423 Put put = new Put(Bytes.toBytes("r1"));
424 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
425 region.put(put);
426 region.flushcache();
427
428 Scan scan = new Scan();
429 scan.setMaxVersions(3);
430
431 RegionScanner scanner1 = region.getScanner(scan);
432
433 Delete delete = new Delete(Bytes.toBytes("r1"));
434 region.delete(delete);
435 region.flushcache();
436
437
438 RegionScanner scanner2 = region.getScanner(scan);
439
440 List<Cell> results = new ArrayList<Cell>();
441
442 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
443
444
445 region.compactStores(true);
446
447
448 RegionScanner scanner3 = region.getScanner(scan);
449
450
451 scanner1.next(results);
452 System.out.println(results);
453 assertEquals(1, results.size());
454
455 results.clear();
456 scanner2.next(results);
457 System.out.println(results);
458 assertEquals(0, results.size());
459
460 results.clear();
461 scanner3.next(results);
462 System.out.println(results);
463 assertEquals(0, results.size());
464 }
465
466 @Test
467 public void testToShowNPEOnRegionScannerReseek() throws Exception {
468 byte[] family = Bytes.toBytes("family");
469 this.region = initHRegion(tableName, method, CONF, family);
470
471 Put put = new Put(Bytes.toBytes("r1"));
472 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
473 region.put(put);
474 put = new Put(Bytes.toBytes("r2"));
475 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
476 region.put(put);
477 region.flushcache();
478
479 Scan scan = new Scan();
480 scan.setMaxVersions(3);
481
482 RegionScanner scanner1 = region.getScanner(scan);
483
484 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
485
486 region.compactStores(true);
487
488 scanner1.reseek(Bytes.toBytes("r2"));
489 List<Cell> results = new ArrayList<Cell>();
490 scanner1.next(results);
491 Cell keyValue = results.get(0);
492 Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
493 scanner1.close();
494 }
495
496 @Test
497 public void testSkipRecoveredEditsReplay() throws Exception {
498 String method = "testSkipRecoveredEditsReplay";
499 TableName tableName = TableName.valueOf(method);
500 byte[] family = Bytes.toBytes("family");
501 this.region = initHRegion(tableName, method, CONF, family);
502 try {
503 Path regiondir = region.getRegionFileSystem().getRegionDir();
504 FileSystem fs = region.getRegionFileSystem().getFileSystem();
505 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
506
507 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
508
509 long maxSeqId = 1050;
510 long minSeqId = 1000;
511
512 for (long i = minSeqId; i <= maxSeqId; i += 10) {
513 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
514 fs.create(recoveredEdits);
515 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
516
517 long time = System.nanoTime();
518 WALEdit edit = new WALEdit();
519 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
520 .toBytes(i)));
521 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
522 HConstants.DEFAULT_CLUSTER_ID), edit));
523
524 writer.close();
525 }
526 MonitoredTask status = TaskMonitor.get().createStatus(method);
527 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
528 for (Store store : region.getStores().values()) {
529 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
530 }
531 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
532 assertEquals(maxSeqId, seqId);
533 Get get = new Get(row);
534 Result result = region.get(get);
535 for (long i = minSeqId; i <= maxSeqId; i += 10) {
536 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
537 assertEquals(1, kvs.size());
538 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
539 }
540 } finally {
541 HRegion.closeHRegion(this.region);
542 this.region = null;
543 }
544 }
545
546 @Test
547 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
548 String method = "testSkipRecoveredEditsReplaySomeIgnored";
549 TableName tableName = TableName.valueOf(method);
550 byte[] family = Bytes.toBytes("family");
551 this.region = initHRegion(tableName, method, CONF, family);
552 try {
553 Path regiondir = region.getRegionFileSystem().getRegionDir();
554 FileSystem fs = region.getRegionFileSystem().getFileSystem();
555 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
556
557 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
558
559 long maxSeqId = 1050;
560 long minSeqId = 1000;
561
562 for (long i = minSeqId; i <= maxSeqId; i += 10) {
563 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
564 fs.create(recoveredEdits);
565 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
566
567 long time = System.nanoTime();
568 WALEdit edit = new WALEdit();
569 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
570 .toBytes(i)));
571 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
572 HConstants.DEFAULT_CLUSTER_ID), edit));
573
574 writer.close();
575 }
576 long recoverSeqId = 1030;
577 MonitoredTask status = TaskMonitor.get().createStatus(method);
578 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
579 for (Store store : region.getStores().values()) {
580 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
581 }
582 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
583 assertEquals(maxSeqId, seqId);
584 Get get = new Get(row);
585 Result result = region.get(get);
586 for (long i = minSeqId; i <= maxSeqId; i += 10) {
587 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
588 if (i < recoverSeqId) {
589 assertEquals(0, kvs.size());
590 } else {
591 assertEquals(1, kvs.size());
592 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
593 }
594 }
595 } finally {
596 HRegion.closeHRegion(this.region);
597 this.region = null;
598 }
599 }
600
601 @Test
602 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
603 byte[] family = Bytes.toBytes("family");
604 this.region = initHRegion(tableName, method, CONF, family);
605 try {
606 Path regiondir = region.getRegionFileSystem().getRegionDir();
607 FileSystem fs = region.getRegionFileSystem().getFileSystem();
608
609 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
610 for (int i = 1000; i < 1050; i += 10) {
611 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
612 FSDataOutputStream dos = fs.create(recoveredEdits);
613 dos.writeInt(i);
614 dos.close();
615 }
616 long minSeqId = 2000;
617 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
618 FSDataOutputStream dos = fs.create(recoveredEdits);
619 dos.close();
620
621 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
622 for (Store store : region.getStores().values()) {
623 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
624 }
625 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
626 assertEquals(minSeqId, seqId);
627 } finally {
628 HRegion.closeHRegion(this.region);
629 this.region = null;
630 }
631 }
632
633 @Test
634 public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
635 String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
636 TableName tableName = TableName.valueOf(method);
637 byte[] family = Bytes.toBytes("family");
638 this.region = initHRegion(tableName, method, CONF, family);
639 try {
640 Path regiondir = region.getRegionFileSystem().getRegionDir();
641 FileSystem fs = region.getRegionFileSystem().getFileSystem();
642 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
643
644 assertEquals(0, region.getStoreFileList(
645 region.getStores().keySet().toArray(new byte[0][])).size());
646
647 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
648
649 long maxSeqId = 1050;
650 long minSeqId = 1000;
651
652 for (long i = minSeqId; i <= maxSeqId; i += 10) {
653 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
654 fs.create(recoveredEdits);
655 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
656
657 long time = System.nanoTime();
658 WALEdit edit = null;
659 if (i == maxSeqId) {
660 edit = WALEdit.createCompaction(region.getRegionInfo(),
661 CompactionDescriptor.newBuilder()
662 .setTableName(ByteString.copyFrom(tableName.getName()))
663 .setFamilyName(ByteString.copyFrom(regionName))
664 .setEncodedRegionName(ByteString.copyFrom(regionName))
665 .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
666 .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName()))
667 .build());
668 } else {
669 edit = new WALEdit();
670 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
671 .toBytes(i)));
672 }
673 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
674 HConstants.DEFAULT_CLUSTER_ID), edit));
675 writer.close();
676 }
677
678 long recoverSeqId = 1030;
679 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
680 MonitoredTask status = TaskMonitor.get().createStatus(method);
681 for (Store store : region.getStores().values()) {
682 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
683 }
684 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
685 assertEquals(maxSeqId, seqId);
686
687
688 assertEquals(1, region.getStoreFileList(
689 region.getStores().keySet().toArray(new byte[0][])).size());
690
691 } finally {
692 HRegion.closeHRegion(this.region);
693 this.region = null;
694 } }
695
696 @Test
697 public void testRecoveredEditsReplayCompaction() throws Exception {
698 String method = name.getMethodName();
699 TableName tableName = TableName.valueOf(method);
700 byte[] family = Bytes.toBytes("family");
701 this.region = initHRegion(tableName, method, CONF, family);
702 try {
703 Path regiondir = region.getRegionFileSystem().getRegionDir();
704 FileSystem fs = region.getRegionFileSystem().getFileSystem();
705 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
706
707 long maxSeqId = 3;
708 long minSeqId = 0;
709
710 for (long i = minSeqId; i < maxSeqId; i++) {
711 Put put = new Put(Bytes.toBytes(i));
712 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
713 region.put(put);
714 region.flushcache();
715 }
716
717
718 assertEquals(3, region.getStore(family).getStorefilesCount());
719 List<Path> storeFiles = new ArrayList<Path>(3);
720 for (StoreFile sf : region.getStore(family).getStorefiles()) {
721 storeFiles.add(sf.getPath());
722 }
723
724
725 CONF.setBoolean("hbase.hstore.compaction.complete", false);
726 region.compactStores();
727
728
729 assertEquals(3, region.getStore(family).getStorefilesCount());
730
731
732 Path tmpDir = region.getRegionFileSystem().getTempDir();
733 FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
734 String errorMsg = "Expected to find 1 file in the region temp directory "
735 + "from the compaction, could not find any";
736 assertNotNull(errorMsg, files);
737 assertEquals(errorMsg, 1, files.length);
738
739 Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
740 files[0].getPath());
741
742 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
743 .getRegionInfo(), family, storeFiles, Lists.newArrayList(newFile), region
744 .getRegionFileSystem().getStoreDir(Bytes.toString(family)));
745
746 HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
747 this.region.getRegionInfo(), compactionDescriptor, new AtomicLong(1));
748
749 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
750
751 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
752 fs.create(recoveredEdits);
753 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
754
755 long time = System.nanoTime();
756
757 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, 10, time,
758 HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(),
759 compactionDescriptor)));
760 writer.close();
761
762
763 region.getTableDesc();
764 region.getRegionInfo();
765 region.close();
766 region = HRegion.openHRegion(region, null);
767
768
769 Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
770 for (StoreFile sf : sfs) {
771 LOG.info(sf.getPath());
772 }
773 assertEquals(1, region.getStore(family).getStorefilesCount());
774 files = FSUtils.listStatus(fs, tmpDir);
775 assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
776
777 for (long i = minSeqId; i < maxSeqId; i++) {
778 Get get = new Get(Bytes.toBytes(i));
779 Result result = region.get(get);
780 byte[] value = result.getValue(family, Bytes.toBytes(i));
781 assertArrayEquals(Bytes.toBytes(i), value);
782 }
783 } finally {
784 HRegion.closeHRegion(this.region);
785 this.region = null;
786 }
787 }
788
789 @Test
790 public void testGetWhileRegionClose() throws IOException {
791 TableName tableName = TableName.valueOf(name.getMethodName());
792 Configuration hc = initSplit();
793 int numRows = 100;
794 byte[][] families = { fam1, fam2, fam3 };
795
796
797 String method = name.getMethodName();
798 this.region = initHRegion(tableName, method, hc, families);
799 try {
800
801 final int startRow = 100;
802 putData(startRow, numRows, qual1, families);
803 putData(startRow, numRows, qual2, families);
804 putData(startRow, numRows, qual3, families);
805 final AtomicBoolean done = new AtomicBoolean(false);
806 final AtomicInteger gets = new AtomicInteger(0);
807 GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
808 try {
809
810 for (int i = 0; i < threads.length / 2; i++) {
811 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
812 threads[i].setDaemon(true);
813 threads[i].start();
814 }
815
816
817 this.region.closing.set(true);
818 for (int i = threads.length / 2; i < threads.length; i++) {
819 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
820 threads[i].setDaemon(true);
821 threads[i].start();
822 }
823 } finally {
824 if (this.region != null) {
825 HRegion.closeHRegion(this.region);
826 }
827 }
828 done.set(true);
829 for (GetTillDoneOrException t : threads) {
830 try {
831 t.join();
832 } catch (InterruptedException e) {
833 e.printStackTrace();
834 }
835 if (t.e != null) {
836 LOG.info("Exception=" + t.e);
837 assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
838 }
839 }
840 } finally {
841 HRegion.closeHRegion(this.region);
842 this.region = null;
843 }
844 }
845
846
847
848
849
850 class GetTillDoneOrException extends Thread {
851 private final Get g;
852 private final AtomicBoolean done;
853 private final AtomicInteger count;
854 private Exception e;
855
856 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
857 super("getter." + i);
858 this.g = new Get(r);
859 this.done = d;
860 this.count = c;
861 }
862
863 @Override
864 public void run() {
865 while (!this.done.get()) {
866 try {
867 assertTrue(region.get(g).size() > 0);
868 this.count.incrementAndGet();
869 } catch (Exception e) {
870 this.e = e;
871 break;
872 }
873 }
874 }
875 }
876
877
878
879
880 @Test
881 public void testWeirdCacheBehaviour() throws Exception {
882 byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
883 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
884 Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
885 this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
886 try {
887 String value = "this is the value";
888 String value2 = "this is some other value";
889 String keyPrefix1 = "prefix1";
890 String keyPrefix2 = "prefix2";
891 String keyPrefix3 = "prefix3";
892 putRows(this.region, 3, value, keyPrefix1);
893 putRows(this.region, 3, value, keyPrefix2);
894 putRows(this.region, 3, value, keyPrefix3);
895 putRows(this.region, 3, value2, keyPrefix1);
896 putRows(this.region, 3, value2, keyPrefix2);
897 putRows(this.region, 3, value2, keyPrefix3);
898 System.out.println("Checking values for key: " + keyPrefix1);
899 assertEquals("Got back incorrect number of rows from scan", 3,
900 getNumberOfRows(keyPrefix1, value2, this.region));
901 System.out.println("Checking values for key: " + keyPrefix2);
902 assertEquals("Got back incorrect number of rows from scan", 3,
903 getNumberOfRows(keyPrefix2, value2, this.region));
904 System.out.println("Checking values for key: " + keyPrefix3);
905 assertEquals("Got back incorrect number of rows from scan", 3,
906 getNumberOfRows(keyPrefix3, value2, this.region));
907 deleteColumns(this.region, value2, keyPrefix1);
908 deleteColumns(this.region, value2, keyPrefix2);
909 deleteColumns(this.region, value2, keyPrefix3);
910 System.out.println("Starting important checks.....");
911 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
912 getNumberOfRows(keyPrefix1, value2, this.region));
913 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
914 getNumberOfRows(keyPrefix2, value2, this.region));
915 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
916 getNumberOfRows(keyPrefix3, value2, this.region));
917 } finally {
918 HRegion.closeHRegion(this.region);
919 this.region = null;
920 }
921 }
922
923 @Test
924 public void testAppendWithReadOnlyTable() throws Exception {
925 byte[] TABLE = Bytes.toBytes("readOnlyTable");
926 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
927 boolean exceptionCaught = false;
928 Append append = new Append(Bytes.toBytes("somerow"));
929 append.setDurability(Durability.SKIP_WAL);
930 append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
931 Bytes.toBytes("somevalue"));
932 try {
933 region.append(append);
934 } catch (IOException e) {
935 exceptionCaught = true;
936 } finally {
937 HRegion.closeHRegion(this.region);
938 this.region = null;
939 }
940 assertTrue(exceptionCaught == true);
941 }
942
943 @Test
944 public void testIncrWithReadOnlyTable() throws Exception {
945 byte[] TABLE = Bytes.toBytes("readOnlyTable");
946 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
947 boolean exceptionCaught = false;
948 Increment inc = new Increment(Bytes.toBytes("somerow"));
949 inc.setDurability(Durability.SKIP_WAL);
950 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
951 try {
952 region.increment(inc);
953 } catch (IOException e) {
954 exceptionCaught = true;
955 } finally {
956 HRegion.closeHRegion(this.region);
957 this.region = null;
958 }
959 assertTrue(exceptionCaught == true);
960 }
961
962 private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
963 InternalScanner scanner = buildScanner(keyPrefix, value, r);
964 int count = 0;
965 boolean more = false;
966 List<Cell> results = new ArrayList<Cell>();
967 do {
968 more = scanner.next(results);
969 if (results != null && !results.isEmpty())
970 count++;
971 else
972 break;
973 Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
974 delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
975 r.delete(delete);
976 results.clear();
977 } while (more);
978 assertEquals("Did not perform correct number of deletes", 3, count);
979 }
980
981 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
982 InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
983 int numberOfResults = 0;
984 List<Cell> results = new ArrayList<Cell>();
985 boolean more = false;
986 do {
987 more = resultScanner.next(results);
988 if (results != null && !results.isEmpty())
989 numberOfResults++;
990 else
991 break;
992 for (Cell kv : results) {
993 System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
994 }
995 results.clear();
996 } while (more);
997 return numberOfResults;
998 }
999
1000 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
1001 throws IOException {
1002
1003 FilterList allFilters = new FilterList();
1004 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
1005
1006 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
1007 Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
1008 filter.setFilterIfMissing(true);
1009 allFilters.addFilter(filter);
1010 Scan scan = new Scan();
1011 scan.addFamily(Bytes.toBytes("trans-blob"));
1012 scan.addFamily(Bytes.toBytes("trans-type"));
1013 scan.addFamily(Bytes.toBytes("trans-date"));
1014 scan.addFamily(Bytes.toBytes("trans-tags"));
1015 scan.addFamily(Bytes.toBytes("trans-group"));
1016 scan.setFilter(allFilters);
1017 return r.getScanner(scan);
1018 }
1019
1020 private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
1021 for (int i = 0; i < numRows; i++) {
1022 String row = key + "_" + i
1023 System.out.println(String.format("Saving row: %s, with value %s", row, value));
1024 Put put = new Put(Bytes.toBytes(row));
1025 put.setDurability(Durability.SKIP_WAL);
1026 put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
1027 put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
1028 put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
1029 put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
1030 put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
1031 r.put(put);
1032 }
1033 }
1034
1035 @Test
1036 public void testFamilyWithAndWithoutColon() throws Exception {
1037 byte[] b = Bytes.toBytes(getName());
1038 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1039 this.region = initHRegion(b, getName(), CONF, cf);
1040 try {
1041 Put p = new Put(b);
1042 byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
1043 p.add(cfwithcolon, cfwithcolon, cfwithcolon);
1044 boolean exception = false;
1045 try {
1046 this.region.put(p);
1047 } catch (NoSuchColumnFamilyException e) {
1048 exception = true;
1049 }
1050 assertTrue(exception);
1051 } finally {
1052 HRegion.closeHRegion(this.region);
1053 this.region = null;
1054 }
1055 }
1056
1057 @Test
1058 public void testBatchPut_whileNoRowLocksHeld() throws IOException {
1059 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1060 byte[] qual = Bytes.toBytes("qual");
1061 byte[] val = Bytes.toBytes("val");
1062 this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1063 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1064 try {
1065 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1066 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1067
1068 LOG.info("First a batch put with all valid puts");
1069 final Put[] puts = new Put[10];
1070 for (int i = 0; i < 10; i++) {
1071 puts[i] = new Put(Bytes.toBytes("row_" + i));
1072 puts[i].add(cf, qual, val);
1073 }
1074
1075 OperationStatus[] codes = this.region.batchMutate(puts);
1076 assertEquals(10, codes.length);
1077 for (int i = 0; i < 10; i++) {
1078 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1079 }
1080 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1081
1082 LOG.info("Next a batch put with one invalid family");
1083 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1084 codes = this.region.batchMutate(puts);
1085 assertEquals(10, codes.length);
1086 for (int i = 0; i < 10; i++) {
1087 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1088 codes[i].getOperationStatusCode());
1089 }
1090
1091 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1092 } finally {
1093 HRegion.closeHRegion(this.region);
1094 this.region = null;
1095 }
1096 }
1097
1098 @Test
1099 public void testBatchPut_whileMultipleRowLocksHeld() throws Exception {
1100 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1101 byte[] qual = Bytes.toBytes("qual");
1102 byte[] val = Bytes.toBytes("val");
1103 this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1104 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1105 try {
1106 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1107 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1108
1109 final Put[] puts = new Put[10];
1110 for (int i = 0; i < 10; i++) {
1111 puts[i] = new Put(Bytes.toBytes("row_" + i));
1112 puts[i].add(cf, qual, val);
1113 }
1114 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1115
1116 LOG.info("batchPut will have to break into four batches to avoid row locks");
1117 RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2"));
1118 RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_4"));
1119 RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_6"));
1120
1121 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1122 final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
1123 TestThread putter = new TestThread(ctx) {
1124 @Override
1125 public void doWork() throws IOException {
1126 retFromThread.set(region.batchMutate(puts));
1127 }
1128 };
1129 LOG.info("...starting put thread while holding locks");
1130 ctx.addThread(putter);
1131 ctx.startThreads();
1132
1133 LOG.info("...waiting for put thread to sync 1st time");
1134 waitForCounter(source, "syncTimeNumOps", syncs + 1);
1135
1136
1137
1138
1139
1140
1141 Thread regionCloseThread = new Thread() {
1142 @Override
1143 public void run() {
1144 try {
1145 HRegion.closeHRegion(region);
1146 } catch (IOException e) {
1147 throw new RuntimeException(e);
1148 }
1149 }
1150 };
1151 regionCloseThread.start();
1152
1153 LOG.info("...releasing row lock 1, which should let put thread continue");
1154 rowLock1.release();
1155
1156 LOG.info("...waiting for put thread to sync 2nd time");
1157 waitForCounter(source, "syncTimeNumOps", syncs + 2);
1158
1159 LOG.info("...releasing row lock 2, which should let put thread continue");
1160 rowLock2.release();
1161
1162 LOG.info("...waiting for put thread to sync 3rd time");
1163 waitForCounter(source, "syncTimeNumOps", syncs + 3);
1164
1165 LOG.info("...releasing row lock 3, which should let put thread continue");
1166 rowLock3.release();
1167
1168 LOG.info("...waiting for put thread to sync 4th time");
1169 waitForCounter(source, "syncTimeNumOps", syncs + 4);
1170
1171 LOG.info("...joining on put thread");
1172 ctx.stop();
1173 regionCloseThread.join();
1174
1175 OperationStatus[] codes = retFromThread.get();
1176 for (int i = 0; i < codes.length; i++) {
1177 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1178 codes[i].getOperationStatusCode());
1179 }
1180 } finally {
1181 HRegion.closeHRegion(this.region);
1182 this.region = null;
1183 }
1184 }
1185
1186 private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount)
1187 throws InterruptedException {
1188 long startWait = System.currentTimeMillis();
1189 long currentCount;
1190 while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) {
1191 Thread.sleep(100);
1192 if (System.currentTimeMillis() - startWait > 10000) {
1193 fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName,
1194 expectedCount, currentCount));
1195 }
1196 }
1197 }
1198
1199
1200 @Test
1201 public void testBatchPutWithTsSlop() throws Exception {
1202 byte[] b = Bytes.toBytes(getName());
1203 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1204 byte[] qual = Bytes.toBytes("qual");
1205 byte[] val = Bytes.toBytes("val");
1206
1207
1208 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1209 this.region = initHRegion(b, getName(), CONF, cf);
1210
1211 try {
1212 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1213 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1214 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1215
1216 final Put[] puts = new Put[10];
1217 for (int i = 0; i < 10; i++) {
1218 puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
1219 puts[i].add(cf, qual, val);
1220 }
1221
1222 OperationStatus[] codes = this.region.batchMutate(puts);
1223 assertEquals(10, codes.length);
1224 for (int i = 0; i < 10; i++) {
1225 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1226 }
1227 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1228
1229 } finally {
1230 HRegion.closeHRegion(this.region);
1231 this.region = null;
1232 }
1233
1234 }
1235
1236
1237
1238
1239 @Test
1240 public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1241 byte[] row1 = Bytes.toBytes("row1");
1242 byte[] fam1 = Bytes.toBytes("fam1");
1243 byte[] qf1 = Bytes.toBytes("qualifier");
1244 byte[] emptyVal = new byte[] {};
1245 byte[] val1 = Bytes.toBytes("value1");
1246 byte[] val2 = Bytes.toBytes("value2");
1247
1248
1249 String method = this.getName();
1250 this.region = initHRegion(tableName, method, CONF, fam1);
1251 try {
1252
1253 Put put = new Put(row1);
1254 put.add(fam1, qf1, emptyVal);
1255
1256
1257 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1258 emptyVal), put, true);
1259 assertTrue(res);
1260
1261
1262 put = new Put(row1);
1263 put.add(fam1, qf1, val1);
1264
1265
1266 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1267 put, true);
1268 assertTrue(res);
1269
1270
1271 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1272 put, true);
1273 assertFalse(res);
1274
1275 Delete delete = new Delete(row1);
1276 delete.deleteColumn(fam1, qf1);
1277 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1278 delete, true);
1279 assertFalse(res);
1280
1281 put = new Put(row1);
1282 put.add(fam1, qf1, val2);
1283
1284 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1285 put, true);
1286 assertTrue(res);
1287
1288
1289 delete = new Delete(row1);
1290 delete.deleteColumn(fam1, qf1);
1291 delete.deleteColumn(fam1, qf1);
1292 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1293 delete, true);
1294 assertTrue(res);
1295
1296 delete = new Delete(row1);
1297 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1298 delete, true);
1299 assertTrue(res);
1300
1301
1302 put = new Put(row1);
1303 put.add(fam1, qf1, val1);
1304
1305 res = region
1306 .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
1307 assertTrue(res);
1308 } finally {
1309 HRegion.closeHRegion(this.region);
1310 this.region = null;
1311 }
1312 }
1313
1314 @Test
1315 public void testCheckAndMutate_WithWrongValue() throws IOException {
1316 byte[] row1 = Bytes.toBytes("row1");
1317 byte[] fam1 = Bytes.toBytes("fam1");
1318 byte[] qf1 = Bytes.toBytes("qualifier");
1319 byte[] val1 = Bytes.toBytes("value1");
1320 byte[] val2 = Bytes.toBytes("value2");
1321
1322
1323 String method = this.getName();
1324 this.region = initHRegion(tableName, method, CONF, fam1);
1325 try {
1326
1327 Put put = new Put(row1);
1328 put.add(fam1, qf1, val1);
1329 region.put(put);
1330
1331
1332 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1333 val2), put, true);
1334 assertEquals(false, res);
1335
1336
1337 Delete delete = new Delete(row1);
1338 delete.deleteFamily(fam1);
1339 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1340 put, true);
1341 assertEquals(false, res);
1342 } finally {
1343 HRegion.closeHRegion(this.region);
1344 this.region = null;
1345 }
1346 }
1347
1348 @Test
1349 public void testCheckAndMutate_WithCorrectValue() throws IOException {
1350 byte[] row1 = Bytes.toBytes("row1");
1351 byte[] fam1 = Bytes.toBytes("fam1");
1352 byte[] qf1 = Bytes.toBytes("qualifier");
1353 byte[] val1 = Bytes.toBytes("value1");
1354
1355
1356 String method = this.getName();
1357 this.region = initHRegion(tableName, method, CONF, fam1);
1358 try {
1359
1360 Put put = new Put(row1);
1361 put.add(fam1, qf1, val1);
1362 region.put(put);
1363
1364
1365 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1366 val1), put, true);
1367 assertEquals(true, res);
1368
1369
1370 Delete delete = new Delete(row1);
1371 delete.deleteColumn(fam1, qf1);
1372 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1373 delete, true);
1374 assertEquals(true, res);
1375 } finally {
1376 HRegion.closeHRegion(this.region);
1377 this.region = null;
1378 }
1379 }
1380
1381 @Test
1382 public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1383 byte[] row1 = Bytes.toBytes("row1");
1384 byte[] fam1 = Bytes.toBytes("fam1");
1385 byte[] qf1 = Bytes.toBytes("qualifier");
1386 byte[] val1 = Bytes.toBytes("value1");
1387 byte[] val2 = Bytes.toBytes("value2");
1388 byte[] val3 = Bytes.toBytes("value3");
1389 byte[] val4 = Bytes.toBytes("value4");
1390
1391
1392 String method = this.getName();
1393 this.region = initHRegion(tableName, method, CONF, fam1);
1394 try {
1395
1396 Put put = new Put(row1);
1397 put.add(fam1, qf1, val3);
1398 region.put(put);
1399
1400
1401 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1402 new BinaryComparator(val3), put, true);
1403 assertEquals(false, res);
1404
1405
1406 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1407 new BinaryComparator(val4), put, true);
1408 assertEquals(false, res);
1409
1410
1411
1412 put = new Put(row1);
1413 put.add(fam1, qf1, val2);
1414 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1415 new BinaryComparator(val2), put, true);
1416 assertEquals(true, res);
1417
1418
1419 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1420 new BinaryComparator(val3), put, true);
1421 assertEquals(false, res);
1422
1423
1424
1425 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1426 new BinaryComparator(val2), put, true);
1427 assertEquals(true, res);
1428
1429
1430
1431 put = new Put(row1);
1432 put.add(fam1, qf1, val3);
1433 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1434 new BinaryComparator(val1), put, true);
1435 assertEquals(true, res);
1436
1437
1438 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1439 new BinaryComparator(val3), put, true);
1440 assertEquals(false, res);
1441
1442
1443 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1444 new BinaryComparator(val2), put, true);
1445 assertEquals(false, res);
1446
1447
1448
1449 put = new Put(row1);
1450 put.add(fam1, qf1, val2);
1451 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1452 new BinaryComparator(val4), put, true);
1453 assertEquals(true, res);
1454
1455
1456 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1457 new BinaryComparator(val1), put, true);
1458 assertEquals(false, res);
1459
1460
1461
1462 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1463 new BinaryComparator(val2), put, true);
1464 assertEquals(true, res);
1465
1466
1467 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1468 new BinaryComparator(val3), put, true);
1469 assertEquals(true, res);
1470 } finally {
1471 HRegion.closeHRegion(this.region);
1472 this.region = null;
1473 }
1474 }
1475
1476 @Test
1477 public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1478 byte[] row1 = Bytes.toBytes("row1");
1479 byte[] fam1 = Bytes.toBytes("fam1");
1480 byte[] fam2 = Bytes.toBytes("fam2");
1481 byte[] qf1 = Bytes.toBytes("qualifier");
1482 byte[] val1 = Bytes.toBytes("value1");
1483 byte[] val2 = Bytes.toBytes("value2");
1484
1485 byte[][] families = { fam1, fam2 };
1486
1487
1488 String method = this.getName();
1489 this.region = initHRegion(tableName, method, CONF, families);
1490 try {
1491
1492 Put put = new Put(row1);
1493 put.add(fam1, qf1, val1);
1494 region.put(put);
1495
1496
1497 long ts = System.currentTimeMillis();
1498 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1499 put = new Put(row1);
1500 put.add(kv);
1501
1502
1503 HStore store = (HStore) region.getStore(fam1);
1504 store.memstore.kvset.size();
1505
1506 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1507 val1), put, true);
1508 assertEquals(true, res);
1509 store.memstore.kvset.size();
1510
1511 Get get = new Get(row1);
1512 get.addColumn(fam2, qf1);
1513 Cell[] actual = region.get(get).rawCells();
1514
1515 Cell[] expected = { kv };
1516
1517 assertEquals(expected.length, actual.length);
1518 for (int i = 0; i < actual.length; i++) {
1519 assertEquals(expected[i], actual[i]);
1520 }
1521 } finally {
1522 HRegion.closeHRegion(this.region);
1523 this.region = null;
1524 }
1525 }
1526
1527 @Test
1528 public void testCheckAndPut_wrongRowInPut() throws IOException {
1529 TableName tableName = TableName.valueOf(name.getMethodName());
1530 this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
1531 try {
1532 Put put = new Put(row2);
1533 put.add(fam1, qual1, value1);
1534 try {
1535 region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1536 new BinaryComparator(value2), put, false);
1537 fail();
1538 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1539
1540 }
1541 } finally {
1542 HRegion.closeHRegion(this.region);
1543 this.region = null;
1544 }
1545 }
1546
1547 @Test
1548 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1549 byte[] row1 = Bytes.toBytes("row1");
1550 byte[] fam1 = Bytes.toBytes("fam1");
1551 byte[] fam2 = Bytes.toBytes("fam2");
1552 byte[] qf1 = Bytes.toBytes("qualifier1");
1553 byte[] qf2 = Bytes.toBytes("qualifier2");
1554 byte[] qf3 = Bytes.toBytes("qualifier3");
1555 byte[] val1 = Bytes.toBytes("value1");
1556 byte[] val2 = Bytes.toBytes("value2");
1557 byte[] val3 = Bytes.toBytes("value3");
1558 byte[] emptyVal = new byte[] {};
1559
1560 byte[][] families = { fam1, fam2 };
1561
1562
1563 String method = this.getName();
1564 this.region = initHRegion(tableName, method, CONF, families);
1565 try {
1566
1567 Put put = new Put(row1);
1568 put.add(fam1, qf1, val1);
1569 region.put(put);
1570 Threads.sleep(2);
1571
1572 put = new Put(row1);
1573 put.add(fam1, qf1, val2);
1574 put.add(fam2, qf1, val3);
1575 put.add(fam2, qf2, val2);
1576 put.add(fam2, qf3, val1);
1577 put.add(fam1, qf3, val1);
1578 region.put(put);
1579
1580
1581 Delete delete = new Delete(row1);
1582 delete.deleteColumn(fam1, qf1);
1583 delete.deleteColumn(fam2, qf1);
1584 delete.deleteColumn(fam1, qf3);
1585 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1586 val2), delete, true);
1587 assertEquals(true, res);
1588
1589 Get get = new Get(row1);
1590 get.addColumn(fam1, qf1);
1591 get.addColumn(fam1, qf3);
1592 get.addColumn(fam2, qf2);
1593 Result r = region.get(get);
1594 assertEquals(2, r.size());
1595 assertArrayEquals(val1, r.getValue(fam1, qf1));
1596 assertArrayEquals(val2, r.getValue(fam2, qf2));
1597
1598
1599 delete = new Delete(row1);
1600 delete.deleteFamily(fam2);
1601 res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1602 delete, true);
1603 assertEquals(true, res);
1604
1605 get = new Get(row1);
1606 r = region.get(get);
1607 assertEquals(1, r.size());
1608 assertArrayEquals(val1, r.getValue(fam1, qf1));
1609
1610
1611 delete = new Delete(row1);
1612 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1613 delete, true);
1614 assertEquals(true, res);
1615 get = new Get(row1);
1616 r = region.get(get);
1617 assertEquals(0, r.size());
1618 } finally {
1619 HRegion.closeHRegion(this.region);
1620 this.region = null;
1621 }
1622 }
1623
1624
1625
1626
1627 @Test
1628 public void testDelete_multiDeleteColumn() throws IOException {
1629 byte[] row1 = Bytes.toBytes("row1");
1630 byte[] fam1 = Bytes.toBytes("fam1");
1631 byte[] qual = Bytes.toBytes("qualifier");
1632 byte[] value = Bytes.toBytes("value");
1633
1634 Put put = new Put(row1);
1635 put.add(fam1, qual, 1, value);
1636 put.add(fam1, qual, 2, value);
1637
1638 String method = this.getName();
1639 this.region = initHRegion(tableName, method, CONF, fam1);
1640 try {
1641 region.put(put);
1642
1643
1644 Delete delete = new Delete(row1);
1645 delete.deleteColumn(fam1, qual);
1646 delete.deleteColumn(fam1, qual);
1647 region.delete(delete);
1648
1649 Get get = new Get(row1);
1650 get.addFamily(fam1);
1651 Result r = region.get(get);
1652 assertEquals(0, r.size());
1653 } finally {
1654 HRegion.closeHRegion(this.region);
1655 this.region = null;
1656 }
1657 }
1658
1659 @Test
1660 public void testDelete_CheckFamily() throws IOException {
1661 byte[] row1 = Bytes.toBytes("row1");
1662 byte[] fam1 = Bytes.toBytes("fam1");
1663 byte[] fam2 = Bytes.toBytes("fam2");
1664 byte[] fam3 = Bytes.toBytes("fam3");
1665 byte[] fam4 = Bytes.toBytes("fam4");
1666
1667
1668 String method = this.getName();
1669 this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
1670 try {
1671 List<Cell> kvs = new ArrayList<Cell>();
1672 kvs.add(new KeyValue(row1, fam4, null, null));
1673
1674
1675 byte[] family = fam2;
1676 try {
1677 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1678 Bytes.BYTES_COMPARATOR);
1679 deleteMap.put(family, kvs);
1680 region.delete(deleteMap, Durability.SYNC_WAL);
1681 } catch (Exception e) {
1682 assertTrue("Family " + new String(family) + " does not exist", false);
1683 }
1684
1685
1686 boolean ok = false;
1687 family = fam4;
1688 try {
1689 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1690 Bytes.BYTES_COMPARATOR);
1691 deleteMap.put(family, kvs);
1692 region.delete(deleteMap, Durability.SYNC_WAL);
1693 } catch (Exception e) {
1694 ok = true;
1695 }
1696 assertEquals("Family " + new String(family) + " does exist", true, ok);
1697 } finally {
1698 HRegion.closeHRegion(this.region);
1699 this.region = null;
1700 }
1701 }
1702
1703 @Test
1704 public void testDelete_mixed() throws IOException, InterruptedException {
1705 byte[] fam = Bytes.toBytes("info");
1706 byte[][] families = { fam };
1707 String method = this.getName();
1708 this.region = initHRegion(tableName, method, CONF, families);
1709 try {
1710 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1711
1712 byte[] row = Bytes.toBytes("table_name");
1713
1714 byte[] serverinfo = Bytes.toBytes("serverinfo");
1715 byte[] splitA = Bytes.toBytes("splitA");
1716 byte[] splitB = Bytes.toBytes("splitB");
1717
1718
1719 Put put = new Put(row);
1720 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1721 region.put(put);
1722
1723 put = new Put(row);
1724 put.add(fam, splitB, Bytes.toBytes("reference_B"));
1725 region.put(put);
1726
1727 put = new Put(row);
1728 put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1729 region.put(put);
1730
1731
1732 Delete delete = new Delete(row);
1733 delete.deleteColumns(fam, splitA);
1734 region.delete(delete);
1735
1736
1737 Get get = new Get(row).addColumn(fam, serverinfo);
1738 Result result = region.get(get);
1739 assertEquals(1, result.size());
1740
1741 get = new Get(row).addColumn(fam, splitA);
1742 result = region.get(get);
1743 assertEquals(0, result.size());
1744
1745 get = new Get(row).addColumn(fam, splitB);
1746 result = region.get(get);
1747 assertEquals(1, result.size());
1748
1749
1750 put = new Put(row);
1751 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1752 region.put(put);
1753 get = new Get(row);
1754 result = region.get(get);
1755 assertEquals(3, result.size());
1756
1757
1758 delete = new Delete(row);
1759 region.delete(delete);
1760 assertEquals(0, region.get(get).size());
1761
1762 region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
1763 result = region.get(get);
1764 assertEquals(1, result.size());
1765 } finally {
1766 HRegion.closeHRegion(this.region);
1767 this.region = null;
1768 }
1769 }
1770
1771 @Test
1772 public void testDeleteRowWithFutureTs() throws IOException {
1773 byte[] fam = Bytes.toBytes("info");
1774 byte[][] families = { fam };
1775 String method = this.getName();
1776 this.region = initHRegion(tableName, method, CONF, families);
1777 try {
1778 byte[] row = Bytes.toBytes("table_name");
1779
1780 byte[] serverinfo = Bytes.toBytes("serverinfo");
1781
1782
1783 Put put = new Put(row);
1784 put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
1785 region.put(put);
1786
1787
1788 Delete delete = new Delete(row);
1789 region.delete(delete);
1790
1791
1792 Get get = new Get(row).addColumn(fam, serverinfo);
1793 Result result = region.get(get);
1794 assertEquals(1, result.size());
1795
1796
1797 delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
1798 region.delete(delete);
1799
1800
1801 get = new Get(row).addColumn(fam, serverinfo);
1802 result = region.get(get);
1803 assertEquals(0, result.size());
1804 } finally {
1805 HRegion.closeHRegion(this.region);
1806 this.region = null;
1807 }
1808 }
1809
1810
1811
1812
1813
1814 @Test
1815 public void testPutWithLatestTS() throws IOException {
1816 byte[] fam = Bytes.toBytes("info");
1817 byte[][] families = { fam };
1818 String method = this.getName();
1819 this.region = initHRegion(tableName, method, CONF, families);
1820 try {
1821 byte[] row = Bytes.toBytes("row1");
1822
1823 byte[] qual = Bytes.toBytes("qual");
1824
1825
1826 Put put = new Put(row);
1827 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1828 region.put(put);
1829
1830
1831 Get get = new Get(row).addColumn(fam, qual);
1832 Result result = region.get(get);
1833 assertEquals(1, result.size());
1834 Cell kv = result.rawCells()[0];
1835 LOG.info("Got: " + kv);
1836 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1837 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1838
1839
1840
1841 row = Bytes.toBytes("row2");
1842 put = new Put(row);
1843 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1844 region.put(put);
1845
1846
1847 get = new Get(row).addColumn(fam, qual);
1848 result = region.get(get);
1849 assertEquals(1, result.size());
1850 kv = result.rawCells()[0];
1851 LOG.info("Got: " + kv);
1852 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1853 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1854 } finally {
1855 HRegion.closeHRegion(this.region);
1856 this.region = null;
1857 }
1858
1859 }
1860
1861
1862
1863
1864
1865
1866 @Test
1867 public void testPutWithTsSlop() throws IOException {
1868 byte[] fam = Bytes.toBytes("info");
1869 byte[][] families = { fam };
1870 String method = this.getName();
1871
1872
1873 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1874 this.region = initHRegion(tableName, method, CONF, families);
1875 boolean caughtExcep = false;
1876 try {
1877 try {
1878
1879 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
1880
1881 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
1882 Bytes.toBytes("value")));
1883 fail("Expected IOE for TS out of configured timerange");
1884 } catch (FailedSanityCheckException ioe) {
1885 LOG.debug("Received expected exception", ioe);
1886 caughtExcep = true;
1887 }
1888 assertTrue("Should catch FailedSanityCheckException", caughtExcep);
1889 } finally {
1890 HRegion.closeHRegion(this.region);
1891 this.region = null;
1892 }
1893 }
1894
1895 @Test
1896 public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
1897 byte[] fam1 = Bytes.toBytes("columnA");
1898 byte[] fam2 = Bytes.toBytes("columnB");
1899 this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
1900 try {
1901 byte[] rowA = Bytes.toBytes("rowA");
1902 byte[] rowB = Bytes.toBytes("rowB");
1903
1904 byte[] value = Bytes.toBytes("value");
1905
1906 Delete delete = new Delete(rowA);
1907 delete.deleteFamily(fam1);
1908
1909 region.delete(delete);
1910
1911
1912 Put put = new Put(rowA);
1913 put.add(fam2, null, value);
1914 region.put(put);
1915
1916 put = new Put(rowB);
1917 put.add(fam1, null, value);
1918 put.add(fam2, null, value);
1919 region.put(put);
1920
1921 Scan scan = new Scan();
1922 scan.addFamily(fam1).addFamily(fam2);
1923 InternalScanner s = region.getScanner(scan);
1924 List<Cell> results = new ArrayList<Cell>();
1925 s.next(results);
1926 assertTrue(CellUtil.matchingRow(results.get(0), rowA));
1927
1928 results.clear();
1929 s.next(results);
1930 assertTrue(CellUtil.matchingRow(results.get(0), rowB));
1931 } finally {
1932 HRegion.closeHRegion(this.region);
1933 this.region = null;
1934 }
1935 }
1936
1937 @Test
1938 public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
1939 Delete delete = new Delete(row);
1940 delete.deleteColumns(fam1, qual1);
1941 doTestDelete_AndPostInsert(delete);
1942 }
1943
1944 @Test
1945 public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
1946 Delete delete = new Delete(row);
1947 delete.deleteFamily(fam1);
1948 doTestDelete_AndPostInsert(delete);
1949 }
1950
1951 public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
1952 TableName tableName = TableName.valueOf(name.getMethodName());
1953 this.region = initHRegion(tableName, getName(), CONF, fam1);
1954 try {
1955 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1956 Put put = new Put(row);
1957 put.add(fam1, qual1, value1);
1958 region.put(put);
1959
1960
1961 region.delete(delete);
1962
1963
1964 put = new Put(row);
1965 put.add(fam1, qual1, value2);
1966 region.put(put);
1967
1968
1969 Get get = new Get(row);
1970 get.addColumn(fam1, qual1);
1971
1972 Result r = region.get(get);
1973 assertEquals(1, r.size());
1974 assertArrayEquals(value2, r.getValue(fam1, qual1));
1975
1976
1977 Scan scan = new Scan(row);
1978 scan.addColumn(fam1, qual1);
1979 InternalScanner s = region.getScanner(scan);
1980
1981 List<Cell> results = new ArrayList<Cell>();
1982 assertEquals(false, s.next(results));
1983 assertEquals(1, results.size());
1984 Cell kv = results.get(0);
1985
1986 assertArrayEquals(value2, CellUtil.cloneValue(kv));
1987 assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
1988 assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
1989 assertArrayEquals(row, CellUtil.cloneRow(kv));
1990 } finally {
1991 HRegion.closeHRegion(this.region);
1992 this.region = null;
1993 }
1994 }
1995
1996 @Test
1997 public void testDelete_CheckTimestampUpdated() throws IOException {
1998 TableName tableName = TableName.valueOf(name.getMethodName());
1999 byte[] row1 = Bytes.toBytes("row1");
2000 byte[] col1 = Bytes.toBytes("col1");
2001 byte[] col2 = Bytes.toBytes("col2");
2002 byte[] col3 = Bytes.toBytes("col3");
2003
2004
2005 String method = this.getName();
2006 this.region = initHRegion(tableName, method, CONF, fam1);
2007 try {
2008
2009 List<Cell> kvs = new ArrayList<Cell>();
2010 kvs.add(new KeyValue(row1, fam1, col1, null));
2011 kvs.add(new KeyValue(row1, fam1, col2, null));
2012 kvs.add(new KeyValue(row1, fam1, col3, null));
2013
2014 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
2015 Bytes.BYTES_COMPARATOR);
2016 deleteMap.put(fam1, kvs);
2017 region.delete(deleteMap, Durability.SYNC_WAL);
2018
2019
2020
2021 long now = System.currentTimeMillis();
2022 KeyValue firstKv = ((HStore) region.getStore(fam1)).memstore.kvset.first();
2023 assertTrue(firstKv.getTimestamp() <= now);
2024 now = firstKv.getTimestamp();
2025 for (KeyValue kv : ((HStore) region.getStore(fam1)).memstore.kvset) {
2026 assertTrue(kv.getTimestamp() <= now);
2027 now = kv.getTimestamp();
2028 }
2029 } finally {
2030 HRegion.closeHRegion(this.region);
2031 this.region = null;
2032 }
2033 }
2034
2035
2036
2037
2038 @Test
2039 public void testGet_FamilyChecker() throws IOException {
2040 byte[] row1 = Bytes.toBytes("row1");
2041 byte[] fam1 = Bytes.toBytes("fam1");
2042 byte[] fam2 = Bytes.toBytes("False");
2043 byte[] col1 = Bytes.toBytes("col1");
2044
2045
2046 String method = this.getName();
2047 this.region = initHRegion(tableName, method, CONF, fam1);
2048 try {
2049 Get get = new Get(row1);
2050 get.addColumn(fam2, col1);
2051
2052
2053 try {
2054 region.get(get);
2055 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
2056 assertFalse(false);
2057 return;
2058 }
2059 assertFalse(true);
2060 } finally {
2061 HRegion.closeHRegion(this.region);
2062 this.region = null;
2063 }
2064 }
2065
2066 @Test
2067 public void testGet_Basic() throws IOException {
2068 byte[] row1 = Bytes.toBytes("row1");
2069 byte[] fam1 = Bytes.toBytes("fam1");
2070 byte[] col1 = Bytes.toBytes("col1");
2071 byte[] col2 = Bytes.toBytes("col2");
2072 byte[] col3 = Bytes.toBytes("col3");
2073 byte[] col4 = Bytes.toBytes("col4");
2074 byte[] col5 = Bytes.toBytes("col5");
2075
2076
2077 String method = this.getName();
2078 this.region = initHRegion(tableName, method, CONF, fam1);
2079 try {
2080
2081 Put put = new Put(row1);
2082 put.add(fam1, col1, null);
2083 put.add(fam1, col2, null);
2084 put.add(fam1, col3, null);
2085 put.add(fam1, col4, null);
2086 put.add(fam1, col5, null);
2087 region.put(put);
2088
2089 Get get = new Get(row1);
2090 get.addColumn(fam1, col2);
2091 get.addColumn(fam1, col4);
2092
2093 KeyValue kv1 = new KeyValue(row1, fam1, col2);
2094 KeyValue kv2 = new KeyValue(row1, fam1, col4);
2095 KeyValue[] expected = { kv1, kv2 };
2096
2097
2098 Result res = region.get(get);
2099 assertEquals(expected.length, res.size());
2100 for (int i = 0; i < res.size(); i++) {
2101 assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
2102 assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
2103 assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
2104 }
2105
2106
2107 Get g = new Get(row1);
2108 final int count = 2;
2109 g.setFilter(new ColumnCountGetFilter(count));
2110 res = region.get(g);
2111 assertEquals(count, res.size());
2112 } finally {
2113 HRegion.closeHRegion(this.region);
2114 this.region = null;
2115 }
2116 }
2117
2118 @Test
2119 public void testGet_Empty() throws IOException {
2120 byte[] row = Bytes.toBytes("row");
2121 byte[] fam = Bytes.toBytes("fam");
2122
2123 String method = this.getName();
2124 this.region = initHRegion(tableName, method, CONF, fam);
2125 try {
2126 Get get = new Get(row);
2127 get.addFamily(fam);
2128 Result r = region.get(get);
2129
2130 assertTrue(r.isEmpty());
2131 } finally {
2132 HRegion.closeHRegion(this.region);
2133 this.region = null;
2134 }
2135 }
2136
2137
2138
2139
2140 @Test
2141 public void testMerge() throws IOException {
2142 byte[][] families = { fam1, fam2, fam3 };
2143 Configuration hc = initSplit();
2144
2145 String method = this.getName();
2146 this.region = initHRegion(tableName, method, hc, families);
2147 try {
2148 LOG.info("" + HBaseTestCase.addContent(region, fam3));
2149 region.flushcache();
2150 region.compactStores();
2151 byte[] splitRow = region.checkSplit();
2152 assertNotNull(splitRow);
2153 LOG.info("SplitRow: " + Bytes.toString(splitRow));
2154 HRegion[] subregions = splitRegion(region, splitRow);
2155 try {
2156
2157 for (int i = 0; i < subregions.length; i++) {
2158 HRegion.openHRegion(subregions[i], null);
2159 subregions[i].compactStores();
2160 }
2161 Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
2162 Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
2163 Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
2164 long startTime = System.currentTimeMillis();
2165 region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
2166 LOG.info("Merge regions elapsed time: "
2167 + ((System.currentTimeMillis() - startTime) / 1000.0));
2168 FILESYSTEM.delete(oldRegion1, true);
2169 FILESYSTEM.delete(oldRegion2, true);
2170 FILESYSTEM.delete(oldRegionPath, true);
2171 LOG.info("splitAndMerge completed.");
2172 } finally {
2173 for (int i = 0; i < subregions.length; i++) {
2174 try {
2175 HRegion.closeHRegion(subregions[i]);
2176 } catch (IOException e) {
2177
2178 }
2179 }
2180 }
2181 } finally {
2182 HRegion.closeHRegion(this.region);
2183 this.region = null;
2184 }
2185 }
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195 HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
2196 PairOfSameType<HRegion> result = null;
2197 SplitTransaction st = new SplitTransaction(parent, midkey);
2198
2199
2200 if (!st.prepare()) {
2201 parent.clearSplit();
2202 return null;
2203 }
2204 try {
2205 result = st.execute(null, null);
2206 } catch (IOException ioe) {
2207 try {
2208 LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; "
2209 + ioe.getMessage());
2210 st.rollback(null, null);
2211 LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString());
2212 return null;
2213 } catch (RuntimeException e) {
2214
2215 LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString()
2216 + " -- aborting server", e);
2217 }
2218 }
2219 finally {
2220 parent.clearSplit();
2221 }
2222 return new HRegion[] { result.getFirst(), result.getSecond() };
2223 }
2224
2225
2226
2227
2228 @Test
2229 public void testGetScanner_WithOkFamilies() throws IOException {
2230 byte[] fam1 = Bytes.toBytes("fam1");
2231 byte[] fam2 = Bytes.toBytes("fam2");
2232
2233 byte[][] families = { fam1, fam2 };
2234
2235
2236 String method = this.getName();
2237 this.region = initHRegion(tableName, method, CONF, families);
2238 try {
2239 Scan scan = new Scan();
2240 scan.addFamily(fam1);
2241 scan.addFamily(fam2);
2242 try {
2243 region.getScanner(scan);
2244 } catch (Exception e) {
2245 assertTrue("Families could not be found in Region", false);
2246 }
2247 } finally {
2248 HRegion.closeHRegion(this.region);
2249 this.region = null;
2250 }
2251 }
2252
2253 @Test
2254 public void testGetScanner_WithNotOkFamilies() throws IOException {
2255 byte[] fam1 = Bytes.toBytes("fam1");
2256 byte[] fam2 = Bytes.toBytes("fam2");
2257
2258 byte[][] families = { fam1 };
2259
2260
2261 String method = this.getName();
2262 this.region = initHRegion(tableName, method, CONF, families);
2263 try {
2264 Scan scan = new Scan();
2265 scan.addFamily(fam2);
2266 boolean ok = false;
2267 try {
2268 region.getScanner(scan);
2269 } catch (Exception e) {
2270 ok = true;
2271 }
2272 assertTrue("Families could not be found in Region", ok);
2273 } finally {
2274 HRegion.closeHRegion(this.region);
2275 this.region = null;
2276 }
2277 }
2278
2279 @Test
2280 public void testGetScanner_WithNoFamilies() throws IOException {
2281 byte[] row1 = Bytes.toBytes("row1");
2282 byte[] fam1 = Bytes.toBytes("fam1");
2283 byte[] fam2 = Bytes.toBytes("fam2");
2284 byte[] fam3 = Bytes.toBytes("fam3");
2285 byte[] fam4 = Bytes.toBytes("fam4");
2286
2287 byte[][] families = { fam1, fam2, fam3, fam4 };
2288
2289
2290 String method = this.getName();
2291 this.region = initHRegion(tableName, method, CONF, families);
2292 try {
2293
2294
2295 Put put = new Put(row1);
2296 put.add(fam1, null, null);
2297 put.add(fam2, null, null);
2298 put.add(fam3, null, null);
2299 put.add(fam4, null, null);
2300 region.put(put);
2301
2302 Scan scan = null;
2303 HRegion.RegionScannerImpl is = null;
2304
2305
2306
2307
2308 scan = new Scan();
2309 scan.addFamily(fam2);
2310 scan.addFamily(fam4);
2311 is = (RegionScannerImpl) region.getScanner(scan);
2312 assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2313
2314 scan = new Scan();
2315 is = (RegionScannerImpl) region.getScanner(scan);
2316 assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2317 } finally {
2318 HRegion.closeHRegion(this.region);
2319 this.region = null;
2320 }
2321 }
2322
2323
2324
2325
2326
2327
2328 @Test
2329 public void testGetScanner_WithRegionClosed() throws IOException {
2330 byte[] fam1 = Bytes.toBytes("fam1");
2331 byte[] fam2 = Bytes.toBytes("fam2");
2332
2333 byte[][] families = { fam1, fam2 };
2334
2335
2336 String method = this.getName();
2337 try {
2338 this.region = initHRegion(tableName, method, CONF, families);
2339 } catch (IOException e) {
2340 e.printStackTrace();
2341 fail("Got IOException during initHRegion, " + e.getMessage());
2342 }
2343 try {
2344 region.closed.set(true);
2345 try {
2346 region.getScanner(null);
2347 fail("Expected to get an exception during getScanner on a region that is closed");
2348 } catch (NotServingRegionException e) {
2349
2350 } catch (IOException e) {
2351 fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2352 + e.getMessage());
2353 }
2354 } finally {
2355 HRegion.closeHRegion(this.region);
2356 this.region = null;
2357 }
2358 }
2359
2360 @Test
2361 public void testRegionScanner_Next() throws IOException {
2362 byte[] row1 = Bytes.toBytes("row1");
2363 byte[] row2 = Bytes.toBytes("row2");
2364 byte[] fam1 = Bytes.toBytes("fam1");
2365 byte[] fam2 = Bytes.toBytes("fam2");
2366 byte[] fam3 = Bytes.toBytes("fam3");
2367 byte[] fam4 = Bytes.toBytes("fam4");
2368
2369 byte[][] families = { fam1, fam2, fam3, fam4 };
2370 long ts = System.currentTimeMillis();
2371
2372
2373 String method = this.getName();
2374 this.region = initHRegion(tableName, method, CONF, families);
2375 try {
2376
2377 Put put = null;
2378 put = new Put(row1);
2379 put.add(fam1, (byte[]) null, ts, null);
2380 put.add(fam2, (byte[]) null, ts, null);
2381 put.add(fam3, (byte[]) null, ts, null);
2382 put.add(fam4, (byte[]) null, ts, null);
2383 region.put(put);
2384
2385 put = new Put(row2);
2386 put.add(fam1, (byte[]) null, ts, null);
2387 put.add(fam2, (byte[]) null, ts, null);
2388 put.add(fam3, (byte[]) null, ts, null);
2389 put.add(fam4, (byte[]) null, ts, null);
2390 region.put(put);
2391
2392 Scan scan = new Scan();
2393 scan.addFamily(fam2);
2394 scan.addFamily(fam4);
2395 InternalScanner is = region.getScanner(scan);
2396
2397 List<Cell> res = null;
2398
2399
2400 List<Cell> expected1 = new ArrayList<Cell>();
2401 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2402 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2403
2404 res = new ArrayList<Cell>();
2405 is.next(res);
2406 for (int i = 0; i < res.size(); i++) {
2407 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2408 }
2409
2410
2411 List<Cell> expected2 = new ArrayList<Cell>();
2412 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2413 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2414
2415 res = new ArrayList<Cell>();
2416 is.next(res);
2417 for (int i = 0; i < res.size(); i++) {
2418 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2419 }
2420 } finally {
2421 HRegion.closeHRegion(this.region);
2422 this.region = null;
2423 }
2424 }
2425
2426 @Test
2427 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2428 byte[] row1 = Bytes.toBytes("row1");
2429 byte[] qf1 = Bytes.toBytes("qualifier1");
2430 byte[] qf2 = Bytes.toBytes("qualifier2");
2431 byte[] fam1 = Bytes.toBytes("fam1");
2432 byte[][] families = { fam1 };
2433
2434 long ts1 = System.currentTimeMillis();
2435 long ts2 = ts1 + 1;
2436 long ts3 = ts1 + 2;
2437
2438
2439 String method = this.getName();
2440 this.region = initHRegion(tableName, method, CONF, families);
2441 try {
2442
2443 Put put = null;
2444 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2445 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2446 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2447
2448 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2449 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2450 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2451
2452 put = new Put(row1);
2453 put.add(kv13);
2454 put.add(kv12);
2455 put.add(kv11);
2456 put.add(kv23);
2457 put.add(kv22);
2458 put.add(kv21);
2459 region.put(put);
2460
2461
2462 List<Cell> expected = new ArrayList<Cell>();
2463 expected.add(kv13);
2464 expected.add(kv12);
2465
2466 Scan scan = new Scan(row1);
2467 scan.addColumn(fam1, qf1);
2468 scan.setMaxVersions(MAX_VERSIONS);
2469 List<Cell> actual = new ArrayList<Cell>();
2470 InternalScanner scanner = region.getScanner(scan);
2471
2472 boolean hasNext = scanner.next(actual);
2473 assertEquals(false, hasNext);
2474
2475
2476 for (int i = 0; i < expected.size(); i++) {
2477 assertEquals(expected.get(i), actual.get(i));
2478 }
2479 } finally {
2480 HRegion.closeHRegion(this.region);
2481 this.region = null;
2482 }
2483 }
2484
2485 @Test
2486 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2487 byte[] row1 = Bytes.toBytes("row1");
2488 byte[] qf1 = Bytes.toBytes("qualifier1");
2489 byte[] qf2 = Bytes.toBytes("qualifier2");
2490 byte[] fam1 = Bytes.toBytes("fam1");
2491 byte[][] families = { fam1 };
2492
2493 long ts1 = 1;
2494 long ts2 = ts1 + 1;
2495 long ts3 = ts1 + 2;
2496
2497
2498 String method = this.getName();
2499 this.region = initHRegion(tableName, method, CONF, families);
2500 try {
2501
2502 Put put = null;
2503 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2504 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2505 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2506
2507 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2508 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2509 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2510
2511 put = new Put(row1);
2512 put.add(kv13);
2513 put.add(kv12);
2514 put.add(kv11);
2515 put.add(kv23);
2516 put.add(kv22);
2517 put.add(kv21);
2518 region.put(put);
2519 region.flushcache();
2520
2521
2522 List<Cell> expected = new ArrayList<Cell>();
2523 expected.add(kv13);
2524 expected.add(kv12);
2525 expected.add(kv23);
2526 expected.add(kv22);
2527
2528 Scan scan = new Scan(row1);
2529 scan.addColumn(fam1, qf1);
2530 scan.addColumn(fam1, qf2);
2531 scan.setMaxVersions(MAX_VERSIONS);
2532 List<Cell> actual = new ArrayList<Cell>();
2533 InternalScanner scanner = region.getScanner(scan);
2534
2535 boolean hasNext = scanner.next(actual);
2536 assertEquals(false, hasNext);
2537
2538
2539 for (int i = 0; i < expected.size(); i++) {
2540 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2541 }
2542 } finally {
2543 HRegion.closeHRegion(this.region);
2544 this.region = null;
2545 }
2546 }
2547
2548 @Test
2549 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2550 byte[] row1 = Bytes.toBytes("row1");
2551 byte[] fam1 = Bytes.toBytes("fam1");
2552 byte[][] families = { fam1 };
2553 byte[] qf1 = Bytes.toBytes("qualifier1");
2554 byte[] qf2 = Bytes.toBytes("qualifier2");
2555
2556 long ts1 = 1;
2557 long ts2 = ts1 + 1;
2558 long ts3 = ts1 + 2;
2559 long ts4 = ts1 + 3;
2560
2561
2562 String method = this.getName();
2563 this.region = initHRegion(tableName, method, CONF, families);
2564 try {
2565
2566 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2567 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2568 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2569 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2570
2571 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2572 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2573 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2574 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2575
2576 Put put = null;
2577 put = new Put(row1);
2578 put.add(kv14);
2579 put.add(kv24);
2580 region.put(put);
2581 region.flushcache();
2582
2583 put = new Put(row1);
2584 put.add(kv23);
2585 put.add(kv13);
2586 region.put(put);
2587 region.flushcache();
2588
2589 put = new Put(row1);
2590 put.add(kv22);
2591 put.add(kv12);
2592 region.put(put);
2593 region.flushcache();
2594
2595 put = new Put(row1);
2596 put.add(kv21);
2597 put.add(kv11);
2598 region.put(put);
2599
2600
2601 List<Cell> expected = new ArrayList<Cell>();
2602 expected.add(kv14);
2603 expected.add(kv13);
2604 expected.add(kv12);
2605 expected.add(kv24);
2606 expected.add(kv23);
2607 expected.add(kv22);
2608
2609 Scan scan = new Scan(row1);
2610 scan.addColumn(fam1, qf1);
2611 scan.addColumn(fam1, qf2);
2612 int versions = 3;
2613 scan.setMaxVersions(versions);
2614 List<Cell> actual = new ArrayList<Cell>();
2615 InternalScanner scanner = region.getScanner(scan);
2616
2617 boolean hasNext = scanner.next(actual);
2618 assertEquals(false, hasNext);
2619
2620
2621 for (int i = 0; i < expected.size(); i++) {
2622 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2623 }
2624 } finally {
2625 HRegion.closeHRegion(this.region);
2626 this.region = null;
2627 }
2628 }
2629
2630 @Test
2631 public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2632 byte[] row1 = Bytes.toBytes("row1");
2633 byte[] qf1 = Bytes.toBytes("qualifier1");
2634 byte[] qf2 = Bytes.toBytes("qualifier2");
2635 byte[] fam1 = Bytes.toBytes("fam1");
2636 byte[][] families = { fam1 };
2637
2638 long ts1 = System.currentTimeMillis();
2639 long ts2 = ts1 + 1;
2640 long ts3 = ts1 + 2;
2641
2642
2643 String method = this.getName();
2644 this.region = initHRegion(tableName, method, CONF, families);
2645 try {
2646
2647 Put put = null;
2648 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2649 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2650 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2651
2652 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2653 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2654 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2655
2656 put = new Put(row1);
2657 put.add(kv13);
2658 put.add(kv12);
2659 put.add(kv11);
2660 put.add(kv23);
2661 put.add(kv22);
2662 put.add(kv21);
2663 region.put(put);
2664
2665
2666 List<Cell> expected = new ArrayList<Cell>();
2667 expected.add(kv13);
2668 expected.add(kv12);
2669 expected.add(kv23);
2670 expected.add(kv22);
2671
2672 Scan scan = new Scan(row1);
2673 scan.addFamily(fam1);
2674 scan.setMaxVersions(MAX_VERSIONS);
2675 List<Cell> actual = new ArrayList<Cell>();
2676 InternalScanner scanner = region.getScanner(scan);
2677
2678 boolean hasNext = scanner.next(actual);
2679 assertEquals(false, hasNext);
2680
2681
2682 for (int i = 0; i < expected.size(); i++) {
2683 assertEquals(expected.get(i), actual.get(i));
2684 }
2685 } finally {
2686 HRegion.closeHRegion(this.region);
2687 this.region = null;
2688 }
2689 }
2690
2691 @Test
2692 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
2693 byte[] row1 = Bytes.toBytes("row1");
2694 byte[] qf1 = Bytes.toBytes("qualifier1");
2695 byte[] qf2 = Bytes.toBytes("qualifier2");
2696 byte[] fam1 = Bytes.toBytes("fam1");
2697
2698 long ts1 = 1;
2699 long ts2 = ts1 + 1;
2700 long ts3 = ts1 + 2;
2701
2702
2703 String method = this.getName();
2704 this.region = initHRegion(tableName, method, CONF, fam1);
2705 try {
2706
2707 Put put = null;
2708 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2709 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2710 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2711
2712 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2713 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2714 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2715
2716 put = new Put(row1);
2717 put.add(kv13);
2718 put.add(kv12);
2719 put.add(kv11);
2720 put.add(kv23);
2721 put.add(kv22);
2722 put.add(kv21);
2723 region.put(put);
2724 region.flushcache();
2725
2726
2727 List<Cell> expected = new ArrayList<Cell>();
2728 expected.add(kv13);
2729 expected.add(kv12);
2730 expected.add(kv23);
2731 expected.add(kv22);
2732
2733 Scan scan = new Scan(row1);
2734 scan.addFamily(fam1);
2735 scan.setMaxVersions(MAX_VERSIONS);
2736 List<Cell> actual = new ArrayList<Cell>();
2737 InternalScanner scanner = region.getScanner(scan);
2738
2739 boolean hasNext = scanner.next(actual);
2740 assertEquals(false, hasNext);
2741
2742
2743 for (int i = 0; i < expected.size(); i++) {
2744 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2745 }
2746 } finally {
2747 HRegion.closeHRegion(this.region);
2748 this.region = null;
2749 }
2750 }
2751
2752 @Test
2753 public void testScanner_StopRow1542() throws IOException {
2754 byte[] family = Bytes.toBytes("testFamily");
2755 this.region = initHRegion(tableName, getName(), CONF, family);
2756 try {
2757 byte[] row1 = Bytes.toBytes("row111");
2758 byte[] row2 = Bytes.toBytes("row222");
2759 byte[] row3 = Bytes.toBytes("row333");
2760 byte[] row4 = Bytes.toBytes("row444");
2761 byte[] row5 = Bytes.toBytes("row555");
2762
2763 byte[] col1 = Bytes.toBytes("Pub111");
2764 byte[] col2 = Bytes.toBytes("Pub222");
2765
2766 Put put = new Put(row1);
2767 put.add(family, col1, Bytes.toBytes(10L));
2768 region.put(put);
2769
2770 put = new Put(row2);
2771 put.add(family, col1, Bytes.toBytes(15L));
2772 region.put(put);
2773
2774 put = new Put(row3);
2775 put.add(family, col2, Bytes.toBytes(20L));
2776 region.put(put);
2777
2778 put = new Put(row4);
2779 put.add(family, col2, Bytes.toBytes(30L));
2780 region.put(put);
2781
2782 put = new Put(row5);
2783 put.add(family, col1, Bytes.toBytes(40L));
2784 region.put(put);
2785
2786 Scan scan = new Scan(row3, row4);
2787 scan.setMaxVersions();
2788 scan.addColumn(family, col1);
2789 InternalScanner s = region.getScanner(scan);
2790
2791 List<Cell> results = new ArrayList<Cell>();
2792 assertEquals(false, s.next(results));
2793 assertEquals(0, results.size());
2794 } finally {
2795 HRegion.closeHRegion(this.region);
2796 this.region = null;
2797 }
2798 }
2799
2800 @Test
2801 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2802 byte[] row1 = Bytes.toBytes("row1");
2803 byte[] fam1 = Bytes.toBytes("fam1");
2804 byte[] qf1 = Bytes.toBytes("qualifier1");
2805 byte[] qf2 = Bytes.toBytes("quateslifier2");
2806
2807 long ts1 = 1;
2808 long ts2 = ts1 + 1;
2809 long ts3 = ts1 + 2;
2810 long ts4 = ts1 + 3;
2811
2812
2813 String method = this.getName();
2814 this.region = initHRegion(tableName, method, CONF, fam1);
2815 try {
2816
2817 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2818 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2819 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2820 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2821
2822 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2823 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2824 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2825 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2826
2827 Put put = null;
2828 put = new Put(row1);
2829 put.add(kv14);
2830 put.add(kv24);
2831 region.put(put);
2832 region.flushcache();
2833
2834 put = new Put(row1);
2835 put.add(kv23);
2836 put.add(kv13);
2837 region.put(put);
2838 region.flushcache();
2839
2840 put = new Put(row1);
2841 put.add(kv22);
2842 put.add(kv12);
2843 region.put(put);
2844 region.flushcache();
2845
2846 put = new Put(row1);
2847 put.add(kv21);
2848 put.add(kv11);
2849 region.put(put);
2850
2851
2852 List<KeyValue> expected = new ArrayList<KeyValue>();
2853 expected.add(kv14);
2854 expected.add(kv13);
2855 expected.add(kv12);
2856 expected.add(kv24);
2857 expected.add(kv23);
2858 expected.add(kv22);
2859
2860 Scan scan = new Scan(row1);
2861 int versions = 3;
2862 scan.setMaxVersions(versions);
2863 List<Cell> actual = new ArrayList<Cell>();
2864 InternalScanner scanner = region.getScanner(scan);
2865
2866 boolean hasNext = scanner.next(actual);
2867 assertEquals(false, hasNext);
2868
2869
2870 for (int i = 0; i < expected.size(); i++) {
2871 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2872 }
2873 } finally {
2874 HRegion.closeHRegion(this.region);
2875 this.region = null;
2876 }
2877 }
2878
2879
2880
2881
2882
2883
2884
2885 @Test
2886 public void testScanner_JoinedScanners() throws IOException {
2887 byte[] cf_essential = Bytes.toBytes("essential");
2888 byte[] cf_joined = Bytes.toBytes("joined");
2889 byte[] cf_alpha = Bytes.toBytes("alpha");
2890 this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
2891 try {
2892 byte[] row1 = Bytes.toBytes("row1");
2893 byte[] row2 = Bytes.toBytes("row2");
2894 byte[] row3 = Bytes.toBytes("row3");
2895
2896 byte[] col_normal = Bytes.toBytes("d");
2897 byte[] col_alpha = Bytes.toBytes("a");
2898
2899 byte[] filtered_val = Bytes.toBytes(3);
2900
2901 Put put = new Put(row1);
2902 put.add(cf_essential, col_normal, Bytes.toBytes(1));
2903 put.add(cf_joined, col_alpha, Bytes.toBytes(1));
2904 region.put(put);
2905
2906 put = new Put(row2);
2907 put.add(cf_essential, col_alpha, Bytes.toBytes(2));
2908 put.add(cf_joined, col_normal, Bytes.toBytes(2));
2909 put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
2910 region.put(put);
2911
2912 put = new Put(row3);
2913 put.add(cf_essential, col_normal, filtered_val);
2914 put.add(cf_joined, col_normal, filtered_val);
2915 region.put(put);
2916
2917
2918
2919
2920
2921 Scan scan = new Scan();
2922 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
2923 CompareOp.NOT_EQUAL, filtered_val);
2924 scan.setFilter(filter);
2925 scan.setLoadColumnFamiliesOnDemand(true);
2926 InternalScanner s = region.getScanner(scan);
2927
2928 List<Cell> results = new ArrayList<Cell>();
2929 assertTrue(s.next(results));
2930 assertEquals(results.size(), 1);
2931 results.clear();
2932
2933 assertTrue(s.next(results));
2934 assertEquals(results.size(), 3);
2935 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
2936 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
2937 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
2938 results.clear();
2939
2940 assertFalse(s.next(results));
2941 assertEquals(results.size(), 0);
2942 } finally {
2943 HRegion.closeHRegion(this.region);
2944 this.region = null;
2945 }
2946 }
2947
2948
2949
2950
2951
2952
2953 @Test
2954 public void testScanner_JoinedScannersWithLimits() throws IOException {
2955 final byte[] cf_first = Bytes.toBytes("first");
2956 final byte[] cf_second = Bytes.toBytes("second");
2957
2958 this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
2959 try {
2960 final byte[] col_a = Bytes.toBytes("a");
2961 final byte[] col_b = Bytes.toBytes("b");
2962
2963 Put put;
2964
2965 for (int i = 0; i < 10; i++) {
2966 put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
2967 put.add(cf_first, col_a, Bytes.toBytes(i));
2968 if (i < 5) {
2969 put.add(cf_first, col_b, Bytes.toBytes(i));
2970 put.add(cf_second, col_a, Bytes.toBytes(i));
2971 put.add(cf_second, col_b, Bytes.toBytes(i));
2972 }
2973 region.put(put);
2974 }
2975
2976 Scan scan = new Scan();
2977 scan.setLoadColumnFamiliesOnDemand(true);
2978 Filter bogusFilter = new FilterBase() {
2979 @Override
2980 public boolean isFamilyEssential(byte[] name) {
2981 return Bytes.equals(name, cf_first);
2982 }
2983 };
2984
2985 scan.setFilter(bogusFilter);
2986 InternalScanner s = region.getScanner(scan);
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017 List<Cell> results = new ArrayList<Cell>();
3018 int index = 0;
3019 while (true) {
3020 boolean more = s.next(results, 3);
3021 if ((index >> 1) < 5) {
3022 if (index % 2 == 0)
3023 assertEquals(results.size(), 3);
3024 else
3025 assertEquals(results.size(), 1);
3026 } else
3027 assertEquals(results.size(), 1);
3028 results.clear();
3029 index++;
3030 if (!more)
3031 break;
3032 }
3033 } finally {
3034 HRegion.closeHRegion(this.region);
3035 this.region = null;
3036 }
3037 }
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047 @Test
3048 public void testBasicSplit() throws Exception {
3049 byte[][] families = { fam1, fam2, fam3 };
3050
3051 Configuration hc = initSplit();
3052
3053 String method = this.getName();
3054 this.region = initHRegion(tableName, method, hc, families);
3055
3056 try {
3057 LOG.info("" + HBaseTestCase.addContent(region, fam3));
3058 region.flushcache();
3059 region.compactStores();
3060 byte[] splitRow = region.checkSplit();
3061 assertNotNull(splitRow);
3062 LOG.info("SplitRow: " + Bytes.toString(splitRow));
3063 HRegion[] regions = splitRegion(region, splitRow);
3064 try {
3065
3066
3067
3068 for (int i = 0; i < regions.length; i++) {
3069 regions[i] = HRegion.openHRegion(regions[i], null);
3070 }
3071
3072
3073 assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
3074 assertGet(regions[1], fam3, splitRow);
3075
3076 assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
3077 assertScan(regions[1], fam3, splitRow);
3078
3079 for (int i = 0; i < regions.length; i++) {
3080
3081
3082 for (int j = 0; j < 2; j++) {
3083 HBaseTestCase.addContent(regions[i], fam3);
3084 }
3085 HBaseTestCase.addContent(regions[i], fam2);
3086 HBaseTestCase.addContent(regions[i], fam1);
3087 regions[i].flushcache();
3088 }
3089
3090 byte[][] midkeys = new byte[regions.length][];
3091
3092 for (int i = 0; i < regions.length; i++) {
3093 regions[i].compactStores();
3094 midkeys[i] = regions[i].checkSplit();
3095 }
3096
3097 TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
3098
3099
3100 for (int i = 0; i < regions.length; i++) {
3101 HRegion[] rs = null;
3102 if (midkeys[i] != null) {
3103 rs = splitRegion(regions[i], midkeys[i]);
3104 for (int j = 0; j < rs.length; j++) {
3105 sortedMap.put(Bytes.toString(rs[j].getRegionName()), HRegion.openHRegion(rs[j], null));
3106 }
3107 }
3108 }
3109 LOG.info("Made 4 regions");
3110
3111
3112 int interval = (LAST_CHAR - FIRST_CHAR) / 3;
3113 byte[] b = Bytes.toBytes(START_KEY);
3114 for (HRegion r : sortedMap.values()) {
3115 assertGet(r, fam3, b);
3116 b[0] += interval;
3117 }
3118 } finally {
3119 for (int i = 0; i < regions.length; i++) {
3120 try {
3121 regions[i].close();
3122 } catch (IOException e) {
3123
3124 }
3125 }
3126 }
3127 } finally {
3128 HRegion.closeHRegion(this.region);
3129 this.region = null;
3130 }
3131 }
3132
3133 @Test
3134 public void testSplitRegion() throws IOException {
3135 byte[] qualifier = Bytes.toBytes("qualifier");
3136 Configuration hc = initSplit();
3137 int numRows = 10;
3138 byte[][] families = { fam1, fam3 };
3139
3140
3141 String method = this.getName();
3142 this.region = initHRegion(tableName, method, hc, families);
3143
3144
3145 int startRow = 100;
3146 putData(startRow, numRows, qualifier, families);
3147 int splitRow = startRow + numRows;
3148 putData(splitRow, numRows, qualifier, families);
3149 region.flushcache();
3150
3151 HRegion[] regions = null;
3152 try {
3153 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
3154
3155 for (int i = 0; i < regions.length; i++) {
3156 regions[i] = HRegion.openHRegion(regions[i], null);
3157 }
3158
3159 assertEquals(2, regions.length);
3160
3161
3162
3163 verifyData(regions[0], startRow, numRows, qualifier, families);
3164 verifyData(regions[1], splitRow, numRows, qualifier, families);
3165
3166 } finally {
3167 HRegion.closeHRegion(this.region);
3168 this.region = null;
3169 }
3170 }
3171
3172 @Test
3173 public void testClearForceSplit() throws IOException {
3174 byte[] qualifier = Bytes.toBytes("qualifier");
3175 Configuration hc = initSplit();
3176 int numRows = 10;
3177 byte[][] families = { fam1, fam3 };
3178
3179
3180 String method = this.getName();
3181 this.region = initHRegion(tableName, method, hc, families);
3182
3183
3184 int startRow = 100;
3185 putData(startRow, numRows, qualifier, families);
3186 int splitRow = startRow + numRows;
3187 byte[] splitRowBytes = Bytes.toBytes("" + splitRow);
3188 putData(splitRow, numRows, qualifier, families);
3189 region.flushcache();
3190
3191 HRegion[] regions = null;
3192 try {
3193
3194 region.forceSplit(splitRowBytes);
3195 assertTrue(region.shouldForceSplit());
3196
3197 assertTrue(Bytes.equals(splitRowBytes, region.checkSplit()));
3198
3199
3200 HStore storeMock = Mockito.mock(HStore.class);
3201 when(storeMock.hasReferences()).thenReturn(true);
3202 when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
3203 when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
3204 when(storeMock.getColumnFamilyName()).thenReturn("cf");
3205 region.stores.put(Bytes.toBytes(storeMock.getColumnFamilyName()), storeMock);
3206 assertTrue(region.hasReferences());
3207
3208
3209 regions = splitRegion(region, splitRowBytes);
3210 assertNull(regions);
3211
3212
3213 assertFalse(region.shouldForceSplit());
3214
3215
3216 region.stores.remove(Bytes.toBytes(storeMock.getColumnFamilyName()));
3217 assertFalse(region.hasReferences());
3218
3219
3220 regions = splitRegion(region, splitRowBytes);
3221
3222
3223 for (int i = 0; i < regions.length; i++) {
3224 regions[i] = HRegion.openHRegion(regions[i], null);
3225 }
3226
3227 assertEquals(2, regions.length);
3228
3229
3230
3231 verifyData(regions[0], startRow, numRows, qualifier, families);
3232 verifyData(regions[1], splitRow, numRows, qualifier, families);
3233
3234 } finally {
3235 HRegion.closeHRegion(this.region);
3236 this.region = null;
3237 }
3238 }
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250 @Test
3251 public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3252 byte[] family = Bytes.toBytes("family");
3253 int numRows = 1000;
3254 int flushAndScanInterval = 10;
3255 int compactInterval = 10 * flushAndScanInterval;
3256
3257 String method = "testFlushCacheWhileScanning";
3258 this.region = initHRegion(tableName, method, CONF, family);
3259 try {
3260 FlushThread flushThread = new FlushThread();
3261 flushThread.start();
3262
3263 Scan scan = new Scan();
3264 scan.addFamily(family);
3265 scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
3266 new BinaryComparator(Bytes.toBytes(5L))));
3267
3268 int expectedCount = 0;
3269 List<Cell> res = new ArrayList<Cell>();
3270
3271 boolean toggle = true;
3272 for (long i = 0; i < numRows; i++) {
3273 Put put = new Put(Bytes.toBytes(i));
3274 put.setDurability(Durability.SKIP_WAL);
3275 put.add(family, qual1, Bytes.toBytes(i % 10));
3276 region.put(put);
3277
3278 if (i != 0 && i % compactInterval == 0) {
3279
3280 region.compactStores(true);
3281 }
3282
3283 if (i % 10 == 5L) {
3284 expectedCount++;
3285 }
3286
3287 if (i != 0 && i % flushAndScanInterval == 0) {
3288 res.clear();
3289 InternalScanner scanner = region.getScanner(scan);
3290 if (toggle) {
3291 flushThread.flush();
3292 }
3293 while (scanner.next(res))
3294 ;
3295 if (!toggle) {
3296 flushThread.flush();
3297 }
3298 assertEquals("i=" + i, expectedCount, res.size());
3299 toggle = !toggle;
3300 }
3301 }
3302
3303 flushThread.done();
3304 flushThread.join();
3305 flushThread.checkNoError();
3306 } finally {
3307 HRegion.closeHRegion(this.region);
3308 this.region = null;
3309 }
3310 }
3311
3312 protected class FlushThread extends Thread {
3313 private volatile boolean done;
3314 private Throwable error = null;
3315
3316 public void done() {
3317 done = true;
3318 synchronized (this) {
3319 interrupt();
3320 }
3321 }
3322
3323 public void checkNoError() {
3324 if (error != null) {
3325 assertNull(error);
3326 }
3327 }
3328
3329 @Override
3330 public void run() {
3331 done = false;
3332 while (!done) {
3333 synchronized (this) {
3334 try {
3335 wait();
3336 } catch (InterruptedException ignored) {
3337 if (done) {
3338 break;
3339 }
3340 }
3341 }
3342 try {
3343 region.flushcache();
3344 } catch (IOException e) {
3345 if (!done) {
3346 LOG.error("Error while flusing cache", e);
3347 error = e;
3348 }
3349 break;
3350 }
3351 }
3352
3353 }
3354
3355 public void flush() {
3356 synchronized (this) {
3357 notify();
3358 }
3359
3360 }
3361 }
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372 @Test
3373 public void testWritesWhileScanning() throws IOException, InterruptedException {
3374 int testCount = 100;
3375 int numRows = 1;
3376 int numFamilies = 10;
3377 int numQualifiers = 100;
3378 int flushInterval = 7;
3379 int compactInterval = 5 * flushInterval;
3380 byte[][] families = new byte[numFamilies][];
3381 for (int i = 0; i < numFamilies; i++) {
3382 families[i] = Bytes.toBytes("family" + i);
3383 }
3384 byte[][] qualifiers = new byte[numQualifiers][];
3385 for (int i = 0; i < numQualifiers; i++) {
3386 qualifiers[i] = Bytes.toBytes("qual" + i);
3387 }
3388
3389 String method = "testWritesWhileScanning";
3390 this.region = initHRegion(tableName, method, CONF, families);
3391 try {
3392 PutThread putThread = new PutThread(numRows, families, qualifiers);
3393 putThread.start();
3394 putThread.waitForFirstPut();
3395
3396 FlushThread flushThread = new FlushThread();
3397 flushThread.start();
3398
3399 Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3400
3401 int expectedCount = numFamilies * numQualifiers;
3402 List<Cell> res = new ArrayList<Cell>();
3403
3404 long prevTimestamp = 0L;
3405 for (int i = 0; i < testCount; i++) {
3406
3407 if (i != 0 && i % compactInterval == 0) {
3408 region.compactStores(true);
3409 }
3410
3411 if (i != 0 && i % flushInterval == 0) {
3412 flushThread.flush();
3413 }
3414
3415 boolean previousEmpty = res.isEmpty();
3416 res.clear();
3417 InternalScanner scanner = region.getScanner(scan);
3418 while (scanner.next(res))
3419 ;
3420 if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3421 assertEquals("i=" + i, expectedCount, res.size());
3422 long timestamp = res.get(0).getTimestamp();
3423 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3424 timestamp >= prevTimestamp);
3425 prevTimestamp = timestamp;
3426 }
3427 }
3428
3429 putThread.done();
3430
3431 region.flushcache();
3432
3433 putThread.join();
3434 putThread.checkNoError();
3435
3436 flushThread.done();
3437 flushThread.join();
3438 flushThread.checkNoError();
3439 } finally {
3440 try {
3441 HRegion.closeHRegion(this.region);
3442 } catch (DroppedSnapshotException dse) {
3443
3444
3445
3446
3447 }
3448 this.region = null;
3449 }
3450 }
3451
3452 protected class PutThread extends Thread {
3453 private volatile boolean done;
3454 private volatile int numPutsFinished = 0;
3455
3456 private Throwable error = null;
3457 private int numRows;
3458 private byte[][] families;
3459 private byte[][] qualifiers;
3460
3461 private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3462 this.numRows = numRows;
3463 this.families = families;
3464 this.qualifiers = qualifiers;
3465 }
3466
3467
3468
3469
3470 public void waitForFirstPut() throws InterruptedException {
3471
3472 while (numPutsFinished == 0) {
3473 checkNoError();
3474 Thread.sleep(50);
3475 }
3476 }
3477
3478 public void done() {
3479 done = true;
3480 synchronized (this) {
3481 interrupt();
3482 }
3483 }
3484
3485 public void checkNoError() {
3486 if (error != null) {
3487 assertNull(error);
3488 }
3489 }
3490
3491 @Override
3492 public void run() {
3493 done = false;
3494 while (!done) {
3495 try {
3496 for (int r = 0; r < numRows; r++) {
3497 byte[] row = Bytes.toBytes("row" + r);
3498 Put put = new Put(row);
3499 put.setDurability(Durability.SKIP_WAL);
3500 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3501 for (byte[] family : families) {
3502 for (byte[] qualifier : qualifiers) {
3503 put.add(family, qualifier, (long) numPutsFinished, value);
3504 }
3505 }
3506 region.put(put);
3507 numPutsFinished++;
3508 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3509 System.out.println("put iteration = " + numPutsFinished);
3510 Delete delete = new Delete(row, (long) numPutsFinished - 30);
3511 region.delete(delete);
3512 }
3513 numPutsFinished++;
3514 }
3515 } catch (InterruptedIOException e) {
3516
3517 } catch (IOException e) {
3518 LOG.error("error while putting records", e);
3519 error = e;
3520 break;
3521 }
3522 }
3523
3524 }
3525
3526 }
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537 @Test
3538 public void testWritesWhileGetting() throws Exception {
3539 int testCount = 100;
3540 int numRows = 1;
3541 int numFamilies = 10;
3542 int numQualifiers = 100;
3543 int compactInterval = 100;
3544 byte[][] families = new byte[numFamilies][];
3545 for (int i = 0; i < numFamilies; i++) {
3546 families[i] = Bytes.toBytes("family" + i);
3547 }
3548 byte[][] qualifiers = new byte[numQualifiers][];
3549 for (int i = 0; i < numQualifiers; i++) {
3550 qualifiers[i] = Bytes.toBytes("qual" + i);
3551 }
3552
3553
3554 String method = "testWritesWhileGetting";
3555
3556
3557
3558
3559
3560 Configuration conf = HBaseConfiguration.create(CONF);
3561 conf.setInt("hbase.hstore.compaction.min", 1);
3562 conf.setInt("hbase.hstore.compaction.max", 1000);
3563 this.region = initHRegion(tableName, method, conf, families);
3564 PutThread putThread = null;
3565 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3566 try {
3567 putThread = new PutThread(numRows, families, qualifiers);
3568 putThread.start();
3569 putThread.waitForFirstPut();
3570
3571
3572 ctx.addThread(new RepeatingTestThread(ctx) {
3573 private int flushesSinceCompact = 0;
3574 private final int maxFlushesSinceCompact = 20;
3575
3576 @Override
3577 public void doAnAction() throws Exception {
3578 if (region.flushcache().isCompactionNeeded()) {
3579 ++flushesSinceCompact;
3580 }
3581
3582
3583 if (flushesSinceCompact == maxFlushesSinceCompact) {
3584 region.compactStores(false);
3585 flushesSinceCompact = 0;
3586 }
3587 }
3588 });
3589 ctx.startThreads();
3590
3591 Get get = new Get(Bytes.toBytes("row0"));
3592 Result result = null;
3593
3594 int expectedCount = numFamilies * numQualifiers;
3595
3596 long prevTimestamp = 0L;
3597 for (int i = 0; i < testCount; i++) {
3598
3599 boolean previousEmpty = result == null || result.isEmpty();
3600 result = region.get(get);
3601 if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3602 assertEquals("i=" + i, expectedCount, result.size());
3603
3604
3605 long timestamp = 0;
3606 for (Cell kv : result.rawCells()) {
3607 if (CellUtil.matchingFamily(kv, families[0])
3608 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3609 timestamp = kv.getTimestamp();
3610 }
3611 }
3612 assertTrue(timestamp >= prevTimestamp);
3613 prevTimestamp = timestamp;
3614 Cell previousKV = null;
3615
3616 for (Cell kv : result.rawCells()) {
3617 byte[] thisValue = CellUtil.cloneValue(kv);
3618 if (previousKV != null) {
3619 if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3620 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3621 + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3622 + "(memStoreTS:" + kv.getMvccVersion() + ")");
3623 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3624 }
3625 }
3626 previousKV = kv;
3627 }
3628 }
3629 }
3630 } finally {
3631 if (putThread != null)
3632 putThread.done();
3633
3634 region.flushcache();
3635
3636 if (putThread != null) {
3637 putThread.join();
3638 putThread.checkNoError();
3639 }
3640
3641 ctx.stop();
3642 HRegion.closeHRegion(this.region);
3643 this.region = null;
3644 }
3645 }
3646
3647 @Test
3648 public void testHolesInMeta() throws Exception {
3649 byte[] family = Bytes.toBytes("family");
3650 this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF,
3651 false, family);
3652 try {
3653 byte[] rowNotServed = Bytes.toBytes("a");
3654 Get g = new Get(rowNotServed);
3655 try {
3656 region.get(g);
3657 fail();
3658 } catch (WrongRegionException x) {
3659
3660 }
3661 byte[] row = Bytes.toBytes("y");
3662 g = new Get(row);
3663 region.get(g);
3664 } finally {
3665 HRegion.closeHRegion(this.region);
3666 this.region = null;
3667 }
3668 }
3669
3670 @Test
3671 public void testIndexesScanWithOneDeletedRow() throws IOException {
3672 byte[] family = Bytes.toBytes("family");
3673
3674
3675 String method = "testIndexesScanWithOneDeletedRow";
3676 this.region = initHRegion(tableName, method, CONF, family);
3677 try {
3678 Put put = new Put(Bytes.toBytes(1L));
3679 put.add(family, qual1, 1L, Bytes.toBytes(1L));
3680 region.put(put);
3681
3682 region.flushcache();
3683
3684 Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3685 region.delete(delete);
3686
3687 put = new Put(Bytes.toBytes(2L));
3688 put.add(family, qual1, 2L, Bytes.toBytes(2L));
3689 region.put(put);
3690
3691 Scan idxScan = new Scan();
3692 idxScan.addFamily(family);
3693 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
3694 new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
3695 new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
3696 CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
3697 InternalScanner scanner = region.getScanner(idxScan);
3698 List<Cell> res = new ArrayList<Cell>();
3699
3700 while (scanner.next(res))
3701 ;
3702 assertEquals(1L, res.size());
3703 } finally {
3704 HRegion.closeHRegion(this.region);
3705 this.region = null;
3706 }
3707 }
3708
3709
3710
3711
3712 @Test
3713 public void testBloomFilterSize() throws IOException {
3714 byte[] fam1 = Bytes.toBytes("fam1");
3715 byte[] qf1 = Bytes.toBytes("col");
3716 byte[] val1 = Bytes.toBytes("value1");
3717
3718 HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
3719 .setBloomFilterType(BloomType.ROWCOL);
3720
3721 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3722 htd.addFamily(hcd);
3723 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3724 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3725 try {
3726 int num_unique_rows = 10;
3727 int duplicate_multiplier = 2;
3728 int num_storefiles = 4;
3729
3730 int version = 0;
3731 for (int f = 0; f < num_storefiles; f++) {
3732 for (int i = 0; i < duplicate_multiplier; i++) {
3733 for (int j = 0; j < num_unique_rows; j++) {
3734 Put put = new Put(Bytes.toBytes("row" + j));
3735 put.setDurability(Durability.SKIP_WAL);
3736 put.add(fam1, qf1, version++, val1);
3737 region.put(put);
3738 }
3739 }
3740 region.flushcache();
3741 }
3742
3743 HStore store = (HStore) region.getStore(fam1);
3744 Collection<StoreFile> storeFiles = store.getStorefiles();
3745 for (StoreFile storefile : storeFiles) {
3746 StoreFile.Reader reader = storefile.getReader();
3747 reader.loadFileInfo();
3748 reader.loadBloomfilter();
3749 assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
3750 assertEquals(num_unique_rows, reader.getFilterEntries());
3751 }
3752
3753 region.compactStores(true);
3754
3755
3756 storeFiles = store.getStorefiles();
3757 for (StoreFile storefile : storeFiles) {
3758 StoreFile.Reader reader = storefile.getReader();
3759 reader.loadFileInfo();
3760 reader.loadBloomfilter();
3761 assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
3762 assertEquals(num_unique_rows, reader.getFilterEntries());
3763 }
3764 } finally {
3765 HRegion.closeHRegion(this.region);
3766 this.region = null;
3767 }
3768 }
3769
3770 @Test
3771 public void testAllColumnsWithBloomFilter() throws IOException {
3772 byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
3773 byte[] FAMILY = Bytes.toBytes("family");
3774
3775
3776 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
3777 .setBloomFilterType(BloomType.ROWCOL);
3778 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
3779 htd.addFamily(hcd);
3780 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3781 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3782 try {
3783
3784 byte row[] = Bytes.toBytes("row:" + 0);
3785 byte column[] = Bytes.toBytes("column:" + 0);
3786 Put put = new Put(row);
3787 put.setDurability(Durability.SKIP_WAL);
3788 for (long idx = 1; idx <= 4; idx++) {
3789 put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
3790 }
3791 region.put(put);
3792
3793
3794 region.flushcache();
3795
3796
3797 Get get = new Get(row);
3798 get.setMaxVersions();
3799 Cell[] kvs = region.get(get).rawCells();
3800
3801
3802 assertEquals(4, kvs.length);
3803 checkOneCell(kvs[0], FAMILY, 0, 0, 4);
3804 checkOneCell(kvs[1], FAMILY, 0, 0, 3);
3805 checkOneCell(kvs[2], FAMILY, 0, 0, 2);
3806 checkOneCell(kvs[3], FAMILY, 0, 0, 1);
3807 } finally {
3808 HRegion.closeHRegion(this.region);
3809 this.region = null;
3810 }
3811 }
3812
3813
3814
3815
3816
3817
3818 @Test
3819 public void testDeleteRowWithBloomFilter() throws IOException {
3820 byte[] familyName = Bytes.toBytes("familyName");
3821
3822
3823 HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
3824 .setBloomFilterType(BloomType.ROWCOL);
3825
3826 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3827 htd.addFamily(hcd);
3828 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3829 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3830 try {
3831
3832 byte row[] = Bytes.toBytes("row1");
3833 byte col[] = Bytes.toBytes("col1");
3834
3835 Put put = new Put(row);
3836 put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
3837 region.put(put);
3838 region.flushcache();
3839
3840 Delete del = new Delete(row);
3841 region.delete(del);
3842 region.flushcache();
3843
3844
3845 Get get = new Get(row);
3846 get.addColumn(familyName, col);
3847
3848 Cell[] keyValues = region.get(get).rawCells();
3849 assertTrue(keyValues.length == 0);
3850 } finally {
3851 HRegion.closeHRegion(this.region);
3852 this.region = null;
3853 }
3854 }
3855
3856 @Test
3857 public void testgetHDFSBlocksDistribution() throws Exception {
3858 HBaseTestingUtility htu = new HBaseTestingUtility();
3859 final int DEFAULT_BLOCK_SIZE = 1024;
3860 htu.getConfiguration().setLong("dfs.namenode.fs-limits.min-block-size", 0);
3861 htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
3862 htu.getConfiguration().setInt("dfs.replication", 2);
3863
3864
3865 MiniHBaseCluster cluster = null;
3866 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
3867 int regionServersCount = 3;
3868
3869 try {
3870 cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
3871 byte[][] families = { fam1, fam2 };
3872 HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);
3873
3874
3875 byte row[] = Bytes.toBytes("row1");
3876 byte col[] = Bytes.toBytes("col1");
3877
3878 Put put = new Put(row);
3879 put.add(fam1, col, 1, Bytes.toBytes("test1"));
3880 put.add(fam2, col, 1, Bytes.toBytes("test2"));
3881 ht.put(put);
3882
3883 HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
3884 .get(0);
3885 firstRegion.flushcache();
3886 HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
3887
3888
3889
3890
3891
3892 long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
3893
3894 String topHost = blocksDistribution1.getTopHosts().get(0);
3895 long topHostWeight = blocksDistribution1.getWeight(topHost);
3896 assertTrue(uniqueBlocksWeight1 == topHostWeight);
3897
3898
3899
3900 HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
3901 htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
3902 long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
3903
3904 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
3905
3906 ht.close();
3907 } finally {
3908 if (cluster != null) {
3909 htu.shutdownMiniCluster();
3910 }
3911 }
3912 }
3913
3914
3915
3916
3917
3918
3919
3920 @Test
3921 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
3922 TableName tableName = TableName.valueOf(name.getMethodName());
3923 HRegionInfo info = null;
3924 try {
3925 FileSystem fs = Mockito.mock(FileSystem.class);
3926 when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
3927 HTableDescriptor htd = new HTableDescriptor(tableName);
3928 htd.addFamily(new HColumnDescriptor("cf"));
3929 info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
3930 HConstants.EMPTY_BYTE_ARRAY, false);
3931 Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
3932 region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
3933
3934 region.initialize();
3935 fail("Region initialization should fail due to IOException");
3936 } catch (IOException io) {
3937 List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
3938 for (MonitoredTask monitoredTask : tasks) {
3939 if (!(monitoredTask instanceof MonitoredRPCHandler)
3940 && monitoredTask.getDescription().contains(region.toString())) {
3941 assertTrue("Region state should be ABORTED.",
3942 monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
3943 break;
3944 }
3945 }
3946 } finally {
3947 HRegion.closeHRegion(region);
3948 }
3949 }
3950
3951
3952
3953
3954
3955 @Test
3956 public void testRegionInfoFileCreation() throws IOException {
3957 Path rootDir = new Path(dir + "testRegionInfoFileCreation");
3958
3959 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
3960 htd.addFamily(new HColumnDescriptor("cf"));
3961
3962 HRegionInfo hri = new HRegionInfo(htd.getTableName());
3963
3964
3965 HRegion region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
3966
3967 Path regionDir = region.getRegionFileSystem().getRegionDir();
3968 FileSystem fs = region.getRegionFileSystem().getFileSystem();
3969 HRegion.closeHRegion(region);
3970
3971 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
3972
3973
3974 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3975 fs.exists(regionInfoFile));
3976
3977
3978 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3979 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3980 HRegion.closeHRegion(region);
3981
3982
3983 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3984 fs.exists(regionInfoFile));
3985
3986
3987 fs.delete(regionInfoFile);
3988 assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
3989 fs.exists(regionInfoFile));
3990
3991 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3992
3993 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3994 HRegion.closeHRegion(region);
3995
3996
3997 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3998 fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
3999 }
4000
4001
4002
4003
4004 private static class Incrementer implements Runnable {
4005 private HRegion region;
4006 private final static byte[] incRow = Bytes.toBytes("incRow");
4007 private final static byte[] family = Bytes.toBytes("family");
4008 private final static byte[] qualifier = Bytes.toBytes("qualifier");
4009 private final static long ONE = 1l;
4010 private int incCounter;
4011
4012 public Incrementer(HRegion region, int incCounter) {
4013 this.region = region;
4014 this.incCounter = incCounter;
4015 }
4016
4017 @Override
4018 public void run() {
4019 int count = 0;
4020 while (count < incCounter) {
4021 Increment inc = new Increment(incRow);
4022 inc.addColumn(family, qualifier, ONE);
4023 count++;
4024 try {
4025 region.increment(inc);
4026 } catch (IOException e) {
4027 e.printStackTrace();
4028 break;
4029 }
4030 }
4031 }
4032 }
4033
4034
4035
4036
4037
4038 @Test
4039 public void testParallelIncrementWithMemStoreFlush() throws Exception {
4040 byte[] family = Incrementer.family;
4041 this.region = initHRegion(tableName, method, CONF, family);
4042 final HRegion region = this.region;
4043 final AtomicBoolean incrementDone = new AtomicBoolean(false);
4044 Runnable flusher = new Runnable() {
4045 @Override
4046 public void run() {
4047 while (!incrementDone.get()) {
4048 try {
4049 region.flushcache();
4050 } catch (Exception e) {
4051 e.printStackTrace();
4052 }
4053 }
4054 }
4055 };
4056
4057
4058 int threadNum = 20;
4059 int incCounter = 100;
4060 long expected = threadNum * incCounter;
4061 Thread[] incrementers = new Thread[threadNum];
4062 Thread flushThread = new Thread(flusher);
4063 for (int i = 0; i < threadNum; i++) {
4064 incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
4065 incrementers[i].start();
4066 }
4067 flushThread.start();
4068 for (int i = 0; i < threadNum; i++) {
4069 incrementers[i].join();
4070 }
4071
4072 incrementDone.set(true);
4073 flushThread.join();
4074
4075 Get get = new Get(Incrementer.incRow);
4076 get.addColumn(Incrementer.family, Incrementer.qualifier);
4077 get.setMaxVersions(1);
4078 Result res = this.region.get(get);
4079 List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
4080
4081
4082 assertEquals(kvs.size(), 1);
4083 Cell kv = kvs.get(0);
4084 assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
4085 this.region = null;
4086 }
4087
4088
4089
4090
4091 private static class Appender implements Runnable {
4092 private HRegion region;
4093 private final static byte[] appendRow = Bytes.toBytes("appendRow");
4094 private final static byte[] family = Bytes.toBytes("family");
4095 private final static byte[] qualifier = Bytes.toBytes("qualifier");
4096 private final static byte[] CHAR = Bytes.toBytes("a");
4097 private int appendCounter;
4098
4099 public Appender(HRegion region, int appendCounter) {
4100 this.region = region;
4101 this.appendCounter = appendCounter;
4102 }
4103
4104 @Override
4105 public void run() {
4106 int count = 0;
4107 while (count < appendCounter) {
4108 Append app = new Append(appendRow);
4109 app.add(family, qualifier, CHAR);
4110 count++;
4111 try {
4112 region.append(app);
4113 } catch (IOException e) {
4114 e.printStackTrace();
4115 break;
4116 }
4117 }
4118 }
4119 }
4120
4121
4122
4123
4124
4125 @Test
4126 public void testParallelAppendWithMemStoreFlush() throws Exception {
4127 byte[] family = Appender.family;
4128 this.region = initHRegion(tableName, method, CONF, family);
4129 final HRegion region = this.region;
4130 final AtomicBoolean appendDone = new AtomicBoolean(false);
4131 Runnable flusher = new Runnable() {
4132 @Override
4133 public void run() {
4134 while (!appendDone.get()) {
4135 try {
4136 region.flushcache();
4137 } catch (Exception e) {
4138 e.printStackTrace();
4139 }
4140 }
4141 }
4142 };
4143
4144
4145
4146 int threadNum = 20;
4147 int appendCounter = 100;
4148 byte[] expected = new byte[threadNum * appendCounter];
4149 for (int i = 0; i < threadNum * appendCounter; i++) {
4150 System.arraycopy(Appender.CHAR, 0, expected, i, 1);
4151 }
4152 Thread[] appenders = new Thread[threadNum];
4153 Thread flushThread = new Thread(flusher);
4154 for (int i = 0; i < threadNum; i++) {
4155 appenders[i] = new Thread(new Appender(this.region, appendCounter));
4156 appenders[i].start();
4157 }
4158 flushThread.start();
4159 for (int i = 0; i < threadNum; i++) {
4160 appenders[i].join();
4161 }
4162
4163 appendDone.set(true);
4164 flushThread.join();
4165
4166 Get get = new Get(Appender.appendRow);
4167 get.addColumn(Appender.family, Appender.qualifier);
4168 get.setMaxVersions(1);
4169 Result res = this.region.get(get);
4170 List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
4171
4172
4173 assertEquals(kvs.size(), 1);
4174 Cell kv = kvs.get(0);
4175 byte[] appendResult = new byte[kv.getValueLength()];
4176 System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
4177 assertArrayEquals(expected, appendResult);
4178 this.region = null;
4179 }
4180
4181
4182
4183
4184
4185 @Test
4186 public void testPutWithMemStoreFlush() throws Exception {
4187 byte[] family = Bytes.toBytes("family");
4188 ;
4189 byte[] qualifier = Bytes.toBytes("qualifier");
4190 byte[] row = Bytes.toBytes("putRow");
4191 byte[] value = null;
4192 this.region = initHRegion(tableName, method, CONF, family);
4193 Put put = null;
4194 Get get = null;
4195 List<Cell> kvs = null;
4196 Result res = null;
4197
4198 put = new Put(row);
4199 value = Bytes.toBytes("value0");
4200 put.add(family, qualifier, 1234567l, value);
4201 region.put(put);
4202 get = new Get(row);
4203 get.addColumn(family, qualifier);
4204 get.setMaxVersions();
4205 res = this.region.get(get);
4206 kvs = res.getColumnCells(family, qualifier);
4207 assertEquals(1, kvs.size());
4208 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4209
4210 region.flushcache();
4211 get = new Get(row);
4212 get.addColumn(family, qualifier);
4213 get.setMaxVersions();
4214 res = this.region.get(get);
4215 kvs = res.getColumnCells(family, qualifier);
4216 assertEquals(1, kvs.size());
4217 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4218
4219 put = new Put(row);
4220 value = Bytes.toBytes("value1");
4221 put.add(family, qualifier, 1234567l, value);
4222 region.put(put);
4223 get = new Get(row);
4224 get.addColumn(family, qualifier);
4225 get.setMaxVersions();
4226 res = this.region.get(get);
4227 kvs = res.getColumnCells(family, qualifier);
4228 assertEquals(1, kvs.size());
4229 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4230
4231 region.flushcache();
4232 get = new Get(row);
4233 get.addColumn(family, qualifier);
4234 get.setMaxVersions();
4235 res = this.region.get(get);
4236 kvs = res.getColumnCells(family, qualifier);
4237 assertEquals(1, kvs.size());
4238 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4239 }
4240
4241 @Test
4242 public void testDurability() throws Exception {
4243 String method = "testDurability";
4244
4245
4246
4247
4248
4249 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4250 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4251 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4252
4253 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4254 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4255 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4256
4257 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4258 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4259
4260 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
4261 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4262
4263 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
4264 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
4265 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
4266
4267
4268 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4269 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4270 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4271 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4272 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
4273 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4274
4275 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4276 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4277 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4278 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4279 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
4280 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
4281
4282
4283 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4284 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4285 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4286 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
4287 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
4288 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
4289
4290 }
4291
4292 private void durabilityTest(String method, Durability tableDurability,
4293 Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
4294 final boolean expectSyncFromLogSyncer) throws Exception {
4295 Configuration conf = HBaseConfiguration.create(CONF);
4296 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
4297 TableName tableName = TableName.valueOf(method);
4298 byte[] family = Bytes.toBytes("family");
4299 Path logDir = new Path(new Path(dir + method), "log");
4300 HLog hlog = HLogFactory.createHLog(FILESYSTEM, logDir, UUID.randomUUID().toString(), conf);
4301 final HLog log = spy(hlog);
4302 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
4303 HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, log,
4304 new byte[][] { family });
4305
4306 Put put = new Put(Bytes.toBytes("r1"));
4307 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
4308 put.setDurability(mutationDurability);
4309 region.put(put);
4310
4311
4312 verify(log, expectAppend ? times(1) : never())
4313 .appendNoSync((HRegionInfo)any(), eq(tableName), (WALEdit)any(), (List<UUID>)any(),
4314 anyLong(), (HTableDescriptor)any(), (AtomicLong)any(), anyBoolean(), anyLong(), anyLong());
4315
4316
4317 if (expectSync || expectSyncFromLogSyncer) {
4318 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
4319 @Override
4320 public boolean evaluate() throws Exception {
4321 try {
4322 if (expectSync) {
4323 verify(log, times(1)).sync(anyLong());
4324 } else if (expectSyncFromLogSyncer) {
4325 verify(log, times(1)).sync();
4326 }
4327 } catch (Throwable ignore) {
4328 }
4329 return true;
4330 }
4331 });
4332 } else {
4333 verify(log, never()).sync(anyLong());
4334 verify(log, never()).sync();
4335 }
4336
4337 HRegion.closeHRegion(this.region);
4338 this.region = null;
4339 }
4340
4341 private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4342 for (int i = startRow; i < startRow + numRows; i++) {
4343 Put put = new Put(Bytes.toBytes("" + i));
4344 put.setDurability(Durability.SKIP_WAL);
4345 for (byte[] family : families) {
4346 put.add(family, qf, null);
4347 }
4348 region.put(put);
4349 }
4350 }
4351
4352 private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
4353 throws IOException {
4354 for (int i = startRow; i < startRow + numRows; i++) {
4355 byte[] row = Bytes.toBytes("" + i);
4356 Get get = new Get(row);
4357 for (byte[] family : families) {
4358 get.addColumn(family, qf);
4359 }
4360 Result result = newReg.get(get);
4361 Cell[] raw = result.rawCells();
4362 assertEquals(families.length, result.size());
4363 for (int j = 0; j < families.length; j++) {
4364 assertTrue(CellUtil.matchingRow(raw[j], row));
4365 assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
4366 assertTrue(CellUtil.matchingQualifier(raw[j], qf));
4367 }
4368 }
4369 }
4370
4371 private void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
4372
4373 Get get = new Get(k).addFamily(family).setMaxVersions();
4374 Cell[] results = r.get(get).rawCells();
4375 for (int j = 0; j < results.length; j++) {
4376 byte[] tmp = CellUtil.cloneValue(results[j]);
4377
4378 assertTrue(Bytes.equals(k, tmp));
4379 }
4380 }
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393 private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
4394 throws IOException {
4395 byte[][] families = { fs };
4396 Scan scan = new Scan();
4397 for (int i = 0; i < families.length; i++)
4398 scan.addFamily(families[i]);
4399 InternalScanner s = r.getScanner(scan);
4400 try {
4401 List<Cell> curVals = new ArrayList<Cell>();
4402 boolean first = true;
4403 OUTER_LOOP: while (s.next(curVals)) {
4404 for (Cell kv : curVals) {
4405 byte[] val = CellUtil.cloneValue(kv);
4406 byte[] curval = val;
4407 if (first) {
4408 first = false;
4409 assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4410 } else {
4411
4412 break OUTER_LOOP;
4413 }
4414 }
4415 }
4416 } finally {
4417 s.close();
4418 }
4419 }
4420
4421
4422
4423
4424
4425 @Test
4426 public void testFlushResult() throws IOException {
4427 String method = name.getMethodName();
4428 byte[] tableName = Bytes.toBytes(method);
4429 byte[] family = Bytes.toBytes("family");
4430
4431 this.region = initHRegion(tableName, method, family);
4432
4433
4434 HRegion.FlushResult fr = region.flushcache();
4435 assertFalse(fr.isFlushSucceeded());
4436 assertFalse(fr.isCompactionNeeded());
4437
4438
4439 for (int i = 0; i < 2; i++) {
4440 Put put = new Put(tableName).add(family, family, tableName);
4441 region.put(put);
4442 fr = region.flushcache();
4443 assertTrue(fr.isFlushSucceeded());
4444 assertFalse(fr.isCompactionNeeded());
4445 }
4446
4447
4448 for (int i = 0; i < 2; i++) {
4449 Put put = new Put(tableName).add(family, family, tableName);
4450 region.put(put);
4451 fr = region.flushcache();
4452 assertTrue(fr.isFlushSucceeded());
4453 assertTrue(fr.isCompactionNeeded());
4454 }
4455 }
4456
4457 private Configuration initSplit() {
4458
4459 CONF.setInt("hbase.hstore.compactionThreshold", 2);
4460
4461
4462 CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4463
4464 CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4465
4466
4467 CONF.setLong("hbase.client.pause", 15 * 1000);
4468
4469
4470
4471 CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4472 return CONF;
4473 }
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484 public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
4485 byte[]... families) throws IOException {
4486 return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
4487 }
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4499 byte[]... families) throws IOException {
4500 return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4501 }
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4514 boolean isReadOnly, byte[]... families) throws IOException {
4515 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4516 }
4517
4518 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4519 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4520 throws IOException {
4521 return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
4522 Durability.SYNC_WAL, null, families);
4523 }
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4538 String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4539 HLog hlog, byte[]... families) throws IOException {
4540 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly, durability, hlog, families);
4541 }
4542
4543
4544
4545
4546
4547 private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
4548 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4549 assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
4550 Bytes.toString(CellUtil.cloneRow(kv)));
4551 assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
4552 Bytes.toString(CellUtil.cloneFamily(kv)));
4553 assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
4554 Bytes.toString(CellUtil.cloneQualifier(kv)));
4555 assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
4556 assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
4557 Bytes.toString(CellUtil.cloneValue(kv)));
4558 }
4559
4560 @Test (timeout=60000)
4561 public void testReverseScanner_FromMemStore_SingleCF_Normal()
4562 throws IOException {
4563 byte[] rowC = Bytes.toBytes("rowC");
4564 byte[] rowA = Bytes.toBytes("rowA");
4565 byte[] rowB = Bytes.toBytes("rowB");
4566 byte[] cf = Bytes.toBytes("CF");
4567 byte[][] families = { cf };
4568 byte[] col = Bytes.toBytes("C");
4569 long ts = 1;
4570 String method = this.getName();
4571 this.region = initHRegion(tableName, method, families);
4572 try {
4573 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4574 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4575 null);
4576 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4577 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4578 Put put = null;
4579 put = new Put(rowC);
4580 put.add(kv1);
4581 put.add(kv11);
4582 region.put(put);
4583 put = new Put(rowA);
4584 put.add(kv2);
4585 region.put(put);
4586 put = new Put(rowB);
4587 put.add(kv3);
4588 region.put(put);
4589
4590 Scan scan = new Scan(rowC);
4591 scan.setMaxVersions(5);
4592 scan.setReversed(true);
4593 InternalScanner scanner = region.getScanner(scan);
4594 List<Cell> currRow = new ArrayList<Cell>();
4595 boolean hasNext = scanner.next(currRow);
4596 assertEquals(2, currRow.size());
4597 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4598 assertTrue(hasNext);
4599 currRow.clear();
4600 hasNext = scanner.next(currRow);
4601 assertEquals(1, currRow.size());
4602 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4603 assertTrue(hasNext);
4604 currRow.clear();
4605 hasNext = scanner.next(currRow);
4606 assertEquals(1, currRow.size());
4607 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4608 assertFalse(hasNext);
4609 scanner.close();
4610 } finally {
4611 HRegion.closeHRegion(this.region);
4612 this.region = null;
4613 }
4614 }
4615
4616 @Test (timeout=60000)
4617 public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
4618 throws IOException {
4619 byte[] rowC = Bytes.toBytes("rowC");
4620 byte[] rowA = Bytes.toBytes("rowA");
4621 byte[] rowB = Bytes.toBytes("rowB");
4622 byte[] rowD = Bytes.toBytes("rowD");
4623 byte[] cf = Bytes.toBytes("CF");
4624 byte[][] families = { cf };
4625 byte[] col = Bytes.toBytes("C");
4626 long ts = 1;
4627 String method = this.getName();
4628 this.region = initHRegion(tableName, method, families);
4629 try {
4630 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4631 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4632 null);
4633 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4634 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4635 Put put = null;
4636 put = new Put(rowC);
4637 put.add(kv1);
4638 put.add(kv11);
4639 region.put(put);
4640 put = new Put(rowA);
4641 put.add(kv2);
4642 region.put(put);
4643 put = new Put(rowB);
4644 put.add(kv3);
4645 region.put(put);
4646
4647 Scan scan = new Scan(rowD);
4648 List<Cell> currRow = new ArrayList<Cell>();
4649 scan.setReversed(true);
4650 scan.setMaxVersions(5);
4651 InternalScanner scanner = region.getScanner(scan);
4652 boolean hasNext = scanner.next(currRow);
4653 assertEquals(2, currRow.size());
4654 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4655 assertTrue(hasNext);
4656 currRow.clear();
4657 hasNext = scanner.next(currRow);
4658 assertEquals(1, currRow.size());
4659 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4660 assertTrue(hasNext);
4661 currRow.clear();
4662 hasNext = scanner.next(currRow);
4663 assertEquals(1, currRow.size());
4664 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4665 assertFalse(hasNext);
4666 scanner.close();
4667 } finally {
4668 HRegion.closeHRegion(this.region);
4669 this.region = null;
4670 }
4671 }
4672
4673 @Test (timeout=60000)
4674 public void testReverseScanner_FromMemStore_SingleCF_FullScan()
4675 throws IOException {
4676 byte[] rowC = Bytes.toBytes("rowC");
4677 byte[] rowA = Bytes.toBytes("rowA");
4678 byte[] rowB = Bytes.toBytes("rowB");
4679 byte[] cf = Bytes.toBytes("CF");
4680 byte[][] families = { cf };
4681 byte[] col = Bytes.toBytes("C");
4682 long ts = 1;
4683 String method = this.getName();
4684 this.region = initHRegion(tableName, method, families);
4685 try {
4686 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4687 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4688 null);
4689 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4690 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4691 Put put = null;
4692 put = new Put(rowC);
4693 put.add(kv1);
4694 put.add(kv11);
4695 region.put(put);
4696 put = new Put(rowA);
4697 put.add(kv2);
4698 region.put(put);
4699 put = new Put(rowB);
4700 put.add(kv3);
4701 region.put(put);
4702 Scan scan = new Scan();
4703 List<Cell> currRow = new ArrayList<Cell>();
4704 scan.setReversed(true);
4705 InternalScanner scanner = region.getScanner(scan);
4706 boolean hasNext = scanner.next(currRow);
4707 assertEquals(1, currRow.size());
4708 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4709 assertTrue(hasNext);
4710 currRow.clear();
4711 hasNext = scanner.next(currRow);
4712 assertEquals(1, currRow.size());
4713 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4714 assertTrue(hasNext);
4715 currRow.clear();
4716 hasNext = scanner.next(currRow);
4717 assertEquals(1, currRow.size());
4718 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4719 assertFalse(hasNext);
4720 scanner.close();
4721 } finally {
4722 HRegion.closeHRegion(this.region);
4723 this.region = null;
4724 }
4725 }
4726
4727 @Test (timeout=60000)
4728 public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
4729
4730 byte[] rowA = Bytes.toBytes("rowA");
4731 byte[] rowB = Bytes.toBytes("rowB");
4732 byte[] rowC = Bytes.toBytes("rowC");
4733 byte[] rowD = Bytes.toBytes("rowD");
4734 byte[] rowE = Bytes.toBytes("rowE");
4735 byte[] cf = Bytes.toBytes("CF");
4736 byte[][] families = { cf };
4737 byte[] col1 = Bytes.toBytes("col1");
4738 byte[] col2 = Bytes.toBytes("col2");
4739 long ts = 1;
4740 String method = this.getName();
4741 this.region = initHRegion(tableName, method, families);
4742 try {
4743 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4744 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4745 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4746 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4747 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4748 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4749 Put put = null;
4750 put = new Put(rowA);
4751 put.add(kv1);
4752 region.put(put);
4753 put = new Put(rowB);
4754 put.add(kv2);
4755 region.put(put);
4756 put = new Put(rowC);
4757 put.add(kv3);
4758 region.put(put);
4759 put = new Put(rowD);
4760 put.add(kv4_1);
4761 region.put(put);
4762 put = new Put(rowD);
4763 put.add(kv4_2);
4764 region.put(put);
4765 put = new Put(rowE);
4766 put.add(kv5);
4767 region.put(put);
4768 region.flushcache();
4769 Scan scan = new Scan(rowD, rowA);
4770 scan.addColumn(families[0], col1);
4771 scan.setReversed(true);
4772 List<Cell> currRow = new ArrayList<Cell>();
4773 InternalScanner scanner = region.getScanner(scan);
4774 boolean hasNext = scanner.next(currRow);
4775 assertEquals(1, currRow.size());
4776 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4777 assertTrue(hasNext);
4778 currRow.clear();
4779 hasNext = scanner.next(currRow);
4780 assertEquals(1, currRow.size());
4781 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4782 assertTrue(hasNext);
4783 currRow.clear();
4784 hasNext = scanner.next(currRow);
4785 assertEquals(1, currRow.size());
4786 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4787 assertFalse(hasNext);
4788 scanner.close();
4789
4790 scan = new Scan(rowD, rowA);
4791 scan.addColumn(families[0], col2);
4792 scan.setReversed(true);
4793 currRow.clear();
4794 scanner = region.getScanner(scan);
4795 hasNext = scanner.next(currRow);
4796 assertEquals(1, currRow.size());
4797 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4798 scanner.close();
4799 } finally {
4800 HRegion.closeHRegion(this.region);
4801 this.region = null;
4802 }
4803 }
4804
4805 @Test (timeout=60000)
4806 public void testReverseScanner_smaller_blocksize() throws IOException {
4807
4808 byte[] rowA = Bytes.toBytes("rowA");
4809 byte[] rowB = Bytes.toBytes("rowB");
4810 byte[] rowC = Bytes.toBytes("rowC");
4811 byte[] rowD = Bytes.toBytes("rowD");
4812 byte[] rowE = Bytes.toBytes("rowE");
4813 byte[] cf = Bytes.toBytes("CF");
4814 byte[][] families = { cf };
4815 byte[] col1 = Bytes.toBytes("col1");
4816 byte[] col2 = Bytes.toBytes("col2");
4817 long ts = 1;
4818 String method = this.getName();
4819 HBaseConfiguration config = new HBaseConfiguration();
4820 config.setInt("test.block.size", 1);
4821 this.region = initHRegion(tableName, method, config, families);
4822 try {
4823 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4824 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4825 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4826 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4827 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4828 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4829 Put put = null;
4830 put = new Put(rowA);
4831 put.add(kv1);
4832 region.put(put);
4833 put = new Put(rowB);
4834 put.add(kv2);
4835 region.put(put);
4836 put = new Put(rowC);
4837 put.add(kv3);
4838 region.put(put);
4839 put = new Put(rowD);
4840 put.add(kv4_1);
4841 region.put(put);
4842 put = new Put(rowD);
4843 put.add(kv4_2);
4844 region.put(put);
4845 put = new Put(rowE);
4846 put.add(kv5);
4847 region.put(put);
4848 region.flushcache();
4849 Scan scan = new Scan(rowD, rowA);
4850 scan.addColumn(families[0], col1);
4851 scan.setReversed(true);
4852 List<Cell> currRow = new ArrayList<Cell>();
4853 InternalScanner scanner = region.getScanner(scan);
4854 boolean hasNext = scanner.next(currRow);
4855 assertEquals(1, currRow.size());
4856 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4857 assertTrue(hasNext);
4858 currRow.clear();
4859 hasNext = scanner.next(currRow);
4860 assertEquals(1, currRow.size());
4861 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4862 assertTrue(hasNext);
4863 currRow.clear();
4864 hasNext = scanner.next(currRow);
4865 assertEquals(1, currRow.size());
4866 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4867 assertFalse(hasNext);
4868 scanner.close();
4869
4870 scan = new Scan(rowD, rowA);
4871 scan.addColumn(families[0], col2);
4872 scan.setReversed(true);
4873 currRow.clear();
4874 scanner = region.getScanner(scan);
4875 hasNext = scanner.next(currRow);
4876 assertEquals(1, currRow.size());
4877 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4878 scanner.close();
4879 } finally {
4880 HRegion.closeHRegion(this.region);
4881 this.region = null;
4882 }
4883 }
4884
4885 @Test (timeout=60000)
4886 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
4887 throws IOException {
4888 byte[] row0 = Bytes.toBytes("row0");
4889 byte[] row1 = Bytes.toBytes("row1");
4890 byte[] row2 = Bytes.toBytes("row2");
4891 byte[] row3 = Bytes.toBytes("row3");
4892 byte[] row4 = Bytes.toBytes("row4");
4893 byte[] row5 = Bytes.toBytes("row5");
4894 byte[] cf1 = Bytes.toBytes("CF1");
4895 byte[] cf2 = Bytes.toBytes("CF2");
4896 byte[] cf3 = Bytes.toBytes("CF3");
4897 byte[][] families = { cf1, cf2, cf3 };
4898 byte[] col = Bytes.toBytes("C");
4899 long ts = 1;
4900 String method = this.getName();
4901 HBaseConfiguration conf = new HBaseConfiguration();
4902
4903 conf.setInt("hbase.hstore.compactionThreshold", 10000);
4904 this.region = initHRegion(tableName, method, conf, families);
4905 try {
4906
4907 KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
4908 null);
4909 KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
4910 null);
4911 KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
4912 KeyValue.Type.Put, null);
4913 KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
4914 null);
4915 KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
4916 null);
4917 KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
4918 null);
4919 KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
4920 KeyValue.Type.Put, null);
4921 KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
4922 null);
4923 KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
4924 KeyValue.Type.Put, null);
4925 KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
4926 null);
4927 KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
4928 null);
4929 KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
4930 KeyValue.Type.Put, null);
4931 KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
4932 null);
4933 KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
4934 KeyValue.Type.Put, null);
4935 KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
4936 null);
4937 KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
4938 null);
4939
4940 Put put = null;
4941 put = new Put(row1);
4942 put.add(kv1_2_1);
4943 region.put(put);
4944 put = new Put(row2);
4945 put.add(kv2_4_1);
4946 region.put(put);
4947 put = new Put(row4);
4948 put.add(kv4_5_4);
4949 put.add(kv4_5_5);
4950 region.put(put);
4951 region.flushcache();
4952
4953 put = new Put(row4);
4954 put.add(kv4_5_1);
4955 put.add(kv4_5_3);
4956 region.put(put);
4957 put = new Put(row1);
4958 put.add(kv1_2_2);
4959 region.put(put);
4960 put = new Put(row2);
4961 put.add(kv2_4_4);
4962 region.put(put);
4963 region.flushcache();
4964
4965 put = new Put(row4);
4966 put.add(kv4_5_2);
4967 region.put(put);
4968 put = new Put(row2);
4969 put.add(kv2_4_2);
4970 put.add(kv2_4_3);
4971 region.put(put);
4972 put = new Put(row3);
4973 put.add(kv3_2_2);
4974 region.put(put);
4975 region.flushcache();
4976
4977
4978 put = new Put(row0);
4979 put.add(kv0_1_1);
4980 region.put(put);
4981 put = new Put(row3);
4982 put.add(kv3_2_1);
4983 region.put(put);
4984 put = new Put(row5);
4985 put.add(kv5_2_1);
4986 put.add(kv5_2_2);
4987 region.put(put);
4988
4989 Scan scan = new Scan(row4);
4990 scan.setMaxVersions(5);
4991 scan.setBatch(3);
4992 scan.setReversed(true);
4993 InternalScanner scanner = region.getScanner(scan);
4994 List<Cell> currRow = new ArrayList<Cell>();
4995 boolean hasNext = false;
4996
4997
4998
4999 hasNext = scanner.next(currRow);
5000 assertEquals(3, currRow.size());
5001 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5002 assertTrue(hasNext);
5003 currRow.clear();
5004 hasNext = scanner.next(currRow);
5005 assertEquals(2, currRow.size());
5006 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5007 assertTrue(hasNext);
5008
5009 currRow.clear();
5010 hasNext = scanner.next(currRow);
5011 assertEquals(2, currRow.size());
5012 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5013 assertTrue(hasNext);
5014
5015
5016 currRow.clear();
5017 hasNext = scanner.next(currRow);
5018 assertEquals(3, currRow.size());
5019 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5020 assertTrue(hasNext);
5021 currRow.clear();
5022 hasNext = scanner.next(currRow);
5023 assertEquals(1, currRow.size());
5024 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5025 assertTrue(hasNext);
5026
5027 currRow.clear();
5028 hasNext = scanner.next(currRow);
5029 assertEquals(2, currRow.size());
5030 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5031 assertTrue(hasNext);
5032
5033 currRow.clear();
5034 hasNext = scanner.next(currRow);
5035 assertEquals(1, currRow.size());
5036 assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
5037 assertFalse(hasNext);
5038
5039 scanner.close();
5040 } finally {
5041 HRegion.closeHRegion(this.region);
5042 this.region = null;
5043 }
5044 }
5045
5046 @Test (timeout=60000)
5047 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
5048 throws IOException {
5049 byte[] row1 = Bytes.toBytes("row1");
5050 byte[] row2 = Bytes.toBytes("row2");
5051 byte[] row3 = Bytes.toBytes("row3");
5052 byte[] row4 = Bytes.toBytes("row4");
5053 byte[] cf1 = Bytes.toBytes("CF1");
5054 byte[] cf2 = Bytes.toBytes("CF2");
5055 byte[] cf3 = Bytes.toBytes("CF3");
5056 byte[] cf4 = Bytes.toBytes("CF4");
5057 byte[][] families = { cf1, cf2, cf3, cf4 };
5058 byte[] col = Bytes.toBytes("C");
5059 long ts = 1;
5060 String method = this.getName();
5061 HBaseConfiguration conf = new HBaseConfiguration();
5062
5063 conf.setInt("hbase.hstore.compactionThreshold", 10000);
5064 this.region = initHRegion(tableName, method, conf, families);
5065 try {
5066 KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
5067 KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
5068 KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
5069 KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
5070
5071 Put put = new Put(row1);
5072 put.add(kv1);
5073 region.put(put);
5074 region.flushcache();
5075
5076 put = new Put(row2);
5077 put.add(kv2);
5078 region.put(put);
5079 region.flushcache();
5080
5081 put = new Put(row3);
5082 put.add(kv3);
5083 region.put(put);
5084 region.flushcache();
5085
5086 put = new Put(row4);
5087 put.add(kv4);
5088 region.put(put);
5089
5090 Scan scan = new Scan(row4);
5091 scan.setReversed(true);
5092 scan.setBatch(10);
5093 InternalScanner scanner = region.getScanner(scan);
5094 List<Cell> currRow = new ArrayList<Cell>();
5095 boolean hasNext = scanner.next(currRow);
5096 assertEquals(1, currRow.size());
5097 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5098 assertTrue(hasNext);
5099 currRow.clear();
5100 hasNext = scanner.next(currRow);
5101 assertEquals(1, currRow.size());
5102 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5103 assertTrue(hasNext);
5104 currRow.clear();
5105 hasNext = scanner.next(currRow);
5106 assertEquals(1, currRow.size());
5107 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5108 assertTrue(hasNext);
5109 currRow.clear();
5110 hasNext = scanner.next(currRow);
5111 assertEquals(1, currRow.size());
5112 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5113 assertFalse(hasNext);
5114 } finally {
5115 HRegion.closeHRegion(this.region);
5116 this.region = null;
5117 }
5118 }
5119
5120
5121
5122
5123 @Test (timeout = 60000)
5124 public void testReverseScanner_StackOverflow() throws IOException {
5125 byte[] cf1 = Bytes.toBytes("CF1");
5126 byte[][] families = {cf1};
5127 byte[] col = Bytes.toBytes("C");
5128 String method = this.getName();
5129 HBaseConfiguration conf = new HBaseConfiguration();
5130
5131 conf.setInt("hbase.hstore.compactionThreshold", 10000);
5132 this.region = initHRegion(tableName, method, conf, families);
5133
5134 try {
5135
5136 Put put = new Put(Bytes.toBytes("19998"));
5137 put.add(cf1, col, Bytes.toBytes("val"));
5138 region.put(put);
5139 region.flushcache();
5140 Put put2 = new Put(Bytes.toBytes("19997"));
5141 put2.add(cf1, col, Bytes.toBytes("val"));
5142 region.put(put2);
5143
5144 Scan scan = new Scan(Bytes.toBytes("19998"));
5145 scan.setReversed(true);
5146 InternalScanner scanner = region.getScanner(scan);
5147
5148
5149
5150 for (int i = 10000; i < 20000; i++) {
5151 Put p = new Put(Bytes.toBytes("" + i));
5152 p.add(cf1, col, Bytes.toBytes("" + i));
5153 region.put(p);
5154 }
5155 region.flushcache();
5156
5157
5158
5159 for (int i = 10000; i < 20000; i++) {
5160 Put p = new Put(Bytes.toBytes("" + i));
5161 p.add(cf1, col, Bytes.toBytes("" + i));
5162 region.put(p);
5163 }
5164
5165 List<Cell> currRow = new ArrayList<Cell>();
5166 boolean hasNext;
5167 do {
5168 hasNext = scanner.next(currRow);
5169 } while (hasNext);
5170 assertEquals(2, currRow.size());
5171 assertArrayEquals(Bytes.toBytes("19998"), currRow.get(0).getRow());
5172 assertArrayEquals(Bytes.toBytes("19997"), currRow.get(1).getRow());
5173 } finally {
5174 HRegion.closeHRegion(this.region);
5175 this.region = null;
5176 }
5177 }
5178
5179 @Test (timeout=60000)
5180 public void testSplitRegionWithReverseScan() throws IOException {
5181 byte [] tableName = Bytes.toBytes("testSplitRegionWithReverseScan");
5182 byte [] qualifier = Bytes.toBytes("qualifier");
5183 Configuration hc = initSplit();
5184 int numRows = 3;
5185 byte [][] families = {fam1};
5186
5187
5188 String method = this.getName();
5189 this.region = initHRegion(tableName, method, hc, families);
5190
5191
5192 int startRow = 100;
5193 putData(startRow, numRows, qualifier, families);
5194 int splitRow = startRow + numRows;
5195 putData(splitRow, numRows, qualifier, families);
5196 int endRow = splitRow + numRows;
5197 region.flushcache();
5198
5199 HRegion [] regions = null;
5200 try {
5201 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
5202
5203 for (int i = 0; i < regions.length; i++) {
5204 regions[i] = HRegion.openHRegion(regions[i], null);
5205 }
5206
5207 assertEquals(2, regions.length);
5208
5209
5210
5211 verifyData(regions[0], startRow, numRows, qualifier, families);
5212 verifyData(regions[1], splitRow, numRows, qualifier, families);
5213
5214
5215 Scan scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 10 * numRows)));
5216 scan.setReversed(true);
5217 InternalScanner scanner = regions[1].getScanner(scan);
5218 List<Cell> currRow = new ArrayList<Cell>();
5219 boolean more = false;
5220 int verify = startRow + 2 * numRows - 1;
5221 do {
5222 more = scanner.next(currRow);
5223 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5224 verify--;
5225 currRow.clear();
5226 } while(more);
5227 assertEquals(verify, startRow + numRows - 1);
5228 scanner.close();
5229
5230 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 2 * numRows - 1)));
5231 scan.setReversed(true);
5232 scanner = regions[1].getScanner(scan);
5233 verify = startRow + 2 * numRows - 1;
5234 do {
5235 more = scanner.next(currRow);
5236 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5237 verify--;
5238 currRow.clear();
5239 } while(more);
5240 assertEquals(verify, startRow + numRows - 1);
5241 scanner.close();
5242
5243 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows)));
5244 scan.setReversed(true);
5245 scanner = regions[0].getScanner(scan);
5246 verify = startRow + numRows - 1;
5247 do {
5248 more = scanner.next(currRow);
5249 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5250 verify--;
5251 currRow.clear();
5252 } while(more);
5253 assertEquals(verify, 99);
5254 scanner.close();
5255
5256 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows - 1)));
5257 scan.setReversed(true);
5258 scanner = regions[0].getScanner(scan);
5259 verify = startRow + numRows - 1;
5260 do {
5261 more = scanner.next(currRow);
5262 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5263 verify--;
5264 currRow.clear();
5265 } while(more);
5266 assertEquals(verify, startRow - 1);
5267 scanner.close();
5268 } finally {
5269 HRegion.closeHRegion(this.region);
5270 this.region = null;
5271 }
5272 }
5273
5274 @Test
5275 public void testWriteRequestsCounter() throws IOException {
5276 byte[] fam = Bytes.toBytes("info");
5277 byte[][] families = { fam };
5278 this.region = initHRegion(tableName, method, CONF, families);
5279
5280 Assert.assertEquals(0L, region.getWriteRequestsCount());
5281
5282 Put put = new Put(row);
5283 put.add(fam, fam, fam);
5284
5285 Assert.assertEquals(0L, region.getWriteRequestsCount());
5286 region.put(put);
5287 Assert.assertEquals(1L, region.getWriteRequestsCount());
5288 region.put(put);
5289 Assert.assertEquals(2L, region.getWriteRequestsCount());
5290 region.put(put);
5291 Assert.assertEquals(3L, region.getWriteRequestsCount());
5292
5293 region.delete(new Delete(row));
5294 Assert.assertEquals(4L, region.getWriteRequestsCount());
5295
5296 HRegion.closeHRegion(this.region);
5297 this.region = null;
5298 }
5299
5300
5301
5302
5303 @Test (timeout=24000)
5304 public void testRegionTooBusy() throws IOException {
5305 String method = "testRegionTooBusy";
5306 byte[] tableName = Bytes.toBytes(method);
5307 byte[] family = Bytes.toBytes("family");
5308 long defaultBusyWaitDuration = CONF.getLong("hbase.busy.wait.duration",
5309 HRegion.DEFAULT_BUSY_WAIT_DURATION);
5310 CONF.setLong("hbase.busy.wait.duration", 1000);
5311 region = initHRegion(tableName, method, CONF, family);
5312 final AtomicBoolean stopped = new AtomicBoolean(true);
5313 Thread t = new Thread(new Runnable() {
5314 @Override
5315 public void run() {
5316 try {
5317 region.lock.writeLock().lock();
5318 stopped.set(false);
5319 while (!stopped.get()) {
5320 Thread.sleep(100);
5321 }
5322 } catch (InterruptedException ie) {
5323 } finally {
5324 region.lock.writeLock().unlock();
5325 }
5326 }
5327 });
5328 t.start();
5329 Get get = new Get(row);
5330 try {
5331 while (stopped.get()) {
5332 Thread.sleep(100);
5333 }
5334 region.get(get);
5335 fail("Should throw RegionTooBusyException");
5336 } catch (InterruptedException ie) {
5337 fail("test interrupted");
5338 } catch (RegionTooBusyException e) {
5339
5340 } finally {
5341 stopped.set(true);
5342 try {
5343 t.join();
5344 } catch (Throwable e) {
5345 }
5346
5347 HRegion.closeHRegion(region);
5348 region = null;
5349 CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration);
5350 }
5351 }
5352
5353 @Test
5354 public void testCellTTLs() throws IOException {
5355 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
5356 EnvironmentEdgeManager.injectEdge(edge);
5357
5358 final byte[] row = Bytes.toBytes("testRow");
5359 final byte[] q1 = Bytes.toBytes("q1");
5360 final byte[] q2 = Bytes.toBytes("q2");
5361 final byte[] q3 = Bytes.toBytes("q3");
5362 final byte[] q4 = Bytes.toBytes("q4");
5363
5364 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCellTTLs"));
5365 HColumnDescriptor hcd = new HColumnDescriptor(fam1);
5366 hcd.setTimeToLive(10);
5367 htd.addFamily(hcd);
5368
5369 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
5370 conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS);
5371
5372 HRegion region = HRegion.createHRegion(new HRegionInfo(htd.getTableName(),
5373 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY),
5374 TEST_UTIL.getDataTestDir(), conf, htd);
5375 assertNotNull(region);
5376 try {
5377 long now = EnvironmentEdgeManager.currentTimeMillis();
5378
5379 region.put(new Put(row).add(new KeyValue(row, fam1, q1, now,
5380 HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
5381
5382 new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
5383
5384 region.put(new Put(row).add(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY));
5385
5386 region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1,
5387 HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
5388
5389 new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
5390
5391 region.put(new Put(row).add(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY));
5392
5393
5394 region.flushcache();
5395
5396
5397 Result r = region.get(new Get(row));
5398 assertNotNull(r.getValue(fam1, q1));
5399 assertNotNull(r.getValue(fam1, q2));
5400 assertNotNull(r.getValue(fam1, q3));
5401 assertNotNull(r.getValue(fam1, q4));
5402
5403
5404 edge.incrementTime(5000);
5405
5406 r = region.get(new Get(row));
5407 assertNull(r.getValue(fam1, q1));
5408 assertNotNull(r.getValue(fam1, q2));
5409 assertNotNull(r.getValue(fam1, q3));
5410 assertNotNull(r.getValue(fam1, q4));
5411
5412
5413 edge.incrementTime(5000);
5414
5415 r = region.get(new Get(row));
5416 assertNull(r.getValue(fam1, q1));
5417 assertNull(r.getValue(fam1, q2));
5418 assertNotNull(r.getValue(fam1, q3));
5419 assertNotNull(r.getValue(fam1, q4));
5420
5421
5422 edge.incrementTime(5000);
5423
5424 r = region.get(new Get(row));
5425 assertNull(r.getValue(fam1, q1));
5426 assertNull(r.getValue(fam1, q2));
5427 assertNull(r.getValue(fam1, q3));
5428 assertNotNull(r.getValue(fam1, q4));
5429
5430
5431 edge.incrementTime(10000);
5432
5433 r = region.get(new Get(row));
5434 assertNull(r.getValue(fam1, q1));
5435 assertNull(r.getValue(fam1, q2));
5436 assertNull(r.getValue(fam1, q3));
5437 assertNull(r.getValue(fam1, q4));
5438
5439
5440
5441
5442 region.put(new Put(row).add(fam1, q1, Bytes.toBytes(1L)));
5443 r = region.get(new Get(row));
5444 byte[] val = r.getValue(fam1, q1);
5445 assertNotNull(val);
5446 assertEquals(Bytes.toLong(val), 1L);
5447
5448
5449 Increment incr = new Increment(row).addColumn(fam1, q1, 1L);
5450 incr.setTTL(5000);
5451 region.increment(incr);
5452
5453
5454 r = region.get(new Get(row));
5455 val = r.getValue(fam1, q1);
5456 assertNotNull(val);
5457 assertEquals(Bytes.toLong(val), 2L);
5458
5459
5460 edge.incrementTime(5000);
5461
5462
5463 r = region.get(new Get(row));
5464 val = r.getValue(fam1, q1);
5465 assertNotNull(val);
5466 assertEquals(Bytes.toLong(val), 1L);
5467
5468
5469 edge.incrementTime(5000);
5470
5471
5472 r = region.get(new Get(row));
5473 assertNull(r.getValue(fam1, q1));
5474
5475 } finally {
5476 HRegion.closeHRegion(region);
5477 }
5478 }
5479
5480 @Test
5481 public void testIncrementTimestampsAreMonotonic() throws IOException {
5482 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5483 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5484 EnvironmentEdgeManager.injectEdge(edge);
5485
5486 edge.setValue(10);
5487 Increment inc = new Increment(row);
5488 inc.setDurability(Durability.SKIP_WAL);
5489 inc.addColumn(fam1, qual1, 1L);
5490 region.increment(inc);
5491
5492 Result result = region.get(new Get(row));
5493 Cell c = result.getColumnLatestCell(fam1, qual1);
5494 assertNotNull(c);
5495 assertEquals(c.getTimestamp(), 10L);
5496
5497 edge.setValue(1);
5498 region.increment(inc);
5499 result = region.get(new Get(row));
5500 c = result.getColumnLatestCell(fam1, qual1);
5501 assertEquals(c.getTimestamp(), 10L);
5502 assertEquals(Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength()), 2L);
5503 }
5504
5505 @Test
5506 public void testAppendTimestampsAreMonotonic() throws IOException {
5507 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5508 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5509 EnvironmentEdgeManager.injectEdge(edge);
5510
5511 edge.setValue(10);
5512 Append a = new Append(row);
5513 a.setDurability(Durability.SKIP_WAL);
5514 a.add(fam1, qual1, qual1);
5515 region.append(a);
5516
5517 Result result = region.get(new Get(row));
5518 Cell c = result.getColumnLatestCell(fam1, qual1);
5519 assertNotNull(c);
5520 assertEquals(c.getTimestamp(), 10L);
5521
5522 edge.setValue(1);
5523 region.append(a);
5524 result = region.get(new Get(row));
5525 c = result.getColumnLatestCell(fam1, qual1);
5526 assertEquals(c.getTimestamp(), 10L);
5527
5528 byte[] expected = new byte[qual1.length*2];
5529 System.arraycopy(qual1, 0, expected, 0, qual1.length);
5530 System.arraycopy(qual1, 0, expected, qual1.length, qual1.length);
5531
5532 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
5533 expected, 0, expected.length));
5534 }
5535
5536 @Test
5537 public void testCheckAndMutateTimestampsAreMonotonic() throws IOException {
5538 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5539 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5540 EnvironmentEdgeManager.injectEdge(edge);
5541
5542 edge.setValue(10);
5543 Put p = new Put(row);
5544 p.setDurability(Durability.SKIP_WAL);
5545 p.add(fam1, qual1, qual1);
5546 region.put(p);
5547
5548 Result result = region.get(new Get(row));
5549 Cell c = result.getColumnLatestCell(fam1, qual1);
5550 assertNotNull(c);
5551 assertEquals(c.getTimestamp(), 10L);
5552
5553 edge.setValue(1);
5554 p = new Put(row);
5555 p.setDurability(Durability.SKIP_WAL);
5556 p.add(fam1, qual1, qual2);
5557 region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(qual1), p, false);
5558 result = region.get(new Get(row));
5559 c = result.getColumnLatestCell(fam1, qual1);
5560 assertEquals(c.getTimestamp(), 10L);
5561
5562 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
5563 qual2, 0, qual2.length));
5564 }
5565
5566 @Test
5567 public void testCheckAndRowMutateTimestampsAreMonotonic() throws IOException {
5568 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5569 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5570 EnvironmentEdgeManager.injectEdge(edge);
5571
5572 edge.setValue(10);
5573 Put p = new Put(row);
5574 p.setDurability(Durability.SKIP_WAL);
5575 p.add(fam1, qual1, qual1);
5576 region.put(p);
5577
5578 Result result = region.get(new Get(row));
5579 Cell c = result.getColumnLatestCell(fam1, qual1);
5580 assertNotNull(c);
5581 assertEquals(c.getTimestamp(), 10L);
5582
5583 edge.setValue(1);
5584 p = new Put(row);
5585 p.setDurability(Durability.SKIP_WAL);
5586 p.add(fam1, qual1, qual2);
5587 RowMutations rm = new RowMutations(row);
5588 rm.add(p);
5589 region.checkAndRowMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(qual1),
5590 rm, false);
5591 result = region.get(new Get(row));
5592 c = result.getColumnLatestCell(fam1, qual1);
5593 assertEquals(c.getTimestamp(), 10L);
5594
5595 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
5596 qual2, 0, qual2.length));
5597 }
5598
5599 private static HRegion initHRegion(byte[] tableName, String callingMethod,
5600 byte[]... families) throws IOException {
5601 return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
5602 families);
5603 }
5604 }