1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21
22 import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23 import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24 import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25 import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29 import static org.junit.Assert.assertArrayEquals;
30 import static org.junit.Assert.assertEquals;
31 import static org.junit.Assert.assertFalse;
32 import static org.junit.Assert.assertNotNull;
33 import static org.junit.Assert.assertNull;
34 import static org.junit.Assert.assertTrue;
35 import static org.junit.Assert.fail;
36 import static org.mockito.Matchers.any;
37 import static org.mockito.Matchers.anyBoolean;
38 import static org.mockito.Matchers.anyLong;
39 import static org.mockito.Matchers.eq;
40 import static org.mockito.Mockito.never;
41 import static org.mockito.Mockito.spy;
42 import static org.mockito.Mockito.times;
43 import static org.mockito.Mockito.verify;
44 import static org.mockito.Mockito.when;
45
46 import java.io.IOException;
47 import java.io.InterruptedIOException;
48 import java.security.PrivilegedExceptionAction;
49 import java.util.ArrayList;
50 import java.util.Arrays;
51 import java.util.Collection;
52 import java.util.List;
53 import java.util.Map;
54 import java.util.NavigableMap;
55 import java.util.TreeMap;
56 import java.util.UUID;
57 import java.util.concurrent.Callable;
58 import java.util.concurrent.CountDownLatch;
59 import java.util.concurrent.ExecutorService;
60 import java.util.concurrent.Executors;
61 import java.util.concurrent.Future;
62 import java.util.concurrent.TimeUnit;
63 import java.util.concurrent.atomic.AtomicBoolean;
64 import java.util.concurrent.atomic.AtomicInteger;
65 import java.util.concurrent.atomic.AtomicLong;
66 import java.util.concurrent.atomic.AtomicReference;
67
68 import org.apache.commons.logging.Log;
69 import org.apache.commons.logging.LogFactory;
70 import org.apache.hadoop.conf.Configuration;
71 import org.apache.hadoop.fs.FSDataOutputStream;
72 import org.apache.hadoop.fs.FileStatus;
73 import org.apache.hadoop.fs.FileSystem;
74 import org.apache.hadoop.fs.Path;
75 import org.apache.hadoop.hbase.Cell;
76 import org.apache.hadoop.hbase.CellComparator;
77 import org.apache.hadoop.hbase.CellUtil;
78 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
79 import org.apache.hadoop.hbase.DroppedSnapshotException;
80 import org.apache.hadoop.hbase.HBaseConfiguration;
81 import org.apache.hadoop.hbase.HBaseTestCase;
82 import org.apache.hadoop.hbase.HBaseTestingUtility;
83 import org.apache.hadoop.hbase.HColumnDescriptor;
84 import org.apache.hadoop.hbase.HConstants;
85 import org.apache.hadoop.hbase.RegionTooBusyException;
86 import org.apache.hadoop.hbase.Tag;
87 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
88 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
89 import org.apache.hadoop.hbase.HRegionInfo;
90 import org.apache.hadoop.hbase.HTableDescriptor;
91 import org.apache.hadoop.hbase.KeyValue;
92 import org.apache.hadoop.hbase.testclassification.MediumTests;
93 import org.apache.hadoop.hbase.MiniHBaseCluster;
94 import org.apache.hadoop.hbase.MultithreadedTestUtil;
95 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
96 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
97 import org.apache.hadoop.hbase.NotServingRegionException;
98 import org.apache.hadoop.hbase.TableName;
99 import org.apache.hadoop.hbase.TagType;
100 import org.apache.hadoop.hbase.Waiter;
101 import org.apache.hadoop.hbase.client.Append;
102 import org.apache.hadoop.hbase.client.Delete;
103 import org.apache.hadoop.hbase.client.Durability;
104 import org.apache.hadoop.hbase.client.Get;
105 import org.apache.hadoop.hbase.client.HTable;
106 import org.apache.hadoop.hbase.client.Increment;
107 import org.apache.hadoop.hbase.client.Mutation;
108 import org.apache.hadoop.hbase.client.Put;
109 import org.apache.hadoop.hbase.client.Result;
110 import org.apache.hadoop.hbase.client.RowMutations;
111 import org.apache.hadoop.hbase.client.Scan;
112 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
113 import org.apache.hadoop.hbase.filter.BinaryComparator;
114 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
115 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
116 import org.apache.hadoop.hbase.filter.Filter;
117 import org.apache.hadoop.hbase.filter.FilterBase;
118 import org.apache.hadoop.hbase.filter.FilterList;
119 import org.apache.hadoop.hbase.filter.NullComparator;
120 import org.apache.hadoop.hbase.filter.PrefixFilter;
121 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
122 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
123 import org.apache.hadoop.hbase.io.hfile.HFile;
124 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
125 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
126 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
127 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
128 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
129 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
130 import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
131 import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
132 import org.apache.hadoop.hbase.regionserver.wal.FaultyHLog;
133 import org.apache.hadoop.hbase.regionserver.wal.HLog;
134 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
135 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
136 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
137 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
138 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
139 import org.apache.hadoop.hbase.security.User;
140 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
141 import org.apache.hadoop.hbase.util.Bytes;
142 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
143 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
144 import org.apache.hadoop.hbase.util.FSUtils;
145 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
146 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
147 import org.apache.hadoop.hbase.util.PairOfSameType;
148 import org.apache.hadoop.hbase.util.Threads;
149 import org.junit.After;
150 import org.junit.Assert;
151 import org.junit.Before;
152 import org.junit.Rule;
153 import org.junit.Test;
154 import org.junit.experimental.categories.Category;
155 import org.junit.rules.TestName;
156 import org.mockito.Mockito;
157
158 import com.google.common.collect.ImmutableList;
159 import com.google.common.collect.Lists;
160 import com.google.protobuf.ByteString;
161
162
163
164
165
166
167
168 @Category(MediumTests.class)
169 @SuppressWarnings("deprecation")
170 public class TestHRegion {
171
172
173 static final Log LOG = LogFactory.getLog(TestHRegion.class);
174 @Rule public TestName name = new TestName();
175
176 private static final String COLUMN_FAMILY = "MyCF";
177 private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
178
179 HRegion region = null;
180
181 private static HBaseTestingUtility TEST_UTIL;
182 public static Configuration CONF ;
183 private String dir;
184 private static FileSystem FILESYSTEM;
185 private final int MAX_VERSIONS = 2;
186
187
188 protected byte[] tableName;
189 protected String method;
190 protected final byte[] qual1 = Bytes.toBytes("qual1");
191 protected final byte[] qual2 = Bytes.toBytes("qual2");
192 protected final byte[] qual3 = Bytes.toBytes("qual3");
193 protected final byte[] value1 = Bytes.toBytes("value1");
194 protected final byte[] value2 = Bytes.toBytes("value2");
195 protected final byte[] row = Bytes.toBytes("rowA");
196 protected final byte[] row2 = Bytes.toBytes("rowB");
197
198 protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
199 .getInstance(MetricsAssertHelper.class);
200
201 @Before
202 public void setup() throws IOException {
203 TEST_UTIL = HBaseTestingUtility.createLocalHTU();
204 FILESYSTEM = TEST_UTIL.getTestFileSystem();
205 CONF = TEST_UTIL.getConfiguration();
206 dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
207 method = name.getMethodName();
208 tableName = Bytes.toBytes(name.getMethodName());
209 }
210
211 @After
212 public void tearDown() throws Exception {
213 EnvironmentEdgeManagerTestHelper.reset();
214 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
215 TEST_UTIL.cleanupTestDir();
216 }
217
218 String getName() {
219 return name.getMethodName();
220 }
221
222
223
224
225
226
227
228
229
230
231
232 @Test (timeout=60000)
233 public void testCloseCarryingSnapshot() throws IOException {
234 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
235 Store store = region.getStore(COLUMN_FAMILY_BYTES);
236
237 byte [] value = Bytes.toBytes(name.getMethodName());
238
239 Put put = new Put(value);
240 put.add(COLUMN_FAMILY_BYTES, null, value);
241
242 region.put(put);
243 StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
244 storeFlushCtx.prepare();
245
246 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
247 region.put(put);
248
249 region.close();
250 assertEquals(0, region.getMemstoreSize().get());
251 HRegion.closeHRegion(region);
252 }
253
254
255
256
257
258 @Test (timeout=60000)
259 public void testMemstoreSnapshotSize() throws IOException {
260 class MyFaultyHLog extends FaultyHLog {
261 StoreFlushContext storeFlushCtx;
262 public MyFaultyHLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
263 throws IOException {
264 super(fs, rootDir, logName, conf);
265 }
266
267 void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
268 this.storeFlushCtx = storeFlushCtx;
269 }
270
271 @Override
272 public void sync(long txid) throws IOException {
273 storeFlushCtx.prepare();
274 super.sync(txid);
275 }
276 }
277
278 FileSystem fs = FileSystem.get(CONF);
279 Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
280 MyFaultyHLog faultyLog = new MyFaultyHLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
281 HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
282 CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES);
283
284 Store store = region.getStore(COLUMN_FAMILY_BYTES);
285
286 byte [] value = Bytes.toBytes(name.getMethodName());
287 faultyLog.setStoreFlushCtx(store.createFlushContext(12345));
288
289 Put put = new Put(value);
290 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
291 faultyLog.setFailureType(FaultyHLog.FailureType.SYNC);
292
293 boolean threwIOE = false;
294 try {
295 region.put(put);
296 } catch (IOException ioe) {
297 threwIOE = true;
298 } finally {
299 assertTrue("The regionserver should have thrown an exception", threwIOE);
300 }
301 long sz = store.getFlushableSize();
302 assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
303 HRegion.closeHRegion(region);
304 }
305
306
307
308
309 @Test
310 public void testMemstoreSizeWithFlushCanceling() throws IOException {
311 FileSystem fs = FileSystem.get(CONF);
312 Path rootDir = new Path(dir + "testMemstoreSizeWithFlushCanceling");
313 FaultyHLog hLog = new FaultyHLog(fs, rootDir, "testMemstoreSizeWithFlushCanceling", CONF);
314 HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
315 CONF, false, Durability.SYNC_WAL, hLog, COLUMN_FAMILY_BYTES);
316 Store store = region.getStore(COLUMN_FAMILY_BYTES);
317 assertEquals(0, region.getMemstoreSize().get());
318
319
320 byte [] value = Bytes.toBytes(name.getMethodName());
321 Put put = new Put(value);
322 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
323 region.put(put);
324 long onePutSize = region.getMemstoreSize().get();
325 assertTrue(onePutSize > 0);
326 region.flushcache();
327 assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize().get());
328 assertEquals("flushable size should be zero", 0, store.getFlushableSize());
329
330
331 RegionCoprocessorHost normalCPHost = region.getCoprocessorHost();
332 RegionCoprocessorHost mockedCPHost = Mockito.mock(RegionCoprocessorHost.class);
333 when(mockedCPHost.preFlush(any(HStore.class), any(InternalScanner.class))).thenReturn(null);
334 region.setCoprocessorHost(mockedCPHost);
335 region.put(put);
336 region.flushcache();
337 assertEquals("memstoreSize should NOT be zero", onePutSize, region.getMemstoreSize().get());
338 assertEquals("flushable size should NOT be zero", onePutSize, store.getFlushableSize());
339
340
341 region.setCoprocessorHost(normalCPHost);
342 region.flushcache();
343 assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize().get());
344 assertEquals("flushable size should be zero", 0, store.getFlushableSize());
345 HRegion.closeHRegion(region);
346 }
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363 @Test (timeout=60000)
364 public void testFlushSizeAccounting() throws Exception {
365 final Configuration conf = HBaseConfiguration.create(CONF);
366
367 conf.setInt("hbase.hstore.flush.retries.number", 1);
368 final User user =
369 User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
370
371 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
372 user.runAs(new PrivilegedExceptionAction<Object>() {
373 @Override
374 public Object run() throws Exception {
375
376 FileSystem fs = FileSystem.get(conf);
377 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
378 FaultyFileSystem ffs = (FaultyFileSystem)fs;
379 HRegion region = null;
380 try {
381
382 region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
383 long size = region.getMemstoreSize().get();
384 Assert.assertEquals(0, size);
385
386 Put p1 = new Put(row);
387 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
388 region.put(p1);
389 final long sizeOfOnePut = region.getMemstoreSize().get();
390
391 try {
392 LOG.info("Flushing");
393 region.flushcache();
394 Assert.fail("Didn't bubble up IOE!");
395 } catch (DroppedSnapshotException dse) {
396
397 region.closing.set(false);
398 }
399
400 ffs.fault.set(false);
401
402 Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize().get());
403
404
405 Put p2 = new Put(row);
406 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
407 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
408 region.put(p2);
409 Assert.assertEquals(sizeOfOnePut * 3, region.getMemstoreSize().get());
410
411
412
413 region.flushcache();
414
415 Assert.assertEquals(sizeOfOnePut * 2, region.getMemstoreSize().get());
416 } finally {
417 HRegion.closeHRegion(region);
418 }
419 return null;
420 }
421 });
422 FileSystem.closeAllForUGI(user.getUGI());
423 }
424
425 @Test
426 public void testCompactionAffectedByScanners() throws Exception {
427 byte[] family = Bytes.toBytes("family");
428 this.region = initHRegion(tableName, method, CONF, family);
429
430 Put put = new Put(Bytes.toBytes("r1"));
431 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
432 region.put(put);
433 region.flushcache();
434
435 Scan scan = new Scan();
436 scan.setMaxVersions(3);
437
438 RegionScanner scanner1 = region.getScanner(scan);
439
440 Delete delete = new Delete(Bytes.toBytes("r1"));
441 region.delete(delete);
442 region.flushcache();
443
444
445 RegionScanner scanner2 = region.getScanner(scan);
446
447 List<Cell> results = new ArrayList<Cell>();
448
449 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
450
451
452 region.compactStores(true);
453
454
455 RegionScanner scanner3 = region.getScanner(scan);
456
457
458 scanner1.next(results);
459 System.out.println(results);
460 assertEquals(1, results.size());
461
462 results.clear();
463 scanner2.next(results);
464 System.out.println(results);
465 assertEquals(0, results.size());
466
467 results.clear();
468 scanner3.next(results);
469 System.out.println(results);
470 assertEquals(0, results.size());
471 }
472
473 @Test
474 public void testToShowNPEOnRegionScannerReseek() throws Exception {
475 byte[] family = Bytes.toBytes("family");
476 this.region = initHRegion(tableName, method, CONF, family);
477
478 Put put = new Put(Bytes.toBytes("r1"));
479 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
480 region.put(put);
481 put = new Put(Bytes.toBytes("r2"));
482 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
483 region.put(put);
484 region.flushcache();
485
486 Scan scan = new Scan();
487 scan.setMaxVersions(3);
488
489 RegionScanner scanner1 = region.getScanner(scan);
490
491 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
492
493 region.compactStores(true);
494
495 scanner1.reseek(Bytes.toBytes("r2"));
496 List<Cell> results = new ArrayList<Cell>();
497 scanner1.next(results);
498 Cell keyValue = results.get(0);
499 Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
500 scanner1.close();
501 }
502
503 @Test
504 public void testSkipRecoveredEditsReplay() throws Exception {
505 String method = "testSkipRecoveredEditsReplay";
506 TableName tableName = TableName.valueOf(method);
507 byte[] family = Bytes.toBytes("family");
508 this.region = initHRegion(tableName, method, CONF, family);
509 try {
510 Path regiondir = region.getRegionFileSystem().getRegionDir();
511 FileSystem fs = region.getRegionFileSystem().getFileSystem();
512 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
513
514 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
515
516 long maxSeqId = 1050;
517 long minSeqId = 1000;
518
519 for (long i = minSeqId; i <= maxSeqId; i += 10) {
520 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
521 fs.create(recoveredEdits);
522 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
523
524 long time = System.nanoTime();
525 WALEdit edit = new WALEdit();
526 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
527 .toBytes(i)));
528 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
529 HConstants.DEFAULT_CLUSTER_ID), edit));
530
531 writer.close();
532 }
533 MonitoredTask status = TaskMonitor.get().createStatus(method);
534 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
535 for (Store store : region.getStores().values()) {
536 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
537 }
538 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
539 assertEquals(maxSeqId, seqId);
540 Get get = new Get(row);
541 Result result = region.get(get);
542 for (long i = minSeqId; i <= maxSeqId; i += 10) {
543 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
544 assertEquals(1, kvs.size());
545 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
546 }
547 } finally {
548 HRegion.closeHRegion(this.region);
549 this.region = null;
550 }
551 }
552
553 @Test
554 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
555 String method = "testSkipRecoveredEditsReplaySomeIgnored";
556 TableName tableName = TableName.valueOf(method);
557 byte[] family = Bytes.toBytes("family");
558 this.region = initHRegion(tableName, method, CONF, family);
559 try {
560 Path regiondir = region.getRegionFileSystem().getRegionDir();
561 FileSystem fs = region.getRegionFileSystem().getFileSystem();
562 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
563
564 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
565
566 long maxSeqId = 1050;
567 long minSeqId = 1000;
568
569 for (long i = minSeqId; i <= maxSeqId; i += 10) {
570 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
571 fs.create(recoveredEdits);
572 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
573
574 long time = System.nanoTime();
575 WALEdit edit = new WALEdit();
576 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
577 .toBytes(i)));
578 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
579 HConstants.DEFAULT_CLUSTER_ID), edit));
580
581 writer.close();
582 }
583 long recoverSeqId = 1030;
584 MonitoredTask status = TaskMonitor.get().createStatus(method);
585 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
586 for (Store store : region.getStores().values()) {
587 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
588 }
589 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
590 assertEquals(maxSeqId, seqId);
591 Get get = new Get(row);
592 Result result = region.get(get);
593 for (long i = minSeqId; i <= maxSeqId; i += 10) {
594 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
595 if (i < recoverSeqId) {
596 assertEquals(0, kvs.size());
597 } else {
598 assertEquals(1, kvs.size());
599 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
600 }
601 }
602 } finally {
603 HRegion.closeHRegion(this.region);
604 this.region = null;
605 }
606 }
607
608 @Test
609 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
610 byte[] family = Bytes.toBytes("family");
611 this.region = initHRegion(tableName, method, CONF, family);
612 try {
613 Path regiondir = region.getRegionFileSystem().getRegionDir();
614 FileSystem fs = region.getRegionFileSystem().getFileSystem();
615
616 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
617 for (int i = 1000; i < 1050; i += 10) {
618 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
619 FSDataOutputStream dos = fs.create(recoveredEdits);
620 dos.writeInt(i);
621 dos.close();
622 }
623 long minSeqId = 2000;
624 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
625 FSDataOutputStream dos = fs.create(recoveredEdits);
626 dos.close();
627
628 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
629 for (Store store : region.getStores().values()) {
630 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
631 }
632 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
633 assertEquals(minSeqId, seqId);
634 } finally {
635 HRegion.closeHRegion(this.region);
636 this.region = null;
637 }
638 }
639
640 @Test
641 public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
642 String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
643 TableName tableName = TableName.valueOf(method);
644 byte[] family = Bytes.toBytes("family");
645 this.region = initHRegion(tableName, method, CONF, family);
646 try {
647 Path regiondir = region.getRegionFileSystem().getRegionDir();
648 FileSystem fs = region.getRegionFileSystem().getFileSystem();
649 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
650
651 assertEquals(0, region.getStoreFileList(
652 region.getStores().keySet().toArray(new byte[0][])).size());
653
654 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
655
656 long maxSeqId = 1050;
657 long minSeqId = 1000;
658
659 for (long i = minSeqId; i <= maxSeqId; i += 10) {
660 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
661 fs.create(recoveredEdits);
662 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
663
664 long time = System.nanoTime();
665 WALEdit edit = null;
666 if (i == maxSeqId) {
667 edit = WALEdit.createCompaction(region.getRegionInfo(),
668 CompactionDescriptor.newBuilder()
669 .setTableName(ByteString.copyFrom(tableName.getName()))
670 .setFamilyName(ByteString.copyFrom(regionName))
671 .setEncodedRegionName(ByteString.copyFrom(regionName))
672 .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
673 .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName()))
674 .build());
675 } else {
676 edit = new WALEdit();
677 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
678 .toBytes(i)));
679 }
680 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
681 HConstants.DEFAULT_CLUSTER_ID), edit));
682 writer.close();
683 }
684
685 long recoverSeqId = 1030;
686 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
687 MonitoredTask status = TaskMonitor.get().createStatus(method);
688 for (Store store : region.getStores().values()) {
689 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
690 }
691 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
692 assertEquals(maxSeqId, seqId);
693
694
695 assertEquals(1, region.getStoreFileList(
696 region.getStores().keySet().toArray(new byte[0][])).size());
697
698 } finally {
699 HRegion.closeHRegion(this.region);
700 this.region = null;
701 } }
702
703 @Test
704 public void testRecoveredEditsReplayCompaction() throws Exception {
705 testRecoveredEditsReplayCompaction(false);
706 testRecoveredEditsReplayCompaction(true);
707 }
708 public void testRecoveredEditsReplayCompaction(boolean mismatchedRegionName) throws Exception {
709 String method = name.getMethodName();
710 TableName tableName = TableName.valueOf(method);
711 byte[] family = Bytes.toBytes("family");
712 this.region = initHRegion(tableName, method, CONF, family);
713 try {
714 Path regiondir = region.getRegionFileSystem().getRegionDir();
715 FileSystem fs = region.getRegionFileSystem().getFileSystem();
716 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
717
718 long maxSeqId = 3;
719 long minSeqId = 0;
720
721 for (long i = minSeqId; i < maxSeqId; i++) {
722 Put put = new Put(Bytes.toBytes(i));
723 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
724 region.put(put);
725 region.flushcache();
726 }
727
728
729 assertEquals(3, region.getStore(family).getStorefilesCount());
730 List<Path> storeFiles = new ArrayList<Path>(3);
731 for (StoreFile sf : region.getStore(family).getStorefiles()) {
732 storeFiles.add(sf.getPath());
733 }
734
735
736 CONF.setBoolean("hbase.hstore.compaction.complete", false);
737 region.compactStores();
738
739
740 assertEquals(3, region.getStore(family).getStorefilesCount());
741
742
743 Path tmpDir = region.getRegionFileSystem().getTempDir();
744 FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
745 String errorMsg = "Expected to find 1 file in the region temp directory "
746 + "from the compaction, could not find any";
747 assertNotNull(errorMsg, files);
748 assertEquals(errorMsg, 1, files.length);
749
750 Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
751 files[0].getPath());
752
753 byte[] encodedNameAsBytes = this.region.getRegionInfo().getEncodedNameAsBytes();
754 byte[] fakeEncodedNameAsBytes = new byte [encodedNameAsBytes.length];
755 for (int i=0; i < encodedNameAsBytes.length; i++) {
756
757 fakeEncodedNameAsBytes[i] = (byte) (encodedNameAsBytes[i] + 1);
758 }
759
760 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
761 .getRegionInfo(), mismatchedRegionName ? fakeEncodedNameAsBytes : null, family,
762 storeFiles, Lists.newArrayList(newFile),
763 region.getRegionFileSystem().getStoreDir(Bytes.toString(family)));
764
765 HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
766 this.region.getRegionInfo(), compactionDescriptor, new AtomicLong(1));
767
768 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
769
770 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
771 fs.create(recoveredEdits);
772 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
773
774 long time = System.nanoTime();
775
776 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, 10, time,
777 HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(),
778 compactionDescriptor)));
779 writer.close();
780
781
782 region.getTableDesc();
783 region.getRegionInfo();
784 region.close();
785 try {
786 region = HRegion.openHRegion(region, null);
787 } catch (WrongRegionException wre) {
788 fail("Matching encoded region name should not have produced WrongRegionException");
789 }
790
791
792 Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
793 for (StoreFile sf : sfs) {
794 LOG.info(sf.getPath());
795 }
796 if (!mismatchedRegionName) {
797 assertEquals(1, region.getStore(family).getStorefilesCount());
798 }
799 files = FSUtils.listStatus(fs, tmpDir);
800 assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
801
802 for (long i = minSeqId; i < maxSeqId; i++) {
803 Get get = new Get(Bytes.toBytes(i));
804 Result result = region.get(get);
805 byte[] value = result.getValue(family, Bytes.toBytes(i));
806 assertArrayEquals(Bytes.toBytes(i), value);
807 }
808 } finally {
809 HRegion.closeHRegion(this.region);
810 this.region = null;
811 }
812 }
813
814 @Test
815 public void testGetWhileRegionClose() throws IOException {
816 TableName tableName = TableName.valueOf(name.getMethodName());
817 Configuration hc = initSplit();
818 int numRows = 100;
819 byte[][] families = { fam1, fam2, fam3 };
820
821
822 String method = name.getMethodName();
823 this.region = initHRegion(tableName, method, hc, families);
824 try {
825
826 final int startRow = 100;
827 putData(startRow, numRows, qual1, families);
828 putData(startRow, numRows, qual2, families);
829 putData(startRow, numRows, qual3, families);
830 final AtomicBoolean done = new AtomicBoolean(false);
831 final AtomicInteger gets = new AtomicInteger(0);
832 GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
833 try {
834
835 for (int i = 0; i < threads.length / 2; i++) {
836 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
837 threads[i].setDaemon(true);
838 threads[i].start();
839 }
840
841
842 this.region.closing.set(true);
843 for (int i = threads.length / 2; i < threads.length; i++) {
844 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
845 threads[i].setDaemon(true);
846 threads[i].start();
847 }
848 } finally {
849 if (this.region != null) {
850 HRegion.closeHRegion(this.region);
851 }
852 }
853 done.set(true);
854 for (GetTillDoneOrException t : threads) {
855 try {
856 t.join();
857 } catch (InterruptedException e) {
858 e.printStackTrace();
859 }
860 if (t.e != null) {
861 LOG.info("Exception=" + t.e);
862 assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
863 }
864 }
865 } finally {
866 HRegion.closeHRegion(this.region);
867 this.region = null;
868 }
869 }
870
871
872
873
874
875 class GetTillDoneOrException extends Thread {
876 private final Get g;
877 private final AtomicBoolean done;
878 private final AtomicInteger count;
879 private Exception e;
880
881 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
882 super("getter." + i);
883 this.g = new Get(r);
884 this.done = d;
885 this.count = c;
886 }
887
888 @Override
889 public void run() {
890 while (!this.done.get()) {
891 try {
892 assertTrue(region.get(g).size() > 0);
893 this.count.incrementAndGet();
894 } catch (Exception e) {
895 this.e = e;
896 break;
897 }
898 }
899 }
900 }
901
902
903
904
905 @Test
906 public void testWeirdCacheBehaviour() throws Exception {
907 byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
908 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
909 Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
910 this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
911 try {
912 String value = "this is the value";
913 String value2 = "this is some other value";
914 String keyPrefix1 = "prefix1";
915 String keyPrefix2 = "prefix2";
916 String keyPrefix3 = "prefix3";
917 putRows(this.region, 3, value, keyPrefix1);
918 putRows(this.region, 3, value, keyPrefix2);
919 putRows(this.region, 3, value, keyPrefix3);
920 putRows(this.region, 3, value2, keyPrefix1);
921 putRows(this.region, 3, value2, keyPrefix2);
922 putRows(this.region, 3, value2, keyPrefix3);
923 System.out.println("Checking values for key: " + keyPrefix1);
924 assertEquals("Got back incorrect number of rows from scan", 3,
925 getNumberOfRows(keyPrefix1, value2, this.region));
926 System.out.println("Checking values for key: " + keyPrefix2);
927 assertEquals("Got back incorrect number of rows from scan", 3,
928 getNumberOfRows(keyPrefix2, value2, this.region));
929 System.out.println("Checking values for key: " + keyPrefix3);
930 assertEquals("Got back incorrect number of rows from scan", 3,
931 getNumberOfRows(keyPrefix3, value2, this.region));
932 deleteColumns(this.region, value2, keyPrefix1);
933 deleteColumns(this.region, value2, keyPrefix2);
934 deleteColumns(this.region, value2, keyPrefix3);
935 System.out.println("Starting important checks.....");
936 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
937 getNumberOfRows(keyPrefix1, value2, this.region));
938 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
939 getNumberOfRows(keyPrefix2, value2, this.region));
940 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
941 getNumberOfRows(keyPrefix3, value2, this.region));
942 } finally {
943 HRegion.closeHRegion(this.region);
944 this.region = null;
945 }
946 }
947
948 @Test
949 public void testAppendWithReadOnlyTable() throws Exception {
950 byte[] TABLE = Bytes.toBytes("readOnlyTable");
951 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
952 boolean exceptionCaught = false;
953 Append append = new Append(Bytes.toBytes("somerow"));
954 append.setDurability(Durability.SKIP_WAL);
955 append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
956 Bytes.toBytes("somevalue"));
957 try {
958 region.append(append);
959 } catch (IOException e) {
960 exceptionCaught = true;
961 } finally {
962 HRegion.closeHRegion(this.region);
963 this.region = null;
964 }
965 assertTrue(exceptionCaught == true);
966 }
967
968 @Test
969 public void testIncrWithReadOnlyTable() throws Exception {
970 byte[] TABLE = Bytes.toBytes("readOnlyTable");
971 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
972 boolean exceptionCaught = false;
973 Increment inc = new Increment(Bytes.toBytes("somerow"));
974 inc.setDurability(Durability.SKIP_WAL);
975 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
976 try {
977 region.increment(inc);
978 } catch (IOException e) {
979 exceptionCaught = true;
980 } finally {
981 HRegion.closeHRegion(this.region);
982 this.region = null;
983 }
984 assertTrue(exceptionCaught == true);
985 }
986
987 private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
988 InternalScanner scanner = buildScanner(keyPrefix, value, r);
989 int count = 0;
990 boolean more = false;
991 List<Cell> results = new ArrayList<Cell>();
992 do {
993 more = scanner.next(results);
994 if (results != null && !results.isEmpty())
995 count++;
996 else
997 break;
998 Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
999 delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
1000 r.delete(delete);
1001 results.clear();
1002 } while (more);
1003 assertEquals("Did not perform correct number of deletes", 3, count);
1004 }
1005
1006 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
1007 InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
1008 int numberOfResults = 0;
1009 List<Cell> results = new ArrayList<Cell>();
1010 boolean more = false;
1011 do {
1012 more = resultScanner.next(results);
1013 if (results != null && !results.isEmpty())
1014 numberOfResults++;
1015 else
1016 break;
1017 for (Cell kv : results) {
1018 System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
1019 }
1020 results.clear();
1021 } while (more);
1022 return numberOfResults;
1023 }
1024
1025 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
1026 throws IOException {
1027
1028 FilterList allFilters = new FilterList();
1029 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
1030
1031 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
1032 Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
1033 filter.setFilterIfMissing(true);
1034 allFilters.addFilter(filter);
1035 Scan scan = new Scan();
1036 scan.addFamily(Bytes.toBytes("trans-blob"));
1037 scan.addFamily(Bytes.toBytes("trans-type"));
1038 scan.addFamily(Bytes.toBytes("trans-date"));
1039 scan.addFamily(Bytes.toBytes("trans-tags"));
1040 scan.addFamily(Bytes.toBytes("trans-group"));
1041 scan.setFilter(allFilters);
1042 return r.getScanner(scan);
1043 }
1044
1045 private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
1046 for (int i = 0; i < numRows; i++) {
1047 String row = key + "_" + i
1048 System.out.println(String.format("Saving row: %s, with value %s", row, value));
1049 Put put = new Put(Bytes.toBytes(row));
1050 put.setDurability(Durability.SKIP_WAL);
1051 put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
1052 put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
1053 put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
1054 put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
1055 put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
1056 r.put(put);
1057 }
1058 }
1059
1060 @Test
1061 public void testFamilyWithAndWithoutColon() throws Exception {
1062 byte[] b = Bytes.toBytes(getName());
1063 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1064 this.region = initHRegion(b, getName(), CONF, cf);
1065 try {
1066 Put p = new Put(b);
1067 byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
1068 p.add(cfwithcolon, cfwithcolon, cfwithcolon);
1069 boolean exception = false;
1070 try {
1071 this.region.put(p);
1072 } catch (NoSuchColumnFamilyException e) {
1073 exception = true;
1074 }
1075 assertTrue(exception);
1076 } finally {
1077 HRegion.closeHRegion(this.region);
1078 this.region = null;
1079 }
1080 }
1081
1082 @Test
1083 public void testBatchPut_whileNoRowLocksHeld() throws IOException {
1084 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1085 byte[] qual = Bytes.toBytes("qual");
1086 byte[] val = Bytes.toBytes("val");
1087 this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1088 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1089 try {
1090 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1091 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1092
1093 LOG.info("First a batch put with all valid puts");
1094 final Put[] puts = new Put[10];
1095 for (int i = 0; i < 10; i++) {
1096 puts[i] = new Put(Bytes.toBytes("row_" + i));
1097 puts[i].add(cf, qual, val);
1098 }
1099
1100 OperationStatus[] codes = this.region.batchMutate(puts);
1101 assertEquals(10, codes.length);
1102 for (int i = 0; i < 10; i++) {
1103 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1104 }
1105 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1106
1107 LOG.info("Next a batch put with one invalid family");
1108 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1109 codes = this.region.batchMutate(puts);
1110 assertEquals(10, codes.length);
1111 for (int i = 0; i < 10; i++) {
1112 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1113 codes[i].getOperationStatusCode());
1114 }
1115
1116 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1117 } finally {
1118 HRegion.closeHRegion(this.region);
1119 this.region = null;
1120 }
1121 }
1122
1123 @Test
1124 public void testBatchPut_whileMultipleRowLocksHeld() throws Exception {
1125 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1126 byte[] qual = Bytes.toBytes("qual");
1127 byte[] val = Bytes.toBytes("val");
1128 this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1129 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1130 try {
1131 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1132 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1133
1134 final Put[] puts = new Put[10];
1135 for (int i = 0; i < 10; i++) {
1136 puts[i] = new Put(Bytes.toBytes("row_" + i));
1137 puts[i].add(cf, qual, val);
1138 }
1139 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1140
1141 LOG.info("batchPut will have to break into four batches to avoid row locks");
1142 RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2"));
1143 RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_4"));
1144 RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_6"));
1145
1146 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1147 final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
1148 TestThread putter = new TestThread(ctx) {
1149 @Override
1150 public void doWork() throws IOException {
1151 retFromThread.set(region.batchMutate(puts));
1152 }
1153 };
1154 LOG.info("...starting put thread while holding locks");
1155 ctx.addThread(putter);
1156 ctx.startThreads();
1157
1158 LOG.info("...waiting for put thread to sync 1st time");
1159 waitForCounter(source, "syncTimeNumOps", syncs + 1);
1160
1161
1162
1163
1164
1165
1166 Thread regionCloseThread = new Thread() {
1167 @Override
1168 public void run() {
1169 try {
1170 HRegion.closeHRegion(region);
1171 } catch (IOException e) {
1172 throw new RuntimeException(e);
1173 }
1174 }
1175 };
1176 regionCloseThread.start();
1177
1178 LOG.info("...releasing row lock 1, which should let put thread continue");
1179 rowLock1.release();
1180
1181 LOG.info("...waiting for put thread to sync 2nd time");
1182 waitForCounter(source, "syncTimeNumOps", syncs + 2);
1183
1184 LOG.info("...releasing row lock 2, which should let put thread continue");
1185 rowLock2.release();
1186
1187 LOG.info("...waiting for put thread to sync 3rd time");
1188 waitForCounter(source, "syncTimeNumOps", syncs + 3);
1189
1190 LOG.info("...releasing row lock 3, which should let put thread continue");
1191 rowLock3.release();
1192
1193 LOG.info("...waiting for put thread to sync 4th time");
1194 waitForCounter(source, "syncTimeNumOps", syncs + 4);
1195
1196 LOG.info("...joining on put thread");
1197 ctx.stop();
1198 regionCloseThread.join();
1199
1200 OperationStatus[] codes = retFromThread.get();
1201 for (int i = 0; i < codes.length; i++) {
1202 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1203 codes[i].getOperationStatusCode());
1204 }
1205 } finally {
1206 HRegion.closeHRegion(this.region);
1207 this.region = null;
1208 }
1209 }
1210
1211 private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount)
1212 throws InterruptedException {
1213 long startWait = System.currentTimeMillis();
1214 long currentCount;
1215 while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) {
1216 Thread.sleep(100);
1217 if (System.currentTimeMillis() - startWait > 10000) {
1218 fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName,
1219 expectedCount, currentCount));
1220 }
1221 }
1222 }
1223
1224
1225 @Test
1226 public void testBatchPutWithTsSlop() throws Exception {
1227 byte[] b = Bytes.toBytes(getName());
1228 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1229 byte[] qual = Bytes.toBytes("qual");
1230 byte[] val = Bytes.toBytes("val");
1231
1232
1233 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1234 this.region = initHRegion(b, getName(), CONF, cf);
1235
1236 try {
1237 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1238 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1239 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1240
1241 final Put[] puts = new Put[10];
1242 for (int i = 0; i < 10; i++) {
1243 puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
1244 puts[i].add(cf, qual, val);
1245 }
1246
1247 OperationStatus[] codes = this.region.batchMutate(puts);
1248 assertEquals(10, codes.length);
1249 for (int i = 0; i < 10; i++) {
1250 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1251 }
1252 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1253
1254 } finally {
1255 HRegion.closeHRegion(this.region);
1256 this.region = null;
1257 }
1258
1259 }
1260
1261
1262
1263
1264 @Test
1265 public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1266 byte[] row1 = Bytes.toBytes("row1");
1267 byte[] fam1 = Bytes.toBytes("fam1");
1268 byte[] qf1 = Bytes.toBytes("qualifier");
1269 byte[] emptyVal = new byte[] {};
1270 byte[] val1 = Bytes.toBytes("value1");
1271 byte[] val2 = Bytes.toBytes("value2");
1272
1273
1274 String method = this.getName();
1275 this.region = initHRegion(tableName, method, CONF, fam1);
1276 try {
1277
1278 Put put = new Put(row1);
1279 put.add(fam1, qf1, emptyVal);
1280
1281
1282 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1283 emptyVal), put, true);
1284 assertTrue(res);
1285
1286
1287 put = new Put(row1);
1288 put.add(fam1, qf1, val1);
1289
1290
1291 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1292 put, true);
1293 assertTrue(res);
1294
1295
1296 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1297 put, true);
1298 assertFalse(res);
1299
1300 Delete delete = new Delete(row1);
1301 delete.deleteColumn(fam1, qf1);
1302 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1303 delete, true);
1304 assertFalse(res);
1305
1306 put = new Put(row1);
1307 put.add(fam1, qf1, val2);
1308
1309 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1310 put, true);
1311 assertTrue(res);
1312
1313
1314 delete = new Delete(row1);
1315 delete.deleteColumn(fam1, qf1);
1316 delete.deleteColumn(fam1, qf1);
1317 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1318 delete, true);
1319 assertTrue(res);
1320
1321 delete = new Delete(row1);
1322 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1323 delete, true);
1324 assertTrue(res);
1325
1326
1327 put = new Put(row1);
1328 put.add(fam1, qf1, val1);
1329
1330 res = region
1331 .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
1332 assertTrue(res);
1333 } finally {
1334 HRegion.closeHRegion(this.region);
1335 this.region = null;
1336 }
1337 }
1338
1339 @Test
1340 public void testCheckAndMutate_WithWrongValue() throws IOException {
1341 byte[] row1 = Bytes.toBytes("row1");
1342 byte[] fam1 = Bytes.toBytes("fam1");
1343 byte[] qf1 = Bytes.toBytes("qualifier");
1344 byte[] val1 = Bytes.toBytes("value1");
1345 byte[] val2 = Bytes.toBytes("value2");
1346
1347
1348 String method = this.getName();
1349 this.region = initHRegion(tableName, method, CONF, fam1);
1350 try {
1351
1352 Put put = new Put(row1);
1353 put.add(fam1, qf1, val1);
1354 region.put(put);
1355
1356
1357 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1358 val2), put, true);
1359 assertEquals(false, res);
1360
1361
1362 Delete delete = new Delete(row1);
1363 delete.deleteFamily(fam1);
1364 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1365 put, true);
1366 assertEquals(false, res);
1367 } finally {
1368 HRegion.closeHRegion(this.region);
1369 this.region = null;
1370 }
1371 }
1372
1373 @Test
1374 public void testCheckAndMutate_WithCorrectValue() throws IOException {
1375 byte[] row1 = Bytes.toBytes("row1");
1376 byte[] fam1 = Bytes.toBytes("fam1");
1377 byte[] qf1 = Bytes.toBytes("qualifier");
1378 byte[] val1 = Bytes.toBytes("value1");
1379
1380
1381 String method = this.getName();
1382 this.region = initHRegion(tableName, method, CONF, fam1);
1383 try {
1384
1385 Put put = new Put(row1);
1386 put.add(fam1, qf1, val1);
1387 region.put(put);
1388
1389
1390 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1391 val1), put, true);
1392 assertEquals(true, res);
1393
1394
1395 Delete delete = new Delete(row1);
1396 delete.deleteColumn(fam1, qf1);
1397 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1398 delete, true);
1399 assertEquals(true, res);
1400 } finally {
1401 HRegion.closeHRegion(this.region);
1402 this.region = null;
1403 }
1404 }
1405
1406 @Test
1407 public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1408 byte[] row1 = Bytes.toBytes("row1");
1409 byte[] fam1 = Bytes.toBytes("fam1");
1410 byte[] qf1 = Bytes.toBytes("qualifier");
1411 byte[] val1 = Bytes.toBytes("value1");
1412 byte[] val2 = Bytes.toBytes("value2");
1413 byte[] val3 = Bytes.toBytes("value3");
1414 byte[] val4 = Bytes.toBytes("value4");
1415
1416
1417 String method = this.getName();
1418 this.region = initHRegion(tableName, method, CONF, fam1);
1419 try {
1420
1421 Put put = new Put(row1);
1422 put.add(fam1, qf1, val3);
1423 region.put(put);
1424
1425
1426 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1427 new BinaryComparator(val3), put, true);
1428 assertEquals(false, res);
1429
1430
1431 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1432 new BinaryComparator(val4), put, true);
1433 assertEquals(false, res);
1434
1435
1436
1437 put = new Put(row1);
1438 put.add(fam1, qf1, val2);
1439 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1440 new BinaryComparator(val2), put, true);
1441 assertEquals(true, res);
1442
1443
1444 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1445 new BinaryComparator(val3), put, true);
1446 assertEquals(false, res);
1447
1448
1449
1450 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1451 new BinaryComparator(val2), put, true);
1452 assertEquals(true, res);
1453
1454
1455
1456 put = new Put(row1);
1457 put.add(fam1, qf1, val3);
1458 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1459 new BinaryComparator(val1), put, true);
1460 assertEquals(true, res);
1461
1462
1463 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1464 new BinaryComparator(val3), put, true);
1465 assertEquals(false, res);
1466
1467
1468 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1469 new BinaryComparator(val2), put, true);
1470 assertEquals(false, res);
1471
1472
1473
1474 put = new Put(row1);
1475 put.add(fam1, qf1, val2);
1476 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1477 new BinaryComparator(val4), put, true);
1478 assertEquals(true, res);
1479
1480
1481 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1482 new BinaryComparator(val1), put, true);
1483 assertEquals(false, res);
1484
1485
1486
1487 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1488 new BinaryComparator(val2), put, true);
1489 assertEquals(true, res);
1490
1491
1492 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1493 new BinaryComparator(val3), put, true);
1494 assertEquals(true, res);
1495 } finally {
1496 HRegion.closeHRegion(this.region);
1497 this.region = null;
1498 }
1499 }
1500
1501 @Test
1502 public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1503 byte[] row1 = Bytes.toBytes("row1");
1504 byte[] fam1 = Bytes.toBytes("fam1");
1505 byte[] fam2 = Bytes.toBytes("fam2");
1506 byte[] qf1 = Bytes.toBytes("qualifier");
1507 byte[] val1 = Bytes.toBytes("value1");
1508 byte[] val2 = Bytes.toBytes("value2");
1509
1510 byte[][] families = { fam1, fam2 };
1511
1512
1513 String method = this.getName();
1514 this.region = initHRegion(tableName, method, CONF, families);
1515 try {
1516
1517 Put put = new Put(row1);
1518 put.add(fam1, qf1, val1);
1519 region.put(put);
1520
1521
1522 long ts = System.currentTimeMillis();
1523 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1524 put = new Put(row1);
1525 put.add(kv);
1526
1527
1528 HStore store = (HStore) region.getStore(fam1);
1529 store.memstore.kvset.size();
1530
1531 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1532 val1), put, true);
1533 assertEquals(true, res);
1534 store.memstore.kvset.size();
1535
1536 Get get = new Get(row1);
1537 get.addColumn(fam2, qf1);
1538 Cell[] actual = region.get(get).rawCells();
1539
1540 Cell[] expected = { kv };
1541
1542 assertEquals(expected.length, actual.length);
1543 for (int i = 0; i < actual.length; i++) {
1544 assertEquals(expected[i], actual[i]);
1545 }
1546 } finally {
1547 HRegion.closeHRegion(this.region);
1548 this.region = null;
1549 }
1550 }
1551
1552 @Test
1553 public void testCheckAndPut_wrongRowInPut() throws IOException {
1554 TableName tableName = TableName.valueOf(name.getMethodName());
1555 this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
1556 try {
1557 Put put = new Put(row2);
1558 put.add(fam1, qual1, value1);
1559 try {
1560 region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1561 new BinaryComparator(value2), put, false);
1562 fail();
1563 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1564
1565 }
1566 } finally {
1567 HRegion.closeHRegion(this.region);
1568 this.region = null;
1569 }
1570 }
1571
1572 @Test
1573 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1574 byte[] row1 = Bytes.toBytes("row1");
1575 byte[] fam1 = Bytes.toBytes("fam1");
1576 byte[] fam2 = Bytes.toBytes("fam2");
1577 byte[] qf1 = Bytes.toBytes("qualifier1");
1578 byte[] qf2 = Bytes.toBytes("qualifier2");
1579 byte[] qf3 = Bytes.toBytes("qualifier3");
1580 byte[] val1 = Bytes.toBytes("value1");
1581 byte[] val2 = Bytes.toBytes("value2");
1582 byte[] val3 = Bytes.toBytes("value3");
1583 byte[] emptyVal = new byte[] {};
1584
1585 byte[][] families = { fam1, fam2 };
1586
1587
1588 String method = this.getName();
1589 this.region = initHRegion(tableName, method, CONF, families);
1590 try {
1591
1592 Put put = new Put(row1);
1593 put.add(fam1, qf1, val1);
1594 region.put(put);
1595 Threads.sleep(2);
1596
1597 put = new Put(row1);
1598 put.add(fam1, qf1, val2);
1599 put.add(fam2, qf1, val3);
1600 put.add(fam2, qf2, val2);
1601 put.add(fam2, qf3, val1);
1602 put.add(fam1, qf3, val1);
1603 region.put(put);
1604
1605
1606 Delete delete = new Delete(row1);
1607 delete.deleteColumn(fam1, qf1);
1608 delete.deleteColumn(fam2, qf1);
1609 delete.deleteColumn(fam1, qf3);
1610 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1611 val2), delete, true);
1612 assertEquals(true, res);
1613
1614 Get get = new Get(row1);
1615 get.addColumn(fam1, qf1);
1616 get.addColumn(fam1, qf3);
1617 get.addColumn(fam2, qf2);
1618 Result r = region.get(get);
1619 assertEquals(2, r.size());
1620 assertArrayEquals(val1, r.getValue(fam1, qf1));
1621 assertArrayEquals(val2, r.getValue(fam2, qf2));
1622
1623
1624 delete = new Delete(row1);
1625 delete.deleteFamily(fam2);
1626 res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1627 delete, true);
1628 assertEquals(true, res);
1629
1630 get = new Get(row1);
1631 r = region.get(get);
1632 assertEquals(1, r.size());
1633 assertArrayEquals(val1, r.getValue(fam1, qf1));
1634
1635
1636 delete = new Delete(row1);
1637 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1638 delete, true);
1639 assertEquals(true, res);
1640 get = new Get(row1);
1641 r = region.get(get);
1642 assertEquals(0, r.size());
1643 } finally {
1644 HRegion.closeHRegion(this.region);
1645 this.region = null;
1646 }
1647 }
1648
1649
1650
1651
1652 @Test
1653 public void testDelete_multiDeleteColumn() throws IOException {
1654 byte[] row1 = Bytes.toBytes("row1");
1655 byte[] fam1 = Bytes.toBytes("fam1");
1656 byte[] qual = Bytes.toBytes("qualifier");
1657 byte[] value = Bytes.toBytes("value");
1658
1659 Put put = new Put(row1);
1660 put.add(fam1, qual, 1, value);
1661 put.add(fam1, qual, 2, value);
1662
1663 String method = this.getName();
1664 this.region = initHRegion(tableName, method, CONF, fam1);
1665 try {
1666 region.put(put);
1667
1668
1669 Delete delete = new Delete(row1);
1670 delete.deleteColumn(fam1, qual);
1671 delete.deleteColumn(fam1, qual);
1672 region.delete(delete);
1673
1674 Get get = new Get(row1);
1675 get.addFamily(fam1);
1676 Result r = region.get(get);
1677 assertEquals(0, r.size());
1678 } finally {
1679 HRegion.closeHRegion(this.region);
1680 this.region = null;
1681 }
1682 }
1683
1684 @Test
1685 public void testDelete_CheckFamily() throws IOException {
1686 byte[] row1 = Bytes.toBytes("row1");
1687 byte[] fam1 = Bytes.toBytes("fam1");
1688 byte[] fam2 = Bytes.toBytes("fam2");
1689 byte[] fam3 = Bytes.toBytes("fam3");
1690 byte[] fam4 = Bytes.toBytes("fam4");
1691
1692
1693 String method = this.getName();
1694 this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
1695 try {
1696 List<Cell> kvs = new ArrayList<Cell>();
1697 kvs.add(new KeyValue(row1, fam4, null, null));
1698
1699
1700 byte[] family = fam2;
1701 try {
1702 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1703 Bytes.BYTES_COMPARATOR);
1704 deleteMap.put(family, kvs);
1705 region.delete(deleteMap, Durability.SYNC_WAL);
1706 } catch (Exception e) {
1707 assertTrue("Family " + new String(family) + " does not exist", false);
1708 }
1709
1710
1711 boolean ok = false;
1712 family = fam4;
1713 try {
1714 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1715 Bytes.BYTES_COMPARATOR);
1716 deleteMap.put(family, kvs);
1717 region.delete(deleteMap, Durability.SYNC_WAL);
1718 } catch (Exception e) {
1719 ok = true;
1720 }
1721 assertEquals("Family " + new String(family) + " does exist", true, ok);
1722 } finally {
1723 HRegion.closeHRegion(this.region);
1724 this.region = null;
1725 }
1726 }
1727
1728 @Test
1729 public void testDelete_mixed() throws IOException, InterruptedException {
1730 byte[] fam = Bytes.toBytes("info");
1731 byte[][] families = { fam };
1732 String method = this.getName();
1733 this.region = initHRegion(tableName, method, CONF, families);
1734 try {
1735 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1736
1737 byte[] row = Bytes.toBytes("table_name");
1738
1739 byte[] serverinfo = Bytes.toBytes("serverinfo");
1740 byte[] splitA = Bytes.toBytes("splitA");
1741 byte[] splitB = Bytes.toBytes("splitB");
1742
1743
1744 Put put = new Put(row);
1745 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1746 region.put(put);
1747
1748 put = new Put(row);
1749 put.add(fam, splitB, Bytes.toBytes("reference_B"));
1750 region.put(put);
1751
1752 put = new Put(row);
1753 put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1754 region.put(put);
1755
1756
1757 Delete delete = new Delete(row);
1758 delete.deleteColumns(fam, splitA);
1759 region.delete(delete);
1760
1761
1762 Get get = new Get(row).addColumn(fam, serverinfo);
1763 Result result = region.get(get);
1764 assertEquals(1, result.size());
1765
1766 get = new Get(row).addColumn(fam, splitA);
1767 result = region.get(get);
1768 assertEquals(0, result.size());
1769
1770 get = new Get(row).addColumn(fam, splitB);
1771 result = region.get(get);
1772 assertEquals(1, result.size());
1773
1774
1775 put = new Put(row);
1776 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1777 region.put(put);
1778 get = new Get(row);
1779 result = region.get(get);
1780 assertEquals(3, result.size());
1781
1782
1783 delete = new Delete(row);
1784 region.delete(delete);
1785 assertEquals(0, region.get(get).size());
1786
1787 region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
1788 result = region.get(get);
1789 assertEquals(1, result.size());
1790 } finally {
1791 HRegion.closeHRegion(this.region);
1792 this.region = null;
1793 }
1794 }
1795
1796 @Test
1797 public void testDeleteRowWithFutureTs() throws IOException {
1798 byte[] fam = Bytes.toBytes("info");
1799 byte[][] families = { fam };
1800 String method = this.getName();
1801 this.region = initHRegion(tableName, method, CONF, families);
1802 try {
1803 byte[] row = Bytes.toBytes("table_name");
1804
1805 byte[] serverinfo = Bytes.toBytes("serverinfo");
1806
1807
1808 Put put = new Put(row);
1809 put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
1810 region.put(put);
1811
1812
1813 Delete delete = new Delete(row);
1814 region.delete(delete);
1815
1816
1817 Get get = new Get(row).addColumn(fam, serverinfo);
1818 Result result = region.get(get);
1819 assertEquals(1, result.size());
1820
1821
1822 delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
1823 region.delete(delete);
1824
1825
1826 get = new Get(row).addColumn(fam, serverinfo);
1827 result = region.get(get);
1828 assertEquals(0, result.size());
1829 } finally {
1830 HRegion.closeHRegion(this.region);
1831 this.region = null;
1832 }
1833 }
1834
1835
1836
1837
1838
1839 @Test
1840 public void testPutWithLatestTS() throws IOException {
1841 byte[] fam = Bytes.toBytes("info");
1842 byte[][] families = { fam };
1843 String method = this.getName();
1844 this.region = initHRegion(tableName, method, CONF, families);
1845 try {
1846 byte[] row = Bytes.toBytes("row1");
1847
1848 byte[] qual = Bytes.toBytes("qual");
1849
1850
1851 Put put = new Put(row);
1852 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1853 region.put(put);
1854
1855
1856 Get get = new Get(row).addColumn(fam, qual);
1857 Result result = region.get(get);
1858 assertEquals(1, result.size());
1859 Cell kv = result.rawCells()[0];
1860 LOG.info("Got: " + kv);
1861 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1862 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1863
1864
1865
1866 row = Bytes.toBytes("row2");
1867 put = new Put(row);
1868 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1869 region.put(put);
1870
1871
1872 get = new Get(row).addColumn(fam, qual);
1873 result = region.get(get);
1874 assertEquals(1, result.size());
1875 kv = result.rawCells()[0];
1876 LOG.info("Got: " + kv);
1877 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1878 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1879 } finally {
1880 HRegion.closeHRegion(this.region);
1881 this.region = null;
1882 }
1883
1884 }
1885
1886
1887
1888
1889
1890
1891 @Test
1892 public void testPutWithTsSlop() throws IOException {
1893 byte[] fam = Bytes.toBytes("info");
1894 byte[][] families = { fam };
1895 String method = this.getName();
1896
1897
1898 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1899 this.region = initHRegion(tableName, method, CONF, families);
1900 boolean caughtExcep = false;
1901 try {
1902 try {
1903
1904 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
1905
1906 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
1907 Bytes.toBytes("value")));
1908 fail("Expected IOE for TS out of configured timerange");
1909 } catch (FailedSanityCheckException ioe) {
1910 LOG.debug("Received expected exception", ioe);
1911 caughtExcep = true;
1912 }
1913 assertTrue("Should catch FailedSanityCheckException", caughtExcep);
1914 } finally {
1915 HRegion.closeHRegion(this.region);
1916 this.region = null;
1917 }
1918 }
1919
1920 @Test
1921 public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
1922 byte[] fam1 = Bytes.toBytes("columnA");
1923 byte[] fam2 = Bytes.toBytes("columnB");
1924 this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
1925 try {
1926 byte[] rowA = Bytes.toBytes("rowA");
1927 byte[] rowB = Bytes.toBytes("rowB");
1928
1929 byte[] value = Bytes.toBytes("value");
1930
1931 Delete delete = new Delete(rowA);
1932 delete.deleteFamily(fam1);
1933
1934 region.delete(delete);
1935
1936
1937 Put put = new Put(rowA);
1938 put.add(fam2, null, value);
1939 region.put(put);
1940
1941 put = new Put(rowB);
1942 put.add(fam1, null, value);
1943 put.add(fam2, null, value);
1944 region.put(put);
1945
1946 Scan scan = new Scan();
1947 scan.addFamily(fam1).addFamily(fam2);
1948 InternalScanner s = region.getScanner(scan);
1949 List<Cell> results = new ArrayList<Cell>();
1950 s.next(results);
1951 assertTrue(CellUtil.matchingRow(results.get(0), rowA));
1952
1953 results.clear();
1954 s.next(results);
1955 assertTrue(CellUtil.matchingRow(results.get(0), rowB));
1956 } finally {
1957 HRegion.closeHRegion(this.region);
1958 this.region = null;
1959 }
1960 }
1961
1962 @Test
1963 public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
1964 Delete delete = new Delete(row);
1965 delete.deleteColumns(fam1, qual1);
1966 doTestDelete_AndPostInsert(delete);
1967 }
1968
1969 @Test
1970 public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
1971 Delete delete = new Delete(row);
1972 delete.deleteFamily(fam1);
1973 doTestDelete_AndPostInsert(delete);
1974 }
1975
1976 public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
1977 TableName tableName = TableName.valueOf(name.getMethodName());
1978 this.region = initHRegion(tableName, getName(), CONF, fam1);
1979 try {
1980 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1981 Put put = new Put(row);
1982 put.add(fam1, qual1, value1);
1983 region.put(put);
1984
1985
1986 region.delete(delete);
1987
1988
1989 put = new Put(row);
1990 put.add(fam1, qual1, value2);
1991 region.put(put);
1992
1993
1994 Get get = new Get(row);
1995 get.addColumn(fam1, qual1);
1996
1997 Result r = region.get(get);
1998 assertEquals(1, r.size());
1999 assertArrayEquals(value2, r.getValue(fam1, qual1));
2000
2001
2002 Scan scan = new Scan(row);
2003 scan.addColumn(fam1, qual1);
2004 InternalScanner s = region.getScanner(scan);
2005
2006 List<Cell> results = new ArrayList<Cell>();
2007 assertEquals(false, s.next(results));
2008 assertEquals(1, results.size());
2009 Cell kv = results.get(0);
2010
2011 assertArrayEquals(value2, CellUtil.cloneValue(kv));
2012 assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
2013 assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
2014 assertArrayEquals(row, CellUtil.cloneRow(kv));
2015 } finally {
2016 HRegion.closeHRegion(this.region);
2017 this.region = null;
2018 }
2019 }
2020
2021 @Test
2022 public void testDelete_CheckTimestampUpdated() throws IOException {
2023 TableName tableName = TableName.valueOf(name.getMethodName());
2024 byte[] row1 = Bytes.toBytes("row1");
2025 byte[] col1 = Bytes.toBytes("col1");
2026 byte[] col2 = Bytes.toBytes("col2");
2027 byte[] col3 = Bytes.toBytes("col3");
2028
2029
2030 String method = this.getName();
2031 this.region = initHRegion(tableName, method, CONF, fam1);
2032 try {
2033
2034 List<Cell> kvs = new ArrayList<Cell>();
2035 kvs.add(new KeyValue(row1, fam1, col1, null));
2036 kvs.add(new KeyValue(row1, fam1, col2, null));
2037 kvs.add(new KeyValue(row1, fam1, col3, null));
2038
2039 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
2040 Bytes.BYTES_COMPARATOR);
2041 deleteMap.put(fam1, kvs);
2042 region.delete(deleteMap, Durability.SYNC_WAL);
2043
2044
2045
2046 long now = System.currentTimeMillis();
2047 KeyValue firstKv = ((HStore) region.getStore(fam1)).memstore.kvset.first();
2048 assertTrue(firstKv.getTimestamp() <= now);
2049 now = firstKv.getTimestamp();
2050 for (KeyValue kv : ((HStore) region.getStore(fam1)).memstore.kvset) {
2051 assertTrue(kv.getTimestamp() <= now);
2052 now = kv.getTimestamp();
2053 }
2054 } finally {
2055 HRegion.closeHRegion(this.region);
2056 this.region = null;
2057 }
2058 }
2059
2060
2061
2062
2063 @Test
2064 public void testGet_FamilyChecker() throws IOException {
2065 byte[] row1 = Bytes.toBytes("row1");
2066 byte[] fam1 = Bytes.toBytes("fam1");
2067 byte[] fam2 = Bytes.toBytes("False");
2068 byte[] col1 = Bytes.toBytes("col1");
2069
2070
2071 String method = this.getName();
2072 this.region = initHRegion(tableName, method, CONF, fam1);
2073 try {
2074 Get get = new Get(row1);
2075 get.addColumn(fam2, col1);
2076
2077
2078 try {
2079 region.get(get);
2080 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
2081 assertFalse(false);
2082 return;
2083 }
2084 assertFalse(true);
2085 } finally {
2086 HRegion.closeHRegion(this.region);
2087 this.region = null;
2088 }
2089 }
2090
2091 @Test
2092 public void testGet_Basic() throws IOException {
2093 byte[] row1 = Bytes.toBytes("row1");
2094 byte[] fam1 = Bytes.toBytes("fam1");
2095 byte[] col1 = Bytes.toBytes("col1");
2096 byte[] col2 = Bytes.toBytes("col2");
2097 byte[] col3 = Bytes.toBytes("col3");
2098 byte[] col4 = Bytes.toBytes("col4");
2099 byte[] col5 = Bytes.toBytes("col5");
2100
2101
2102 String method = this.getName();
2103 this.region = initHRegion(tableName, method, CONF, fam1);
2104 try {
2105
2106 Put put = new Put(row1);
2107 put.add(fam1, col1, null);
2108 put.add(fam1, col2, null);
2109 put.add(fam1, col3, null);
2110 put.add(fam1, col4, null);
2111 put.add(fam1, col5, null);
2112 region.put(put);
2113
2114 Get get = new Get(row1);
2115 get.addColumn(fam1, col2);
2116 get.addColumn(fam1, col4);
2117
2118 KeyValue kv1 = new KeyValue(row1, fam1, col2);
2119 KeyValue kv2 = new KeyValue(row1, fam1, col4);
2120 KeyValue[] expected = { kv1, kv2 };
2121
2122
2123 Result res = region.get(get);
2124 assertEquals(expected.length, res.size());
2125 for (int i = 0; i < res.size(); i++) {
2126 assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
2127 assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
2128 assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
2129 }
2130
2131
2132 Get g = new Get(row1);
2133 final int count = 2;
2134 g.setFilter(new ColumnCountGetFilter(count));
2135 res = region.get(g);
2136 assertEquals(count, res.size());
2137 } finally {
2138 HRegion.closeHRegion(this.region);
2139 this.region = null;
2140 }
2141 }
2142
2143 @Test
2144 public void testGet_Empty() throws IOException {
2145 byte[] row = Bytes.toBytes("row");
2146 byte[] fam = Bytes.toBytes("fam");
2147
2148 String method = this.getName();
2149 this.region = initHRegion(tableName, method, CONF, fam);
2150 try {
2151 Get get = new Get(row);
2152 get.addFamily(fam);
2153 Result r = region.get(get);
2154
2155 assertTrue(r.isEmpty());
2156 } finally {
2157 HRegion.closeHRegion(this.region);
2158 this.region = null;
2159 }
2160 }
2161
2162
2163
2164
2165 @Test
2166 public void testMerge() throws IOException {
2167 byte[][] families = { fam1, fam2, fam3 };
2168 Configuration hc = initSplit();
2169
2170 String method = this.getName();
2171 this.region = initHRegion(tableName, method, hc, families);
2172 try {
2173 LOG.info("" + HBaseTestCase.addContent(region, fam3));
2174 region.flushcache();
2175 region.compactStores();
2176 byte[] splitRow = region.checkSplit();
2177 assertNotNull(splitRow);
2178 LOG.info("SplitRow: " + Bytes.toString(splitRow));
2179 HRegion[] subregions = splitRegion(region, splitRow);
2180 try {
2181
2182 for (int i = 0; i < subregions.length; i++) {
2183 HRegion.openHRegion(subregions[i], null);
2184 subregions[i].compactStores();
2185 }
2186 Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
2187 Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
2188 Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
2189 long startTime = System.currentTimeMillis();
2190 region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
2191 LOG.info("Merge regions elapsed time: "
2192 + ((System.currentTimeMillis() - startTime) / 1000.0));
2193 FILESYSTEM.delete(oldRegion1, true);
2194 FILESYSTEM.delete(oldRegion2, true);
2195 FILESYSTEM.delete(oldRegionPath, true);
2196 LOG.info("splitAndMerge completed.");
2197 } finally {
2198 for (int i = 0; i < subregions.length; i++) {
2199 try {
2200 HRegion.closeHRegion(subregions[i]);
2201 } catch (IOException e) {
2202
2203 }
2204 }
2205 }
2206 } finally {
2207 HRegion.closeHRegion(this.region);
2208 this.region = null;
2209 }
2210 }
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220 HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
2221 PairOfSameType<HRegion> result = null;
2222 SplitTransaction st = new SplitTransaction(parent, midkey);
2223
2224
2225 if (!st.prepare()) {
2226 parent.clearSplit();
2227 return null;
2228 }
2229 try {
2230 result = st.execute(null, null);
2231 } catch (IOException ioe) {
2232 try {
2233 LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; "
2234 + ioe.getMessage());
2235 st.rollback(null, null);
2236 LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString());
2237 return null;
2238 } catch (RuntimeException e) {
2239
2240 LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString()
2241 + " -- aborting server", e);
2242 }
2243 }
2244 finally {
2245 parent.clearSplit();
2246 }
2247 return new HRegion[] { result.getFirst(), result.getSecond() };
2248 }
2249
2250
2251
2252
2253 @Test
2254 public void testGetScanner_WithOkFamilies() throws IOException {
2255 byte[] fam1 = Bytes.toBytes("fam1");
2256 byte[] fam2 = Bytes.toBytes("fam2");
2257
2258 byte[][] families = { fam1, fam2 };
2259
2260
2261 String method = this.getName();
2262 this.region = initHRegion(tableName, method, CONF, families);
2263 try {
2264 Scan scan = new Scan();
2265 scan.addFamily(fam1);
2266 scan.addFamily(fam2);
2267 try {
2268 region.getScanner(scan);
2269 } catch (Exception e) {
2270 assertTrue("Families could not be found in Region", false);
2271 }
2272 } finally {
2273 HRegion.closeHRegion(this.region);
2274 this.region = null;
2275 }
2276 }
2277
2278 @Test
2279 public void testGetScanner_WithNotOkFamilies() throws IOException {
2280 byte[] fam1 = Bytes.toBytes("fam1");
2281 byte[] fam2 = Bytes.toBytes("fam2");
2282
2283 byte[][] families = { fam1 };
2284
2285
2286 String method = this.getName();
2287 this.region = initHRegion(tableName, method, CONF, families);
2288 try {
2289 Scan scan = new Scan();
2290 scan.addFamily(fam2);
2291 boolean ok = false;
2292 try {
2293 region.getScanner(scan);
2294 } catch (Exception e) {
2295 ok = true;
2296 }
2297 assertTrue("Families could not be found in Region", ok);
2298 } finally {
2299 HRegion.closeHRegion(this.region);
2300 this.region = null;
2301 }
2302 }
2303
2304 @Test
2305 public void testGetScanner_WithNoFamilies() throws IOException {
2306 byte[] row1 = Bytes.toBytes("row1");
2307 byte[] fam1 = Bytes.toBytes("fam1");
2308 byte[] fam2 = Bytes.toBytes("fam2");
2309 byte[] fam3 = Bytes.toBytes("fam3");
2310 byte[] fam4 = Bytes.toBytes("fam4");
2311
2312 byte[][] families = { fam1, fam2, fam3, fam4 };
2313
2314
2315 String method = this.getName();
2316 this.region = initHRegion(tableName, method, CONF, families);
2317 try {
2318
2319
2320 Put put = new Put(row1);
2321 put.add(fam1, null, null);
2322 put.add(fam2, null, null);
2323 put.add(fam3, null, null);
2324 put.add(fam4, null, null);
2325 region.put(put);
2326
2327 Scan scan = null;
2328 HRegion.RegionScannerImpl is = null;
2329
2330
2331
2332
2333 scan = new Scan();
2334 scan.addFamily(fam2);
2335 scan.addFamily(fam4);
2336 is = (RegionScannerImpl) region.getScanner(scan);
2337 assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2338
2339 scan = new Scan();
2340 is = (RegionScannerImpl) region.getScanner(scan);
2341 assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2342 } finally {
2343 HRegion.closeHRegion(this.region);
2344 this.region = null;
2345 }
2346 }
2347
2348
2349
2350
2351
2352
2353 @Test
2354 public void testGetScanner_WithRegionClosed() throws IOException {
2355 byte[] fam1 = Bytes.toBytes("fam1");
2356 byte[] fam2 = Bytes.toBytes("fam2");
2357
2358 byte[][] families = { fam1, fam2 };
2359
2360
2361 String method = this.getName();
2362 try {
2363 this.region = initHRegion(tableName, method, CONF, families);
2364 } catch (IOException e) {
2365 e.printStackTrace();
2366 fail("Got IOException during initHRegion, " + e.getMessage());
2367 }
2368 try {
2369 region.closed.set(true);
2370 try {
2371 region.getScanner(null);
2372 fail("Expected to get an exception during getScanner on a region that is closed");
2373 } catch (NotServingRegionException e) {
2374
2375 } catch (IOException e) {
2376 fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2377 + e.getMessage());
2378 }
2379 } finally {
2380 HRegion.closeHRegion(this.region);
2381 this.region = null;
2382 }
2383 }
2384
2385 @Test
2386 public void testRegionScanner_Next() throws IOException {
2387 byte[] row1 = Bytes.toBytes("row1");
2388 byte[] row2 = Bytes.toBytes("row2");
2389 byte[] fam1 = Bytes.toBytes("fam1");
2390 byte[] fam2 = Bytes.toBytes("fam2");
2391 byte[] fam3 = Bytes.toBytes("fam3");
2392 byte[] fam4 = Bytes.toBytes("fam4");
2393
2394 byte[][] families = { fam1, fam2, fam3, fam4 };
2395 long ts = System.currentTimeMillis();
2396
2397
2398 String method = this.getName();
2399 this.region = initHRegion(tableName, method, CONF, families);
2400 try {
2401
2402 Put put = null;
2403 put = new Put(row1);
2404 put.add(fam1, (byte[]) null, ts, null);
2405 put.add(fam2, (byte[]) null, ts, null);
2406 put.add(fam3, (byte[]) null, ts, null);
2407 put.add(fam4, (byte[]) null, ts, null);
2408 region.put(put);
2409
2410 put = new Put(row2);
2411 put.add(fam1, (byte[]) null, ts, null);
2412 put.add(fam2, (byte[]) null, ts, null);
2413 put.add(fam3, (byte[]) null, ts, null);
2414 put.add(fam4, (byte[]) null, ts, null);
2415 region.put(put);
2416
2417 Scan scan = new Scan();
2418 scan.addFamily(fam2);
2419 scan.addFamily(fam4);
2420 InternalScanner is = region.getScanner(scan);
2421
2422 List<Cell> res = null;
2423
2424
2425 List<Cell> expected1 = new ArrayList<Cell>();
2426 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2427 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2428
2429 res = new ArrayList<Cell>();
2430 is.next(res);
2431 for (int i = 0; i < res.size(); i++) {
2432 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2433 }
2434
2435
2436 List<Cell> expected2 = new ArrayList<Cell>();
2437 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2438 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2439
2440 res = new ArrayList<Cell>();
2441 is.next(res);
2442 for (int i = 0; i < res.size(); i++) {
2443 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2444 }
2445 } finally {
2446 HRegion.closeHRegion(this.region);
2447 this.region = null;
2448 }
2449 }
2450
2451 @Test
2452 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2453 byte[] row1 = Bytes.toBytes("row1");
2454 byte[] qf1 = Bytes.toBytes("qualifier1");
2455 byte[] qf2 = Bytes.toBytes("qualifier2");
2456 byte[] fam1 = Bytes.toBytes("fam1");
2457 byte[][] families = { fam1 };
2458
2459 long ts1 = System.currentTimeMillis();
2460 long ts2 = ts1 + 1;
2461 long ts3 = ts1 + 2;
2462
2463
2464 String method = this.getName();
2465 this.region = initHRegion(tableName, method, CONF, families);
2466 try {
2467
2468 Put put = null;
2469 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2470 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2471 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2472
2473 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2474 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2475 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2476
2477 put = new Put(row1);
2478 put.add(kv13);
2479 put.add(kv12);
2480 put.add(kv11);
2481 put.add(kv23);
2482 put.add(kv22);
2483 put.add(kv21);
2484 region.put(put);
2485
2486
2487 List<Cell> expected = new ArrayList<Cell>();
2488 expected.add(kv13);
2489 expected.add(kv12);
2490
2491 Scan scan = new Scan(row1);
2492 scan.addColumn(fam1, qf1);
2493 scan.setMaxVersions(MAX_VERSIONS);
2494 List<Cell> actual = new ArrayList<Cell>();
2495 InternalScanner scanner = region.getScanner(scan);
2496
2497 boolean hasNext = scanner.next(actual);
2498 assertEquals(false, hasNext);
2499
2500
2501 for (int i = 0; i < expected.size(); i++) {
2502 assertEquals(expected.get(i), actual.get(i));
2503 }
2504 } finally {
2505 HRegion.closeHRegion(this.region);
2506 this.region = null;
2507 }
2508 }
2509
2510 @Test
2511 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2512 byte[] row1 = Bytes.toBytes("row1");
2513 byte[] qf1 = Bytes.toBytes("qualifier1");
2514 byte[] qf2 = Bytes.toBytes("qualifier2");
2515 byte[] fam1 = Bytes.toBytes("fam1");
2516 byte[][] families = { fam1 };
2517
2518 long ts1 = 1;
2519 long ts2 = ts1 + 1;
2520 long ts3 = ts1 + 2;
2521
2522
2523 String method = this.getName();
2524 this.region = initHRegion(tableName, method, CONF, families);
2525 try {
2526
2527 Put put = null;
2528 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2529 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2530 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2531
2532 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2533 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2534 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2535
2536 put = new Put(row1);
2537 put.add(kv13);
2538 put.add(kv12);
2539 put.add(kv11);
2540 put.add(kv23);
2541 put.add(kv22);
2542 put.add(kv21);
2543 region.put(put);
2544 region.flushcache();
2545
2546
2547 List<Cell> expected = new ArrayList<Cell>();
2548 expected.add(kv13);
2549 expected.add(kv12);
2550 expected.add(kv23);
2551 expected.add(kv22);
2552
2553 Scan scan = new Scan(row1);
2554 scan.addColumn(fam1, qf1);
2555 scan.addColumn(fam1, qf2);
2556 scan.setMaxVersions(MAX_VERSIONS);
2557 List<Cell> actual = new ArrayList<Cell>();
2558 InternalScanner scanner = region.getScanner(scan);
2559
2560 boolean hasNext = scanner.next(actual);
2561 assertEquals(false, hasNext);
2562
2563
2564 for (int i = 0; i < expected.size(); i++) {
2565 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2566 }
2567 } finally {
2568 HRegion.closeHRegion(this.region);
2569 this.region = null;
2570 }
2571 }
2572
2573 @Test
2574 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2575 byte[] row1 = Bytes.toBytes("row1");
2576 byte[] fam1 = Bytes.toBytes("fam1");
2577 byte[][] families = { fam1 };
2578 byte[] qf1 = Bytes.toBytes("qualifier1");
2579 byte[] qf2 = Bytes.toBytes("qualifier2");
2580
2581 long ts1 = 1;
2582 long ts2 = ts1 + 1;
2583 long ts3 = ts1 + 2;
2584 long ts4 = ts1 + 3;
2585
2586
2587 String method = this.getName();
2588 this.region = initHRegion(tableName, method, CONF, families);
2589 try {
2590
2591 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2592 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2593 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2594 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2595
2596 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2597 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2598 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2599 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2600
2601 Put put = null;
2602 put = new Put(row1);
2603 put.add(kv14);
2604 put.add(kv24);
2605 region.put(put);
2606 region.flushcache();
2607
2608 put = new Put(row1);
2609 put.add(kv23);
2610 put.add(kv13);
2611 region.put(put);
2612 region.flushcache();
2613
2614 put = new Put(row1);
2615 put.add(kv22);
2616 put.add(kv12);
2617 region.put(put);
2618 region.flushcache();
2619
2620 put = new Put(row1);
2621 put.add(kv21);
2622 put.add(kv11);
2623 region.put(put);
2624
2625
2626 List<Cell> expected = new ArrayList<Cell>();
2627 expected.add(kv14);
2628 expected.add(kv13);
2629 expected.add(kv12);
2630 expected.add(kv24);
2631 expected.add(kv23);
2632 expected.add(kv22);
2633
2634 Scan scan = new Scan(row1);
2635 scan.addColumn(fam1, qf1);
2636 scan.addColumn(fam1, qf2);
2637 int versions = 3;
2638 scan.setMaxVersions(versions);
2639 List<Cell> actual = new ArrayList<Cell>();
2640 InternalScanner scanner = region.getScanner(scan);
2641
2642 boolean hasNext = scanner.next(actual);
2643 assertEquals(false, hasNext);
2644
2645
2646 for (int i = 0; i < expected.size(); i++) {
2647 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2648 }
2649 } finally {
2650 HRegion.closeHRegion(this.region);
2651 this.region = null;
2652 }
2653 }
2654
2655 @Test
2656 public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2657 byte[] row1 = Bytes.toBytes("row1");
2658 byte[] qf1 = Bytes.toBytes("qualifier1");
2659 byte[] qf2 = Bytes.toBytes("qualifier2");
2660 byte[] fam1 = Bytes.toBytes("fam1");
2661 byte[][] families = { fam1 };
2662
2663 long ts1 = System.currentTimeMillis();
2664 long ts2 = ts1 + 1;
2665 long ts3 = ts1 + 2;
2666
2667
2668 String method = this.getName();
2669 this.region = initHRegion(tableName, method, CONF, families);
2670 try {
2671
2672 Put put = null;
2673 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2674 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2675 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2676
2677 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2678 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2679 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2680
2681 put = new Put(row1);
2682 put.add(kv13);
2683 put.add(kv12);
2684 put.add(kv11);
2685 put.add(kv23);
2686 put.add(kv22);
2687 put.add(kv21);
2688 region.put(put);
2689
2690
2691 List<Cell> expected = new ArrayList<Cell>();
2692 expected.add(kv13);
2693 expected.add(kv12);
2694 expected.add(kv23);
2695 expected.add(kv22);
2696
2697 Scan scan = new Scan(row1);
2698 scan.addFamily(fam1);
2699 scan.setMaxVersions(MAX_VERSIONS);
2700 List<Cell> actual = new ArrayList<Cell>();
2701 InternalScanner scanner = region.getScanner(scan);
2702
2703 boolean hasNext = scanner.next(actual);
2704 assertEquals(false, hasNext);
2705
2706
2707 for (int i = 0; i < expected.size(); i++) {
2708 assertEquals(expected.get(i), actual.get(i));
2709 }
2710 } finally {
2711 HRegion.closeHRegion(this.region);
2712 this.region = null;
2713 }
2714 }
2715
2716 @Test
2717 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
2718 byte[] row1 = Bytes.toBytes("row1");
2719 byte[] qf1 = Bytes.toBytes("qualifier1");
2720 byte[] qf2 = Bytes.toBytes("qualifier2");
2721 byte[] fam1 = Bytes.toBytes("fam1");
2722
2723 long ts1 = 1;
2724 long ts2 = ts1 + 1;
2725 long ts3 = ts1 + 2;
2726
2727
2728 String method = this.getName();
2729 this.region = initHRegion(tableName, method, CONF, fam1);
2730 try {
2731
2732 Put put = null;
2733 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2734 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2735 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2736
2737 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2738 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2739 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2740
2741 put = new Put(row1);
2742 put.add(kv13);
2743 put.add(kv12);
2744 put.add(kv11);
2745 put.add(kv23);
2746 put.add(kv22);
2747 put.add(kv21);
2748 region.put(put);
2749 region.flushcache();
2750
2751
2752 List<Cell> expected = new ArrayList<Cell>();
2753 expected.add(kv13);
2754 expected.add(kv12);
2755 expected.add(kv23);
2756 expected.add(kv22);
2757
2758 Scan scan = new Scan(row1);
2759 scan.addFamily(fam1);
2760 scan.setMaxVersions(MAX_VERSIONS);
2761 List<Cell> actual = new ArrayList<Cell>();
2762 InternalScanner scanner = region.getScanner(scan);
2763
2764 boolean hasNext = scanner.next(actual);
2765 assertEquals(false, hasNext);
2766
2767
2768 for (int i = 0; i < expected.size(); i++) {
2769 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2770 }
2771 } finally {
2772 HRegion.closeHRegion(this.region);
2773 this.region = null;
2774 }
2775 }
2776
2777 @Test
2778 public void testScanner_StopRow1542() throws IOException {
2779 byte[] family = Bytes.toBytes("testFamily");
2780 this.region = initHRegion(tableName, getName(), CONF, family);
2781 try {
2782 byte[] row1 = Bytes.toBytes("row111");
2783 byte[] row2 = Bytes.toBytes("row222");
2784 byte[] row3 = Bytes.toBytes("row333");
2785 byte[] row4 = Bytes.toBytes("row444");
2786 byte[] row5 = Bytes.toBytes("row555");
2787
2788 byte[] col1 = Bytes.toBytes("Pub111");
2789 byte[] col2 = Bytes.toBytes("Pub222");
2790
2791 Put put = new Put(row1);
2792 put.add(family, col1, Bytes.toBytes(10L));
2793 region.put(put);
2794
2795 put = new Put(row2);
2796 put.add(family, col1, Bytes.toBytes(15L));
2797 region.put(put);
2798
2799 put = new Put(row3);
2800 put.add(family, col2, Bytes.toBytes(20L));
2801 region.put(put);
2802
2803 put = new Put(row4);
2804 put.add(family, col2, Bytes.toBytes(30L));
2805 region.put(put);
2806
2807 put = new Put(row5);
2808 put.add(family, col1, Bytes.toBytes(40L));
2809 region.put(put);
2810
2811 Scan scan = new Scan(row3, row4);
2812 scan.setMaxVersions();
2813 scan.addColumn(family, col1);
2814 InternalScanner s = region.getScanner(scan);
2815
2816 List<Cell> results = new ArrayList<Cell>();
2817 assertEquals(false, s.next(results));
2818 assertEquals(0, results.size());
2819 } finally {
2820 HRegion.closeHRegion(this.region);
2821 this.region = null;
2822 }
2823 }
2824
2825 @Test
2826 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2827 byte[] row1 = Bytes.toBytes("row1");
2828 byte[] fam1 = Bytes.toBytes("fam1");
2829 byte[] qf1 = Bytes.toBytes("qualifier1");
2830 byte[] qf2 = Bytes.toBytes("quateslifier2");
2831
2832 long ts1 = 1;
2833 long ts2 = ts1 + 1;
2834 long ts3 = ts1 + 2;
2835 long ts4 = ts1 + 3;
2836
2837
2838 String method = this.getName();
2839 this.region = initHRegion(tableName, method, CONF, fam1);
2840 try {
2841
2842 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2843 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2844 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2845 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2846
2847 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2848 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2849 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2850 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2851
2852 Put put = null;
2853 put = new Put(row1);
2854 put.add(kv14);
2855 put.add(kv24);
2856 region.put(put);
2857 region.flushcache();
2858
2859 put = new Put(row1);
2860 put.add(kv23);
2861 put.add(kv13);
2862 region.put(put);
2863 region.flushcache();
2864
2865 put = new Put(row1);
2866 put.add(kv22);
2867 put.add(kv12);
2868 region.put(put);
2869 region.flushcache();
2870
2871 put = new Put(row1);
2872 put.add(kv21);
2873 put.add(kv11);
2874 region.put(put);
2875
2876
2877 List<KeyValue> expected = new ArrayList<KeyValue>();
2878 expected.add(kv14);
2879 expected.add(kv13);
2880 expected.add(kv12);
2881 expected.add(kv24);
2882 expected.add(kv23);
2883 expected.add(kv22);
2884
2885 Scan scan = new Scan(row1);
2886 int versions = 3;
2887 scan.setMaxVersions(versions);
2888 List<Cell> actual = new ArrayList<Cell>();
2889 InternalScanner scanner = region.getScanner(scan);
2890
2891 boolean hasNext = scanner.next(actual);
2892 assertEquals(false, hasNext);
2893
2894
2895 for (int i = 0; i < expected.size(); i++) {
2896 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2897 }
2898 } finally {
2899 HRegion.closeHRegion(this.region);
2900 this.region = null;
2901 }
2902 }
2903
2904
2905
2906
2907
2908
2909
2910 @Test
2911 public void testScanner_JoinedScanners() throws IOException {
2912 byte[] cf_essential = Bytes.toBytes("essential");
2913 byte[] cf_joined = Bytes.toBytes("joined");
2914 byte[] cf_alpha = Bytes.toBytes("alpha");
2915 this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
2916 try {
2917 byte[] row1 = Bytes.toBytes("row1");
2918 byte[] row2 = Bytes.toBytes("row2");
2919 byte[] row3 = Bytes.toBytes("row3");
2920
2921 byte[] col_normal = Bytes.toBytes("d");
2922 byte[] col_alpha = Bytes.toBytes("a");
2923
2924 byte[] filtered_val = Bytes.toBytes(3);
2925
2926 Put put = new Put(row1);
2927 put.add(cf_essential, col_normal, Bytes.toBytes(1));
2928 put.add(cf_joined, col_alpha, Bytes.toBytes(1));
2929 region.put(put);
2930
2931 put = new Put(row2);
2932 put.add(cf_essential, col_alpha, Bytes.toBytes(2));
2933 put.add(cf_joined, col_normal, Bytes.toBytes(2));
2934 put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
2935 region.put(put);
2936
2937 put = new Put(row3);
2938 put.add(cf_essential, col_normal, filtered_val);
2939 put.add(cf_joined, col_normal, filtered_val);
2940 region.put(put);
2941
2942
2943
2944
2945
2946 Scan scan = new Scan();
2947 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
2948 CompareOp.NOT_EQUAL, filtered_val);
2949 scan.setFilter(filter);
2950 scan.setLoadColumnFamiliesOnDemand(true);
2951 InternalScanner s = region.getScanner(scan);
2952
2953 List<Cell> results = new ArrayList<Cell>();
2954 assertTrue(s.next(results));
2955 assertEquals(results.size(), 1);
2956 results.clear();
2957
2958 assertTrue(s.next(results));
2959 assertEquals(results.size(), 3);
2960 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
2961 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
2962 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
2963 results.clear();
2964
2965 assertFalse(s.next(results));
2966 assertEquals(results.size(), 0);
2967 } finally {
2968 HRegion.closeHRegion(this.region);
2969 this.region = null;
2970 }
2971 }
2972
2973
2974
2975
2976
2977
2978 @Test
2979 public void testScanner_JoinedScannersWithLimits() throws IOException {
2980 final byte[] cf_first = Bytes.toBytes("first");
2981 final byte[] cf_second = Bytes.toBytes("second");
2982
2983 this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
2984 try {
2985 final byte[] col_a = Bytes.toBytes("a");
2986 final byte[] col_b = Bytes.toBytes("b");
2987
2988 Put put;
2989
2990 for (int i = 0; i < 10; i++) {
2991 put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
2992 put.add(cf_first, col_a, Bytes.toBytes(i));
2993 if (i < 5) {
2994 put.add(cf_first, col_b, Bytes.toBytes(i));
2995 put.add(cf_second, col_a, Bytes.toBytes(i));
2996 put.add(cf_second, col_b, Bytes.toBytes(i));
2997 }
2998 region.put(put);
2999 }
3000
3001 Scan scan = new Scan();
3002 scan.setLoadColumnFamiliesOnDemand(true);
3003 Filter bogusFilter = new FilterBase() {
3004 @Override
3005 public boolean isFamilyEssential(byte[] name) {
3006 return Bytes.equals(name, cf_first);
3007 }
3008 };
3009
3010 scan.setFilter(bogusFilter);
3011 InternalScanner s = region.getScanner(scan);
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042 List<Cell> results = new ArrayList<Cell>();
3043 int index = 0;
3044 while (true) {
3045 boolean more = s.next(results, 3);
3046 if ((index >> 1) < 5) {
3047 if (index % 2 == 0)
3048 assertEquals(results.size(), 3);
3049 else
3050 assertEquals(results.size(), 1);
3051 } else
3052 assertEquals(results.size(), 1);
3053 results.clear();
3054 index++;
3055 if (!more)
3056 break;
3057 }
3058 } finally {
3059 HRegion.closeHRegion(this.region);
3060 this.region = null;
3061 }
3062 }
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072 @Test
3073 public void testBasicSplit() throws Exception {
3074 byte[][] families = { fam1, fam2, fam3 };
3075
3076 Configuration hc = initSplit();
3077
3078 String method = this.getName();
3079 this.region = initHRegion(tableName, method, hc, families);
3080
3081 try {
3082 LOG.info("" + HBaseTestCase.addContent(region, fam3));
3083 region.flushcache();
3084 region.compactStores();
3085 byte[] splitRow = region.checkSplit();
3086 assertNotNull(splitRow);
3087 LOG.info("SplitRow: " + Bytes.toString(splitRow));
3088 HRegion[] regions = splitRegion(region, splitRow);
3089 try {
3090
3091
3092
3093 for (int i = 0; i < regions.length; i++) {
3094 regions[i] = HRegion.openHRegion(regions[i], null);
3095 }
3096
3097
3098 assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
3099 assertGet(regions[1], fam3, splitRow);
3100
3101 assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
3102 assertScan(regions[1], fam3, splitRow);
3103
3104 for (int i = 0; i < regions.length; i++) {
3105
3106
3107 for (int j = 0; j < 2; j++) {
3108 HBaseTestCase.addContent(regions[i], fam3);
3109 }
3110 HBaseTestCase.addContent(regions[i], fam2);
3111 HBaseTestCase.addContent(regions[i], fam1);
3112 regions[i].flushcache();
3113 }
3114
3115 byte[][] midkeys = new byte[regions.length][];
3116
3117 for (int i = 0; i < regions.length; i++) {
3118 regions[i].compactStores();
3119 midkeys[i] = regions[i].checkSplit();
3120 }
3121
3122 TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
3123
3124
3125 for (int i = 0; i < regions.length; i++) {
3126 HRegion[] rs = null;
3127 if (midkeys[i] != null) {
3128 rs = splitRegion(regions[i], midkeys[i]);
3129 for (int j = 0; j < rs.length; j++) {
3130 sortedMap.put(Bytes.toString(rs[j].getRegionName()), HRegion.openHRegion(rs[j], null));
3131 }
3132 }
3133 }
3134 LOG.info("Made 4 regions");
3135
3136
3137 int interval = (LAST_CHAR - FIRST_CHAR) / 3;
3138 byte[] b = Bytes.toBytes(START_KEY);
3139 for (HRegion r : sortedMap.values()) {
3140 assertGet(r, fam3, b);
3141 b[0] += interval;
3142 }
3143 } finally {
3144 for (int i = 0; i < regions.length; i++) {
3145 try {
3146 regions[i].close();
3147 } catch (IOException e) {
3148
3149 }
3150 }
3151 }
3152 } finally {
3153 HRegion.closeHRegion(this.region);
3154 this.region = null;
3155 }
3156 }
3157
3158 @Test
3159 public void testSplitRegion() throws IOException {
3160 byte[] qualifier = Bytes.toBytes("qualifier");
3161 Configuration hc = initSplit();
3162 int numRows = 10;
3163 byte[][] families = { fam1, fam3 };
3164
3165
3166 String method = this.getName();
3167 this.region = initHRegion(tableName, method, hc, families);
3168
3169
3170 int startRow = 100;
3171 putData(startRow, numRows, qualifier, families);
3172 int splitRow = startRow + numRows;
3173 putData(splitRow, numRows, qualifier, families);
3174 region.flushcache();
3175
3176 HRegion[] regions = null;
3177 try {
3178 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
3179
3180 for (int i = 0; i < regions.length; i++) {
3181 regions[i] = HRegion.openHRegion(regions[i], null);
3182 }
3183
3184 assertEquals(2, regions.length);
3185
3186
3187
3188 verifyData(regions[0], startRow, numRows, qualifier, families);
3189 verifyData(regions[1], splitRow, numRows, qualifier, families);
3190
3191 } finally {
3192 HRegion.closeHRegion(this.region);
3193 this.region = null;
3194 }
3195 }
3196
3197 @Test
3198 public void testClearForceSplit() throws IOException {
3199 byte[] qualifier = Bytes.toBytes("qualifier");
3200 Configuration hc = initSplit();
3201 int numRows = 10;
3202 byte[][] families = { fam1, fam3 };
3203
3204
3205 String method = this.getName();
3206 this.region = initHRegion(tableName, method, hc, families);
3207
3208
3209 int startRow = 100;
3210 putData(startRow, numRows, qualifier, families);
3211 int splitRow = startRow + numRows;
3212 byte[] splitRowBytes = Bytes.toBytes("" + splitRow);
3213 putData(splitRow, numRows, qualifier, families);
3214 region.flushcache();
3215
3216 HRegion[] regions = null;
3217 try {
3218
3219 region.forceSplit(splitRowBytes);
3220 assertTrue(region.shouldForceSplit());
3221
3222 assertTrue(Bytes.equals(splitRowBytes, region.checkSplit()));
3223
3224
3225 HStore storeMock = Mockito.mock(HStore.class);
3226 when(storeMock.hasReferences()).thenReturn(true);
3227 when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
3228 when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
3229 when(storeMock.getColumnFamilyName()).thenReturn("cf");
3230 region.stores.put(Bytes.toBytes(storeMock.getColumnFamilyName()), storeMock);
3231 assertTrue(region.hasReferences());
3232
3233
3234 regions = splitRegion(region, splitRowBytes);
3235 assertNull(regions);
3236
3237
3238 assertFalse(region.shouldForceSplit());
3239
3240
3241 region.stores.remove(Bytes.toBytes(storeMock.getColumnFamilyName()));
3242 assertFalse(region.hasReferences());
3243
3244
3245 regions = splitRegion(region, splitRowBytes);
3246
3247
3248 for (int i = 0; i < regions.length; i++) {
3249 regions[i] = HRegion.openHRegion(regions[i], null);
3250 }
3251
3252 assertEquals(2, regions.length);
3253
3254
3255
3256 verifyData(regions[0], startRow, numRows, qualifier, families);
3257 verifyData(regions[1], splitRow, numRows, qualifier, families);
3258
3259 } finally {
3260 HRegion.closeHRegion(this.region);
3261 this.region = null;
3262 }
3263 }
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275 @Test
3276 public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3277 byte[] family = Bytes.toBytes("family");
3278 int numRows = 1000;
3279 int flushAndScanInterval = 10;
3280 int compactInterval = 10 * flushAndScanInterval;
3281
3282 String method = "testFlushCacheWhileScanning";
3283 this.region = initHRegion(tableName, method, CONF, family);
3284 try {
3285 FlushThread flushThread = new FlushThread();
3286 flushThread.start();
3287
3288 Scan scan = new Scan();
3289 scan.addFamily(family);
3290 scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
3291 new BinaryComparator(Bytes.toBytes(5L))));
3292
3293 int expectedCount = 0;
3294 List<Cell> res = new ArrayList<Cell>();
3295
3296 boolean toggle = true;
3297 for (long i = 0; i < numRows; i++) {
3298 Put put = new Put(Bytes.toBytes(i));
3299 put.setDurability(Durability.SKIP_WAL);
3300 put.add(family, qual1, Bytes.toBytes(i % 10));
3301 region.put(put);
3302
3303 if (i != 0 && i % compactInterval == 0) {
3304
3305 region.compactStores(true);
3306 }
3307
3308 if (i % 10 == 5L) {
3309 expectedCount++;
3310 }
3311
3312 if (i != 0 && i % flushAndScanInterval == 0) {
3313 res.clear();
3314 InternalScanner scanner = region.getScanner(scan);
3315 if (toggle) {
3316 flushThread.flush();
3317 }
3318 while (scanner.next(res))
3319 ;
3320 if (!toggle) {
3321 flushThread.flush();
3322 }
3323 assertEquals("i=" + i, expectedCount, res.size());
3324 toggle = !toggle;
3325 }
3326 }
3327
3328 flushThread.done();
3329 flushThread.join();
3330 flushThread.checkNoError();
3331 } finally {
3332 HRegion.closeHRegion(this.region);
3333 this.region = null;
3334 }
3335 }
3336
3337 protected class FlushThread extends Thread {
3338 private volatile boolean done;
3339 private Throwable error = null;
3340
3341 public void done() {
3342 done = true;
3343 synchronized (this) {
3344 interrupt();
3345 }
3346 }
3347
3348 public void checkNoError() {
3349 if (error != null) {
3350 assertNull(error);
3351 }
3352 }
3353
3354 @Override
3355 public void run() {
3356 done = false;
3357 while (!done) {
3358 synchronized (this) {
3359 try {
3360 wait();
3361 } catch (InterruptedException ignored) {
3362 if (done) {
3363 break;
3364 }
3365 }
3366 }
3367 try {
3368 region.flushcache();
3369 } catch (IOException e) {
3370 if (!done) {
3371 LOG.error("Error while flusing cache", e);
3372 error = e;
3373 }
3374 break;
3375 }
3376 }
3377
3378 }
3379
3380 public void flush() {
3381 synchronized (this) {
3382 notify();
3383 }
3384
3385 }
3386 }
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397 @Test
3398 public void testWritesWhileScanning() throws IOException, InterruptedException {
3399 int testCount = 100;
3400 int numRows = 1;
3401 int numFamilies = 10;
3402 int numQualifiers = 100;
3403 int flushInterval = 7;
3404 int compactInterval = 5 * flushInterval;
3405 byte[][] families = new byte[numFamilies][];
3406 for (int i = 0; i < numFamilies; i++) {
3407 families[i] = Bytes.toBytes("family" + i);
3408 }
3409 byte[][] qualifiers = new byte[numQualifiers][];
3410 for (int i = 0; i < numQualifiers; i++) {
3411 qualifiers[i] = Bytes.toBytes("qual" + i);
3412 }
3413
3414 String method = "testWritesWhileScanning";
3415 this.region = initHRegion(tableName, method, CONF, families);
3416 try {
3417 PutThread putThread = new PutThread(numRows, families, qualifiers);
3418 putThread.start();
3419 putThread.waitForFirstPut();
3420
3421 FlushThread flushThread = new FlushThread();
3422 flushThread.start();
3423
3424 Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3425
3426 int expectedCount = numFamilies * numQualifiers;
3427 List<Cell> res = new ArrayList<Cell>();
3428
3429 long prevTimestamp = 0L;
3430 for (int i = 0; i < testCount; i++) {
3431
3432 if (i != 0 && i % compactInterval == 0) {
3433 region.compactStores(true);
3434 }
3435
3436 if (i != 0 && i % flushInterval == 0) {
3437 flushThread.flush();
3438 }
3439
3440 boolean previousEmpty = res.isEmpty();
3441 res.clear();
3442 InternalScanner scanner = region.getScanner(scan);
3443 while (scanner.next(res))
3444 ;
3445 if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3446 assertEquals("i=" + i, expectedCount, res.size());
3447 long timestamp = res.get(0).getTimestamp();
3448 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3449 timestamp >= prevTimestamp);
3450 prevTimestamp = timestamp;
3451 }
3452 }
3453
3454 putThread.done();
3455
3456 region.flushcache();
3457
3458 putThread.join();
3459 putThread.checkNoError();
3460
3461 flushThread.done();
3462 flushThread.join();
3463 flushThread.checkNoError();
3464 } finally {
3465 try {
3466 HRegion.closeHRegion(this.region);
3467 } catch (DroppedSnapshotException dse) {
3468
3469
3470
3471
3472 }
3473 this.region = null;
3474 }
3475 }
3476
3477 protected class PutThread extends Thread {
3478 private volatile boolean done;
3479 private volatile int numPutsFinished = 0;
3480
3481 private Throwable error = null;
3482 private int numRows;
3483 private byte[][] families;
3484 private byte[][] qualifiers;
3485
3486 private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3487 this.numRows = numRows;
3488 this.families = families;
3489 this.qualifiers = qualifiers;
3490 }
3491
3492
3493
3494
3495 public void waitForFirstPut() throws InterruptedException {
3496
3497 while (numPutsFinished == 0) {
3498 checkNoError();
3499 Thread.sleep(50);
3500 }
3501 }
3502
3503 public void done() {
3504 done = true;
3505 synchronized (this) {
3506 interrupt();
3507 }
3508 }
3509
3510 public void checkNoError() {
3511 if (error != null) {
3512 assertNull(error);
3513 }
3514 }
3515
3516 @Override
3517 public void run() {
3518 done = false;
3519 while (!done) {
3520 try {
3521 for (int r = 0; r < numRows; r++) {
3522 byte[] row = Bytes.toBytes("row" + r);
3523 Put put = new Put(row);
3524 put.setDurability(Durability.SKIP_WAL);
3525 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3526 for (byte[] family : families) {
3527 for (byte[] qualifier : qualifiers) {
3528 put.add(family, qualifier, (long) numPutsFinished, value);
3529 }
3530 }
3531 region.put(put);
3532 numPutsFinished++;
3533 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3534 System.out.println("put iteration = " + numPutsFinished);
3535 Delete delete = new Delete(row, (long) numPutsFinished - 30);
3536 region.delete(delete);
3537 }
3538 numPutsFinished++;
3539 }
3540 } catch (InterruptedIOException e) {
3541
3542 } catch (IOException e) {
3543 LOG.error("error while putting records", e);
3544 error = e;
3545 break;
3546 }
3547 }
3548
3549 }
3550
3551 }
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562 @Test
3563 public void testWritesWhileGetting() throws Exception {
3564 int testCount = 100;
3565 int numRows = 1;
3566 int numFamilies = 10;
3567 int numQualifiers = 100;
3568 int compactInterval = 100;
3569 byte[][] families = new byte[numFamilies][];
3570 for (int i = 0; i < numFamilies; i++) {
3571 families[i] = Bytes.toBytes("family" + i);
3572 }
3573 byte[][] qualifiers = new byte[numQualifiers][];
3574 for (int i = 0; i < numQualifiers; i++) {
3575 qualifiers[i] = Bytes.toBytes("qual" + i);
3576 }
3577
3578
3579 String method = "testWritesWhileGetting";
3580
3581
3582
3583
3584
3585 Configuration conf = HBaseConfiguration.create(CONF);
3586 conf.setInt("hbase.hstore.compaction.min", 1);
3587 conf.setInt("hbase.hstore.compaction.max", 1000);
3588 this.region = initHRegion(tableName, method, conf, families);
3589 PutThread putThread = null;
3590 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3591 try {
3592 putThread = new PutThread(numRows, families, qualifiers);
3593 putThread.start();
3594 putThread.waitForFirstPut();
3595
3596
3597 ctx.addThread(new RepeatingTestThread(ctx) {
3598 private int flushesSinceCompact = 0;
3599 private final int maxFlushesSinceCompact = 20;
3600
3601 @Override
3602 public void doAnAction() throws Exception {
3603 if (region.flushcache().isCompactionNeeded()) {
3604 ++flushesSinceCompact;
3605 }
3606
3607
3608 if (flushesSinceCompact == maxFlushesSinceCompact) {
3609 region.compactStores(false);
3610 flushesSinceCompact = 0;
3611 }
3612 }
3613 });
3614 ctx.startThreads();
3615
3616 Get get = new Get(Bytes.toBytes("row0"));
3617 Result result = null;
3618
3619 int expectedCount = numFamilies * numQualifiers;
3620
3621 long prevTimestamp = 0L;
3622 for (int i = 0; i < testCount; i++) {
3623
3624 boolean previousEmpty = result == null || result.isEmpty();
3625 result = region.get(get);
3626 if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3627 assertEquals("i=" + i, expectedCount, result.size());
3628
3629
3630 long timestamp = 0;
3631 for (Cell kv : result.rawCells()) {
3632 if (CellUtil.matchingFamily(kv, families[0])
3633 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3634 timestamp = kv.getTimestamp();
3635 }
3636 }
3637 assertTrue(timestamp >= prevTimestamp);
3638 prevTimestamp = timestamp;
3639 Cell previousKV = null;
3640
3641 for (Cell kv : result.rawCells()) {
3642 byte[] thisValue = CellUtil.cloneValue(kv);
3643 if (previousKV != null) {
3644 if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3645 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3646 + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3647 + "(memStoreTS:" + kv.getMvccVersion() + ")");
3648 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3649 }
3650 }
3651 previousKV = kv;
3652 }
3653 }
3654 }
3655 } finally {
3656 if (putThread != null)
3657 putThread.done();
3658
3659 region.flushcache();
3660
3661 if (putThread != null) {
3662 putThread.join();
3663 putThread.checkNoError();
3664 }
3665
3666 ctx.stop();
3667 HRegion.closeHRegion(this.region);
3668 this.region = null;
3669 }
3670 }
3671
3672 @Test
3673 public void testHolesInMeta() throws Exception {
3674 byte[] family = Bytes.toBytes("family");
3675 this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF,
3676 false, family);
3677 try {
3678 byte[] rowNotServed = Bytes.toBytes("a");
3679 Get g = new Get(rowNotServed);
3680 try {
3681 region.get(g);
3682 fail();
3683 } catch (WrongRegionException x) {
3684
3685 }
3686 byte[] row = Bytes.toBytes("y");
3687 g = new Get(row);
3688 region.get(g);
3689 } finally {
3690 HRegion.closeHRegion(this.region);
3691 this.region = null;
3692 }
3693 }
3694
3695 @Test
3696 public void testIndexesScanWithOneDeletedRow() throws IOException {
3697 byte[] family = Bytes.toBytes("family");
3698
3699
3700 String method = "testIndexesScanWithOneDeletedRow";
3701 this.region = initHRegion(tableName, method, CONF, family);
3702 try {
3703 Put put = new Put(Bytes.toBytes(1L));
3704 put.add(family, qual1, 1L, Bytes.toBytes(1L));
3705 region.put(put);
3706
3707 region.flushcache();
3708
3709 Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3710 region.delete(delete);
3711
3712 put = new Put(Bytes.toBytes(2L));
3713 put.add(family, qual1, 2L, Bytes.toBytes(2L));
3714 region.put(put);
3715
3716 Scan idxScan = new Scan();
3717 idxScan.addFamily(family);
3718 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
3719 new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
3720 new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
3721 CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
3722 InternalScanner scanner = region.getScanner(idxScan);
3723 List<Cell> res = new ArrayList<Cell>();
3724
3725 while (scanner.next(res))
3726 ;
3727 assertEquals(1L, res.size());
3728 } finally {
3729 HRegion.closeHRegion(this.region);
3730 this.region = null;
3731 }
3732 }
3733
3734
3735
3736
3737 @Test
3738 public void testBloomFilterSize() throws IOException {
3739 byte[] fam1 = Bytes.toBytes("fam1");
3740 byte[] qf1 = Bytes.toBytes("col");
3741 byte[] val1 = Bytes.toBytes("value1");
3742
3743 HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
3744 .setBloomFilterType(BloomType.ROWCOL);
3745
3746 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3747 htd.addFamily(hcd);
3748 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3749 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3750 try {
3751 int num_unique_rows = 10;
3752 int duplicate_multiplier = 2;
3753 int num_storefiles = 4;
3754
3755 int version = 0;
3756 for (int f = 0; f < num_storefiles; f++) {
3757 for (int i = 0; i < duplicate_multiplier; i++) {
3758 for (int j = 0; j < num_unique_rows; j++) {
3759 Put put = new Put(Bytes.toBytes("row" + j));
3760 put.setDurability(Durability.SKIP_WAL);
3761 put.add(fam1, qf1, version++, val1);
3762 region.put(put);
3763 }
3764 }
3765 region.flushcache();
3766 }
3767
3768 HStore store = (HStore) region.getStore(fam1);
3769 Collection<StoreFile> storeFiles = store.getStorefiles();
3770 for (StoreFile storefile : storeFiles) {
3771 StoreFile.Reader reader = storefile.getReader();
3772 reader.loadFileInfo();
3773 reader.loadBloomfilter();
3774 assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
3775 assertEquals(num_unique_rows, reader.getFilterEntries());
3776 }
3777
3778 region.compactStores(true);
3779
3780
3781 storeFiles = store.getStorefiles();
3782 for (StoreFile storefile : storeFiles) {
3783 StoreFile.Reader reader = storefile.getReader();
3784 reader.loadFileInfo();
3785 reader.loadBloomfilter();
3786 assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
3787 assertEquals(num_unique_rows, reader.getFilterEntries());
3788 }
3789 } finally {
3790 HRegion.closeHRegion(this.region);
3791 this.region = null;
3792 }
3793 }
3794
3795 @Test
3796 public void testAllColumnsWithBloomFilter() throws IOException {
3797 byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
3798 byte[] FAMILY = Bytes.toBytes("family");
3799
3800
3801 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
3802 .setBloomFilterType(BloomType.ROWCOL);
3803 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
3804 htd.addFamily(hcd);
3805 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3806 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3807 try {
3808
3809 byte row[] = Bytes.toBytes("row:" + 0);
3810 byte column[] = Bytes.toBytes("column:" + 0);
3811 Put put = new Put(row);
3812 put.setDurability(Durability.SKIP_WAL);
3813 for (long idx = 1; idx <= 4; idx++) {
3814 put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
3815 }
3816 region.put(put);
3817
3818
3819 region.flushcache();
3820
3821
3822 Get get = new Get(row);
3823 get.setMaxVersions();
3824 Cell[] kvs = region.get(get).rawCells();
3825
3826
3827 assertEquals(4, kvs.length);
3828 checkOneCell(kvs[0], FAMILY, 0, 0, 4);
3829 checkOneCell(kvs[1], FAMILY, 0, 0, 3);
3830 checkOneCell(kvs[2], FAMILY, 0, 0, 2);
3831 checkOneCell(kvs[3], FAMILY, 0, 0, 1);
3832 } finally {
3833 HRegion.closeHRegion(this.region);
3834 this.region = null;
3835 }
3836 }
3837
3838
3839
3840
3841
3842
3843 @Test
3844 public void testDeleteRowWithBloomFilter() throws IOException {
3845 byte[] familyName = Bytes.toBytes("familyName");
3846
3847
3848 HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
3849 .setBloomFilterType(BloomType.ROWCOL);
3850
3851 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3852 htd.addFamily(hcd);
3853 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3854 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3855 try {
3856
3857 byte row[] = Bytes.toBytes("row1");
3858 byte col[] = Bytes.toBytes("col1");
3859
3860 Put put = new Put(row);
3861 put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
3862 region.put(put);
3863 region.flushcache();
3864
3865 Delete del = new Delete(row);
3866 region.delete(del);
3867 region.flushcache();
3868
3869
3870 Get get = new Get(row);
3871 get.addColumn(familyName, col);
3872
3873 Cell[] keyValues = region.get(get).rawCells();
3874 assertTrue(keyValues.length == 0);
3875 } finally {
3876 HRegion.closeHRegion(this.region);
3877 this.region = null;
3878 }
3879 }
3880
3881 @Test
3882 public void testgetHDFSBlocksDistribution() throws Exception {
3883 HBaseTestingUtility htu = new HBaseTestingUtility();
3884 final int DEFAULT_BLOCK_SIZE = 1024;
3885 htu.getConfiguration().setLong("dfs.namenode.fs-limits.min-block-size", 0);
3886 htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
3887 htu.getConfiguration().setInt("dfs.replication", 2);
3888
3889
3890 MiniHBaseCluster cluster = null;
3891 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
3892 int regionServersCount = 3;
3893
3894 try {
3895 cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
3896 byte[][] families = { fam1, fam2 };
3897 HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);
3898
3899
3900 byte row[] = Bytes.toBytes("row1");
3901 byte col[] = Bytes.toBytes("col1");
3902
3903 Put put = new Put(row);
3904 put.add(fam1, col, 1, Bytes.toBytes("test1"));
3905 put.add(fam2, col, 1, Bytes.toBytes("test2"));
3906 ht.put(put);
3907
3908 HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
3909 .get(0);
3910 firstRegion.flushcache();
3911 HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
3912
3913
3914
3915
3916
3917 long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
3918
3919 String topHost = blocksDistribution1.getTopHosts().get(0);
3920 long topHostWeight = blocksDistribution1.getWeight(topHost);
3921 assertTrue(uniqueBlocksWeight1 == topHostWeight);
3922
3923
3924
3925 HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
3926 htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
3927 long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
3928
3929 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
3930
3931 ht.close();
3932 } finally {
3933 if (cluster != null) {
3934 htu.shutdownMiniCluster();
3935 }
3936 }
3937 }
3938
3939
3940
3941
3942
3943
3944
3945 @Test
3946 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
3947 TableName tableName = TableName.valueOf(name.getMethodName());
3948 HRegionInfo info = null;
3949 try {
3950 FileSystem fs = Mockito.mock(FileSystem.class);
3951 when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
3952 HTableDescriptor htd = new HTableDescriptor(tableName);
3953 htd.addFamily(new HColumnDescriptor("cf"));
3954 info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
3955 HConstants.EMPTY_BYTE_ARRAY, false);
3956 Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
3957 region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
3958
3959 region.initialize();
3960 fail("Region initialization should fail due to IOException");
3961 } catch (IOException io) {
3962 List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
3963 for (MonitoredTask monitoredTask : tasks) {
3964 if (!(monitoredTask instanceof MonitoredRPCHandler)
3965 && monitoredTask.getDescription().contains(region.toString())) {
3966 assertTrue("Region state should be ABORTED.",
3967 monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
3968 break;
3969 }
3970 }
3971 } finally {
3972 HRegion.closeHRegion(region);
3973 }
3974 }
3975
3976
3977
3978
3979
3980 @Test
3981 public void testRegionInfoFileCreation() throws IOException {
3982 Path rootDir = new Path(dir + "testRegionInfoFileCreation");
3983
3984 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
3985 htd.addFamily(new HColumnDescriptor("cf"));
3986
3987 HRegionInfo hri = new HRegionInfo(htd.getTableName());
3988
3989
3990 HRegion region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
3991
3992 Path regionDir = region.getRegionFileSystem().getRegionDir();
3993 FileSystem fs = region.getRegionFileSystem().getFileSystem();
3994 HRegion.closeHRegion(region);
3995
3996 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
3997
3998
3999 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4000 fs.exists(regionInfoFile));
4001
4002
4003 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
4004 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
4005 HRegion.closeHRegion(region);
4006
4007
4008 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4009 fs.exists(regionInfoFile));
4010
4011
4012 fs.delete(regionInfoFile);
4013 assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
4014 fs.exists(regionInfoFile));
4015
4016 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
4017
4018 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
4019 HRegion.closeHRegion(region);
4020
4021
4022 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4023 fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
4024 }
4025
4026
4027
4028
4029 private static class Incrementer implements Runnable {
4030 private HRegion region;
4031 private final static byte[] incRow = Bytes.toBytes("incRow");
4032 private final static byte[] family = Bytes.toBytes("family");
4033 private final static byte[] qualifier = Bytes.toBytes("qualifier");
4034 private final static long ONE = 1l;
4035 private int incCounter;
4036
4037 public Incrementer(HRegion region, int incCounter) {
4038 this.region = region;
4039 this.incCounter = incCounter;
4040 }
4041
4042 @Override
4043 public void run() {
4044 int count = 0;
4045 while (count < incCounter) {
4046 Increment inc = new Increment(incRow);
4047 inc.addColumn(family, qualifier, ONE);
4048 count++;
4049 try {
4050 region.increment(inc);
4051 } catch (IOException e) {
4052 e.printStackTrace();
4053 break;
4054 }
4055 }
4056 }
4057 }
4058
4059
4060
4061
4062
4063 @Test
4064 public void testParallelIncrementWithMemStoreFlush() throws Exception {
4065 byte[] family = Incrementer.family;
4066 this.region = initHRegion(tableName, method, CONF, family);
4067 final HRegion region = this.region;
4068 final AtomicBoolean incrementDone = new AtomicBoolean(false);
4069 Runnable flusher = new Runnable() {
4070 @Override
4071 public void run() {
4072 while (!incrementDone.get()) {
4073 try {
4074 region.flushcache();
4075 } catch (Exception e) {
4076 e.printStackTrace();
4077 }
4078 }
4079 }
4080 };
4081
4082
4083 int threadNum = 20;
4084 int incCounter = 100;
4085 long expected = threadNum * incCounter;
4086 Thread[] incrementers = new Thread[threadNum];
4087 Thread flushThread = new Thread(flusher);
4088 for (int i = 0; i < threadNum; i++) {
4089 incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
4090 incrementers[i].start();
4091 }
4092 flushThread.start();
4093 for (int i = 0; i < threadNum; i++) {
4094 incrementers[i].join();
4095 }
4096
4097 incrementDone.set(true);
4098 flushThread.join();
4099
4100 Get get = new Get(Incrementer.incRow);
4101 get.addColumn(Incrementer.family, Incrementer.qualifier);
4102 get.setMaxVersions(1);
4103 Result res = this.region.get(get);
4104 List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
4105
4106
4107 assertEquals(kvs.size(), 1);
4108 Cell kv = kvs.get(0);
4109 assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
4110 this.region = null;
4111 }
4112
4113
4114
4115
4116 private static class Appender implements Runnable {
4117 private HRegion region;
4118 private final static byte[] appendRow = Bytes.toBytes("appendRow");
4119 private final static byte[] family = Bytes.toBytes("family");
4120 private final static byte[] qualifier = Bytes.toBytes("qualifier");
4121 private final static byte[] CHAR = Bytes.toBytes("a");
4122 private int appendCounter;
4123
4124 public Appender(HRegion region, int appendCounter) {
4125 this.region = region;
4126 this.appendCounter = appendCounter;
4127 }
4128
4129 @Override
4130 public void run() {
4131 int count = 0;
4132 while (count < appendCounter) {
4133 Append app = new Append(appendRow);
4134 app.add(family, qualifier, CHAR);
4135 count++;
4136 try {
4137 region.append(app);
4138 } catch (IOException e) {
4139 e.printStackTrace();
4140 break;
4141 }
4142 }
4143 }
4144 }
4145
4146
4147
4148
4149
4150 @Test
4151 public void testParallelAppendWithMemStoreFlush() throws Exception {
4152 byte[] family = Appender.family;
4153 this.region = initHRegion(tableName, method, CONF, family);
4154 final HRegion region = this.region;
4155 final AtomicBoolean appendDone = new AtomicBoolean(false);
4156 Runnable flusher = new Runnable() {
4157 @Override
4158 public void run() {
4159 while (!appendDone.get()) {
4160 try {
4161 region.flushcache();
4162 } catch (Exception e) {
4163 e.printStackTrace();
4164 }
4165 }
4166 }
4167 };
4168
4169
4170
4171 int threadNum = 20;
4172 int appendCounter = 100;
4173 byte[] expected = new byte[threadNum * appendCounter];
4174 for (int i = 0; i < threadNum * appendCounter; i++) {
4175 System.arraycopy(Appender.CHAR, 0, expected, i, 1);
4176 }
4177 Thread[] appenders = new Thread[threadNum];
4178 Thread flushThread = new Thread(flusher);
4179 for (int i = 0; i < threadNum; i++) {
4180 appenders[i] = new Thread(new Appender(this.region, appendCounter));
4181 appenders[i].start();
4182 }
4183 flushThread.start();
4184 for (int i = 0; i < threadNum; i++) {
4185 appenders[i].join();
4186 }
4187
4188 appendDone.set(true);
4189 flushThread.join();
4190
4191 Get get = new Get(Appender.appendRow);
4192 get.addColumn(Appender.family, Appender.qualifier);
4193 get.setMaxVersions(1);
4194 Result res = this.region.get(get);
4195 List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
4196
4197
4198 assertEquals(kvs.size(), 1);
4199 Cell kv = kvs.get(0);
4200 byte[] appendResult = new byte[kv.getValueLength()];
4201 System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
4202 assertArrayEquals(expected, appendResult);
4203 this.region = null;
4204 }
4205
4206
4207
4208
4209
4210 @Test
4211 public void testPutWithMemStoreFlush() throws Exception {
4212 byte[] family = Bytes.toBytes("family");
4213 ;
4214 byte[] qualifier = Bytes.toBytes("qualifier");
4215 byte[] row = Bytes.toBytes("putRow");
4216 byte[] value = null;
4217 this.region = initHRegion(tableName, method, CONF, family);
4218 Put put = null;
4219 Get get = null;
4220 List<Cell> kvs = null;
4221 Result res = null;
4222
4223 put = new Put(row);
4224 value = Bytes.toBytes("value0");
4225 put.add(family, qualifier, 1234567l, value);
4226 region.put(put);
4227 get = new Get(row);
4228 get.addColumn(family, qualifier);
4229 get.setMaxVersions();
4230 res = this.region.get(get);
4231 kvs = res.getColumnCells(family, qualifier);
4232 assertEquals(1, kvs.size());
4233 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4234
4235 region.flushcache();
4236 get = new Get(row);
4237 get.addColumn(family, qualifier);
4238 get.setMaxVersions();
4239 res = this.region.get(get);
4240 kvs = res.getColumnCells(family, qualifier);
4241 assertEquals(1, kvs.size());
4242 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4243
4244 put = new Put(row);
4245 value = Bytes.toBytes("value1");
4246 put.add(family, qualifier, 1234567l, value);
4247 region.put(put);
4248 get = new Get(row);
4249 get.addColumn(family, qualifier);
4250 get.setMaxVersions();
4251 res = this.region.get(get);
4252 kvs = res.getColumnCells(family, qualifier);
4253 assertEquals(1, kvs.size());
4254 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4255
4256 region.flushcache();
4257 get = new Get(row);
4258 get.addColumn(family, qualifier);
4259 get.setMaxVersions();
4260 res = this.region.get(get);
4261 kvs = res.getColumnCells(family, qualifier);
4262 assertEquals(1, kvs.size());
4263 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4264 }
4265
4266 @Test
4267 public void testDurability() throws Exception {
4268 String method = "testDurability";
4269
4270
4271
4272
4273
4274 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4275 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4276 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4277
4278 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4279 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4280 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4281
4282 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4283 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4284
4285 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
4286 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4287
4288 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
4289 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
4290 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
4291
4292
4293 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4294 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4295 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4296 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4297 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
4298 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4299
4300 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4301 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4302 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4303 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4304 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
4305 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
4306
4307
4308 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4309 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4310 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4311 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
4312 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
4313 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
4314
4315 }
4316
4317 private void durabilityTest(String method, Durability tableDurability,
4318 Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
4319 final boolean expectSyncFromLogSyncer) throws Exception {
4320 Configuration conf = HBaseConfiguration.create(CONF);
4321 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
4322 TableName tableName = TableName.valueOf(method);
4323 byte[] family = Bytes.toBytes("family");
4324 Path logDir = new Path(new Path(dir + method), "log");
4325 HLog hlog = HLogFactory.createHLog(FILESYSTEM, logDir, UUID.randomUUID().toString(), conf);
4326 final HLog log = spy(hlog);
4327 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
4328 HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, log,
4329 new byte[][] { family });
4330
4331 Put put = new Put(Bytes.toBytes("r1"));
4332 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
4333 put.setDurability(mutationDurability);
4334 region.put(put);
4335
4336
4337 verify(log, expectAppend ? times(1) : never())
4338 .appendNoSync((HRegionInfo)any(), eq(tableName), (WALEdit)any(), (List<UUID>)any(),
4339 anyLong(), (HTableDescriptor)any(), (AtomicLong)any(), anyBoolean(), anyLong(), anyLong());
4340
4341
4342 if (expectSync || expectSyncFromLogSyncer) {
4343 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
4344 @Override
4345 public boolean evaluate() throws Exception {
4346 try {
4347 if (expectSync) {
4348 verify(log, times(1)).sync(anyLong());
4349 } else if (expectSyncFromLogSyncer) {
4350 verify(log, times(1)).sync();
4351 }
4352 } catch (Throwable ignore) {
4353 }
4354 return true;
4355 }
4356 });
4357 } else {
4358 verify(log, never()).sync(anyLong());
4359 verify(log, never()).sync();
4360 }
4361
4362 HRegion.closeHRegion(this.region);
4363 this.region = null;
4364 }
4365
4366 private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4367 for (int i = startRow; i < startRow + numRows; i++) {
4368 Put put = new Put(Bytes.toBytes("" + i));
4369 put.setDurability(Durability.SKIP_WAL);
4370 for (byte[] family : families) {
4371 put.add(family, qf, null);
4372 }
4373 region.put(put);
4374 }
4375 }
4376
4377 private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
4378 throws IOException {
4379 for (int i = startRow; i < startRow + numRows; i++) {
4380 byte[] row = Bytes.toBytes("" + i);
4381 Get get = new Get(row);
4382 for (byte[] family : families) {
4383 get.addColumn(family, qf);
4384 }
4385 Result result = newReg.get(get);
4386 Cell[] raw = result.rawCells();
4387 assertEquals(families.length, result.size());
4388 for (int j = 0; j < families.length; j++) {
4389 assertTrue(CellUtil.matchingRow(raw[j], row));
4390 assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
4391 assertTrue(CellUtil.matchingQualifier(raw[j], qf));
4392 }
4393 }
4394 }
4395
4396 private void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
4397
4398 Get get = new Get(k).addFamily(family).setMaxVersions();
4399 Cell[] results = r.get(get).rawCells();
4400 for (int j = 0; j < results.length; j++) {
4401 byte[] tmp = CellUtil.cloneValue(results[j]);
4402
4403 assertTrue(Bytes.equals(k, tmp));
4404 }
4405 }
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418 private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
4419 throws IOException {
4420 byte[][] families = { fs };
4421 Scan scan = new Scan();
4422 for (int i = 0; i < families.length; i++)
4423 scan.addFamily(families[i]);
4424 InternalScanner s = r.getScanner(scan);
4425 try {
4426 List<Cell> curVals = new ArrayList<Cell>();
4427 boolean first = true;
4428 OUTER_LOOP: while (s.next(curVals)) {
4429 for (Cell kv : curVals) {
4430 byte[] val = CellUtil.cloneValue(kv);
4431 byte[] curval = val;
4432 if (first) {
4433 first = false;
4434 assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4435 } else {
4436
4437 break OUTER_LOOP;
4438 }
4439 }
4440 }
4441 } finally {
4442 s.close();
4443 }
4444 }
4445
4446
4447
4448
4449
4450 @Test
4451 public void testFlushResult() throws IOException {
4452 String method = name.getMethodName();
4453 byte[] tableName = Bytes.toBytes(method);
4454 byte[] family = Bytes.toBytes("family");
4455
4456 this.region = initHRegion(tableName, method, family);
4457
4458
4459 HRegion.FlushResult fr = region.flushcache();
4460 assertFalse(fr.isFlushSucceeded());
4461 assertFalse(fr.isCompactionNeeded());
4462
4463
4464 for (int i = 0; i < 2; i++) {
4465 Put put = new Put(tableName).add(family, family, tableName);
4466 region.put(put);
4467 fr = region.flushcache();
4468 assertTrue(fr.isFlushSucceeded());
4469 assertFalse(fr.isCompactionNeeded());
4470 }
4471
4472
4473 for (int i = 0; i < 2; i++) {
4474 Put put = new Put(tableName).add(family, family, tableName);
4475 region.put(put);
4476 fr = region.flushcache();
4477 assertTrue(fr.isFlushSucceeded());
4478 assertTrue(fr.isCompactionNeeded());
4479 }
4480 }
4481
4482 private Configuration initSplit() {
4483
4484 CONF.setInt("hbase.hstore.compactionThreshold", 2);
4485
4486
4487 CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4488
4489 CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4490
4491
4492 CONF.setLong("hbase.client.pause", 15 * 1000);
4493
4494
4495
4496 CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4497 return CONF;
4498 }
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509 public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
4510 byte[]... families) throws IOException {
4511 return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
4512 }
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4524 byte[]... families) throws IOException {
4525 return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4526 }
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4539 boolean isReadOnly, byte[]... families) throws IOException {
4540 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4541 }
4542
4543 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4544 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4545 throws IOException {
4546 return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
4547 Durability.SYNC_WAL, null, families);
4548 }
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4563 String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4564 HLog hlog, byte[]... families) throws IOException {
4565 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly, durability, hlog, families);
4566 }
4567
4568
4569
4570
4571
4572 private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
4573 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4574 assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
4575 Bytes.toString(CellUtil.cloneRow(kv)));
4576 assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
4577 Bytes.toString(CellUtil.cloneFamily(kv)));
4578 assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
4579 Bytes.toString(CellUtil.cloneQualifier(kv)));
4580 assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
4581 assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
4582 Bytes.toString(CellUtil.cloneValue(kv)));
4583 }
4584
4585 @Test (timeout=60000)
4586 public void testReverseScanner_FromMemStore_SingleCF_Normal()
4587 throws IOException {
4588 byte[] rowC = Bytes.toBytes("rowC");
4589 byte[] rowA = Bytes.toBytes("rowA");
4590 byte[] rowB = Bytes.toBytes("rowB");
4591 byte[] cf = Bytes.toBytes("CF");
4592 byte[][] families = { cf };
4593 byte[] col = Bytes.toBytes("C");
4594 long ts = 1;
4595 String method = this.getName();
4596 this.region = initHRegion(tableName, method, families);
4597 try {
4598 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4599 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4600 null);
4601 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4602 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4603 Put put = null;
4604 put = new Put(rowC);
4605 put.add(kv1);
4606 put.add(kv11);
4607 region.put(put);
4608 put = new Put(rowA);
4609 put.add(kv2);
4610 region.put(put);
4611 put = new Put(rowB);
4612 put.add(kv3);
4613 region.put(put);
4614
4615 Scan scan = new Scan(rowC);
4616 scan.setMaxVersions(5);
4617 scan.setReversed(true);
4618 InternalScanner scanner = region.getScanner(scan);
4619 List<Cell> currRow = new ArrayList<Cell>();
4620 boolean hasNext = scanner.next(currRow);
4621 assertEquals(2, currRow.size());
4622 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4623 assertTrue(hasNext);
4624 currRow.clear();
4625 hasNext = scanner.next(currRow);
4626 assertEquals(1, currRow.size());
4627 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4628 assertTrue(hasNext);
4629 currRow.clear();
4630 hasNext = scanner.next(currRow);
4631 assertEquals(1, currRow.size());
4632 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4633 assertFalse(hasNext);
4634 scanner.close();
4635 } finally {
4636 HRegion.closeHRegion(this.region);
4637 this.region = null;
4638 }
4639 }
4640
4641 @Test (timeout=60000)
4642 public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
4643 throws IOException {
4644 byte[] rowC = Bytes.toBytes("rowC");
4645 byte[] rowA = Bytes.toBytes("rowA");
4646 byte[] rowB = Bytes.toBytes("rowB");
4647 byte[] rowD = Bytes.toBytes("rowD");
4648 byte[] cf = Bytes.toBytes("CF");
4649 byte[][] families = { cf };
4650 byte[] col = Bytes.toBytes("C");
4651 long ts = 1;
4652 String method = this.getName();
4653 this.region = initHRegion(tableName, method, families);
4654 try {
4655 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4656 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4657 null);
4658 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4659 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4660 Put put = null;
4661 put = new Put(rowC);
4662 put.add(kv1);
4663 put.add(kv11);
4664 region.put(put);
4665 put = new Put(rowA);
4666 put.add(kv2);
4667 region.put(put);
4668 put = new Put(rowB);
4669 put.add(kv3);
4670 region.put(put);
4671
4672 Scan scan = new Scan(rowD);
4673 List<Cell> currRow = new ArrayList<Cell>();
4674 scan.setReversed(true);
4675 scan.setMaxVersions(5);
4676 InternalScanner scanner = region.getScanner(scan);
4677 boolean hasNext = scanner.next(currRow);
4678 assertEquals(2, currRow.size());
4679 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4680 assertTrue(hasNext);
4681 currRow.clear();
4682 hasNext = scanner.next(currRow);
4683 assertEquals(1, currRow.size());
4684 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4685 assertTrue(hasNext);
4686 currRow.clear();
4687 hasNext = scanner.next(currRow);
4688 assertEquals(1, currRow.size());
4689 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4690 assertFalse(hasNext);
4691 scanner.close();
4692 } finally {
4693 HRegion.closeHRegion(this.region);
4694 this.region = null;
4695 }
4696 }
4697
4698 @Test (timeout=60000)
4699 public void testReverseScanner_FromMemStore_SingleCF_FullScan()
4700 throws IOException {
4701 byte[] rowC = Bytes.toBytes("rowC");
4702 byte[] rowA = Bytes.toBytes("rowA");
4703 byte[] rowB = Bytes.toBytes("rowB");
4704 byte[] cf = Bytes.toBytes("CF");
4705 byte[][] families = { cf };
4706 byte[] col = Bytes.toBytes("C");
4707 long ts = 1;
4708 String method = this.getName();
4709 this.region = initHRegion(tableName, method, families);
4710 try {
4711 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4712 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4713 null);
4714 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4715 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4716 Put put = null;
4717 put = new Put(rowC);
4718 put.add(kv1);
4719 put.add(kv11);
4720 region.put(put);
4721 put = new Put(rowA);
4722 put.add(kv2);
4723 region.put(put);
4724 put = new Put(rowB);
4725 put.add(kv3);
4726 region.put(put);
4727 Scan scan = new Scan();
4728 List<Cell> currRow = new ArrayList<Cell>();
4729 scan.setReversed(true);
4730 InternalScanner scanner = region.getScanner(scan);
4731 boolean hasNext = scanner.next(currRow);
4732 assertEquals(1, currRow.size());
4733 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4734 assertTrue(hasNext);
4735 currRow.clear();
4736 hasNext = scanner.next(currRow);
4737 assertEquals(1, currRow.size());
4738 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4739 assertTrue(hasNext);
4740 currRow.clear();
4741 hasNext = scanner.next(currRow);
4742 assertEquals(1, currRow.size());
4743 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4744 assertFalse(hasNext);
4745 scanner.close();
4746 } finally {
4747 HRegion.closeHRegion(this.region);
4748 this.region = null;
4749 }
4750 }
4751
4752 @Test (timeout=60000)
4753 public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
4754
4755 byte[] rowA = Bytes.toBytes("rowA");
4756 byte[] rowB = Bytes.toBytes("rowB");
4757 byte[] rowC = Bytes.toBytes("rowC");
4758 byte[] rowD = Bytes.toBytes("rowD");
4759 byte[] rowE = Bytes.toBytes("rowE");
4760 byte[] cf = Bytes.toBytes("CF");
4761 byte[][] families = { cf };
4762 byte[] col1 = Bytes.toBytes("col1");
4763 byte[] col2 = Bytes.toBytes("col2");
4764 long ts = 1;
4765 String method = this.getName();
4766 this.region = initHRegion(tableName, method, families);
4767 try {
4768 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4769 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4770 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4771 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4772 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4773 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4774 Put put = null;
4775 put = new Put(rowA);
4776 put.add(kv1);
4777 region.put(put);
4778 put = new Put(rowB);
4779 put.add(kv2);
4780 region.put(put);
4781 put = new Put(rowC);
4782 put.add(kv3);
4783 region.put(put);
4784 put = new Put(rowD);
4785 put.add(kv4_1);
4786 region.put(put);
4787 put = new Put(rowD);
4788 put.add(kv4_2);
4789 region.put(put);
4790 put = new Put(rowE);
4791 put.add(kv5);
4792 region.put(put);
4793 region.flushcache();
4794 Scan scan = new Scan(rowD, rowA);
4795 scan.addColumn(families[0], col1);
4796 scan.setReversed(true);
4797 List<Cell> currRow = new ArrayList<Cell>();
4798 InternalScanner scanner = region.getScanner(scan);
4799 boolean hasNext = scanner.next(currRow);
4800 assertEquals(1, currRow.size());
4801 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4802 assertTrue(hasNext);
4803 currRow.clear();
4804 hasNext = scanner.next(currRow);
4805 assertEquals(1, currRow.size());
4806 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4807 assertTrue(hasNext);
4808 currRow.clear();
4809 hasNext = scanner.next(currRow);
4810 assertEquals(1, currRow.size());
4811 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4812 assertFalse(hasNext);
4813 scanner.close();
4814
4815 scan = new Scan(rowD, rowA);
4816 scan.addColumn(families[0], col2);
4817 scan.setReversed(true);
4818 currRow.clear();
4819 scanner = region.getScanner(scan);
4820 hasNext = scanner.next(currRow);
4821 assertEquals(1, currRow.size());
4822 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4823 scanner.close();
4824 } finally {
4825 HRegion.closeHRegion(this.region);
4826 this.region = null;
4827 }
4828 }
4829
4830 @Test (timeout=60000)
4831 public void testReverseScanner_smaller_blocksize() throws IOException {
4832
4833 byte[] rowA = Bytes.toBytes("rowA");
4834 byte[] rowB = Bytes.toBytes("rowB");
4835 byte[] rowC = Bytes.toBytes("rowC");
4836 byte[] rowD = Bytes.toBytes("rowD");
4837 byte[] rowE = Bytes.toBytes("rowE");
4838 byte[] cf = Bytes.toBytes("CF");
4839 byte[][] families = { cf };
4840 byte[] col1 = Bytes.toBytes("col1");
4841 byte[] col2 = Bytes.toBytes("col2");
4842 long ts = 1;
4843 String method = this.getName();
4844 HBaseConfiguration config = new HBaseConfiguration();
4845 config.setInt("test.block.size", 1);
4846 this.region = initHRegion(tableName, method, config, families);
4847 try {
4848 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4849 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4850 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4851 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4852 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4853 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4854 Put put = null;
4855 put = new Put(rowA);
4856 put.add(kv1);
4857 region.put(put);
4858 put = new Put(rowB);
4859 put.add(kv2);
4860 region.put(put);
4861 put = new Put(rowC);
4862 put.add(kv3);
4863 region.put(put);
4864 put = new Put(rowD);
4865 put.add(kv4_1);
4866 region.put(put);
4867 put = new Put(rowD);
4868 put.add(kv4_2);
4869 region.put(put);
4870 put = new Put(rowE);
4871 put.add(kv5);
4872 region.put(put);
4873 region.flushcache();
4874 Scan scan = new Scan(rowD, rowA);
4875 scan.addColumn(families[0], col1);
4876 scan.setReversed(true);
4877 List<Cell> currRow = new ArrayList<Cell>();
4878 InternalScanner scanner = region.getScanner(scan);
4879 boolean hasNext = scanner.next(currRow);
4880 assertEquals(1, currRow.size());
4881 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4882 assertTrue(hasNext);
4883 currRow.clear();
4884 hasNext = scanner.next(currRow);
4885 assertEquals(1, currRow.size());
4886 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4887 assertTrue(hasNext);
4888 currRow.clear();
4889 hasNext = scanner.next(currRow);
4890 assertEquals(1, currRow.size());
4891 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4892 assertFalse(hasNext);
4893 scanner.close();
4894
4895 scan = new Scan(rowD, rowA);
4896 scan.addColumn(families[0], col2);
4897 scan.setReversed(true);
4898 currRow.clear();
4899 scanner = region.getScanner(scan);
4900 hasNext = scanner.next(currRow);
4901 assertEquals(1, currRow.size());
4902 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4903 scanner.close();
4904 } finally {
4905 HRegion.closeHRegion(this.region);
4906 this.region = null;
4907 }
4908 }
4909
4910 @Test (timeout=60000)
4911 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
4912 throws IOException {
4913 byte[] row0 = Bytes.toBytes("row0");
4914 byte[] row1 = Bytes.toBytes("row1");
4915 byte[] row2 = Bytes.toBytes("row2");
4916 byte[] row3 = Bytes.toBytes("row3");
4917 byte[] row4 = Bytes.toBytes("row4");
4918 byte[] row5 = Bytes.toBytes("row5");
4919 byte[] cf1 = Bytes.toBytes("CF1");
4920 byte[] cf2 = Bytes.toBytes("CF2");
4921 byte[] cf3 = Bytes.toBytes("CF3");
4922 byte[][] families = { cf1, cf2, cf3 };
4923 byte[] col = Bytes.toBytes("C");
4924 long ts = 1;
4925 String method = this.getName();
4926 HBaseConfiguration conf = new HBaseConfiguration();
4927
4928 conf.setInt("hbase.hstore.compactionThreshold", 10000);
4929 this.region = initHRegion(tableName, method, conf, families);
4930 try {
4931
4932 KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
4933 null);
4934 KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
4935 null);
4936 KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
4937 KeyValue.Type.Put, null);
4938 KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
4939 null);
4940 KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
4941 null);
4942 KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
4943 null);
4944 KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
4945 KeyValue.Type.Put, null);
4946 KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
4947 null);
4948 KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
4949 KeyValue.Type.Put, null);
4950 KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
4951 null);
4952 KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
4953 null);
4954 KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
4955 KeyValue.Type.Put, null);
4956 KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
4957 null);
4958 KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
4959 KeyValue.Type.Put, null);
4960 KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
4961 null);
4962 KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
4963 null);
4964
4965 Put put = null;
4966 put = new Put(row1);
4967 put.add(kv1_2_1);
4968 region.put(put);
4969 put = new Put(row2);
4970 put.add(kv2_4_1);
4971 region.put(put);
4972 put = new Put(row4);
4973 put.add(kv4_5_4);
4974 put.add(kv4_5_5);
4975 region.put(put);
4976 region.flushcache();
4977
4978 put = new Put(row4);
4979 put.add(kv4_5_1);
4980 put.add(kv4_5_3);
4981 region.put(put);
4982 put = new Put(row1);
4983 put.add(kv1_2_2);
4984 region.put(put);
4985 put = new Put(row2);
4986 put.add(kv2_4_4);
4987 region.put(put);
4988 region.flushcache();
4989
4990 put = new Put(row4);
4991 put.add(kv4_5_2);
4992 region.put(put);
4993 put = new Put(row2);
4994 put.add(kv2_4_2);
4995 put.add(kv2_4_3);
4996 region.put(put);
4997 put = new Put(row3);
4998 put.add(kv3_2_2);
4999 region.put(put);
5000 region.flushcache();
5001
5002
5003 put = new Put(row0);
5004 put.add(kv0_1_1);
5005 region.put(put);
5006 put = new Put(row3);
5007 put.add(kv3_2_1);
5008 region.put(put);
5009 put = new Put(row5);
5010 put.add(kv5_2_1);
5011 put.add(kv5_2_2);
5012 region.put(put);
5013
5014 Scan scan = new Scan(row4);
5015 scan.setMaxVersions(5);
5016 scan.setBatch(3);
5017 scan.setReversed(true);
5018 InternalScanner scanner = region.getScanner(scan);
5019 List<Cell> currRow = new ArrayList<Cell>();
5020 boolean hasNext = false;
5021
5022
5023
5024 hasNext = scanner.next(currRow);
5025 assertEquals(3, currRow.size());
5026 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5027 assertTrue(hasNext);
5028 currRow.clear();
5029 hasNext = scanner.next(currRow);
5030 assertEquals(2, currRow.size());
5031 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5032 assertTrue(hasNext);
5033
5034 currRow.clear();
5035 hasNext = scanner.next(currRow);
5036 assertEquals(2, currRow.size());
5037 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5038 assertTrue(hasNext);
5039
5040
5041 currRow.clear();
5042 hasNext = scanner.next(currRow);
5043 assertEquals(3, currRow.size());
5044 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5045 assertTrue(hasNext);
5046 currRow.clear();
5047 hasNext = scanner.next(currRow);
5048 assertEquals(1, currRow.size());
5049 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5050 assertTrue(hasNext);
5051
5052 currRow.clear();
5053 hasNext = scanner.next(currRow);
5054 assertEquals(2, currRow.size());
5055 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5056 assertTrue(hasNext);
5057
5058 currRow.clear();
5059 hasNext = scanner.next(currRow);
5060 assertEquals(1, currRow.size());
5061 assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
5062 assertFalse(hasNext);
5063
5064 scanner.close();
5065 } finally {
5066 HRegion.closeHRegion(this.region);
5067 this.region = null;
5068 }
5069 }
5070
5071 @Test (timeout=60000)
5072 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
5073 throws IOException {
5074 byte[] row1 = Bytes.toBytes("row1");
5075 byte[] row2 = Bytes.toBytes("row2");
5076 byte[] row3 = Bytes.toBytes("row3");
5077 byte[] row4 = Bytes.toBytes("row4");
5078 byte[] cf1 = Bytes.toBytes("CF1");
5079 byte[] cf2 = Bytes.toBytes("CF2");
5080 byte[] cf3 = Bytes.toBytes("CF3");
5081 byte[] cf4 = Bytes.toBytes("CF4");
5082 byte[][] families = { cf1, cf2, cf3, cf4 };
5083 byte[] col = Bytes.toBytes("C");
5084 long ts = 1;
5085 String method = this.getName();
5086 HBaseConfiguration conf = new HBaseConfiguration();
5087
5088 conf.setInt("hbase.hstore.compactionThreshold", 10000);
5089 this.region = initHRegion(tableName, method, conf, families);
5090 try {
5091 KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
5092 KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
5093 KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
5094 KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
5095
5096 Put put = new Put(row1);
5097 put.add(kv1);
5098 region.put(put);
5099 region.flushcache();
5100
5101 put = new Put(row2);
5102 put.add(kv2);
5103 region.put(put);
5104 region.flushcache();
5105
5106 put = new Put(row3);
5107 put.add(kv3);
5108 region.put(put);
5109 region.flushcache();
5110
5111 put = new Put(row4);
5112 put.add(kv4);
5113 region.put(put);
5114
5115 Scan scan = new Scan(row4);
5116 scan.setReversed(true);
5117 scan.setBatch(10);
5118 InternalScanner scanner = region.getScanner(scan);
5119 List<Cell> currRow = new ArrayList<Cell>();
5120 boolean hasNext = scanner.next(currRow);
5121 assertEquals(1, currRow.size());
5122 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5123 assertTrue(hasNext);
5124 currRow.clear();
5125 hasNext = scanner.next(currRow);
5126 assertEquals(1, currRow.size());
5127 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5128 assertTrue(hasNext);
5129 currRow.clear();
5130 hasNext = scanner.next(currRow);
5131 assertEquals(1, currRow.size());
5132 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5133 assertTrue(hasNext);
5134 currRow.clear();
5135 hasNext = scanner.next(currRow);
5136 assertEquals(1, currRow.size());
5137 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5138 assertFalse(hasNext);
5139 } finally {
5140 HRegion.closeHRegion(this.region);
5141 this.region = null;
5142 }
5143 }
5144
5145
5146
5147
5148 @Test (timeout = 60000)
5149 public void testReverseScanner_StackOverflow() throws IOException {
5150 byte[] cf1 = Bytes.toBytes("CF1");
5151 byte[][] families = {cf1};
5152 byte[] col = Bytes.toBytes("C");
5153 String method = this.getName();
5154 HBaseConfiguration conf = new HBaseConfiguration();
5155
5156 conf.setInt("hbase.hstore.compactionThreshold", 10000);
5157 this.region = initHRegion(tableName, method, conf, families);
5158
5159 try {
5160
5161 Put put = new Put(Bytes.toBytes("19998"));
5162 put.add(cf1, col, Bytes.toBytes("val"));
5163 region.put(put);
5164 region.flushcache();
5165 Put put2 = new Put(Bytes.toBytes("19997"));
5166 put2.add(cf1, col, Bytes.toBytes("val"));
5167 region.put(put2);
5168
5169 Scan scan = new Scan(Bytes.toBytes("19998"));
5170 scan.setReversed(true);
5171 InternalScanner scanner = region.getScanner(scan);
5172
5173
5174
5175 for (int i = 10000; i < 20000; i++) {
5176 Put p = new Put(Bytes.toBytes("" + i));
5177 p.add(cf1, col, Bytes.toBytes("" + i));
5178 region.put(p);
5179 }
5180 region.flushcache();
5181
5182
5183
5184 for (int i = 10000; i < 20000; i++) {
5185 Put p = new Put(Bytes.toBytes("" + i));
5186 p.add(cf1, col, Bytes.toBytes("" + i));
5187 region.put(p);
5188 }
5189
5190 List<Cell> currRow = new ArrayList<Cell>();
5191 boolean hasNext;
5192 do {
5193 hasNext = scanner.next(currRow);
5194 } while (hasNext);
5195 assertEquals(2, currRow.size());
5196 assertArrayEquals(Bytes.toBytes("19998"), currRow.get(0).getRow());
5197 assertArrayEquals(Bytes.toBytes("19997"), currRow.get(1).getRow());
5198 } finally {
5199 HRegion.closeHRegion(this.region);
5200 this.region = null;
5201 }
5202 }
5203
5204 @Test (timeout=60000)
5205 public void testSplitRegionWithReverseScan() throws IOException {
5206 byte [] tableName = Bytes.toBytes("testSplitRegionWithReverseScan");
5207 byte [] qualifier = Bytes.toBytes("qualifier");
5208 Configuration hc = initSplit();
5209 int numRows = 3;
5210 byte [][] families = {fam1};
5211
5212
5213 String method = this.getName();
5214 this.region = initHRegion(tableName, method, hc, families);
5215
5216
5217 int startRow = 100;
5218 putData(startRow, numRows, qualifier, families);
5219 int splitRow = startRow + numRows;
5220 putData(splitRow, numRows, qualifier, families);
5221 int endRow = splitRow + numRows;
5222 region.flushcache();
5223
5224 HRegion [] regions = null;
5225 try {
5226 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
5227
5228 for (int i = 0; i < regions.length; i++) {
5229 regions[i] = HRegion.openHRegion(regions[i], null);
5230 }
5231
5232 assertEquals(2, regions.length);
5233
5234
5235
5236 verifyData(regions[0], startRow, numRows, qualifier, families);
5237 verifyData(regions[1], splitRow, numRows, qualifier, families);
5238
5239
5240 Scan scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 10 * numRows)));
5241 scan.setReversed(true);
5242 InternalScanner scanner = regions[1].getScanner(scan);
5243 List<Cell> currRow = new ArrayList<Cell>();
5244 boolean more = false;
5245 int verify = startRow + 2 * numRows - 1;
5246 do {
5247 more = scanner.next(currRow);
5248 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5249 verify--;
5250 currRow.clear();
5251 } while(more);
5252 assertEquals(verify, startRow + numRows - 1);
5253 scanner.close();
5254
5255 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 2 * numRows - 1)));
5256 scan.setReversed(true);
5257 scanner = regions[1].getScanner(scan);
5258 verify = startRow + 2 * numRows - 1;
5259 do {
5260 more = scanner.next(currRow);
5261 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5262 verify--;
5263 currRow.clear();
5264 } while(more);
5265 assertEquals(verify, startRow + numRows - 1);
5266 scanner.close();
5267
5268 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows)));
5269 scan.setReversed(true);
5270 scanner = regions[0].getScanner(scan);
5271 verify = startRow + numRows - 1;
5272 do {
5273 more = scanner.next(currRow);
5274 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5275 verify--;
5276 currRow.clear();
5277 } while(more);
5278 assertEquals(verify, 99);
5279 scanner.close();
5280
5281 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows - 1)));
5282 scan.setReversed(true);
5283 scanner = regions[0].getScanner(scan);
5284 verify = startRow + numRows - 1;
5285 do {
5286 more = scanner.next(currRow);
5287 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5288 verify--;
5289 currRow.clear();
5290 } while(more);
5291 assertEquals(verify, startRow - 1);
5292 scanner.close();
5293 } finally {
5294 HRegion.closeHRegion(this.region);
5295 this.region = null;
5296 }
5297 }
5298
5299 @Test
5300 public void testWriteRequestsCounter() throws IOException {
5301 byte[] fam = Bytes.toBytes("info");
5302 byte[][] families = { fam };
5303 this.region = initHRegion(tableName, method, CONF, families);
5304
5305 Assert.assertEquals(0L, region.getWriteRequestsCount());
5306
5307 Put put = new Put(row);
5308 put.add(fam, fam, fam);
5309
5310 Assert.assertEquals(0L, region.getWriteRequestsCount());
5311 region.put(put);
5312 Assert.assertEquals(1L, region.getWriteRequestsCount());
5313 region.put(put);
5314 Assert.assertEquals(2L, region.getWriteRequestsCount());
5315 region.put(put);
5316 Assert.assertEquals(3L, region.getWriteRequestsCount());
5317
5318 region.delete(new Delete(row));
5319 Assert.assertEquals(4L, region.getWriteRequestsCount());
5320
5321 HRegion.closeHRegion(this.region);
5322 this.region = null;
5323 }
5324
5325
5326
5327
5328 @Test (timeout=24000)
5329 public void testRegionTooBusy() throws IOException {
5330 String method = "testRegionTooBusy";
5331 byte[] tableName = Bytes.toBytes(method);
5332 byte[] family = Bytes.toBytes("family");
5333 long defaultBusyWaitDuration = CONF.getLong("hbase.busy.wait.duration",
5334 HRegion.DEFAULT_BUSY_WAIT_DURATION);
5335 CONF.setLong("hbase.busy.wait.duration", 1000);
5336 region = initHRegion(tableName, method, CONF, family);
5337 final AtomicBoolean stopped = new AtomicBoolean(true);
5338 Thread t = new Thread(new Runnable() {
5339 @Override
5340 public void run() {
5341 try {
5342 region.lock.writeLock().lock();
5343 stopped.set(false);
5344 while (!stopped.get()) {
5345 Thread.sleep(100);
5346 }
5347 } catch (InterruptedException ie) {
5348 } finally {
5349 region.lock.writeLock().unlock();
5350 }
5351 }
5352 });
5353 t.start();
5354 Get get = new Get(row);
5355 try {
5356 while (stopped.get()) {
5357 Thread.sleep(100);
5358 }
5359 region.get(get);
5360 fail("Should throw RegionTooBusyException");
5361 } catch (InterruptedException ie) {
5362 fail("test interrupted");
5363 } catch (RegionTooBusyException e) {
5364
5365 } finally {
5366 stopped.set(true);
5367 try {
5368 t.join();
5369 } catch (Throwable e) {
5370 }
5371
5372 HRegion.closeHRegion(region);
5373 region = null;
5374 CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration);
5375 }
5376 }
5377
5378 @Test
5379 public void testCellTTLs() throws IOException {
5380 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
5381 EnvironmentEdgeManager.injectEdge(edge);
5382
5383 final byte[] row = Bytes.toBytes("testRow");
5384 final byte[] q1 = Bytes.toBytes("q1");
5385 final byte[] q2 = Bytes.toBytes("q2");
5386 final byte[] q3 = Bytes.toBytes("q3");
5387 final byte[] q4 = Bytes.toBytes("q4");
5388
5389 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCellTTLs"));
5390 HColumnDescriptor hcd = new HColumnDescriptor(fam1);
5391 hcd.setTimeToLive(10);
5392 htd.addFamily(hcd);
5393
5394 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
5395 conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS);
5396
5397 HRegion region = HRegion.createHRegion(new HRegionInfo(htd.getTableName(),
5398 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY),
5399 TEST_UTIL.getDataTestDir(), conf, htd);
5400 assertNotNull(region);
5401 try {
5402 long now = EnvironmentEdgeManager.currentTimeMillis();
5403
5404 region.put(new Put(row).add(new KeyValue(row, fam1, q1, now,
5405 HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
5406
5407 new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
5408
5409 region.put(new Put(row).add(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY));
5410
5411 region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1,
5412 HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
5413
5414 new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
5415
5416 region.put(new Put(row).add(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY));
5417
5418
5419 region.flushcache();
5420
5421
5422 Result r = region.get(new Get(row));
5423 assertNotNull(r.getValue(fam1, q1));
5424 assertNotNull(r.getValue(fam1, q2));
5425 assertNotNull(r.getValue(fam1, q3));
5426 assertNotNull(r.getValue(fam1, q4));
5427
5428
5429 edge.incrementTime(5000);
5430
5431 r = region.get(new Get(row));
5432 assertNull(r.getValue(fam1, q1));
5433 assertNotNull(r.getValue(fam1, q2));
5434 assertNotNull(r.getValue(fam1, q3));
5435 assertNotNull(r.getValue(fam1, q4));
5436
5437
5438 edge.incrementTime(5000);
5439
5440 r = region.get(new Get(row));
5441 assertNull(r.getValue(fam1, q1));
5442 assertNull(r.getValue(fam1, q2));
5443 assertNotNull(r.getValue(fam1, q3));
5444 assertNotNull(r.getValue(fam1, q4));
5445
5446
5447 edge.incrementTime(5000);
5448
5449 r = region.get(new Get(row));
5450 assertNull(r.getValue(fam1, q1));
5451 assertNull(r.getValue(fam1, q2));
5452 assertNull(r.getValue(fam1, q3));
5453 assertNotNull(r.getValue(fam1, q4));
5454
5455
5456 edge.incrementTime(10000);
5457
5458 r = region.get(new Get(row));
5459 assertNull(r.getValue(fam1, q1));
5460 assertNull(r.getValue(fam1, q2));
5461 assertNull(r.getValue(fam1, q3));
5462 assertNull(r.getValue(fam1, q4));
5463
5464
5465
5466
5467 region.put(new Put(row).add(fam1, q1, Bytes.toBytes(1L)));
5468 r = region.get(new Get(row));
5469 byte[] val = r.getValue(fam1, q1);
5470 assertNotNull(val);
5471 assertEquals(Bytes.toLong(val), 1L);
5472
5473
5474 Increment incr = new Increment(row).addColumn(fam1, q1, 1L);
5475 incr.setTTL(5000);
5476 region.increment(incr);
5477
5478
5479 r = region.get(new Get(row));
5480 val = r.getValue(fam1, q1);
5481 assertNotNull(val);
5482 assertEquals(Bytes.toLong(val), 2L);
5483
5484
5485 edge.incrementTime(5000);
5486
5487
5488 r = region.get(new Get(row));
5489 val = r.getValue(fam1, q1);
5490 assertNotNull(val);
5491 assertEquals(Bytes.toLong(val), 1L);
5492
5493
5494 edge.incrementTime(5000);
5495
5496
5497 r = region.get(new Get(row));
5498 assertNull(r.getValue(fam1, q1));
5499
5500 } finally {
5501 HRegion.closeHRegion(region);
5502 }
5503 }
5504
5505 @Test
5506 public void testIncrementTimestampsAreMonotonic() throws IOException {
5507 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5508 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5509 EnvironmentEdgeManager.injectEdge(edge);
5510
5511 edge.setValue(10);
5512 Increment inc = new Increment(row);
5513 inc.setDurability(Durability.SKIP_WAL);
5514 inc.addColumn(fam1, qual1, 1L);
5515 region.increment(inc);
5516
5517 Result result = region.get(new Get(row));
5518 Cell c = result.getColumnLatestCell(fam1, qual1);
5519 assertNotNull(c);
5520 assertEquals(c.getTimestamp(), 10L);
5521
5522 edge.setValue(1);
5523 region.increment(inc);
5524 result = region.get(new Get(row));
5525 c = result.getColumnLatestCell(fam1, qual1);
5526 assertEquals(c.getTimestamp(), 10L);
5527 assertEquals(Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength()), 2L);
5528 }
5529
5530 @Test
5531 public void testAppendTimestampsAreMonotonic() throws IOException {
5532 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5533 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5534 EnvironmentEdgeManager.injectEdge(edge);
5535
5536 edge.setValue(10);
5537 Append a = new Append(row);
5538 a.setDurability(Durability.SKIP_WAL);
5539 a.add(fam1, qual1, qual1);
5540 region.append(a);
5541
5542 Result result = region.get(new Get(row));
5543 Cell c = result.getColumnLatestCell(fam1, qual1);
5544 assertNotNull(c);
5545 assertEquals(c.getTimestamp(), 10L);
5546
5547 edge.setValue(1);
5548 region.append(a);
5549 result = region.get(new Get(row));
5550 c = result.getColumnLatestCell(fam1, qual1);
5551 assertEquals(c.getTimestamp(), 10L);
5552
5553 byte[] expected = new byte[qual1.length*2];
5554 System.arraycopy(qual1, 0, expected, 0, qual1.length);
5555 System.arraycopy(qual1, 0, expected, qual1.length, qual1.length);
5556
5557 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
5558 expected, 0, expected.length));
5559 }
5560
5561 @Test
5562 public void testCheckAndMutateTimestampsAreMonotonic() throws IOException {
5563 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5564 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5565 EnvironmentEdgeManager.injectEdge(edge);
5566
5567 edge.setValue(10);
5568 Put p = new Put(row);
5569 p.setDurability(Durability.SKIP_WAL);
5570 p.add(fam1, qual1, qual1);
5571 region.put(p);
5572
5573 Result result = region.get(new Get(row));
5574 Cell c = result.getColumnLatestCell(fam1, qual1);
5575 assertNotNull(c);
5576 assertEquals(c.getTimestamp(), 10L);
5577
5578 edge.setValue(1);
5579 p = new Put(row);
5580 p.setDurability(Durability.SKIP_WAL);
5581 p.add(fam1, qual1, qual2);
5582 region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(qual1), p, false);
5583 result = region.get(new Get(row));
5584 c = result.getColumnLatestCell(fam1, qual1);
5585 assertEquals(c.getTimestamp(), 10L);
5586
5587 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
5588 qual2, 0, qual2.length));
5589 }
5590
5591 @Test(timeout = 60000)
5592 public void testBatchMutateWithWrongRegionException() throws Exception {
5593 final byte[] a = Bytes.toBytes("a");
5594 final byte[] b = Bytes.toBytes("b");
5595 final byte[] c = Bytes.toBytes("c");
5596
5597 int prevLockTimeout = CONF.getInt("hbase.rowlock.wait.duration", 30000);
5598 CONF.setInt("hbase.rowlock.wait.duration", 1000);
5599 final HRegion region = initHRegion(tableName, a, c, name.getMethodName(), CONF, false, fam1);
5600
5601 Mutation[] mutations = new Mutation[] {
5602 new Put(a).addImmutable(fam1, null, null),
5603 new Put(c).addImmutable(fam1, null, null),
5604 new Put(b).addImmutable(fam1, null, null),
5605 };
5606
5607 OperationStatus[] status = region.batchMutate(mutations);
5608 assertEquals(status[0].getOperationStatusCode(), OperationStatusCode.SUCCESS);
5609 assertEquals(status[1].getOperationStatusCode(), OperationStatusCode.SANITY_CHECK_FAILURE);
5610 assertEquals(status[2].getOperationStatusCode(), OperationStatusCode.SUCCESS);
5611
5612
5613
5614 final CountDownLatch obtainedRowLock = new CountDownLatch(1);
5615 ExecutorService exec = Executors.newFixedThreadPool(2);
5616 Future<Void> f1 = exec.submit(new Callable<Void>() {
5617 @Override
5618 public Void call() throws Exception {
5619 LOG.info("Acquiring row lock");
5620 RowLock rl = region.getRowLock(b);
5621 obtainedRowLock.countDown();
5622 LOG.info("Waiting for 5 seconds before releasing lock");
5623 Threads.sleep(5000);
5624 LOG.info("Releasing row lock");
5625 rl.release();
5626 return null;
5627 }
5628 });
5629 obtainedRowLock.await(30, TimeUnit.SECONDS);
5630
5631 Future<Void> f2 = exec.submit(new Callable<Void>() {
5632 @Override
5633 public Void call() throws Exception {
5634 Mutation[] mutations = new Mutation[] {
5635 new Put(a).addImmutable(fam1, null, null),
5636 new Put(b).addImmutable(fam1, null, null),
5637 };
5638
5639
5640 OperationStatus[] status = region.batchMutate(mutations);
5641 assertEquals(status[0].getOperationStatusCode(), OperationStatusCode.SUCCESS);
5642 assertEquals(status[1].getOperationStatusCode(), OperationStatusCode.SUCCESS);
5643 return null;
5644 }
5645 });
5646
5647 f1.get();
5648 f2.get();
5649
5650 CONF.setInt("hbase.rowlock.wait.duration", prevLockTimeout);
5651 }
5652
5653 @Test
5654 public void testCheckAndRowMutateTimestampsAreMonotonic() throws IOException {
5655 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5656 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5657 EnvironmentEdgeManager.injectEdge(edge);
5658
5659 edge.setValue(10);
5660 Put p = new Put(row);
5661 p.setDurability(Durability.SKIP_WAL);
5662 p.add(fam1, qual1, qual1);
5663 region.put(p);
5664
5665 Result result = region.get(new Get(row));
5666 Cell c = result.getColumnLatestCell(fam1, qual1);
5667 assertNotNull(c);
5668 assertEquals(c.getTimestamp(), 10L);
5669
5670 edge.setValue(1);
5671 p = new Put(row);
5672 p.setDurability(Durability.SKIP_WAL);
5673 p.add(fam1, qual1, qual2);
5674 RowMutations rm = new RowMutations(row);
5675 rm.add(p);
5676 region.checkAndRowMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(qual1),
5677 rm, false);
5678 result = region.get(new Get(row));
5679 c = result.getColumnLatestCell(fam1, qual1);
5680 assertEquals(c.getTimestamp(), 10L);
5681
5682 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
5683 qual2, 0, qual2.length));
5684 }
5685
5686 private static HRegion initHRegion(byte[] tableName, String callingMethod,
5687 byte[]... families) throws IOException {
5688 return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
5689 families);
5690 }
5691 }