1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21
22 import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23 import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24 import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25 import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29 import static org.junit.Assert.assertArrayEquals;
30 import static org.junit.Assert.assertEquals;
31 import static org.junit.Assert.assertFalse;
32 import static org.junit.Assert.assertNotNull;
33 import static org.junit.Assert.assertNull;
34 import static org.junit.Assert.assertTrue;
35 import static org.junit.Assert.fail;
36 import static org.mockito.Matchers.any;
37 import static org.mockito.Matchers.anyBoolean;
38 import static org.mockito.Matchers.anyLong;
39 import static org.mockito.Matchers.eq;
40 import static org.mockito.Mockito.never;
41 import static org.mockito.Mockito.spy;
42 import static org.mockito.Mockito.times;
43 import static org.mockito.Mockito.verify;
44
45 import java.io.IOException;
46 import java.io.InterruptedIOException;
47 import java.security.PrivilegedExceptionAction;
48 import java.util.ArrayList;
49 import java.util.Arrays;
50 import java.util.Collection;
51 import java.util.List;
52 import java.util.Map;
53 import java.util.NavigableMap;
54 import java.util.TreeMap;
55 import java.util.UUID;
56 import java.util.concurrent.atomic.AtomicBoolean;
57 import java.util.concurrent.atomic.AtomicInteger;
58 import java.util.concurrent.atomic.AtomicLong;
59 import java.util.concurrent.atomic.AtomicReference;
60
61 import org.apache.commons.logging.Log;
62 import org.apache.commons.logging.LogFactory;
63 import org.apache.hadoop.conf.Configuration;
64 import org.apache.hadoop.fs.FSDataOutputStream;
65 import org.apache.hadoop.fs.FileStatus;
66 import org.apache.hadoop.fs.FileSystem;
67 import org.apache.hadoop.fs.Path;
68 import org.apache.hadoop.hbase.Cell;
69 import org.apache.hadoop.hbase.CellComparator;
70 import org.apache.hadoop.hbase.CellUtil;
71 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
72 import org.apache.hadoop.hbase.DroppedSnapshotException;
73 import org.apache.hadoop.hbase.HBaseConfiguration;
74 import org.apache.hadoop.hbase.HBaseTestCase;
75 import org.apache.hadoop.hbase.HBaseTestingUtility;
76 import org.apache.hadoop.hbase.HColumnDescriptor;
77 import org.apache.hadoop.hbase.HConstants;
78 import org.apache.hadoop.hbase.RegionTooBusyException;
79 import org.apache.hadoop.hbase.Tag;
80 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
81 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
82 import org.apache.hadoop.hbase.HRegionInfo;
83 import org.apache.hadoop.hbase.HTableDescriptor;
84 import org.apache.hadoop.hbase.KeyValue;
85 import org.apache.hadoop.hbase.testclassification.MediumTests;
86 import org.apache.hadoop.hbase.MiniHBaseCluster;
87 import org.apache.hadoop.hbase.MultithreadedTestUtil;
88 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
89 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
90 import org.apache.hadoop.hbase.NotServingRegionException;
91 import org.apache.hadoop.hbase.TableName;
92 import org.apache.hadoop.hbase.TagType;
93 import org.apache.hadoop.hbase.Waiter;
94 import org.apache.hadoop.hbase.client.Append;
95 import org.apache.hadoop.hbase.client.Delete;
96 import org.apache.hadoop.hbase.client.Durability;
97 import org.apache.hadoop.hbase.client.Get;
98 import org.apache.hadoop.hbase.client.HTable;
99 import org.apache.hadoop.hbase.client.Increment;
100 import org.apache.hadoop.hbase.client.Put;
101 import org.apache.hadoop.hbase.client.Result;
102 import org.apache.hadoop.hbase.client.RowMutations;
103 import org.apache.hadoop.hbase.client.Scan;
104 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
105 import org.apache.hadoop.hbase.filter.BinaryComparator;
106 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
107 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
108 import org.apache.hadoop.hbase.filter.Filter;
109 import org.apache.hadoop.hbase.filter.FilterBase;
110 import org.apache.hadoop.hbase.filter.FilterList;
111 import org.apache.hadoop.hbase.filter.NullComparator;
112 import org.apache.hadoop.hbase.filter.PrefixFilter;
113 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
114 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
115 import org.apache.hadoop.hbase.io.hfile.HFile;
116 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
117 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
118 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
119 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
120 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
121 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
122 import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
123 import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
124 import org.apache.hadoop.hbase.regionserver.wal.FaultyHLog;
125 import org.apache.hadoop.hbase.regionserver.wal.HLog;
126 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
127 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
128 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
129 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
130 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
131 import org.apache.hadoop.hbase.security.User;
132 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
133 import org.apache.hadoop.hbase.util.Bytes;
134 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
135 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
136 import org.apache.hadoop.hbase.util.FSUtils;
137 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
138 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
139 import org.apache.hadoop.hbase.util.PairOfSameType;
140 import org.apache.hadoop.hbase.util.Threads;
141 import org.junit.After;
142 import org.junit.Assert;
143 import org.junit.Before;
144 import org.junit.Rule;
145 import org.junit.Test;
146 import org.junit.experimental.categories.Category;
147 import org.junit.rules.TestName;
148 import org.mockito.Mockito;
149
150 import com.google.common.collect.ImmutableList;
151 import com.google.common.collect.Lists;
152 import com.google.protobuf.ByteString;
153
154
155
156
157
158
159
160 @Category(MediumTests.class)
161 @SuppressWarnings("deprecation")
162 public class TestHRegion {
163
164
165 static final Log LOG = LogFactory.getLog(TestHRegion.class);
166 @Rule public TestName name = new TestName();
167
168 private static final String COLUMN_FAMILY = "MyCF";
169 private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
170
171 HRegion region = null;
172
173 private static HBaseTestingUtility TEST_UTIL;
174 public static Configuration CONF ;
175 private String dir;
176 private static FileSystem FILESYSTEM;
177 private final int MAX_VERSIONS = 2;
178
179
180 protected byte[] tableName;
181 protected String method;
182 protected final byte[] qual1 = Bytes.toBytes("qual1");
183 protected final byte[] qual2 = Bytes.toBytes("qual2");
184 protected final byte[] qual3 = Bytes.toBytes("qual3");
185 protected final byte[] value1 = Bytes.toBytes("value1");
186 protected final byte[] value2 = Bytes.toBytes("value2");
187 protected final byte[] row = Bytes.toBytes("rowA");
188 protected final byte[] row2 = Bytes.toBytes("rowB");
189
190 protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
191 .getInstance(MetricsAssertHelper.class);
192
193 @Before
194 public void setup() throws IOException {
195 TEST_UTIL = HBaseTestingUtility.createLocalHTU();
196 FILESYSTEM = TEST_UTIL.getTestFileSystem();
197 CONF = TEST_UTIL.getConfiguration();
198 dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
199 method = name.getMethodName();
200 tableName = Bytes.toBytes(name.getMethodName());
201 }
202
203 @After
204 public void tearDown() throws Exception {
205 EnvironmentEdgeManagerTestHelper.reset();
206 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
207 TEST_UTIL.cleanupTestDir();
208 }
209
210 String getName() {
211 return name.getMethodName();
212 }
213
214
215
216
217
218
219
220
221
222
223
224 @Test (timeout=60000)
225 public void testCloseCarryingSnapshot() throws IOException {
226 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
227 Store store = region.getStore(COLUMN_FAMILY_BYTES);
228
229 byte [] value = Bytes.toBytes(name.getMethodName());
230
231 Put put = new Put(value);
232 put.add(COLUMN_FAMILY_BYTES, null, value);
233
234 region.put(put);
235 StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
236 storeFlushCtx.prepare();
237
238 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
239 region.put(put);
240
241 region.close();
242 assertEquals(0, region.getMemstoreSize().get());
243 HRegion.closeHRegion(region);
244 }
245
246
247
248
249
250 @Test (timeout=60000)
251 public void testMemstoreSnapshotSize() throws IOException {
252 class MyFaultyHLog extends FaultyHLog {
253 StoreFlushContext storeFlushCtx;
254 public MyFaultyHLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
255 throws IOException {
256 super(fs, rootDir, logName, conf);
257 }
258
259 void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
260 this.storeFlushCtx = storeFlushCtx;
261 }
262
263 @Override
264 public void sync(long txid) throws IOException {
265 storeFlushCtx.prepare();
266 super.sync(txid);
267 }
268 }
269
270 FileSystem fs = FileSystem.get(CONF);
271 Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
272 MyFaultyHLog faultyLog = new MyFaultyHLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
273 HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
274 CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES);
275
276 Store store = region.getStore(COLUMN_FAMILY_BYTES);
277
278 byte [] value = Bytes.toBytes(name.getMethodName());
279 faultyLog.setStoreFlushCtx(store.createFlushContext(12345));
280
281 Put put = new Put(value);
282 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
283 faultyLog.setFailureType(FaultyHLog.FailureType.SYNC);
284
285 boolean threwIOE = false;
286 try {
287 region.put(put);
288 } catch (IOException ioe) {
289 threwIOE = true;
290 } finally {
291 assertTrue("The regionserver should have thrown an exception", threwIOE);
292 }
293 long sz = store.getFlushableSize();
294 assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
295 HRegion.closeHRegion(region);
296 }
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313 @Test (timeout=60000)
314 public void testFlushSizeAccounting() throws Exception {
315 final Configuration conf = HBaseConfiguration.create(CONF);
316
317 conf.setInt("hbase.hstore.flush.retries.number", 1);
318 final User user =
319 User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
320
321 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
322 user.runAs(new PrivilegedExceptionAction<Object>() {
323 @Override
324 public Object run() throws Exception {
325
326 FileSystem fs = FileSystem.get(conf);
327 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
328 FaultyFileSystem ffs = (FaultyFileSystem)fs;
329 HRegion region = null;
330 try {
331
332 region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
333 long size = region.getMemstoreSize().get();
334 Assert.assertEquals(0, size);
335
336 Put p1 = new Put(row);
337 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
338 region.put(p1);
339 final long sizeOfOnePut = region.getMemstoreSize().get();
340
341 try {
342 LOG.info("Flushing");
343 region.flushcache();
344 Assert.fail("Didn't bubble up IOE!");
345 } catch (DroppedSnapshotException dse) {
346
347 region.closing.set(false);
348 }
349
350 ffs.fault.set(false);
351
352 Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize().get());
353
354
355 Put p2 = new Put(row);
356 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
357 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
358 region.put(p2);
359 Assert.assertEquals(sizeOfOnePut * 3, region.getMemstoreSize().get());
360
361
362
363 region.flushcache();
364
365 Assert.assertEquals(sizeOfOnePut * 2, region.getMemstoreSize().get());
366 } finally {
367 HRegion.closeHRegion(region);
368 }
369 return null;
370 }
371 });
372 FileSystem.closeAllForUGI(user.getUGI());
373 }
374
375 @Test
376 public void testCompactionAffectedByScanners() throws Exception {
377 byte[] family = Bytes.toBytes("family");
378 this.region = initHRegion(tableName, method, CONF, family);
379
380 Put put = new Put(Bytes.toBytes("r1"));
381 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
382 region.put(put);
383 region.flushcache();
384
385 Scan scan = new Scan();
386 scan.setMaxVersions(3);
387
388 RegionScanner scanner1 = region.getScanner(scan);
389
390 Delete delete = new Delete(Bytes.toBytes("r1"));
391 region.delete(delete);
392 region.flushcache();
393
394
395 RegionScanner scanner2 = region.getScanner(scan);
396
397 List<Cell> results = new ArrayList<Cell>();
398
399 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
400
401
402 region.compactStores(true);
403
404
405 RegionScanner scanner3 = region.getScanner(scan);
406
407
408 scanner1.next(results);
409 System.out.println(results);
410 assertEquals(1, results.size());
411
412 results.clear();
413 scanner2.next(results);
414 System.out.println(results);
415 assertEquals(0, results.size());
416
417 results.clear();
418 scanner3.next(results);
419 System.out.println(results);
420 assertEquals(0, results.size());
421 }
422
423 @Test
424 public void testToShowNPEOnRegionScannerReseek() throws Exception {
425 byte[] family = Bytes.toBytes("family");
426 this.region = initHRegion(tableName, method, CONF, family);
427
428 Put put = new Put(Bytes.toBytes("r1"));
429 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
430 region.put(put);
431 put = new Put(Bytes.toBytes("r2"));
432 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
433 region.put(put);
434 region.flushcache();
435
436 Scan scan = new Scan();
437 scan.setMaxVersions(3);
438
439 RegionScanner scanner1 = region.getScanner(scan);
440
441 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
442
443 region.compactStores(true);
444
445 scanner1.reseek(Bytes.toBytes("r2"));
446 List<Cell> results = new ArrayList<Cell>();
447 scanner1.next(results);
448 Cell keyValue = results.get(0);
449 Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
450 scanner1.close();
451 }
452
453 @Test
454 public void testSkipRecoveredEditsReplay() throws Exception {
455 String method = "testSkipRecoveredEditsReplay";
456 TableName tableName = TableName.valueOf(method);
457 byte[] family = Bytes.toBytes("family");
458 this.region = initHRegion(tableName, method, CONF, family);
459 try {
460 Path regiondir = region.getRegionFileSystem().getRegionDir();
461 FileSystem fs = region.getRegionFileSystem().getFileSystem();
462 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
463
464 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
465
466 long maxSeqId = 1050;
467 long minSeqId = 1000;
468
469 for (long i = minSeqId; i <= maxSeqId; i += 10) {
470 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
471 fs.create(recoveredEdits);
472 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
473
474 long time = System.nanoTime();
475 WALEdit edit = new WALEdit();
476 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
477 .toBytes(i)));
478 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
479 HConstants.DEFAULT_CLUSTER_ID), edit));
480
481 writer.close();
482 }
483 MonitoredTask status = TaskMonitor.get().createStatus(method);
484 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
485 for (Store store : region.getStores().values()) {
486 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
487 }
488 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
489 assertEquals(maxSeqId, seqId);
490 Get get = new Get(row);
491 Result result = region.get(get);
492 for (long i = minSeqId; i <= maxSeqId; i += 10) {
493 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
494 assertEquals(1, kvs.size());
495 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
496 }
497 } finally {
498 HRegion.closeHRegion(this.region);
499 this.region = null;
500 }
501 }
502
503 @Test
504 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
505 String method = "testSkipRecoveredEditsReplaySomeIgnored";
506 TableName tableName = TableName.valueOf(method);
507 byte[] family = Bytes.toBytes("family");
508 this.region = initHRegion(tableName, method, CONF, family);
509 try {
510 Path regiondir = region.getRegionFileSystem().getRegionDir();
511 FileSystem fs = region.getRegionFileSystem().getFileSystem();
512 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
513
514 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
515
516 long maxSeqId = 1050;
517 long minSeqId = 1000;
518
519 for (long i = minSeqId; i <= maxSeqId; i += 10) {
520 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
521 fs.create(recoveredEdits);
522 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
523
524 long time = System.nanoTime();
525 WALEdit edit = new WALEdit();
526 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
527 .toBytes(i)));
528 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
529 HConstants.DEFAULT_CLUSTER_ID), edit));
530
531 writer.close();
532 }
533 long recoverSeqId = 1030;
534 MonitoredTask status = TaskMonitor.get().createStatus(method);
535 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
536 for (Store store : region.getStores().values()) {
537 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
538 }
539 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
540 assertEquals(maxSeqId, seqId);
541 Get get = new Get(row);
542 Result result = region.get(get);
543 for (long i = minSeqId; i <= maxSeqId; i += 10) {
544 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
545 if (i < recoverSeqId) {
546 assertEquals(0, kvs.size());
547 } else {
548 assertEquals(1, kvs.size());
549 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
550 }
551 }
552 } finally {
553 HRegion.closeHRegion(this.region);
554 this.region = null;
555 }
556 }
557
558 @Test
559 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
560 byte[] family = Bytes.toBytes("family");
561 this.region = initHRegion(tableName, method, CONF, family);
562 try {
563 Path regiondir = region.getRegionFileSystem().getRegionDir();
564 FileSystem fs = region.getRegionFileSystem().getFileSystem();
565
566 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
567 for (int i = 1000; i < 1050; i += 10) {
568 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
569 FSDataOutputStream dos = fs.create(recoveredEdits);
570 dos.writeInt(i);
571 dos.close();
572 }
573 long minSeqId = 2000;
574 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
575 FSDataOutputStream dos = fs.create(recoveredEdits);
576 dos.close();
577
578 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
579 for (Store store : region.getStores().values()) {
580 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
581 }
582 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
583 assertEquals(minSeqId, seqId);
584 } finally {
585 HRegion.closeHRegion(this.region);
586 this.region = null;
587 }
588 }
589
590 @Test
591 public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
592 String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
593 TableName tableName = TableName.valueOf(method);
594 byte[] family = Bytes.toBytes("family");
595 this.region = initHRegion(tableName, method, CONF, family);
596 try {
597 Path regiondir = region.getRegionFileSystem().getRegionDir();
598 FileSystem fs = region.getRegionFileSystem().getFileSystem();
599 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
600
601 assertEquals(0, region.getStoreFileList(
602 region.getStores().keySet().toArray(new byte[0][])).size());
603
604 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
605
606 long maxSeqId = 1050;
607 long minSeqId = 1000;
608
609 for (long i = minSeqId; i <= maxSeqId; i += 10) {
610 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
611 fs.create(recoveredEdits);
612 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
613
614 long time = System.nanoTime();
615 WALEdit edit = null;
616 if (i == maxSeqId) {
617 edit = WALEdit.createCompaction(region.getRegionInfo(),
618 CompactionDescriptor.newBuilder()
619 .setTableName(ByteString.copyFrom(tableName.getName()))
620 .setFamilyName(ByteString.copyFrom(regionName))
621 .setEncodedRegionName(ByteString.copyFrom(regionName))
622 .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
623 .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName()))
624 .build());
625 } else {
626 edit = new WALEdit();
627 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
628 .toBytes(i)));
629 }
630 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
631 HConstants.DEFAULT_CLUSTER_ID), edit));
632 writer.close();
633 }
634
635 long recoverSeqId = 1030;
636 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
637 MonitoredTask status = TaskMonitor.get().createStatus(method);
638 for (Store store : region.getStores().values()) {
639 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
640 }
641 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
642 assertEquals(maxSeqId, seqId);
643
644
645 assertEquals(1, region.getStoreFileList(
646 region.getStores().keySet().toArray(new byte[0][])).size());
647
648 } finally {
649 HRegion.closeHRegion(this.region);
650 this.region = null;
651 } }
652
653 @Test
654 public void testRecoveredEditsReplayCompaction() throws Exception {
655 String method = name.getMethodName();
656 TableName tableName = TableName.valueOf(method);
657 byte[] family = Bytes.toBytes("family");
658 this.region = initHRegion(tableName, method, CONF, family);
659 try {
660 Path regiondir = region.getRegionFileSystem().getRegionDir();
661 FileSystem fs = region.getRegionFileSystem().getFileSystem();
662 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
663
664 long maxSeqId = 3;
665 long minSeqId = 0;
666
667 for (long i = minSeqId; i < maxSeqId; i++) {
668 Put put = new Put(Bytes.toBytes(i));
669 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
670 region.put(put);
671 region.flushcache();
672 }
673
674
675 assertEquals(3, region.getStore(family).getStorefilesCount());
676 List<Path> storeFiles = new ArrayList<Path>(3);
677 for (StoreFile sf : region.getStore(family).getStorefiles()) {
678 storeFiles.add(sf.getPath());
679 }
680
681
682 CONF.setBoolean("hbase.hstore.compaction.complete", false);
683 region.compactStores();
684
685
686 assertEquals(3, region.getStore(family).getStorefilesCount());
687
688
689 Path tmpDir = region.getRegionFileSystem().getTempDir();
690 FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
691 String errorMsg = "Expected to find 1 file in the region temp directory "
692 + "from the compaction, could not find any";
693 assertNotNull(errorMsg, files);
694 assertEquals(errorMsg, 1, files.length);
695
696 Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
697 files[0].getPath());
698
699 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
700 .getRegionInfo(), family, storeFiles, Lists.newArrayList(newFile), region
701 .getRegionFileSystem().getStoreDir(Bytes.toString(family)));
702
703 HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
704 this.region.getRegionInfo(), compactionDescriptor, new AtomicLong(1));
705
706 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
707
708 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
709 fs.create(recoveredEdits);
710 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
711
712 long time = System.nanoTime();
713
714 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, 10, time,
715 HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(),
716 compactionDescriptor)));
717 writer.close();
718
719
720 region.getTableDesc();
721 region.getRegionInfo();
722 region.close();
723 region = HRegion.openHRegion(region, null);
724
725
726 Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
727 for (StoreFile sf : sfs) {
728 LOG.info(sf.getPath());
729 }
730 assertEquals(1, region.getStore(family).getStorefilesCount());
731 files = FSUtils.listStatus(fs, tmpDir);
732 assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
733
734 for (long i = minSeqId; i < maxSeqId; i++) {
735 Get get = new Get(Bytes.toBytes(i));
736 Result result = region.get(get);
737 byte[] value = result.getValue(family, Bytes.toBytes(i));
738 assertArrayEquals(Bytes.toBytes(i), value);
739 }
740 } finally {
741 HRegion.closeHRegion(this.region);
742 this.region = null;
743 }
744 }
745
746 @Test
747 public void testGetWhileRegionClose() throws IOException {
748 TableName tableName = TableName.valueOf(name.getMethodName());
749 Configuration hc = initSplit();
750 int numRows = 100;
751 byte[][] families = { fam1, fam2, fam3 };
752
753
754 String method = name.getMethodName();
755 this.region = initHRegion(tableName, method, hc, families);
756 try {
757
758 final int startRow = 100;
759 putData(startRow, numRows, qual1, families);
760 putData(startRow, numRows, qual2, families);
761 putData(startRow, numRows, qual3, families);
762 final AtomicBoolean done = new AtomicBoolean(false);
763 final AtomicInteger gets = new AtomicInteger(0);
764 GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
765 try {
766
767 for (int i = 0; i < threads.length / 2; i++) {
768 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
769 threads[i].setDaemon(true);
770 threads[i].start();
771 }
772
773
774 this.region.closing.set(true);
775 for (int i = threads.length / 2; i < threads.length; i++) {
776 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
777 threads[i].setDaemon(true);
778 threads[i].start();
779 }
780 } finally {
781 if (this.region != null) {
782 HRegion.closeHRegion(this.region);
783 }
784 }
785 done.set(true);
786 for (GetTillDoneOrException t : threads) {
787 try {
788 t.join();
789 } catch (InterruptedException e) {
790 e.printStackTrace();
791 }
792 if (t.e != null) {
793 LOG.info("Exception=" + t.e);
794 assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
795 }
796 }
797 } finally {
798 HRegion.closeHRegion(this.region);
799 this.region = null;
800 }
801 }
802
803
804
805
806
807 class GetTillDoneOrException extends Thread {
808 private final Get g;
809 private final AtomicBoolean done;
810 private final AtomicInteger count;
811 private Exception e;
812
813 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
814 super("getter." + i);
815 this.g = new Get(r);
816 this.done = d;
817 this.count = c;
818 }
819
820 @Override
821 public void run() {
822 while (!this.done.get()) {
823 try {
824 assertTrue(region.get(g).size() > 0);
825 this.count.incrementAndGet();
826 } catch (Exception e) {
827 this.e = e;
828 break;
829 }
830 }
831 }
832 }
833
834
835
836
837 @Test
838 public void testWeirdCacheBehaviour() throws Exception {
839 byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
840 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
841 Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
842 this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
843 try {
844 String value = "this is the value";
845 String value2 = "this is some other value";
846 String keyPrefix1 = "prefix1";
847 String keyPrefix2 = "prefix2";
848 String keyPrefix3 = "prefix3";
849 putRows(this.region, 3, value, keyPrefix1);
850 putRows(this.region, 3, value, keyPrefix2);
851 putRows(this.region, 3, value, keyPrefix3);
852 putRows(this.region, 3, value2, keyPrefix1);
853 putRows(this.region, 3, value2, keyPrefix2);
854 putRows(this.region, 3, value2, keyPrefix3);
855 System.out.println("Checking values for key: " + keyPrefix1);
856 assertEquals("Got back incorrect number of rows from scan", 3,
857 getNumberOfRows(keyPrefix1, value2, this.region));
858 System.out.println("Checking values for key: " + keyPrefix2);
859 assertEquals("Got back incorrect number of rows from scan", 3,
860 getNumberOfRows(keyPrefix2, value2, this.region));
861 System.out.println("Checking values for key: " + keyPrefix3);
862 assertEquals("Got back incorrect number of rows from scan", 3,
863 getNumberOfRows(keyPrefix3, value2, this.region));
864 deleteColumns(this.region, value2, keyPrefix1);
865 deleteColumns(this.region, value2, keyPrefix2);
866 deleteColumns(this.region, value2, keyPrefix3);
867 System.out.println("Starting important checks.....");
868 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
869 getNumberOfRows(keyPrefix1, value2, this.region));
870 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
871 getNumberOfRows(keyPrefix2, value2, this.region));
872 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
873 getNumberOfRows(keyPrefix3, value2, this.region));
874 } finally {
875 HRegion.closeHRegion(this.region);
876 this.region = null;
877 }
878 }
879
880 @Test
881 public void testAppendWithReadOnlyTable() throws Exception {
882 byte[] TABLE = Bytes.toBytes("readOnlyTable");
883 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
884 boolean exceptionCaught = false;
885 Append append = new Append(Bytes.toBytes("somerow"));
886 append.setDurability(Durability.SKIP_WAL);
887 append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
888 Bytes.toBytes("somevalue"));
889 try {
890 region.append(append);
891 } catch (IOException e) {
892 exceptionCaught = true;
893 } finally {
894 HRegion.closeHRegion(this.region);
895 this.region = null;
896 }
897 assertTrue(exceptionCaught == true);
898 }
899
900 @Test
901 public void testIncrWithReadOnlyTable() throws Exception {
902 byte[] TABLE = Bytes.toBytes("readOnlyTable");
903 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
904 boolean exceptionCaught = false;
905 Increment inc = new Increment(Bytes.toBytes("somerow"));
906 inc.setDurability(Durability.SKIP_WAL);
907 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
908 try {
909 region.increment(inc);
910 } catch (IOException e) {
911 exceptionCaught = true;
912 } finally {
913 HRegion.closeHRegion(this.region);
914 this.region = null;
915 }
916 assertTrue(exceptionCaught == true);
917 }
918
919 private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
920 InternalScanner scanner = buildScanner(keyPrefix, value, r);
921 int count = 0;
922 boolean more = false;
923 List<Cell> results = new ArrayList<Cell>();
924 do {
925 more = scanner.next(results);
926 if (results != null && !results.isEmpty())
927 count++;
928 else
929 break;
930 Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
931 delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
932 r.delete(delete);
933 results.clear();
934 } while (more);
935 assertEquals("Did not perform correct number of deletes", 3, count);
936 }
937
938 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
939 InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
940 int numberOfResults = 0;
941 List<Cell> results = new ArrayList<Cell>();
942 boolean more = false;
943 do {
944 more = resultScanner.next(results);
945 if (results != null && !results.isEmpty())
946 numberOfResults++;
947 else
948 break;
949 for (Cell kv : results) {
950 System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
951 }
952 results.clear();
953 } while (more);
954 return numberOfResults;
955 }
956
957 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
958 throws IOException {
959
960 FilterList allFilters = new FilterList();
961 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
962
963 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
964 Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
965 filter.setFilterIfMissing(true);
966 allFilters.addFilter(filter);
967 Scan scan = new Scan();
968 scan.addFamily(Bytes.toBytes("trans-blob"));
969 scan.addFamily(Bytes.toBytes("trans-type"));
970 scan.addFamily(Bytes.toBytes("trans-date"));
971 scan.addFamily(Bytes.toBytes("trans-tags"));
972 scan.addFamily(Bytes.toBytes("trans-group"));
973 scan.setFilter(allFilters);
974 return r.getScanner(scan);
975 }
976
977 private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
978 for (int i = 0; i < numRows; i++) {
979 String row = key + "_" + i
980 System.out.println(String.format("Saving row: %s, with value %s", row, value));
981 Put put = new Put(Bytes.toBytes(row));
982 put.setDurability(Durability.SKIP_WAL);
983 put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
984 put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
985 put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
986 put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
987 put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
988 r.put(put);
989 }
990 }
991
992 @Test
993 public void testFamilyWithAndWithoutColon() throws Exception {
994 byte[] b = Bytes.toBytes(getName());
995 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
996 this.region = initHRegion(b, getName(), CONF, cf);
997 try {
998 Put p = new Put(b);
999 byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
1000 p.add(cfwithcolon, cfwithcolon, cfwithcolon);
1001 boolean exception = false;
1002 try {
1003 this.region.put(p);
1004 } catch (NoSuchColumnFamilyException e) {
1005 exception = true;
1006 }
1007 assertTrue(exception);
1008 } finally {
1009 HRegion.closeHRegion(this.region);
1010 this.region = null;
1011 }
1012 }
1013
1014 @Test
1015 public void testBatchPut_whileNoRowLocksHeld() throws IOException {
1016 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1017 byte[] qual = Bytes.toBytes("qual");
1018 byte[] val = Bytes.toBytes("val");
1019 this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1020 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1021 try {
1022 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1023 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1024
1025 LOG.info("First a batch put with all valid puts");
1026 final Put[] puts = new Put[10];
1027 for (int i = 0; i < 10; i++) {
1028 puts[i] = new Put(Bytes.toBytes("row_" + i));
1029 puts[i].add(cf, qual, val);
1030 }
1031
1032 OperationStatus[] codes = this.region.batchMutate(puts);
1033 assertEquals(10, codes.length);
1034 for (int i = 0; i < 10; i++) {
1035 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1036 }
1037 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1038
1039 LOG.info("Next a batch put with one invalid family");
1040 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1041 codes = this.region.batchMutate(puts);
1042 assertEquals(10, codes.length);
1043 for (int i = 0; i < 10; i++) {
1044 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1045 codes[i].getOperationStatusCode());
1046 }
1047
1048 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1049 } finally {
1050 HRegion.closeHRegion(this.region);
1051 this.region = null;
1052 }
1053 }
1054
1055 @Test
1056 public void testBatchPut_whileMultipleRowLocksHeld() throws Exception {
1057 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1058 byte[] qual = Bytes.toBytes("qual");
1059 byte[] val = Bytes.toBytes("val");
1060 this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1061 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1062 try {
1063 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1064 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1065
1066 final Put[] puts = new Put[10];
1067 for (int i = 0; i < 10; i++) {
1068 puts[i] = new Put(Bytes.toBytes("row_" + i));
1069 puts[i].add(cf, qual, val);
1070 }
1071 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1072
1073 LOG.info("batchPut will have to break into four batches to avoid row locks");
1074 RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2"));
1075 RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_4"));
1076 RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_6"));
1077
1078 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1079 final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
1080 TestThread putter = new TestThread(ctx) {
1081 @Override
1082 public void doWork() throws IOException {
1083 retFromThread.set(region.batchMutate(puts));
1084 }
1085 };
1086 LOG.info("...starting put thread while holding locks");
1087 ctx.addThread(putter);
1088 ctx.startThreads();
1089
1090 LOG.info("...waiting for put thread to sync 1st time");
1091 waitForCounter(source, "syncTimeNumOps", syncs + 1);
1092
1093
1094
1095
1096
1097
1098 Thread regionCloseThread = new Thread() {
1099 @Override
1100 public void run() {
1101 try {
1102 HRegion.closeHRegion(region);
1103 } catch (IOException e) {
1104 throw new RuntimeException(e);
1105 }
1106 }
1107 };
1108 regionCloseThread.start();
1109
1110 LOG.info("...releasing row lock 1, which should let put thread continue");
1111 rowLock1.release();
1112
1113 LOG.info("...waiting for put thread to sync 2nd time");
1114 waitForCounter(source, "syncTimeNumOps", syncs + 2);
1115
1116 LOG.info("...releasing row lock 2, which should let put thread continue");
1117 rowLock2.release();
1118
1119 LOG.info("...waiting for put thread to sync 3rd time");
1120 waitForCounter(source, "syncTimeNumOps", syncs + 3);
1121
1122 LOG.info("...releasing row lock 3, which should let put thread continue");
1123 rowLock3.release();
1124
1125 LOG.info("...waiting for put thread to sync 4th time");
1126 waitForCounter(source, "syncTimeNumOps", syncs + 4);
1127
1128 LOG.info("...joining on put thread");
1129 ctx.stop();
1130 regionCloseThread.join();
1131
1132 OperationStatus[] codes = retFromThread.get();
1133 for (int i = 0; i < codes.length; i++) {
1134 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1135 codes[i].getOperationStatusCode());
1136 }
1137 } finally {
1138 HRegion.closeHRegion(this.region);
1139 this.region = null;
1140 }
1141 }
1142
1143 private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount)
1144 throws InterruptedException {
1145 long startWait = System.currentTimeMillis();
1146 long currentCount;
1147 while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) {
1148 Thread.sleep(100);
1149 if (System.currentTimeMillis() - startWait > 10000) {
1150 fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName,
1151 expectedCount, currentCount));
1152 }
1153 }
1154 }
1155
1156
1157 @Test
1158 public void testBatchPutWithTsSlop() throws Exception {
1159 byte[] b = Bytes.toBytes(getName());
1160 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1161 byte[] qual = Bytes.toBytes("qual");
1162 byte[] val = Bytes.toBytes("val");
1163
1164
1165 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1166 this.region = initHRegion(b, getName(), CONF, cf);
1167
1168 try {
1169 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1170 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1171 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1172
1173 final Put[] puts = new Put[10];
1174 for (int i = 0; i < 10; i++) {
1175 puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
1176 puts[i].add(cf, qual, val);
1177 }
1178
1179 OperationStatus[] codes = this.region.batchMutate(puts);
1180 assertEquals(10, codes.length);
1181 for (int i = 0; i < 10; i++) {
1182 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1183 }
1184 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1185
1186 } finally {
1187 HRegion.closeHRegion(this.region);
1188 this.region = null;
1189 }
1190
1191 }
1192
1193
1194
1195
1196 @Test
1197 public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1198 byte[] row1 = Bytes.toBytes("row1");
1199 byte[] fam1 = Bytes.toBytes("fam1");
1200 byte[] qf1 = Bytes.toBytes("qualifier");
1201 byte[] emptyVal = new byte[] {};
1202 byte[] val1 = Bytes.toBytes("value1");
1203 byte[] val2 = Bytes.toBytes("value2");
1204
1205
1206 String method = this.getName();
1207 this.region = initHRegion(tableName, method, CONF, fam1);
1208 try {
1209
1210 Put put = new Put(row1);
1211 put.add(fam1, qf1, emptyVal);
1212
1213
1214 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1215 emptyVal), put, true);
1216 assertTrue(res);
1217
1218
1219 put = new Put(row1);
1220 put.add(fam1, qf1, val1);
1221
1222
1223 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1224 put, true);
1225 assertTrue(res);
1226
1227
1228 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1229 put, true);
1230 assertFalse(res);
1231
1232 Delete delete = new Delete(row1);
1233 delete.deleteColumn(fam1, qf1);
1234 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1235 delete, true);
1236 assertFalse(res);
1237
1238 put = new Put(row1);
1239 put.add(fam1, qf1, val2);
1240
1241 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1242 put, true);
1243 assertTrue(res);
1244
1245
1246 delete = new Delete(row1);
1247 delete.deleteColumn(fam1, qf1);
1248 delete.deleteColumn(fam1, qf1);
1249 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1250 delete, true);
1251 assertTrue(res);
1252
1253 delete = new Delete(row1);
1254 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1255 delete, true);
1256 assertTrue(res);
1257
1258
1259 put = new Put(row1);
1260 put.add(fam1, qf1, val1);
1261
1262 res = region
1263 .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
1264 assertTrue(res);
1265 } finally {
1266 HRegion.closeHRegion(this.region);
1267 this.region = null;
1268 }
1269 }
1270
1271 @Test
1272 public void testCheckAndMutate_WithWrongValue() throws IOException {
1273 byte[] row1 = Bytes.toBytes("row1");
1274 byte[] fam1 = Bytes.toBytes("fam1");
1275 byte[] qf1 = Bytes.toBytes("qualifier");
1276 byte[] val1 = Bytes.toBytes("value1");
1277 byte[] val2 = Bytes.toBytes("value2");
1278
1279
1280 String method = this.getName();
1281 this.region = initHRegion(tableName, method, CONF, fam1);
1282 try {
1283
1284 Put put = new Put(row1);
1285 put.add(fam1, qf1, val1);
1286 region.put(put);
1287
1288
1289 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1290 val2), put, true);
1291 assertEquals(false, res);
1292
1293
1294 Delete delete = new Delete(row1);
1295 delete.deleteFamily(fam1);
1296 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1297 put, true);
1298 assertEquals(false, res);
1299 } finally {
1300 HRegion.closeHRegion(this.region);
1301 this.region = null;
1302 }
1303 }
1304
1305 @Test
1306 public void testCheckAndMutate_WithCorrectValue() throws IOException {
1307 byte[] row1 = Bytes.toBytes("row1");
1308 byte[] fam1 = Bytes.toBytes("fam1");
1309 byte[] qf1 = Bytes.toBytes("qualifier");
1310 byte[] val1 = Bytes.toBytes("value1");
1311
1312
1313 String method = this.getName();
1314 this.region = initHRegion(tableName, method, CONF, fam1);
1315 try {
1316
1317 Put put = new Put(row1);
1318 put.add(fam1, qf1, val1);
1319 region.put(put);
1320
1321
1322 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1323 val1), put, true);
1324 assertEquals(true, res);
1325
1326
1327 Delete delete = new Delete(row1);
1328 delete.deleteColumn(fam1, qf1);
1329 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1330 delete, true);
1331 assertEquals(true, res);
1332 } finally {
1333 HRegion.closeHRegion(this.region);
1334 this.region = null;
1335 }
1336 }
1337
1338 @Test
1339 public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1340 byte[] row1 = Bytes.toBytes("row1");
1341 byte[] fam1 = Bytes.toBytes("fam1");
1342 byte[] qf1 = Bytes.toBytes("qualifier");
1343 byte[] val1 = Bytes.toBytes("value1");
1344 byte[] val2 = Bytes.toBytes("value2");
1345 byte[] val3 = Bytes.toBytes("value3");
1346 byte[] val4 = Bytes.toBytes("value4");
1347
1348
1349 String method = this.getName();
1350 this.region = initHRegion(tableName, method, CONF, fam1);
1351 try {
1352
1353 Put put = new Put(row1);
1354 put.add(fam1, qf1, val3);
1355 region.put(put);
1356
1357
1358 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1359 new BinaryComparator(val3), put, true);
1360 assertEquals(false, res);
1361
1362
1363 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1364 new BinaryComparator(val4), put, true);
1365 assertEquals(false, res);
1366
1367
1368
1369 put = new Put(row1);
1370 put.add(fam1, qf1, val2);
1371 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1372 new BinaryComparator(val2), put, true);
1373 assertEquals(true, res);
1374
1375
1376 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1377 new BinaryComparator(val3), put, true);
1378 assertEquals(false, res);
1379
1380
1381
1382 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1383 new BinaryComparator(val2), put, true);
1384 assertEquals(true, res);
1385
1386
1387
1388 put = new Put(row1);
1389 put.add(fam1, qf1, val3);
1390 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1391 new BinaryComparator(val1), put, true);
1392 assertEquals(true, res);
1393
1394
1395 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1396 new BinaryComparator(val3), put, true);
1397 assertEquals(false, res);
1398
1399
1400 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1401 new BinaryComparator(val2), put, true);
1402 assertEquals(false, res);
1403
1404
1405
1406 put = new Put(row1);
1407 put.add(fam1, qf1, val2);
1408 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1409 new BinaryComparator(val4), put, true);
1410 assertEquals(true, res);
1411
1412
1413 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1414 new BinaryComparator(val1), put, true);
1415 assertEquals(false, res);
1416
1417
1418
1419 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1420 new BinaryComparator(val2), put, true);
1421 assertEquals(true, res);
1422
1423
1424 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1425 new BinaryComparator(val3), put, true);
1426 assertEquals(true, res);
1427 } finally {
1428 HRegion.closeHRegion(this.region);
1429 this.region = null;
1430 }
1431 }
1432
1433 @Test
1434 public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1435 byte[] row1 = Bytes.toBytes("row1");
1436 byte[] fam1 = Bytes.toBytes("fam1");
1437 byte[] fam2 = Bytes.toBytes("fam2");
1438 byte[] qf1 = Bytes.toBytes("qualifier");
1439 byte[] val1 = Bytes.toBytes("value1");
1440 byte[] val2 = Bytes.toBytes("value2");
1441
1442 byte[][] families = { fam1, fam2 };
1443
1444
1445 String method = this.getName();
1446 this.region = initHRegion(tableName, method, CONF, families);
1447 try {
1448
1449 Put put = new Put(row1);
1450 put.add(fam1, qf1, val1);
1451 region.put(put);
1452
1453
1454 long ts = System.currentTimeMillis();
1455 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1456 put = new Put(row1);
1457 put.add(kv);
1458
1459
1460 HStore store = (HStore) region.getStore(fam1);
1461 store.memstore.kvset.size();
1462
1463 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1464 val1), put, true);
1465 assertEquals(true, res);
1466 store.memstore.kvset.size();
1467
1468 Get get = new Get(row1);
1469 get.addColumn(fam2, qf1);
1470 Cell[] actual = region.get(get).rawCells();
1471
1472 Cell[] expected = { kv };
1473
1474 assertEquals(expected.length, actual.length);
1475 for (int i = 0; i < actual.length; i++) {
1476 assertEquals(expected[i], actual[i]);
1477 }
1478 } finally {
1479 HRegion.closeHRegion(this.region);
1480 this.region = null;
1481 }
1482 }
1483
1484 @Test
1485 public void testCheckAndPut_wrongRowInPut() throws IOException {
1486 TableName tableName = TableName.valueOf(name.getMethodName());
1487 this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
1488 try {
1489 Put put = new Put(row2);
1490 put.add(fam1, qual1, value1);
1491 try {
1492 region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1493 new BinaryComparator(value2), put, false);
1494 fail();
1495 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1496
1497 }
1498 } finally {
1499 HRegion.closeHRegion(this.region);
1500 this.region = null;
1501 }
1502 }
1503
1504 @Test
1505 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1506 byte[] row1 = Bytes.toBytes("row1");
1507 byte[] fam1 = Bytes.toBytes("fam1");
1508 byte[] fam2 = Bytes.toBytes("fam2");
1509 byte[] qf1 = Bytes.toBytes("qualifier1");
1510 byte[] qf2 = Bytes.toBytes("qualifier2");
1511 byte[] qf3 = Bytes.toBytes("qualifier3");
1512 byte[] val1 = Bytes.toBytes("value1");
1513 byte[] val2 = Bytes.toBytes("value2");
1514 byte[] val3 = Bytes.toBytes("value3");
1515 byte[] emptyVal = new byte[] {};
1516
1517 byte[][] families = { fam1, fam2 };
1518
1519
1520 String method = this.getName();
1521 this.region = initHRegion(tableName, method, CONF, families);
1522 try {
1523
1524 Put put = new Put(row1);
1525 put.add(fam1, qf1, val1);
1526 region.put(put);
1527 Threads.sleep(2);
1528
1529 put = new Put(row1);
1530 put.add(fam1, qf1, val2);
1531 put.add(fam2, qf1, val3);
1532 put.add(fam2, qf2, val2);
1533 put.add(fam2, qf3, val1);
1534 put.add(fam1, qf3, val1);
1535 region.put(put);
1536
1537
1538 Delete delete = new Delete(row1);
1539 delete.deleteColumn(fam1, qf1);
1540 delete.deleteColumn(fam2, qf1);
1541 delete.deleteColumn(fam1, qf3);
1542 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1543 val2), delete, true);
1544 assertEquals(true, res);
1545
1546 Get get = new Get(row1);
1547 get.addColumn(fam1, qf1);
1548 get.addColumn(fam1, qf3);
1549 get.addColumn(fam2, qf2);
1550 Result r = region.get(get);
1551 assertEquals(2, r.size());
1552 assertArrayEquals(val1, r.getValue(fam1, qf1));
1553 assertArrayEquals(val2, r.getValue(fam2, qf2));
1554
1555
1556 delete = new Delete(row1);
1557 delete.deleteFamily(fam2);
1558 res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1559 delete, true);
1560 assertEquals(true, res);
1561
1562 get = new Get(row1);
1563 r = region.get(get);
1564 assertEquals(1, r.size());
1565 assertArrayEquals(val1, r.getValue(fam1, qf1));
1566
1567
1568 delete = new Delete(row1);
1569 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1570 delete, true);
1571 assertEquals(true, res);
1572 get = new Get(row1);
1573 r = region.get(get);
1574 assertEquals(0, r.size());
1575 } finally {
1576 HRegion.closeHRegion(this.region);
1577 this.region = null;
1578 }
1579 }
1580
1581
1582
1583
1584 @Test
1585 public void testDelete_multiDeleteColumn() throws IOException {
1586 byte[] row1 = Bytes.toBytes("row1");
1587 byte[] fam1 = Bytes.toBytes("fam1");
1588 byte[] qual = Bytes.toBytes("qualifier");
1589 byte[] value = Bytes.toBytes("value");
1590
1591 Put put = new Put(row1);
1592 put.add(fam1, qual, 1, value);
1593 put.add(fam1, qual, 2, value);
1594
1595 String method = this.getName();
1596 this.region = initHRegion(tableName, method, CONF, fam1);
1597 try {
1598 region.put(put);
1599
1600
1601 Delete delete = new Delete(row1);
1602 delete.deleteColumn(fam1, qual);
1603 delete.deleteColumn(fam1, qual);
1604 region.delete(delete);
1605
1606 Get get = new Get(row1);
1607 get.addFamily(fam1);
1608 Result r = region.get(get);
1609 assertEquals(0, r.size());
1610 } finally {
1611 HRegion.closeHRegion(this.region);
1612 this.region = null;
1613 }
1614 }
1615
1616 @Test
1617 public void testDelete_CheckFamily() throws IOException {
1618 byte[] row1 = Bytes.toBytes("row1");
1619 byte[] fam1 = Bytes.toBytes("fam1");
1620 byte[] fam2 = Bytes.toBytes("fam2");
1621 byte[] fam3 = Bytes.toBytes("fam3");
1622 byte[] fam4 = Bytes.toBytes("fam4");
1623
1624
1625 String method = this.getName();
1626 this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
1627 try {
1628 List<Cell> kvs = new ArrayList<Cell>();
1629 kvs.add(new KeyValue(row1, fam4, null, null));
1630
1631
1632 byte[] family = fam2;
1633 try {
1634 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1635 Bytes.BYTES_COMPARATOR);
1636 deleteMap.put(family, kvs);
1637 region.delete(deleteMap, Durability.SYNC_WAL);
1638 } catch (Exception e) {
1639 assertTrue("Family " + new String(family) + " does not exist", false);
1640 }
1641
1642
1643 boolean ok = false;
1644 family = fam4;
1645 try {
1646 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1647 Bytes.BYTES_COMPARATOR);
1648 deleteMap.put(family, kvs);
1649 region.delete(deleteMap, Durability.SYNC_WAL);
1650 } catch (Exception e) {
1651 ok = true;
1652 }
1653 assertEquals("Family " + new String(family) + " does exist", true, ok);
1654 } finally {
1655 HRegion.closeHRegion(this.region);
1656 this.region = null;
1657 }
1658 }
1659
1660 @Test
1661 public void testDelete_mixed() throws IOException, InterruptedException {
1662 byte[] fam = Bytes.toBytes("info");
1663 byte[][] families = { fam };
1664 String method = this.getName();
1665 this.region = initHRegion(tableName, method, CONF, families);
1666 try {
1667 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1668
1669 byte[] row = Bytes.toBytes("table_name");
1670
1671 byte[] serverinfo = Bytes.toBytes("serverinfo");
1672 byte[] splitA = Bytes.toBytes("splitA");
1673 byte[] splitB = Bytes.toBytes("splitB");
1674
1675
1676 Put put = new Put(row);
1677 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1678 region.put(put);
1679
1680 put = new Put(row);
1681 put.add(fam, splitB, Bytes.toBytes("reference_B"));
1682 region.put(put);
1683
1684 put = new Put(row);
1685 put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1686 region.put(put);
1687
1688
1689 Delete delete = new Delete(row);
1690 delete.deleteColumns(fam, splitA);
1691 region.delete(delete);
1692
1693
1694 Get get = new Get(row).addColumn(fam, serverinfo);
1695 Result result = region.get(get);
1696 assertEquals(1, result.size());
1697
1698 get = new Get(row).addColumn(fam, splitA);
1699 result = region.get(get);
1700 assertEquals(0, result.size());
1701
1702 get = new Get(row).addColumn(fam, splitB);
1703 result = region.get(get);
1704 assertEquals(1, result.size());
1705
1706
1707 put = new Put(row);
1708 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1709 region.put(put);
1710 get = new Get(row);
1711 result = region.get(get);
1712 assertEquals(3, result.size());
1713
1714
1715 delete = new Delete(row);
1716 region.delete(delete);
1717 assertEquals(0, region.get(get).size());
1718
1719 region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
1720 result = region.get(get);
1721 assertEquals(1, result.size());
1722 } finally {
1723 HRegion.closeHRegion(this.region);
1724 this.region = null;
1725 }
1726 }
1727
1728 @Test
1729 public void testDeleteRowWithFutureTs() throws IOException {
1730 byte[] fam = Bytes.toBytes("info");
1731 byte[][] families = { fam };
1732 String method = this.getName();
1733 this.region = initHRegion(tableName, method, CONF, families);
1734 try {
1735 byte[] row = Bytes.toBytes("table_name");
1736
1737 byte[] serverinfo = Bytes.toBytes("serverinfo");
1738
1739
1740 Put put = new Put(row);
1741 put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
1742 region.put(put);
1743
1744
1745 Delete delete = new Delete(row);
1746 region.delete(delete);
1747
1748
1749 Get get = new Get(row).addColumn(fam, serverinfo);
1750 Result result = region.get(get);
1751 assertEquals(1, result.size());
1752
1753
1754 delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
1755 region.delete(delete);
1756
1757
1758 get = new Get(row).addColumn(fam, serverinfo);
1759 result = region.get(get);
1760 assertEquals(0, result.size());
1761 } finally {
1762 HRegion.closeHRegion(this.region);
1763 this.region = null;
1764 }
1765 }
1766
1767
1768
1769
1770
1771 @Test
1772 public void testPutWithLatestTS() throws IOException {
1773 byte[] fam = Bytes.toBytes("info");
1774 byte[][] families = { fam };
1775 String method = this.getName();
1776 this.region = initHRegion(tableName, method, CONF, families);
1777 try {
1778 byte[] row = Bytes.toBytes("row1");
1779
1780 byte[] qual = Bytes.toBytes("qual");
1781
1782
1783 Put put = new Put(row);
1784 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1785 region.put(put);
1786
1787
1788 Get get = new Get(row).addColumn(fam, qual);
1789 Result result = region.get(get);
1790 assertEquals(1, result.size());
1791 Cell kv = result.rawCells()[0];
1792 LOG.info("Got: " + kv);
1793 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1794 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1795
1796
1797
1798 row = Bytes.toBytes("row2");
1799 put = new Put(row);
1800 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1801 region.put(put);
1802
1803
1804 get = new Get(row).addColumn(fam, qual);
1805 result = region.get(get);
1806 assertEquals(1, result.size());
1807 kv = result.rawCells()[0];
1808 LOG.info("Got: " + kv);
1809 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1810 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1811 } finally {
1812 HRegion.closeHRegion(this.region);
1813 this.region = null;
1814 }
1815
1816 }
1817
1818
1819
1820
1821
1822
1823 @Test
1824 public void testPutWithTsSlop() throws IOException {
1825 byte[] fam = Bytes.toBytes("info");
1826 byte[][] families = { fam };
1827 String method = this.getName();
1828
1829
1830 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1831 this.region = initHRegion(tableName, method, CONF, families);
1832 boolean caughtExcep = false;
1833 try {
1834 try {
1835
1836 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
1837
1838 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
1839 Bytes.toBytes("value")));
1840 fail("Expected IOE for TS out of configured timerange");
1841 } catch (FailedSanityCheckException ioe) {
1842 LOG.debug("Received expected exception", ioe);
1843 caughtExcep = true;
1844 }
1845 assertTrue("Should catch FailedSanityCheckException", caughtExcep);
1846 } finally {
1847 HRegion.closeHRegion(this.region);
1848 this.region = null;
1849 }
1850 }
1851
1852 @Test
1853 public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
1854 byte[] fam1 = Bytes.toBytes("columnA");
1855 byte[] fam2 = Bytes.toBytes("columnB");
1856 this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
1857 try {
1858 byte[] rowA = Bytes.toBytes("rowA");
1859 byte[] rowB = Bytes.toBytes("rowB");
1860
1861 byte[] value = Bytes.toBytes("value");
1862
1863 Delete delete = new Delete(rowA);
1864 delete.deleteFamily(fam1);
1865
1866 region.delete(delete);
1867
1868
1869 Put put = new Put(rowA);
1870 put.add(fam2, null, value);
1871 region.put(put);
1872
1873 put = new Put(rowB);
1874 put.add(fam1, null, value);
1875 put.add(fam2, null, value);
1876 region.put(put);
1877
1878 Scan scan = new Scan();
1879 scan.addFamily(fam1).addFamily(fam2);
1880 InternalScanner s = region.getScanner(scan);
1881 List<Cell> results = new ArrayList<Cell>();
1882 s.next(results);
1883 assertTrue(CellUtil.matchingRow(results.get(0), rowA));
1884
1885 results.clear();
1886 s.next(results);
1887 assertTrue(CellUtil.matchingRow(results.get(0), rowB));
1888 } finally {
1889 HRegion.closeHRegion(this.region);
1890 this.region = null;
1891 }
1892 }
1893
1894 @Test
1895 public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
1896 Delete delete = new Delete(row);
1897 delete.deleteColumns(fam1, qual1);
1898 doTestDelete_AndPostInsert(delete);
1899 }
1900
1901 @Test
1902 public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
1903 Delete delete = new Delete(row);
1904 delete.deleteFamily(fam1);
1905 doTestDelete_AndPostInsert(delete);
1906 }
1907
1908 public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
1909 TableName tableName = TableName.valueOf(name.getMethodName());
1910 this.region = initHRegion(tableName, getName(), CONF, fam1);
1911 try {
1912 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1913 Put put = new Put(row);
1914 put.add(fam1, qual1, value1);
1915 region.put(put);
1916
1917
1918 region.delete(delete);
1919
1920
1921 put = new Put(row);
1922 put.add(fam1, qual1, value2);
1923 region.put(put);
1924
1925
1926 Get get = new Get(row);
1927 get.addColumn(fam1, qual1);
1928
1929 Result r = region.get(get);
1930 assertEquals(1, r.size());
1931 assertArrayEquals(value2, r.getValue(fam1, qual1));
1932
1933
1934 Scan scan = new Scan(row);
1935 scan.addColumn(fam1, qual1);
1936 InternalScanner s = region.getScanner(scan);
1937
1938 List<Cell> results = new ArrayList<Cell>();
1939 assertEquals(false, s.next(results));
1940 assertEquals(1, results.size());
1941 Cell kv = results.get(0);
1942
1943 assertArrayEquals(value2, CellUtil.cloneValue(kv));
1944 assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
1945 assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
1946 assertArrayEquals(row, CellUtil.cloneRow(kv));
1947 } finally {
1948 HRegion.closeHRegion(this.region);
1949 this.region = null;
1950 }
1951 }
1952
1953 @Test
1954 public void testDelete_CheckTimestampUpdated() throws IOException {
1955 TableName tableName = TableName.valueOf(name.getMethodName());
1956 byte[] row1 = Bytes.toBytes("row1");
1957 byte[] col1 = Bytes.toBytes("col1");
1958 byte[] col2 = Bytes.toBytes("col2");
1959 byte[] col3 = Bytes.toBytes("col3");
1960
1961
1962 String method = this.getName();
1963 this.region = initHRegion(tableName, method, CONF, fam1);
1964 try {
1965
1966 List<Cell> kvs = new ArrayList<Cell>();
1967 kvs.add(new KeyValue(row1, fam1, col1, null));
1968 kvs.add(new KeyValue(row1, fam1, col2, null));
1969 kvs.add(new KeyValue(row1, fam1, col3, null));
1970
1971 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1972 Bytes.BYTES_COMPARATOR);
1973 deleteMap.put(fam1, kvs);
1974 region.delete(deleteMap, Durability.SYNC_WAL);
1975
1976
1977
1978 long now = System.currentTimeMillis();
1979 KeyValue firstKv = ((HStore) region.getStore(fam1)).memstore.kvset.first();
1980 assertTrue(firstKv.getTimestamp() <= now);
1981 now = firstKv.getTimestamp();
1982 for (KeyValue kv : ((HStore) region.getStore(fam1)).memstore.kvset) {
1983 assertTrue(kv.getTimestamp() <= now);
1984 now = kv.getTimestamp();
1985 }
1986 } finally {
1987 HRegion.closeHRegion(this.region);
1988 this.region = null;
1989 }
1990 }
1991
1992
1993
1994
1995 @Test
1996 public void testGet_FamilyChecker() throws IOException {
1997 byte[] row1 = Bytes.toBytes("row1");
1998 byte[] fam1 = Bytes.toBytes("fam1");
1999 byte[] fam2 = Bytes.toBytes("False");
2000 byte[] col1 = Bytes.toBytes("col1");
2001
2002
2003 String method = this.getName();
2004 this.region = initHRegion(tableName, method, CONF, fam1);
2005 try {
2006 Get get = new Get(row1);
2007 get.addColumn(fam2, col1);
2008
2009
2010 try {
2011 region.get(get);
2012 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
2013 assertFalse(false);
2014 return;
2015 }
2016 assertFalse(true);
2017 } finally {
2018 HRegion.closeHRegion(this.region);
2019 this.region = null;
2020 }
2021 }
2022
2023 @Test
2024 public void testGet_Basic() throws IOException {
2025 byte[] row1 = Bytes.toBytes("row1");
2026 byte[] fam1 = Bytes.toBytes("fam1");
2027 byte[] col1 = Bytes.toBytes("col1");
2028 byte[] col2 = Bytes.toBytes("col2");
2029 byte[] col3 = Bytes.toBytes("col3");
2030 byte[] col4 = Bytes.toBytes("col4");
2031 byte[] col5 = Bytes.toBytes("col5");
2032
2033
2034 String method = this.getName();
2035 this.region = initHRegion(tableName, method, CONF, fam1);
2036 try {
2037
2038 Put put = new Put(row1);
2039 put.add(fam1, col1, null);
2040 put.add(fam1, col2, null);
2041 put.add(fam1, col3, null);
2042 put.add(fam1, col4, null);
2043 put.add(fam1, col5, null);
2044 region.put(put);
2045
2046 Get get = new Get(row1);
2047 get.addColumn(fam1, col2);
2048 get.addColumn(fam1, col4);
2049
2050 KeyValue kv1 = new KeyValue(row1, fam1, col2);
2051 KeyValue kv2 = new KeyValue(row1, fam1, col4);
2052 KeyValue[] expected = { kv1, kv2 };
2053
2054
2055 Result res = region.get(get);
2056 assertEquals(expected.length, res.size());
2057 for (int i = 0; i < res.size(); i++) {
2058 assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
2059 assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
2060 assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
2061 }
2062
2063
2064 Get g = new Get(row1);
2065 final int count = 2;
2066 g.setFilter(new ColumnCountGetFilter(count));
2067 res = region.get(g);
2068 assertEquals(count, res.size());
2069 } finally {
2070 HRegion.closeHRegion(this.region);
2071 this.region = null;
2072 }
2073 }
2074
2075 @Test
2076 public void testGet_Empty() throws IOException {
2077 byte[] row = Bytes.toBytes("row");
2078 byte[] fam = Bytes.toBytes("fam");
2079
2080 String method = this.getName();
2081 this.region = initHRegion(tableName, method, CONF, fam);
2082 try {
2083 Get get = new Get(row);
2084 get.addFamily(fam);
2085 Result r = region.get(get);
2086
2087 assertTrue(r.isEmpty());
2088 } finally {
2089 HRegion.closeHRegion(this.region);
2090 this.region = null;
2091 }
2092 }
2093
2094
2095
2096
2097 @Test
2098 public void testMerge() throws IOException {
2099 byte[][] families = { fam1, fam2, fam3 };
2100 Configuration hc = initSplit();
2101
2102 String method = this.getName();
2103 this.region = initHRegion(tableName, method, hc, families);
2104 try {
2105 LOG.info("" + HBaseTestCase.addContent(region, fam3));
2106 region.flushcache();
2107 region.compactStores();
2108 byte[] splitRow = region.checkSplit();
2109 assertNotNull(splitRow);
2110 LOG.info("SplitRow: " + Bytes.toString(splitRow));
2111 HRegion[] subregions = splitRegion(region, splitRow);
2112 try {
2113
2114 for (int i = 0; i < subregions.length; i++) {
2115 HRegion.openHRegion(subregions[i], null);
2116 subregions[i].compactStores();
2117 }
2118 Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
2119 Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
2120 Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
2121 long startTime = System.currentTimeMillis();
2122 region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
2123 LOG.info("Merge regions elapsed time: "
2124 + ((System.currentTimeMillis() - startTime) / 1000.0));
2125 FILESYSTEM.delete(oldRegion1, true);
2126 FILESYSTEM.delete(oldRegion2, true);
2127 FILESYSTEM.delete(oldRegionPath, true);
2128 LOG.info("splitAndMerge completed.");
2129 } finally {
2130 for (int i = 0; i < subregions.length; i++) {
2131 try {
2132 HRegion.closeHRegion(subregions[i]);
2133 } catch (IOException e) {
2134
2135 }
2136 }
2137 }
2138 } finally {
2139 HRegion.closeHRegion(this.region);
2140 this.region = null;
2141 }
2142 }
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152 HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
2153 PairOfSameType<HRegion> result = null;
2154 SplitTransaction st = new SplitTransaction(parent, midkey);
2155
2156
2157 if (!st.prepare()) {
2158 parent.clearSplit();
2159 return null;
2160 }
2161 try {
2162 result = st.execute(null, null);
2163 } catch (IOException ioe) {
2164 try {
2165 LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; "
2166 + ioe.getMessage());
2167 st.rollback(null, null);
2168 LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString());
2169 return null;
2170 } catch (RuntimeException e) {
2171
2172 LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString()
2173 + " -- aborting server", e);
2174 }
2175 }
2176 finally {
2177 parent.clearSplit();
2178 }
2179 return new HRegion[] { result.getFirst(), result.getSecond() };
2180 }
2181
2182
2183
2184
2185 @Test
2186 public void testGetScanner_WithOkFamilies() throws IOException {
2187 byte[] fam1 = Bytes.toBytes("fam1");
2188 byte[] fam2 = Bytes.toBytes("fam2");
2189
2190 byte[][] families = { fam1, fam2 };
2191
2192
2193 String method = this.getName();
2194 this.region = initHRegion(tableName, method, CONF, families);
2195 try {
2196 Scan scan = new Scan();
2197 scan.addFamily(fam1);
2198 scan.addFamily(fam2);
2199 try {
2200 region.getScanner(scan);
2201 } catch (Exception e) {
2202 assertTrue("Families could not be found in Region", false);
2203 }
2204 } finally {
2205 HRegion.closeHRegion(this.region);
2206 this.region = null;
2207 }
2208 }
2209
2210 @Test
2211 public void testGetScanner_WithNotOkFamilies() throws IOException {
2212 byte[] fam1 = Bytes.toBytes("fam1");
2213 byte[] fam2 = Bytes.toBytes("fam2");
2214
2215 byte[][] families = { fam1 };
2216
2217
2218 String method = this.getName();
2219 this.region = initHRegion(tableName, method, CONF, families);
2220 try {
2221 Scan scan = new Scan();
2222 scan.addFamily(fam2);
2223 boolean ok = false;
2224 try {
2225 region.getScanner(scan);
2226 } catch (Exception e) {
2227 ok = true;
2228 }
2229 assertTrue("Families could not be found in Region", ok);
2230 } finally {
2231 HRegion.closeHRegion(this.region);
2232 this.region = null;
2233 }
2234 }
2235
2236 @Test
2237 public void testGetScanner_WithNoFamilies() throws IOException {
2238 byte[] row1 = Bytes.toBytes("row1");
2239 byte[] fam1 = Bytes.toBytes("fam1");
2240 byte[] fam2 = Bytes.toBytes("fam2");
2241 byte[] fam3 = Bytes.toBytes("fam3");
2242 byte[] fam4 = Bytes.toBytes("fam4");
2243
2244 byte[][] families = { fam1, fam2, fam3, fam4 };
2245
2246
2247 String method = this.getName();
2248 this.region = initHRegion(tableName, method, CONF, families);
2249 try {
2250
2251
2252 Put put = new Put(row1);
2253 put.add(fam1, null, null);
2254 put.add(fam2, null, null);
2255 put.add(fam3, null, null);
2256 put.add(fam4, null, null);
2257 region.put(put);
2258
2259 Scan scan = null;
2260 HRegion.RegionScannerImpl is = null;
2261
2262
2263
2264
2265 scan = new Scan();
2266 scan.addFamily(fam2);
2267 scan.addFamily(fam4);
2268 is = (RegionScannerImpl) region.getScanner(scan);
2269 assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2270
2271 scan = new Scan();
2272 is = (RegionScannerImpl) region.getScanner(scan);
2273 assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2274 } finally {
2275 HRegion.closeHRegion(this.region);
2276 this.region = null;
2277 }
2278 }
2279
2280
2281
2282
2283
2284
2285 @Test
2286 public void testGetScanner_WithRegionClosed() throws IOException {
2287 byte[] fam1 = Bytes.toBytes("fam1");
2288 byte[] fam2 = Bytes.toBytes("fam2");
2289
2290 byte[][] families = { fam1, fam2 };
2291
2292
2293 String method = this.getName();
2294 try {
2295 this.region = initHRegion(tableName, method, CONF, families);
2296 } catch (IOException e) {
2297 e.printStackTrace();
2298 fail("Got IOException during initHRegion, " + e.getMessage());
2299 }
2300 try {
2301 region.closed.set(true);
2302 try {
2303 region.getScanner(null);
2304 fail("Expected to get an exception during getScanner on a region that is closed");
2305 } catch (NotServingRegionException e) {
2306
2307 } catch (IOException e) {
2308 fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2309 + e.getMessage());
2310 }
2311 } finally {
2312 HRegion.closeHRegion(this.region);
2313 this.region = null;
2314 }
2315 }
2316
2317 @Test
2318 public void testRegionScanner_Next() throws IOException {
2319 byte[] row1 = Bytes.toBytes("row1");
2320 byte[] row2 = Bytes.toBytes("row2");
2321 byte[] fam1 = Bytes.toBytes("fam1");
2322 byte[] fam2 = Bytes.toBytes("fam2");
2323 byte[] fam3 = Bytes.toBytes("fam3");
2324 byte[] fam4 = Bytes.toBytes("fam4");
2325
2326 byte[][] families = { fam1, fam2, fam3, fam4 };
2327 long ts = System.currentTimeMillis();
2328
2329
2330 String method = this.getName();
2331 this.region = initHRegion(tableName, method, CONF, families);
2332 try {
2333
2334 Put put = null;
2335 put = new Put(row1);
2336 put.add(fam1, (byte[]) null, ts, null);
2337 put.add(fam2, (byte[]) null, ts, null);
2338 put.add(fam3, (byte[]) null, ts, null);
2339 put.add(fam4, (byte[]) null, ts, null);
2340 region.put(put);
2341
2342 put = new Put(row2);
2343 put.add(fam1, (byte[]) null, ts, null);
2344 put.add(fam2, (byte[]) null, ts, null);
2345 put.add(fam3, (byte[]) null, ts, null);
2346 put.add(fam4, (byte[]) null, ts, null);
2347 region.put(put);
2348
2349 Scan scan = new Scan();
2350 scan.addFamily(fam2);
2351 scan.addFamily(fam4);
2352 InternalScanner is = region.getScanner(scan);
2353
2354 List<Cell> res = null;
2355
2356
2357 List<Cell> expected1 = new ArrayList<Cell>();
2358 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2359 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2360
2361 res = new ArrayList<Cell>();
2362 is.next(res);
2363 for (int i = 0; i < res.size(); i++) {
2364 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2365 }
2366
2367
2368 List<Cell> expected2 = new ArrayList<Cell>();
2369 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2370 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2371
2372 res = new ArrayList<Cell>();
2373 is.next(res);
2374 for (int i = 0; i < res.size(); i++) {
2375 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2376 }
2377 } finally {
2378 HRegion.closeHRegion(this.region);
2379 this.region = null;
2380 }
2381 }
2382
2383 @Test
2384 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2385 byte[] row1 = Bytes.toBytes("row1");
2386 byte[] qf1 = Bytes.toBytes("qualifier1");
2387 byte[] qf2 = Bytes.toBytes("qualifier2");
2388 byte[] fam1 = Bytes.toBytes("fam1");
2389 byte[][] families = { fam1 };
2390
2391 long ts1 = System.currentTimeMillis();
2392 long ts2 = ts1 + 1;
2393 long ts3 = ts1 + 2;
2394
2395
2396 String method = this.getName();
2397 this.region = initHRegion(tableName, method, CONF, families);
2398 try {
2399
2400 Put put = null;
2401 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2402 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2403 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2404
2405 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2406 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2407 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2408
2409 put = new Put(row1);
2410 put.add(kv13);
2411 put.add(kv12);
2412 put.add(kv11);
2413 put.add(kv23);
2414 put.add(kv22);
2415 put.add(kv21);
2416 region.put(put);
2417
2418
2419 List<Cell> expected = new ArrayList<Cell>();
2420 expected.add(kv13);
2421 expected.add(kv12);
2422
2423 Scan scan = new Scan(row1);
2424 scan.addColumn(fam1, qf1);
2425 scan.setMaxVersions(MAX_VERSIONS);
2426 List<Cell> actual = new ArrayList<Cell>();
2427 InternalScanner scanner = region.getScanner(scan);
2428
2429 boolean hasNext = scanner.next(actual);
2430 assertEquals(false, hasNext);
2431
2432
2433 for (int i = 0; i < expected.size(); i++) {
2434 assertEquals(expected.get(i), actual.get(i));
2435 }
2436 } finally {
2437 HRegion.closeHRegion(this.region);
2438 this.region = null;
2439 }
2440 }
2441
2442 @Test
2443 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2444 byte[] row1 = Bytes.toBytes("row1");
2445 byte[] qf1 = Bytes.toBytes("qualifier1");
2446 byte[] qf2 = Bytes.toBytes("qualifier2");
2447 byte[] fam1 = Bytes.toBytes("fam1");
2448 byte[][] families = { fam1 };
2449
2450 long ts1 = 1;
2451 long ts2 = ts1 + 1;
2452 long ts3 = ts1 + 2;
2453
2454
2455 String method = this.getName();
2456 this.region = initHRegion(tableName, method, CONF, families);
2457 try {
2458
2459 Put put = null;
2460 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2461 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2462 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2463
2464 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2465 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2466 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2467
2468 put = new Put(row1);
2469 put.add(kv13);
2470 put.add(kv12);
2471 put.add(kv11);
2472 put.add(kv23);
2473 put.add(kv22);
2474 put.add(kv21);
2475 region.put(put);
2476 region.flushcache();
2477
2478
2479 List<Cell> expected = new ArrayList<Cell>();
2480 expected.add(kv13);
2481 expected.add(kv12);
2482 expected.add(kv23);
2483 expected.add(kv22);
2484
2485 Scan scan = new Scan(row1);
2486 scan.addColumn(fam1, qf1);
2487 scan.addColumn(fam1, qf2);
2488 scan.setMaxVersions(MAX_VERSIONS);
2489 List<Cell> actual = new ArrayList<Cell>();
2490 InternalScanner scanner = region.getScanner(scan);
2491
2492 boolean hasNext = scanner.next(actual);
2493 assertEquals(false, hasNext);
2494
2495
2496 for (int i = 0; i < expected.size(); i++) {
2497 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2498 }
2499 } finally {
2500 HRegion.closeHRegion(this.region);
2501 this.region = null;
2502 }
2503 }
2504
2505 @Test
2506 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2507 byte[] row1 = Bytes.toBytes("row1");
2508 byte[] fam1 = Bytes.toBytes("fam1");
2509 byte[][] families = { fam1 };
2510 byte[] qf1 = Bytes.toBytes("qualifier1");
2511 byte[] qf2 = Bytes.toBytes("qualifier2");
2512
2513 long ts1 = 1;
2514 long ts2 = ts1 + 1;
2515 long ts3 = ts1 + 2;
2516 long ts4 = ts1 + 3;
2517
2518
2519 String method = this.getName();
2520 this.region = initHRegion(tableName, method, CONF, families);
2521 try {
2522
2523 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2524 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2525 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2526 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2527
2528 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2529 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2530 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2531 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2532
2533 Put put = null;
2534 put = new Put(row1);
2535 put.add(kv14);
2536 put.add(kv24);
2537 region.put(put);
2538 region.flushcache();
2539
2540 put = new Put(row1);
2541 put.add(kv23);
2542 put.add(kv13);
2543 region.put(put);
2544 region.flushcache();
2545
2546 put = new Put(row1);
2547 put.add(kv22);
2548 put.add(kv12);
2549 region.put(put);
2550 region.flushcache();
2551
2552 put = new Put(row1);
2553 put.add(kv21);
2554 put.add(kv11);
2555 region.put(put);
2556
2557
2558 List<Cell> expected = new ArrayList<Cell>();
2559 expected.add(kv14);
2560 expected.add(kv13);
2561 expected.add(kv12);
2562 expected.add(kv24);
2563 expected.add(kv23);
2564 expected.add(kv22);
2565
2566 Scan scan = new Scan(row1);
2567 scan.addColumn(fam1, qf1);
2568 scan.addColumn(fam1, qf2);
2569 int versions = 3;
2570 scan.setMaxVersions(versions);
2571 List<Cell> actual = new ArrayList<Cell>();
2572 InternalScanner scanner = region.getScanner(scan);
2573
2574 boolean hasNext = scanner.next(actual);
2575 assertEquals(false, hasNext);
2576
2577
2578 for (int i = 0; i < expected.size(); i++) {
2579 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2580 }
2581 } finally {
2582 HRegion.closeHRegion(this.region);
2583 this.region = null;
2584 }
2585 }
2586
2587 @Test
2588 public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2589 byte[] row1 = Bytes.toBytes("row1");
2590 byte[] qf1 = Bytes.toBytes("qualifier1");
2591 byte[] qf2 = Bytes.toBytes("qualifier2");
2592 byte[] fam1 = Bytes.toBytes("fam1");
2593 byte[][] families = { fam1 };
2594
2595 long ts1 = System.currentTimeMillis();
2596 long ts2 = ts1 + 1;
2597 long ts3 = ts1 + 2;
2598
2599
2600 String method = this.getName();
2601 this.region = initHRegion(tableName, method, CONF, families);
2602 try {
2603
2604 Put put = null;
2605 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2606 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2607 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2608
2609 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2610 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2611 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2612
2613 put = new Put(row1);
2614 put.add(kv13);
2615 put.add(kv12);
2616 put.add(kv11);
2617 put.add(kv23);
2618 put.add(kv22);
2619 put.add(kv21);
2620 region.put(put);
2621
2622
2623 List<Cell> expected = new ArrayList<Cell>();
2624 expected.add(kv13);
2625 expected.add(kv12);
2626 expected.add(kv23);
2627 expected.add(kv22);
2628
2629 Scan scan = new Scan(row1);
2630 scan.addFamily(fam1);
2631 scan.setMaxVersions(MAX_VERSIONS);
2632 List<Cell> actual = new ArrayList<Cell>();
2633 InternalScanner scanner = region.getScanner(scan);
2634
2635 boolean hasNext = scanner.next(actual);
2636 assertEquals(false, hasNext);
2637
2638
2639 for (int i = 0; i < expected.size(); i++) {
2640 assertEquals(expected.get(i), actual.get(i));
2641 }
2642 } finally {
2643 HRegion.closeHRegion(this.region);
2644 this.region = null;
2645 }
2646 }
2647
2648 @Test
2649 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
2650 byte[] row1 = Bytes.toBytes("row1");
2651 byte[] qf1 = Bytes.toBytes("qualifier1");
2652 byte[] qf2 = Bytes.toBytes("qualifier2");
2653 byte[] fam1 = Bytes.toBytes("fam1");
2654
2655 long ts1 = 1;
2656 long ts2 = ts1 + 1;
2657 long ts3 = ts1 + 2;
2658
2659
2660 String method = this.getName();
2661 this.region = initHRegion(tableName, method, CONF, fam1);
2662 try {
2663
2664 Put put = null;
2665 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2666 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2667 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2668
2669 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2670 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2671 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2672
2673 put = new Put(row1);
2674 put.add(kv13);
2675 put.add(kv12);
2676 put.add(kv11);
2677 put.add(kv23);
2678 put.add(kv22);
2679 put.add(kv21);
2680 region.put(put);
2681 region.flushcache();
2682
2683
2684 List<Cell> expected = new ArrayList<Cell>();
2685 expected.add(kv13);
2686 expected.add(kv12);
2687 expected.add(kv23);
2688 expected.add(kv22);
2689
2690 Scan scan = new Scan(row1);
2691 scan.addFamily(fam1);
2692 scan.setMaxVersions(MAX_VERSIONS);
2693 List<Cell> actual = new ArrayList<Cell>();
2694 InternalScanner scanner = region.getScanner(scan);
2695
2696 boolean hasNext = scanner.next(actual);
2697 assertEquals(false, hasNext);
2698
2699
2700 for (int i = 0; i < expected.size(); i++) {
2701 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2702 }
2703 } finally {
2704 HRegion.closeHRegion(this.region);
2705 this.region = null;
2706 }
2707 }
2708
2709 @Test
2710 public void testScanner_StopRow1542() throws IOException {
2711 byte[] family = Bytes.toBytes("testFamily");
2712 this.region = initHRegion(tableName, getName(), CONF, family);
2713 try {
2714 byte[] row1 = Bytes.toBytes("row111");
2715 byte[] row2 = Bytes.toBytes("row222");
2716 byte[] row3 = Bytes.toBytes("row333");
2717 byte[] row4 = Bytes.toBytes("row444");
2718 byte[] row5 = Bytes.toBytes("row555");
2719
2720 byte[] col1 = Bytes.toBytes("Pub111");
2721 byte[] col2 = Bytes.toBytes("Pub222");
2722
2723 Put put = new Put(row1);
2724 put.add(family, col1, Bytes.toBytes(10L));
2725 region.put(put);
2726
2727 put = new Put(row2);
2728 put.add(family, col1, Bytes.toBytes(15L));
2729 region.put(put);
2730
2731 put = new Put(row3);
2732 put.add(family, col2, Bytes.toBytes(20L));
2733 region.put(put);
2734
2735 put = new Put(row4);
2736 put.add(family, col2, Bytes.toBytes(30L));
2737 region.put(put);
2738
2739 put = new Put(row5);
2740 put.add(family, col1, Bytes.toBytes(40L));
2741 region.put(put);
2742
2743 Scan scan = new Scan(row3, row4);
2744 scan.setMaxVersions();
2745 scan.addColumn(family, col1);
2746 InternalScanner s = region.getScanner(scan);
2747
2748 List<Cell> results = new ArrayList<Cell>();
2749 assertEquals(false, s.next(results));
2750 assertEquals(0, results.size());
2751 } finally {
2752 HRegion.closeHRegion(this.region);
2753 this.region = null;
2754 }
2755 }
2756
2757 @Test
2758 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2759 byte[] row1 = Bytes.toBytes("row1");
2760 byte[] fam1 = Bytes.toBytes("fam1");
2761 byte[] qf1 = Bytes.toBytes("qualifier1");
2762 byte[] qf2 = Bytes.toBytes("quateslifier2");
2763
2764 long ts1 = 1;
2765 long ts2 = ts1 + 1;
2766 long ts3 = ts1 + 2;
2767 long ts4 = ts1 + 3;
2768
2769
2770 String method = this.getName();
2771 this.region = initHRegion(tableName, method, CONF, fam1);
2772 try {
2773
2774 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2775 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2776 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2777 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2778
2779 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2780 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2781 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2782 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2783
2784 Put put = null;
2785 put = new Put(row1);
2786 put.add(kv14);
2787 put.add(kv24);
2788 region.put(put);
2789 region.flushcache();
2790
2791 put = new Put(row1);
2792 put.add(kv23);
2793 put.add(kv13);
2794 region.put(put);
2795 region.flushcache();
2796
2797 put = new Put(row1);
2798 put.add(kv22);
2799 put.add(kv12);
2800 region.put(put);
2801 region.flushcache();
2802
2803 put = new Put(row1);
2804 put.add(kv21);
2805 put.add(kv11);
2806 region.put(put);
2807
2808
2809 List<KeyValue> expected = new ArrayList<KeyValue>();
2810 expected.add(kv14);
2811 expected.add(kv13);
2812 expected.add(kv12);
2813 expected.add(kv24);
2814 expected.add(kv23);
2815 expected.add(kv22);
2816
2817 Scan scan = new Scan(row1);
2818 int versions = 3;
2819 scan.setMaxVersions(versions);
2820 List<Cell> actual = new ArrayList<Cell>();
2821 InternalScanner scanner = region.getScanner(scan);
2822
2823 boolean hasNext = scanner.next(actual);
2824 assertEquals(false, hasNext);
2825
2826
2827 for (int i = 0; i < expected.size(); i++) {
2828 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2829 }
2830 } finally {
2831 HRegion.closeHRegion(this.region);
2832 this.region = null;
2833 }
2834 }
2835
2836
2837
2838
2839
2840
2841
2842 @Test
2843 public void testScanner_JoinedScanners() throws IOException {
2844 byte[] cf_essential = Bytes.toBytes("essential");
2845 byte[] cf_joined = Bytes.toBytes("joined");
2846 byte[] cf_alpha = Bytes.toBytes("alpha");
2847 this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
2848 try {
2849 byte[] row1 = Bytes.toBytes("row1");
2850 byte[] row2 = Bytes.toBytes("row2");
2851 byte[] row3 = Bytes.toBytes("row3");
2852
2853 byte[] col_normal = Bytes.toBytes("d");
2854 byte[] col_alpha = Bytes.toBytes("a");
2855
2856 byte[] filtered_val = Bytes.toBytes(3);
2857
2858 Put put = new Put(row1);
2859 put.add(cf_essential, col_normal, Bytes.toBytes(1));
2860 put.add(cf_joined, col_alpha, Bytes.toBytes(1));
2861 region.put(put);
2862
2863 put = new Put(row2);
2864 put.add(cf_essential, col_alpha, Bytes.toBytes(2));
2865 put.add(cf_joined, col_normal, Bytes.toBytes(2));
2866 put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
2867 region.put(put);
2868
2869 put = new Put(row3);
2870 put.add(cf_essential, col_normal, filtered_val);
2871 put.add(cf_joined, col_normal, filtered_val);
2872 region.put(put);
2873
2874
2875
2876
2877
2878 Scan scan = new Scan();
2879 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
2880 CompareOp.NOT_EQUAL, filtered_val);
2881 scan.setFilter(filter);
2882 scan.setLoadColumnFamiliesOnDemand(true);
2883 InternalScanner s = region.getScanner(scan);
2884
2885 List<Cell> results = new ArrayList<Cell>();
2886 assertTrue(s.next(results));
2887 assertEquals(results.size(), 1);
2888 results.clear();
2889
2890 assertTrue(s.next(results));
2891 assertEquals(results.size(), 3);
2892 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
2893 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
2894 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
2895 results.clear();
2896
2897 assertFalse(s.next(results));
2898 assertEquals(results.size(), 0);
2899 } finally {
2900 HRegion.closeHRegion(this.region);
2901 this.region = null;
2902 }
2903 }
2904
2905
2906
2907
2908
2909
2910 @Test
2911 public void testScanner_JoinedScannersWithLimits() throws IOException {
2912 final byte[] cf_first = Bytes.toBytes("first");
2913 final byte[] cf_second = Bytes.toBytes("second");
2914
2915 this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
2916 try {
2917 final byte[] col_a = Bytes.toBytes("a");
2918 final byte[] col_b = Bytes.toBytes("b");
2919
2920 Put put;
2921
2922 for (int i = 0; i < 10; i++) {
2923 put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
2924 put.add(cf_first, col_a, Bytes.toBytes(i));
2925 if (i < 5) {
2926 put.add(cf_first, col_b, Bytes.toBytes(i));
2927 put.add(cf_second, col_a, Bytes.toBytes(i));
2928 put.add(cf_second, col_b, Bytes.toBytes(i));
2929 }
2930 region.put(put);
2931 }
2932
2933 Scan scan = new Scan();
2934 scan.setLoadColumnFamiliesOnDemand(true);
2935 Filter bogusFilter = new FilterBase() {
2936 @Override
2937 public boolean isFamilyEssential(byte[] name) {
2938 return Bytes.equals(name, cf_first);
2939 }
2940 };
2941
2942 scan.setFilter(bogusFilter);
2943 InternalScanner s = region.getScanner(scan);
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974 List<Cell> results = new ArrayList<Cell>();
2975 int index = 0;
2976 while (true) {
2977 boolean more = s.next(results, 3);
2978 if ((index >> 1) < 5) {
2979 if (index % 2 == 0)
2980 assertEquals(results.size(), 3);
2981 else
2982 assertEquals(results.size(), 1);
2983 } else
2984 assertEquals(results.size(), 1);
2985 results.clear();
2986 index++;
2987 if (!more)
2988 break;
2989 }
2990 } finally {
2991 HRegion.closeHRegion(this.region);
2992 this.region = null;
2993 }
2994 }
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004 @Test
3005 public void testBasicSplit() throws Exception {
3006 byte[][] families = { fam1, fam2, fam3 };
3007
3008 Configuration hc = initSplit();
3009
3010 String method = this.getName();
3011 this.region = initHRegion(tableName, method, hc, families);
3012
3013 try {
3014 LOG.info("" + HBaseTestCase.addContent(region, fam3));
3015 region.flushcache();
3016 region.compactStores();
3017 byte[] splitRow = region.checkSplit();
3018 assertNotNull(splitRow);
3019 LOG.info("SplitRow: " + Bytes.toString(splitRow));
3020 HRegion[] regions = splitRegion(region, splitRow);
3021 try {
3022
3023
3024
3025 for (int i = 0; i < regions.length; i++) {
3026 regions[i] = HRegion.openHRegion(regions[i], null);
3027 }
3028
3029
3030 assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
3031 assertGet(regions[1], fam3, splitRow);
3032
3033 assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
3034 assertScan(regions[1], fam3, splitRow);
3035
3036 for (int i = 0; i < regions.length; i++) {
3037
3038
3039 for (int j = 0; j < 2; j++) {
3040 HBaseTestCase.addContent(regions[i], fam3);
3041 }
3042 HBaseTestCase.addContent(regions[i], fam2);
3043 HBaseTestCase.addContent(regions[i], fam1);
3044 regions[i].flushcache();
3045 }
3046
3047 byte[][] midkeys = new byte[regions.length][];
3048
3049 for (int i = 0; i < regions.length; i++) {
3050 regions[i].compactStores();
3051 midkeys[i] = regions[i].checkSplit();
3052 }
3053
3054 TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
3055
3056
3057 for (int i = 0; i < regions.length; i++) {
3058 HRegion[] rs = null;
3059 if (midkeys[i] != null) {
3060 rs = splitRegion(regions[i], midkeys[i]);
3061 for (int j = 0; j < rs.length; j++) {
3062 sortedMap.put(Bytes.toString(rs[j].getRegionName()), HRegion.openHRegion(rs[j], null));
3063 }
3064 }
3065 }
3066 LOG.info("Made 4 regions");
3067
3068
3069 int interval = (LAST_CHAR - FIRST_CHAR) / 3;
3070 byte[] b = Bytes.toBytes(START_KEY);
3071 for (HRegion r : sortedMap.values()) {
3072 assertGet(r, fam3, b);
3073 b[0] += interval;
3074 }
3075 } finally {
3076 for (int i = 0; i < regions.length; i++) {
3077 try {
3078 regions[i].close();
3079 } catch (IOException e) {
3080
3081 }
3082 }
3083 }
3084 } finally {
3085 HRegion.closeHRegion(this.region);
3086 this.region = null;
3087 }
3088 }
3089
3090 @Test
3091 public void testSplitRegion() throws IOException {
3092 byte[] qualifier = Bytes.toBytes("qualifier");
3093 Configuration hc = initSplit();
3094 int numRows = 10;
3095 byte[][] families = { fam1, fam3 };
3096
3097
3098 String method = this.getName();
3099 this.region = initHRegion(tableName, method, hc, families);
3100
3101
3102 int startRow = 100;
3103 putData(startRow, numRows, qualifier, families);
3104 int splitRow = startRow + numRows;
3105 putData(splitRow, numRows, qualifier, families);
3106 region.flushcache();
3107
3108 HRegion[] regions = null;
3109 try {
3110 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
3111
3112 for (int i = 0; i < regions.length; i++) {
3113 regions[i] = HRegion.openHRegion(regions[i], null);
3114 }
3115
3116 assertEquals(2, regions.length);
3117
3118
3119
3120 verifyData(regions[0], startRow, numRows, qualifier, families);
3121 verifyData(regions[1], splitRow, numRows, qualifier, families);
3122
3123 } finally {
3124 HRegion.closeHRegion(this.region);
3125 this.region = null;
3126 }
3127 }
3128
3129 @Test
3130 public void testClearForceSplit() throws IOException {
3131 byte[] qualifier = Bytes.toBytes("qualifier");
3132 Configuration hc = initSplit();
3133 int numRows = 10;
3134 byte[][] families = { fam1, fam3 };
3135
3136
3137 String method = this.getName();
3138 this.region = initHRegion(tableName, method, hc, families);
3139
3140
3141 int startRow = 100;
3142 putData(startRow, numRows, qualifier, families);
3143 int splitRow = startRow + numRows;
3144 byte[] splitRowBytes = Bytes.toBytes("" + splitRow);
3145 putData(splitRow, numRows, qualifier, families);
3146 region.flushcache();
3147
3148 HRegion[] regions = null;
3149 try {
3150
3151 region.forceSplit(splitRowBytes);
3152 assertTrue(region.shouldForceSplit());
3153
3154 assertTrue(Bytes.equals(splitRowBytes, region.checkSplit()));
3155
3156
3157 HStore storeMock = Mockito.mock(HStore.class);
3158 Mockito.when(storeMock.hasReferences()).thenReturn(true);
3159 Mockito.when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
3160 Mockito.when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
3161 Mockito.when(storeMock.getColumnFamilyName()).thenReturn("cf");
3162 region.stores.put(Bytes.toBytes(storeMock.getColumnFamilyName()), storeMock);
3163 assertTrue(region.hasReferences());
3164
3165
3166 regions = splitRegion(region, splitRowBytes);
3167 assertNull(regions);
3168
3169
3170 assertFalse(region.shouldForceSplit());
3171
3172
3173 region.stores.remove(Bytes.toBytes(storeMock.getColumnFamilyName()));
3174 assertFalse(region.hasReferences());
3175
3176
3177 regions = splitRegion(region, splitRowBytes);
3178
3179
3180 for (int i = 0; i < regions.length; i++) {
3181 regions[i] = HRegion.openHRegion(regions[i], null);
3182 }
3183
3184 assertEquals(2, regions.length);
3185
3186
3187
3188 verifyData(regions[0], startRow, numRows, qualifier, families);
3189 verifyData(regions[1], splitRow, numRows, qualifier, families);
3190
3191 } finally {
3192 HRegion.closeHRegion(this.region);
3193 this.region = null;
3194 }
3195 }
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207 @Test
3208 public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3209 byte[] family = Bytes.toBytes("family");
3210 int numRows = 1000;
3211 int flushAndScanInterval = 10;
3212 int compactInterval = 10 * flushAndScanInterval;
3213
3214 String method = "testFlushCacheWhileScanning";
3215 this.region = initHRegion(tableName, method, CONF, family);
3216 try {
3217 FlushThread flushThread = new FlushThread();
3218 flushThread.start();
3219
3220 Scan scan = new Scan();
3221 scan.addFamily(family);
3222 scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
3223 new BinaryComparator(Bytes.toBytes(5L))));
3224
3225 int expectedCount = 0;
3226 List<Cell> res = new ArrayList<Cell>();
3227
3228 boolean toggle = true;
3229 for (long i = 0; i < numRows; i++) {
3230 Put put = new Put(Bytes.toBytes(i));
3231 put.setDurability(Durability.SKIP_WAL);
3232 put.add(family, qual1, Bytes.toBytes(i % 10));
3233 region.put(put);
3234
3235 if (i != 0 && i % compactInterval == 0) {
3236
3237 region.compactStores(true);
3238 }
3239
3240 if (i % 10 == 5L) {
3241 expectedCount++;
3242 }
3243
3244 if (i != 0 && i % flushAndScanInterval == 0) {
3245 res.clear();
3246 InternalScanner scanner = region.getScanner(scan);
3247 if (toggle) {
3248 flushThread.flush();
3249 }
3250 while (scanner.next(res))
3251 ;
3252 if (!toggle) {
3253 flushThread.flush();
3254 }
3255 assertEquals("i=" + i, expectedCount, res.size());
3256 toggle = !toggle;
3257 }
3258 }
3259
3260 flushThread.done();
3261 flushThread.join();
3262 flushThread.checkNoError();
3263 } finally {
3264 HRegion.closeHRegion(this.region);
3265 this.region = null;
3266 }
3267 }
3268
3269 protected class FlushThread extends Thread {
3270 private volatile boolean done;
3271 private Throwable error = null;
3272
3273 public void done() {
3274 done = true;
3275 synchronized (this) {
3276 interrupt();
3277 }
3278 }
3279
3280 public void checkNoError() {
3281 if (error != null) {
3282 assertNull(error);
3283 }
3284 }
3285
3286 @Override
3287 public void run() {
3288 done = false;
3289 while (!done) {
3290 synchronized (this) {
3291 try {
3292 wait();
3293 } catch (InterruptedException ignored) {
3294 if (done) {
3295 break;
3296 }
3297 }
3298 }
3299 try {
3300 region.flushcache();
3301 } catch (IOException e) {
3302 if (!done) {
3303 LOG.error("Error while flusing cache", e);
3304 error = e;
3305 }
3306 break;
3307 }
3308 }
3309
3310 }
3311
3312 public void flush() {
3313 synchronized (this) {
3314 notify();
3315 }
3316
3317 }
3318 }
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329 @Test
3330 public void testWritesWhileScanning() throws IOException, InterruptedException {
3331 int testCount = 100;
3332 int numRows = 1;
3333 int numFamilies = 10;
3334 int numQualifiers = 100;
3335 int flushInterval = 7;
3336 int compactInterval = 5 * flushInterval;
3337 byte[][] families = new byte[numFamilies][];
3338 for (int i = 0; i < numFamilies; i++) {
3339 families[i] = Bytes.toBytes("family" + i);
3340 }
3341 byte[][] qualifiers = new byte[numQualifiers][];
3342 for (int i = 0; i < numQualifiers; i++) {
3343 qualifiers[i] = Bytes.toBytes("qual" + i);
3344 }
3345
3346 String method = "testWritesWhileScanning";
3347 this.region = initHRegion(tableName, method, CONF, families);
3348 try {
3349 PutThread putThread = new PutThread(numRows, families, qualifiers);
3350 putThread.start();
3351 putThread.waitForFirstPut();
3352
3353 FlushThread flushThread = new FlushThread();
3354 flushThread.start();
3355
3356 Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3357
3358 int expectedCount = numFamilies * numQualifiers;
3359 List<Cell> res = new ArrayList<Cell>();
3360
3361 long prevTimestamp = 0L;
3362 for (int i = 0; i < testCount; i++) {
3363
3364 if (i != 0 && i % compactInterval == 0) {
3365 region.compactStores(true);
3366 }
3367
3368 if (i != 0 && i % flushInterval == 0) {
3369 flushThread.flush();
3370 }
3371
3372 boolean previousEmpty = res.isEmpty();
3373 res.clear();
3374 InternalScanner scanner = region.getScanner(scan);
3375 while (scanner.next(res))
3376 ;
3377 if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3378 assertEquals("i=" + i, expectedCount, res.size());
3379 long timestamp = res.get(0).getTimestamp();
3380 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3381 timestamp >= prevTimestamp);
3382 prevTimestamp = timestamp;
3383 }
3384 }
3385
3386 putThread.done();
3387
3388 region.flushcache();
3389
3390 putThread.join();
3391 putThread.checkNoError();
3392
3393 flushThread.done();
3394 flushThread.join();
3395 flushThread.checkNoError();
3396 } finally {
3397 try {
3398 HRegion.closeHRegion(this.region);
3399 } catch (DroppedSnapshotException dse) {
3400
3401
3402
3403
3404 }
3405 this.region = null;
3406 }
3407 }
3408
3409 protected class PutThread extends Thread {
3410 private volatile boolean done;
3411 private volatile int numPutsFinished = 0;
3412
3413 private Throwable error = null;
3414 private int numRows;
3415 private byte[][] families;
3416 private byte[][] qualifiers;
3417
3418 private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3419 this.numRows = numRows;
3420 this.families = families;
3421 this.qualifiers = qualifiers;
3422 }
3423
3424
3425
3426
3427 public void waitForFirstPut() throws InterruptedException {
3428
3429 while (numPutsFinished == 0) {
3430 checkNoError();
3431 Thread.sleep(50);
3432 }
3433 }
3434
3435 public void done() {
3436 done = true;
3437 synchronized (this) {
3438 interrupt();
3439 }
3440 }
3441
3442 public void checkNoError() {
3443 if (error != null) {
3444 assertNull(error);
3445 }
3446 }
3447
3448 @Override
3449 public void run() {
3450 done = false;
3451 while (!done) {
3452 try {
3453 for (int r = 0; r < numRows; r++) {
3454 byte[] row = Bytes.toBytes("row" + r);
3455 Put put = new Put(row);
3456 put.setDurability(Durability.SKIP_WAL);
3457 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3458 for (byte[] family : families) {
3459 for (byte[] qualifier : qualifiers) {
3460 put.add(family, qualifier, (long) numPutsFinished, value);
3461 }
3462 }
3463 region.put(put);
3464 numPutsFinished++;
3465 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3466 System.out.println("put iteration = " + numPutsFinished);
3467 Delete delete = new Delete(row, (long) numPutsFinished - 30);
3468 region.delete(delete);
3469 }
3470 numPutsFinished++;
3471 }
3472 } catch (InterruptedIOException e) {
3473
3474 } catch (IOException e) {
3475 LOG.error("error while putting records", e);
3476 error = e;
3477 break;
3478 }
3479 }
3480
3481 }
3482
3483 }
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494 @Test
3495 public void testWritesWhileGetting() throws Exception {
3496 int testCount = 100;
3497 int numRows = 1;
3498 int numFamilies = 10;
3499 int numQualifiers = 100;
3500 int compactInterval = 100;
3501 byte[][] families = new byte[numFamilies][];
3502 for (int i = 0; i < numFamilies; i++) {
3503 families[i] = Bytes.toBytes("family" + i);
3504 }
3505 byte[][] qualifiers = new byte[numQualifiers][];
3506 for (int i = 0; i < numQualifiers; i++) {
3507 qualifiers[i] = Bytes.toBytes("qual" + i);
3508 }
3509
3510
3511 String method = "testWritesWhileGetting";
3512
3513
3514
3515
3516
3517 Configuration conf = HBaseConfiguration.create(CONF);
3518 conf.setInt("hbase.hstore.compaction.min", 1);
3519 conf.setInt("hbase.hstore.compaction.max", 1000);
3520 this.region = initHRegion(tableName, method, conf, families);
3521 PutThread putThread = null;
3522 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3523 try {
3524 putThread = new PutThread(numRows, families, qualifiers);
3525 putThread.start();
3526 putThread.waitForFirstPut();
3527
3528
3529 ctx.addThread(new RepeatingTestThread(ctx) {
3530 private int flushesSinceCompact = 0;
3531 private final int maxFlushesSinceCompact = 20;
3532
3533 @Override
3534 public void doAnAction() throws Exception {
3535 if (region.flushcache().isCompactionNeeded()) {
3536 ++flushesSinceCompact;
3537 }
3538
3539
3540 if (flushesSinceCompact == maxFlushesSinceCompact) {
3541 region.compactStores(false);
3542 flushesSinceCompact = 0;
3543 }
3544 }
3545 });
3546 ctx.startThreads();
3547
3548 Get get = new Get(Bytes.toBytes("row0"));
3549 Result result = null;
3550
3551 int expectedCount = numFamilies * numQualifiers;
3552
3553 long prevTimestamp = 0L;
3554 for (int i = 0; i < testCount; i++) {
3555
3556 boolean previousEmpty = result == null || result.isEmpty();
3557 result = region.get(get);
3558 if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3559 assertEquals("i=" + i, expectedCount, result.size());
3560
3561
3562 long timestamp = 0;
3563 for (Cell kv : result.rawCells()) {
3564 if (CellUtil.matchingFamily(kv, families[0])
3565 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3566 timestamp = kv.getTimestamp();
3567 }
3568 }
3569 assertTrue(timestamp >= prevTimestamp);
3570 prevTimestamp = timestamp;
3571 Cell previousKV = null;
3572
3573 for (Cell kv : result.rawCells()) {
3574 byte[] thisValue = CellUtil.cloneValue(kv);
3575 if (previousKV != null) {
3576 if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3577 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3578 + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3579 + "(memStoreTS:" + kv.getMvccVersion() + ")");
3580 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3581 }
3582 }
3583 previousKV = kv;
3584 }
3585 }
3586 }
3587 } finally {
3588 if (putThread != null)
3589 putThread.done();
3590
3591 region.flushcache();
3592
3593 if (putThread != null) {
3594 putThread.join();
3595 putThread.checkNoError();
3596 }
3597
3598 ctx.stop();
3599 HRegion.closeHRegion(this.region);
3600 this.region = null;
3601 }
3602 }
3603
3604 @Test
3605 public void testHolesInMeta() throws Exception {
3606 byte[] family = Bytes.toBytes("family");
3607 this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF,
3608 false, family);
3609 try {
3610 byte[] rowNotServed = Bytes.toBytes("a");
3611 Get g = new Get(rowNotServed);
3612 try {
3613 region.get(g);
3614 fail();
3615 } catch (WrongRegionException x) {
3616
3617 }
3618 byte[] row = Bytes.toBytes("y");
3619 g = new Get(row);
3620 region.get(g);
3621 } finally {
3622 HRegion.closeHRegion(this.region);
3623 this.region = null;
3624 }
3625 }
3626
3627 @Test
3628 public void testIndexesScanWithOneDeletedRow() throws IOException {
3629 byte[] family = Bytes.toBytes("family");
3630
3631
3632 String method = "testIndexesScanWithOneDeletedRow";
3633 this.region = initHRegion(tableName, method, CONF, family);
3634 try {
3635 Put put = new Put(Bytes.toBytes(1L));
3636 put.add(family, qual1, 1L, Bytes.toBytes(1L));
3637 region.put(put);
3638
3639 region.flushcache();
3640
3641 Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3642 region.delete(delete);
3643
3644 put = new Put(Bytes.toBytes(2L));
3645 put.add(family, qual1, 2L, Bytes.toBytes(2L));
3646 region.put(put);
3647
3648 Scan idxScan = new Scan();
3649 idxScan.addFamily(family);
3650 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
3651 new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
3652 new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
3653 CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
3654 InternalScanner scanner = region.getScanner(idxScan);
3655 List<Cell> res = new ArrayList<Cell>();
3656
3657 while (scanner.next(res))
3658 ;
3659 assertEquals(1L, res.size());
3660 } finally {
3661 HRegion.closeHRegion(this.region);
3662 this.region = null;
3663 }
3664 }
3665
3666
3667
3668
3669 @Test
3670 public void testBloomFilterSize() throws IOException {
3671 byte[] fam1 = Bytes.toBytes("fam1");
3672 byte[] qf1 = Bytes.toBytes("col");
3673 byte[] val1 = Bytes.toBytes("value1");
3674
3675 HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
3676 .setBloomFilterType(BloomType.ROWCOL);
3677
3678 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3679 htd.addFamily(hcd);
3680 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3681 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3682 try {
3683 int num_unique_rows = 10;
3684 int duplicate_multiplier = 2;
3685 int num_storefiles = 4;
3686
3687 int version = 0;
3688 for (int f = 0; f < num_storefiles; f++) {
3689 for (int i = 0; i < duplicate_multiplier; i++) {
3690 for (int j = 0; j < num_unique_rows; j++) {
3691 Put put = new Put(Bytes.toBytes("row" + j));
3692 put.setDurability(Durability.SKIP_WAL);
3693 put.add(fam1, qf1, version++, val1);
3694 region.put(put);
3695 }
3696 }
3697 region.flushcache();
3698 }
3699
3700 HStore store = (HStore) region.getStore(fam1);
3701 Collection<StoreFile> storeFiles = store.getStorefiles();
3702 for (StoreFile storefile : storeFiles) {
3703 StoreFile.Reader reader = storefile.getReader();
3704 reader.loadFileInfo();
3705 reader.loadBloomfilter();
3706 assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
3707 assertEquals(num_unique_rows, reader.getFilterEntries());
3708 }
3709
3710 region.compactStores(true);
3711
3712
3713 storeFiles = store.getStorefiles();
3714 for (StoreFile storefile : storeFiles) {
3715 StoreFile.Reader reader = storefile.getReader();
3716 reader.loadFileInfo();
3717 reader.loadBloomfilter();
3718 assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
3719 assertEquals(num_unique_rows, reader.getFilterEntries());
3720 }
3721 } finally {
3722 HRegion.closeHRegion(this.region);
3723 this.region = null;
3724 }
3725 }
3726
3727 @Test
3728 public void testAllColumnsWithBloomFilter() throws IOException {
3729 byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
3730 byte[] FAMILY = Bytes.toBytes("family");
3731
3732
3733 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
3734 .setBloomFilterType(BloomType.ROWCOL);
3735 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
3736 htd.addFamily(hcd);
3737 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3738 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3739 try {
3740
3741 byte row[] = Bytes.toBytes("row:" + 0);
3742 byte column[] = Bytes.toBytes("column:" + 0);
3743 Put put = new Put(row);
3744 put.setDurability(Durability.SKIP_WAL);
3745 for (long idx = 1; idx <= 4; idx++) {
3746 put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
3747 }
3748 region.put(put);
3749
3750
3751 region.flushcache();
3752
3753
3754 Get get = new Get(row);
3755 get.setMaxVersions();
3756 Cell[] kvs = region.get(get).rawCells();
3757
3758
3759 assertEquals(4, kvs.length);
3760 checkOneCell(kvs[0], FAMILY, 0, 0, 4);
3761 checkOneCell(kvs[1], FAMILY, 0, 0, 3);
3762 checkOneCell(kvs[2], FAMILY, 0, 0, 2);
3763 checkOneCell(kvs[3], FAMILY, 0, 0, 1);
3764 } finally {
3765 HRegion.closeHRegion(this.region);
3766 this.region = null;
3767 }
3768 }
3769
3770
3771
3772
3773
3774
3775 @Test
3776 public void testDeleteRowWithBloomFilter() throws IOException {
3777 byte[] familyName = Bytes.toBytes("familyName");
3778
3779
3780 HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
3781 .setBloomFilterType(BloomType.ROWCOL);
3782
3783 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3784 htd.addFamily(hcd);
3785 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3786 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3787 try {
3788
3789 byte row[] = Bytes.toBytes("row1");
3790 byte col[] = Bytes.toBytes("col1");
3791
3792 Put put = new Put(row);
3793 put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
3794 region.put(put);
3795 region.flushcache();
3796
3797 Delete del = new Delete(row);
3798 region.delete(del);
3799 region.flushcache();
3800
3801
3802 Get get = new Get(row);
3803 get.addColumn(familyName, col);
3804
3805 Cell[] keyValues = region.get(get).rawCells();
3806 assertTrue(keyValues.length == 0);
3807 } finally {
3808 HRegion.closeHRegion(this.region);
3809 this.region = null;
3810 }
3811 }
3812
3813 @Test
3814 public void testgetHDFSBlocksDistribution() throws Exception {
3815 HBaseTestingUtility htu = new HBaseTestingUtility();
3816 final int DEFAULT_BLOCK_SIZE = 1024;
3817 htu.getConfiguration().setLong("dfs.namenode.fs-limits.min-block-size", 0);
3818 htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
3819 htu.getConfiguration().setInt("dfs.replication", 2);
3820
3821
3822 MiniHBaseCluster cluster = null;
3823 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
3824 int regionServersCount = 3;
3825
3826 try {
3827 cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
3828 byte[][] families = { fam1, fam2 };
3829 HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);
3830
3831
3832 byte row[] = Bytes.toBytes("row1");
3833 byte col[] = Bytes.toBytes("col1");
3834
3835 Put put = new Put(row);
3836 put.add(fam1, col, 1, Bytes.toBytes("test1"));
3837 put.add(fam2, col, 1, Bytes.toBytes("test2"));
3838 ht.put(put);
3839
3840 HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
3841 .get(0);
3842 firstRegion.flushcache();
3843 HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
3844
3845
3846
3847
3848
3849 long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
3850
3851 String topHost = blocksDistribution1.getTopHosts().get(0);
3852 long topHostWeight = blocksDistribution1.getWeight(topHost);
3853 assertTrue(uniqueBlocksWeight1 == topHostWeight);
3854
3855
3856
3857 HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
3858 htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
3859 long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
3860
3861 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
3862
3863 ht.close();
3864 } finally {
3865 if (cluster != null) {
3866 htu.shutdownMiniCluster();
3867 }
3868 }
3869 }
3870
3871
3872
3873
3874
3875
3876
3877 @Test
3878 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
3879 TableName tableName = TableName.valueOf(name.getMethodName());
3880 HRegionInfo info = null;
3881 try {
3882 FileSystem fs = Mockito.mock(FileSystem.class);
3883 Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
3884 HTableDescriptor htd = new HTableDescriptor(tableName);
3885 htd.addFamily(new HColumnDescriptor("cf"));
3886 info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
3887 HConstants.EMPTY_BYTE_ARRAY, false);
3888 Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
3889 region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
3890
3891 region.initialize();
3892 fail("Region initialization should fail due to IOException");
3893 } catch (IOException io) {
3894 List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
3895 for (MonitoredTask monitoredTask : tasks) {
3896 if (!(monitoredTask instanceof MonitoredRPCHandler)
3897 && monitoredTask.getDescription().contains(region.toString())) {
3898 assertTrue("Region state should be ABORTED.",
3899 monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
3900 break;
3901 }
3902 }
3903 } finally {
3904 HRegion.closeHRegion(region);
3905 }
3906 }
3907
3908
3909
3910
3911
3912 @Test
3913 public void testRegionInfoFileCreation() throws IOException {
3914 Path rootDir = new Path(dir + "testRegionInfoFileCreation");
3915
3916 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
3917 htd.addFamily(new HColumnDescriptor("cf"));
3918
3919 HRegionInfo hri = new HRegionInfo(htd.getTableName());
3920
3921
3922 HRegion region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
3923
3924 Path regionDir = region.getRegionFileSystem().getRegionDir();
3925 FileSystem fs = region.getRegionFileSystem().getFileSystem();
3926 HRegion.closeHRegion(region);
3927
3928 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
3929
3930
3931 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3932 fs.exists(regionInfoFile));
3933
3934
3935 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3936 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3937 HRegion.closeHRegion(region);
3938
3939
3940 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3941 fs.exists(regionInfoFile));
3942
3943
3944 fs.delete(regionInfoFile);
3945 assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
3946 fs.exists(regionInfoFile));
3947
3948 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3949
3950 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3951 HRegion.closeHRegion(region);
3952
3953
3954 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3955 fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
3956 }
3957
3958
3959
3960
3961 private static class Incrementer implements Runnable {
3962 private HRegion region;
3963 private final static byte[] incRow = Bytes.toBytes("incRow");
3964 private final static byte[] family = Bytes.toBytes("family");
3965 private final static byte[] qualifier = Bytes.toBytes("qualifier");
3966 private final static long ONE = 1l;
3967 private int incCounter;
3968
3969 public Incrementer(HRegion region, int incCounter) {
3970 this.region = region;
3971 this.incCounter = incCounter;
3972 }
3973
3974 @Override
3975 public void run() {
3976 int count = 0;
3977 while (count < incCounter) {
3978 Increment inc = new Increment(incRow);
3979 inc.addColumn(family, qualifier, ONE);
3980 count++;
3981 try {
3982 region.increment(inc);
3983 } catch (IOException e) {
3984 e.printStackTrace();
3985 break;
3986 }
3987 }
3988 }
3989 }
3990
3991
3992
3993
3994
3995 @Test
3996 public void testParallelIncrementWithMemStoreFlush() throws Exception {
3997 byte[] family = Incrementer.family;
3998 this.region = initHRegion(tableName, method, CONF, family);
3999 final HRegion region = this.region;
4000 final AtomicBoolean incrementDone = new AtomicBoolean(false);
4001 Runnable flusher = new Runnable() {
4002 @Override
4003 public void run() {
4004 while (!incrementDone.get()) {
4005 try {
4006 region.flushcache();
4007 } catch (Exception e) {
4008 e.printStackTrace();
4009 }
4010 }
4011 }
4012 };
4013
4014
4015 int threadNum = 20;
4016 int incCounter = 100;
4017 long expected = threadNum * incCounter;
4018 Thread[] incrementers = new Thread[threadNum];
4019 Thread flushThread = new Thread(flusher);
4020 for (int i = 0; i < threadNum; i++) {
4021 incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
4022 incrementers[i].start();
4023 }
4024 flushThread.start();
4025 for (int i = 0; i < threadNum; i++) {
4026 incrementers[i].join();
4027 }
4028
4029 incrementDone.set(true);
4030 flushThread.join();
4031
4032 Get get = new Get(Incrementer.incRow);
4033 get.addColumn(Incrementer.family, Incrementer.qualifier);
4034 get.setMaxVersions(1);
4035 Result res = this.region.get(get);
4036 List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
4037
4038
4039 assertEquals(kvs.size(), 1);
4040 Cell kv = kvs.get(0);
4041 assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
4042 this.region = null;
4043 }
4044
4045
4046
4047
4048 private static class Appender implements Runnable {
4049 private HRegion region;
4050 private final static byte[] appendRow = Bytes.toBytes("appendRow");
4051 private final static byte[] family = Bytes.toBytes("family");
4052 private final static byte[] qualifier = Bytes.toBytes("qualifier");
4053 private final static byte[] CHAR = Bytes.toBytes("a");
4054 private int appendCounter;
4055
4056 public Appender(HRegion region, int appendCounter) {
4057 this.region = region;
4058 this.appendCounter = appendCounter;
4059 }
4060
4061 @Override
4062 public void run() {
4063 int count = 0;
4064 while (count < appendCounter) {
4065 Append app = new Append(appendRow);
4066 app.add(family, qualifier, CHAR);
4067 count++;
4068 try {
4069 region.append(app);
4070 } catch (IOException e) {
4071 e.printStackTrace();
4072 break;
4073 }
4074 }
4075 }
4076 }
4077
4078
4079
4080
4081
4082 @Test
4083 public void testParallelAppendWithMemStoreFlush() throws Exception {
4084 byte[] family = Appender.family;
4085 this.region = initHRegion(tableName, method, CONF, family);
4086 final HRegion region = this.region;
4087 final AtomicBoolean appendDone = new AtomicBoolean(false);
4088 Runnable flusher = new Runnable() {
4089 @Override
4090 public void run() {
4091 while (!appendDone.get()) {
4092 try {
4093 region.flushcache();
4094 } catch (Exception e) {
4095 e.printStackTrace();
4096 }
4097 }
4098 }
4099 };
4100
4101
4102
4103 int threadNum = 20;
4104 int appendCounter = 100;
4105 byte[] expected = new byte[threadNum * appendCounter];
4106 for (int i = 0; i < threadNum * appendCounter; i++) {
4107 System.arraycopy(Appender.CHAR, 0, expected, i, 1);
4108 }
4109 Thread[] appenders = new Thread[threadNum];
4110 Thread flushThread = new Thread(flusher);
4111 for (int i = 0; i < threadNum; i++) {
4112 appenders[i] = new Thread(new Appender(this.region, appendCounter));
4113 appenders[i].start();
4114 }
4115 flushThread.start();
4116 for (int i = 0; i < threadNum; i++) {
4117 appenders[i].join();
4118 }
4119
4120 appendDone.set(true);
4121 flushThread.join();
4122
4123 Get get = new Get(Appender.appendRow);
4124 get.addColumn(Appender.family, Appender.qualifier);
4125 get.setMaxVersions(1);
4126 Result res = this.region.get(get);
4127 List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
4128
4129
4130 assertEquals(kvs.size(), 1);
4131 Cell kv = kvs.get(0);
4132 byte[] appendResult = new byte[kv.getValueLength()];
4133 System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
4134 assertArrayEquals(expected, appendResult);
4135 this.region = null;
4136 }
4137
4138
4139
4140
4141
4142 @Test
4143 public void testPutWithMemStoreFlush() throws Exception {
4144 byte[] family = Bytes.toBytes("family");
4145 ;
4146 byte[] qualifier = Bytes.toBytes("qualifier");
4147 byte[] row = Bytes.toBytes("putRow");
4148 byte[] value = null;
4149 this.region = initHRegion(tableName, method, CONF, family);
4150 Put put = null;
4151 Get get = null;
4152 List<Cell> kvs = null;
4153 Result res = null;
4154
4155 put = new Put(row);
4156 value = Bytes.toBytes("value0");
4157 put.add(family, qualifier, 1234567l, value);
4158 region.put(put);
4159 get = new Get(row);
4160 get.addColumn(family, qualifier);
4161 get.setMaxVersions();
4162 res = this.region.get(get);
4163 kvs = res.getColumnCells(family, qualifier);
4164 assertEquals(1, kvs.size());
4165 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4166
4167 region.flushcache();
4168 get = new Get(row);
4169 get.addColumn(family, qualifier);
4170 get.setMaxVersions();
4171 res = this.region.get(get);
4172 kvs = res.getColumnCells(family, qualifier);
4173 assertEquals(1, kvs.size());
4174 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4175
4176 put = new Put(row);
4177 value = Bytes.toBytes("value1");
4178 put.add(family, qualifier, 1234567l, value);
4179 region.put(put);
4180 get = new Get(row);
4181 get.addColumn(family, qualifier);
4182 get.setMaxVersions();
4183 res = this.region.get(get);
4184 kvs = res.getColumnCells(family, qualifier);
4185 assertEquals(1, kvs.size());
4186 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4187
4188 region.flushcache();
4189 get = new Get(row);
4190 get.addColumn(family, qualifier);
4191 get.setMaxVersions();
4192 res = this.region.get(get);
4193 kvs = res.getColumnCells(family, qualifier);
4194 assertEquals(1, kvs.size());
4195 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4196 }
4197
4198 @Test
4199 public void testDurability() throws Exception {
4200 String method = "testDurability";
4201
4202
4203
4204
4205
4206 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4207 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4208 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4209
4210 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4211 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4212 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4213
4214 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4215 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4216
4217 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
4218 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4219
4220 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
4221 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
4222 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
4223
4224
4225 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4226 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4227 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4228 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4229 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
4230 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4231
4232 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4233 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4234 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4235 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4236 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
4237 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
4238
4239
4240 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4241 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4242 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4243 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
4244 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
4245 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
4246
4247 }
4248
4249 private void durabilityTest(String method, Durability tableDurability,
4250 Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
4251 final boolean expectSyncFromLogSyncer) throws Exception {
4252 Configuration conf = HBaseConfiguration.create(CONF);
4253 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
4254 TableName tableName = TableName.valueOf(method);
4255 byte[] family = Bytes.toBytes("family");
4256 Path logDir = new Path(new Path(dir + method), "log");
4257 HLog hlog = HLogFactory.createHLog(FILESYSTEM, logDir, UUID.randomUUID().toString(), conf);
4258 final HLog log = spy(hlog);
4259 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
4260 HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, log,
4261 new byte[][] { family });
4262
4263 Put put = new Put(Bytes.toBytes("r1"));
4264 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
4265 put.setDurability(mutationDurability);
4266 region.put(put);
4267
4268
4269 verify(log, expectAppend ? times(1) : never())
4270 .appendNoSync((HRegionInfo)any(), eq(tableName), (WALEdit)any(), (List<UUID>)any(),
4271 anyLong(), (HTableDescriptor)any(), (AtomicLong)any(), anyBoolean(), anyLong(), anyLong());
4272
4273
4274 if (expectSync || expectSyncFromLogSyncer) {
4275 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
4276 @Override
4277 public boolean evaluate() throws Exception {
4278 try {
4279 if (expectSync) {
4280 verify(log, times(1)).sync(anyLong());
4281 } else if (expectSyncFromLogSyncer) {
4282 verify(log, times(1)).sync();
4283 }
4284 } catch (Throwable ignore) {
4285 }
4286 return true;
4287 }
4288 });
4289 } else {
4290 verify(log, never()).sync(anyLong());
4291 verify(log, never()).sync();
4292 }
4293
4294 HRegion.closeHRegion(this.region);
4295 this.region = null;
4296 }
4297
4298 private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4299 for (int i = startRow; i < startRow + numRows; i++) {
4300 Put put = new Put(Bytes.toBytes("" + i));
4301 put.setDurability(Durability.SKIP_WAL);
4302 for (byte[] family : families) {
4303 put.add(family, qf, null);
4304 }
4305 region.put(put);
4306 }
4307 }
4308
4309 private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
4310 throws IOException {
4311 for (int i = startRow; i < startRow + numRows; i++) {
4312 byte[] row = Bytes.toBytes("" + i);
4313 Get get = new Get(row);
4314 for (byte[] family : families) {
4315 get.addColumn(family, qf);
4316 }
4317 Result result = newReg.get(get);
4318 Cell[] raw = result.rawCells();
4319 assertEquals(families.length, result.size());
4320 for (int j = 0; j < families.length; j++) {
4321 assertTrue(CellUtil.matchingRow(raw[j], row));
4322 assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
4323 assertTrue(CellUtil.matchingQualifier(raw[j], qf));
4324 }
4325 }
4326 }
4327
4328 private void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
4329
4330 Get get = new Get(k).addFamily(family).setMaxVersions();
4331 Cell[] results = r.get(get).rawCells();
4332 for (int j = 0; j < results.length; j++) {
4333 byte[] tmp = CellUtil.cloneValue(results[j]);
4334
4335 assertTrue(Bytes.equals(k, tmp));
4336 }
4337 }
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350 private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
4351 throws IOException {
4352 byte[][] families = { fs };
4353 Scan scan = new Scan();
4354 for (int i = 0; i < families.length; i++)
4355 scan.addFamily(families[i]);
4356 InternalScanner s = r.getScanner(scan);
4357 try {
4358 List<Cell> curVals = new ArrayList<Cell>();
4359 boolean first = true;
4360 OUTER_LOOP: while (s.next(curVals)) {
4361 for (Cell kv : curVals) {
4362 byte[] val = CellUtil.cloneValue(kv);
4363 byte[] curval = val;
4364 if (first) {
4365 first = false;
4366 assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4367 } else {
4368
4369 break OUTER_LOOP;
4370 }
4371 }
4372 }
4373 } finally {
4374 s.close();
4375 }
4376 }
4377
4378
4379
4380
4381
4382 @Test
4383 public void testFlushResult() throws IOException {
4384 String method = name.getMethodName();
4385 byte[] tableName = Bytes.toBytes(method);
4386 byte[] family = Bytes.toBytes("family");
4387
4388 this.region = initHRegion(tableName, method, family);
4389
4390
4391 HRegion.FlushResult fr = region.flushcache();
4392 assertFalse(fr.isFlushSucceeded());
4393 assertFalse(fr.isCompactionNeeded());
4394
4395
4396 for (int i = 0; i < 2; i++) {
4397 Put put = new Put(tableName).add(family, family, tableName);
4398 region.put(put);
4399 fr = region.flushcache();
4400 assertTrue(fr.isFlushSucceeded());
4401 assertFalse(fr.isCompactionNeeded());
4402 }
4403
4404
4405 for (int i = 0; i < 2; i++) {
4406 Put put = new Put(tableName).add(family, family, tableName);
4407 region.put(put);
4408 fr = region.flushcache();
4409 assertTrue(fr.isFlushSucceeded());
4410 assertTrue(fr.isCompactionNeeded());
4411 }
4412 }
4413
4414 private Configuration initSplit() {
4415
4416 CONF.setInt("hbase.hstore.compactionThreshold", 2);
4417
4418
4419 CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4420
4421 CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4422
4423
4424 CONF.setLong("hbase.client.pause", 15 * 1000);
4425
4426
4427
4428 CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4429 return CONF;
4430 }
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441 public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
4442 byte[]... families) throws IOException {
4443 return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
4444 }
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4456 byte[]... families) throws IOException {
4457 return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4458 }
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4471 boolean isReadOnly, byte[]... families) throws IOException {
4472 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4473 }
4474
4475 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4476 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4477 throws IOException {
4478 return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
4479 Durability.SYNC_WAL, null, families);
4480 }
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4495 String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4496 HLog hlog, byte[]... families) throws IOException {
4497 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly, durability, hlog, families);
4498 }
4499
4500
4501
4502
4503
4504 private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
4505 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4506 assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
4507 Bytes.toString(CellUtil.cloneRow(kv)));
4508 assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
4509 Bytes.toString(CellUtil.cloneFamily(kv)));
4510 assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
4511 Bytes.toString(CellUtil.cloneQualifier(kv)));
4512 assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
4513 assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
4514 Bytes.toString(CellUtil.cloneValue(kv)));
4515 }
4516
4517 @Test (timeout=60000)
4518 public void testReverseScanner_FromMemStore_SingleCF_Normal()
4519 throws IOException {
4520 byte[] rowC = Bytes.toBytes("rowC");
4521 byte[] rowA = Bytes.toBytes("rowA");
4522 byte[] rowB = Bytes.toBytes("rowB");
4523 byte[] cf = Bytes.toBytes("CF");
4524 byte[][] families = { cf };
4525 byte[] col = Bytes.toBytes("C");
4526 long ts = 1;
4527 String method = this.getName();
4528 this.region = initHRegion(tableName, method, families);
4529 try {
4530 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4531 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4532 null);
4533 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4534 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4535 Put put = null;
4536 put = new Put(rowC);
4537 put.add(kv1);
4538 put.add(kv11);
4539 region.put(put);
4540 put = new Put(rowA);
4541 put.add(kv2);
4542 region.put(put);
4543 put = new Put(rowB);
4544 put.add(kv3);
4545 region.put(put);
4546
4547 Scan scan = new Scan(rowC);
4548 scan.setMaxVersions(5);
4549 scan.setReversed(true);
4550 InternalScanner scanner = region.getScanner(scan);
4551 List<Cell> currRow = new ArrayList<Cell>();
4552 boolean hasNext = scanner.next(currRow);
4553 assertEquals(2, currRow.size());
4554 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4555 assertTrue(hasNext);
4556 currRow.clear();
4557 hasNext = scanner.next(currRow);
4558 assertEquals(1, currRow.size());
4559 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4560 assertTrue(hasNext);
4561 currRow.clear();
4562 hasNext = scanner.next(currRow);
4563 assertEquals(1, currRow.size());
4564 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4565 assertFalse(hasNext);
4566 scanner.close();
4567 } finally {
4568 HRegion.closeHRegion(this.region);
4569 this.region = null;
4570 }
4571 }
4572
4573 @Test (timeout=60000)
4574 public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
4575 throws IOException {
4576 byte[] rowC = Bytes.toBytes("rowC");
4577 byte[] rowA = Bytes.toBytes("rowA");
4578 byte[] rowB = Bytes.toBytes("rowB");
4579 byte[] rowD = Bytes.toBytes("rowD");
4580 byte[] cf = Bytes.toBytes("CF");
4581 byte[][] families = { cf };
4582 byte[] col = Bytes.toBytes("C");
4583 long ts = 1;
4584 String method = this.getName();
4585 this.region = initHRegion(tableName, method, families);
4586 try {
4587 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4588 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4589 null);
4590 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4591 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4592 Put put = null;
4593 put = new Put(rowC);
4594 put.add(kv1);
4595 put.add(kv11);
4596 region.put(put);
4597 put = new Put(rowA);
4598 put.add(kv2);
4599 region.put(put);
4600 put = new Put(rowB);
4601 put.add(kv3);
4602 region.put(put);
4603
4604 Scan scan = new Scan(rowD);
4605 List<Cell> currRow = new ArrayList<Cell>();
4606 scan.setReversed(true);
4607 scan.setMaxVersions(5);
4608 InternalScanner scanner = region.getScanner(scan);
4609 boolean hasNext = scanner.next(currRow);
4610 assertEquals(2, currRow.size());
4611 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4612 assertTrue(hasNext);
4613 currRow.clear();
4614 hasNext = scanner.next(currRow);
4615 assertEquals(1, currRow.size());
4616 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4617 assertTrue(hasNext);
4618 currRow.clear();
4619 hasNext = scanner.next(currRow);
4620 assertEquals(1, currRow.size());
4621 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4622 assertFalse(hasNext);
4623 scanner.close();
4624 } finally {
4625 HRegion.closeHRegion(this.region);
4626 this.region = null;
4627 }
4628 }
4629
4630 @Test (timeout=60000)
4631 public void testReverseScanner_FromMemStore_SingleCF_FullScan()
4632 throws IOException {
4633 byte[] rowC = Bytes.toBytes("rowC");
4634 byte[] rowA = Bytes.toBytes("rowA");
4635 byte[] rowB = Bytes.toBytes("rowB");
4636 byte[] cf = Bytes.toBytes("CF");
4637 byte[][] families = { cf };
4638 byte[] col = Bytes.toBytes("C");
4639 long ts = 1;
4640 String method = this.getName();
4641 this.region = initHRegion(tableName, method, families);
4642 try {
4643 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4644 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4645 null);
4646 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4647 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4648 Put put = null;
4649 put = new Put(rowC);
4650 put.add(kv1);
4651 put.add(kv11);
4652 region.put(put);
4653 put = new Put(rowA);
4654 put.add(kv2);
4655 region.put(put);
4656 put = new Put(rowB);
4657 put.add(kv3);
4658 region.put(put);
4659 Scan scan = new Scan();
4660 List<Cell> currRow = new ArrayList<Cell>();
4661 scan.setReversed(true);
4662 InternalScanner scanner = region.getScanner(scan);
4663 boolean hasNext = scanner.next(currRow);
4664 assertEquals(1, currRow.size());
4665 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4666 assertTrue(hasNext);
4667 currRow.clear();
4668 hasNext = scanner.next(currRow);
4669 assertEquals(1, currRow.size());
4670 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4671 assertTrue(hasNext);
4672 currRow.clear();
4673 hasNext = scanner.next(currRow);
4674 assertEquals(1, currRow.size());
4675 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4676 assertFalse(hasNext);
4677 scanner.close();
4678 } finally {
4679 HRegion.closeHRegion(this.region);
4680 this.region = null;
4681 }
4682 }
4683
4684 @Test (timeout=60000)
4685 public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
4686
4687 byte[] rowA = Bytes.toBytes("rowA");
4688 byte[] rowB = Bytes.toBytes("rowB");
4689 byte[] rowC = Bytes.toBytes("rowC");
4690 byte[] rowD = Bytes.toBytes("rowD");
4691 byte[] rowE = Bytes.toBytes("rowE");
4692 byte[] cf = Bytes.toBytes("CF");
4693 byte[][] families = { cf };
4694 byte[] col1 = Bytes.toBytes("col1");
4695 byte[] col2 = Bytes.toBytes("col2");
4696 long ts = 1;
4697 String method = this.getName();
4698 this.region = initHRegion(tableName, method, families);
4699 try {
4700 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4701 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4702 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4703 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4704 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4705 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4706 Put put = null;
4707 put = new Put(rowA);
4708 put.add(kv1);
4709 region.put(put);
4710 put = new Put(rowB);
4711 put.add(kv2);
4712 region.put(put);
4713 put = new Put(rowC);
4714 put.add(kv3);
4715 region.put(put);
4716 put = new Put(rowD);
4717 put.add(kv4_1);
4718 region.put(put);
4719 put = new Put(rowD);
4720 put.add(kv4_2);
4721 region.put(put);
4722 put = new Put(rowE);
4723 put.add(kv5);
4724 region.put(put);
4725 region.flushcache();
4726 Scan scan = new Scan(rowD, rowA);
4727 scan.addColumn(families[0], col1);
4728 scan.setReversed(true);
4729 List<Cell> currRow = new ArrayList<Cell>();
4730 InternalScanner scanner = region.getScanner(scan);
4731 boolean hasNext = scanner.next(currRow);
4732 assertEquals(1, currRow.size());
4733 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4734 assertTrue(hasNext);
4735 currRow.clear();
4736 hasNext = scanner.next(currRow);
4737 assertEquals(1, currRow.size());
4738 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4739 assertTrue(hasNext);
4740 currRow.clear();
4741 hasNext = scanner.next(currRow);
4742 assertEquals(1, currRow.size());
4743 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4744 assertFalse(hasNext);
4745 scanner.close();
4746
4747 scan = new Scan(rowD, rowA);
4748 scan.addColumn(families[0], col2);
4749 scan.setReversed(true);
4750 currRow.clear();
4751 scanner = region.getScanner(scan);
4752 hasNext = scanner.next(currRow);
4753 assertEquals(1, currRow.size());
4754 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4755 scanner.close();
4756 } finally {
4757 HRegion.closeHRegion(this.region);
4758 this.region = null;
4759 }
4760 }
4761
4762 @Test (timeout=60000)
4763 public void testReverseScanner_smaller_blocksize() throws IOException {
4764
4765 byte[] rowA = Bytes.toBytes("rowA");
4766 byte[] rowB = Bytes.toBytes("rowB");
4767 byte[] rowC = Bytes.toBytes("rowC");
4768 byte[] rowD = Bytes.toBytes("rowD");
4769 byte[] rowE = Bytes.toBytes("rowE");
4770 byte[] cf = Bytes.toBytes("CF");
4771 byte[][] families = { cf };
4772 byte[] col1 = Bytes.toBytes("col1");
4773 byte[] col2 = Bytes.toBytes("col2");
4774 long ts = 1;
4775 String method = this.getName();
4776 HBaseConfiguration config = new HBaseConfiguration();
4777 config.setInt("test.block.size", 1);
4778 this.region = initHRegion(tableName, method, config, families);
4779 try {
4780 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4781 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4782 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4783 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4784 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4785 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4786 Put put = null;
4787 put = new Put(rowA);
4788 put.add(kv1);
4789 region.put(put);
4790 put = new Put(rowB);
4791 put.add(kv2);
4792 region.put(put);
4793 put = new Put(rowC);
4794 put.add(kv3);
4795 region.put(put);
4796 put = new Put(rowD);
4797 put.add(kv4_1);
4798 region.put(put);
4799 put = new Put(rowD);
4800 put.add(kv4_2);
4801 region.put(put);
4802 put = new Put(rowE);
4803 put.add(kv5);
4804 region.put(put);
4805 region.flushcache();
4806 Scan scan = new Scan(rowD, rowA);
4807 scan.addColumn(families[0], col1);
4808 scan.setReversed(true);
4809 List<Cell> currRow = new ArrayList<Cell>();
4810 InternalScanner scanner = region.getScanner(scan);
4811 boolean hasNext = scanner.next(currRow);
4812 assertEquals(1, currRow.size());
4813 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4814 assertTrue(hasNext);
4815 currRow.clear();
4816 hasNext = scanner.next(currRow);
4817 assertEquals(1, currRow.size());
4818 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4819 assertTrue(hasNext);
4820 currRow.clear();
4821 hasNext = scanner.next(currRow);
4822 assertEquals(1, currRow.size());
4823 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4824 assertFalse(hasNext);
4825 scanner.close();
4826
4827 scan = new Scan(rowD, rowA);
4828 scan.addColumn(families[0], col2);
4829 scan.setReversed(true);
4830 currRow.clear();
4831 scanner = region.getScanner(scan);
4832 hasNext = scanner.next(currRow);
4833 assertEquals(1, currRow.size());
4834 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4835 scanner.close();
4836 } finally {
4837 HRegion.closeHRegion(this.region);
4838 this.region = null;
4839 }
4840 }
4841
4842 @Test (timeout=60000)
4843 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
4844 throws IOException {
4845 byte[] row0 = Bytes.toBytes("row0");
4846 byte[] row1 = Bytes.toBytes("row1");
4847 byte[] row2 = Bytes.toBytes("row2");
4848 byte[] row3 = Bytes.toBytes("row3");
4849 byte[] row4 = Bytes.toBytes("row4");
4850 byte[] row5 = Bytes.toBytes("row5");
4851 byte[] cf1 = Bytes.toBytes("CF1");
4852 byte[] cf2 = Bytes.toBytes("CF2");
4853 byte[] cf3 = Bytes.toBytes("CF3");
4854 byte[][] families = { cf1, cf2, cf3 };
4855 byte[] col = Bytes.toBytes("C");
4856 long ts = 1;
4857 String method = this.getName();
4858 HBaseConfiguration conf = new HBaseConfiguration();
4859
4860 conf.setInt("hbase.hstore.compactionThreshold", 10000);
4861 this.region = initHRegion(tableName, method, conf, families);
4862 try {
4863
4864 KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
4865 null);
4866 KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
4867 null);
4868 KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
4869 KeyValue.Type.Put, null);
4870 KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
4871 null);
4872 KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
4873 null);
4874 KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
4875 null);
4876 KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
4877 KeyValue.Type.Put, null);
4878 KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
4879 null);
4880 KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
4881 KeyValue.Type.Put, null);
4882 KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
4883 null);
4884 KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
4885 null);
4886 KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
4887 KeyValue.Type.Put, null);
4888 KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
4889 null);
4890 KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
4891 KeyValue.Type.Put, null);
4892 KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
4893 null);
4894 KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
4895 null);
4896
4897 Put put = null;
4898 put = new Put(row1);
4899 put.add(kv1_2_1);
4900 region.put(put);
4901 put = new Put(row2);
4902 put.add(kv2_4_1);
4903 region.put(put);
4904 put = new Put(row4);
4905 put.add(kv4_5_4);
4906 put.add(kv4_5_5);
4907 region.put(put);
4908 region.flushcache();
4909
4910 put = new Put(row4);
4911 put.add(kv4_5_1);
4912 put.add(kv4_5_3);
4913 region.put(put);
4914 put = new Put(row1);
4915 put.add(kv1_2_2);
4916 region.put(put);
4917 put = new Put(row2);
4918 put.add(kv2_4_4);
4919 region.put(put);
4920 region.flushcache();
4921
4922 put = new Put(row4);
4923 put.add(kv4_5_2);
4924 region.put(put);
4925 put = new Put(row2);
4926 put.add(kv2_4_2);
4927 put.add(kv2_4_3);
4928 region.put(put);
4929 put = new Put(row3);
4930 put.add(kv3_2_2);
4931 region.put(put);
4932 region.flushcache();
4933
4934
4935 put = new Put(row0);
4936 put.add(kv0_1_1);
4937 region.put(put);
4938 put = new Put(row3);
4939 put.add(kv3_2_1);
4940 region.put(put);
4941 put = new Put(row5);
4942 put.add(kv5_2_1);
4943 put.add(kv5_2_2);
4944 region.put(put);
4945
4946 Scan scan = new Scan(row4);
4947 scan.setMaxVersions(5);
4948 scan.setBatch(3);
4949 scan.setReversed(true);
4950 InternalScanner scanner = region.getScanner(scan);
4951 List<Cell> currRow = new ArrayList<Cell>();
4952 boolean hasNext = false;
4953
4954
4955
4956 hasNext = scanner.next(currRow);
4957 assertEquals(3, currRow.size());
4958 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4959 assertTrue(hasNext);
4960 currRow.clear();
4961 hasNext = scanner.next(currRow);
4962 assertEquals(2, currRow.size());
4963 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4964 assertTrue(hasNext);
4965
4966 currRow.clear();
4967 hasNext = scanner.next(currRow);
4968 assertEquals(2, currRow.size());
4969 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
4970 assertTrue(hasNext);
4971
4972
4973 currRow.clear();
4974 hasNext = scanner.next(currRow);
4975 assertEquals(3, currRow.size());
4976 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4977 assertTrue(hasNext);
4978 currRow.clear();
4979 hasNext = scanner.next(currRow);
4980 assertEquals(1, currRow.size());
4981 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4982 assertTrue(hasNext);
4983
4984 currRow.clear();
4985 hasNext = scanner.next(currRow);
4986 assertEquals(2, currRow.size());
4987 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
4988 assertTrue(hasNext);
4989
4990 currRow.clear();
4991 hasNext = scanner.next(currRow);
4992 assertEquals(1, currRow.size());
4993 assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
4994 assertFalse(hasNext);
4995
4996 scanner.close();
4997 } finally {
4998 HRegion.closeHRegion(this.region);
4999 this.region = null;
5000 }
5001 }
5002
5003 @Test (timeout=60000)
5004 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
5005 throws IOException {
5006 byte[] row1 = Bytes.toBytes("row1");
5007 byte[] row2 = Bytes.toBytes("row2");
5008 byte[] row3 = Bytes.toBytes("row3");
5009 byte[] row4 = Bytes.toBytes("row4");
5010 byte[] cf1 = Bytes.toBytes("CF1");
5011 byte[] cf2 = Bytes.toBytes("CF2");
5012 byte[] cf3 = Bytes.toBytes("CF3");
5013 byte[] cf4 = Bytes.toBytes("CF4");
5014 byte[][] families = { cf1, cf2, cf3, cf4 };
5015 byte[] col = Bytes.toBytes("C");
5016 long ts = 1;
5017 String method = this.getName();
5018 HBaseConfiguration conf = new HBaseConfiguration();
5019
5020 conf.setInt("hbase.hstore.compactionThreshold", 10000);
5021 this.region = initHRegion(tableName, method, conf, families);
5022 try {
5023 KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
5024 KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
5025 KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
5026 KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
5027
5028 Put put = new Put(row1);
5029 put.add(kv1);
5030 region.put(put);
5031 region.flushcache();
5032
5033 put = new Put(row2);
5034 put.add(kv2);
5035 region.put(put);
5036 region.flushcache();
5037
5038 put = new Put(row3);
5039 put.add(kv3);
5040 region.put(put);
5041 region.flushcache();
5042
5043 put = new Put(row4);
5044 put.add(kv4);
5045 region.put(put);
5046
5047 Scan scan = new Scan(row4);
5048 scan.setReversed(true);
5049 scan.setBatch(10);
5050 InternalScanner scanner = region.getScanner(scan);
5051 List<Cell> currRow = new ArrayList<Cell>();
5052 boolean hasNext = scanner.next(currRow);
5053 assertEquals(1, currRow.size());
5054 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5055 assertTrue(hasNext);
5056 currRow.clear();
5057 hasNext = scanner.next(currRow);
5058 assertEquals(1, currRow.size());
5059 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5060 assertTrue(hasNext);
5061 currRow.clear();
5062 hasNext = scanner.next(currRow);
5063 assertEquals(1, currRow.size());
5064 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5065 assertTrue(hasNext);
5066 currRow.clear();
5067 hasNext = scanner.next(currRow);
5068 assertEquals(1, currRow.size());
5069 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5070 assertFalse(hasNext);
5071 } finally {
5072 HRegion.closeHRegion(this.region);
5073 this.region = null;
5074 }
5075 }
5076
5077 @Test (timeout=60000)
5078 public void testSplitRegionWithReverseScan() throws IOException {
5079 byte [] tableName = Bytes.toBytes("testSplitRegionWithReverseScan");
5080 byte [] qualifier = Bytes.toBytes("qualifier");
5081 Configuration hc = initSplit();
5082 int numRows = 3;
5083 byte [][] families = {fam1};
5084
5085
5086 String method = this.getName();
5087 this.region = initHRegion(tableName, method, hc, families);
5088
5089
5090 int startRow = 100;
5091 putData(startRow, numRows, qualifier, families);
5092 int splitRow = startRow + numRows;
5093 putData(splitRow, numRows, qualifier, families);
5094 int endRow = splitRow + numRows;
5095 region.flushcache();
5096
5097 HRegion [] regions = null;
5098 try {
5099 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
5100
5101 for (int i = 0; i < regions.length; i++) {
5102 regions[i] = HRegion.openHRegion(regions[i], null);
5103 }
5104
5105 assertEquals(2, regions.length);
5106
5107
5108
5109 verifyData(regions[0], startRow, numRows, qualifier, families);
5110 verifyData(regions[1], splitRow, numRows, qualifier, families);
5111
5112
5113 Scan scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 10 * numRows)));
5114 scan.setReversed(true);
5115 InternalScanner scanner = regions[1].getScanner(scan);
5116 List<Cell> currRow = new ArrayList<Cell>();
5117 boolean more = false;
5118 int verify = startRow + 2 * numRows - 1;
5119 do {
5120 more = scanner.next(currRow);
5121 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5122 verify--;
5123 currRow.clear();
5124 } while(more);
5125 assertEquals(verify, startRow + numRows - 1);
5126 scanner.close();
5127
5128 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 2 * numRows - 1)));
5129 scan.setReversed(true);
5130 scanner = regions[1].getScanner(scan);
5131 verify = startRow + 2 * numRows - 1;
5132 do {
5133 more = scanner.next(currRow);
5134 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5135 verify--;
5136 currRow.clear();
5137 } while(more);
5138 assertEquals(verify, startRow + numRows - 1);
5139 scanner.close();
5140
5141 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows)));
5142 scan.setReversed(true);
5143 scanner = regions[0].getScanner(scan);
5144 verify = startRow + numRows - 1;
5145 do {
5146 more = scanner.next(currRow);
5147 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5148 verify--;
5149 currRow.clear();
5150 } while(more);
5151 assertEquals(verify, 99);
5152 scanner.close();
5153
5154 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows - 1)));
5155 scan.setReversed(true);
5156 scanner = regions[0].getScanner(scan);
5157 verify = startRow + numRows - 1;
5158 do {
5159 more = scanner.next(currRow);
5160 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5161 verify--;
5162 currRow.clear();
5163 } while(more);
5164 assertEquals(verify, startRow - 1);
5165 scanner.close();
5166 } finally {
5167 HRegion.closeHRegion(this.region);
5168 this.region = null;
5169 }
5170 }
5171
5172 @Test
5173 public void testWriteRequestsCounter() throws IOException {
5174 byte[] fam = Bytes.toBytes("info");
5175 byte[][] families = { fam };
5176 this.region = initHRegion(tableName, method, CONF, families);
5177
5178 Assert.assertEquals(0L, region.getWriteRequestsCount());
5179
5180 Put put = new Put(row);
5181 put.add(fam, fam, fam);
5182
5183 Assert.assertEquals(0L, region.getWriteRequestsCount());
5184 region.put(put);
5185 Assert.assertEquals(1L, region.getWriteRequestsCount());
5186 region.put(put);
5187 Assert.assertEquals(2L, region.getWriteRequestsCount());
5188 region.put(put);
5189 Assert.assertEquals(3L, region.getWriteRequestsCount());
5190
5191 region.delete(new Delete(row));
5192 Assert.assertEquals(4L, region.getWriteRequestsCount());
5193
5194 HRegion.closeHRegion(this.region);
5195 this.region = null;
5196 }
5197
5198
5199
5200
5201 @Test (timeout=24000)
5202 public void testRegionTooBusy() throws IOException {
5203 String method = "testRegionTooBusy";
5204 byte[] tableName = Bytes.toBytes(method);
5205 byte[] family = Bytes.toBytes("family");
5206 long defaultBusyWaitDuration = CONF.getLong("hbase.busy.wait.duration",
5207 HRegion.DEFAULT_BUSY_WAIT_DURATION);
5208 CONF.setLong("hbase.busy.wait.duration", 1000);
5209 region = initHRegion(tableName, method, CONF, family);
5210 final AtomicBoolean stopped = new AtomicBoolean(true);
5211 Thread t = new Thread(new Runnable() {
5212 @Override
5213 public void run() {
5214 try {
5215 region.lock.writeLock().lock();
5216 stopped.set(false);
5217 while (!stopped.get()) {
5218 Thread.sleep(100);
5219 }
5220 } catch (InterruptedException ie) {
5221 } finally {
5222 region.lock.writeLock().unlock();
5223 }
5224 }
5225 });
5226 t.start();
5227 Get get = new Get(row);
5228 try {
5229 while (stopped.get()) {
5230 Thread.sleep(100);
5231 }
5232 region.get(get);
5233 fail("Should throw RegionTooBusyException");
5234 } catch (InterruptedException ie) {
5235 fail("test interrupted");
5236 } catch (RegionTooBusyException e) {
5237
5238 } finally {
5239 stopped.set(true);
5240 try {
5241 t.join();
5242 } catch (Throwable e) {
5243 }
5244
5245 HRegion.closeHRegion(region);
5246 region = null;
5247 CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration);
5248 }
5249 }
5250
5251 @Test
5252 public void testCellTTLs() throws IOException {
5253 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
5254 EnvironmentEdgeManager.injectEdge(edge);
5255
5256 final byte[] row = Bytes.toBytes("testRow");
5257 final byte[] q1 = Bytes.toBytes("q1");
5258 final byte[] q2 = Bytes.toBytes("q2");
5259 final byte[] q3 = Bytes.toBytes("q3");
5260 final byte[] q4 = Bytes.toBytes("q4");
5261
5262 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCellTTLs"));
5263 HColumnDescriptor hcd = new HColumnDescriptor(fam1);
5264 hcd.setTimeToLive(10);
5265 htd.addFamily(hcd);
5266
5267 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
5268 conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS);
5269
5270 HRegion region = HRegion.createHRegion(new HRegionInfo(htd.getTableName(),
5271 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY),
5272 TEST_UTIL.getDataTestDir(), conf, htd);
5273 assertNotNull(region);
5274 try {
5275 long now = EnvironmentEdgeManager.currentTimeMillis();
5276
5277 region.put(new Put(row).add(new KeyValue(row, fam1, q1, now,
5278 HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
5279
5280 new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
5281
5282 region.put(new Put(row).add(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY));
5283
5284 region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1,
5285 HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
5286
5287 new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
5288
5289 region.put(new Put(row).add(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY));
5290
5291
5292 region.flushcache();
5293
5294
5295 Result r = region.get(new Get(row));
5296 assertNotNull(r.getValue(fam1, q1));
5297 assertNotNull(r.getValue(fam1, q2));
5298 assertNotNull(r.getValue(fam1, q3));
5299 assertNotNull(r.getValue(fam1, q4));
5300
5301
5302 edge.incrementTime(5000);
5303
5304 r = region.get(new Get(row));
5305 assertNull(r.getValue(fam1, q1));
5306 assertNotNull(r.getValue(fam1, q2));
5307 assertNotNull(r.getValue(fam1, q3));
5308 assertNotNull(r.getValue(fam1, q4));
5309
5310
5311 edge.incrementTime(5000);
5312
5313 r = region.get(new Get(row));
5314 assertNull(r.getValue(fam1, q1));
5315 assertNull(r.getValue(fam1, q2));
5316 assertNotNull(r.getValue(fam1, q3));
5317 assertNotNull(r.getValue(fam1, q4));
5318
5319
5320 edge.incrementTime(5000);
5321
5322 r = region.get(new Get(row));
5323 assertNull(r.getValue(fam1, q1));
5324 assertNull(r.getValue(fam1, q2));
5325 assertNull(r.getValue(fam1, q3));
5326 assertNotNull(r.getValue(fam1, q4));
5327
5328
5329 edge.incrementTime(10000);
5330
5331 r = region.get(new Get(row));
5332 assertNull(r.getValue(fam1, q1));
5333 assertNull(r.getValue(fam1, q2));
5334 assertNull(r.getValue(fam1, q3));
5335 assertNull(r.getValue(fam1, q4));
5336
5337
5338
5339
5340 region.put(new Put(row).add(fam1, q1, Bytes.toBytes(1L)));
5341 r = region.get(new Get(row));
5342 byte[] val = r.getValue(fam1, q1);
5343 assertNotNull(val);
5344 assertEquals(Bytes.toLong(val), 1L);
5345
5346
5347 Increment incr = new Increment(row).addColumn(fam1, q1, 1L);
5348 incr.setTTL(5000);
5349 region.increment(incr);
5350
5351
5352 r = region.get(new Get(row));
5353 val = r.getValue(fam1, q1);
5354 assertNotNull(val);
5355 assertEquals(Bytes.toLong(val), 2L);
5356
5357
5358 edge.incrementTime(5000);
5359
5360
5361 r = region.get(new Get(row));
5362 val = r.getValue(fam1, q1);
5363 assertNotNull(val);
5364 assertEquals(Bytes.toLong(val), 1L);
5365
5366
5367 edge.incrementTime(5000);
5368
5369
5370 r = region.get(new Get(row));
5371 assertNull(r.getValue(fam1, q1));
5372
5373 } finally {
5374 HRegion.closeHRegion(region);
5375 }
5376 }
5377
5378 @Test
5379 public void testIncrementTimestampsAreMonotonic() throws IOException {
5380 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5381 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5382 EnvironmentEdgeManager.injectEdge(edge);
5383
5384 edge.setValue(10);
5385 Increment inc = new Increment(row);
5386 inc.setDurability(Durability.SKIP_WAL);
5387 inc.addColumn(fam1, qual1, 1L);
5388 region.increment(inc);
5389
5390 Result result = region.get(new Get(row));
5391 Cell c = result.getColumnLatestCell(fam1, qual1);
5392 assertNotNull(c);
5393 assertEquals(c.getTimestamp(), 10L);
5394
5395 edge.setValue(1);
5396 region.increment(inc);
5397 result = region.get(new Get(row));
5398 c = result.getColumnLatestCell(fam1, qual1);
5399 assertEquals(c.getTimestamp(), 10L);
5400 assertEquals(Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength()), 2L);
5401 }
5402
5403 @Test
5404 public void testAppendTimestampsAreMonotonic() throws IOException {
5405 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5406 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5407 EnvironmentEdgeManager.injectEdge(edge);
5408
5409 edge.setValue(10);
5410 Append a = new Append(row);
5411 a.setDurability(Durability.SKIP_WAL);
5412 a.add(fam1, qual1, qual1);
5413 region.append(a);
5414
5415 Result result = region.get(new Get(row));
5416 Cell c = result.getColumnLatestCell(fam1, qual1);
5417 assertNotNull(c);
5418 assertEquals(c.getTimestamp(), 10L);
5419
5420 edge.setValue(1);
5421 region.append(a);
5422 result = region.get(new Get(row));
5423 c = result.getColumnLatestCell(fam1, qual1);
5424 assertEquals(c.getTimestamp(), 10L);
5425
5426 byte[] expected = new byte[qual1.length*2];
5427 System.arraycopy(qual1, 0, expected, 0, qual1.length);
5428 System.arraycopy(qual1, 0, expected, qual1.length, qual1.length);
5429
5430 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
5431 expected, 0, expected.length));
5432 }
5433
5434 @Test
5435 public void testCheckAndMutateTimestampsAreMonotonic() throws IOException {
5436 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5437 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5438 EnvironmentEdgeManager.injectEdge(edge);
5439
5440 edge.setValue(10);
5441 Put p = new Put(row);
5442 p.setDurability(Durability.SKIP_WAL);
5443 p.add(fam1, qual1, qual1);
5444 region.put(p);
5445
5446 Result result = region.get(new Get(row));
5447 Cell c = result.getColumnLatestCell(fam1, qual1);
5448 assertNotNull(c);
5449 assertEquals(c.getTimestamp(), 10L);
5450
5451 edge.setValue(1);
5452 p = new Put(row);
5453 p.setDurability(Durability.SKIP_WAL);
5454 p.add(fam1, qual1, qual2);
5455 region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(qual1), p, false);
5456 result = region.get(new Get(row));
5457 c = result.getColumnLatestCell(fam1, qual1);
5458 assertEquals(c.getTimestamp(), 10L);
5459
5460 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
5461 qual2, 0, qual2.length));
5462 }
5463
5464 @Test
5465 public void testCheckAndRowMutateTimestampsAreMonotonic() throws IOException {
5466 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
5467 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
5468 EnvironmentEdgeManager.injectEdge(edge);
5469
5470 edge.setValue(10);
5471 Put p = new Put(row);
5472 p.setDurability(Durability.SKIP_WAL);
5473 p.add(fam1, qual1, qual1);
5474 region.put(p);
5475
5476 Result result = region.get(new Get(row));
5477 Cell c = result.getColumnLatestCell(fam1, qual1);
5478 assertNotNull(c);
5479 assertEquals(c.getTimestamp(), 10L);
5480
5481 edge.setValue(1);
5482 p = new Put(row);
5483 p.setDurability(Durability.SKIP_WAL);
5484 p.add(fam1, qual1, qual2);
5485 RowMutations rm = new RowMutations(row);
5486 rm.add(p);
5487 region.checkAndRowMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(qual1),
5488 rm, false);
5489 result = region.get(new Get(row));
5490 c = result.getColumnLatestCell(fam1, qual1);
5491 assertEquals(c.getTimestamp(), 10L);
5492
5493 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
5494 qual2, 0, qual2.length));
5495 }
5496
5497 private static HRegion initHRegion(byte[] tableName, String callingMethod,
5498 byte[]... families) throws IOException {
5499 return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
5500 families);
5501 }
5502 }