1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21
22 import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23 import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24 import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25 import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29 import static org.junit.Assert.assertArrayEquals;
30 import static org.junit.Assert.assertEquals;
31 import static org.junit.Assert.assertFalse;
32 import static org.junit.Assert.assertNotNull;
33 import static org.junit.Assert.assertNull;
34 import static org.junit.Assert.assertTrue;
35 import static org.junit.Assert.fail;
36 import static org.mockito.Matchers.any;
37 import static org.mockito.Matchers.anyBoolean;
38 import static org.mockito.Matchers.anyLong;
39 import static org.mockito.Matchers.eq;
40 import static org.mockito.Mockito.never;
41 import static org.mockito.Mockito.spy;
42 import static org.mockito.Mockito.times;
43 import static org.mockito.Mockito.verify;
44
45 import java.io.IOException;
46 import java.io.InterruptedIOException;
47 import java.security.PrivilegedExceptionAction;
48 import java.util.ArrayList;
49 import java.util.Arrays;
50 import java.util.Collection;
51 import java.util.List;
52 import java.util.Map;
53 import java.util.NavigableMap;
54 import java.util.TreeMap;
55 import java.util.UUID;
56 import java.util.concurrent.atomic.AtomicBoolean;
57 import java.util.concurrent.atomic.AtomicInteger;
58 import java.util.concurrent.atomic.AtomicLong;
59 import java.util.concurrent.atomic.AtomicReference;
60
61 import org.apache.commons.logging.Log;
62 import org.apache.commons.logging.LogFactory;
63 import org.apache.hadoop.conf.Configuration;
64 import org.apache.hadoop.fs.FSDataOutputStream;
65 import org.apache.hadoop.fs.FileStatus;
66 import org.apache.hadoop.fs.FileSystem;
67 import org.apache.hadoop.fs.Path;
68 import org.apache.hadoop.hbase.Cell;
69 import org.apache.hadoop.hbase.CellComparator;
70 import org.apache.hadoop.hbase.CellUtil;
71 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
72 import org.apache.hadoop.hbase.DroppedSnapshotException;
73 import org.apache.hadoop.hbase.HBaseConfiguration;
74 import org.apache.hadoop.hbase.HBaseTestCase;
75 import org.apache.hadoop.hbase.HBaseTestingUtility;
76 import org.apache.hadoop.hbase.HColumnDescriptor;
77 import org.apache.hadoop.hbase.HConstants;
78 import org.apache.hadoop.hbase.RegionTooBusyException;
79 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
80 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
81 import org.apache.hadoop.hbase.HRegionInfo;
82 import org.apache.hadoop.hbase.HTableDescriptor;
83 import org.apache.hadoop.hbase.KeyValue;
84 import org.apache.hadoop.hbase.MediumTests;
85 import org.apache.hadoop.hbase.MiniHBaseCluster;
86 import org.apache.hadoop.hbase.MultithreadedTestUtil;
87 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
88 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
89 import org.apache.hadoop.hbase.NotServingRegionException;
90 import org.apache.hadoop.hbase.TableName;
91 import org.apache.hadoop.hbase.Waiter;
92 import org.apache.hadoop.hbase.client.Append;
93 import org.apache.hadoop.hbase.client.Delete;
94 import org.apache.hadoop.hbase.client.Durability;
95 import org.apache.hadoop.hbase.client.Get;
96 import org.apache.hadoop.hbase.client.HTable;
97 import org.apache.hadoop.hbase.client.Increment;
98 import org.apache.hadoop.hbase.client.Put;
99 import org.apache.hadoop.hbase.client.Result;
100 import org.apache.hadoop.hbase.client.Scan;
101 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
102 import org.apache.hadoop.hbase.filter.BinaryComparator;
103 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
104 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
105 import org.apache.hadoop.hbase.filter.Filter;
106 import org.apache.hadoop.hbase.filter.FilterBase;
107 import org.apache.hadoop.hbase.filter.FilterList;
108 import org.apache.hadoop.hbase.filter.NullComparator;
109 import org.apache.hadoop.hbase.filter.PrefixFilter;
110 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
111 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
112 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
113 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
114 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
115 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
116 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
117 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
118 import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
119 import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
120 import org.apache.hadoop.hbase.regionserver.wal.FaultyHLog;
121 import org.apache.hadoop.hbase.regionserver.wal.HLog;
122 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
123 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
124 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
125 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
126 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
127 import org.apache.hadoop.hbase.security.User;
128 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
129 import org.apache.hadoop.hbase.util.Bytes;
130 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
131 import org.apache.hadoop.hbase.util.FSUtils;
132 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
133 import org.apache.hadoop.hbase.util.PairOfSameType;
134 import org.apache.hadoop.hbase.util.Threads;
135 import org.junit.After;
136 import org.junit.Assert;
137 import org.junit.Before;
138 import org.junit.Rule;
139 import org.junit.Test;
140 import org.junit.experimental.categories.Category;
141 import org.junit.rules.TestName;
142 import org.mockito.Mockito;
143
144 import com.google.common.collect.ImmutableList;
145 import com.google.common.collect.Lists;
146 import com.google.protobuf.ByteString;
147
148
149
150
151
152
153
154 @Category(MediumTests.class)
155 @SuppressWarnings("deprecation")
156 public class TestHRegion {
157
158
159 static final Log LOG = LogFactory.getLog(TestHRegion.class);
160 @Rule public TestName name = new TestName();
161
162 private static final String COLUMN_FAMILY = "MyCF";
163 private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
164
165 HRegion region = null;
166
167 private static HBaseTestingUtility TEST_UTIL;
168 public static Configuration CONF ;
169 private String dir;
170 private static FileSystem FILESYSTEM;
171 private final int MAX_VERSIONS = 2;
172
173
174 protected byte[] tableName;
175 protected String method;
176 protected final byte[] qual1 = Bytes.toBytes("qual1");
177 protected final byte[] qual2 = Bytes.toBytes("qual2");
178 protected final byte[] qual3 = Bytes.toBytes("qual3");
179 protected final byte[] value1 = Bytes.toBytes("value1");
180 protected final byte[] value2 = Bytes.toBytes("value2");
181 protected final byte[] row = Bytes.toBytes("rowA");
182 protected final byte[] row2 = Bytes.toBytes("rowB");
183
184 protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
185 .getInstance(MetricsAssertHelper.class);
186
187 @Before
188 public void setup() throws IOException {
189 TEST_UTIL = HBaseTestingUtility.createLocalHTU();
190 FILESYSTEM = TEST_UTIL.getTestFileSystem();
191 CONF = TEST_UTIL.getConfiguration();
192 dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
193 method = name.getMethodName();
194 tableName = Bytes.toBytes(name.getMethodName());
195 }
196
197 @After
198 public void tearDown() throws Exception {
199 EnvironmentEdgeManagerTestHelper.reset();
200 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
201 TEST_UTIL.cleanupTestDir();
202 }
203
204 String getName() {
205 return name.getMethodName();
206 }
207
208
209
210
211
212
213
214
215
216
217
218 @Test (timeout=60000)
219 public void testCloseCarryingSnapshot() throws IOException {
220 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
221 Store store = region.getStore(COLUMN_FAMILY_BYTES);
222
223 byte [] value = Bytes.toBytes(name.getMethodName());
224
225 Put put = new Put(value);
226 put.add(COLUMN_FAMILY_BYTES, null, value);
227
228 region.put(put);
229 StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
230 storeFlushCtx.prepare();
231
232 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
233 region.put(put);
234
235 region.close();
236 assertEquals(0, region.getMemstoreSize().get());
237 HRegion.closeHRegion(region);
238 }
239
240
241
242
243
244 @Test (timeout=60000)
245 public void testMemstoreSnapshotSize() throws IOException {
246 class MyFaultyHLog extends FaultyHLog {
247 StoreFlushContext storeFlushCtx;
248 public MyFaultyHLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
249 throws IOException {
250 super(fs, rootDir, logName, conf);
251 }
252
253 void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
254 this.storeFlushCtx = storeFlushCtx;
255 }
256
257 @Override
258 public void sync(long txid) throws IOException {
259 storeFlushCtx.prepare();
260 super.sync(txid);
261 }
262 }
263
264 FileSystem fs = FileSystem.get(CONF);
265 Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
266 MyFaultyHLog faultyLog = new MyFaultyHLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
267 HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
268 CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES);
269
270 Store store = region.getStore(COLUMN_FAMILY_BYTES);
271
272 byte [] value = Bytes.toBytes(name.getMethodName());
273 faultyLog.setStoreFlushCtx(store.createFlushContext(12345));
274
275 Put put = new Put(value);
276 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
277 faultyLog.setFailureType(FaultyHLog.FailureType.SYNC);
278
279 boolean threwIOE = false;
280 try {
281 region.put(put);
282 } catch (IOException ioe) {
283 threwIOE = true;
284 } finally {
285 assertTrue("The regionserver should have thrown an exception", threwIOE);
286 }
287 long sz = store.getFlushableSize();
288 assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
289 HRegion.closeHRegion(region);
290 }
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307 @Test (timeout=60000)
308 public void testFlushSizeAccounting() throws Exception {
309 final Configuration conf = HBaseConfiguration.create(CONF);
310
311 conf.setInt("hbase.hstore.flush.retries.number", 1);
312 final User user =
313 User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
314
315 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
316 user.runAs(new PrivilegedExceptionAction<Object>() {
317 @Override
318 public Object run() throws Exception {
319
320 FileSystem fs = FileSystem.get(conf);
321 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
322 FaultyFileSystem ffs = (FaultyFileSystem)fs;
323 HRegion region = null;
324 try {
325
326 region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
327 long size = region.getMemstoreSize().get();
328 Assert.assertEquals(0, size);
329
330 Put p1 = new Put(row);
331 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
332 region.put(p1);
333 final long sizeOfOnePut = region.getMemstoreSize().get();
334
335 try {
336 LOG.info("Flushing");
337 region.flushcache();
338 Assert.fail("Didn't bubble up IOE!");
339 } catch (DroppedSnapshotException dse) {
340
341 }
342
343 ffs.fault.set(false);
344
345 Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize().get());
346
347
348 Put p2 = new Put(row);
349 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
350 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
351 region.put(p2);
352 Assert.assertEquals(sizeOfOnePut * 3, region.getMemstoreSize().get());
353
354
355
356 region.flushcache();
357
358 Assert.assertEquals(sizeOfOnePut * 2, region.getMemstoreSize().get());
359 } finally {
360 HRegion.closeHRegion(region);
361 }
362 return null;
363 }
364 });
365 FileSystem.closeAllForUGI(user.getUGI());
366 }
367
368 @Test
369 public void testCompactionAffectedByScanners() throws Exception {
370 byte[] family = Bytes.toBytes("family");
371 this.region = initHRegion(tableName, method, CONF, family);
372
373 Put put = new Put(Bytes.toBytes("r1"));
374 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
375 region.put(put);
376 region.flushcache();
377
378 Scan scan = new Scan();
379 scan.setMaxVersions(3);
380
381 RegionScanner scanner1 = region.getScanner(scan);
382
383 Delete delete = new Delete(Bytes.toBytes("r1"));
384 region.delete(delete);
385 region.flushcache();
386
387
388 RegionScanner scanner2 = region.getScanner(scan);
389
390 List<Cell> results = new ArrayList<Cell>();
391
392 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
393
394
395 region.compactStores(true);
396
397
398 RegionScanner scanner3 = region.getScanner(scan);
399
400
401 scanner1.next(results);
402 System.out.println(results);
403 assertEquals(1, results.size());
404
405 results.clear();
406 scanner2.next(results);
407 System.out.println(results);
408 assertEquals(0, results.size());
409
410 results.clear();
411 scanner3.next(results);
412 System.out.println(results);
413 assertEquals(0, results.size());
414 }
415
416 @Test
417 public void testToShowNPEOnRegionScannerReseek() throws Exception {
418 byte[] family = Bytes.toBytes("family");
419 this.region = initHRegion(tableName, method, CONF, family);
420
421 Put put = new Put(Bytes.toBytes("r1"));
422 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
423 region.put(put);
424 put = new Put(Bytes.toBytes("r2"));
425 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
426 region.put(put);
427 region.flushcache();
428
429 Scan scan = new Scan();
430 scan.setMaxVersions(3);
431
432 RegionScanner scanner1 = region.getScanner(scan);
433
434 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
435
436 region.compactStores(true);
437
438 scanner1.reseek(Bytes.toBytes("r2"));
439 List<Cell> results = new ArrayList<Cell>();
440 scanner1.next(results);
441 Cell keyValue = results.get(0);
442 Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
443 scanner1.close();
444 }
445
446 @Test
447 public void testSkipRecoveredEditsReplay() throws Exception {
448 String method = "testSkipRecoveredEditsReplay";
449 TableName tableName = TableName.valueOf(method);
450 byte[] family = Bytes.toBytes("family");
451 this.region = initHRegion(tableName, method, CONF, family);
452 try {
453 Path regiondir = region.getRegionFileSystem().getRegionDir();
454 FileSystem fs = region.getRegionFileSystem().getFileSystem();
455 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
456
457 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
458
459 long maxSeqId = 1050;
460 long minSeqId = 1000;
461
462 for (long i = minSeqId; i <= maxSeqId; i += 10) {
463 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
464 fs.create(recoveredEdits);
465 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
466
467 long time = System.nanoTime();
468 WALEdit edit = new WALEdit();
469 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
470 .toBytes(i)));
471 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
472 HConstants.DEFAULT_CLUSTER_ID), edit));
473
474 writer.close();
475 }
476 MonitoredTask status = TaskMonitor.get().createStatus(method);
477 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
478 for (Store store : region.getStores().values()) {
479 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
480 }
481 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
482 assertEquals(maxSeqId, seqId);
483 Get get = new Get(row);
484 Result result = region.get(get);
485 for (long i = minSeqId; i <= maxSeqId; i += 10) {
486 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
487 assertEquals(1, kvs.size());
488 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
489 }
490 } finally {
491 HRegion.closeHRegion(this.region);
492 this.region = null;
493 }
494 }
495
496 @Test
497 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
498 String method = "testSkipRecoveredEditsReplaySomeIgnored";
499 TableName tableName = TableName.valueOf(method);
500 byte[] family = Bytes.toBytes("family");
501 this.region = initHRegion(tableName, method, CONF, family);
502 try {
503 Path regiondir = region.getRegionFileSystem().getRegionDir();
504 FileSystem fs = region.getRegionFileSystem().getFileSystem();
505 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
506
507 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
508
509 long maxSeqId = 1050;
510 long minSeqId = 1000;
511
512 for (long i = minSeqId; i <= maxSeqId; i += 10) {
513 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
514 fs.create(recoveredEdits);
515 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
516
517 long time = System.nanoTime();
518 WALEdit edit = new WALEdit();
519 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
520 .toBytes(i)));
521 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
522 HConstants.DEFAULT_CLUSTER_ID), edit));
523
524 writer.close();
525 }
526 long recoverSeqId = 1030;
527 MonitoredTask status = TaskMonitor.get().createStatus(method);
528 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
529 for (Store store : region.getStores().values()) {
530 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
531 }
532 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
533 assertEquals(maxSeqId, seqId);
534 Get get = new Get(row);
535 Result result = region.get(get);
536 for (long i = minSeqId; i <= maxSeqId; i += 10) {
537 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
538 if (i < recoverSeqId) {
539 assertEquals(0, kvs.size());
540 } else {
541 assertEquals(1, kvs.size());
542 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
543 }
544 }
545 } finally {
546 HRegion.closeHRegion(this.region);
547 this.region = null;
548 }
549 }
550
551 @Test
552 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
553 byte[] family = Bytes.toBytes("family");
554 this.region = initHRegion(tableName, method, CONF, family);
555 try {
556 Path regiondir = region.getRegionFileSystem().getRegionDir();
557 FileSystem fs = region.getRegionFileSystem().getFileSystem();
558
559 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
560 for (int i = 1000; i < 1050; i += 10) {
561 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
562 FSDataOutputStream dos = fs.create(recoveredEdits);
563 dos.writeInt(i);
564 dos.close();
565 }
566 long minSeqId = 2000;
567 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
568 FSDataOutputStream dos = fs.create(recoveredEdits);
569 dos.close();
570
571 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
572 for (Store store : region.getStores().values()) {
573 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
574 }
575 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
576 assertEquals(minSeqId, seqId);
577 } finally {
578 HRegion.closeHRegion(this.region);
579 this.region = null;
580 }
581 }
582
583 @Test
584 public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
585 String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
586 TableName tableName = TableName.valueOf(method);
587 byte[] family = Bytes.toBytes("family");
588 this.region = initHRegion(tableName, method, CONF, family);
589 try {
590 Path regiondir = region.getRegionFileSystem().getRegionDir();
591 FileSystem fs = region.getRegionFileSystem().getFileSystem();
592 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
593
594 assertEquals(0, region.getStoreFileList(
595 region.getStores().keySet().toArray(new byte[0][])).size());
596
597 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
598
599 long maxSeqId = 1050;
600 long minSeqId = 1000;
601
602 for (long i = minSeqId; i <= maxSeqId; i += 10) {
603 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
604 fs.create(recoveredEdits);
605 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
606
607 long time = System.nanoTime();
608 WALEdit edit = null;
609 if (i == maxSeqId) {
610 edit = WALEdit.createCompaction(region.getRegionInfo(),
611 CompactionDescriptor.newBuilder()
612 .setTableName(ByteString.copyFrom(tableName.getName()))
613 .setFamilyName(ByteString.copyFrom(regionName))
614 .setEncodedRegionName(ByteString.copyFrom(regionName))
615 .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
616 .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName()))
617 .build());
618 } else {
619 edit = new WALEdit();
620 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
621 .toBytes(i)));
622 }
623 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
624 HConstants.DEFAULT_CLUSTER_ID), edit));
625 writer.close();
626 }
627
628 long recoverSeqId = 1030;
629 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
630 MonitoredTask status = TaskMonitor.get().createStatus(method);
631 for (Store store : region.getStores().values()) {
632 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
633 }
634 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
635 assertEquals(maxSeqId, seqId);
636
637
638 assertEquals(1, region.getStoreFileList(
639 region.getStores().keySet().toArray(new byte[0][])).size());
640
641 } finally {
642 HRegion.closeHRegion(this.region);
643 this.region = null;
644 } }
645
646 @Test
647 public void testRecoveredEditsReplayCompaction() throws Exception {
648 String method = name.getMethodName();
649 TableName tableName = TableName.valueOf(method);
650 byte[] family = Bytes.toBytes("family");
651 this.region = initHRegion(tableName, method, CONF, family);
652 try {
653 Path regiondir = region.getRegionFileSystem().getRegionDir();
654 FileSystem fs = region.getRegionFileSystem().getFileSystem();
655 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
656
657 long maxSeqId = 3;
658 long minSeqId = 0;
659
660 for (long i = minSeqId; i < maxSeqId; i++) {
661 Put put = new Put(Bytes.toBytes(i));
662 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
663 region.put(put);
664 region.flushcache();
665 }
666
667
668 assertEquals(3, region.getStore(family).getStorefilesCount());
669 List<Path> storeFiles = new ArrayList<Path>(3);
670 for (StoreFile sf : region.getStore(family).getStorefiles()) {
671 storeFiles.add(sf.getPath());
672 }
673
674
675 CONF.setBoolean("hbase.hstore.compaction.complete", false);
676 region.compactStores();
677
678
679 assertEquals(3, region.getStore(family).getStorefilesCount());
680
681
682 Path tmpDir = region.getRegionFileSystem().getTempDir();
683 FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
684 String errorMsg = "Expected to find 1 file in the region temp directory "
685 + "from the compaction, could not find any";
686 assertNotNull(errorMsg, files);
687 assertEquals(errorMsg, 1, files.length);
688
689 Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
690 files[0].getPath());
691
692 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
693 .getRegionInfo(), family, storeFiles, Lists.newArrayList(newFile), region
694 .getRegionFileSystem().getStoreDir(Bytes.toString(family)));
695
696 HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
697 this.region.getRegionInfo(), compactionDescriptor, new AtomicLong(1));
698
699 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
700
701 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
702 fs.create(recoveredEdits);
703 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
704
705 long time = System.nanoTime();
706
707 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, 10, time,
708 HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(),
709 compactionDescriptor)));
710 writer.close();
711
712
713 region.getTableDesc();
714 region.getRegionInfo();
715 region.close();
716 region = HRegion.openHRegion(region, null);
717
718
719 Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
720 for (StoreFile sf : sfs) {
721 LOG.info(sf.getPath());
722 }
723 assertEquals(1, region.getStore(family).getStorefilesCount());
724 files = FSUtils.listStatus(fs, tmpDir);
725 assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
726
727 for (long i = minSeqId; i < maxSeqId; i++) {
728 Get get = new Get(Bytes.toBytes(i));
729 Result result = region.get(get);
730 byte[] value = result.getValue(family, Bytes.toBytes(i));
731 assertArrayEquals(Bytes.toBytes(i), value);
732 }
733 } finally {
734 HRegion.closeHRegion(this.region);
735 this.region = null;
736 }
737 }
738
739 @Test
740 public void testGetWhileRegionClose() throws IOException {
741 TableName tableName = TableName.valueOf(name.getMethodName());
742 Configuration hc = initSplit();
743 int numRows = 100;
744 byte[][] families = { fam1, fam2, fam3 };
745
746
747 String method = name.getMethodName();
748 this.region = initHRegion(tableName, method, hc, families);
749 try {
750
751 final int startRow = 100;
752 putData(startRow, numRows, qual1, families);
753 putData(startRow, numRows, qual2, families);
754 putData(startRow, numRows, qual3, families);
755 final AtomicBoolean done = new AtomicBoolean(false);
756 final AtomicInteger gets = new AtomicInteger(0);
757 GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
758 try {
759
760 for (int i = 0; i < threads.length / 2; i++) {
761 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
762 threads[i].setDaemon(true);
763 threads[i].start();
764 }
765
766
767 this.region.closing.set(true);
768 for (int i = threads.length / 2; i < threads.length; i++) {
769 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
770 threads[i].setDaemon(true);
771 threads[i].start();
772 }
773 } finally {
774 if (this.region != null) {
775 HRegion.closeHRegion(this.region);
776 }
777 }
778 done.set(true);
779 for (GetTillDoneOrException t : threads) {
780 try {
781 t.join();
782 } catch (InterruptedException e) {
783 e.printStackTrace();
784 }
785 if (t.e != null) {
786 LOG.info("Exception=" + t.e);
787 assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
788 }
789 }
790 } finally {
791 HRegion.closeHRegion(this.region);
792 this.region = null;
793 }
794 }
795
796
797
798
799
800 class GetTillDoneOrException extends Thread {
801 private final Get g;
802 private final AtomicBoolean done;
803 private final AtomicInteger count;
804 private Exception e;
805
806 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
807 super("getter." + i);
808 this.g = new Get(r);
809 this.done = d;
810 this.count = c;
811 }
812
813 @Override
814 public void run() {
815 while (!this.done.get()) {
816 try {
817 assertTrue(region.get(g).size() > 0);
818 this.count.incrementAndGet();
819 } catch (Exception e) {
820 this.e = e;
821 break;
822 }
823 }
824 }
825 }
826
827
828
829
830 @Test
831 public void testWeirdCacheBehaviour() throws Exception {
832 byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
833 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
834 Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
835 this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
836 try {
837 String value = "this is the value";
838 String value2 = "this is some other value";
839 String keyPrefix1 = "prefix1";
840 String keyPrefix2 = "prefix2";
841 String keyPrefix3 = "prefix3";
842 putRows(this.region, 3, value, keyPrefix1);
843 putRows(this.region, 3, value, keyPrefix2);
844 putRows(this.region, 3, value, keyPrefix3);
845 putRows(this.region, 3, value2, keyPrefix1);
846 putRows(this.region, 3, value2, keyPrefix2);
847 putRows(this.region, 3, value2, keyPrefix3);
848 System.out.println("Checking values for key: " + keyPrefix1);
849 assertEquals("Got back incorrect number of rows from scan", 3,
850 getNumberOfRows(keyPrefix1, value2, this.region));
851 System.out.println("Checking values for key: " + keyPrefix2);
852 assertEquals("Got back incorrect number of rows from scan", 3,
853 getNumberOfRows(keyPrefix2, value2, this.region));
854 System.out.println("Checking values for key: " + keyPrefix3);
855 assertEquals("Got back incorrect number of rows from scan", 3,
856 getNumberOfRows(keyPrefix3, value2, this.region));
857 deleteColumns(this.region, value2, keyPrefix1);
858 deleteColumns(this.region, value2, keyPrefix2);
859 deleteColumns(this.region, value2, keyPrefix3);
860 System.out.println("Starting important checks.....");
861 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
862 getNumberOfRows(keyPrefix1, value2, this.region));
863 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
864 getNumberOfRows(keyPrefix2, value2, this.region));
865 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
866 getNumberOfRows(keyPrefix3, value2, this.region));
867 } finally {
868 HRegion.closeHRegion(this.region);
869 this.region = null;
870 }
871 }
872
873 @Test
874 public void testAppendWithReadOnlyTable() throws Exception {
875 byte[] TABLE = Bytes.toBytes("readOnlyTable");
876 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
877 boolean exceptionCaught = false;
878 Append append = new Append(Bytes.toBytes("somerow"));
879 append.setDurability(Durability.SKIP_WAL);
880 append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
881 Bytes.toBytes("somevalue"));
882 try {
883 region.append(append);
884 } catch (IOException e) {
885 exceptionCaught = true;
886 } finally {
887 HRegion.closeHRegion(this.region);
888 this.region = null;
889 }
890 assertTrue(exceptionCaught == true);
891 }
892
893 @Test
894 public void testIncrWithReadOnlyTable() throws Exception {
895 byte[] TABLE = Bytes.toBytes("readOnlyTable");
896 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
897 boolean exceptionCaught = false;
898 Increment inc = new Increment(Bytes.toBytes("somerow"));
899 inc.setDurability(Durability.SKIP_WAL);
900 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
901 try {
902 region.increment(inc);
903 } catch (IOException e) {
904 exceptionCaught = true;
905 } finally {
906 HRegion.closeHRegion(this.region);
907 this.region = null;
908 }
909 assertTrue(exceptionCaught == true);
910 }
911
912 private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
913 InternalScanner scanner = buildScanner(keyPrefix, value, r);
914 int count = 0;
915 boolean more = false;
916 List<Cell> results = new ArrayList<Cell>();
917 do {
918 more = scanner.next(results);
919 if (results != null && !results.isEmpty())
920 count++;
921 else
922 break;
923 Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
924 delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
925 r.delete(delete);
926 results.clear();
927 } while (more);
928 assertEquals("Did not perform correct number of deletes", 3, count);
929 }
930
931 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
932 InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
933 int numberOfResults = 0;
934 List<Cell> results = new ArrayList<Cell>();
935 boolean more = false;
936 do {
937 more = resultScanner.next(results);
938 if (results != null && !results.isEmpty())
939 numberOfResults++;
940 else
941 break;
942 for (Cell kv : results) {
943 System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
944 }
945 results.clear();
946 } while (more);
947 return numberOfResults;
948 }
949
950 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
951 throws IOException {
952
953 FilterList allFilters = new FilterList();
954 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
955
956 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
957 Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
958 filter.setFilterIfMissing(true);
959 allFilters.addFilter(filter);
960 Scan scan = new Scan();
961 scan.addFamily(Bytes.toBytes("trans-blob"));
962 scan.addFamily(Bytes.toBytes("trans-type"));
963 scan.addFamily(Bytes.toBytes("trans-date"));
964 scan.addFamily(Bytes.toBytes("trans-tags"));
965 scan.addFamily(Bytes.toBytes("trans-group"));
966 scan.setFilter(allFilters);
967 return r.getScanner(scan);
968 }
969
970 private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
971 for (int i = 0; i < numRows; i++) {
972 String row = key + "_" + i
973 System.out.println(String.format("Saving row: %s, with value %s", row, value));
974 Put put = new Put(Bytes.toBytes(row));
975 put.setDurability(Durability.SKIP_WAL);
976 put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
977 put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
978 put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
979 put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
980 put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
981 r.put(put);
982 }
983 }
984
985 @Test
986 public void testFamilyWithAndWithoutColon() throws Exception {
987 byte[] b = Bytes.toBytes(getName());
988 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
989 this.region = initHRegion(b, getName(), CONF, cf);
990 try {
991 Put p = new Put(b);
992 byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
993 p.add(cfwithcolon, cfwithcolon, cfwithcolon);
994 boolean exception = false;
995 try {
996 this.region.put(p);
997 } catch (NoSuchColumnFamilyException e) {
998 exception = true;
999 }
1000 assertTrue(exception);
1001 } finally {
1002 HRegion.closeHRegion(this.region);
1003 this.region = null;
1004 }
1005 }
1006
1007 @Test
1008 public void testBatchPut() throws Exception {
1009 byte[] b = Bytes.toBytes(getName());
1010 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1011 byte[] qual = Bytes.toBytes("qual");
1012 byte[] val = Bytes.toBytes("val");
1013 this.region = initHRegion(b, getName(), CONF, cf);
1014 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1015 try {
1016 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1017 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1018
1019 LOG.info("First a batch put with all valid puts");
1020 final Put[] puts = new Put[10];
1021 for (int i = 0; i < 10; i++) {
1022 puts[i] = new Put(Bytes.toBytes("row_" + i));
1023 puts[i].add(cf, qual, val);
1024 }
1025
1026 OperationStatus[] codes = this.region.batchMutate(puts);
1027 assertEquals(10, codes.length);
1028 for (int i = 0; i < 10; i++) {
1029 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1030 }
1031 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1032
1033 LOG.info("Next a batch put with one invalid family");
1034 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1035 codes = this.region.batchMutate(puts);
1036 assertEquals(10, codes.length);
1037 for (int i = 0; i < 10; i++) {
1038 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1039 codes[i].getOperationStatusCode());
1040 }
1041
1042 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1043
1044 LOG.info("Next a batch put that has to break into two batches to avoid a lock");
1045 RowLock rowLock = region.getRowLock(Bytes.toBytes("row_2"));
1046
1047 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1048 final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
1049 TestThread putter = new TestThread(ctx) {
1050 @Override
1051 public void doWork() throws IOException {
1052 retFromThread.set(region.batchMutate(puts));
1053 }
1054 };
1055 LOG.info("...starting put thread while holding lock");
1056 ctx.addThread(putter);
1057 ctx.startThreads();
1058
1059 LOG.info("...waiting for put thread to sync first time");
1060 long startWait = System.currentTimeMillis();
1061 while (metricsAssertHelper.getCounter("syncTimeNumOps", source) == syncs + 2) {
1062 Thread.sleep(100);
1063 if (System.currentTimeMillis() - startWait > 10000) {
1064 fail("Timed out waiting for thread to sync first minibatch");
1065 }
1066 }
1067 LOG.info("...releasing row lock, which should let put thread continue");
1068 rowLock.release();
1069 LOG.info("...joining on thread");
1070 ctx.stop();
1071 LOG.info("...checking that next batch was synced");
1072 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 4, source);
1073 codes = retFromThread.get();
1074 for (int i = 0; i < 10; i++) {
1075 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1076 codes[i].getOperationStatusCode());
1077 }
1078
1079 } finally {
1080 HRegion.closeHRegion(this.region);
1081 this.region = null;
1082 }
1083 }
1084
1085 @Test
1086 public void testBatchPutWithTsSlop() throws Exception {
1087 byte[] b = Bytes.toBytes(getName());
1088 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1089 byte[] qual = Bytes.toBytes("qual");
1090 byte[] val = Bytes.toBytes("val");
1091
1092
1093 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1094 this.region = initHRegion(b, getName(), CONF, cf);
1095
1096 try {
1097 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1098 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1099 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1100
1101 final Put[] puts = new Put[10];
1102 for (int i = 0; i < 10; i++) {
1103 puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
1104 puts[i].add(cf, qual, val);
1105 }
1106
1107 OperationStatus[] codes = this.region.batchMutate(puts);
1108 assertEquals(10, codes.length);
1109 for (int i = 0; i < 10; i++) {
1110 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1111 }
1112 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1113
1114 } finally {
1115 HRegion.closeHRegion(this.region);
1116 this.region = null;
1117 }
1118
1119 }
1120
1121
1122
1123
1124 @Test
1125 public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1126 byte[] row1 = Bytes.toBytes("row1");
1127 byte[] fam1 = Bytes.toBytes("fam1");
1128 byte[] qf1 = Bytes.toBytes("qualifier");
1129 byte[] emptyVal = new byte[] {};
1130 byte[] val1 = Bytes.toBytes("value1");
1131 byte[] val2 = Bytes.toBytes("value2");
1132
1133
1134 String method = this.getName();
1135 this.region = initHRegion(tableName, method, CONF, fam1);
1136 try {
1137
1138 Put put = new Put(row1);
1139 put.add(fam1, qf1, emptyVal);
1140
1141
1142 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1143 emptyVal), put, true);
1144 assertTrue(res);
1145
1146
1147 put = new Put(row1);
1148 put.add(fam1, qf1, val1);
1149
1150
1151 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1152 put, true);
1153 assertTrue(res);
1154
1155
1156 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1157 put, true);
1158 assertFalse(res);
1159
1160 Delete delete = new Delete(row1);
1161 delete.deleteColumn(fam1, qf1);
1162 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1163 delete, true);
1164 assertFalse(res);
1165
1166 put = new Put(row1);
1167 put.add(fam1, qf1, val2);
1168
1169 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1170 put, true);
1171 assertTrue(res);
1172
1173
1174 delete = new Delete(row1);
1175 delete.deleteColumn(fam1, qf1);
1176 delete.deleteColumn(fam1, qf1);
1177 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1178 delete, true);
1179 assertTrue(res);
1180
1181 delete = new Delete(row1);
1182 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1183 delete, true);
1184 assertTrue(res);
1185
1186
1187 put = new Put(row1);
1188 put.add(fam1, qf1, val1);
1189
1190 res = region
1191 .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
1192 assertTrue(res);
1193 } finally {
1194 HRegion.closeHRegion(this.region);
1195 this.region = null;
1196 }
1197 }
1198
1199 @Test
1200 public void testCheckAndMutate_WithWrongValue() throws IOException {
1201 byte[] row1 = Bytes.toBytes("row1");
1202 byte[] fam1 = Bytes.toBytes("fam1");
1203 byte[] qf1 = Bytes.toBytes("qualifier");
1204 byte[] val1 = Bytes.toBytes("value1");
1205 byte[] val2 = Bytes.toBytes("value2");
1206
1207
1208 String method = this.getName();
1209 this.region = initHRegion(tableName, method, CONF, fam1);
1210 try {
1211
1212 Put put = new Put(row1);
1213 put.add(fam1, qf1, val1);
1214 region.put(put);
1215
1216
1217 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1218 val2), put, true);
1219 assertEquals(false, res);
1220
1221
1222 Delete delete = new Delete(row1);
1223 delete.deleteFamily(fam1);
1224 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1225 put, true);
1226 assertEquals(false, res);
1227 } finally {
1228 HRegion.closeHRegion(this.region);
1229 this.region = null;
1230 }
1231 }
1232
1233 @Test
1234 public void testCheckAndMutate_WithCorrectValue() throws IOException {
1235 byte[] row1 = Bytes.toBytes("row1");
1236 byte[] fam1 = Bytes.toBytes("fam1");
1237 byte[] qf1 = Bytes.toBytes("qualifier");
1238 byte[] val1 = Bytes.toBytes("value1");
1239
1240
1241 String method = this.getName();
1242 this.region = initHRegion(tableName, method, CONF, fam1);
1243 try {
1244
1245 Put put = new Put(row1);
1246 put.add(fam1, qf1, val1);
1247 region.put(put);
1248
1249
1250 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1251 val1), put, true);
1252 assertEquals(true, res);
1253
1254
1255 Delete delete = new Delete(row1);
1256 delete.deleteColumn(fam1, qf1);
1257 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1258 delete, true);
1259 assertEquals(true, res);
1260 } finally {
1261 HRegion.closeHRegion(this.region);
1262 this.region = null;
1263 }
1264 }
1265
1266 @Test
1267 public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1268 byte[] row1 = Bytes.toBytes("row1");
1269 byte[] fam1 = Bytes.toBytes("fam1");
1270 byte[] qf1 = Bytes.toBytes("qualifier");
1271 byte[] val1 = Bytes.toBytes("value1");
1272 byte[] val2 = Bytes.toBytes("value2");
1273 byte[] val3 = Bytes.toBytes("value3");
1274 byte[] val4 = Bytes.toBytes("value4");
1275
1276
1277 String method = this.getName();
1278 this.region = initHRegion(tableName, method, CONF, fam1);
1279 try {
1280
1281 Put put = new Put(row1);
1282 put.add(fam1, qf1, val3);
1283 region.put(put);
1284
1285
1286 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1287 new BinaryComparator(val3), put, true);
1288 assertEquals(false, res);
1289
1290
1291 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1292 new BinaryComparator(val4), put, true);
1293 assertEquals(false, res);
1294
1295
1296
1297 put = new Put(row1);
1298 put.add(fam1, qf1, val2);
1299 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1300 new BinaryComparator(val2), put, true);
1301 assertEquals(true, res);
1302
1303
1304 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1305 new BinaryComparator(val3), put, true);
1306 assertEquals(false, res);
1307
1308
1309
1310 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1311 new BinaryComparator(val2), put, true);
1312 assertEquals(true, res);
1313
1314
1315
1316 put = new Put(row1);
1317 put.add(fam1, qf1, val3);
1318 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1319 new BinaryComparator(val1), put, true);
1320 assertEquals(true, res);
1321
1322
1323 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1324 new BinaryComparator(val3), put, true);
1325 assertEquals(false, res);
1326
1327
1328 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1329 new BinaryComparator(val2), put, true);
1330 assertEquals(false, res);
1331
1332
1333
1334 put = new Put(row1);
1335 put.add(fam1, qf1, val2);
1336 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1337 new BinaryComparator(val4), put, true);
1338 assertEquals(true, res);
1339
1340
1341 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1342 new BinaryComparator(val1), put, true);
1343 assertEquals(false, res);
1344
1345
1346
1347 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1348 new BinaryComparator(val2), put, true);
1349 assertEquals(true, res);
1350
1351
1352 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1353 new BinaryComparator(val3), put, true);
1354 assertEquals(true, res);
1355 } finally {
1356 HRegion.closeHRegion(this.region);
1357 this.region = null;
1358 }
1359 }
1360
1361 @Test
1362 public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1363 byte[] row1 = Bytes.toBytes("row1");
1364 byte[] fam1 = Bytes.toBytes("fam1");
1365 byte[] fam2 = Bytes.toBytes("fam2");
1366 byte[] qf1 = Bytes.toBytes("qualifier");
1367 byte[] val1 = Bytes.toBytes("value1");
1368 byte[] val2 = Bytes.toBytes("value2");
1369
1370 byte[][] families = { fam1, fam2 };
1371
1372
1373 String method = this.getName();
1374 this.region = initHRegion(tableName, method, CONF, families);
1375 try {
1376
1377 Put put = new Put(row1);
1378 put.add(fam1, qf1, val1);
1379 region.put(put);
1380
1381
1382 long ts = System.currentTimeMillis();
1383 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1384 put = new Put(row1);
1385 put.add(kv);
1386
1387
1388 HStore store = (HStore) region.getStore(fam1);
1389 store.memstore.kvset.size();
1390
1391 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1392 val1), put, true);
1393 assertEquals(true, res);
1394 store.memstore.kvset.size();
1395
1396 Get get = new Get(row1);
1397 get.addColumn(fam2, qf1);
1398 Cell[] actual = region.get(get).rawCells();
1399
1400 Cell[] expected = { kv };
1401
1402 assertEquals(expected.length, actual.length);
1403 for (int i = 0; i < actual.length; i++) {
1404 assertEquals(expected[i], actual[i]);
1405 }
1406 } finally {
1407 HRegion.closeHRegion(this.region);
1408 this.region = null;
1409 }
1410 }
1411
1412 @Test
1413 public void testCheckAndPut_wrongRowInPut() throws IOException {
1414 TableName tableName = TableName.valueOf(name.getMethodName());
1415 this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
1416 try {
1417 Put put = new Put(row2);
1418 put.add(fam1, qual1, value1);
1419 try {
1420 region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1421 new BinaryComparator(value2), put, false);
1422 fail();
1423 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1424
1425 }
1426 } finally {
1427 HRegion.closeHRegion(this.region);
1428 this.region = null;
1429 }
1430 }
1431
1432 @Test
1433 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1434 byte[] row1 = Bytes.toBytes("row1");
1435 byte[] fam1 = Bytes.toBytes("fam1");
1436 byte[] fam2 = Bytes.toBytes("fam2");
1437 byte[] qf1 = Bytes.toBytes("qualifier1");
1438 byte[] qf2 = Bytes.toBytes("qualifier2");
1439 byte[] qf3 = Bytes.toBytes("qualifier3");
1440 byte[] val1 = Bytes.toBytes("value1");
1441 byte[] val2 = Bytes.toBytes("value2");
1442 byte[] val3 = Bytes.toBytes("value3");
1443 byte[] emptyVal = new byte[] {};
1444
1445 byte[][] families = { fam1, fam2 };
1446
1447
1448 String method = this.getName();
1449 this.region = initHRegion(tableName, method, CONF, families);
1450 try {
1451
1452 Put put = new Put(row1);
1453 put.add(fam1, qf1, val1);
1454 region.put(put);
1455 Threads.sleep(2);
1456
1457 put = new Put(row1);
1458 put.add(fam1, qf1, val2);
1459 put.add(fam2, qf1, val3);
1460 put.add(fam2, qf2, val2);
1461 put.add(fam2, qf3, val1);
1462 put.add(fam1, qf3, val1);
1463 region.put(put);
1464
1465
1466 Delete delete = new Delete(row1);
1467 delete.deleteColumn(fam1, qf1);
1468 delete.deleteColumn(fam2, qf1);
1469 delete.deleteColumn(fam1, qf3);
1470 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1471 val2), delete, true);
1472 assertEquals(true, res);
1473
1474 Get get = new Get(row1);
1475 get.addColumn(fam1, qf1);
1476 get.addColumn(fam1, qf3);
1477 get.addColumn(fam2, qf2);
1478 Result r = region.get(get);
1479 assertEquals(2, r.size());
1480 assertArrayEquals(val1, r.getValue(fam1, qf1));
1481 assertArrayEquals(val2, r.getValue(fam2, qf2));
1482
1483
1484 delete = new Delete(row1);
1485 delete.deleteFamily(fam2);
1486 res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1487 delete, true);
1488 assertEquals(true, res);
1489
1490 get = new Get(row1);
1491 r = region.get(get);
1492 assertEquals(1, r.size());
1493 assertArrayEquals(val1, r.getValue(fam1, qf1));
1494
1495
1496 delete = new Delete(row1);
1497 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1498 delete, true);
1499 assertEquals(true, res);
1500 get = new Get(row1);
1501 r = region.get(get);
1502 assertEquals(0, r.size());
1503 } finally {
1504 HRegion.closeHRegion(this.region);
1505 this.region = null;
1506 }
1507 }
1508
1509
1510
1511
1512 @Test
1513 public void testDelete_multiDeleteColumn() throws IOException {
1514 byte[] row1 = Bytes.toBytes("row1");
1515 byte[] fam1 = Bytes.toBytes("fam1");
1516 byte[] qual = Bytes.toBytes("qualifier");
1517 byte[] value = Bytes.toBytes("value");
1518
1519 Put put = new Put(row1);
1520 put.add(fam1, qual, 1, value);
1521 put.add(fam1, qual, 2, value);
1522
1523 String method = this.getName();
1524 this.region = initHRegion(tableName, method, CONF, fam1);
1525 try {
1526 region.put(put);
1527
1528
1529 Delete delete = new Delete(row1);
1530 delete.deleteColumn(fam1, qual);
1531 delete.deleteColumn(fam1, qual);
1532 region.delete(delete);
1533
1534 Get get = new Get(row1);
1535 get.addFamily(fam1);
1536 Result r = region.get(get);
1537 assertEquals(0, r.size());
1538 } finally {
1539 HRegion.closeHRegion(this.region);
1540 this.region = null;
1541 }
1542 }
1543
1544 @Test
1545 public void testDelete_CheckFamily() throws IOException {
1546 byte[] row1 = Bytes.toBytes("row1");
1547 byte[] fam1 = Bytes.toBytes("fam1");
1548 byte[] fam2 = Bytes.toBytes("fam2");
1549 byte[] fam3 = Bytes.toBytes("fam3");
1550 byte[] fam4 = Bytes.toBytes("fam4");
1551
1552
1553 String method = this.getName();
1554 this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
1555 try {
1556 List<Cell> kvs = new ArrayList<Cell>();
1557 kvs.add(new KeyValue(row1, fam4, null, null));
1558
1559
1560 byte[] family = fam2;
1561 try {
1562 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1563 Bytes.BYTES_COMPARATOR);
1564 deleteMap.put(family, kvs);
1565 region.delete(deleteMap, Durability.SYNC_WAL);
1566 } catch (Exception e) {
1567 assertTrue("Family " + new String(family) + " does not exist", false);
1568 }
1569
1570
1571 boolean ok = false;
1572 family = fam4;
1573 try {
1574 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1575 Bytes.BYTES_COMPARATOR);
1576 deleteMap.put(family, kvs);
1577 region.delete(deleteMap, Durability.SYNC_WAL);
1578 } catch (Exception e) {
1579 ok = true;
1580 }
1581 assertEquals("Family " + new String(family) + " does exist", true, ok);
1582 } finally {
1583 HRegion.closeHRegion(this.region);
1584 this.region = null;
1585 }
1586 }
1587
1588 @Test
1589 public void testDelete_mixed() throws IOException, InterruptedException {
1590 byte[] fam = Bytes.toBytes("info");
1591 byte[][] families = { fam };
1592 String method = this.getName();
1593 this.region = initHRegion(tableName, method, CONF, families);
1594 try {
1595 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1596
1597 byte[] row = Bytes.toBytes("table_name");
1598
1599 byte[] serverinfo = Bytes.toBytes("serverinfo");
1600 byte[] splitA = Bytes.toBytes("splitA");
1601 byte[] splitB = Bytes.toBytes("splitB");
1602
1603
1604 Put put = new Put(row);
1605 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1606 region.put(put);
1607
1608 put = new Put(row);
1609 put.add(fam, splitB, Bytes.toBytes("reference_B"));
1610 region.put(put);
1611
1612 put = new Put(row);
1613 put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1614 region.put(put);
1615
1616
1617 Delete delete = new Delete(row);
1618 delete.deleteColumns(fam, splitA);
1619 region.delete(delete);
1620
1621
1622 Get get = new Get(row).addColumn(fam, serverinfo);
1623 Result result = region.get(get);
1624 assertEquals(1, result.size());
1625
1626 get = new Get(row).addColumn(fam, splitA);
1627 result = region.get(get);
1628 assertEquals(0, result.size());
1629
1630 get = new Get(row).addColumn(fam, splitB);
1631 result = region.get(get);
1632 assertEquals(1, result.size());
1633
1634
1635 put = new Put(row);
1636 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1637 region.put(put);
1638 get = new Get(row);
1639 result = region.get(get);
1640 assertEquals(3, result.size());
1641
1642
1643 delete = new Delete(row);
1644 region.delete(delete);
1645 assertEquals(0, region.get(get).size());
1646
1647 region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
1648 result = region.get(get);
1649 assertEquals(1, result.size());
1650 } finally {
1651 HRegion.closeHRegion(this.region);
1652 this.region = null;
1653 }
1654 }
1655
1656 @Test
1657 public void testDeleteRowWithFutureTs() throws IOException {
1658 byte[] fam = Bytes.toBytes("info");
1659 byte[][] families = { fam };
1660 String method = this.getName();
1661 this.region = initHRegion(tableName, method, CONF, families);
1662 try {
1663 byte[] row = Bytes.toBytes("table_name");
1664
1665 byte[] serverinfo = Bytes.toBytes("serverinfo");
1666
1667
1668 Put put = new Put(row);
1669 put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
1670 region.put(put);
1671
1672
1673 Delete delete = new Delete(row);
1674 region.delete(delete);
1675
1676
1677 Get get = new Get(row).addColumn(fam, serverinfo);
1678 Result result = region.get(get);
1679 assertEquals(1, result.size());
1680
1681
1682 delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
1683 region.delete(delete);
1684
1685
1686 get = new Get(row).addColumn(fam, serverinfo);
1687 result = region.get(get);
1688 assertEquals(0, result.size());
1689 } finally {
1690 HRegion.closeHRegion(this.region);
1691 this.region = null;
1692 }
1693 }
1694
1695
1696
1697
1698
1699 @Test
1700 public void testPutWithLatestTS() throws IOException {
1701 byte[] fam = Bytes.toBytes("info");
1702 byte[][] families = { fam };
1703 String method = this.getName();
1704 this.region = initHRegion(tableName, method, CONF, families);
1705 try {
1706 byte[] row = Bytes.toBytes("row1");
1707
1708 byte[] qual = Bytes.toBytes("qual");
1709
1710
1711 Put put = new Put(row);
1712 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1713 region.put(put);
1714
1715
1716 Get get = new Get(row).addColumn(fam, qual);
1717 Result result = region.get(get);
1718 assertEquals(1, result.size());
1719 Cell kv = result.rawCells()[0];
1720 LOG.info("Got: " + kv);
1721 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1722 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1723
1724
1725
1726 row = Bytes.toBytes("row2");
1727 put = new Put(row);
1728 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1729 region.put(put);
1730
1731
1732 get = new Get(row).addColumn(fam, qual);
1733 result = region.get(get);
1734 assertEquals(1, result.size());
1735 kv = result.rawCells()[0];
1736 LOG.info("Got: " + kv);
1737 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1738 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1739 } finally {
1740 HRegion.closeHRegion(this.region);
1741 this.region = null;
1742 }
1743
1744 }
1745
1746
1747
1748
1749
1750
1751 @Test
1752 public void testPutWithTsSlop() throws IOException {
1753 byte[] fam = Bytes.toBytes("info");
1754 byte[][] families = { fam };
1755 String method = this.getName();
1756
1757
1758 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1759 this.region = initHRegion(tableName, method, CONF, families);
1760 boolean caughtExcep = false;
1761 try {
1762 try {
1763
1764 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
1765
1766 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
1767 Bytes.toBytes("value")));
1768 fail("Expected IOE for TS out of configured timerange");
1769 } catch (FailedSanityCheckException ioe) {
1770 LOG.debug("Received expected exception", ioe);
1771 caughtExcep = true;
1772 }
1773 assertTrue("Should catch FailedSanityCheckException", caughtExcep);
1774 } finally {
1775 HRegion.closeHRegion(this.region);
1776 this.region = null;
1777 }
1778 }
1779
1780 @Test
1781 public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
1782 byte[] fam1 = Bytes.toBytes("columnA");
1783 byte[] fam2 = Bytes.toBytes("columnB");
1784 this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
1785 try {
1786 byte[] rowA = Bytes.toBytes("rowA");
1787 byte[] rowB = Bytes.toBytes("rowB");
1788
1789 byte[] value = Bytes.toBytes("value");
1790
1791 Delete delete = new Delete(rowA);
1792 delete.deleteFamily(fam1);
1793
1794 region.delete(delete);
1795
1796
1797 Put put = new Put(rowA);
1798 put.add(fam2, null, value);
1799 region.put(put);
1800
1801 put = new Put(rowB);
1802 put.add(fam1, null, value);
1803 put.add(fam2, null, value);
1804 region.put(put);
1805
1806 Scan scan = new Scan();
1807 scan.addFamily(fam1).addFamily(fam2);
1808 InternalScanner s = region.getScanner(scan);
1809 List<Cell> results = new ArrayList<Cell>();
1810 s.next(results);
1811 assertTrue(CellUtil.matchingRow(results.get(0), rowA));
1812
1813 results.clear();
1814 s.next(results);
1815 assertTrue(CellUtil.matchingRow(results.get(0), rowB));
1816 } finally {
1817 HRegion.closeHRegion(this.region);
1818 this.region = null;
1819 }
1820 }
1821
1822 @Test
1823 public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
1824 Delete delete = new Delete(row);
1825 delete.deleteColumns(fam1, qual1);
1826 doTestDelete_AndPostInsert(delete);
1827 }
1828
1829 @Test
1830 public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
1831 Delete delete = new Delete(row);
1832 delete.deleteFamily(fam1);
1833 doTestDelete_AndPostInsert(delete);
1834 }
1835
1836 public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
1837 TableName tableName = TableName.valueOf(name.getMethodName());
1838 this.region = initHRegion(tableName, getName(), CONF, fam1);
1839 try {
1840 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1841 Put put = new Put(row);
1842 put.add(fam1, qual1, value1);
1843 region.put(put);
1844
1845
1846 region.delete(delete);
1847
1848
1849 put = new Put(row);
1850 put.add(fam1, qual1, value2);
1851 region.put(put);
1852
1853
1854 Get get = new Get(row);
1855 get.addColumn(fam1, qual1);
1856
1857 Result r = region.get(get);
1858 assertEquals(1, r.size());
1859 assertArrayEquals(value2, r.getValue(fam1, qual1));
1860
1861
1862 Scan scan = new Scan(row);
1863 scan.addColumn(fam1, qual1);
1864 InternalScanner s = region.getScanner(scan);
1865
1866 List<Cell> results = new ArrayList<Cell>();
1867 assertEquals(false, s.next(results));
1868 assertEquals(1, results.size());
1869 Cell kv = results.get(0);
1870
1871 assertArrayEquals(value2, CellUtil.cloneValue(kv));
1872 assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
1873 assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
1874 assertArrayEquals(row, CellUtil.cloneRow(kv));
1875 } finally {
1876 HRegion.closeHRegion(this.region);
1877 this.region = null;
1878 }
1879 }
1880
1881 @Test
1882 public void testDelete_CheckTimestampUpdated() throws IOException {
1883 TableName tableName = TableName.valueOf(name.getMethodName());
1884 byte[] row1 = Bytes.toBytes("row1");
1885 byte[] col1 = Bytes.toBytes("col1");
1886 byte[] col2 = Bytes.toBytes("col2");
1887 byte[] col3 = Bytes.toBytes("col3");
1888
1889
1890 String method = this.getName();
1891 this.region = initHRegion(tableName, method, CONF, fam1);
1892 try {
1893
1894 List<Cell> kvs = new ArrayList<Cell>();
1895 kvs.add(new KeyValue(row1, fam1, col1, null));
1896 kvs.add(new KeyValue(row1, fam1, col2, null));
1897 kvs.add(new KeyValue(row1, fam1, col3, null));
1898
1899 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1900 Bytes.BYTES_COMPARATOR);
1901 deleteMap.put(fam1, kvs);
1902 region.delete(deleteMap, Durability.SYNC_WAL);
1903
1904
1905
1906 long now = System.currentTimeMillis();
1907 KeyValue firstKv = ((HStore) region.getStore(fam1)).memstore.kvset.first();
1908 assertTrue(firstKv.getTimestamp() <= now);
1909 now = firstKv.getTimestamp();
1910 for (KeyValue kv : ((HStore) region.getStore(fam1)).memstore.kvset) {
1911 assertTrue(kv.getTimestamp() <= now);
1912 now = kv.getTimestamp();
1913 }
1914 } finally {
1915 HRegion.closeHRegion(this.region);
1916 this.region = null;
1917 }
1918 }
1919
1920
1921
1922
1923 @Test
1924 public void testGet_FamilyChecker() throws IOException {
1925 byte[] row1 = Bytes.toBytes("row1");
1926 byte[] fam1 = Bytes.toBytes("fam1");
1927 byte[] fam2 = Bytes.toBytes("False");
1928 byte[] col1 = Bytes.toBytes("col1");
1929
1930
1931 String method = this.getName();
1932 this.region = initHRegion(tableName, method, CONF, fam1);
1933 try {
1934 Get get = new Get(row1);
1935 get.addColumn(fam2, col1);
1936
1937
1938 try {
1939 region.get(get);
1940 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
1941 assertFalse(false);
1942 return;
1943 }
1944 assertFalse(true);
1945 } finally {
1946 HRegion.closeHRegion(this.region);
1947 this.region = null;
1948 }
1949 }
1950
1951 @Test
1952 public void testGet_Basic() throws IOException {
1953 byte[] row1 = Bytes.toBytes("row1");
1954 byte[] fam1 = Bytes.toBytes("fam1");
1955 byte[] col1 = Bytes.toBytes("col1");
1956 byte[] col2 = Bytes.toBytes("col2");
1957 byte[] col3 = Bytes.toBytes("col3");
1958 byte[] col4 = Bytes.toBytes("col4");
1959 byte[] col5 = Bytes.toBytes("col5");
1960
1961
1962 String method = this.getName();
1963 this.region = initHRegion(tableName, method, CONF, fam1);
1964 try {
1965
1966 Put put = new Put(row1);
1967 put.add(fam1, col1, null);
1968 put.add(fam1, col2, null);
1969 put.add(fam1, col3, null);
1970 put.add(fam1, col4, null);
1971 put.add(fam1, col5, null);
1972 region.put(put);
1973
1974 Get get = new Get(row1);
1975 get.addColumn(fam1, col2);
1976 get.addColumn(fam1, col4);
1977
1978 KeyValue kv1 = new KeyValue(row1, fam1, col2);
1979 KeyValue kv2 = new KeyValue(row1, fam1, col4);
1980 KeyValue[] expected = { kv1, kv2 };
1981
1982
1983 Result res = region.get(get);
1984 assertEquals(expected.length, res.size());
1985 for (int i = 0; i < res.size(); i++) {
1986 assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
1987 assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
1988 assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
1989 }
1990
1991
1992 Get g = new Get(row1);
1993 final int count = 2;
1994 g.setFilter(new ColumnCountGetFilter(count));
1995 res = region.get(g);
1996 assertEquals(count, res.size());
1997 } finally {
1998 HRegion.closeHRegion(this.region);
1999 this.region = null;
2000 }
2001 }
2002
2003 @Test
2004 public void testGet_Empty() throws IOException {
2005 byte[] row = Bytes.toBytes("row");
2006 byte[] fam = Bytes.toBytes("fam");
2007
2008 String method = this.getName();
2009 this.region = initHRegion(tableName, method, CONF, fam);
2010 try {
2011 Get get = new Get(row);
2012 get.addFamily(fam);
2013 Result r = region.get(get);
2014
2015 assertTrue(r.isEmpty());
2016 } finally {
2017 HRegion.closeHRegion(this.region);
2018 this.region = null;
2019 }
2020 }
2021
2022
2023
2024
2025 @Test
2026 public void testMerge() throws IOException {
2027 byte[][] families = { fam1, fam2, fam3 };
2028 Configuration hc = initSplit();
2029
2030 String method = this.getName();
2031 this.region = initHRegion(tableName, method, hc, families);
2032 try {
2033 LOG.info("" + HBaseTestCase.addContent(region, fam3));
2034 region.flushcache();
2035 region.compactStores();
2036 byte[] splitRow = region.checkSplit();
2037 assertNotNull(splitRow);
2038 LOG.info("SplitRow: " + Bytes.toString(splitRow));
2039 HRegion[] subregions = splitRegion(region, splitRow);
2040 try {
2041
2042 for (int i = 0; i < subregions.length; i++) {
2043 HRegion.openHRegion(subregions[i], null);
2044 subregions[i].compactStores();
2045 }
2046 Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
2047 Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
2048 Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
2049 long startTime = System.currentTimeMillis();
2050 region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
2051 LOG.info("Merge regions elapsed time: "
2052 + ((System.currentTimeMillis() - startTime) / 1000.0));
2053 FILESYSTEM.delete(oldRegion1, true);
2054 FILESYSTEM.delete(oldRegion2, true);
2055 FILESYSTEM.delete(oldRegionPath, true);
2056 LOG.info("splitAndMerge completed.");
2057 } finally {
2058 for (int i = 0; i < subregions.length; i++) {
2059 try {
2060 HRegion.closeHRegion(subregions[i]);
2061 } catch (IOException e) {
2062
2063 }
2064 }
2065 }
2066 } finally {
2067 HRegion.closeHRegion(this.region);
2068 this.region = null;
2069 }
2070 }
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080 HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
2081 PairOfSameType<HRegion> result = null;
2082 SplitTransaction st = new SplitTransaction(parent, midkey);
2083
2084
2085 if (!st.prepare()) {
2086 parent.clearSplit();
2087 return null;
2088 }
2089 try {
2090 result = st.execute(null, null);
2091 } catch (IOException ioe) {
2092 try {
2093 LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; "
2094 + ioe.getMessage());
2095 st.rollback(null, null);
2096 LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString());
2097 return null;
2098 } catch (RuntimeException e) {
2099
2100 LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString()
2101 + " -- aborting server", e);
2102 }
2103 }
2104 finally {
2105 parent.clearSplit();
2106 }
2107 return new HRegion[] { result.getFirst(), result.getSecond() };
2108 }
2109
2110
2111
2112
2113 @Test
2114 public void testGetScanner_WithOkFamilies() throws IOException {
2115 byte[] fam1 = Bytes.toBytes("fam1");
2116 byte[] fam2 = Bytes.toBytes("fam2");
2117
2118 byte[][] families = { fam1, fam2 };
2119
2120
2121 String method = this.getName();
2122 this.region = initHRegion(tableName, method, CONF, families);
2123 try {
2124 Scan scan = new Scan();
2125 scan.addFamily(fam1);
2126 scan.addFamily(fam2);
2127 try {
2128 region.getScanner(scan);
2129 } catch (Exception e) {
2130 assertTrue("Families could not be found in Region", false);
2131 }
2132 } finally {
2133 HRegion.closeHRegion(this.region);
2134 this.region = null;
2135 }
2136 }
2137
2138 @Test
2139 public void testGetScanner_WithNotOkFamilies() throws IOException {
2140 byte[] fam1 = Bytes.toBytes("fam1");
2141 byte[] fam2 = Bytes.toBytes("fam2");
2142
2143 byte[][] families = { fam1 };
2144
2145
2146 String method = this.getName();
2147 this.region = initHRegion(tableName, method, CONF, families);
2148 try {
2149 Scan scan = new Scan();
2150 scan.addFamily(fam2);
2151 boolean ok = false;
2152 try {
2153 region.getScanner(scan);
2154 } catch (Exception e) {
2155 ok = true;
2156 }
2157 assertTrue("Families could not be found in Region", ok);
2158 } finally {
2159 HRegion.closeHRegion(this.region);
2160 this.region = null;
2161 }
2162 }
2163
2164 @Test
2165 public void testGetScanner_WithNoFamilies() throws IOException {
2166 byte[] row1 = Bytes.toBytes("row1");
2167 byte[] fam1 = Bytes.toBytes("fam1");
2168 byte[] fam2 = Bytes.toBytes("fam2");
2169 byte[] fam3 = Bytes.toBytes("fam3");
2170 byte[] fam4 = Bytes.toBytes("fam4");
2171
2172 byte[][] families = { fam1, fam2, fam3, fam4 };
2173
2174
2175 String method = this.getName();
2176 this.region = initHRegion(tableName, method, CONF, families);
2177 try {
2178
2179
2180 Put put = new Put(row1);
2181 put.add(fam1, null, null);
2182 put.add(fam2, null, null);
2183 put.add(fam3, null, null);
2184 put.add(fam4, null, null);
2185 region.put(put);
2186
2187 Scan scan = null;
2188 HRegion.RegionScannerImpl is = null;
2189
2190
2191
2192
2193 scan = new Scan();
2194 scan.addFamily(fam2);
2195 scan.addFamily(fam4);
2196 is = (RegionScannerImpl) region.getScanner(scan);
2197 assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2198
2199 scan = new Scan();
2200 is = (RegionScannerImpl) region.getScanner(scan);
2201 assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2202 } finally {
2203 HRegion.closeHRegion(this.region);
2204 this.region = null;
2205 }
2206 }
2207
2208
2209
2210
2211
2212
2213 @Test
2214 public void testGetScanner_WithRegionClosed() throws IOException {
2215 byte[] fam1 = Bytes.toBytes("fam1");
2216 byte[] fam2 = Bytes.toBytes("fam2");
2217
2218 byte[][] families = { fam1, fam2 };
2219
2220
2221 String method = this.getName();
2222 try {
2223 this.region = initHRegion(tableName, method, CONF, families);
2224 } catch (IOException e) {
2225 e.printStackTrace();
2226 fail("Got IOException during initHRegion, " + e.getMessage());
2227 }
2228 try {
2229 region.closed.set(true);
2230 try {
2231 region.getScanner(null);
2232 fail("Expected to get an exception during getScanner on a region that is closed");
2233 } catch (NotServingRegionException e) {
2234
2235 } catch (IOException e) {
2236 fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2237 + e.getMessage());
2238 }
2239 } finally {
2240 HRegion.closeHRegion(this.region);
2241 this.region = null;
2242 }
2243 }
2244
2245 @Test
2246 public void testRegionScanner_Next() throws IOException {
2247 byte[] row1 = Bytes.toBytes("row1");
2248 byte[] row2 = Bytes.toBytes("row2");
2249 byte[] fam1 = Bytes.toBytes("fam1");
2250 byte[] fam2 = Bytes.toBytes("fam2");
2251 byte[] fam3 = Bytes.toBytes("fam3");
2252 byte[] fam4 = Bytes.toBytes("fam4");
2253
2254 byte[][] families = { fam1, fam2, fam3, fam4 };
2255 long ts = System.currentTimeMillis();
2256
2257
2258 String method = this.getName();
2259 this.region = initHRegion(tableName, method, CONF, families);
2260 try {
2261
2262 Put put = null;
2263 put = new Put(row1);
2264 put.add(fam1, (byte[]) null, ts, null);
2265 put.add(fam2, (byte[]) null, ts, null);
2266 put.add(fam3, (byte[]) null, ts, null);
2267 put.add(fam4, (byte[]) null, ts, null);
2268 region.put(put);
2269
2270 put = new Put(row2);
2271 put.add(fam1, (byte[]) null, ts, null);
2272 put.add(fam2, (byte[]) null, ts, null);
2273 put.add(fam3, (byte[]) null, ts, null);
2274 put.add(fam4, (byte[]) null, ts, null);
2275 region.put(put);
2276
2277 Scan scan = new Scan();
2278 scan.addFamily(fam2);
2279 scan.addFamily(fam4);
2280 InternalScanner is = region.getScanner(scan);
2281
2282 List<Cell> res = null;
2283
2284
2285 List<Cell> expected1 = new ArrayList<Cell>();
2286 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2287 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2288
2289 res = new ArrayList<Cell>();
2290 is.next(res);
2291 for (int i = 0; i < res.size(); i++) {
2292 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2293 }
2294
2295
2296 List<Cell> expected2 = new ArrayList<Cell>();
2297 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2298 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2299
2300 res = new ArrayList<Cell>();
2301 is.next(res);
2302 for (int i = 0; i < res.size(); i++) {
2303 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2304 }
2305 } finally {
2306 HRegion.closeHRegion(this.region);
2307 this.region = null;
2308 }
2309 }
2310
2311 @Test
2312 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2313 byte[] row1 = Bytes.toBytes("row1");
2314 byte[] qf1 = Bytes.toBytes("qualifier1");
2315 byte[] qf2 = Bytes.toBytes("qualifier2");
2316 byte[] fam1 = Bytes.toBytes("fam1");
2317 byte[][] families = { fam1 };
2318
2319 long ts1 = System.currentTimeMillis();
2320 long ts2 = ts1 + 1;
2321 long ts3 = ts1 + 2;
2322
2323
2324 String method = this.getName();
2325 this.region = initHRegion(tableName, method, CONF, families);
2326 try {
2327
2328 Put put = null;
2329 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2330 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2331 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2332
2333 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2334 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2335 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2336
2337 put = new Put(row1);
2338 put.add(kv13);
2339 put.add(kv12);
2340 put.add(kv11);
2341 put.add(kv23);
2342 put.add(kv22);
2343 put.add(kv21);
2344 region.put(put);
2345
2346
2347 List<Cell> expected = new ArrayList<Cell>();
2348 expected.add(kv13);
2349 expected.add(kv12);
2350
2351 Scan scan = new Scan(row1);
2352 scan.addColumn(fam1, qf1);
2353 scan.setMaxVersions(MAX_VERSIONS);
2354 List<Cell> actual = new ArrayList<Cell>();
2355 InternalScanner scanner = region.getScanner(scan);
2356
2357 boolean hasNext = scanner.next(actual);
2358 assertEquals(false, hasNext);
2359
2360
2361 for (int i = 0; i < expected.size(); i++) {
2362 assertEquals(expected.get(i), actual.get(i));
2363 }
2364 } finally {
2365 HRegion.closeHRegion(this.region);
2366 this.region = null;
2367 }
2368 }
2369
2370 @Test
2371 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2372 byte[] row1 = Bytes.toBytes("row1");
2373 byte[] qf1 = Bytes.toBytes("qualifier1");
2374 byte[] qf2 = Bytes.toBytes("qualifier2");
2375 byte[] fam1 = Bytes.toBytes("fam1");
2376 byte[][] families = { fam1 };
2377
2378 long ts1 = 1;
2379 long ts2 = ts1 + 1;
2380 long ts3 = ts1 + 2;
2381
2382
2383 String method = this.getName();
2384 this.region = initHRegion(tableName, method, CONF, families);
2385 try {
2386
2387 Put put = null;
2388 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2389 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2390 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2391
2392 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2393 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2394 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2395
2396 put = new Put(row1);
2397 put.add(kv13);
2398 put.add(kv12);
2399 put.add(kv11);
2400 put.add(kv23);
2401 put.add(kv22);
2402 put.add(kv21);
2403 region.put(put);
2404 region.flushcache();
2405
2406
2407 List<Cell> expected = new ArrayList<Cell>();
2408 expected.add(kv13);
2409 expected.add(kv12);
2410 expected.add(kv23);
2411 expected.add(kv22);
2412
2413 Scan scan = new Scan(row1);
2414 scan.addColumn(fam1, qf1);
2415 scan.addColumn(fam1, qf2);
2416 scan.setMaxVersions(MAX_VERSIONS);
2417 List<Cell> actual = new ArrayList<Cell>();
2418 InternalScanner scanner = region.getScanner(scan);
2419
2420 boolean hasNext = scanner.next(actual);
2421 assertEquals(false, hasNext);
2422
2423
2424 for (int i = 0; i < expected.size(); i++) {
2425 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2426 }
2427 } finally {
2428 HRegion.closeHRegion(this.region);
2429 this.region = null;
2430 }
2431 }
2432
2433 @Test
2434 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2435 byte[] row1 = Bytes.toBytes("row1");
2436 byte[] fam1 = Bytes.toBytes("fam1");
2437 byte[][] families = { fam1 };
2438 byte[] qf1 = Bytes.toBytes("qualifier1");
2439 byte[] qf2 = Bytes.toBytes("qualifier2");
2440
2441 long ts1 = 1;
2442 long ts2 = ts1 + 1;
2443 long ts3 = ts1 + 2;
2444 long ts4 = ts1 + 3;
2445
2446
2447 String method = this.getName();
2448 this.region = initHRegion(tableName, method, CONF, families);
2449 try {
2450
2451 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2452 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2453 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2454 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2455
2456 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2457 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2458 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2459 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2460
2461 Put put = null;
2462 put = new Put(row1);
2463 put.add(kv14);
2464 put.add(kv24);
2465 region.put(put);
2466 region.flushcache();
2467
2468 put = new Put(row1);
2469 put.add(kv23);
2470 put.add(kv13);
2471 region.put(put);
2472 region.flushcache();
2473
2474 put = new Put(row1);
2475 put.add(kv22);
2476 put.add(kv12);
2477 region.put(put);
2478 region.flushcache();
2479
2480 put = new Put(row1);
2481 put.add(kv21);
2482 put.add(kv11);
2483 region.put(put);
2484
2485
2486 List<Cell> expected = new ArrayList<Cell>();
2487 expected.add(kv14);
2488 expected.add(kv13);
2489 expected.add(kv12);
2490 expected.add(kv24);
2491 expected.add(kv23);
2492 expected.add(kv22);
2493
2494 Scan scan = new Scan(row1);
2495 scan.addColumn(fam1, qf1);
2496 scan.addColumn(fam1, qf2);
2497 int versions = 3;
2498 scan.setMaxVersions(versions);
2499 List<Cell> actual = new ArrayList<Cell>();
2500 InternalScanner scanner = region.getScanner(scan);
2501
2502 boolean hasNext = scanner.next(actual);
2503 assertEquals(false, hasNext);
2504
2505
2506 for (int i = 0; i < expected.size(); i++) {
2507 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2508 }
2509 } finally {
2510 HRegion.closeHRegion(this.region);
2511 this.region = null;
2512 }
2513 }
2514
2515 @Test
2516 public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2517 byte[] row1 = Bytes.toBytes("row1");
2518 byte[] qf1 = Bytes.toBytes("qualifier1");
2519 byte[] qf2 = Bytes.toBytes("qualifier2");
2520 byte[] fam1 = Bytes.toBytes("fam1");
2521 byte[][] families = { fam1 };
2522
2523 long ts1 = System.currentTimeMillis();
2524 long ts2 = ts1 + 1;
2525 long ts3 = ts1 + 2;
2526
2527
2528 String method = this.getName();
2529 this.region = initHRegion(tableName, method, CONF, families);
2530 try {
2531
2532 Put put = null;
2533 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2534 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2535 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2536
2537 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2538 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2539 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2540
2541 put = new Put(row1);
2542 put.add(kv13);
2543 put.add(kv12);
2544 put.add(kv11);
2545 put.add(kv23);
2546 put.add(kv22);
2547 put.add(kv21);
2548 region.put(put);
2549
2550
2551 List<Cell> expected = new ArrayList<Cell>();
2552 expected.add(kv13);
2553 expected.add(kv12);
2554 expected.add(kv23);
2555 expected.add(kv22);
2556
2557 Scan scan = new Scan(row1);
2558 scan.addFamily(fam1);
2559 scan.setMaxVersions(MAX_VERSIONS);
2560 List<Cell> actual = new ArrayList<Cell>();
2561 InternalScanner scanner = region.getScanner(scan);
2562
2563 boolean hasNext = scanner.next(actual);
2564 assertEquals(false, hasNext);
2565
2566
2567 for (int i = 0; i < expected.size(); i++) {
2568 assertEquals(expected.get(i), actual.get(i));
2569 }
2570 } finally {
2571 HRegion.closeHRegion(this.region);
2572 this.region = null;
2573 }
2574 }
2575
2576 @Test
2577 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
2578 byte[] row1 = Bytes.toBytes("row1");
2579 byte[] qf1 = Bytes.toBytes("qualifier1");
2580 byte[] qf2 = Bytes.toBytes("qualifier2");
2581 byte[] fam1 = Bytes.toBytes("fam1");
2582
2583 long ts1 = 1;
2584 long ts2 = ts1 + 1;
2585 long ts3 = ts1 + 2;
2586
2587
2588 String method = this.getName();
2589 this.region = initHRegion(tableName, method, CONF, fam1);
2590 try {
2591
2592 Put put = null;
2593 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2594 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2595 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2596
2597 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2598 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2599 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2600
2601 put = new Put(row1);
2602 put.add(kv13);
2603 put.add(kv12);
2604 put.add(kv11);
2605 put.add(kv23);
2606 put.add(kv22);
2607 put.add(kv21);
2608 region.put(put);
2609 region.flushcache();
2610
2611
2612 List<Cell> expected = new ArrayList<Cell>();
2613 expected.add(kv13);
2614 expected.add(kv12);
2615 expected.add(kv23);
2616 expected.add(kv22);
2617
2618 Scan scan = new Scan(row1);
2619 scan.addFamily(fam1);
2620 scan.setMaxVersions(MAX_VERSIONS);
2621 List<Cell> actual = new ArrayList<Cell>();
2622 InternalScanner scanner = region.getScanner(scan);
2623
2624 boolean hasNext = scanner.next(actual);
2625 assertEquals(false, hasNext);
2626
2627
2628 for (int i = 0; i < expected.size(); i++) {
2629 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2630 }
2631 } finally {
2632 HRegion.closeHRegion(this.region);
2633 this.region = null;
2634 }
2635 }
2636
2637 @Test
2638 public void testScanner_StopRow1542() throws IOException {
2639 byte[] family = Bytes.toBytes("testFamily");
2640 this.region = initHRegion(tableName, getName(), CONF, family);
2641 try {
2642 byte[] row1 = Bytes.toBytes("row111");
2643 byte[] row2 = Bytes.toBytes("row222");
2644 byte[] row3 = Bytes.toBytes("row333");
2645 byte[] row4 = Bytes.toBytes("row444");
2646 byte[] row5 = Bytes.toBytes("row555");
2647
2648 byte[] col1 = Bytes.toBytes("Pub111");
2649 byte[] col2 = Bytes.toBytes("Pub222");
2650
2651 Put put = new Put(row1);
2652 put.add(family, col1, Bytes.toBytes(10L));
2653 region.put(put);
2654
2655 put = new Put(row2);
2656 put.add(family, col1, Bytes.toBytes(15L));
2657 region.put(put);
2658
2659 put = new Put(row3);
2660 put.add(family, col2, Bytes.toBytes(20L));
2661 region.put(put);
2662
2663 put = new Put(row4);
2664 put.add(family, col2, Bytes.toBytes(30L));
2665 region.put(put);
2666
2667 put = new Put(row5);
2668 put.add(family, col1, Bytes.toBytes(40L));
2669 region.put(put);
2670
2671 Scan scan = new Scan(row3, row4);
2672 scan.setMaxVersions();
2673 scan.addColumn(family, col1);
2674 InternalScanner s = region.getScanner(scan);
2675
2676 List<Cell> results = new ArrayList<Cell>();
2677 assertEquals(false, s.next(results));
2678 assertEquals(0, results.size());
2679 } finally {
2680 HRegion.closeHRegion(this.region);
2681 this.region = null;
2682 }
2683 }
2684
2685 @Test
2686 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2687 byte[] row1 = Bytes.toBytes("row1");
2688 byte[] fam1 = Bytes.toBytes("fam1");
2689 byte[] qf1 = Bytes.toBytes("qualifier1");
2690 byte[] qf2 = Bytes.toBytes("quateslifier2");
2691
2692 long ts1 = 1;
2693 long ts2 = ts1 + 1;
2694 long ts3 = ts1 + 2;
2695 long ts4 = ts1 + 3;
2696
2697
2698 String method = this.getName();
2699 this.region = initHRegion(tableName, method, CONF, fam1);
2700 try {
2701
2702 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2703 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2704 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2705 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2706
2707 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2708 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2709 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2710 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2711
2712 Put put = null;
2713 put = new Put(row1);
2714 put.add(kv14);
2715 put.add(kv24);
2716 region.put(put);
2717 region.flushcache();
2718
2719 put = new Put(row1);
2720 put.add(kv23);
2721 put.add(kv13);
2722 region.put(put);
2723 region.flushcache();
2724
2725 put = new Put(row1);
2726 put.add(kv22);
2727 put.add(kv12);
2728 region.put(put);
2729 region.flushcache();
2730
2731 put = new Put(row1);
2732 put.add(kv21);
2733 put.add(kv11);
2734 region.put(put);
2735
2736
2737 List<KeyValue> expected = new ArrayList<KeyValue>();
2738 expected.add(kv14);
2739 expected.add(kv13);
2740 expected.add(kv12);
2741 expected.add(kv24);
2742 expected.add(kv23);
2743 expected.add(kv22);
2744
2745 Scan scan = new Scan(row1);
2746 int versions = 3;
2747 scan.setMaxVersions(versions);
2748 List<Cell> actual = new ArrayList<Cell>();
2749 InternalScanner scanner = region.getScanner(scan);
2750
2751 boolean hasNext = scanner.next(actual);
2752 assertEquals(false, hasNext);
2753
2754
2755 for (int i = 0; i < expected.size(); i++) {
2756 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2757 }
2758 } finally {
2759 HRegion.closeHRegion(this.region);
2760 this.region = null;
2761 }
2762 }
2763
2764
2765
2766
2767
2768
2769
2770 @Test
2771 public void testScanner_JoinedScanners() throws IOException {
2772 byte[] cf_essential = Bytes.toBytes("essential");
2773 byte[] cf_joined = Bytes.toBytes("joined");
2774 byte[] cf_alpha = Bytes.toBytes("alpha");
2775 this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
2776 try {
2777 byte[] row1 = Bytes.toBytes("row1");
2778 byte[] row2 = Bytes.toBytes("row2");
2779 byte[] row3 = Bytes.toBytes("row3");
2780
2781 byte[] col_normal = Bytes.toBytes("d");
2782 byte[] col_alpha = Bytes.toBytes("a");
2783
2784 byte[] filtered_val = Bytes.toBytes(3);
2785
2786 Put put = new Put(row1);
2787 put.add(cf_essential, col_normal, Bytes.toBytes(1));
2788 put.add(cf_joined, col_alpha, Bytes.toBytes(1));
2789 region.put(put);
2790
2791 put = new Put(row2);
2792 put.add(cf_essential, col_alpha, Bytes.toBytes(2));
2793 put.add(cf_joined, col_normal, Bytes.toBytes(2));
2794 put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
2795 region.put(put);
2796
2797 put = new Put(row3);
2798 put.add(cf_essential, col_normal, filtered_val);
2799 put.add(cf_joined, col_normal, filtered_val);
2800 region.put(put);
2801
2802
2803
2804
2805
2806 Scan scan = new Scan();
2807 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
2808 CompareOp.NOT_EQUAL, filtered_val);
2809 scan.setFilter(filter);
2810 scan.setLoadColumnFamiliesOnDemand(true);
2811 InternalScanner s = region.getScanner(scan);
2812
2813 List<Cell> results = new ArrayList<Cell>();
2814 assertTrue(s.next(results));
2815 assertEquals(results.size(), 1);
2816 results.clear();
2817
2818 assertTrue(s.next(results));
2819 assertEquals(results.size(), 3);
2820 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
2821 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
2822 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
2823 results.clear();
2824
2825 assertFalse(s.next(results));
2826 assertEquals(results.size(), 0);
2827 } finally {
2828 HRegion.closeHRegion(this.region);
2829 this.region = null;
2830 }
2831 }
2832
2833
2834
2835
2836
2837
2838 @Test
2839 public void testScanner_JoinedScannersWithLimits() throws IOException {
2840 final byte[] cf_first = Bytes.toBytes("first");
2841 final byte[] cf_second = Bytes.toBytes("second");
2842
2843 this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
2844 try {
2845 final byte[] col_a = Bytes.toBytes("a");
2846 final byte[] col_b = Bytes.toBytes("b");
2847
2848 Put put;
2849
2850 for (int i = 0; i < 10; i++) {
2851 put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
2852 put.add(cf_first, col_a, Bytes.toBytes(i));
2853 if (i < 5) {
2854 put.add(cf_first, col_b, Bytes.toBytes(i));
2855 put.add(cf_second, col_a, Bytes.toBytes(i));
2856 put.add(cf_second, col_b, Bytes.toBytes(i));
2857 }
2858 region.put(put);
2859 }
2860
2861 Scan scan = new Scan();
2862 scan.setLoadColumnFamiliesOnDemand(true);
2863 Filter bogusFilter = new FilterBase() {
2864 @Override
2865 public boolean isFamilyEssential(byte[] name) {
2866 return Bytes.equals(name, cf_first);
2867 }
2868 };
2869
2870 scan.setFilter(bogusFilter);
2871 InternalScanner s = region.getScanner(scan);
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902 List<Cell> results = new ArrayList<Cell>();
2903 int index = 0;
2904 while (true) {
2905 boolean more = s.next(results, 3);
2906 if ((index >> 1) < 5) {
2907 if (index % 2 == 0)
2908 assertEquals(results.size(), 3);
2909 else
2910 assertEquals(results.size(), 1);
2911 } else
2912 assertEquals(results.size(), 1);
2913 results.clear();
2914 index++;
2915 if (!more)
2916 break;
2917 }
2918 } finally {
2919 HRegion.closeHRegion(this.region);
2920 this.region = null;
2921 }
2922 }
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932 @Test
2933 public void testBasicSplit() throws Exception {
2934 byte[][] families = { fam1, fam2, fam3 };
2935
2936 Configuration hc = initSplit();
2937
2938 String method = this.getName();
2939 this.region = initHRegion(tableName, method, hc, families);
2940
2941 try {
2942 LOG.info("" + HBaseTestCase.addContent(region, fam3));
2943 region.flushcache();
2944 region.compactStores();
2945 byte[] splitRow = region.checkSplit();
2946 assertNotNull(splitRow);
2947 LOG.info("SplitRow: " + Bytes.toString(splitRow));
2948 HRegion[] regions = splitRegion(region, splitRow);
2949 try {
2950
2951
2952
2953 for (int i = 0; i < regions.length; i++) {
2954 regions[i] = HRegion.openHRegion(regions[i], null);
2955 }
2956
2957
2958 assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
2959 assertGet(regions[1], fam3, splitRow);
2960
2961 assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
2962 assertScan(regions[1], fam3, splitRow);
2963
2964 for (int i = 0; i < regions.length; i++) {
2965
2966
2967 for (int j = 0; j < 2; j++) {
2968 HBaseTestCase.addContent(regions[i], fam3);
2969 }
2970 HBaseTestCase.addContent(regions[i], fam2);
2971 HBaseTestCase.addContent(regions[i], fam1);
2972 regions[i].flushcache();
2973 }
2974
2975 byte[][] midkeys = new byte[regions.length][];
2976
2977 for (int i = 0; i < regions.length; i++) {
2978 regions[i].compactStores();
2979 midkeys[i] = regions[i].checkSplit();
2980 }
2981
2982 TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
2983
2984
2985 for (int i = 0; i < regions.length; i++) {
2986 HRegion[] rs = null;
2987 if (midkeys[i] != null) {
2988 rs = splitRegion(regions[i], midkeys[i]);
2989 for (int j = 0; j < rs.length; j++) {
2990 sortedMap.put(Bytes.toString(rs[j].getRegionName()), HRegion.openHRegion(rs[j], null));
2991 }
2992 }
2993 }
2994 LOG.info("Made 4 regions");
2995
2996
2997 int interval = (LAST_CHAR - FIRST_CHAR) / 3;
2998 byte[] b = Bytes.toBytes(START_KEY);
2999 for (HRegion r : sortedMap.values()) {
3000 assertGet(r, fam3, b);
3001 b[0] += interval;
3002 }
3003 } finally {
3004 for (int i = 0; i < regions.length; i++) {
3005 try {
3006 regions[i].close();
3007 } catch (IOException e) {
3008
3009 }
3010 }
3011 }
3012 } finally {
3013 HRegion.closeHRegion(this.region);
3014 this.region = null;
3015 }
3016 }
3017
3018 @Test
3019 public void testSplitRegion() throws IOException {
3020 byte[] qualifier = Bytes.toBytes("qualifier");
3021 Configuration hc = initSplit();
3022 int numRows = 10;
3023 byte[][] families = { fam1, fam3 };
3024
3025
3026 String method = this.getName();
3027 this.region = initHRegion(tableName, method, hc, families);
3028
3029
3030 int startRow = 100;
3031 putData(startRow, numRows, qualifier, families);
3032 int splitRow = startRow + numRows;
3033 putData(splitRow, numRows, qualifier, families);
3034 region.flushcache();
3035
3036 HRegion[] regions = null;
3037 try {
3038 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
3039
3040 for (int i = 0; i < regions.length; i++) {
3041 regions[i] = HRegion.openHRegion(regions[i], null);
3042 }
3043
3044 assertEquals(2, regions.length);
3045
3046
3047
3048 verifyData(regions[0], startRow, numRows, qualifier, families);
3049 verifyData(regions[1], splitRow, numRows, qualifier, families);
3050
3051 } finally {
3052 HRegion.closeHRegion(this.region);
3053 this.region = null;
3054 }
3055 }
3056
3057 @Test
3058 public void testClearForceSplit() throws IOException {
3059 byte[] qualifier = Bytes.toBytes("qualifier");
3060 Configuration hc = initSplit();
3061 int numRows = 10;
3062 byte[][] families = { fam1, fam3 };
3063
3064
3065 String method = this.getName();
3066 this.region = initHRegion(tableName, method, hc, families);
3067
3068
3069 int startRow = 100;
3070 putData(startRow, numRows, qualifier, families);
3071 int splitRow = startRow + numRows;
3072 byte[] splitRowBytes = Bytes.toBytes("" + splitRow);
3073 putData(splitRow, numRows, qualifier, families);
3074 region.flushcache();
3075
3076 HRegion[] regions = null;
3077 try {
3078
3079 region.forceSplit(splitRowBytes);
3080 assertTrue(region.shouldForceSplit());
3081
3082 assertTrue(Bytes.equals(splitRowBytes, region.checkSplit()));
3083
3084
3085 HStore storeMock = Mockito.mock(HStore.class);
3086 Mockito.when(storeMock.hasReferences()).thenReturn(true);
3087 Mockito.when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
3088 Mockito.when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
3089 Mockito.when(storeMock.getColumnFamilyName()).thenReturn("cf");
3090 region.stores.put(Bytes.toBytes(storeMock.getColumnFamilyName()), storeMock);
3091 assertTrue(region.hasReferences());
3092
3093
3094 regions = splitRegion(region, splitRowBytes);
3095 assertNull(regions);
3096
3097
3098 assertFalse(region.shouldForceSplit());
3099
3100
3101 region.stores.remove(Bytes.toBytes(storeMock.getColumnFamilyName()));
3102 assertFalse(region.hasReferences());
3103
3104
3105 regions = splitRegion(region, splitRowBytes);
3106
3107
3108 for (int i = 0; i < regions.length; i++) {
3109 regions[i] = HRegion.openHRegion(regions[i], null);
3110 }
3111
3112 assertEquals(2, regions.length);
3113
3114
3115
3116 verifyData(regions[0], startRow, numRows, qualifier, families);
3117 verifyData(regions[1], splitRow, numRows, qualifier, families);
3118
3119 } finally {
3120 HRegion.closeHRegion(this.region);
3121 this.region = null;
3122 }
3123 }
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135 @Test
3136 public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3137 byte[] family = Bytes.toBytes("family");
3138 int numRows = 1000;
3139 int flushAndScanInterval = 10;
3140 int compactInterval = 10 * flushAndScanInterval;
3141
3142 String method = "testFlushCacheWhileScanning";
3143 this.region = initHRegion(tableName, method, CONF, family);
3144 try {
3145 FlushThread flushThread = new FlushThread();
3146 flushThread.start();
3147
3148 Scan scan = new Scan();
3149 scan.addFamily(family);
3150 scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
3151 new BinaryComparator(Bytes.toBytes(5L))));
3152
3153 int expectedCount = 0;
3154 List<Cell> res = new ArrayList<Cell>();
3155
3156 boolean toggle = true;
3157 for (long i = 0; i < numRows; i++) {
3158 Put put = new Put(Bytes.toBytes(i));
3159 put.setDurability(Durability.SKIP_WAL);
3160 put.add(family, qual1, Bytes.toBytes(i % 10));
3161 region.put(put);
3162
3163 if (i != 0 && i % compactInterval == 0) {
3164
3165 region.compactStores(true);
3166 }
3167
3168 if (i % 10 == 5L) {
3169 expectedCount++;
3170 }
3171
3172 if (i != 0 && i % flushAndScanInterval == 0) {
3173 res.clear();
3174 InternalScanner scanner = region.getScanner(scan);
3175 if (toggle) {
3176 flushThread.flush();
3177 }
3178 while (scanner.next(res))
3179 ;
3180 if (!toggle) {
3181 flushThread.flush();
3182 }
3183 assertEquals("i=" + i, expectedCount, res.size());
3184 toggle = !toggle;
3185 }
3186 }
3187
3188 flushThread.done();
3189 flushThread.join();
3190 flushThread.checkNoError();
3191 } finally {
3192 HRegion.closeHRegion(this.region);
3193 this.region = null;
3194 }
3195 }
3196
3197 protected class FlushThread extends Thread {
3198 private volatile boolean done;
3199 private Throwable error = null;
3200
3201 public void done() {
3202 done = true;
3203 synchronized (this) {
3204 interrupt();
3205 }
3206 }
3207
3208 public void checkNoError() {
3209 if (error != null) {
3210 assertNull(error);
3211 }
3212 }
3213
3214 @Override
3215 public void run() {
3216 done = false;
3217 while (!done) {
3218 synchronized (this) {
3219 try {
3220 wait();
3221 } catch (InterruptedException ignored) {
3222 if (done) {
3223 break;
3224 }
3225 }
3226 }
3227 try {
3228 region.flushcache();
3229 } catch (IOException e) {
3230 if (!done) {
3231 LOG.error("Error while flusing cache", e);
3232 error = e;
3233 }
3234 break;
3235 }
3236 }
3237
3238 }
3239
3240 public void flush() {
3241 synchronized (this) {
3242 notify();
3243 }
3244
3245 }
3246 }
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257 @Test
3258 public void testWritesWhileScanning() throws IOException, InterruptedException {
3259 int testCount = 100;
3260 int numRows = 1;
3261 int numFamilies = 10;
3262 int numQualifiers = 100;
3263 int flushInterval = 7;
3264 int compactInterval = 5 * flushInterval;
3265 byte[][] families = new byte[numFamilies][];
3266 for (int i = 0; i < numFamilies; i++) {
3267 families[i] = Bytes.toBytes("family" + i);
3268 }
3269 byte[][] qualifiers = new byte[numQualifiers][];
3270 for (int i = 0; i < numQualifiers; i++) {
3271 qualifiers[i] = Bytes.toBytes("qual" + i);
3272 }
3273
3274 String method = "testWritesWhileScanning";
3275 this.region = initHRegion(tableName, method, CONF, families);
3276 try {
3277 PutThread putThread = new PutThread(numRows, families, qualifiers);
3278 putThread.start();
3279 putThread.waitForFirstPut();
3280
3281 FlushThread flushThread = new FlushThread();
3282 flushThread.start();
3283
3284 Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3285
3286 int expectedCount = numFamilies * numQualifiers;
3287 List<Cell> res = new ArrayList<Cell>();
3288
3289 long prevTimestamp = 0L;
3290 for (int i = 0; i < testCount; i++) {
3291
3292 if (i != 0 && i % compactInterval == 0) {
3293 region.compactStores(true);
3294 }
3295
3296 if (i != 0 && i % flushInterval == 0) {
3297 flushThread.flush();
3298 }
3299
3300 boolean previousEmpty = res.isEmpty();
3301 res.clear();
3302 InternalScanner scanner = region.getScanner(scan);
3303 while (scanner.next(res))
3304 ;
3305 if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3306 assertEquals("i=" + i, expectedCount, res.size());
3307 long timestamp = res.get(0).getTimestamp();
3308 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3309 timestamp >= prevTimestamp);
3310 prevTimestamp = timestamp;
3311 }
3312 }
3313
3314 putThread.done();
3315
3316 region.flushcache();
3317
3318 putThread.join();
3319 putThread.checkNoError();
3320
3321 flushThread.done();
3322 flushThread.join();
3323 flushThread.checkNoError();
3324 } finally {
3325 try {
3326 HRegion.closeHRegion(this.region);
3327 } catch (DroppedSnapshotException dse) {
3328
3329
3330
3331
3332 }
3333 this.region = null;
3334 }
3335 }
3336
3337 protected class PutThread extends Thread {
3338 private volatile boolean done;
3339 private volatile int numPutsFinished = 0;
3340
3341 private Throwable error = null;
3342 private int numRows;
3343 private byte[][] families;
3344 private byte[][] qualifiers;
3345
3346 private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3347 this.numRows = numRows;
3348 this.families = families;
3349 this.qualifiers = qualifiers;
3350 }
3351
3352
3353
3354
3355 public void waitForFirstPut() throws InterruptedException {
3356
3357 while (numPutsFinished == 0) {
3358 checkNoError();
3359 Thread.sleep(50);
3360 }
3361 }
3362
3363 public void done() {
3364 done = true;
3365 synchronized (this) {
3366 interrupt();
3367 }
3368 }
3369
3370 public void checkNoError() {
3371 if (error != null) {
3372 assertNull(error);
3373 }
3374 }
3375
3376 @Override
3377 public void run() {
3378 done = false;
3379 while (!done) {
3380 try {
3381 for (int r = 0; r < numRows; r++) {
3382 byte[] row = Bytes.toBytes("row" + r);
3383 Put put = new Put(row);
3384 put.setDurability(Durability.SKIP_WAL);
3385 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3386 for (byte[] family : families) {
3387 for (byte[] qualifier : qualifiers) {
3388 put.add(family, qualifier, (long) numPutsFinished, value);
3389 }
3390 }
3391 region.put(put);
3392 numPutsFinished++;
3393 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3394 System.out.println("put iteration = " + numPutsFinished);
3395 Delete delete = new Delete(row, (long) numPutsFinished - 30);
3396 region.delete(delete);
3397 }
3398 numPutsFinished++;
3399 }
3400 } catch (InterruptedIOException e) {
3401
3402 } catch (IOException e) {
3403 LOG.error("error while putting records", e);
3404 error = e;
3405 break;
3406 }
3407 }
3408
3409 }
3410
3411 }
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422 @Test
3423 public void testWritesWhileGetting() throws Exception {
3424 int testCount = 100;
3425 int numRows = 1;
3426 int numFamilies = 10;
3427 int numQualifiers = 100;
3428 int compactInterval = 100;
3429 byte[][] families = new byte[numFamilies][];
3430 for (int i = 0; i < numFamilies; i++) {
3431 families[i] = Bytes.toBytes("family" + i);
3432 }
3433 byte[][] qualifiers = new byte[numQualifiers][];
3434 for (int i = 0; i < numQualifiers; i++) {
3435 qualifiers[i] = Bytes.toBytes("qual" + i);
3436 }
3437
3438
3439 String method = "testWritesWhileGetting";
3440
3441
3442
3443
3444
3445 Configuration conf = HBaseConfiguration.create(CONF);
3446 conf.setInt("hbase.hstore.compaction.min", 1);
3447 conf.setInt("hbase.hstore.compaction.max", 1000);
3448 this.region = initHRegion(tableName, method, conf, families);
3449 PutThread putThread = null;
3450 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3451 try {
3452 putThread = new PutThread(numRows, families, qualifiers);
3453 putThread.start();
3454 putThread.waitForFirstPut();
3455
3456
3457 ctx.addThread(new RepeatingTestThread(ctx) {
3458 private int flushesSinceCompact = 0;
3459 private final int maxFlushesSinceCompact = 20;
3460
3461 @Override
3462 public void doAnAction() throws Exception {
3463 if (region.flushcache().isCompactionNeeded()) {
3464 ++flushesSinceCompact;
3465 }
3466
3467
3468 if (flushesSinceCompact == maxFlushesSinceCompact) {
3469 region.compactStores(false);
3470 flushesSinceCompact = 0;
3471 }
3472 }
3473 });
3474 ctx.startThreads();
3475
3476 Get get = new Get(Bytes.toBytes("row0"));
3477 Result result = null;
3478
3479 int expectedCount = numFamilies * numQualifiers;
3480
3481 long prevTimestamp = 0L;
3482 for (int i = 0; i < testCount; i++) {
3483
3484 boolean previousEmpty = result == null || result.isEmpty();
3485 result = region.get(get);
3486 if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3487 assertEquals("i=" + i, expectedCount, result.size());
3488
3489
3490 long timestamp = 0;
3491 for (Cell kv : result.rawCells()) {
3492 if (CellUtil.matchingFamily(kv, families[0])
3493 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3494 timestamp = kv.getTimestamp();
3495 }
3496 }
3497 assertTrue(timestamp >= prevTimestamp);
3498 prevTimestamp = timestamp;
3499 Cell previousKV = null;
3500
3501 for (Cell kv : result.rawCells()) {
3502 byte[] thisValue = CellUtil.cloneValue(kv);
3503 if (previousKV != null) {
3504 if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3505 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3506 + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3507 + "(memStoreTS:" + kv.getMvccVersion() + ")");
3508 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3509 }
3510 }
3511 previousKV = kv;
3512 }
3513 }
3514 }
3515 } finally {
3516 if (putThread != null)
3517 putThread.done();
3518
3519 region.flushcache();
3520
3521 if (putThread != null) {
3522 putThread.join();
3523 putThread.checkNoError();
3524 }
3525
3526 ctx.stop();
3527 HRegion.closeHRegion(this.region);
3528 this.region = null;
3529 }
3530 }
3531
3532 @Test
3533 public void testHolesInMeta() throws Exception {
3534 byte[] family = Bytes.toBytes("family");
3535 this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF,
3536 false, family);
3537 try {
3538 byte[] rowNotServed = Bytes.toBytes("a");
3539 Get g = new Get(rowNotServed);
3540 try {
3541 region.get(g);
3542 fail();
3543 } catch (WrongRegionException x) {
3544
3545 }
3546 byte[] row = Bytes.toBytes("y");
3547 g = new Get(row);
3548 region.get(g);
3549 } finally {
3550 HRegion.closeHRegion(this.region);
3551 this.region = null;
3552 }
3553 }
3554
3555 @Test
3556 public void testIndexesScanWithOneDeletedRow() throws IOException {
3557 byte[] family = Bytes.toBytes("family");
3558
3559
3560 String method = "testIndexesScanWithOneDeletedRow";
3561 this.region = initHRegion(tableName, method, CONF, family);
3562 try {
3563 Put put = new Put(Bytes.toBytes(1L));
3564 put.add(family, qual1, 1L, Bytes.toBytes(1L));
3565 region.put(put);
3566
3567 region.flushcache();
3568
3569 Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3570 region.delete(delete);
3571
3572 put = new Put(Bytes.toBytes(2L));
3573 put.add(family, qual1, 2L, Bytes.toBytes(2L));
3574 region.put(put);
3575
3576 Scan idxScan = new Scan();
3577 idxScan.addFamily(family);
3578 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
3579 new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
3580 new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
3581 CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
3582 InternalScanner scanner = region.getScanner(idxScan);
3583 List<Cell> res = new ArrayList<Cell>();
3584
3585 while (scanner.next(res))
3586 ;
3587 assertEquals(1L, res.size());
3588 } finally {
3589 HRegion.closeHRegion(this.region);
3590 this.region = null;
3591 }
3592 }
3593
3594
3595
3596
3597 @Test
3598 public void testBloomFilterSize() throws IOException {
3599 byte[] fam1 = Bytes.toBytes("fam1");
3600 byte[] qf1 = Bytes.toBytes("col");
3601 byte[] val1 = Bytes.toBytes("value1");
3602
3603 HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
3604 .setBloomFilterType(BloomType.ROWCOL);
3605
3606 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3607 htd.addFamily(hcd);
3608 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3609 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3610 try {
3611 int num_unique_rows = 10;
3612 int duplicate_multiplier = 2;
3613 int num_storefiles = 4;
3614
3615 int version = 0;
3616 for (int f = 0; f < num_storefiles; f++) {
3617 for (int i = 0; i < duplicate_multiplier; i++) {
3618 for (int j = 0; j < num_unique_rows; j++) {
3619 Put put = new Put(Bytes.toBytes("row" + j));
3620 put.setDurability(Durability.SKIP_WAL);
3621 put.add(fam1, qf1, version++, val1);
3622 region.put(put);
3623 }
3624 }
3625 region.flushcache();
3626 }
3627
3628 HStore store = (HStore) region.getStore(fam1);
3629 Collection<StoreFile> storeFiles = store.getStorefiles();
3630 for (StoreFile storefile : storeFiles) {
3631 StoreFile.Reader reader = storefile.getReader();
3632 reader.loadFileInfo();
3633 reader.loadBloomfilter();
3634 assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
3635 assertEquals(num_unique_rows, reader.getFilterEntries());
3636 }
3637
3638 region.compactStores(true);
3639
3640
3641 storeFiles = store.getStorefiles();
3642 for (StoreFile storefile : storeFiles) {
3643 StoreFile.Reader reader = storefile.getReader();
3644 reader.loadFileInfo();
3645 reader.loadBloomfilter();
3646 assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
3647 assertEquals(num_unique_rows, reader.getFilterEntries());
3648 }
3649 } finally {
3650 HRegion.closeHRegion(this.region);
3651 this.region = null;
3652 }
3653 }
3654
3655 @Test
3656 public void testAllColumnsWithBloomFilter() throws IOException {
3657 byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
3658 byte[] FAMILY = Bytes.toBytes("family");
3659
3660
3661 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
3662 .setBloomFilterType(BloomType.ROWCOL);
3663 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
3664 htd.addFamily(hcd);
3665 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3666 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3667 try {
3668
3669 byte row[] = Bytes.toBytes("row:" + 0);
3670 byte column[] = Bytes.toBytes("column:" + 0);
3671 Put put = new Put(row);
3672 put.setDurability(Durability.SKIP_WAL);
3673 for (long idx = 1; idx <= 4; idx++) {
3674 put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
3675 }
3676 region.put(put);
3677
3678
3679 region.flushcache();
3680
3681
3682 Get get = new Get(row);
3683 get.setMaxVersions();
3684 Cell[] kvs = region.get(get).rawCells();
3685
3686
3687 assertEquals(4, kvs.length);
3688 checkOneCell(kvs[0], FAMILY, 0, 0, 4);
3689 checkOneCell(kvs[1], FAMILY, 0, 0, 3);
3690 checkOneCell(kvs[2], FAMILY, 0, 0, 2);
3691 checkOneCell(kvs[3], FAMILY, 0, 0, 1);
3692 } finally {
3693 HRegion.closeHRegion(this.region);
3694 this.region = null;
3695 }
3696 }
3697
3698
3699
3700
3701
3702
3703 @Test
3704 public void testDeleteRowWithBloomFilter() throws IOException {
3705 byte[] familyName = Bytes.toBytes("familyName");
3706
3707
3708 HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
3709 .setBloomFilterType(BloomType.ROWCOL);
3710
3711 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3712 htd.addFamily(hcd);
3713 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3714 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3715 try {
3716
3717 byte row[] = Bytes.toBytes("row1");
3718 byte col[] = Bytes.toBytes("col1");
3719
3720 Put put = new Put(row);
3721 put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
3722 region.put(put);
3723 region.flushcache();
3724
3725 Delete del = new Delete(row);
3726 region.delete(del);
3727 region.flushcache();
3728
3729
3730 Get get = new Get(row);
3731 get.addColumn(familyName, col);
3732
3733 Cell[] keyValues = region.get(get).rawCells();
3734 assertTrue(keyValues.length == 0);
3735 } finally {
3736 HRegion.closeHRegion(this.region);
3737 this.region = null;
3738 }
3739 }
3740
3741 @Test
3742 public void testgetHDFSBlocksDistribution() throws Exception {
3743 HBaseTestingUtility htu = new HBaseTestingUtility();
3744 final int DEFAULT_BLOCK_SIZE = 1024;
3745 htu.getConfiguration().setLong("dfs.namenode.fs-limits.min-block-size", 0);
3746 htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
3747 htu.getConfiguration().setInt("dfs.replication", 2);
3748
3749
3750 MiniHBaseCluster cluster = null;
3751 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
3752 int regionServersCount = 3;
3753
3754 try {
3755 cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
3756 byte[][] families = { fam1, fam2 };
3757 HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);
3758
3759
3760 byte row[] = Bytes.toBytes("row1");
3761 byte col[] = Bytes.toBytes("col1");
3762
3763 Put put = new Put(row);
3764 put.add(fam1, col, 1, Bytes.toBytes("test1"));
3765 put.add(fam2, col, 1, Bytes.toBytes("test2"));
3766 ht.put(put);
3767
3768 HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
3769 .get(0);
3770 firstRegion.flushcache();
3771 HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
3772
3773
3774
3775
3776
3777 long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
3778
3779 String topHost = blocksDistribution1.getTopHosts().get(0);
3780 long topHostWeight = blocksDistribution1.getWeight(topHost);
3781 assertTrue(uniqueBlocksWeight1 == topHostWeight);
3782
3783
3784
3785 HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
3786 htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
3787 long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
3788
3789 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
3790
3791 ht.close();
3792 } finally {
3793 if (cluster != null) {
3794 htu.shutdownMiniCluster();
3795 }
3796 }
3797 }
3798
3799
3800
3801
3802
3803
3804
3805 @Test
3806 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
3807 TableName tableName = TableName.valueOf(name.getMethodName());
3808 HRegionInfo info = null;
3809 try {
3810 FileSystem fs = Mockito.mock(FileSystem.class);
3811 Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
3812 HTableDescriptor htd = new HTableDescriptor(tableName);
3813 htd.addFamily(new HColumnDescriptor("cf"));
3814 info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
3815 HConstants.EMPTY_BYTE_ARRAY, false);
3816 Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
3817 region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
3818
3819 region.initialize();
3820 fail("Region initialization should fail due to IOException");
3821 } catch (IOException io) {
3822 List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
3823 for (MonitoredTask monitoredTask : tasks) {
3824 if (!(monitoredTask instanceof MonitoredRPCHandler)
3825 && monitoredTask.getDescription().contains(region.toString())) {
3826 assertTrue("Region state should be ABORTED.",
3827 monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
3828 break;
3829 }
3830 }
3831 } finally {
3832 HRegion.closeHRegion(region);
3833 }
3834 }
3835
3836
3837
3838
3839
3840 @Test
3841 public void testRegionInfoFileCreation() throws IOException {
3842 Path rootDir = new Path(dir + "testRegionInfoFileCreation");
3843
3844 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
3845 htd.addFamily(new HColumnDescriptor("cf"));
3846
3847 HRegionInfo hri = new HRegionInfo(htd.getTableName());
3848
3849
3850 HRegion region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
3851
3852 Path regionDir = region.getRegionFileSystem().getRegionDir();
3853 FileSystem fs = region.getRegionFileSystem().getFileSystem();
3854 HRegion.closeHRegion(region);
3855
3856 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
3857
3858
3859 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3860 fs.exists(regionInfoFile));
3861
3862
3863 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3864 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3865 HRegion.closeHRegion(region);
3866
3867
3868 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3869 fs.exists(regionInfoFile));
3870
3871
3872 fs.delete(regionInfoFile);
3873 assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
3874 fs.exists(regionInfoFile));
3875
3876 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3877
3878 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3879 HRegion.closeHRegion(region);
3880
3881
3882 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3883 fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
3884 }
3885
3886
3887
3888
3889 private static class Incrementer implements Runnable {
3890 private HRegion region;
3891 private final static byte[] incRow = Bytes.toBytes("incRow");
3892 private final static byte[] family = Bytes.toBytes("family");
3893 private final static byte[] qualifier = Bytes.toBytes("qualifier");
3894 private final static long ONE = 1l;
3895 private int incCounter;
3896
3897 public Incrementer(HRegion region, int incCounter) {
3898 this.region = region;
3899 this.incCounter = incCounter;
3900 }
3901
3902 @Override
3903 public void run() {
3904 int count = 0;
3905 while (count < incCounter) {
3906 Increment inc = new Increment(incRow);
3907 inc.addColumn(family, qualifier, ONE);
3908 count++;
3909 try {
3910 region.increment(inc);
3911 } catch (IOException e) {
3912 e.printStackTrace();
3913 break;
3914 }
3915 }
3916 }
3917 }
3918
3919
3920
3921
3922
3923 @Test
3924 public void testParallelIncrementWithMemStoreFlush() throws Exception {
3925 byte[] family = Incrementer.family;
3926 this.region = initHRegion(tableName, method, CONF, family);
3927 final HRegion region = this.region;
3928 final AtomicBoolean incrementDone = new AtomicBoolean(false);
3929 Runnable flusher = new Runnable() {
3930 @Override
3931 public void run() {
3932 while (!incrementDone.get()) {
3933 try {
3934 region.flushcache();
3935 } catch (Exception e) {
3936 e.printStackTrace();
3937 }
3938 }
3939 }
3940 };
3941
3942
3943 int threadNum = 20;
3944 int incCounter = 100;
3945 long expected = threadNum * incCounter;
3946 Thread[] incrementers = new Thread[threadNum];
3947 Thread flushThread = new Thread(flusher);
3948 for (int i = 0; i < threadNum; i++) {
3949 incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
3950 incrementers[i].start();
3951 }
3952 flushThread.start();
3953 for (int i = 0; i < threadNum; i++) {
3954 incrementers[i].join();
3955 }
3956
3957 incrementDone.set(true);
3958 flushThread.join();
3959
3960 Get get = new Get(Incrementer.incRow);
3961 get.addColumn(Incrementer.family, Incrementer.qualifier);
3962 get.setMaxVersions(1);
3963 Result res = this.region.get(get);
3964 List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
3965
3966
3967 assertEquals(kvs.size(), 1);
3968 Cell kv = kvs.get(0);
3969 assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
3970 this.region = null;
3971 }
3972
3973
3974
3975
3976 private static class Appender implements Runnable {
3977 private HRegion region;
3978 private final static byte[] appendRow = Bytes.toBytes("appendRow");
3979 private final static byte[] family = Bytes.toBytes("family");
3980 private final static byte[] qualifier = Bytes.toBytes("qualifier");
3981 private final static byte[] CHAR = Bytes.toBytes("a");
3982 private int appendCounter;
3983
3984 public Appender(HRegion region, int appendCounter) {
3985 this.region = region;
3986 this.appendCounter = appendCounter;
3987 }
3988
3989 @Override
3990 public void run() {
3991 int count = 0;
3992 while (count < appendCounter) {
3993 Append app = new Append(appendRow);
3994 app.add(family, qualifier, CHAR);
3995 count++;
3996 try {
3997 region.append(app);
3998 } catch (IOException e) {
3999 e.printStackTrace();
4000 break;
4001 }
4002 }
4003 }
4004 }
4005
4006
4007
4008
4009
4010 @Test
4011 public void testParallelAppendWithMemStoreFlush() throws Exception {
4012 byte[] family = Appender.family;
4013 this.region = initHRegion(tableName, method, CONF, family);
4014 final HRegion region = this.region;
4015 final AtomicBoolean appendDone = new AtomicBoolean(false);
4016 Runnable flusher = new Runnable() {
4017 @Override
4018 public void run() {
4019 while (!appendDone.get()) {
4020 try {
4021 region.flushcache();
4022 } catch (Exception e) {
4023 e.printStackTrace();
4024 }
4025 }
4026 }
4027 };
4028
4029
4030
4031 int threadNum = 20;
4032 int appendCounter = 100;
4033 byte[] expected = new byte[threadNum * appendCounter];
4034 for (int i = 0; i < threadNum * appendCounter; i++) {
4035 System.arraycopy(Appender.CHAR, 0, expected, i, 1);
4036 }
4037 Thread[] appenders = new Thread[threadNum];
4038 Thread flushThread = new Thread(flusher);
4039 for (int i = 0; i < threadNum; i++) {
4040 appenders[i] = new Thread(new Appender(this.region, appendCounter));
4041 appenders[i].start();
4042 }
4043 flushThread.start();
4044 for (int i = 0; i < threadNum; i++) {
4045 appenders[i].join();
4046 }
4047
4048 appendDone.set(true);
4049 flushThread.join();
4050
4051 Get get = new Get(Appender.appendRow);
4052 get.addColumn(Appender.family, Appender.qualifier);
4053 get.setMaxVersions(1);
4054 Result res = this.region.get(get);
4055 List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
4056
4057
4058 assertEquals(kvs.size(), 1);
4059 Cell kv = kvs.get(0);
4060 byte[] appendResult = new byte[kv.getValueLength()];
4061 System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
4062 assertArrayEquals(expected, appendResult);
4063 this.region = null;
4064 }
4065
4066
4067
4068
4069
4070 @Test
4071 public void testPutWithMemStoreFlush() throws Exception {
4072 byte[] family = Bytes.toBytes("family");
4073 ;
4074 byte[] qualifier = Bytes.toBytes("qualifier");
4075 byte[] row = Bytes.toBytes("putRow");
4076 byte[] value = null;
4077 this.region = initHRegion(tableName, method, CONF, family);
4078 Put put = null;
4079 Get get = null;
4080 List<Cell> kvs = null;
4081 Result res = null;
4082
4083 put = new Put(row);
4084 value = Bytes.toBytes("value0");
4085 put.add(family, qualifier, 1234567l, value);
4086 region.put(put);
4087 get = new Get(row);
4088 get.addColumn(family, qualifier);
4089 get.setMaxVersions();
4090 res = this.region.get(get);
4091 kvs = res.getColumnCells(family, qualifier);
4092 assertEquals(1, kvs.size());
4093 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4094
4095 region.flushcache();
4096 get = new Get(row);
4097 get.addColumn(family, qualifier);
4098 get.setMaxVersions();
4099 res = this.region.get(get);
4100 kvs = res.getColumnCells(family, qualifier);
4101 assertEquals(1, kvs.size());
4102 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4103
4104 put = new Put(row);
4105 value = Bytes.toBytes("value1");
4106 put.add(family, qualifier, 1234567l, value);
4107 region.put(put);
4108 get = new Get(row);
4109 get.addColumn(family, qualifier);
4110 get.setMaxVersions();
4111 res = this.region.get(get);
4112 kvs = res.getColumnCells(family, qualifier);
4113 assertEquals(1, kvs.size());
4114 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4115
4116 region.flushcache();
4117 get = new Get(row);
4118 get.addColumn(family, qualifier);
4119 get.setMaxVersions();
4120 res = this.region.get(get);
4121 kvs = res.getColumnCells(family, qualifier);
4122 assertEquals(1, kvs.size());
4123 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4124 }
4125
4126 @Test
4127 public void testDurability() throws Exception {
4128 String method = "testDurability";
4129
4130
4131
4132
4133
4134 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4135 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4136 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4137
4138 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4139 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4140 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4141
4142 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4143 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4144
4145 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
4146 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4147
4148 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
4149 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
4150 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
4151
4152
4153 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4154 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4155 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4156 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4157 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
4158 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4159
4160 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4161 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4162 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4163 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4164 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
4165 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
4166
4167
4168 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4169 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4170 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4171 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
4172 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
4173 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
4174
4175 }
4176
4177 private void durabilityTest(String method, Durability tableDurability,
4178 Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
4179 final boolean expectSyncFromLogSyncer) throws Exception {
4180 Configuration conf = HBaseConfiguration.create(CONF);
4181 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
4182 TableName tableName = TableName.valueOf(method);
4183 byte[] family = Bytes.toBytes("family");
4184 Path logDir = new Path(new Path(dir + method), "log");
4185 HLog hlog = HLogFactory.createHLog(FILESYSTEM, logDir, UUID.randomUUID().toString(), conf);
4186 final HLog log = spy(hlog);
4187 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
4188 HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, log,
4189 new byte[][] { family });
4190
4191 Put put = new Put(Bytes.toBytes("r1"));
4192 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
4193 put.setDurability(mutationDurability);
4194 region.put(put);
4195
4196
4197 verify(log, expectAppend ? times(1) : never())
4198 .appendNoSync((HRegionInfo)any(), eq(tableName), (WALEdit)any(), (List<UUID>)any(),
4199 anyLong(), (HTableDescriptor)any(), (AtomicLong)any(), anyBoolean(), anyLong(), anyLong());
4200
4201
4202 if (expectSync || expectSyncFromLogSyncer) {
4203 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
4204 @Override
4205 public boolean evaluate() throws Exception {
4206 try {
4207 if (expectSync) {
4208 verify(log, times(1)).sync(anyLong());
4209 } else if (expectSyncFromLogSyncer) {
4210 verify(log, times(1)).sync();
4211 }
4212 } catch (Throwable ignore) {
4213 }
4214 return true;
4215 }
4216 });
4217 } else {
4218 verify(log, never()).sync(anyLong());
4219 verify(log, never()).sync();
4220 }
4221
4222 HRegion.closeHRegion(this.region);
4223 this.region = null;
4224 }
4225
4226 private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4227 for (int i = startRow; i < startRow + numRows; i++) {
4228 Put put = new Put(Bytes.toBytes("" + i));
4229 put.setDurability(Durability.SKIP_WAL);
4230 for (byte[] family : families) {
4231 put.add(family, qf, null);
4232 }
4233 region.put(put);
4234 }
4235 }
4236
4237 private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
4238 throws IOException {
4239 for (int i = startRow; i < startRow + numRows; i++) {
4240 byte[] row = Bytes.toBytes("" + i);
4241 Get get = new Get(row);
4242 for (byte[] family : families) {
4243 get.addColumn(family, qf);
4244 }
4245 Result result = newReg.get(get);
4246 Cell[] raw = result.rawCells();
4247 assertEquals(families.length, result.size());
4248 for (int j = 0; j < families.length; j++) {
4249 assertTrue(CellUtil.matchingRow(raw[j], row));
4250 assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
4251 assertTrue(CellUtil.matchingQualifier(raw[j], qf));
4252 }
4253 }
4254 }
4255
4256 private void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
4257
4258 Get get = new Get(k).addFamily(family).setMaxVersions();
4259 Cell[] results = r.get(get).rawCells();
4260 for (int j = 0; j < results.length; j++) {
4261 byte[] tmp = CellUtil.cloneValue(results[j]);
4262
4263 assertTrue(Bytes.equals(k, tmp));
4264 }
4265 }
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278 private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
4279 throws IOException {
4280 byte[][] families = { fs };
4281 Scan scan = new Scan();
4282 for (int i = 0; i < families.length; i++)
4283 scan.addFamily(families[i]);
4284 InternalScanner s = r.getScanner(scan);
4285 try {
4286 List<Cell> curVals = new ArrayList<Cell>();
4287 boolean first = true;
4288 OUTER_LOOP: while (s.next(curVals)) {
4289 for (Cell kv : curVals) {
4290 byte[] val = CellUtil.cloneValue(kv);
4291 byte[] curval = val;
4292 if (first) {
4293 first = false;
4294 assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4295 } else {
4296
4297 break OUTER_LOOP;
4298 }
4299 }
4300 }
4301 } finally {
4302 s.close();
4303 }
4304 }
4305
4306
4307
4308
4309
4310 @Test
4311 public void testFlushResult() throws IOException {
4312 String method = name.getMethodName();
4313 byte[] tableName = Bytes.toBytes(method);
4314 byte[] family = Bytes.toBytes("family");
4315
4316 this.region = initHRegion(tableName, method, family);
4317
4318
4319 HRegion.FlushResult fr = region.flushcache();
4320 assertFalse(fr.isFlushSucceeded());
4321 assertFalse(fr.isCompactionNeeded());
4322
4323
4324 for (int i = 0; i < 2; i++) {
4325 Put put = new Put(tableName).add(family, family, tableName);
4326 region.put(put);
4327 fr = region.flushcache();
4328 assertTrue(fr.isFlushSucceeded());
4329 assertFalse(fr.isCompactionNeeded());
4330 }
4331
4332
4333 for (int i = 0; i < 2; i++) {
4334 Put put = new Put(tableName).add(family, family, tableName);
4335 region.put(put);
4336 fr = region.flushcache();
4337 assertTrue(fr.isFlushSucceeded());
4338 assertTrue(fr.isCompactionNeeded());
4339 }
4340 }
4341
4342 private Configuration initSplit() {
4343
4344 CONF.setInt("hbase.hstore.compactionThreshold", 2);
4345
4346
4347 CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4348
4349 CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4350
4351
4352 CONF.setLong("hbase.client.pause", 15 * 1000);
4353
4354
4355
4356 CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4357 return CONF;
4358 }
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369 public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
4370 byte[]... families) throws IOException {
4371 return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
4372 }
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4384 byte[]... families) throws IOException {
4385 return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4386 }
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4399 boolean isReadOnly, byte[]... families) throws IOException {
4400 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4401 }
4402
4403 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4404 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4405 throws IOException {
4406 return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
4407 Durability.SYNC_WAL, null, families);
4408 }
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4423 String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4424 HLog hlog, byte[]... families) throws IOException {
4425 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly, durability, hlog, families);
4426 }
4427
4428
4429
4430
4431
4432 private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
4433 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4434 assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
4435 Bytes.toString(CellUtil.cloneRow(kv)));
4436 assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
4437 Bytes.toString(CellUtil.cloneFamily(kv)));
4438 assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
4439 Bytes.toString(CellUtil.cloneQualifier(kv)));
4440 assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
4441 assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
4442 Bytes.toString(CellUtil.cloneValue(kv)));
4443 }
4444
4445 public void testReverseScanner_FromMemStore_SingleCF_Normal()
4446 throws IOException {
4447 byte[] rowC = Bytes.toBytes("rowC");
4448 byte[] rowA = Bytes.toBytes("rowA");
4449 byte[] rowB = Bytes.toBytes("rowB");
4450 byte[] cf = Bytes.toBytes("CF");
4451 byte[][] families = { cf };
4452 byte[] col = Bytes.toBytes("C");
4453 long ts = 1;
4454 String method = this.getName();
4455 this.region = initHRegion(tableName, method, families);
4456 try {
4457 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4458 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4459 null);
4460 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4461 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4462 Put put = null;
4463 put = new Put(rowC);
4464 put.add(kv1);
4465 put.add(kv11);
4466 region.put(put);
4467 put = new Put(rowA);
4468 put.add(kv2);
4469 region.put(put);
4470 put = new Put(rowB);
4471 put.add(kv3);
4472 region.put(put);
4473
4474 Scan scan = new Scan(rowC);
4475 scan.setMaxVersions(5);
4476 scan.setReversed(true);
4477 InternalScanner scanner = region.getScanner(scan);
4478 List<Cell> currRow = new ArrayList<Cell>();
4479 boolean hasNext = scanner.next(currRow);
4480 assertEquals(2, currRow.size());
4481 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4482 assertTrue(hasNext);
4483 currRow.clear();
4484 hasNext = scanner.next(currRow);
4485 assertEquals(1, currRow.size());
4486 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4487 assertTrue(hasNext);
4488 currRow.clear();
4489 hasNext = scanner.next(currRow);
4490 assertEquals(1, currRow.size());
4491 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4492 assertFalse(hasNext);
4493 scanner.close();
4494 } finally {
4495 HRegion.closeHRegion(this.region);
4496 this.region = null;
4497 }
4498 }
4499
4500 public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
4501 throws IOException {
4502 byte[] rowC = Bytes.toBytes("rowC");
4503 byte[] rowA = Bytes.toBytes("rowA");
4504 byte[] rowB = Bytes.toBytes("rowB");
4505 byte[] rowD = Bytes.toBytes("rowD");
4506 byte[] cf = Bytes.toBytes("CF");
4507 byte[][] families = { cf };
4508 byte[] col = Bytes.toBytes("C");
4509 long ts = 1;
4510 String method = this.getName();
4511 this.region = initHRegion(tableName, method, families);
4512 try {
4513 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4514 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4515 null);
4516 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4517 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4518 Put put = null;
4519 put = new Put(rowC);
4520 put.add(kv1);
4521 put.add(kv11);
4522 region.put(put);
4523 put = new Put(rowA);
4524 put.add(kv2);
4525 region.put(put);
4526 put = new Put(rowB);
4527 put.add(kv3);
4528 region.put(put);
4529
4530 Scan scan = new Scan(rowD);
4531 List<Cell> currRow = new ArrayList<Cell>();
4532 scan.setReversed(true);
4533 scan.setMaxVersions(5);
4534 InternalScanner scanner = region.getScanner(scan);
4535 boolean hasNext = scanner.next(currRow);
4536 assertEquals(2, currRow.size());
4537 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4538 assertTrue(hasNext);
4539 currRow.clear();
4540 hasNext = scanner.next(currRow);
4541 assertEquals(1, currRow.size());
4542 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4543 assertTrue(hasNext);
4544 currRow.clear();
4545 hasNext = scanner.next(currRow);
4546 assertEquals(1, currRow.size());
4547 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4548 assertFalse(hasNext);
4549 scanner.close();
4550 } finally {
4551 HRegion.closeHRegion(this.region);
4552 this.region = null;
4553 }
4554 }
4555
4556 public void testReverseScanner_FromMemStore_SingleCF_FullScan()
4557 throws IOException {
4558 byte[] rowC = Bytes.toBytes("rowC");
4559 byte[] rowA = Bytes.toBytes("rowA");
4560 byte[] rowB = Bytes.toBytes("rowB");
4561 byte[] cf = Bytes.toBytes("CF");
4562 byte[][] families = { cf };
4563 byte[] col = Bytes.toBytes("C");
4564 long ts = 1;
4565 String method = this.getName();
4566 this.region = initHRegion(tableName, method, families);
4567 try {
4568 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4569 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4570 null);
4571 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4572 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4573 Put put = null;
4574 put = new Put(rowC);
4575 put.add(kv1);
4576 put.add(kv11);
4577 region.put(put);
4578 put = new Put(rowA);
4579 put.add(kv2);
4580 region.put(put);
4581 put = new Put(rowB);
4582 put.add(kv3);
4583 region.put(put);
4584 Scan scan = new Scan();
4585 List<Cell> currRow = new ArrayList<Cell>();
4586 scan.setReversed(true);
4587 InternalScanner scanner = region.getScanner(scan);
4588 boolean hasNext = scanner.next(currRow);
4589 assertEquals(1, currRow.size());
4590 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4591 assertTrue(hasNext);
4592 currRow.clear();
4593 hasNext = scanner.next(currRow);
4594 assertEquals(1, currRow.size());
4595 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4596 assertTrue(hasNext);
4597 currRow.clear();
4598 hasNext = scanner.next(currRow);
4599 assertEquals(1, currRow.size());
4600 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4601 assertFalse(hasNext);
4602 scanner.close();
4603 } finally {
4604 HRegion.closeHRegion(this.region);
4605 this.region = null;
4606 }
4607 }
4608
4609 public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
4610
4611 byte[] rowA = Bytes.toBytes("rowA");
4612 byte[] rowB = Bytes.toBytes("rowB");
4613 byte[] rowC = Bytes.toBytes("rowC");
4614 byte[] rowD = Bytes.toBytes("rowD");
4615 byte[] rowE = Bytes.toBytes("rowE");
4616 byte[] cf = Bytes.toBytes("CF");
4617 byte[][] families = { cf };
4618 byte[] col1 = Bytes.toBytes("col1");
4619 byte[] col2 = Bytes.toBytes("col2");
4620 long ts = 1;
4621 String method = this.getName();
4622 this.region = initHRegion(tableName, method, families);
4623 try {
4624 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4625 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4626 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4627 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4628 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4629 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4630 Put put = null;
4631 put = new Put(rowA);
4632 put.add(kv1);
4633 region.put(put);
4634 put = new Put(rowB);
4635 put.add(kv2);
4636 region.put(put);
4637 put = new Put(rowC);
4638 put.add(kv3);
4639 region.put(put);
4640 put = new Put(rowD);
4641 put.add(kv4_1);
4642 region.put(put);
4643 put = new Put(rowD);
4644 put.add(kv4_2);
4645 region.put(put);
4646 put = new Put(rowE);
4647 put.add(kv5);
4648 region.put(put);
4649 region.flushcache();
4650 Scan scan = new Scan(rowD, rowA);
4651 scan.addColumn(families[0], col1);
4652 scan.setReversed(true);
4653 List<Cell> currRow = new ArrayList<Cell>();
4654 InternalScanner scanner = region.getScanner(scan);
4655 boolean hasNext = scanner.next(currRow);
4656 assertEquals(1, currRow.size());
4657 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4658 assertTrue(hasNext);
4659 currRow.clear();
4660 hasNext = scanner.next(currRow);
4661 assertEquals(1, currRow.size());
4662 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4663 assertTrue(hasNext);
4664 currRow.clear();
4665 hasNext = scanner.next(currRow);
4666 assertEquals(1, currRow.size());
4667 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4668 assertFalse(hasNext);
4669 scanner.close();
4670
4671 scan = new Scan(rowD, rowA);
4672 scan.addColumn(families[0], col2);
4673 scan.setReversed(true);
4674 currRow.clear();
4675 scanner = region.getScanner(scan);
4676 hasNext = scanner.next(currRow);
4677 assertEquals(1, currRow.size());
4678 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4679 scanner.close();
4680 } finally {
4681 HRegion.closeHRegion(this.region);
4682 this.region = null;
4683 }
4684 }
4685
4686 public void testReverseScanner_smaller_blocksize() throws IOException {
4687
4688 byte[] rowA = Bytes.toBytes("rowA");
4689 byte[] rowB = Bytes.toBytes("rowB");
4690 byte[] rowC = Bytes.toBytes("rowC");
4691 byte[] rowD = Bytes.toBytes("rowD");
4692 byte[] rowE = Bytes.toBytes("rowE");
4693 byte[] cf = Bytes.toBytes("CF");
4694 byte[][] families = { cf };
4695 byte[] col1 = Bytes.toBytes("col1");
4696 byte[] col2 = Bytes.toBytes("col2");
4697 long ts = 1;
4698 String method = this.getName();
4699 HBaseConfiguration config = new HBaseConfiguration();
4700 config.setInt("test.block.size", 1);
4701 this.region = initHRegion(tableName, method, config, families);
4702 try {
4703 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4704 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4705 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4706 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4707 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4708 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4709 Put put = null;
4710 put = new Put(rowA);
4711 put.add(kv1);
4712 region.put(put);
4713 put = new Put(rowB);
4714 put.add(kv2);
4715 region.put(put);
4716 put = new Put(rowC);
4717 put.add(kv3);
4718 region.put(put);
4719 put = new Put(rowD);
4720 put.add(kv4_1);
4721 region.put(put);
4722 put = new Put(rowD);
4723 put.add(kv4_2);
4724 region.put(put);
4725 put = new Put(rowE);
4726 put.add(kv5);
4727 region.put(put);
4728 region.flushcache();
4729 Scan scan = new Scan(rowD, rowA);
4730 scan.addColumn(families[0], col1);
4731 scan.setReversed(true);
4732 List<Cell> currRow = new ArrayList<Cell>();
4733 InternalScanner scanner = region.getScanner(scan);
4734 boolean hasNext = scanner.next(currRow);
4735 assertEquals(1, currRow.size());
4736 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4737 assertTrue(hasNext);
4738 currRow.clear();
4739 hasNext = scanner.next(currRow);
4740 assertEquals(1, currRow.size());
4741 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4742 assertTrue(hasNext);
4743 currRow.clear();
4744 hasNext = scanner.next(currRow);
4745 assertEquals(1, currRow.size());
4746 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4747 assertFalse(hasNext);
4748 scanner.close();
4749
4750 scan = new Scan(rowD, rowA);
4751 scan.addColumn(families[0], col2);
4752 scan.setReversed(true);
4753 currRow.clear();
4754 scanner = region.getScanner(scan);
4755 hasNext = scanner.next(currRow);
4756 assertEquals(1, currRow.size());
4757 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4758 scanner.close();
4759 } finally {
4760 HRegion.closeHRegion(this.region);
4761 this.region = null;
4762 }
4763 }
4764
4765 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
4766 throws IOException {
4767 byte[] row0 = Bytes.toBytes("row0");
4768 byte[] row1 = Bytes.toBytes("row1");
4769 byte[] row2 = Bytes.toBytes("row2");
4770 byte[] row3 = Bytes.toBytes("row3");
4771 byte[] row4 = Bytes.toBytes("row4");
4772 byte[] row5 = Bytes.toBytes("row5");
4773 byte[] cf1 = Bytes.toBytes("CF1");
4774 byte[] cf2 = Bytes.toBytes("CF2");
4775 byte[] cf3 = Bytes.toBytes("CF3");
4776 byte[][] families = { cf1, cf2, cf3 };
4777 byte[] col = Bytes.toBytes("C");
4778 long ts = 1;
4779 String method = this.getName();
4780 HBaseConfiguration conf = new HBaseConfiguration();
4781
4782 conf.setInt("hbase.hstore.compactionThreshold", 10000);
4783 this.region = initHRegion(tableName, method, conf, families);
4784 try {
4785
4786 KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
4787 null);
4788 KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
4789 null);
4790 KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
4791 KeyValue.Type.Put, null);
4792 KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
4793 null);
4794 KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
4795 null);
4796 KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
4797 null);
4798 KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
4799 KeyValue.Type.Put, null);
4800 KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
4801 null);
4802 KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
4803 KeyValue.Type.Put, null);
4804 KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
4805 null);
4806 KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
4807 null);
4808 KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
4809 KeyValue.Type.Put, null);
4810 KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
4811 null);
4812 KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
4813 KeyValue.Type.Put, null);
4814 KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
4815 null);
4816 KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
4817 null);
4818
4819 Put put = null;
4820 put = new Put(row1);
4821 put.add(kv1_2_1);
4822 region.put(put);
4823 put = new Put(row2);
4824 put.add(kv2_4_1);
4825 region.put(put);
4826 put = new Put(row4);
4827 put.add(kv4_5_4);
4828 put.add(kv4_5_5);
4829 region.put(put);
4830 region.flushcache();
4831
4832 put = new Put(row4);
4833 put.add(kv4_5_1);
4834 put.add(kv4_5_3);
4835 region.put(put);
4836 put = new Put(row1);
4837 put.add(kv1_2_2);
4838 region.put(put);
4839 put = new Put(row2);
4840 put.add(kv2_4_4);
4841 region.put(put);
4842 region.flushcache();
4843
4844 put = new Put(row4);
4845 put.add(kv4_5_2);
4846 region.put(put);
4847 put = new Put(row2);
4848 put.add(kv2_4_2);
4849 put.add(kv2_4_3);
4850 region.put(put);
4851 put = new Put(row3);
4852 put.add(kv3_2_2);
4853 region.put(put);
4854 region.flushcache();
4855
4856
4857 put = new Put(row0);
4858 put.add(kv0_1_1);
4859 region.put(put);
4860 put = new Put(row3);
4861 put.add(kv3_2_1);
4862 region.put(put);
4863 put = new Put(row5);
4864 put.add(kv5_2_1);
4865 put.add(kv5_2_2);
4866 region.put(put);
4867
4868 Scan scan = new Scan(row4);
4869 scan.setMaxVersions(5);
4870 scan.setBatch(3);
4871 scan.setReversed(true);
4872 InternalScanner scanner = region.getScanner(scan);
4873 List<Cell> currRow = new ArrayList<Cell>();
4874 boolean hasNext = false;
4875
4876
4877
4878 hasNext = scanner.next(currRow);
4879 assertEquals(3, currRow.size());
4880 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4881 assertTrue(hasNext);
4882 currRow.clear();
4883 hasNext = scanner.next(currRow);
4884 assertEquals(2, currRow.size());
4885 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4886 assertTrue(hasNext);
4887
4888 currRow.clear();
4889 hasNext = scanner.next(currRow);
4890 assertEquals(2, currRow.size());
4891 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
4892 assertTrue(hasNext);
4893
4894
4895 currRow.clear();
4896 hasNext = scanner.next(currRow);
4897 assertEquals(3, currRow.size());
4898 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4899 assertTrue(hasNext);
4900 currRow.clear();
4901 hasNext = scanner.next(currRow);
4902 assertEquals(1, currRow.size());
4903 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4904 assertTrue(hasNext);
4905
4906 currRow.clear();
4907 hasNext = scanner.next(currRow);
4908 assertEquals(2, currRow.size());
4909 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
4910 assertTrue(hasNext);
4911
4912 currRow.clear();
4913 hasNext = scanner.next(currRow);
4914 assertEquals(1, currRow.size());
4915 assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
4916 assertFalse(hasNext);
4917
4918 scanner.close();
4919 } finally {
4920 HRegion.closeHRegion(this.region);
4921 this.region = null;
4922 }
4923 }
4924
4925 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
4926 throws IOException {
4927 byte[] row1 = Bytes.toBytes("row1");
4928 byte[] row2 = Bytes.toBytes("row2");
4929 byte[] row3 = Bytes.toBytes("row3");
4930 byte[] row4 = Bytes.toBytes("row4");
4931 byte[] cf1 = Bytes.toBytes("CF1");
4932 byte[] cf2 = Bytes.toBytes("CF2");
4933 byte[] cf3 = Bytes.toBytes("CF3");
4934 byte[] cf4 = Bytes.toBytes("CF4");
4935 byte[][] families = { cf1, cf2, cf3, cf4 };
4936 byte[] col = Bytes.toBytes("C");
4937 long ts = 1;
4938 String method = this.getName();
4939 HBaseConfiguration conf = new HBaseConfiguration();
4940
4941 conf.setInt("hbase.hstore.compactionThreshold", 10000);
4942 this.region = initHRegion(tableName, method, conf, families);
4943 try {
4944 KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
4945 KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
4946 KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
4947 KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
4948
4949 Put put = new Put(row1);
4950 put.add(kv1);
4951 region.put(put);
4952 region.flushcache();
4953
4954 put = new Put(row2);
4955 put.add(kv2);
4956 region.put(put);
4957 region.flushcache();
4958
4959 put = new Put(row3);
4960 put.add(kv3);
4961 region.put(put);
4962 region.flushcache();
4963
4964 put = new Put(row4);
4965 put.add(kv4);
4966 region.put(put);
4967
4968 Scan scan = new Scan(row4);
4969 scan.setReversed(true);
4970 scan.setBatch(10);
4971 InternalScanner scanner = region.getScanner(scan);
4972 List<Cell> currRow = new ArrayList<Cell>();
4973 boolean hasNext = scanner.next(currRow);
4974 assertEquals(1, currRow.size());
4975 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4976 assertTrue(hasNext);
4977 currRow.clear();
4978 hasNext = scanner.next(currRow);
4979 assertEquals(1, currRow.size());
4980 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
4981 assertTrue(hasNext);
4982 currRow.clear();
4983 hasNext = scanner.next(currRow);
4984 assertEquals(1, currRow.size());
4985 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4986 assertTrue(hasNext);
4987 currRow.clear();
4988 hasNext = scanner.next(currRow);
4989 assertEquals(1, currRow.size());
4990 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
4991 assertFalse(hasNext);
4992 } finally {
4993 HRegion.closeHRegion(this.region);
4994 this.region = null;
4995 }
4996 }
4997
4998 @Test
4999 public void testWriteRequestsCounter() throws IOException {
5000 byte[] fam = Bytes.toBytes("info");
5001 byte[][] families = { fam };
5002 this.region = initHRegion(tableName, method, CONF, families);
5003
5004 Assert.assertEquals(0L, region.getWriteRequestsCount());
5005
5006 Put put = new Put(row);
5007 put.add(fam, fam, fam);
5008
5009 Assert.assertEquals(0L, region.getWriteRequestsCount());
5010 region.put(put);
5011 Assert.assertEquals(1L, region.getWriteRequestsCount());
5012 region.put(put);
5013 Assert.assertEquals(2L, region.getWriteRequestsCount());
5014 region.put(put);
5015 Assert.assertEquals(3L, region.getWriteRequestsCount());
5016
5017 region.delete(new Delete(row));
5018 Assert.assertEquals(4L, region.getWriteRequestsCount());
5019
5020 HRegion.closeHRegion(this.region);
5021 this.region = null;
5022 }
5023
5024
5025
5026
5027 @Test (timeout=24000)
5028 public void testRegionTooBusy() throws IOException {
5029 String method = "testRegionTooBusy";
5030 byte[] tableName = Bytes.toBytes(method);
5031 byte[] family = Bytes.toBytes("family");
5032 long defaultBusyWaitDuration = CONF.getLong("hbase.busy.wait.duration",
5033 HRegion.DEFAULT_BUSY_WAIT_DURATION);
5034 CONF.setLong("hbase.busy.wait.duration", 1000);
5035 region = initHRegion(tableName, method, CONF, family);
5036 final AtomicBoolean stopped = new AtomicBoolean(true);
5037 Thread t = new Thread(new Runnable() {
5038 @Override
5039 public void run() {
5040 try {
5041 region.lock.writeLock().lock();
5042 stopped.set(false);
5043 while (!stopped.get()) {
5044 Thread.sleep(100);
5045 }
5046 } catch (InterruptedException ie) {
5047 } finally {
5048 region.lock.writeLock().unlock();
5049 }
5050 }
5051 });
5052 t.start();
5053 Get get = new Get(row);
5054 try {
5055 while (stopped.get()) {
5056 Thread.sleep(100);
5057 }
5058 region.get(get);
5059 fail("Should throw RegionTooBusyException");
5060 } catch (InterruptedException ie) {
5061 fail("test interrupted");
5062 } catch (RegionTooBusyException e) {
5063
5064 } finally {
5065 stopped.set(true);
5066 try {
5067 t.join();
5068 } catch (Throwable e) {
5069 }
5070
5071 HRegion.closeHRegion(region);
5072 region = null;
5073 CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration);
5074 }
5075 }
5076
5077 private static HRegion initHRegion(byte[] tableName, String callingMethod,
5078 byte[]... families) throws IOException {
5079 return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
5080 families);
5081 }
5082 }