1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21
22 import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23 import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24 import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25 import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29 import static org.junit.Assert.assertArrayEquals;
30 import static org.junit.Assert.assertEquals;
31 import static org.junit.Assert.assertFalse;
32 import static org.junit.Assert.assertNotNull;
33 import static org.junit.Assert.assertNull;
34 import static org.junit.Assert.assertTrue;
35 import static org.junit.Assert.fail;
36 import static org.mockito.Matchers.any;
37 import static org.mockito.Matchers.anyBoolean;
38 import static org.mockito.Matchers.anyLong;
39 import static org.mockito.Matchers.eq;
40 import static org.mockito.Mockito.never;
41 import static org.mockito.Mockito.spy;
42 import static org.mockito.Mockito.times;
43 import static org.mockito.Mockito.verify;
44
45 import java.io.IOException;
46 import java.io.InterruptedIOException;
47 import java.security.PrivilegedExceptionAction;
48 import java.util.ArrayList;
49 import java.util.Arrays;
50 import java.util.Collection;
51 import java.util.List;
52 import java.util.Map;
53 import java.util.NavigableMap;
54 import java.util.TreeMap;
55 import java.util.UUID;
56 import java.util.concurrent.atomic.AtomicBoolean;
57 import java.util.concurrent.atomic.AtomicInteger;
58 import java.util.concurrent.atomic.AtomicLong;
59 import java.util.concurrent.atomic.AtomicReference;
60
61 import org.apache.commons.logging.Log;
62 import org.apache.commons.logging.LogFactory;
63 import org.apache.hadoop.conf.Configuration;
64 import org.apache.hadoop.fs.FSDataOutputStream;
65 import org.apache.hadoop.fs.FileStatus;
66 import org.apache.hadoop.fs.FileSystem;
67 import org.apache.hadoop.fs.Path;
68 import org.apache.hadoop.hbase.Cell;
69 import org.apache.hadoop.hbase.CellComparator;
70 import org.apache.hadoop.hbase.CellUtil;
71 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
72 import org.apache.hadoop.hbase.DroppedSnapshotException;
73 import org.apache.hadoop.hbase.HBaseConfiguration;
74 import org.apache.hadoop.hbase.HBaseTestCase;
75 import org.apache.hadoop.hbase.HBaseTestingUtility;
76 import org.apache.hadoop.hbase.HColumnDescriptor;
77 import org.apache.hadoop.hbase.HConstants;
78 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
79 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
80 import org.apache.hadoop.hbase.HRegionInfo;
81 import org.apache.hadoop.hbase.HTableDescriptor;
82 import org.apache.hadoop.hbase.KeyValue;
83 import org.apache.hadoop.hbase.MediumTests;
84 import org.apache.hadoop.hbase.MiniHBaseCluster;
85 import org.apache.hadoop.hbase.MultithreadedTestUtil;
86 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
87 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
88 import org.apache.hadoop.hbase.NotServingRegionException;
89 import org.apache.hadoop.hbase.TableName;
90 import org.apache.hadoop.hbase.Waiter;
91 import org.apache.hadoop.hbase.client.Append;
92 import org.apache.hadoop.hbase.client.Delete;
93 import org.apache.hadoop.hbase.client.Durability;
94 import org.apache.hadoop.hbase.client.Get;
95 import org.apache.hadoop.hbase.client.HTable;
96 import org.apache.hadoop.hbase.client.Increment;
97 import org.apache.hadoop.hbase.client.Put;
98 import org.apache.hadoop.hbase.client.Result;
99 import org.apache.hadoop.hbase.client.Scan;
100 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
101 import org.apache.hadoop.hbase.filter.BinaryComparator;
102 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
103 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
104 import org.apache.hadoop.hbase.filter.Filter;
105 import org.apache.hadoop.hbase.filter.FilterBase;
106 import org.apache.hadoop.hbase.filter.FilterList;
107 import org.apache.hadoop.hbase.filter.NullComparator;
108 import org.apache.hadoop.hbase.filter.PrefixFilter;
109 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
110 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
111 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
112 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
113 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
114 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
115 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
116 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
117 import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
118 import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
119 import org.apache.hadoop.hbase.regionserver.wal.FaultyHLog;
120 import org.apache.hadoop.hbase.regionserver.wal.HLog;
121 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
122 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
123 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
124 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
125 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
126 import org.apache.hadoop.hbase.security.User;
127 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
128 import org.apache.hadoop.hbase.util.Bytes;
129 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
130 import org.apache.hadoop.hbase.util.FSUtils;
131 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
132 import org.apache.hadoop.hbase.util.PairOfSameType;
133 import org.apache.hadoop.hbase.util.Threads;
134 import org.junit.After;
135 import org.junit.Assert;
136 import org.junit.Before;
137 import org.junit.Rule;
138 import org.junit.Test;
139 import org.junit.experimental.categories.Category;
140 import org.junit.rules.TestName;
141 import org.mockito.Mockito;
142
143 import com.google.common.collect.ImmutableList;
144 import com.google.common.collect.Lists;
145 import com.google.protobuf.ByteString;
146
147
148
149
150
151
152
153 @Category(MediumTests.class)
154 @SuppressWarnings("deprecation")
155 public class TestHRegion {
156
157
158 static final Log LOG = LogFactory.getLog(TestHRegion.class);
159 @Rule public TestName name = new TestName();
160
161 private static final String COLUMN_FAMILY = "MyCF";
162 private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
163
164 HRegion region = null;
165
166 private static HBaseTestingUtility TEST_UTIL;
167 public static Configuration CONF ;
168 private String dir;
169 private static FileSystem FILESYSTEM;
170 private final int MAX_VERSIONS = 2;
171
172
173 protected byte[] tableName;
174 protected String method;
175 protected final byte[] qual1 = Bytes.toBytes("qual1");
176 protected final byte[] qual2 = Bytes.toBytes("qual2");
177 protected final byte[] qual3 = Bytes.toBytes("qual3");
178 protected final byte[] value1 = Bytes.toBytes("value1");
179 protected final byte[] value2 = Bytes.toBytes("value2");
180 protected final byte[] row = Bytes.toBytes("rowA");
181 protected final byte[] row2 = Bytes.toBytes("rowB");
182
183 protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
184 .getInstance(MetricsAssertHelper.class);
185
186 @Before
187 public void setup() throws IOException {
188 TEST_UTIL = HBaseTestingUtility.createLocalHTU();
189 FILESYSTEM = TEST_UTIL.getTestFileSystem();
190 CONF = TEST_UTIL.getConfiguration();
191 dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
192 method = name.getMethodName();
193 tableName = Bytes.toBytes(name.getMethodName());
194 }
195
196 @After
197 public void tearDown() throws Exception {
198 EnvironmentEdgeManagerTestHelper.reset();
199 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
200 TEST_UTIL.cleanupTestDir();
201 }
202
203 String getName() {
204 return name.getMethodName();
205 }
206
207
208
209
210
211
212
213
214
215
216
217 @Test (timeout=60000)
218 public void testCloseCarryingSnapshot() throws IOException {
219 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
220 Store store = region.getStore(COLUMN_FAMILY_BYTES);
221
222 byte [] value = Bytes.toBytes(name.getMethodName());
223
224 Put put = new Put(value);
225 put.add(COLUMN_FAMILY_BYTES, null, value);
226
227 region.put(put);
228 StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
229 storeFlushCtx.prepare();
230
231 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
232 region.put(put);
233
234 region.close();
235 assertEquals(0, region.getMemstoreSize().get());
236 HRegion.closeHRegion(region);
237 }
238
239
240
241
242
243 @Test (timeout=60000)
244 public void testMemstoreSnapshotSize() throws IOException {
245 class MyFaultyHLog extends FaultyHLog {
246 StoreFlushContext storeFlushCtx;
247 public MyFaultyHLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
248 throws IOException {
249 super(fs, rootDir, logName, conf);
250 }
251
252 void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
253 this.storeFlushCtx = storeFlushCtx;
254 }
255
256 @Override
257 public void sync(long txid) throws IOException {
258 storeFlushCtx.prepare();
259 super.sync(txid);
260 }
261 }
262
263 FileSystem fs = FileSystem.get(CONF);
264 Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
265 MyFaultyHLog faultyLog = new MyFaultyHLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
266 HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
267 CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES);
268
269 Store store = region.getStore(COLUMN_FAMILY_BYTES);
270
271 byte [] value = Bytes.toBytes(name.getMethodName());
272 faultyLog.setStoreFlushCtx(store.createFlushContext(12345));
273
274 Put put = new Put(value);
275 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
276 faultyLog.setFailureType(FaultyHLog.FailureType.SYNC);
277
278 boolean threwIOE = false;
279 try {
280 region.put(put);
281 } catch (IOException ioe) {
282 threwIOE = true;
283 } finally {
284 assertTrue("The regionserver should have thrown an exception", threwIOE);
285 }
286 long sz = store.getFlushableSize();
287 assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
288 HRegion.closeHRegion(region);
289 }
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306 @Test (timeout=60000)
307 public void testFlushSizeAccounting() throws Exception {
308 final Configuration conf = HBaseConfiguration.create(CONF);
309
310 conf.setInt("hbase.hstore.flush.retries.number", 1);
311 final User user =
312 User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
313
314 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
315 user.runAs(new PrivilegedExceptionAction<Object>() {
316 @Override
317 public Object run() throws Exception {
318
319 FileSystem fs = FileSystem.get(conf);
320 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
321 FaultyFileSystem ffs = (FaultyFileSystem)fs;
322 HRegion region = null;
323 try {
324
325 region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
326 long size = region.getMemstoreSize().get();
327 Assert.assertEquals(0, size);
328
329 Put p1 = new Put(row);
330 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
331 region.put(p1);
332 final long sizeOfOnePut = region.getMemstoreSize().get();
333
334 try {
335 LOG.info("Flushing");
336 region.flushcache();
337 Assert.fail("Didn't bubble up IOE!");
338 } catch (DroppedSnapshotException dse) {
339
340 }
341
342 ffs.fault.set(false);
343
344 Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize().get());
345
346
347 Put p2 = new Put(row);
348 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
349 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
350 region.put(p2);
351 Assert.assertEquals(sizeOfOnePut * 3, region.getMemstoreSize().get());
352
353
354
355 region.flushcache();
356
357 Assert.assertEquals(sizeOfOnePut * 2, region.getMemstoreSize().get());
358 } finally {
359 HRegion.closeHRegion(region);
360 }
361 return null;
362 }
363 });
364 FileSystem.closeAllForUGI(user.getUGI());
365 }
366
367 @Test
368 public void testCompactionAffectedByScanners() throws Exception {
369 byte[] family = Bytes.toBytes("family");
370 this.region = initHRegion(tableName, method, CONF, family);
371
372 Put put = new Put(Bytes.toBytes("r1"));
373 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
374 region.put(put);
375 region.flushcache();
376
377 Scan scan = new Scan();
378 scan.setMaxVersions(3);
379
380 RegionScanner scanner1 = region.getScanner(scan);
381
382 Delete delete = new Delete(Bytes.toBytes("r1"));
383 region.delete(delete);
384 region.flushcache();
385
386
387 RegionScanner scanner2 = region.getScanner(scan);
388
389 List<Cell> results = new ArrayList<Cell>();
390
391 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
392
393
394 region.compactStores(true);
395
396
397 RegionScanner scanner3 = region.getScanner(scan);
398
399
400 scanner1.next(results);
401 System.out.println(results);
402 assertEquals(1, results.size());
403
404 results.clear();
405 scanner2.next(results);
406 System.out.println(results);
407 assertEquals(0, results.size());
408
409 results.clear();
410 scanner3.next(results);
411 System.out.println(results);
412 assertEquals(0, results.size());
413 }
414
415 @Test
416 public void testToShowNPEOnRegionScannerReseek() throws Exception {
417 byte[] family = Bytes.toBytes("family");
418 this.region = initHRegion(tableName, method, CONF, family);
419
420 Put put = new Put(Bytes.toBytes("r1"));
421 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
422 region.put(put);
423 put = new Put(Bytes.toBytes("r2"));
424 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
425 region.put(put);
426 region.flushcache();
427
428 Scan scan = new Scan();
429 scan.setMaxVersions(3);
430
431 RegionScanner scanner1 = region.getScanner(scan);
432
433 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
434
435 region.compactStores(true);
436
437 scanner1.reseek(Bytes.toBytes("r2"));
438 List<Cell> results = new ArrayList<Cell>();
439 scanner1.next(results);
440 Cell keyValue = results.get(0);
441 Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
442 scanner1.close();
443 }
444
445 @Test
446 public void testSkipRecoveredEditsReplay() throws Exception {
447 String method = "testSkipRecoveredEditsReplay";
448 TableName tableName = TableName.valueOf(method);
449 byte[] family = Bytes.toBytes("family");
450 this.region = initHRegion(tableName, method, CONF, family);
451 try {
452 Path regiondir = region.getRegionFileSystem().getRegionDir();
453 FileSystem fs = region.getRegionFileSystem().getFileSystem();
454 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
455
456 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
457
458 long maxSeqId = 1050;
459 long minSeqId = 1000;
460
461 for (long i = minSeqId; i <= maxSeqId; i += 10) {
462 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
463 fs.create(recoveredEdits);
464 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
465
466 long time = System.nanoTime();
467 WALEdit edit = new WALEdit();
468 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
469 .toBytes(i)));
470 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
471 HConstants.DEFAULT_CLUSTER_ID), edit));
472
473 writer.close();
474 }
475 MonitoredTask status = TaskMonitor.get().createStatus(method);
476 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
477 for (Store store : region.getStores().values()) {
478 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
479 }
480 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
481 assertEquals(maxSeqId, seqId);
482 Get get = new Get(row);
483 Result result = region.get(get);
484 for (long i = minSeqId; i <= maxSeqId; i += 10) {
485 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
486 assertEquals(1, kvs.size());
487 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
488 }
489 } finally {
490 HRegion.closeHRegion(this.region);
491 this.region = null;
492 }
493 }
494
495 @Test
496 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
497 String method = "testSkipRecoveredEditsReplaySomeIgnored";
498 TableName tableName = TableName.valueOf(method);
499 byte[] family = Bytes.toBytes("family");
500 this.region = initHRegion(tableName, method, CONF, family);
501 try {
502 Path regiondir = region.getRegionFileSystem().getRegionDir();
503 FileSystem fs = region.getRegionFileSystem().getFileSystem();
504 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
505
506 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
507
508 long maxSeqId = 1050;
509 long minSeqId = 1000;
510
511 for (long i = minSeqId; i <= maxSeqId; i += 10) {
512 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
513 fs.create(recoveredEdits);
514 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
515
516 long time = System.nanoTime();
517 WALEdit edit = new WALEdit();
518 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
519 .toBytes(i)));
520 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
521 HConstants.DEFAULT_CLUSTER_ID), edit));
522
523 writer.close();
524 }
525 long recoverSeqId = 1030;
526 MonitoredTask status = TaskMonitor.get().createStatus(method);
527 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
528 for (Store store : region.getStores().values()) {
529 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
530 }
531 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
532 assertEquals(maxSeqId, seqId);
533 Get get = new Get(row);
534 Result result = region.get(get);
535 for (long i = minSeqId; i <= maxSeqId; i += 10) {
536 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
537 if (i < recoverSeqId) {
538 assertEquals(0, kvs.size());
539 } else {
540 assertEquals(1, kvs.size());
541 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
542 }
543 }
544 } finally {
545 HRegion.closeHRegion(this.region);
546 this.region = null;
547 }
548 }
549
550 @Test
551 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
552 byte[] family = Bytes.toBytes("family");
553 this.region = initHRegion(tableName, method, CONF, family);
554 try {
555 Path regiondir = region.getRegionFileSystem().getRegionDir();
556 FileSystem fs = region.getRegionFileSystem().getFileSystem();
557
558 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
559 for (int i = 1000; i < 1050; i += 10) {
560 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
561 FSDataOutputStream dos = fs.create(recoveredEdits);
562 dos.writeInt(i);
563 dos.close();
564 }
565 long minSeqId = 2000;
566 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
567 FSDataOutputStream dos = fs.create(recoveredEdits);
568 dos.close();
569
570 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
571 for (Store store : region.getStores().values()) {
572 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
573 }
574 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
575 assertEquals(minSeqId, seqId);
576 } finally {
577 HRegion.closeHRegion(this.region);
578 this.region = null;
579 }
580 }
581
582 @Test
583 public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
584 String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
585 TableName tableName = TableName.valueOf(method);
586 byte[] family = Bytes.toBytes("family");
587 this.region = initHRegion(tableName, method, CONF, family);
588 try {
589 Path regiondir = region.getRegionFileSystem().getRegionDir();
590 FileSystem fs = region.getRegionFileSystem().getFileSystem();
591 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
592
593 assertEquals(0, region.getStoreFileList(
594 region.getStores().keySet().toArray(new byte[0][])).size());
595
596 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
597
598 long maxSeqId = 1050;
599 long minSeqId = 1000;
600
601 for (long i = minSeqId; i <= maxSeqId; i += 10) {
602 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
603 fs.create(recoveredEdits);
604 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
605
606 long time = System.nanoTime();
607 WALEdit edit = null;
608 if (i == maxSeqId) {
609 edit = WALEdit.createCompaction(region.getRegionInfo(),
610 CompactionDescriptor.newBuilder()
611 .setTableName(ByteString.copyFrom(tableName.getName()))
612 .setFamilyName(ByteString.copyFrom(regionName))
613 .setEncodedRegionName(ByteString.copyFrom(regionName))
614 .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
615 .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName()))
616 .build());
617 } else {
618 edit = new WALEdit();
619 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
620 .toBytes(i)));
621 }
622 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
623 HConstants.DEFAULT_CLUSTER_ID), edit));
624 writer.close();
625 }
626
627 long recoverSeqId = 1030;
628 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
629 MonitoredTask status = TaskMonitor.get().createStatus(method);
630 for (Store store : region.getStores().values()) {
631 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
632 }
633 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
634 assertEquals(maxSeqId, seqId);
635
636
637 assertEquals(1, region.getStoreFileList(
638 region.getStores().keySet().toArray(new byte[0][])).size());
639
640 } finally {
641 HRegion.closeHRegion(this.region);
642 this.region = null;
643 } }
644
645 @Test
646 public void testRecoveredEditsReplayCompaction() throws Exception {
647 String method = name.getMethodName();
648 TableName tableName = TableName.valueOf(method);
649 byte[] family = Bytes.toBytes("family");
650 this.region = initHRegion(tableName, method, CONF, family);
651 try {
652 Path regiondir = region.getRegionFileSystem().getRegionDir();
653 FileSystem fs = region.getRegionFileSystem().getFileSystem();
654 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
655
656 long maxSeqId = 3;
657 long minSeqId = 0;
658
659 for (long i = minSeqId; i < maxSeqId; i++) {
660 Put put = new Put(Bytes.toBytes(i));
661 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
662 region.put(put);
663 region.flushcache();
664 }
665
666
667 assertEquals(3, region.getStore(family).getStorefilesCount());
668 List<Path> storeFiles = new ArrayList<Path>(3);
669 for (StoreFile sf : region.getStore(family).getStorefiles()) {
670 storeFiles.add(sf.getPath());
671 }
672
673
674 CONF.setBoolean("hbase.hstore.compaction.complete", false);
675 region.compactStores();
676
677
678 assertEquals(3, region.getStore(family).getStorefilesCount());
679
680
681 Path tmpDir = region.getRegionFileSystem().getTempDir();
682 FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
683 String errorMsg = "Expected to find 1 file in the region temp directory "
684 + "from the compaction, could not find any";
685 assertNotNull(errorMsg, files);
686 assertEquals(errorMsg, 1, files.length);
687
688 Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
689 files[0].getPath());
690
691 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
692 .getRegionInfo(), family, storeFiles, Lists.newArrayList(newFile), region
693 .getRegionFileSystem().getStoreDir(Bytes.toString(family)));
694
695 HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
696 this.region.getRegionInfo(), compactionDescriptor, new AtomicLong(1));
697
698 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
699
700 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
701 fs.create(recoveredEdits);
702 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
703
704 long time = System.nanoTime();
705
706 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, 10, time,
707 HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(),
708 compactionDescriptor)));
709 writer.close();
710
711
712 region.getTableDesc();
713 region.getRegionInfo();
714 region.close();
715 region = HRegion.openHRegion(region, null);
716
717
718 Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
719 for (StoreFile sf : sfs) {
720 LOG.info(sf.getPath());
721 }
722 assertEquals(1, region.getStore(family).getStorefilesCount());
723 files = FSUtils.listStatus(fs, tmpDir);
724 assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
725
726 for (long i = minSeqId; i < maxSeqId; i++) {
727 Get get = new Get(Bytes.toBytes(i));
728 Result result = region.get(get);
729 byte[] value = result.getValue(family, Bytes.toBytes(i));
730 assertArrayEquals(Bytes.toBytes(i), value);
731 }
732 } finally {
733 HRegion.closeHRegion(this.region);
734 this.region = null;
735 }
736 }
737
738 @Test
739 public void testGetWhileRegionClose() throws IOException {
740 TableName tableName = TableName.valueOf(name.getMethodName());
741 Configuration hc = initSplit();
742 int numRows = 100;
743 byte[][] families = { fam1, fam2, fam3 };
744
745
746 String method = name.getMethodName();
747 this.region = initHRegion(tableName, method, hc, families);
748 try {
749
750 final int startRow = 100;
751 putData(startRow, numRows, qual1, families);
752 putData(startRow, numRows, qual2, families);
753 putData(startRow, numRows, qual3, families);
754 final AtomicBoolean done = new AtomicBoolean(false);
755 final AtomicInteger gets = new AtomicInteger(0);
756 GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
757 try {
758
759 for (int i = 0; i < threads.length / 2; i++) {
760 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
761 threads[i].setDaemon(true);
762 threads[i].start();
763 }
764
765
766 this.region.closing.set(true);
767 for (int i = threads.length / 2; i < threads.length; i++) {
768 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
769 threads[i].setDaemon(true);
770 threads[i].start();
771 }
772 } finally {
773 if (this.region != null) {
774 HRegion.closeHRegion(this.region);
775 }
776 }
777 done.set(true);
778 for (GetTillDoneOrException t : threads) {
779 try {
780 t.join();
781 } catch (InterruptedException e) {
782 e.printStackTrace();
783 }
784 if (t.e != null) {
785 LOG.info("Exception=" + t.e);
786 assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
787 }
788 }
789 } finally {
790 HRegion.closeHRegion(this.region);
791 this.region = null;
792 }
793 }
794
795
796
797
798
799 class GetTillDoneOrException extends Thread {
800 private final Get g;
801 private final AtomicBoolean done;
802 private final AtomicInteger count;
803 private Exception e;
804
805 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
806 super("getter." + i);
807 this.g = new Get(r);
808 this.done = d;
809 this.count = c;
810 }
811
812 @Override
813 public void run() {
814 while (!this.done.get()) {
815 try {
816 assertTrue(region.get(g).size() > 0);
817 this.count.incrementAndGet();
818 } catch (Exception e) {
819 this.e = e;
820 break;
821 }
822 }
823 }
824 }
825
826
827
828
829 @Test
830 public void testWeirdCacheBehaviour() throws Exception {
831 byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
832 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
833 Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
834 this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
835 try {
836 String value = "this is the value";
837 String value2 = "this is some other value";
838 String keyPrefix1 = "prefix1";
839 String keyPrefix2 = "prefix2";
840 String keyPrefix3 = "prefix3";
841 putRows(this.region, 3, value, keyPrefix1);
842 putRows(this.region, 3, value, keyPrefix2);
843 putRows(this.region, 3, value, keyPrefix3);
844 putRows(this.region, 3, value2, keyPrefix1);
845 putRows(this.region, 3, value2, keyPrefix2);
846 putRows(this.region, 3, value2, keyPrefix3);
847 System.out.println("Checking values for key: " + keyPrefix1);
848 assertEquals("Got back incorrect number of rows from scan", 3,
849 getNumberOfRows(keyPrefix1, value2, this.region));
850 System.out.println("Checking values for key: " + keyPrefix2);
851 assertEquals("Got back incorrect number of rows from scan", 3,
852 getNumberOfRows(keyPrefix2, value2, this.region));
853 System.out.println("Checking values for key: " + keyPrefix3);
854 assertEquals("Got back incorrect number of rows from scan", 3,
855 getNumberOfRows(keyPrefix3, value2, this.region));
856 deleteColumns(this.region, value2, keyPrefix1);
857 deleteColumns(this.region, value2, keyPrefix2);
858 deleteColumns(this.region, value2, keyPrefix3);
859 System.out.println("Starting important checks.....");
860 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
861 getNumberOfRows(keyPrefix1, value2, this.region));
862 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
863 getNumberOfRows(keyPrefix2, value2, this.region));
864 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
865 getNumberOfRows(keyPrefix3, value2, this.region));
866 } finally {
867 HRegion.closeHRegion(this.region);
868 this.region = null;
869 }
870 }
871
872 @Test
873 public void testAppendWithReadOnlyTable() throws Exception {
874 byte[] TABLE = Bytes.toBytes("readOnlyTable");
875 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
876 boolean exceptionCaught = false;
877 Append append = new Append(Bytes.toBytes("somerow"));
878 append.setDurability(Durability.SKIP_WAL);
879 append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
880 Bytes.toBytes("somevalue"));
881 try {
882 region.append(append);
883 } catch (IOException e) {
884 exceptionCaught = true;
885 } finally {
886 HRegion.closeHRegion(this.region);
887 this.region = null;
888 }
889 assertTrue(exceptionCaught == true);
890 }
891
892 @Test
893 public void testIncrWithReadOnlyTable() throws Exception {
894 byte[] TABLE = Bytes.toBytes("readOnlyTable");
895 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
896 boolean exceptionCaught = false;
897 Increment inc = new Increment(Bytes.toBytes("somerow"));
898 inc.setDurability(Durability.SKIP_WAL);
899 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
900 try {
901 region.increment(inc);
902 } catch (IOException e) {
903 exceptionCaught = true;
904 } finally {
905 HRegion.closeHRegion(this.region);
906 this.region = null;
907 }
908 assertTrue(exceptionCaught == true);
909 }
910
911 private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
912 InternalScanner scanner = buildScanner(keyPrefix, value, r);
913 int count = 0;
914 boolean more = false;
915 List<Cell> results = new ArrayList<Cell>();
916 do {
917 more = scanner.next(results);
918 if (results != null && !results.isEmpty())
919 count++;
920 else
921 break;
922 Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
923 delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
924 r.delete(delete);
925 results.clear();
926 } while (more);
927 assertEquals("Did not perform correct number of deletes", 3, count);
928 }
929
930 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
931 InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
932 int numberOfResults = 0;
933 List<Cell> results = new ArrayList<Cell>();
934 boolean more = false;
935 do {
936 more = resultScanner.next(results);
937 if (results != null && !results.isEmpty())
938 numberOfResults++;
939 else
940 break;
941 for (Cell kv : results) {
942 System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
943 }
944 results.clear();
945 } while (more);
946 return numberOfResults;
947 }
948
949 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
950 throws IOException {
951
952 FilterList allFilters = new FilterList();
953 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
954
955 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
956 Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
957 filter.setFilterIfMissing(true);
958 allFilters.addFilter(filter);
959 Scan scan = new Scan();
960 scan.addFamily(Bytes.toBytes("trans-blob"));
961 scan.addFamily(Bytes.toBytes("trans-type"));
962 scan.addFamily(Bytes.toBytes("trans-date"));
963 scan.addFamily(Bytes.toBytes("trans-tags"));
964 scan.addFamily(Bytes.toBytes("trans-group"));
965 scan.setFilter(allFilters);
966 return r.getScanner(scan);
967 }
968
969 private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
970 for (int i = 0; i < numRows; i++) {
971 String row = key + "_" + i
972 System.out.println(String.format("Saving row: %s, with value %s", row, value));
973 Put put = new Put(Bytes.toBytes(row));
974 put.setDurability(Durability.SKIP_WAL);
975 put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
976 put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
977 put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
978 put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
979 put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
980 r.put(put);
981 }
982 }
983
984 @Test
985 public void testFamilyWithAndWithoutColon() throws Exception {
986 byte[] b = Bytes.toBytes(getName());
987 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
988 this.region = initHRegion(b, getName(), CONF, cf);
989 try {
990 Put p = new Put(b);
991 byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
992 p.add(cfwithcolon, cfwithcolon, cfwithcolon);
993 boolean exception = false;
994 try {
995 this.region.put(p);
996 } catch (NoSuchColumnFamilyException e) {
997 exception = true;
998 }
999 assertTrue(exception);
1000 } finally {
1001 HRegion.closeHRegion(this.region);
1002 this.region = null;
1003 }
1004 }
1005
1006 @Test
1007 public void testBatchPut() throws Exception {
1008 byte[] b = Bytes.toBytes(getName());
1009 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1010 byte[] qual = Bytes.toBytes("qual");
1011 byte[] val = Bytes.toBytes("val");
1012 this.region = initHRegion(b, getName(), CONF, cf);
1013 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1014 try {
1015 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1016 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1017
1018 LOG.info("First a batch put with all valid puts");
1019 final Put[] puts = new Put[10];
1020 for (int i = 0; i < 10; i++) {
1021 puts[i] = new Put(Bytes.toBytes("row_" + i));
1022 puts[i].add(cf, qual, val);
1023 }
1024
1025 OperationStatus[] codes = this.region.batchMutate(puts);
1026 assertEquals(10, codes.length);
1027 for (int i = 0; i < 10; i++) {
1028 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1029 }
1030 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1031
1032 LOG.info("Next a batch put with one invalid family");
1033 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1034 codes = this.region.batchMutate(puts);
1035 assertEquals(10, codes.length);
1036 for (int i = 0; i < 10; i++) {
1037 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1038 codes[i].getOperationStatusCode());
1039 }
1040
1041 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1042
1043 LOG.info("Next a batch put that has to break into two batches to avoid a lock");
1044 RowLock rowLock = region.getRowLock(Bytes.toBytes("row_2"));
1045
1046 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1047 final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
1048 TestThread putter = new TestThread(ctx) {
1049 @Override
1050 public void doWork() throws IOException {
1051 retFromThread.set(region.batchMutate(puts));
1052 }
1053 };
1054 LOG.info("...starting put thread while holding lock");
1055 ctx.addThread(putter);
1056 ctx.startThreads();
1057
1058 LOG.info("...waiting for put thread to sync first time");
1059 long startWait = System.currentTimeMillis();
1060 while (metricsAssertHelper.getCounter("syncTimeNumOps", source) == syncs + 2) {
1061 Thread.sleep(100);
1062 if (System.currentTimeMillis() - startWait > 10000) {
1063 fail("Timed out waiting for thread to sync first minibatch");
1064 }
1065 }
1066 LOG.info("...releasing row lock, which should let put thread continue");
1067 rowLock.release();
1068 LOG.info("...joining on thread");
1069 ctx.stop();
1070 LOG.info("...checking that next batch was synced");
1071 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 4, source);
1072 codes = retFromThread.get();
1073 for (int i = 0; i < 10; i++) {
1074 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1075 codes[i].getOperationStatusCode());
1076 }
1077
1078 } finally {
1079 HRegion.closeHRegion(this.region);
1080 this.region = null;
1081 }
1082 }
1083
1084 @Test
1085 public void testBatchPutWithTsSlop() throws Exception {
1086 byte[] b = Bytes.toBytes(getName());
1087 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1088 byte[] qual = Bytes.toBytes("qual");
1089 byte[] val = Bytes.toBytes("val");
1090
1091
1092 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1093 this.region = initHRegion(b, getName(), CONF, cf);
1094
1095 try {
1096 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1097 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1098 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1099
1100 final Put[] puts = new Put[10];
1101 for (int i = 0; i < 10; i++) {
1102 puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
1103 puts[i].add(cf, qual, val);
1104 }
1105
1106 OperationStatus[] codes = this.region.batchMutate(puts);
1107 assertEquals(10, codes.length);
1108 for (int i = 0; i < 10; i++) {
1109 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1110 }
1111 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1112
1113 } finally {
1114 HRegion.closeHRegion(this.region);
1115 this.region = null;
1116 }
1117
1118 }
1119
1120
1121
1122
1123 @Test
1124 public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1125 byte[] row1 = Bytes.toBytes("row1");
1126 byte[] fam1 = Bytes.toBytes("fam1");
1127 byte[] qf1 = Bytes.toBytes("qualifier");
1128 byte[] emptyVal = new byte[] {};
1129 byte[] val1 = Bytes.toBytes("value1");
1130 byte[] val2 = Bytes.toBytes("value2");
1131
1132
1133 String method = this.getName();
1134 this.region = initHRegion(tableName, method, CONF, fam1);
1135 try {
1136
1137 Put put = new Put(row1);
1138 put.add(fam1, qf1, emptyVal);
1139
1140
1141 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1142 emptyVal), put, true);
1143 assertTrue(res);
1144
1145
1146 put = new Put(row1);
1147 put.add(fam1, qf1, val1);
1148
1149
1150 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1151 put, true);
1152 assertTrue(res);
1153
1154
1155 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1156 put, true);
1157 assertFalse(res);
1158
1159 Delete delete = new Delete(row1);
1160 delete.deleteColumn(fam1, qf1);
1161 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1162 delete, true);
1163 assertFalse(res);
1164
1165 put = new Put(row1);
1166 put.add(fam1, qf1, val2);
1167
1168 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1169 put, true);
1170 assertTrue(res);
1171
1172
1173 delete = new Delete(row1);
1174 delete.deleteColumn(fam1, qf1);
1175 delete.deleteColumn(fam1, qf1);
1176 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1177 delete, true);
1178 assertTrue(res);
1179
1180 delete = new Delete(row1);
1181 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1182 delete, true);
1183 assertTrue(res);
1184
1185
1186 put = new Put(row1);
1187 put.add(fam1, qf1, val1);
1188
1189 res = region
1190 .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
1191 assertTrue(res);
1192 } finally {
1193 HRegion.closeHRegion(this.region);
1194 this.region = null;
1195 }
1196 }
1197
1198 @Test
1199 public void testCheckAndMutate_WithWrongValue() throws IOException {
1200 byte[] row1 = Bytes.toBytes("row1");
1201 byte[] fam1 = Bytes.toBytes("fam1");
1202 byte[] qf1 = Bytes.toBytes("qualifier");
1203 byte[] val1 = Bytes.toBytes("value1");
1204 byte[] val2 = Bytes.toBytes("value2");
1205
1206
1207 String method = this.getName();
1208 this.region = initHRegion(tableName, method, CONF, fam1);
1209 try {
1210
1211 Put put = new Put(row1);
1212 put.add(fam1, qf1, val1);
1213 region.put(put);
1214
1215
1216 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1217 val2), put, true);
1218 assertEquals(false, res);
1219
1220
1221 Delete delete = new Delete(row1);
1222 delete.deleteFamily(fam1);
1223 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1224 delete, true);
1225 assertEquals(false, res);
1226 } finally {
1227 HRegion.closeHRegion(this.region);
1228 this.region = null;
1229 }
1230 }
1231
1232 @Test
1233 public void testCheckAndMutate_WithCorrectValue() throws IOException {
1234 byte[] row1 = Bytes.toBytes("row1");
1235 byte[] fam1 = Bytes.toBytes("fam1");
1236 byte[] qf1 = Bytes.toBytes("qualifier");
1237 byte[] val1 = Bytes.toBytes("value1");
1238
1239
1240 String method = this.getName();
1241 this.region = initHRegion(tableName, method, CONF, fam1);
1242 try {
1243
1244 Put put = new Put(row1);
1245 put.add(fam1, qf1, val1);
1246 region.put(put);
1247
1248
1249 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1250 val1), put, true);
1251 assertEquals(true, res);
1252
1253
1254 Delete delete = new Delete(row1);
1255 delete.deleteColumn(fam1, qf1);
1256 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1257 put, true);
1258 assertEquals(true, res);
1259 } finally {
1260 HRegion.closeHRegion(this.region);
1261 this.region = null;
1262 }
1263 }
1264
1265 @Test
1266 public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1267 byte[] row1 = Bytes.toBytes("row1");
1268 byte[] fam1 = Bytes.toBytes("fam1");
1269 byte[] qf1 = Bytes.toBytes("qualifier");
1270 byte[] val1 = Bytes.toBytes("value1");
1271 byte[] val2 = Bytes.toBytes("value2");
1272 byte[] val3 = Bytes.toBytes("value3");
1273 byte[] val4 = Bytes.toBytes("value4");
1274
1275
1276 String method = this.getName();
1277 this.region = initHRegion(tableName, method, CONF, fam1);
1278 try {
1279
1280 Put put = new Put(row1);
1281 put.add(fam1, qf1, val3);
1282 region.put(put);
1283
1284
1285 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1286 new BinaryComparator(val3), put, true);
1287 assertEquals(false, res);
1288
1289
1290 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1291 new BinaryComparator(val4), put, true);
1292 assertEquals(false, res);
1293
1294
1295
1296 put = new Put(row1);
1297 put.add(fam1, qf1, val2);
1298 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1299 new BinaryComparator(val2), put, true);
1300 assertEquals(true, res);
1301
1302
1303 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1304 new BinaryComparator(val3), put, true);
1305 assertEquals(false, res);
1306
1307
1308
1309 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1310 new BinaryComparator(val2), put, true);
1311 assertEquals(true, res);
1312
1313
1314
1315 put = new Put(row1);
1316 put.add(fam1, qf1, val3);
1317 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1318 new BinaryComparator(val1), put, true);
1319 assertEquals(true, res);
1320
1321
1322 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1323 new BinaryComparator(val3), put, true);
1324 assertEquals(false, res);
1325
1326
1327 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1328 new BinaryComparator(val2), put, true);
1329 assertEquals(false, res);
1330
1331
1332
1333 put = new Put(row1);
1334 put.add(fam1, qf1, val2);
1335 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1336 new BinaryComparator(val4), put, true);
1337 assertEquals(true, res);
1338
1339
1340 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1341 new BinaryComparator(val1), put, true);
1342 assertEquals(false, res);
1343
1344
1345
1346 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1347 new BinaryComparator(val2), put, true);
1348 assertEquals(true, res);
1349
1350
1351 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1352 new BinaryComparator(val3), put, true);
1353 assertEquals(true, res);
1354 } finally {
1355 HRegion.closeHRegion(this.region);
1356 this.region = null;
1357 }
1358 }
1359
1360 @Test
1361 public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1362 byte[] row1 = Bytes.toBytes("row1");
1363 byte[] fam1 = Bytes.toBytes("fam1");
1364 byte[] fam2 = Bytes.toBytes("fam2");
1365 byte[] qf1 = Bytes.toBytes("qualifier");
1366 byte[] val1 = Bytes.toBytes("value1");
1367 byte[] val2 = Bytes.toBytes("value2");
1368
1369 byte[][] families = { fam1, fam2 };
1370
1371
1372 String method = this.getName();
1373 this.region = initHRegion(tableName, method, CONF, families);
1374 try {
1375
1376 Put put = new Put(row1);
1377 put.add(fam1, qf1, val1);
1378 region.put(put);
1379
1380
1381 long ts = System.currentTimeMillis();
1382 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1383 put = new Put(row1);
1384 put.add(kv);
1385
1386
1387 HStore store = (HStore) region.getStore(fam1);
1388 store.memstore.kvset.size();
1389
1390 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1391 val1), put, true);
1392 assertEquals(true, res);
1393 store.memstore.kvset.size();
1394
1395 Get get = new Get(row1);
1396 get.addColumn(fam2, qf1);
1397 Cell[] actual = region.get(get).rawCells();
1398
1399 Cell[] expected = { kv };
1400
1401 assertEquals(expected.length, actual.length);
1402 for (int i = 0; i < actual.length; i++) {
1403 assertEquals(expected[i], actual[i]);
1404 }
1405 } finally {
1406 HRegion.closeHRegion(this.region);
1407 this.region = null;
1408 }
1409 }
1410
1411 @Test
1412 public void testCheckAndPut_wrongRowInPut() throws IOException {
1413 TableName tableName = TableName.valueOf(name.getMethodName());
1414 this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
1415 try {
1416 Put put = new Put(row2);
1417 put.add(fam1, qual1, value1);
1418 try {
1419 region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1420 new BinaryComparator(value2), put, false);
1421 fail();
1422 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1423
1424 }
1425 } finally {
1426 HRegion.closeHRegion(this.region);
1427 this.region = null;
1428 }
1429 }
1430
1431 @Test
1432 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1433 byte[] row1 = Bytes.toBytes("row1");
1434 byte[] fam1 = Bytes.toBytes("fam1");
1435 byte[] fam2 = Bytes.toBytes("fam2");
1436 byte[] qf1 = Bytes.toBytes("qualifier1");
1437 byte[] qf2 = Bytes.toBytes("qualifier2");
1438 byte[] qf3 = Bytes.toBytes("qualifier3");
1439 byte[] val1 = Bytes.toBytes("value1");
1440 byte[] val2 = Bytes.toBytes("value2");
1441 byte[] val3 = Bytes.toBytes("value3");
1442 byte[] emptyVal = new byte[] {};
1443
1444 byte[][] families = { fam1, fam2 };
1445
1446
1447 String method = this.getName();
1448 this.region = initHRegion(tableName, method, CONF, families);
1449 try {
1450
1451 Put put = new Put(row1);
1452 put.add(fam1, qf1, val1);
1453 region.put(put);
1454 Threads.sleep(2);
1455
1456 put = new Put(row1);
1457 put.add(fam1, qf1, val2);
1458 put.add(fam2, qf1, val3);
1459 put.add(fam2, qf2, val2);
1460 put.add(fam2, qf3, val1);
1461 put.add(fam1, qf3, val1);
1462 region.put(put);
1463
1464
1465 Delete delete = new Delete(row1);
1466 delete.deleteColumn(fam1, qf1);
1467 delete.deleteColumn(fam2, qf1);
1468 delete.deleteColumn(fam1, qf3);
1469 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1470 val2), delete, true);
1471 assertEquals(true, res);
1472
1473 Get get = new Get(row1);
1474 get.addColumn(fam1, qf1);
1475 get.addColumn(fam1, qf3);
1476 get.addColumn(fam2, qf2);
1477 Result r = region.get(get);
1478 assertEquals(2, r.size());
1479 assertArrayEquals(val1, r.getValue(fam1, qf1));
1480 assertArrayEquals(val2, r.getValue(fam2, qf2));
1481
1482
1483 delete = new Delete(row1);
1484 delete.deleteFamily(fam2);
1485 res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1486 delete, true);
1487 assertEquals(true, res);
1488
1489 get = new Get(row1);
1490 r = region.get(get);
1491 assertEquals(1, r.size());
1492 assertArrayEquals(val1, r.getValue(fam1, qf1));
1493
1494
1495 delete = new Delete(row1);
1496 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1497 delete, true);
1498 assertEquals(true, res);
1499 get = new Get(row1);
1500 r = region.get(get);
1501 assertEquals(0, r.size());
1502 } finally {
1503 HRegion.closeHRegion(this.region);
1504 this.region = null;
1505 }
1506 }
1507
1508
1509
1510
1511 @Test
1512 public void testDelete_multiDeleteColumn() throws IOException {
1513 byte[] row1 = Bytes.toBytes("row1");
1514 byte[] fam1 = Bytes.toBytes("fam1");
1515 byte[] qual = Bytes.toBytes("qualifier");
1516 byte[] value = Bytes.toBytes("value");
1517
1518 Put put = new Put(row1);
1519 put.add(fam1, qual, 1, value);
1520 put.add(fam1, qual, 2, value);
1521
1522 String method = this.getName();
1523 this.region = initHRegion(tableName, method, CONF, fam1);
1524 try {
1525 region.put(put);
1526
1527
1528 Delete delete = new Delete(row1);
1529 delete.deleteColumn(fam1, qual);
1530 delete.deleteColumn(fam1, qual);
1531 region.delete(delete);
1532
1533 Get get = new Get(row1);
1534 get.addFamily(fam1);
1535 Result r = region.get(get);
1536 assertEquals(0, r.size());
1537 } finally {
1538 HRegion.closeHRegion(this.region);
1539 this.region = null;
1540 }
1541 }
1542
1543 @Test
1544 public void testDelete_CheckFamily() throws IOException {
1545 byte[] row1 = Bytes.toBytes("row1");
1546 byte[] fam1 = Bytes.toBytes("fam1");
1547 byte[] fam2 = Bytes.toBytes("fam2");
1548 byte[] fam3 = Bytes.toBytes("fam3");
1549 byte[] fam4 = Bytes.toBytes("fam4");
1550
1551
1552 String method = this.getName();
1553 this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
1554 try {
1555 List<Cell> kvs = new ArrayList<Cell>();
1556 kvs.add(new KeyValue(row1, fam4, null, null));
1557
1558
1559 byte[] family = fam2;
1560 try {
1561 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1562 Bytes.BYTES_COMPARATOR);
1563 deleteMap.put(family, kvs);
1564 region.delete(deleteMap, Durability.SYNC_WAL);
1565 } catch (Exception e) {
1566 assertTrue("Family " + new String(family) + " does not exist", false);
1567 }
1568
1569
1570 boolean ok = false;
1571 family = fam4;
1572 try {
1573 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1574 Bytes.BYTES_COMPARATOR);
1575 deleteMap.put(family, kvs);
1576 region.delete(deleteMap, Durability.SYNC_WAL);
1577 } catch (Exception e) {
1578 ok = true;
1579 }
1580 assertEquals("Family " + new String(family) + " does exist", true, ok);
1581 } finally {
1582 HRegion.closeHRegion(this.region);
1583 this.region = null;
1584 }
1585 }
1586
1587 @Test
1588 public void testDelete_mixed() throws IOException, InterruptedException {
1589 byte[] fam = Bytes.toBytes("info");
1590 byte[][] families = { fam };
1591 String method = this.getName();
1592 this.region = initHRegion(tableName, method, CONF, families);
1593 try {
1594 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1595
1596 byte[] row = Bytes.toBytes("table_name");
1597
1598 byte[] serverinfo = Bytes.toBytes("serverinfo");
1599 byte[] splitA = Bytes.toBytes("splitA");
1600 byte[] splitB = Bytes.toBytes("splitB");
1601
1602
1603 Put put = new Put(row);
1604 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1605 region.put(put);
1606
1607 put = new Put(row);
1608 put.add(fam, splitB, Bytes.toBytes("reference_B"));
1609 region.put(put);
1610
1611 put = new Put(row);
1612 put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1613 region.put(put);
1614
1615
1616 Delete delete = new Delete(row);
1617 delete.deleteColumns(fam, splitA);
1618 region.delete(delete);
1619
1620
1621 Get get = new Get(row).addColumn(fam, serverinfo);
1622 Result result = region.get(get);
1623 assertEquals(1, result.size());
1624
1625 get = new Get(row).addColumn(fam, splitA);
1626 result = region.get(get);
1627 assertEquals(0, result.size());
1628
1629 get = new Get(row).addColumn(fam, splitB);
1630 result = region.get(get);
1631 assertEquals(1, result.size());
1632
1633
1634 put = new Put(row);
1635 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1636 region.put(put);
1637 get = new Get(row);
1638 result = region.get(get);
1639 assertEquals(3, result.size());
1640
1641
1642 delete = new Delete(row);
1643 region.delete(delete);
1644 assertEquals(0, region.get(get).size());
1645
1646 region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
1647 result = region.get(get);
1648 assertEquals(1, result.size());
1649 } finally {
1650 HRegion.closeHRegion(this.region);
1651 this.region = null;
1652 }
1653 }
1654
1655 @Test
1656 public void testDeleteRowWithFutureTs() throws IOException {
1657 byte[] fam = Bytes.toBytes("info");
1658 byte[][] families = { fam };
1659 String method = this.getName();
1660 this.region = initHRegion(tableName, method, CONF, families);
1661 try {
1662 byte[] row = Bytes.toBytes("table_name");
1663
1664 byte[] serverinfo = Bytes.toBytes("serverinfo");
1665
1666
1667 Put put = new Put(row);
1668 put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
1669 region.put(put);
1670
1671
1672 Delete delete = new Delete(row);
1673 region.delete(delete);
1674
1675
1676 Get get = new Get(row).addColumn(fam, serverinfo);
1677 Result result = region.get(get);
1678 assertEquals(1, result.size());
1679
1680
1681 delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
1682 region.delete(delete);
1683
1684
1685 get = new Get(row).addColumn(fam, serverinfo);
1686 result = region.get(get);
1687 assertEquals(0, result.size());
1688 } finally {
1689 HRegion.closeHRegion(this.region);
1690 this.region = null;
1691 }
1692 }
1693
1694
1695
1696
1697
1698 @Test
1699 public void testPutWithLatestTS() throws IOException {
1700 byte[] fam = Bytes.toBytes("info");
1701 byte[][] families = { fam };
1702 String method = this.getName();
1703 this.region = initHRegion(tableName, method, CONF, families);
1704 try {
1705 byte[] row = Bytes.toBytes("row1");
1706
1707 byte[] qual = Bytes.toBytes("qual");
1708
1709
1710 Put put = new Put(row);
1711 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1712 region.put(put);
1713
1714
1715 Get get = new Get(row).addColumn(fam, qual);
1716 Result result = region.get(get);
1717 assertEquals(1, result.size());
1718 Cell kv = result.rawCells()[0];
1719 LOG.info("Got: " + kv);
1720 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1721 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1722
1723
1724
1725 row = Bytes.toBytes("row2");
1726 put = new Put(row);
1727 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1728 region.put(put);
1729
1730
1731 get = new Get(row).addColumn(fam, qual);
1732 result = region.get(get);
1733 assertEquals(1, result.size());
1734 kv = result.rawCells()[0];
1735 LOG.info("Got: " + kv);
1736 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1737 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1738 } finally {
1739 HRegion.closeHRegion(this.region);
1740 this.region = null;
1741 }
1742
1743 }
1744
1745
1746
1747
1748
1749
1750 @Test
1751 public void testPutWithTsSlop() throws IOException {
1752 byte[] fam = Bytes.toBytes("info");
1753 byte[][] families = { fam };
1754 String method = this.getName();
1755
1756
1757 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1758 this.region = initHRegion(tableName, method, CONF, families);
1759 boolean caughtExcep = false;
1760 try {
1761 try {
1762
1763 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
1764
1765 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
1766 Bytes.toBytes("value")));
1767 fail("Expected IOE for TS out of configured timerange");
1768 } catch (FailedSanityCheckException ioe) {
1769 LOG.debug("Received expected exception", ioe);
1770 caughtExcep = true;
1771 }
1772 assertTrue("Should catch FailedSanityCheckException", caughtExcep);
1773 } finally {
1774 HRegion.closeHRegion(this.region);
1775 this.region = null;
1776 }
1777 }
1778
1779 @Test
1780 public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
1781 byte[] fam1 = Bytes.toBytes("columnA");
1782 byte[] fam2 = Bytes.toBytes("columnB");
1783 this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
1784 try {
1785 byte[] rowA = Bytes.toBytes("rowA");
1786 byte[] rowB = Bytes.toBytes("rowB");
1787
1788 byte[] value = Bytes.toBytes("value");
1789
1790 Delete delete = new Delete(rowA);
1791 delete.deleteFamily(fam1);
1792
1793 region.delete(delete);
1794
1795
1796 Put put = new Put(rowA);
1797 put.add(fam2, null, value);
1798 region.put(put);
1799
1800 put = new Put(rowB);
1801 put.add(fam1, null, value);
1802 put.add(fam2, null, value);
1803 region.put(put);
1804
1805 Scan scan = new Scan();
1806 scan.addFamily(fam1).addFamily(fam2);
1807 InternalScanner s = region.getScanner(scan);
1808 List<Cell> results = new ArrayList<Cell>();
1809 s.next(results);
1810 assertTrue(CellUtil.matchingRow(results.get(0), rowA));
1811
1812 results.clear();
1813 s.next(results);
1814 assertTrue(CellUtil.matchingRow(results.get(0), rowB));
1815 } finally {
1816 HRegion.closeHRegion(this.region);
1817 this.region = null;
1818 }
1819 }
1820
1821 @Test
1822 public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
1823 Delete delete = new Delete(row);
1824 delete.deleteColumns(fam1, qual1);
1825 doTestDelete_AndPostInsert(delete);
1826 }
1827
1828 @Test
1829 public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
1830 Delete delete = new Delete(row);
1831 delete.deleteFamily(fam1);
1832 doTestDelete_AndPostInsert(delete);
1833 }
1834
1835 public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
1836 TableName tableName = TableName.valueOf(name.getMethodName());
1837 this.region = initHRegion(tableName, getName(), CONF, fam1);
1838 try {
1839 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1840 Put put = new Put(row);
1841 put.add(fam1, qual1, value1);
1842 region.put(put);
1843
1844
1845 region.delete(delete);
1846
1847
1848 put = new Put(row);
1849 put.add(fam1, qual1, value2);
1850 region.put(put);
1851
1852
1853 Get get = new Get(row);
1854 get.addColumn(fam1, qual1);
1855
1856 Result r = region.get(get);
1857 assertEquals(1, r.size());
1858 assertArrayEquals(value2, r.getValue(fam1, qual1));
1859
1860
1861 Scan scan = new Scan(row);
1862 scan.addColumn(fam1, qual1);
1863 InternalScanner s = region.getScanner(scan);
1864
1865 List<Cell> results = new ArrayList<Cell>();
1866 assertEquals(false, s.next(results));
1867 assertEquals(1, results.size());
1868 Cell kv = results.get(0);
1869
1870 assertArrayEquals(value2, CellUtil.cloneValue(kv));
1871 assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
1872 assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
1873 assertArrayEquals(row, CellUtil.cloneRow(kv));
1874 } finally {
1875 HRegion.closeHRegion(this.region);
1876 this.region = null;
1877 }
1878 }
1879
1880 @Test
1881 public void testDelete_CheckTimestampUpdated() throws IOException {
1882 TableName tableName = TableName.valueOf(name.getMethodName());
1883 byte[] row1 = Bytes.toBytes("row1");
1884 byte[] col1 = Bytes.toBytes("col1");
1885 byte[] col2 = Bytes.toBytes("col2");
1886 byte[] col3 = Bytes.toBytes("col3");
1887
1888
1889 String method = this.getName();
1890 this.region = initHRegion(tableName, method, CONF, fam1);
1891 try {
1892
1893 List<Cell> kvs = new ArrayList<Cell>();
1894 kvs.add(new KeyValue(row1, fam1, col1, null));
1895 kvs.add(new KeyValue(row1, fam1, col2, null));
1896 kvs.add(new KeyValue(row1, fam1, col3, null));
1897
1898 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1899 Bytes.BYTES_COMPARATOR);
1900 deleteMap.put(fam1, kvs);
1901 region.delete(deleteMap, Durability.SYNC_WAL);
1902
1903
1904
1905 long now = System.currentTimeMillis();
1906 KeyValue firstKv = ((HStore) region.getStore(fam1)).memstore.kvset.first();
1907 assertTrue(firstKv.getTimestamp() <= now);
1908 now = firstKv.getTimestamp();
1909 for (KeyValue kv : ((HStore) region.getStore(fam1)).memstore.kvset) {
1910 assertTrue(kv.getTimestamp() <= now);
1911 now = kv.getTimestamp();
1912 }
1913 } finally {
1914 HRegion.closeHRegion(this.region);
1915 this.region = null;
1916 }
1917 }
1918
1919
1920
1921
1922 @Test
1923 public void testGet_FamilyChecker() throws IOException {
1924 byte[] row1 = Bytes.toBytes("row1");
1925 byte[] fam1 = Bytes.toBytes("fam1");
1926 byte[] fam2 = Bytes.toBytes("False");
1927 byte[] col1 = Bytes.toBytes("col1");
1928
1929
1930 String method = this.getName();
1931 this.region = initHRegion(tableName, method, CONF, fam1);
1932 try {
1933 Get get = new Get(row1);
1934 get.addColumn(fam2, col1);
1935
1936
1937 try {
1938 region.get(get);
1939 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
1940 assertFalse(false);
1941 return;
1942 }
1943 assertFalse(true);
1944 } finally {
1945 HRegion.closeHRegion(this.region);
1946 this.region = null;
1947 }
1948 }
1949
1950 @Test
1951 public void testGet_Basic() throws IOException {
1952 byte[] row1 = Bytes.toBytes("row1");
1953 byte[] fam1 = Bytes.toBytes("fam1");
1954 byte[] col1 = Bytes.toBytes("col1");
1955 byte[] col2 = Bytes.toBytes("col2");
1956 byte[] col3 = Bytes.toBytes("col3");
1957 byte[] col4 = Bytes.toBytes("col4");
1958 byte[] col5 = Bytes.toBytes("col5");
1959
1960
1961 String method = this.getName();
1962 this.region = initHRegion(tableName, method, CONF, fam1);
1963 try {
1964
1965 Put put = new Put(row1);
1966 put.add(fam1, col1, null);
1967 put.add(fam1, col2, null);
1968 put.add(fam1, col3, null);
1969 put.add(fam1, col4, null);
1970 put.add(fam1, col5, null);
1971 region.put(put);
1972
1973 Get get = new Get(row1);
1974 get.addColumn(fam1, col2);
1975 get.addColumn(fam1, col4);
1976
1977 KeyValue kv1 = new KeyValue(row1, fam1, col2);
1978 KeyValue kv2 = new KeyValue(row1, fam1, col4);
1979 KeyValue[] expected = { kv1, kv2 };
1980
1981
1982 Result res = region.get(get);
1983 assertEquals(expected.length, res.size());
1984 for (int i = 0; i < res.size(); i++) {
1985 assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
1986 assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
1987 assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
1988 }
1989
1990
1991 Get g = new Get(row1);
1992 final int count = 2;
1993 g.setFilter(new ColumnCountGetFilter(count));
1994 res = region.get(g);
1995 assertEquals(count, res.size());
1996 } finally {
1997 HRegion.closeHRegion(this.region);
1998 this.region = null;
1999 }
2000 }
2001
2002 @Test
2003 public void testGet_Empty() throws IOException {
2004 byte[] row = Bytes.toBytes("row");
2005 byte[] fam = Bytes.toBytes("fam");
2006
2007 String method = this.getName();
2008 this.region = initHRegion(tableName, method, CONF, fam);
2009 try {
2010 Get get = new Get(row);
2011 get.addFamily(fam);
2012 Result r = region.get(get);
2013
2014 assertTrue(r.isEmpty());
2015 } finally {
2016 HRegion.closeHRegion(this.region);
2017 this.region = null;
2018 }
2019 }
2020
2021
2022
2023
2024 @Test
2025 public void testMerge() throws IOException {
2026 byte[][] families = { fam1, fam2, fam3 };
2027 Configuration hc = initSplit();
2028
2029 String method = this.getName();
2030 this.region = initHRegion(tableName, method, hc, families);
2031 try {
2032 LOG.info("" + HBaseTestCase.addContent(region, fam3));
2033 region.flushcache();
2034 region.compactStores();
2035 byte[] splitRow = region.checkSplit();
2036 assertNotNull(splitRow);
2037 LOG.info("SplitRow: " + Bytes.toString(splitRow));
2038 HRegion[] subregions = splitRegion(region, splitRow);
2039 try {
2040
2041 for (int i = 0; i < subregions.length; i++) {
2042 HRegion.openHRegion(subregions[i], null);
2043 subregions[i].compactStores();
2044 }
2045 Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
2046 Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
2047 Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
2048 long startTime = System.currentTimeMillis();
2049 region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
2050 LOG.info("Merge regions elapsed time: "
2051 + ((System.currentTimeMillis() - startTime) / 1000.0));
2052 FILESYSTEM.delete(oldRegion1, true);
2053 FILESYSTEM.delete(oldRegion2, true);
2054 FILESYSTEM.delete(oldRegionPath, true);
2055 LOG.info("splitAndMerge completed.");
2056 } finally {
2057 for (int i = 0; i < subregions.length; i++) {
2058 try {
2059 HRegion.closeHRegion(subregions[i]);
2060 } catch (IOException e) {
2061
2062 }
2063 }
2064 }
2065 } finally {
2066 HRegion.closeHRegion(this.region);
2067 this.region = null;
2068 }
2069 }
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079 HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
2080 PairOfSameType<HRegion> result = null;
2081 SplitTransaction st = new SplitTransaction(parent, midkey);
2082
2083
2084 if (!st.prepare()) {
2085 parent.clearSplit();
2086 return null;
2087 }
2088 try {
2089 result = st.execute(null, null);
2090 } catch (IOException ioe) {
2091 try {
2092 LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; "
2093 + ioe.getMessage());
2094 st.rollback(null, null);
2095 LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString());
2096 return null;
2097 } catch (RuntimeException e) {
2098
2099 LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString()
2100 + " -- aborting server", e);
2101 }
2102 }
2103 finally {
2104 parent.clearSplit();
2105 }
2106 return new HRegion[] { result.getFirst(), result.getSecond() };
2107 }
2108
2109
2110
2111
2112 @Test
2113 public void testGetScanner_WithOkFamilies() throws IOException {
2114 byte[] fam1 = Bytes.toBytes("fam1");
2115 byte[] fam2 = Bytes.toBytes("fam2");
2116
2117 byte[][] families = { fam1, fam2 };
2118
2119
2120 String method = this.getName();
2121 this.region = initHRegion(tableName, method, CONF, families);
2122 try {
2123 Scan scan = new Scan();
2124 scan.addFamily(fam1);
2125 scan.addFamily(fam2);
2126 try {
2127 region.getScanner(scan);
2128 } catch (Exception e) {
2129 assertTrue("Families could not be found in Region", false);
2130 }
2131 } finally {
2132 HRegion.closeHRegion(this.region);
2133 this.region = null;
2134 }
2135 }
2136
2137 @Test
2138 public void testGetScanner_WithNotOkFamilies() throws IOException {
2139 byte[] fam1 = Bytes.toBytes("fam1");
2140 byte[] fam2 = Bytes.toBytes("fam2");
2141
2142 byte[][] families = { fam1 };
2143
2144
2145 String method = this.getName();
2146 this.region = initHRegion(tableName, method, CONF, families);
2147 try {
2148 Scan scan = new Scan();
2149 scan.addFamily(fam2);
2150 boolean ok = false;
2151 try {
2152 region.getScanner(scan);
2153 } catch (Exception e) {
2154 ok = true;
2155 }
2156 assertTrue("Families could not be found in Region", ok);
2157 } finally {
2158 HRegion.closeHRegion(this.region);
2159 this.region = null;
2160 }
2161 }
2162
2163 @Test
2164 public void testGetScanner_WithNoFamilies() throws IOException {
2165 byte[] row1 = Bytes.toBytes("row1");
2166 byte[] fam1 = Bytes.toBytes("fam1");
2167 byte[] fam2 = Bytes.toBytes("fam2");
2168 byte[] fam3 = Bytes.toBytes("fam3");
2169 byte[] fam4 = Bytes.toBytes("fam4");
2170
2171 byte[][] families = { fam1, fam2, fam3, fam4 };
2172
2173
2174 String method = this.getName();
2175 this.region = initHRegion(tableName, method, CONF, families);
2176 try {
2177
2178
2179 Put put = new Put(row1);
2180 put.add(fam1, null, null);
2181 put.add(fam2, null, null);
2182 put.add(fam3, null, null);
2183 put.add(fam4, null, null);
2184 region.put(put);
2185
2186 Scan scan = null;
2187 HRegion.RegionScannerImpl is = null;
2188
2189
2190
2191
2192 scan = new Scan();
2193 scan.addFamily(fam2);
2194 scan.addFamily(fam4);
2195 is = (RegionScannerImpl) region.getScanner(scan);
2196 assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2197
2198 scan = new Scan();
2199 is = (RegionScannerImpl) region.getScanner(scan);
2200 assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2201 } finally {
2202 HRegion.closeHRegion(this.region);
2203 this.region = null;
2204 }
2205 }
2206
2207
2208
2209
2210
2211
2212 @Test
2213 public void testGetScanner_WithRegionClosed() throws IOException {
2214 byte[] fam1 = Bytes.toBytes("fam1");
2215 byte[] fam2 = Bytes.toBytes("fam2");
2216
2217 byte[][] families = { fam1, fam2 };
2218
2219
2220 String method = this.getName();
2221 try {
2222 this.region = initHRegion(tableName, method, CONF, families);
2223 } catch (IOException e) {
2224 e.printStackTrace();
2225 fail("Got IOException during initHRegion, " + e.getMessage());
2226 }
2227 try {
2228 region.closed.set(true);
2229 try {
2230 region.getScanner(null);
2231 fail("Expected to get an exception during getScanner on a region that is closed");
2232 } catch (NotServingRegionException e) {
2233
2234 } catch (IOException e) {
2235 fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2236 + e.getMessage());
2237 }
2238 } finally {
2239 HRegion.closeHRegion(this.region);
2240 this.region = null;
2241 }
2242 }
2243
2244 @Test
2245 public void testRegionScanner_Next() throws IOException {
2246 byte[] row1 = Bytes.toBytes("row1");
2247 byte[] row2 = Bytes.toBytes("row2");
2248 byte[] fam1 = Bytes.toBytes("fam1");
2249 byte[] fam2 = Bytes.toBytes("fam2");
2250 byte[] fam3 = Bytes.toBytes("fam3");
2251 byte[] fam4 = Bytes.toBytes("fam4");
2252
2253 byte[][] families = { fam1, fam2, fam3, fam4 };
2254 long ts = System.currentTimeMillis();
2255
2256
2257 String method = this.getName();
2258 this.region = initHRegion(tableName, method, CONF, families);
2259 try {
2260
2261 Put put = null;
2262 put = new Put(row1);
2263 put.add(fam1, (byte[]) null, ts, null);
2264 put.add(fam2, (byte[]) null, ts, null);
2265 put.add(fam3, (byte[]) null, ts, null);
2266 put.add(fam4, (byte[]) null, ts, null);
2267 region.put(put);
2268
2269 put = new Put(row2);
2270 put.add(fam1, (byte[]) null, ts, null);
2271 put.add(fam2, (byte[]) null, ts, null);
2272 put.add(fam3, (byte[]) null, ts, null);
2273 put.add(fam4, (byte[]) null, ts, null);
2274 region.put(put);
2275
2276 Scan scan = new Scan();
2277 scan.addFamily(fam2);
2278 scan.addFamily(fam4);
2279 InternalScanner is = region.getScanner(scan);
2280
2281 List<Cell> res = null;
2282
2283
2284 List<Cell> expected1 = new ArrayList<Cell>();
2285 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2286 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2287
2288 res = new ArrayList<Cell>();
2289 is.next(res);
2290 for (int i = 0; i < res.size(); i++) {
2291 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2292 }
2293
2294
2295 List<Cell> expected2 = new ArrayList<Cell>();
2296 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2297 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2298
2299 res = new ArrayList<Cell>();
2300 is.next(res);
2301 for (int i = 0; i < res.size(); i++) {
2302 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2303 }
2304 } finally {
2305 HRegion.closeHRegion(this.region);
2306 this.region = null;
2307 }
2308 }
2309
2310 @Test
2311 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2312 byte[] row1 = Bytes.toBytes("row1");
2313 byte[] qf1 = Bytes.toBytes("qualifier1");
2314 byte[] qf2 = Bytes.toBytes("qualifier2");
2315 byte[] fam1 = Bytes.toBytes("fam1");
2316 byte[][] families = { fam1 };
2317
2318 long ts1 = System.currentTimeMillis();
2319 long ts2 = ts1 + 1;
2320 long ts3 = ts1 + 2;
2321
2322
2323 String method = this.getName();
2324 this.region = initHRegion(tableName, method, CONF, families);
2325 try {
2326
2327 Put put = null;
2328 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2329 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2330 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2331
2332 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2333 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2334 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2335
2336 put = new Put(row1);
2337 put.add(kv13);
2338 put.add(kv12);
2339 put.add(kv11);
2340 put.add(kv23);
2341 put.add(kv22);
2342 put.add(kv21);
2343 region.put(put);
2344
2345
2346 List<Cell> expected = new ArrayList<Cell>();
2347 expected.add(kv13);
2348 expected.add(kv12);
2349
2350 Scan scan = new Scan(row1);
2351 scan.addColumn(fam1, qf1);
2352 scan.setMaxVersions(MAX_VERSIONS);
2353 List<Cell> actual = new ArrayList<Cell>();
2354 InternalScanner scanner = region.getScanner(scan);
2355
2356 boolean hasNext = scanner.next(actual);
2357 assertEquals(false, hasNext);
2358
2359
2360 for (int i = 0; i < expected.size(); i++) {
2361 assertEquals(expected.get(i), actual.get(i));
2362 }
2363 } finally {
2364 HRegion.closeHRegion(this.region);
2365 this.region = null;
2366 }
2367 }
2368
2369 @Test
2370 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2371 byte[] row1 = Bytes.toBytes("row1");
2372 byte[] qf1 = Bytes.toBytes("qualifier1");
2373 byte[] qf2 = Bytes.toBytes("qualifier2");
2374 byte[] fam1 = Bytes.toBytes("fam1");
2375 byte[][] families = { fam1 };
2376
2377 long ts1 = 1;
2378 long ts2 = ts1 + 1;
2379 long ts3 = ts1 + 2;
2380
2381
2382 String method = this.getName();
2383 this.region = initHRegion(tableName, method, CONF, families);
2384 try {
2385
2386 Put put = null;
2387 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2388 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2389 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2390
2391 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2392 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2393 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2394
2395 put = new Put(row1);
2396 put.add(kv13);
2397 put.add(kv12);
2398 put.add(kv11);
2399 put.add(kv23);
2400 put.add(kv22);
2401 put.add(kv21);
2402 region.put(put);
2403 region.flushcache();
2404
2405
2406 List<Cell> expected = new ArrayList<Cell>();
2407 expected.add(kv13);
2408 expected.add(kv12);
2409 expected.add(kv23);
2410 expected.add(kv22);
2411
2412 Scan scan = new Scan(row1);
2413 scan.addColumn(fam1, qf1);
2414 scan.addColumn(fam1, qf2);
2415 scan.setMaxVersions(MAX_VERSIONS);
2416 List<Cell> actual = new ArrayList<Cell>();
2417 InternalScanner scanner = region.getScanner(scan);
2418
2419 boolean hasNext = scanner.next(actual);
2420 assertEquals(false, hasNext);
2421
2422
2423 for (int i = 0; i < expected.size(); i++) {
2424 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2425 }
2426 } finally {
2427 HRegion.closeHRegion(this.region);
2428 this.region = null;
2429 }
2430 }
2431
2432 @Test
2433 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2434 byte[] row1 = Bytes.toBytes("row1");
2435 byte[] fam1 = Bytes.toBytes("fam1");
2436 byte[][] families = { fam1 };
2437 byte[] qf1 = Bytes.toBytes("qualifier1");
2438 byte[] qf2 = Bytes.toBytes("qualifier2");
2439
2440 long ts1 = 1;
2441 long ts2 = ts1 + 1;
2442 long ts3 = ts1 + 2;
2443 long ts4 = ts1 + 3;
2444
2445
2446 String method = this.getName();
2447 this.region = initHRegion(tableName, method, CONF, families);
2448 try {
2449
2450 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2451 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2452 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2453 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2454
2455 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2456 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2457 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2458 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2459
2460 Put put = null;
2461 put = new Put(row1);
2462 put.add(kv14);
2463 put.add(kv24);
2464 region.put(put);
2465 region.flushcache();
2466
2467 put = new Put(row1);
2468 put.add(kv23);
2469 put.add(kv13);
2470 region.put(put);
2471 region.flushcache();
2472
2473 put = new Put(row1);
2474 put.add(kv22);
2475 put.add(kv12);
2476 region.put(put);
2477 region.flushcache();
2478
2479 put = new Put(row1);
2480 put.add(kv21);
2481 put.add(kv11);
2482 region.put(put);
2483
2484
2485 List<Cell> expected = new ArrayList<Cell>();
2486 expected.add(kv14);
2487 expected.add(kv13);
2488 expected.add(kv12);
2489 expected.add(kv24);
2490 expected.add(kv23);
2491 expected.add(kv22);
2492
2493 Scan scan = new Scan(row1);
2494 scan.addColumn(fam1, qf1);
2495 scan.addColumn(fam1, qf2);
2496 int versions = 3;
2497 scan.setMaxVersions(versions);
2498 List<Cell> actual = new ArrayList<Cell>();
2499 InternalScanner scanner = region.getScanner(scan);
2500
2501 boolean hasNext = scanner.next(actual);
2502 assertEquals(false, hasNext);
2503
2504
2505 for (int i = 0; i < expected.size(); i++) {
2506 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2507 }
2508 } finally {
2509 HRegion.closeHRegion(this.region);
2510 this.region = null;
2511 }
2512 }
2513
2514 @Test
2515 public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2516 byte[] row1 = Bytes.toBytes("row1");
2517 byte[] qf1 = Bytes.toBytes("qualifier1");
2518 byte[] qf2 = Bytes.toBytes("qualifier2");
2519 byte[] fam1 = Bytes.toBytes("fam1");
2520 byte[][] families = { fam1 };
2521
2522 long ts1 = System.currentTimeMillis();
2523 long ts2 = ts1 + 1;
2524 long ts3 = ts1 + 2;
2525
2526
2527 String method = this.getName();
2528 this.region = initHRegion(tableName, method, CONF, families);
2529 try {
2530
2531 Put put = null;
2532 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2533 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2534 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2535
2536 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2537 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2538 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2539
2540 put = new Put(row1);
2541 put.add(kv13);
2542 put.add(kv12);
2543 put.add(kv11);
2544 put.add(kv23);
2545 put.add(kv22);
2546 put.add(kv21);
2547 region.put(put);
2548
2549
2550 List<Cell> expected = new ArrayList<Cell>();
2551 expected.add(kv13);
2552 expected.add(kv12);
2553 expected.add(kv23);
2554 expected.add(kv22);
2555
2556 Scan scan = new Scan(row1);
2557 scan.addFamily(fam1);
2558 scan.setMaxVersions(MAX_VERSIONS);
2559 List<Cell> actual = new ArrayList<Cell>();
2560 InternalScanner scanner = region.getScanner(scan);
2561
2562 boolean hasNext = scanner.next(actual);
2563 assertEquals(false, hasNext);
2564
2565
2566 for (int i = 0; i < expected.size(); i++) {
2567 assertEquals(expected.get(i), actual.get(i));
2568 }
2569 } finally {
2570 HRegion.closeHRegion(this.region);
2571 this.region = null;
2572 }
2573 }
2574
2575 @Test
2576 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
2577 byte[] row1 = Bytes.toBytes("row1");
2578 byte[] qf1 = Bytes.toBytes("qualifier1");
2579 byte[] qf2 = Bytes.toBytes("qualifier2");
2580 byte[] fam1 = Bytes.toBytes("fam1");
2581
2582 long ts1 = 1;
2583 long ts2 = ts1 + 1;
2584 long ts3 = ts1 + 2;
2585
2586
2587 String method = this.getName();
2588 this.region = initHRegion(tableName, method, CONF, fam1);
2589 try {
2590
2591 Put put = null;
2592 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2593 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2594 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2595
2596 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2597 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2598 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2599
2600 put = new Put(row1);
2601 put.add(kv13);
2602 put.add(kv12);
2603 put.add(kv11);
2604 put.add(kv23);
2605 put.add(kv22);
2606 put.add(kv21);
2607 region.put(put);
2608 region.flushcache();
2609
2610
2611 List<Cell> expected = new ArrayList<Cell>();
2612 expected.add(kv13);
2613 expected.add(kv12);
2614 expected.add(kv23);
2615 expected.add(kv22);
2616
2617 Scan scan = new Scan(row1);
2618 scan.addFamily(fam1);
2619 scan.setMaxVersions(MAX_VERSIONS);
2620 List<Cell> actual = new ArrayList<Cell>();
2621 InternalScanner scanner = region.getScanner(scan);
2622
2623 boolean hasNext = scanner.next(actual);
2624 assertEquals(false, hasNext);
2625
2626
2627 for (int i = 0; i < expected.size(); i++) {
2628 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2629 }
2630 } finally {
2631 HRegion.closeHRegion(this.region);
2632 this.region = null;
2633 }
2634 }
2635
2636 @Test
2637 public void testScanner_StopRow1542() throws IOException {
2638 byte[] family = Bytes.toBytes("testFamily");
2639 this.region = initHRegion(tableName, getName(), CONF, family);
2640 try {
2641 byte[] row1 = Bytes.toBytes("row111");
2642 byte[] row2 = Bytes.toBytes("row222");
2643 byte[] row3 = Bytes.toBytes("row333");
2644 byte[] row4 = Bytes.toBytes("row444");
2645 byte[] row5 = Bytes.toBytes("row555");
2646
2647 byte[] col1 = Bytes.toBytes("Pub111");
2648 byte[] col2 = Bytes.toBytes("Pub222");
2649
2650 Put put = new Put(row1);
2651 put.add(family, col1, Bytes.toBytes(10L));
2652 region.put(put);
2653
2654 put = new Put(row2);
2655 put.add(family, col1, Bytes.toBytes(15L));
2656 region.put(put);
2657
2658 put = new Put(row3);
2659 put.add(family, col2, Bytes.toBytes(20L));
2660 region.put(put);
2661
2662 put = new Put(row4);
2663 put.add(family, col2, Bytes.toBytes(30L));
2664 region.put(put);
2665
2666 put = new Put(row5);
2667 put.add(family, col1, Bytes.toBytes(40L));
2668 region.put(put);
2669
2670 Scan scan = new Scan(row3, row4);
2671 scan.setMaxVersions();
2672 scan.addColumn(family, col1);
2673 InternalScanner s = region.getScanner(scan);
2674
2675 List<Cell> results = new ArrayList<Cell>();
2676 assertEquals(false, s.next(results));
2677 assertEquals(0, results.size());
2678 } finally {
2679 HRegion.closeHRegion(this.region);
2680 this.region = null;
2681 }
2682 }
2683
2684 @Test
2685 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2686 byte[] row1 = Bytes.toBytes("row1");
2687 byte[] fam1 = Bytes.toBytes("fam1");
2688 byte[] qf1 = Bytes.toBytes("qualifier1");
2689 byte[] qf2 = Bytes.toBytes("quateslifier2");
2690
2691 long ts1 = 1;
2692 long ts2 = ts1 + 1;
2693 long ts3 = ts1 + 2;
2694 long ts4 = ts1 + 3;
2695
2696
2697 String method = this.getName();
2698 this.region = initHRegion(tableName, method, CONF, fam1);
2699 try {
2700
2701 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2702 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2703 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2704 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2705
2706 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2707 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2708 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2709 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2710
2711 Put put = null;
2712 put = new Put(row1);
2713 put.add(kv14);
2714 put.add(kv24);
2715 region.put(put);
2716 region.flushcache();
2717
2718 put = new Put(row1);
2719 put.add(kv23);
2720 put.add(kv13);
2721 region.put(put);
2722 region.flushcache();
2723
2724 put = new Put(row1);
2725 put.add(kv22);
2726 put.add(kv12);
2727 region.put(put);
2728 region.flushcache();
2729
2730 put = new Put(row1);
2731 put.add(kv21);
2732 put.add(kv11);
2733 region.put(put);
2734
2735
2736 List<KeyValue> expected = new ArrayList<KeyValue>();
2737 expected.add(kv14);
2738 expected.add(kv13);
2739 expected.add(kv12);
2740 expected.add(kv24);
2741 expected.add(kv23);
2742 expected.add(kv22);
2743
2744 Scan scan = new Scan(row1);
2745 int versions = 3;
2746 scan.setMaxVersions(versions);
2747 List<Cell> actual = new ArrayList<Cell>();
2748 InternalScanner scanner = region.getScanner(scan);
2749
2750 boolean hasNext = scanner.next(actual);
2751 assertEquals(false, hasNext);
2752
2753
2754 for (int i = 0; i < expected.size(); i++) {
2755 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2756 }
2757 } finally {
2758 HRegion.closeHRegion(this.region);
2759 this.region = null;
2760 }
2761 }
2762
2763
2764
2765
2766
2767
2768
2769 @Test
2770 public void testScanner_JoinedScanners() throws IOException {
2771 byte[] cf_essential = Bytes.toBytes("essential");
2772 byte[] cf_joined = Bytes.toBytes("joined");
2773 byte[] cf_alpha = Bytes.toBytes("alpha");
2774 this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
2775 try {
2776 byte[] row1 = Bytes.toBytes("row1");
2777 byte[] row2 = Bytes.toBytes("row2");
2778 byte[] row3 = Bytes.toBytes("row3");
2779
2780 byte[] col_normal = Bytes.toBytes("d");
2781 byte[] col_alpha = Bytes.toBytes("a");
2782
2783 byte[] filtered_val = Bytes.toBytes(3);
2784
2785 Put put = new Put(row1);
2786 put.add(cf_essential, col_normal, Bytes.toBytes(1));
2787 put.add(cf_joined, col_alpha, Bytes.toBytes(1));
2788 region.put(put);
2789
2790 put = new Put(row2);
2791 put.add(cf_essential, col_alpha, Bytes.toBytes(2));
2792 put.add(cf_joined, col_normal, Bytes.toBytes(2));
2793 put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
2794 region.put(put);
2795
2796 put = new Put(row3);
2797 put.add(cf_essential, col_normal, filtered_val);
2798 put.add(cf_joined, col_normal, filtered_val);
2799 region.put(put);
2800
2801
2802
2803
2804
2805 Scan scan = new Scan();
2806 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
2807 CompareOp.NOT_EQUAL, filtered_val);
2808 scan.setFilter(filter);
2809 scan.setLoadColumnFamiliesOnDemand(true);
2810 InternalScanner s = region.getScanner(scan);
2811
2812 List<Cell> results = new ArrayList<Cell>();
2813 assertTrue(s.next(results));
2814 assertEquals(results.size(), 1);
2815 results.clear();
2816
2817 assertTrue(s.next(results));
2818 assertEquals(results.size(), 3);
2819 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
2820 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
2821 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
2822 results.clear();
2823
2824 assertFalse(s.next(results));
2825 assertEquals(results.size(), 0);
2826 } finally {
2827 HRegion.closeHRegion(this.region);
2828 this.region = null;
2829 }
2830 }
2831
2832
2833
2834
2835
2836
2837 @Test
2838 public void testScanner_JoinedScannersWithLimits() throws IOException {
2839 final byte[] cf_first = Bytes.toBytes("first");
2840 final byte[] cf_second = Bytes.toBytes("second");
2841
2842 this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
2843 try {
2844 final byte[] col_a = Bytes.toBytes("a");
2845 final byte[] col_b = Bytes.toBytes("b");
2846
2847 Put put;
2848
2849 for (int i = 0; i < 10; i++) {
2850 put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
2851 put.add(cf_first, col_a, Bytes.toBytes(i));
2852 if (i < 5) {
2853 put.add(cf_first, col_b, Bytes.toBytes(i));
2854 put.add(cf_second, col_a, Bytes.toBytes(i));
2855 put.add(cf_second, col_b, Bytes.toBytes(i));
2856 }
2857 region.put(put);
2858 }
2859
2860 Scan scan = new Scan();
2861 scan.setLoadColumnFamiliesOnDemand(true);
2862 Filter bogusFilter = new FilterBase() {
2863 @Override
2864 public boolean isFamilyEssential(byte[] name) {
2865 return Bytes.equals(name, cf_first);
2866 }
2867 };
2868
2869 scan.setFilter(bogusFilter);
2870 InternalScanner s = region.getScanner(scan);
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901 List<Cell> results = new ArrayList<Cell>();
2902 int index = 0;
2903 while (true) {
2904 boolean more = s.next(results, 3);
2905 if ((index >> 1) < 5) {
2906 if (index % 2 == 0)
2907 assertEquals(results.size(), 3);
2908 else
2909 assertEquals(results.size(), 1);
2910 } else
2911 assertEquals(results.size(), 1);
2912 results.clear();
2913 index++;
2914 if (!more)
2915 break;
2916 }
2917 } finally {
2918 HRegion.closeHRegion(this.region);
2919 this.region = null;
2920 }
2921 }
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931 @Test
2932 public void testBasicSplit() throws Exception {
2933 byte[][] families = { fam1, fam2, fam3 };
2934
2935 Configuration hc = initSplit();
2936
2937 String method = this.getName();
2938 this.region = initHRegion(tableName, method, hc, families);
2939
2940 try {
2941 LOG.info("" + HBaseTestCase.addContent(region, fam3));
2942 region.flushcache();
2943 region.compactStores();
2944 byte[] splitRow = region.checkSplit();
2945 assertNotNull(splitRow);
2946 LOG.info("SplitRow: " + Bytes.toString(splitRow));
2947 HRegion[] regions = splitRegion(region, splitRow);
2948 try {
2949
2950
2951
2952 for (int i = 0; i < regions.length; i++) {
2953 regions[i] = HRegion.openHRegion(regions[i], null);
2954 }
2955
2956
2957 assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
2958 assertGet(regions[1], fam3, splitRow);
2959
2960 assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
2961 assertScan(regions[1], fam3, splitRow);
2962
2963 for (int i = 0; i < regions.length; i++) {
2964
2965
2966 for (int j = 0; j < 2; j++) {
2967 HBaseTestCase.addContent(regions[i], fam3);
2968 }
2969 HBaseTestCase.addContent(regions[i], fam2);
2970 HBaseTestCase.addContent(regions[i], fam1);
2971 regions[i].flushcache();
2972 }
2973
2974 byte[][] midkeys = new byte[regions.length][];
2975
2976 for (int i = 0; i < regions.length; i++) {
2977 regions[i].compactStores();
2978 midkeys[i] = regions[i].checkSplit();
2979 }
2980
2981 TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
2982
2983
2984 for (int i = 0; i < regions.length; i++) {
2985 HRegion[] rs = null;
2986 if (midkeys[i] != null) {
2987 rs = splitRegion(regions[i], midkeys[i]);
2988 for (int j = 0; j < rs.length; j++) {
2989 sortedMap.put(Bytes.toString(rs[j].getRegionName()), HRegion.openHRegion(rs[j], null));
2990 }
2991 }
2992 }
2993 LOG.info("Made 4 regions");
2994
2995
2996 int interval = (LAST_CHAR - FIRST_CHAR) / 3;
2997 byte[] b = Bytes.toBytes(START_KEY);
2998 for (HRegion r : sortedMap.values()) {
2999 assertGet(r, fam3, b);
3000 b[0] += interval;
3001 }
3002 } finally {
3003 for (int i = 0; i < regions.length; i++) {
3004 try {
3005 regions[i].close();
3006 } catch (IOException e) {
3007
3008 }
3009 }
3010 }
3011 } finally {
3012 HRegion.closeHRegion(this.region);
3013 this.region = null;
3014 }
3015 }
3016
3017 @Test
3018 public void testSplitRegion() throws IOException {
3019 byte[] qualifier = Bytes.toBytes("qualifier");
3020 Configuration hc = initSplit();
3021 int numRows = 10;
3022 byte[][] families = { fam1, fam3 };
3023
3024
3025 String method = this.getName();
3026 this.region = initHRegion(tableName, method, hc, families);
3027
3028
3029 int startRow = 100;
3030 putData(startRow, numRows, qualifier, families);
3031 int splitRow = startRow + numRows;
3032 putData(splitRow, numRows, qualifier, families);
3033 region.flushcache();
3034
3035 HRegion[] regions = null;
3036 try {
3037 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
3038
3039 for (int i = 0; i < regions.length; i++) {
3040 regions[i] = HRegion.openHRegion(regions[i], null);
3041 }
3042
3043 assertEquals(2, regions.length);
3044
3045
3046
3047 verifyData(regions[0], startRow, numRows, qualifier, families);
3048 verifyData(regions[1], splitRow, numRows, qualifier, families);
3049
3050 } finally {
3051 HRegion.closeHRegion(this.region);
3052 this.region = null;
3053 }
3054 }
3055
3056 @Test
3057 public void testClearForceSplit() throws IOException {
3058 byte[] qualifier = Bytes.toBytes("qualifier");
3059 Configuration hc = initSplit();
3060 int numRows = 10;
3061 byte[][] families = { fam1, fam3 };
3062
3063
3064 String method = this.getName();
3065 this.region = initHRegion(tableName, method, hc, families);
3066
3067
3068 int startRow = 100;
3069 putData(startRow, numRows, qualifier, families);
3070 int splitRow = startRow + numRows;
3071 byte[] splitRowBytes = Bytes.toBytes("" + splitRow);
3072 putData(splitRow, numRows, qualifier, families);
3073 region.flushcache();
3074
3075 HRegion[] regions = null;
3076 try {
3077
3078 region.forceSplit(splitRowBytes);
3079 assertTrue(region.shouldForceSplit());
3080
3081 assertTrue(Bytes.equals(splitRowBytes, region.checkSplit()));
3082
3083
3084 HStore storeMock = Mockito.mock(HStore.class);
3085 Mockito.when(storeMock.hasReferences()).thenReturn(true);
3086 Mockito.when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
3087 Mockito.when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
3088 Mockito.when(storeMock.getColumnFamilyName()).thenReturn("cf");
3089 region.stores.put(Bytes.toBytes(storeMock.getColumnFamilyName()), storeMock);
3090 assertTrue(region.hasReferences());
3091
3092
3093 regions = splitRegion(region, splitRowBytes);
3094 assertNull(regions);
3095
3096
3097 assertFalse(region.shouldForceSplit());
3098
3099
3100 region.stores.remove(Bytes.toBytes(storeMock.getColumnFamilyName()));
3101 assertFalse(region.hasReferences());
3102
3103
3104 regions = splitRegion(region, splitRowBytes);
3105
3106
3107 for (int i = 0; i < regions.length; i++) {
3108 regions[i] = HRegion.openHRegion(regions[i], null);
3109 }
3110
3111 assertEquals(2, regions.length);
3112
3113
3114
3115 verifyData(regions[0], startRow, numRows, qualifier, families);
3116 verifyData(regions[1], splitRow, numRows, qualifier, families);
3117
3118 } finally {
3119 HRegion.closeHRegion(this.region);
3120 this.region = null;
3121 }
3122 }
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134 @Test
3135 public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3136 byte[] family = Bytes.toBytes("family");
3137 int numRows = 1000;
3138 int flushAndScanInterval = 10;
3139 int compactInterval = 10 * flushAndScanInterval;
3140
3141 String method = "testFlushCacheWhileScanning";
3142 this.region = initHRegion(tableName, method, CONF, family);
3143 try {
3144 FlushThread flushThread = new FlushThread();
3145 flushThread.start();
3146
3147 Scan scan = new Scan();
3148 scan.addFamily(family);
3149 scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
3150 new BinaryComparator(Bytes.toBytes(5L))));
3151
3152 int expectedCount = 0;
3153 List<Cell> res = new ArrayList<Cell>();
3154
3155 boolean toggle = true;
3156 for (long i = 0; i < numRows; i++) {
3157 Put put = new Put(Bytes.toBytes(i));
3158 put.setDurability(Durability.SKIP_WAL);
3159 put.add(family, qual1, Bytes.toBytes(i % 10));
3160 region.put(put);
3161
3162 if (i != 0 && i % compactInterval == 0) {
3163
3164 region.compactStores(true);
3165 }
3166
3167 if (i % 10 == 5L) {
3168 expectedCount++;
3169 }
3170
3171 if (i != 0 && i % flushAndScanInterval == 0) {
3172 res.clear();
3173 InternalScanner scanner = region.getScanner(scan);
3174 if (toggle) {
3175 flushThread.flush();
3176 }
3177 while (scanner.next(res))
3178 ;
3179 if (!toggle) {
3180 flushThread.flush();
3181 }
3182 assertEquals("i=" + i, expectedCount, res.size());
3183 toggle = !toggle;
3184 }
3185 }
3186
3187 flushThread.done();
3188 flushThread.join();
3189 flushThread.checkNoError();
3190 } finally {
3191 HRegion.closeHRegion(this.region);
3192 this.region = null;
3193 }
3194 }
3195
3196 protected class FlushThread extends Thread {
3197 private volatile boolean done;
3198 private Throwable error = null;
3199
3200 public void done() {
3201 done = true;
3202 synchronized (this) {
3203 interrupt();
3204 }
3205 }
3206
3207 public void checkNoError() {
3208 if (error != null) {
3209 assertNull(error);
3210 }
3211 }
3212
3213 @Override
3214 public void run() {
3215 done = false;
3216 while (!done) {
3217 synchronized (this) {
3218 try {
3219 wait();
3220 } catch (InterruptedException ignored) {
3221 if (done) {
3222 break;
3223 }
3224 }
3225 }
3226 try {
3227 region.flushcache();
3228 } catch (IOException e) {
3229 if (!done) {
3230 LOG.error("Error while flusing cache", e);
3231 error = e;
3232 }
3233 break;
3234 }
3235 }
3236
3237 }
3238
3239 public void flush() {
3240 synchronized (this) {
3241 notify();
3242 }
3243
3244 }
3245 }
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256 @Test
3257 public void testWritesWhileScanning() throws IOException, InterruptedException {
3258 int testCount = 100;
3259 int numRows = 1;
3260 int numFamilies = 10;
3261 int numQualifiers = 100;
3262 int flushInterval = 7;
3263 int compactInterval = 5 * flushInterval;
3264 byte[][] families = new byte[numFamilies][];
3265 for (int i = 0; i < numFamilies; i++) {
3266 families[i] = Bytes.toBytes("family" + i);
3267 }
3268 byte[][] qualifiers = new byte[numQualifiers][];
3269 for (int i = 0; i < numQualifiers; i++) {
3270 qualifiers[i] = Bytes.toBytes("qual" + i);
3271 }
3272
3273 String method = "testWritesWhileScanning";
3274 this.region = initHRegion(tableName, method, CONF, families);
3275 try {
3276 PutThread putThread = new PutThread(numRows, families, qualifiers);
3277 putThread.start();
3278 putThread.waitForFirstPut();
3279
3280 FlushThread flushThread = new FlushThread();
3281 flushThread.start();
3282
3283 Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3284
3285 int expectedCount = numFamilies * numQualifiers;
3286 List<Cell> res = new ArrayList<Cell>();
3287
3288 long prevTimestamp = 0L;
3289 for (int i = 0; i < testCount; i++) {
3290
3291 if (i != 0 && i % compactInterval == 0) {
3292 region.compactStores(true);
3293 }
3294
3295 if (i != 0 && i % flushInterval == 0) {
3296 flushThread.flush();
3297 }
3298
3299 boolean previousEmpty = res.isEmpty();
3300 res.clear();
3301 InternalScanner scanner = region.getScanner(scan);
3302 while (scanner.next(res))
3303 ;
3304 if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3305 assertEquals("i=" + i, expectedCount, res.size());
3306 long timestamp = res.get(0).getTimestamp();
3307 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3308 timestamp >= prevTimestamp);
3309 prevTimestamp = timestamp;
3310 }
3311 }
3312
3313 putThread.done();
3314
3315 region.flushcache();
3316
3317 putThread.join();
3318 putThread.checkNoError();
3319
3320 flushThread.done();
3321 flushThread.join();
3322 flushThread.checkNoError();
3323 } finally {
3324 try {
3325 HRegion.closeHRegion(this.region);
3326 } catch (DroppedSnapshotException dse) {
3327
3328
3329
3330
3331 }
3332 this.region = null;
3333 }
3334 }
3335
3336 protected class PutThread extends Thread {
3337 private volatile boolean done;
3338 private volatile int numPutsFinished = 0;
3339
3340 private Throwable error = null;
3341 private int numRows;
3342 private byte[][] families;
3343 private byte[][] qualifiers;
3344
3345 private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3346 this.numRows = numRows;
3347 this.families = families;
3348 this.qualifiers = qualifiers;
3349 }
3350
3351
3352
3353
3354 public void waitForFirstPut() throws InterruptedException {
3355
3356 while (numPutsFinished == 0) {
3357 checkNoError();
3358 Thread.sleep(50);
3359 }
3360 }
3361
3362 public void done() {
3363 done = true;
3364 synchronized (this) {
3365 interrupt();
3366 }
3367 }
3368
3369 public void checkNoError() {
3370 if (error != null) {
3371 assertNull(error);
3372 }
3373 }
3374
3375 @Override
3376 public void run() {
3377 done = false;
3378 while (!done) {
3379 try {
3380 for (int r = 0; r < numRows; r++) {
3381 byte[] row = Bytes.toBytes("row" + r);
3382 Put put = new Put(row);
3383 put.setDurability(Durability.SKIP_WAL);
3384 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3385 for (byte[] family : families) {
3386 for (byte[] qualifier : qualifiers) {
3387 put.add(family, qualifier, (long) numPutsFinished, value);
3388 }
3389 }
3390 region.put(put);
3391 numPutsFinished++;
3392 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3393 System.out.println("put iteration = " + numPutsFinished);
3394 Delete delete = new Delete(row, (long) numPutsFinished - 30);
3395 region.delete(delete);
3396 }
3397 numPutsFinished++;
3398 }
3399 } catch (InterruptedIOException e) {
3400
3401 } catch (IOException e) {
3402 LOG.error("error while putting records", e);
3403 error = e;
3404 break;
3405 }
3406 }
3407
3408 }
3409
3410 }
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421 @Test
3422 public void testWritesWhileGetting() throws Exception {
3423 int testCount = 100;
3424 int numRows = 1;
3425 int numFamilies = 10;
3426 int numQualifiers = 100;
3427 int compactInterval = 100;
3428 byte[][] families = new byte[numFamilies][];
3429 for (int i = 0; i < numFamilies; i++) {
3430 families[i] = Bytes.toBytes("family" + i);
3431 }
3432 byte[][] qualifiers = new byte[numQualifiers][];
3433 for (int i = 0; i < numQualifiers; i++) {
3434 qualifiers[i] = Bytes.toBytes("qual" + i);
3435 }
3436
3437
3438 String method = "testWritesWhileGetting";
3439
3440
3441
3442
3443
3444 Configuration conf = HBaseConfiguration.create(CONF);
3445 conf.setInt("hbase.hstore.compaction.min", 1);
3446 conf.setInt("hbase.hstore.compaction.max", 1000);
3447 this.region = initHRegion(tableName, method, conf, families);
3448 PutThread putThread = null;
3449 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3450 try {
3451 putThread = new PutThread(numRows, families, qualifiers);
3452 putThread.start();
3453 putThread.waitForFirstPut();
3454
3455
3456 ctx.addThread(new RepeatingTestThread(ctx) {
3457 private int flushesSinceCompact = 0;
3458 private final int maxFlushesSinceCompact = 20;
3459
3460 @Override
3461 public void doAnAction() throws Exception {
3462 if (region.flushcache().isCompactionNeeded()) {
3463 ++flushesSinceCompact;
3464 }
3465
3466
3467 if (flushesSinceCompact == maxFlushesSinceCompact) {
3468 region.compactStores(false);
3469 flushesSinceCompact = 0;
3470 }
3471 }
3472 });
3473 ctx.startThreads();
3474
3475 Get get = new Get(Bytes.toBytes("row0"));
3476 Result result = null;
3477
3478 int expectedCount = numFamilies * numQualifiers;
3479
3480 long prevTimestamp = 0L;
3481 for (int i = 0; i < testCount; i++) {
3482
3483 boolean previousEmpty = result == null || result.isEmpty();
3484 result = region.get(get);
3485 if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3486 assertEquals("i=" + i, expectedCount, result.size());
3487
3488
3489 long timestamp = 0;
3490 for (Cell kv : result.rawCells()) {
3491 if (CellUtil.matchingFamily(kv, families[0])
3492 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3493 timestamp = kv.getTimestamp();
3494 }
3495 }
3496 assertTrue(timestamp >= prevTimestamp);
3497 prevTimestamp = timestamp;
3498 Cell previousKV = null;
3499
3500 for (Cell kv : result.rawCells()) {
3501 byte[] thisValue = CellUtil.cloneValue(kv);
3502 if (previousKV != null) {
3503 if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3504 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3505 + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3506 + "(memStoreTS:" + kv.getMvccVersion() + ")");
3507 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3508 }
3509 }
3510 previousKV = kv;
3511 }
3512 }
3513 }
3514 } finally {
3515 if (putThread != null)
3516 putThread.done();
3517
3518 region.flushcache();
3519
3520 if (putThread != null) {
3521 putThread.join();
3522 putThread.checkNoError();
3523 }
3524
3525 ctx.stop();
3526 HRegion.closeHRegion(this.region);
3527 this.region = null;
3528 }
3529 }
3530
3531 @Test
3532 public void testHolesInMeta() throws Exception {
3533 byte[] family = Bytes.toBytes("family");
3534 this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF,
3535 false, family);
3536 try {
3537 byte[] rowNotServed = Bytes.toBytes("a");
3538 Get g = new Get(rowNotServed);
3539 try {
3540 region.get(g);
3541 fail();
3542 } catch (WrongRegionException x) {
3543
3544 }
3545 byte[] row = Bytes.toBytes("y");
3546 g = new Get(row);
3547 region.get(g);
3548 } finally {
3549 HRegion.closeHRegion(this.region);
3550 this.region = null;
3551 }
3552 }
3553
3554 @Test
3555 public void testIndexesScanWithOneDeletedRow() throws IOException {
3556 byte[] family = Bytes.toBytes("family");
3557
3558
3559 String method = "testIndexesScanWithOneDeletedRow";
3560 this.region = initHRegion(tableName, method, CONF, family);
3561 try {
3562 Put put = new Put(Bytes.toBytes(1L));
3563 put.add(family, qual1, 1L, Bytes.toBytes(1L));
3564 region.put(put);
3565
3566 region.flushcache();
3567
3568 Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3569 region.delete(delete);
3570
3571 put = new Put(Bytes.toBytes(2L));
3572 put.add(family, qual1, 2L, Bytes.toBytes(2L));
3573 region.put(put);
3574
3575 Scan idxScan = new Scan();
3576 idxScan.addFamily(family);
3577 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
3578 new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
3579 new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
3580 CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
3581 InternalScanner scanner = region.getScanner(idxScan);
3582 List<Cell> res = new ArrayList<Cell>();
3583
3584 while (scanner.next(res))
3585 ;
3586 assertEquals(1L, res.size());
3587 } finally {
3588 HRegion.closeHRegion(this.region);
3589 this.region = null;
3590 }
3591 }
3592
3593
3594
3595
3596 @Test
3597 public void testBloomFilterSize() throws IOException {
3598 byte[] fam1 = Bytes.toBytes("fam1");
3599 byte[] qf1 = Bytes.toBytes("col");
3600 byte[] val1 = Bytes.toBytes("value1");
3601
3602 HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
3603 .setBloomFilterType(BloomType.ROWCOL);
3604
3605 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3606 htd.addFamily(hcd);
3607 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3608 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3609 try {
3610 int num_unique_rows = 10;
3611 int duplicate_multiplier = 2;
3612 int num_storefiles = 4;
3613
3614 int version = 0;
3615 for (int f = 0; f < num_storefiles; f++) {
3616 for (int i = 0; i < duplicate_multiplier; i++) {
3617 for (int j = 0; j < num_unique_rows; j++) {
3618 Put put = new Put(Bytes.toBytes("row" + j));
3619 put.setDurability(Durability.SKIP_WAL);
3620 put.add(fam1, qf1, version++, val1);
3621 region.put(put);
3622 }
3623 }
3624 region.flushcache();
3625 }
3626
3627 HStore store = (HStore) region.getStore(fam1);
3628 Collection<StoreFile> storeFiles = store.getStorefiles();
3629 for (StoreFile storefile : storeFiles) {
3630 StoreFile.Reader reader = storefile.getReader();
3631 reader.loadFileInfo();
3632 reader.loadBloomfilter();
3633 assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
3634 assertEquals(num_unique_rows, reader.getFilterEntries());
3635 }
3636
3637 region.compactStores(true);
3638
3639
3640 storeFiles = store.getStorefiles();
3641 for (StoreFile storefile : storeFiles) {
3642 StoreFile.Reader reader = storefile.getReader();
3643 reader.loadFileInfo();
3644 reader.loadBloomfilter();
3645 assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
3646 assertEquals(num_unique_rows, reader.getFilterEntries());
3647 }
3648 } finally {
3649 HRegion.closeHRegion(this.region);
3650 this.region = null;
3651 }
3652 }
3653
3654 @Test
3655 public void testAllColumnsWithBloomFilter() throws IOException {
3656 byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
3657 byte[] FAMILY = Bytes.toBytes("family");
3658
3659
3660 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
3661 .setBloomFilterType(BloomType.ROWCOL);
3662 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
3663 htd.addFamily(hcd);
3664 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3665 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3666 try {
3667
3668 byte row[] = Bytes.toBytes("row:" + 0);
3669 byte column[] = Bytes.toBytes("column:" + 0);
3670 Put put = new Put(row);
3671 put.setDurability(Durability.SKIP_WAL);
3672 for (long idx = 1; idx <= 4; idx++) {
3673 put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
3674 }
3675 region.put(put);
3676
3677
3678 region.flushcache();
3679
3680
3681 Get get = new Get(row);
3682 get.setMaxVersions();
3683 Cell[] kvs = region.get(get).rawCells();
3684
3685
3686 assertEquals(4, kvs.length);
3687 checkOneCell(kvs[0], FAMILY, 0, 0, 4);
3688 checkOneCell(kvs[1], FAMILY, 0, 0, 3);
3689 checkOneCell(kvs[2], FAMILY, 0, 0, 2);
3690 checkOneCell(kvs[3], FAMILY, 0, 0, 1);
3691 } finally {
3692 HRegion.closeHRegion(this.region);
3693 this.region = null;
3694 }
3695 }
3696
3697
3698
3699
3700
3701
3702 @Test
3703 public void testDeleteRowWithBloomFilter() throws IOException {
3704 byte[] familyName = Bytes.toBytes("familyName");
3705
3706
3707 HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
3708 .setBloomFilterType(BloomType.ROWCOL);
3709
3710 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3711 htd.addFamily(hcd);
3712 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3713 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3714 try {
3715
3716 byte row[] = Bytes.toBytes("row1");
3717 byte col[] = Bytes.toBytes("col1");
3718
3719 Put put = new Put(row);
3720 put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
3721 region.put(put);
3722 region.flushcache();
3723
3724 Delete del = new Delete(row);
3725 region.delete(del);
3726 region.flushcache();
3727
3728
3729 Get get = new Get(row);
3730 get.addColumn(familyName, col);
3731
3732 Cell[] keyValues = region.get(get).rawCells();
3733 assertTrue(keyValues.length == 0);
3734 } finally {
3735 HRegion.closeHRegion(this.region);
3736 this.region = null;
3737 }
3738 }
3739
3740 @Test
3741 public void testgetHDFSBlocksDistribution() throws Exception {
3742 HBaseTestingUtility htu = new HBaseTestingUtility();
3743 final int DEFAULT_BLOCK_SIZE = 1024;
3744 htu.getConfiguration().setLong("dfs.namenode.fs-limits.min-block-size", 0);
3745 htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
3746 htu.getConfiguration().setInt("dfs.replication", 2);
3747
3748
3749 MiniHBaseCluster cluster = null;
3750 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
3751 int regionServersCount = 3;
3752
3753 try {
3754 cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
3755 byte[][] families = { fam1, fam2 };
3756 HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);
3757
3758
3759 byte row[] = Bytes.toBytes("row1");
3760 byte col[] = Bytes.toBytes("col1");
3761
3762 Put put = new Put(row);
3763 put.add(fam1, col, 1, Bytes.toBytes("test1"));
3764 put.add(fam2, col, 1, Bytes.toBytes("test2"));
3765 ht.put(put);
3766
3767 HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
3768 .get(0);
3769 firstRegion.flushcache();
3770 HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
3771
3772
3773
3774
3775
3776 long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
3777
3778 String topHost = blocksDistribution1.getTopHosts().get(0);
3779 long topHostWeight = blocksDistribution1.getWeight(topHost);
3780 assertTrue(uniqueBlocksWeight1 == topHostWeight);
3781
3782
3783
3784 HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
3785 htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
3786 long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
3787
3788 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
3789
3790 ht.close();
3791 } finally {
3792 if (cluster != null) {
3793 htu.shutdownMiniCluster();
3794 }
3795 }
3796 }
3797
3798
3799
3800
3801
3802
3803
3804 @Test
3805 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
3806 TableName tableName = TableName.valueOf(name.getMethodName());
3807 HRegionInfo info = null;
3808 try {
3809 FileSystem fs = Mockito.mock(FileSystem.class);
3810 Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
3811 HTableDescriptor htd = new HTableDescriptor(tableName);
3812 htd.addFamily(new HColumnDescriptor("cf"));
3813 info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
3814 HConstants.EMPTY_BYTE_ARRAY, false);
3815 Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
3816 region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
3817
3818 region.initialize();
3819 fail("Region initialization should fail due to IOException");
3820 } catch (IOException io) {
3821 List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
3822 for (MonitoredTask monitoredTask : tasks) {
3823 if (!(monitoredTask instanceof MonitoredRPCHandler)
3824 && monitoredTask.getDescription().contains(region.toString())) {
3825 assertTrue("Region state should be ABORTED.",
3826 monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
3827 break;
3828 }
3829 }
3830 } finally {
3831 HRegion.closeHRegion(region);
3832 }
3833 }
3834
3835
3836
3837
3838
3839 @Test
3840 public void testRegionInfoFileCreation() throws IOException {
3841 Path rootDir = new Path(dir + "testRegionInfoFileCreation");
3842
3843 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
3844 htd.addFamily(new HColumnDescriptor("cf"));
3845
3846 HRegionInfo hri = new HRegionInfo(htd.getTableName());
3847
3848
3849 HRegion region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
3850
3851 Path regionDir = region.getRegionFileSystem().getRegionDir();
3852 FileSystem fs = region.getRegionFileSystem().getFileSystem();
3853 HRegion.closeHRegion(region);
3854
3855 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
3856
3857
3858 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3859 fs.exists(regionInfoFile));
3860
3861
3862 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3863 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3864 HRegion.closeHRegion(region);
3865
3866
3867 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3868 fs.exists(regionInfoFile));
3869
3870
3871 fs.delete(regionInfoFile);
3872 assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
3873 fs.exists(regionInfoFile));
3874
3875 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3876
3877 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3878 HRegion.closeHRegion(region);
3879
3880
3881 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3882 fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
3883 }
3884
3885
3886
3887
3888 private static class Incrementer implements Runnable {
3889 private HRegion region;
3890 private final static byte[] incRow = Bytes.toBytes("incRow");
3891 private final static byte[] family = Bytes.toBytes("family");
3892 private final static byte[] qualifier = Bytes.toBytes("qualifier");
3893 private final static long ONE = 1l;
3894 private int incCounter;
3895
3896 public Incrementer(HRegion region, int incCounter) {
3897 this.region = region;
3898 this.incCounter = incCounter;
3899 }
3900
3901 @Override
3902 public void run() {
3903 int count = 0;
3904 while (count < incCounter) {
3905 Increment inc = new Increment(incRow);
3906 inc.addColumn(family, qualifier, ONE);
3907 count++;
3908 try {
3909 region.increment(inc);
3910 } catch (IOException e) {
3911 e.printStackTrace();
3912 break;
3913 }
3914 }
3915 }
3916 }
3917
3918
3919
3920
3921
3922 @Test
3923 public void testParallelIncrementWithMemStoreFlush() throws Exception {
3924 byte[] family = Incrementer.family;
3925 this.region = initHRegion(tableName, method, CONF, family);
3926 final HRegion region = this.region;
3927 final AtomicBoolean incrementDone = new AtomicBoolean(false);
3928 Runnable flusher = new Runnable() {
3929 @Override
3930 public void run() {
3931 while (!incrementDone.get()) {
3932 try {
3933 region.flushcache();
3934 } catch (Exception e) {
3935 e.printStackTrace();
3936 }
3937 }
3938 }
3939 };
3940
3941
3942 int threadNum = 20;
3943 int incCounter = 100;
3944 long expected = threadNum * incCounter;
3945 Thread[] incrementers = new Thread[threadNum];
3946 Thread flushThread = new Thread(flusher);
3947 for (int i = 0; i < threadNum; i++) {
3948 incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
3949 incrementers[i].start();
3950 }
3951 flushThread.start();
3952 for (int i = 0; i < threadNum; i++) {
3953 incrementers[i].join();
3954 }
3955
3956 incrementDone.set(true);
3957 flushThread.join();
3958
3959 Get get = new Get(Incrementer.incRow);
3960 get.addColumn(Incrementer.family, Incrementer.qualifier);
3961 get.setMaxVersions(1);
3962 Result res = this.region.get(get);
3963 List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
3964
3965
3966 assertEquals(kvs.size(), 1);
3967 Cell kv = kvs.get(0);
3968 assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
3969 this.region = null;
3970 }
3971
3972
3973
3974
3975 private static class Appender implements Runnable {
3976 private HRegion region;
3977 private final static byte[] appendRow = Bytes.toBytes("appendRow");
3978 private final static byte[] family = Bytes.toBytes("family");
3979 private final static byte[] qualifier = Bytes.toBytes("qualifier");
3980 private final static byte[] CHAR = Bytes.toBytes("a");
3981 private int appendCounter;
3982
3983 public Appender(HRegion region, int appendCounter) {
3984 this.region = region;
3985 this.appendCounter = appendCounter;
3986 }
3987
3988 @Override
3989 public void run() {
3990 int count = 0;
3991 while (count < appendCounter) {
3992 Append app = new Append(appendRow);
3993 app.add(family, qualifier, CHAR);
3994 count++;
3995 try {
3996 region.append(app);
3997 } catch (IOException e) {
3998 e.printStackTrace();
3999 break;
4000 }
4001 }
4002 }
4003 }
4004
4005
4006
4007
4008
4009 @Test
4010 public void testParallelAppendWithMemStoreFlush() throws Exception {
4011 byte[] family = Appender.family;
4012 this.region = initHRegion(tableName, method, CONF, family);
4013 final HRegion region = this.region;
4014 final AtomicBoolean appendDone = new AtomicBoolean(false);
4015 Runnable flusher = new Runnable() {
4016 @Override
4017 public void run() {
4018 while (!appendDone.get()) {
4019 try {
4020 region.flushcache();
4021 } catch (Exception e) {
4022 e.printStackTrace();
4023 }
4024 }
4025 }
4026 };
4027
4028
4029
4030 int threadNum = 20;
4031 int appendCounter = 100;
4032 byte[] expected = new byte[threadNum * appendCounter];
4033 for (int i = 0; i < threadNum * appendCounter; i++) {
4034 System.arraycopy(Appender.CHAR, 0, expected, i, 1);
4035 }
4036 Thread[] appenders = new Thread[threadNum];
4037 Thread flushThread = new Thread(flusher);
4038 for (int i = 0; i < threadNum; i++) {
4039 appenders[i] = new Thread(new Appender(this.region, appendCounter));
4040 appenders[i].start();
4041 }
4042 flushThread.start();
4043 for (int i = 0; i < threadNum; i++) {
4044 appenders[i].join();
4045 }
4046
4047 appendDone.set(true);
4048 flushThread.join();
4049
4050 Get get = new Get(Appender.appendRow);
4051 get.addColumn(Appender.family, Appender.qualifier);
4052 get.setMaxVersions(1);
4053 Result res = this.region.get(get);
4054 List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
4055
4056
4057 assertEquals(kvs.size(), 1);
4058 Cell kv = kvs.get(0);
4059 byte[] appendResult = new byte[kv.getValueLength()];
4060 System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
4061 assertArrayEquals(expected, appendResult);
4062 this.region = null;
4063 }
4064
4065
4066
4067
4068
4069 @Test
4070 public void testPutWithMemStoreFlush() throws Exception {
4071 byte[] family = Bytes.toBytes("family");
4072 ;
4073 byte[] qualifier = Bytes.toBytes("qualifier");
4074 byte[] row = Bytes.toBytes("putRow");
4075 byte[] value = null;
4076 this.region = initHRegion(tableName, method, CONF, family);
4077 Put put = null;
4078 Get get = null;
4079 List<Cell> kvs = null;
4080 Result res = null;
4081
4082 put = new Put(row);
4083 value = Bytes.toBytes("value0");
4084 put.add(family, qualifier, 1234567l, value);
4085 region.put(put);
4086 get = new Get(row);
4087 get.addColumn(family, qualifier);
4088 get.setMaxVersions();
4089 res = this.region.get(get);
4090 kvs = res.getColumnCells(family, qualifier);
4091 assertEquals(1, kvs.size());
4092 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4093
4094 region.flushcache();
4095 get = new Get(row);
4096 get.addColumn(family, qualifier);
4097 get.setMaxVersions();
4098 res = this.region.get(get);
4099 kvs = res.getColumnCells(family, qualifier);
4100 assertEquals(1, kvs.size());
4101 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4102
4103 put = new Put(row);
4104 value = Bytes.toBytes("value1");
4105 put.add(family, qualifier, 1234567l, value);
4106 region.put(put);
4107 get = new Get(row);
4108 get.addColumn(family, qualifier);
4109 get.setMaxVersions();
4110 res = this.region.get(get);
4111 kvs = res.getColumnCells(family, qualifier);
4112 assertEquals(1, kvs.size());
4113 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4114
4115 region.flushcache();
4116 get = new Get(row);
4117 get.addColumn(family, qualifier);
4118 get.setMaxVersions();
4119 res = this.region.get(get);
4120 kvs = res.getColumnCells(family, qualifier);
4121 assertEquals(1, kvs.size());
4122 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4123 }
4124
4125 @Test
4126 public void testDurability() throws Exception {
4127 String method = "testDurability";
4128
4129
4130
4131
4132
4133 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4134 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4135 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4136
4137 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4138 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4139 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4140
4141 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4142 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4143
4144 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
4145 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4146
4147 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
4148 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
4149 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
4150
4151
4152 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4153 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4154 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4155 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4156 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
4157 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4158
4159 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4160 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4161 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4162 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4163 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
4164 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
4165
4166
4167 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4168 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4169 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4170 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
4171 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
4172 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
4173
4174 }
4175
4176 private void durabilityTest(String method, Durability tableDurability,
4177 Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
4178 final boolean expectSyncFromLogSyncer) throws Exception {
4179 Configuration conf = HBaseConfiguration.create(CONF);
4180 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
4181 TableName tableName = TableName.valueOf(method);
4182 byte[] family = Bytes.toBytes("family");
4183 Path logDir = new Path(new Path(dir + method), "log");
4184 HLog hlog = HLogFactory.createHLog(FILESYSTEM, logDir, UUID.randomUUID().toString(), conf);
4185 final HLog log = spy(hlog);
4186 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
4187 HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, log,
4188 new byte[][] { family });
4189
4190 Put put = new Put(Bytes.toBytes("r1"));
4191 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
4192 put.setDurability(mutationDurability);
4193 region.put(put);
4194
4195
4196 verify(log, expectAppend ? times(1) : never())
4197 .appendNoSync((HRegionInfo)any(), eq(tableName), (WALEdit)any(), (List<UUID>)any(),
4198 anyLong(), (HTableDescriptor)any(), (AtomicLong)any(), anyBoolean(), anyLong(), anyLong());
4199
4200
4201 if (expectSync || expectSyncFromLogSyncer) {
4202 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
4203 @Override
4204 public boolean evaluate() throws Exception {
4205 try {
4206 if (expectSync) {
4207 verify(log, times(1)).sync(anyLong());
4208 } else if (expectSyncFromLogSyncer) {
4209 verify(log, times(1)).sync();
4210 }
4211 } catch (Throwable ignore) {
4212 }
4213 return true;
4214 }
4215 });
4216 } else {
4217 verify(log, never()).sync(anyLong());
4218 verify(log, never()).sync();
4219 }
4220
4221 HRegion.closeHRegion(this.region);
4222 this.region = null;
4223 }
4224
4225 private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4226 for (int i = startRow; i < startRow + numRows; i++) {
4227 Put put = new Put(Bytes.toBytes("" + i));
4228 put.setDurability(Durability.SKIP_WAL);
4229 for (byte[] family : families) {
4230 put.add(family, qf, null);
4231 }
4232 region.put(put);
4233 }
4234 }
4235
4236 private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
4237 throws IOException {
4238 for (int i = startRow; i < startRow + numRows; i++) {
4239 byte[] row = Bytes.toBytes("" + i);
4240 Get get = new Get(row);
4241 for (byte[] family : families) {
4242 get.addColumn(family, qf);
4243 }
4244 Result result = newReg.get(get);
4245 Cell[] raw = result.rawCells();
4246 assertEquals(families.length, result.size());
4247 for (int j = 0; j < families.length; j++) {
4248 assertTrue(CellUtil.matchingRow(raw[j], row));
4249 assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
4250 assertTrue(CellUtil.matchingQualifier(raw[j], qf));
4251 }
4252 }
4253 }
4254
4255 private void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
4256
4257 Get get = new Get(k).addFamily(family).setMaxVersions();
4258 Cell[] results = r.get(get).rawCells();
4259 for (int j = 0; j < results.length; j++) {
4260 byte[] tmp = CellUtil.cloneValue(results[j]);
4261
4262 assertTrue(Bytes.equals(k, tmp));
4263 }
4264 }
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277 private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
4278 throws IOException {
4279 byte[][] families = { fs };
4280 Scan scan = new Scan();
4281 for (int i = 0; i < families.length; i++)
4282 scan.addFamily(families[i]);
4283 InternalScanner s = r.getScanner(scan);
4284 try {
4285 List<Cell> curVals = new ArrayList<Cell>();
4286 boolean first = true;
4287 OUTER_LOOP: while (s.next(curVals)) {
4288 for (Cell kv : curVals) {
4289 byte[] val = CellUtil.cloneValue(kv);
4290 byte[] curval = val;
4291 if (first) {
4292 first = false;
4293 assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4294 } else {
4295
4296 break OUTER_LOOP;
4297 }
4298 }
4299 }
4300 } finally {
4301 s.close();
4302 }
4303 }
4304
4305
4306
4307
4308
4309 @Test
4310 public void testFlushResult() throws IOException {
4311 String method = name.getMethodName();
4312 byte[] tableName = Bytes.toBytes(method);
4313 byte[] family = Bytes.toBytes("family");
4314
4315 this.region = initHRegion(tableName, method, family);
4316
4317
4318 HRegion.FlushResult fr = region.flushcache();
4319 assertFalse(fr.isFlushSucceeded());
4320 assertFalse(fr.isCompactionNeeded());
4321
4322
4323 for (int i = 0; i < 2; i++) {
4324 Put put = new Put(tableName).add(family, family, tableName);
4325 region.put(put);
4326 fr = region.flushcache();
4327 assertTrue(fr.isFlushSucceeded());
4328 assertFalse(fr.isCompactionNeeded());
4329 }
4330
4331
4332 for (int i = 0; i < 2; i++) {
4333 Put put = new Put(tableName).add(family, family, tableName);
4334 region.put(put);
4335 fr = region.flushcache();
4336 assertTrue(fr.isFlushSucceeded());
4337 assertTrue(fr.isCompactionNeeded());
4338 }
4339 }
4340
4341 private Configuration initSplit() {
4342
4343 CONF.setInt("hbase.hstore.compactionThreshold", 2);
4344
4345
4346 CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4347
4348 CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4349
4350
4351 CONF.setLong("hbase.client.pause", 15 * 1000);
4352
4353
4354
4355 CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4356 return CONF;
4357 }
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368 public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
4369 byte[]... families) throws IOException {
4370 return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
4371 }
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4383 byte[]... families) throws IOException {
4384 return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4385 }
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4398 boolean isReadOnly, byte[]... families) throws IOException {
4399 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4400 }
4401
4402 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4403 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4404 throws IOException {
4405 return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
4406 Durability.SYNC_WAL, null, families);
4407 }
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4422 String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4423 HLog hlog, byte[]... families) throws IOException {
4424 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly, durability, hlog, families);
4425 }
4426
4427
4428
4429
4430
4431 private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
4432 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4433 assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
4434 Bytes.toString(CellUtil.cloneRow(kv)));
4435 assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
4436 Bytes.toString(CellUtil.cloneFamily(kv)));
4437 assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
4438 Bytes.toString(CellUtil.cloneQualifier(kv)));
4439 assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
4440 assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
4441 Bytes.toString(CellUtil.cloneValue(kv)));
4442 }
4443
4444 public void testReverseScanner_FromMemStore_SingleCF_Normal()
4445 throws IOException {
4446 byte[] rowC = Bytes.toBytes("rowC");
4447 byte[] rowA = Bytes.toBytes("rowA");
4448 byte[] rowB = Bytes.toBytes("rowB");
4449 byte[] cf = Bytes.toBytes("CF");
4450 byte[][] families = { cf };
4451 byte[] col = Bytes.toBytes("C");
4452 long ts = 1;
4453 String method = this.getName();
4454 this.region = initHRegion(tableName, method, families);
4455 try {
4456 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4457 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4458 null);
4459 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4460 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4461 Put put = null;
4462 put = new Put(rowC);
4463 put.add(kv1);
4464 put.add(kv11);
4465 region.put(put);
4466 put = new Put(rowA);
4467 put.add(kv2);
4468 region.put(put);
4469 put = new Put(rowB);
4470 put.add(kv3);
4471 region.put(put);
4472
4473 Scan scan = new Scan(rowC);
4474 scan.setMaxVersions(5);
4475 scan.setReversed(true);
4476 InternalScanner scanner = region.getScanner(scan);
4477 List<Cell> currRow = new ArrayList<Cell>();
4478 boolean hasNext = scanner.next(currRow);
4479 assertEquals(2, currRow.size());
4480 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4481 assertTrue(hasNext);
4482 currRow.clear();
4483 hasNext = scanner.next(currRow);
4484 assertEquals(1, currRow.size());
4485 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4486 assertTrue(hasNext);
4487 currRow.clear();
4488 hasNext = scanner.next(currRow);
4489 assertEquals(1, currRow.size());
4490 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4491 assertFalse(hasNext);
4492 scanner.close();
4493 } finally {
4494 HRegion.closeHRegion(this.region);
4495 this.region = null;
4496 }
4497 }
4498
4499 public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
4500 throws IOException {
4501 byte[] rowC = Bytes.toBytes("rowC");
4502 byte[] rowA = Bytes.toBytes("rowA");
4503 byte[] rowB = Bytes.toBytes("rowB");
4504 byte[] rowD = Bytes.toBytes("rowD");
4505 byte[] cf = Bytes.toBytes("CF");
4506 byte[][] families = { cf };
4507 byte[] col = Bytes.toBytes("C");
4508 long ts = 1;
4509 String method = this.getName();
4510 this.region = initHRegion(tableName, method, families);
4511 try {
4512 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4513 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4514 null);
4515 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4516 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4517 Put put = null;
4518 put = new Put(rowC);
4519 put.add(kv1);
4520 put.add(kv11);
4521 region.put(put);
4522 put = new Put(rowA);
4523 put.add(kv2);
4524 region.put(put);
4525 put = new Put(rowB);
4526 put.add(kv3);
4527 region.put(put);
4528
4529 Scan scan = new Scan(rowD);
4530 List<Cell> currRow = new ArrayList<Cell>();
4531 scan.setReversed(true);
4532 scan.setMaxVersions(5);
4533 InternalScanner scanner = region.getScanner(scan);
4534 boolean hasNext = scanner.next(currRow);
4535 assertEquals(2, currRow.size());
4536 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4537 assertTrue(hasNext);
4538 currRow.clear();
4539 hasNext = scanner.next(currRow);
4540 assertEquals(1, currRow.size());
4541 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4542 assertTrue(hasNext);
4543 currRow.clear();
4544 hasNext = scanner.next(currRow);
4545 assertEquals(1, currRow.size());
4546 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4547 assertFalse(hasNext);
4548 scanner.close();
4549 } finally {
4550 HRegion.closeHRegion(this.region);
4551 this.region = null;
4552 }
4553 }
4554
4555 public void testReverseScanner_FromMemStore_SingleCF_FullScan()
4556 throws IOException {
4557 byte[] rowC = Bytes.toBytes("rowC");
4558 byte[] rowA = Bytes.toBytes("rowA");
4559 byte[] rowB = Bytes.toBytes("rowB");
4560 byte[] cf = Bytes.toBytes("CF");
4561 byte[][] families = { cf };
4562 byte[] col = Bytes.toBytes("C");
4563 long ts = 1;
4564 String method = this.getName();
4565 this.region = initHRegion(tableName, method, families);
4566 try {
4567 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4568 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4569 null);
4570 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4571 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4572 Put put = null;
4573 put = new Put(rowC);
4574 put.add(kv1);
4575 put.add(kv11);
4576 region.put(put);
4577 put = new Put(rowA);
4578 put.add(kv2);
4579 region.put(put);
4580 put = new Put(rowB);
4581 put.add(kv3);
4582 region.put(put);
4583 Scan scan = new Scan();
4584 List<Cell> currRow = new ArrayList<Cell>();
4585 scan.setReversed(true);
4586 InternalScanner scanner = region.getScanner(scan);
4587 boolean hasNext = scanner.next(currRow);
4588 assertEquals(1, currRow.size());
4589 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4590 assertTrue(hasNext);
4591 currRow.clear();
4592 hasNext = scanner.next(currRow);
4593 assertEquals(1, currRow.size());
4594 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4595 assertTrue(hasNext);
4596 currRow.clear();
4597 hasNext = scanner.next(currRow);
4598 assertEquals(1, currRow.size());
4599 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4600 assertFalse(hasNext);
4601 scanner.close();
4602 } finally {
4603 HRegion.closeHRegion(this.region);
4604 this.region = null;
4605 }
4606 }
4607
4608 public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
4609
4610 byte[] rowA = Bytes.toBytes("rowA");
4611 byte[] rowB = Bytes.toBytes("rowB");
4612 byte[] rowC = Bytes.toBytes("rowC");
4613 byte[] rowD = Bytes.toBytes("rowD");
4614 byte[] rowE = Bytes.toBytes("rowE");
4615 byte[] cf = Bytes.toBytes("CF");
4616 byte[][] families = { cf };
4617 byte[] col1 = Bytes.toBytes("col1");
4618 byte[] col2 = Bytes.toBytes("col2");
4619 long ts = 1;
4620 String method = this.getName();
4621 this.region = initHRegion(tableName, method, families);
4622 try {
4623 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4624 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4625 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4626 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4627 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4628 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4629 Put put = null;
4630 put = new Put(rowA);
4631 put.add(kv1);
4632 region.put(put);
4633 put = new Put(rowB);
4634 put.add(kv2);
4635 region.put(put);
4636 put = new Put(rowC);
4637 put.add(kv3);
4638 region.put(put);
4639 put = new Put(rowD);
4640 put.add(kv4_1);
4641 region.put(put);
4642 put = new Put(rowD);
4643 put.add(kv4_2);
4644 region.put(put);
4645 put = new Put(rowE);
4646 put.add(kv5);
4647 region.put(put);
4648 region.flushcache();
4649 Scan scan = new Scan(rowD, rowA);
4650 scan.addColumn(families[0], col1);
4651 scan.setReversed(true);
4652 List<Cell> currRow = new ArrayList<Cell>();
4653 InternalScanner scanner = region.getScanner(scan);
4654 boolean hasNext = scanner.next(currRow);
4655 assertEquals(1, currRow.size());
4656 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4657 assertTrue(hasNext);
4658 currRow.clear();
4659 hasNext = scanner.next(currRow);
4660 assertEquals(1, currRow.size());
4661 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4662 assertTrue(hasNext);
4663 currRow.clear();
4664 hasNext = scanner.next(currRow);
4665 assertEquals(1, currRow.size());
4666 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4667 assertFalse(hasNext);
4668 scanner.close();
4669
4670 scan = new Scan(rowD, rowA);
4671 scan.addColumn(families[0], col2);
4672 scan.setReversed(true);
4673 currRow.clear();
4674 scanner = region.getScanner(scan);
4675 hasNext = scanner.next(currRow);
4676 assertEquals(1, currRow.size());
4677 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4678 scanner.close();
4679 } finally {
4680 HRegion.closeHRegion(this.region);
4681 this.region = null;
4682 }
4683 }
4684
4685 public void testReverseScanner_smaller_blocksize() throws IOException {
4686
4687 byte[] rowA = Bytes.toBytes("rowA");
4688 byte[] rowB = Bytes.toBytes("rowB");
4689 byte[] rowC = Bytes.toBytes("rowC");
4690 byte[] rowD = Bytes.toBytes("rowD");
4691 byte[] rowE = Bytes.toBytes("rowE");
4692 byte[] cf = Bytes.toBytes("CF");
4693 byte[][] families = { cf };
4694 byte[] col1 = Bytes.toBytes("col1");
4695 byte[] col2 = Bytes.toBytes("col2");
4696 long ts = 1;
4697 String method = this.getName();
4698 HBaseConfiguration config = new HBaseConfiguration();
4699 config.setInt("test.block.size", 1);
4700 this.region = initHRegion(tableName, method, config, families);
4701 try {
4702 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4703 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4704 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4705 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4706 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4707 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4708 Put put = null;
4709 put = new Put(rowA);
4710 put.add(kv1);
4711 region.put(put);
4712 put = new Put(rowB);
4713 put.add(kv2);
4714 region.put(put);
4715 put = new Put(rowC);
4716 put.add(kv3);
4717 region.put(put);
4718 put = new Put(rowD);
4719 put.add(kv4_1);
4720 region.put(put);
4721 put = new Put(rowD);
4722 put.add(kv4_2);
4723 region.put(put);
4724 put = new Put(rowE);
4725 put.add(kv5);
4726 region.put(put);
4727 region.flushcache();
4728 Scan scan = new Scan(rowD, rowA);
4729 scan.addColumn(families[0], col1);
4730 scan.setReversed(true);
4731 List<Cell> currRow = new ArrayList<Cell>();
4732 InternalScanner scanner = region.getScanner(scan);
4733 boolean hasNext = scanner.next(currRow);
4734 assertEquals(1, currRow.size());
4735 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4736 assertTrue(hasNext);
4737 currRow.clear();
4738 hasNext = scanner.next(currRow);
4739 assertEquals(1, currRow.size());
4740 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4741 assertTrue(hasNext);
4742 currRow.clear();
4743 hasNext = scanner.next(currRow);
4744 assertEquals(1, currRow.size());
4745 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4746 assertFalse(hasNext);
4747 scanner.close();
4748
4749 scan = new Scan(rowD, rowA);
4750 scan.addColumn(families[0], col2);
4751 scan.setReversed(true);
4752 currRow.clear();
4753 scanner = region.getScanner(scan);
4754 hasNext = scanner.next(currRow);
4755 assertEquals(1, currRow.size());
4756 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4757 scanner.close();
4758 } finally {
4759 HRegion.closeHRegion(this.region);
4760 this.region = null;
4761 }
4762 }
4763
4764 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
4765 throws IOException {
4766 byte[] row0 = Bytes.toBytes("row0");
4767 byte[] row1 = Bytes.toBytes("row1");
4768 byte[] row2 = Bytes.toBytes("row2");
4769 byte[] row3 = Bytes.toBytes("row3");
4770 byte[] row4 = Bytes.toBytes("row4");
4771 byte[] row5 = Bytes.toBytes("row5");
4772 byte[] cf1 = Bytes.toBytes("CF1");
4773 byte[] cf2 = Bytes.toBytes("CF2");
4774 byte[] cf3 = Bytes.toBytes("CF3");
4775 byte[][] families = { cf1, cf2, cf3 };
4776 byte[] col = Bytes.toBytes("C");
4777 long ts = 1;
4778 String method = this.getName();
4779 HBaseConfiguration conf = new HBaseConfiguration();
4780
4781 conf.setInt("hbase.hstore.compactionThreshold", 10000);
4782 this.region = initHRegion(tableName, method, conf, families);
4783 try {
4784
4785 KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
4786 null);
4787 KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
4788 null);
4789 KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
4790 KeyValue.Type.Put, null);
4791 KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
4792 null);
4793 KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
4794 null);
4795 KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
4796 null);
4797 KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
4798 KeyValue.Type.Put, null);
4799 KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
4800 null);
4801 KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
4802 KeyValue.Type.Put, null);
4803 KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
4804 null);
4805 KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
4806 null);
4807 KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
4808 KeyValue.Type.Put, null);
4809 KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
4810 null);
4811 KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
4812 KeyValue.Type.Put, null);
4813 KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
4814 null);
4815 KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
4816 null);
4817
4818 Put put = null;
4819 put = new Put(row1);
4820 put.add(kv1_2_1);
4821 region.put(put);
4822 put = new Put(row2);
4823 put.add(kv2_4_1);
4824 region.put(put);
4825 put = new Put(row4);
4826 put.add(kv4_5_4);
4827 put.add(kv4_5_5);
4828 region.put(put);
4829 region.flushcache();
4830
4831 put = new Put(row4);
4832 put.add(kv4_5_1);
4833 put.add(kv4_5_3);
4834 region.put(put);
4835 put = new Put(row1);
4836 put.add(kv1_2_2);
4837 region.put(put);
4838 put = new Put(row2);
4839 put.add(kv2_4_4);
4840 region.put(put);
4841 region.flushcache();
4842
4843 put = new Put(row4);
4844 put.add(kv4_5_2);
4845 region.put(put);
4846 put = new Put(row2);
4847 put.add(kv2_4_2);
4848 put.add(kv2_4_3);
4849 region.put(put);
4850 put = new Put(row3);
4851 put.add(kv3_2_2);
4852 region.put(put);
4853 region.flushcache();
4854
4855
4856 put = new Put(row0);
4857 put.add(kv0_1_1);
4858 region.put(put);
4859 put = new Put(row3);
4860 put.add(kv3_2_1);
4861 region.put(put);
4862 put = new Put(row5);
4863 put.add(kv5_2_1);
4864 put.add(kv5_2_2);
4865 region.put(put);
4866
4867 Scan scan = new Scan(row4);
4868 scan.setMaxVersions(5);
4869 scan.setBatch(3);
4870 scan.setReversed(true);
4871 InternalScanner scanner = region.getScanner(scan);
4872 List<Cell> currRow = new ArrayList<Cell>();
4873 boolean hasNext = false;
4874
4875
4876
4877 hasNext = scanner.next(currRow);
4878 assertEquals(3, currRow.size());
4879 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4880 assertTrue(hasNext);
4881 currRow.clear();
4882 hasNext = scanner.next(currRow);
4883 assertEquals(2, currRow.size());
4884 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4885 assertTrue(hasNext);
4886
4887 currRow.clear();
4888 hasNext = scanner.next(currRow);
4889 assertEquals(2, currRow.size());
4890 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
4891 assertTrue(hasNext);
4892
4893
4894 currRow.clear();
4895 hasNext = scanner.next(currRow);
4896 assertEquals(3, currRow.size());
4897 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4898 assertTrue(hasNext);
4899 currRow.clear();
4900 hasNext = scanner.next(currRow);
4901 assertEquals(1, currRow.size());
4902 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4903 assertTrue(hasNext);
4904
4905 currRow.clear();
4906 hasNext = scanner.next(currRow);
4907 assertEquals(2, currRow.size());
4908 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
4909 assertTrue(hasNext);
4910
4911 currRow.clear();
4912 hasNext = scanner.next(currRow);
4913 assertEquals(1, currRow.size());
4914 assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
4915 assertFalse(hasNext);
4916
4917 scanner.close();
4918 } finally {
4919 HRegion.closeHRegion(this.region);
4920 this.region = null;
4921 }
4922 }
4923
4924 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
4925 throws IOException {
4926 byte[] row1 = Bytes.toBytes("row1");
4927 byte[] row2 = Bytes.toBytes("row2");
4928 byte[] row3 = Bytes.toBytes("row3");
4929 byte[] row4 = Bytes.toBytes("row4");
4930 byte[] cf1 = Bytes.toBytes("CF1");
4931 byte[] cf2 = Bytes.toBytes("CF2");
4932 byte[] cf3 = Bytes.toBytes("CF3");
4933 byte[] cf4 = Bytes.toBytes("CF4");
4934 byte[][] families = { cf1, cf2, cf3, cf4 };
4935 byte[] col = Bytes.toBytes("C");
4936 long ts = 1;
4937 String method = this.getName();
4938 HBaseConfiguration conf = new HBaseConfiguration();
4939
4940 conf.setInt("hbase.hstore.compactionThreshold", 10000);
4941 this.region = initHRegion(tableName, method, conf, families);
4942 try {
4943 KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
4944 KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
4945 KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
4946 KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
4947
4948 Put put = new Put(row1);
4949 put.add(kv1);
4950 region.put(put);
4951 region.flushcache();
4952
4953 put = new Put(row2);
4954 put.add(kv2);
4955 region.put(put);
4956 region.flushcache();
4957
4958 put = new Put(row3);
4959 put.add(kv3);
4960 region.put(put);
4961 region.flushcache();
4962
4963 put = new Put(row4);
4964 put.add(kv4);
4965 region.put(put);
4966
4967 Scan scan = new Scan(row4);
4968 scan.setReversed(true);
4969 scan.setBatch(10);
4970 InternalScanner scanner = region.getScanner(scan);
4971 List<Cell> currRow = new ArrayList<Cell>();
4972 boolean hasNext = scanner.next(currRow);
4973 assertEquals(1, currRow.size());
4974 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4975 assertTrue(hasNext);
4976 currRow.clear();
4977 hasNext = scanner.next(currRow);
4978 assertEquals(1, currRow.size());
4979 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
4980 assertTrue(hasNext);
4981 currRow.clear();
4982 hasNext = scanner.next(currRow);
4983 assertEquals(1, currRow.size());
4984 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4985 assertTrue(hasNext);
4986 currRow.clear();
4987 hasNext = scanner.next(currRow);
4988 assertEquals(1, currRow.size());
4989 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
4990 assertFalse(hasNext);
4991 } finally {
4992 HRegion.closeHRegion(this.region);
4993 this.region = null;
4994 }
4995 }
4996
4997 @Test
4998 public void testWriteRequestsCounter() throws IOException {
4999 byte[] fam = Bytes.toBytes("info");
5000 byte[][] families = { fam };
5001 this.region = initHRegion(tableName, method, CONF, families);
5002
5003 Assert.assertEquals(0L, region.getWriteRequestsCount());
5004
5005 Put put = new Put(row);
5006 put.add(fam, fam, fam);
5007
5008 Assert.assertEquals(0L, region.getWriteRequestsCount());
5009 region.put(put);
5010 Assert.assertEquals(1L, region.getWriteRequestsCount());
5011 region.put(put);
5012 Assert.assertEquals(2L, region.getWriteRequestsCount());
5013 region.put(put);
5014 Assert.assertEquals(3L, region.getWriteRequestsCount());
5015
5016 region.delete(new Delete(row));
5017 Assert.assertEquals(4L, region.getWriteRequestsCount());
5018
5019 HRegion.closeHRegion(this.region);
5020 this.region = null;
5021 }
5022
5023 private static HRegion initHRegion(byte[] tableName, String callingMethod,
5024 byte[]... families) throws IOException {
5025 return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
5026 families);
5027 }
5028 }