1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21
22 import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23 import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24 import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25 import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29 import static org.junit.Assert.assertArrayEquals;
30 import static org.junit.Assert.assertEquals;
31 import static org.junit.Assert.assertFalse;
32 import static org.junit.Assert.assertNotNull;
33 import static org.junit.Assert.assertNull;
34 import static org.junit.Assert.assertTrue;
35 import static org.junit.Assert.fail;
36 import static org.mockito.Matchers.any;
37 import static org.mockito.Matchers.anyBoolean;
38 import static org.mockito.Matchers.anyLong;
39 import static org.mockito.Matchers.eq;
40 import static org.mockito.Mockito.never;
41 import static org.mockito.Mockito.spy;
42 import static org.mockito.Mockito.times;
43 import static org.mockito.Mockito.verify;
44
45 import java.io.IOException;
46 import java.io.InterruptedIOException;
47 import java.security.PrivilegedExceptionAction;
48 import java.util.ArrayList;
49 import java.util.Arrays;
50 import java.util.Collection;
51 import java.util.List;
52 import java.util.Map;
53 import java.util.NavigableMap;
54 import java.util.TreeMap;
55 import java.util.UUID;
56 import java.util.concurrent.atomic.AtomicBoolean;
57 import java.util.concurrent.atomic.AtomicInteger;
58 import java.util.concurrent.atomic.AtomicLong;
59 import java.util.concurrent.atomic.AtomicReference;
60
61 import org.apache.commons.logging.Log;
62 import org.apache.commons.logging.LogFactory;
63 import org.apache.hadoop.conf.Configuration;
64 import org.apache.hadoop.fs.FSDataOutputStream;
65 import org.apache.hadoop.fs.FileStatus;
66 import org.apache.hadoop.fs.FileSystem;
67 import org.apache.hadoop.fs.Path;
68 import org.apache.hadoop.hbase.Cell;
69 import org.apache.hadoop.hbase.CellComparator;
70 import org.apache.hadoop.hbase.CellUtil;
71 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
72 import org.apache.hadoop.hbase.DroppedSnapshotException;
73 import org.apache.hadoop.hbase.HBaseConfiguration;
74 import org.apache.hadoop.hbase.HBaseTestCase;
75 import org.apache.hadoop.hbase.HBaseTestingUtility;
76 import org.apache.hadoop.hbase.HColumnDescriptor;
77 import org.apache.hadoop.hbase.HConstants;
78 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
79 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
80 import org.apache.hadoop.hbase.HRegionInfo;
81 import org.apache.hadoop.hbase.HTableDescriptor;
82 import org.apache.hadoop.hbase.KeyValue;
83 import org.apache.hadoop.hbase.MediumTests;
84 import org.apache.hadoop.hbase.MiniHBaseCluster;
85 import org.apache.hadoop.hbase.MultithreadedTestUtil;
86 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
87 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
88 import org.apache.hadoop.hbase.NotServingRegionException;
89 import org.apache.hadoop.hbase.TableName;
90 import org.apache.hadoop.hbase.Waiter;
91 import org.apache.hadoop.hbase.client.Append;
92 import org.apache.hadoop.hbase.client.Delete;
93 import org.apache.hadoop.hbase.client.Durability;
94 import org.apache.hadoop.hbase.client.Get;
95 import org.apache.hadoop.hbase.client.HTable;
96 import org.apache.hadoop.hbase.client.Increment;
97 import org.apache.hadoop.hbase.client.Put;
98 import org.apache.hadoop.hbase.client.Result;
99 import org.apache.hadoop.hbase.client.Scan;
100 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
101 import org.apache.hadoop.hbase.filter.BinaryComparator;
102 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
103 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
104 import org.apache.hadoop.hbase.filter.Filter;
105 import org.apache.hadoop.hbase.filter.FilterBase;
106 import org.apache.hadoop.hbase.filter.FilterList;
107 import org.apache.hadoop.hbase.filter.NullComparator;
108 import org.apache.hadoop.hbase.filter.PrefixFilter;
109 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
110 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
111 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
112 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
113 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
114 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
115 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
116 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
117 import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
118 import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
119 import org.apache.hadoop.hbase.regionserver.wal.FaultyHLog;
120 import org.apache.hadoop.hbase.regionserver.wal.HLog;
121 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
122 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
123 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
124 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
125 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
126 import org.apache.hadoop.hbase.security.User;
127 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
128 import org.apache.hadoop.hbase.util.Bytes;
129 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
130 import org.apache.hadoop.hbase.util.FSUtils;
131 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
132 import org.apache.hadoop.hbase.util.PairOfSameType;
133 import org.apache.hadoop.hbase.util.Threads;
134 import org.junit.After;
135 import org.junit.Assert;
136 import org.junit.Before;
137 import org.junit.Rule;
138 import org.junit.Test;
139 import org.junit.experimental.categories.Category;
140 import org.junit.rules.TestName;
141 import org.mockito.Mockito;
142
143 import com.google.common.collect.Lists;
144 import com.google.protobuf.ByteString;
145
146
147
148
149
150
151
152 @Category(MediumTests.class)
153 @SuppressWarnings("deprecation")
154 public class TestHRegion {
155
156
157 static final Log LOG = LogFactory.getLog(TestHRegion.class);
158 @Rule public TestName name = new TestName();
159
160 private static final String COLUMN_FAMILY = "MyCF";
161 private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
162
163 HRegion region = null;
164
165 private static HBaseTestingUtility TEST_UTIL;
166 public static Configuration CONF ;
167 private String dir;
168 private static FileSystem FILESYSTEM;
169 private final int MAX_VERSIONS = 2;
170
171
172 protected byte[] tableName;
173 protected String method;
174 protected final byte[] qual1 = Bytes.toBytes("qual1");
175 protected final byte[] qual2 = Bytes.toBytes("qual2");
176 protected final byte[] qual3 = Bytes.toBytes("qual3");
177 protected final byte[] value1 = Bytes.toBytes("value1");
178 protected final byte[] value2 = Bytes.toBytes("value2");
179 protected final byte[] row = Bytes.toBytes("rowA");
180 protected final byte[] row2 = Bytes.toBytes("rowB");
181
182 protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
183 .getInstance(MetricsAssertHelper.class);
184
185 @Before
186 public void setup() throws IOException {
187 TEST_UTIL = HBaseTestingUtility.createLocalHTU();
188 FILESYSTEM = TEST_UTIL.getTestFileSystem();
189 CONF = TEST_UTIL.getConfiguration();
190 dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
191 method = name.getMethodName();
192 tableName = Bytes.toBytes(name.getMethodName());
193 }
194
195 @After
196 public void tearDown() throws Exception {
197 EnvironmentEdgeManagerTestHelper.reset();
198 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
199 TEST_UTIL.cleanupTestDir();
200 }
201
202 String getName() {
203 return name.getMethodName();
204 }
205
206
207
208
209
210
211
212
213
214
215
216 @Test (timeout=60000)
217 public void testCloseCarryingSnapshot() throws IOException {
218 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
219 Store store = region.getStore(COLUMN_FAMILY_BYTES);
220
221 byte [] value = Bytes.toBytes(name.getMethodName());
222
223 Put put = new Put(value);
224 put.add(COLUMN_FAMILY_BYTES, null, value);
225
226 region.put(put);
227 StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
228 storeFlushCtx.prepare();
229
230 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
231 region.put(put);
232
233 region.close();
234 assertEquals(0, region.getMemstoreSize().get());
235 HRegion.closeHRegion(region);
236 }
237
238
239
240
241
242 @Test (timeout=60000)
243 public void testMemstoreSnapshotSize() throws IOException {
244 class MyFaultyHLog extends FaultyHLog {
245 StoreFlushContext storeFlushCtx;
246 public MyFaultyHLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
247 throws IOException {
248 super(fs, rootDir, logName, conf);
249 }
250
251 void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
252 this.storeFlushCtx = storeFlushCtx;
253 }
254
255 @Override
256 public void sync(long txid) throws IOException {
257 storeFlushCtx.prepare();
258 super.sync(txid);
259 }
260 }
261
262 FileSystem fs = FileSystem.get(CONF);
263 Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
264 MyFaultyHLog faultyLog = new MyFaultyHLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
265 HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
266 CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES);
267
268 Store store = region.getStore(COLUMN_FAMILY_BYTES);
269
270 byte [] value = Bytes.toBytes(name.getMethodName());
271 faultyLog.setStoreFlushCtx(store.createFlushContext(12345));
272
273 Put put = new Put(value);
274 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
275 faultyLog.setFailureType(FaultyHLog.FailureType.SYNC);
276
277 boolean threwIOE = false;
278 try {
279 region.put(put);
280 } catch (IOException ioe) {
281 threwIOE = true;
282 } finally {
283 assertTrue("The regionserver should have thrown an exception", threwIOE);
284 }
285 long sz = store.getFlushableSize();
286 assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
287 HRegion.closeHRegion(region);
288 }
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305 @Test (timeout=60000)
306 public void testFlushSizeAccounting() throws Exception {
307 final Configuration conf = HBaseConfiguration.create(CONF);
308
309 conf.setInt("hbase.hstore.flush.retries.number", 1);
310 final User user =
311 User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
312
313 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
314 user.runAs(new PrivilegedExceptionAction<Object>() {
315 @Override
316 public Object run() throws Exception {
317
318 FileSystem fs = FileSystem.get(conf);
319 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
320 FaultyFileSystem ffs = (FaultyFileSystem)fs;
321 HRegion region = null;
322 try {
323
324 region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
325 long size = region.getMemstoreSize().get();
326 Assert.assertEquals(0, size);
327
328 Put p1 = new Put(row);
329 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
330 region.put(p1);
331 final long sizeOfOnePut = region.getMemstoreSize().get();
332
333 try {
334 LOG.info("Flushing");
335 region.flushcache();
336 Assert.fail("Didn't bubble up IOE!");
337 } catch (DroppedSnapshotException dse) {
338
339 }
340
341 ffs.fault.set(false);
342
343 Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize().get());
344
345
346 Put p2 = new Put(row);
347 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
348 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
349 region.put(p2);
350 Assert.assertEquals(sizeOfOnePut * 3, region.getMemstoreSize().get());
351
352
353
354 region.flushcache();
355
356 Assert.assertEquals(sizeOfOnePut * 2, region.getMemstoreSize().get());
357 } finally {
358 HRegion.closeHRegion(region);
359 }
360 return null;
361 }
362 });
363 FileSystem.closeAllForUGI(user.getUGI());
364 }
365
366 @Test
367 public void testCompactionAffectedByScanners() throws Exception {
368 byte[] family = Bytes.toBytes("family");
369 this.region = initHRegion(tableName, method, CONF, family);
370
371 Put put = new Put(Bytes.toBytes("r1"));
372 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
373 region.put(put);
374 region.flushcache();
375
376 Scan scan = new Scan();
377 scan.setMaxVersions(3);
378
379 RegionScanner scanner1 = region.getScanner(scan);
380
381 Delete delete = new Delete(Bytes.toBytes("r1"));
382 region.delete(delete);
383 region.flushcache();
384
385
386 RegionScanner scanner2 = region.getScanner(scan);
387
388 List<Cell> results = new ArrayList<Cell>();
389
390 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
391
392
393 region.compactStores(true);
394
395
396 RegionScanner scanner3 = region.getScanner(scan);
397
398
399 scanner1.next(results);
400 System.out.println(results);
401 assertEquals(1, results.size());
402
403 results.clear();
404 scanner2.next(results);
405 System.out.println(results);
406 assertEquals(0, results.size());
407
408 results.clear();
409 scanner3.next(results);
410 System.out.println(results);
411 assertEquals(0, results.size());
412 }
413
414 @Test
415 public void testToShowNPEOnRegionScannerReseek() throws Exception {
416 byte[] family = Bytes.toBytes("family");
417 this.region = initHRegion(tableName, method, CONF, family);
418
419 Put put = new Put(Bytes.toBytes("r1"));
420 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
421 region.put(put);
422 put = new Put(Bytes.toBytes("r2"));
423 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
424 region.put(put);
425 region.flushcache();
426
427 Scan scan = new Scan();
428 scan.setMaxVersions(3);
429
430 RegionScanner scanner1 = region.getScanner(scan);
431
432 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
433
434 region.compactStores(true);
435
436 scanner1.reseek(Bytes.toBytes("r2"));
437 List<Cell> results = new ArrayList<Cell>();
438 scanner1.next(results);
439 Cell keyValue = results.get(0);
440 Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
441 scanner1.close();
442 }
443
444 @Test
445 public void testSkipRecoveredEditsReplay() throws Exception {
446 String method = "testSkipRecoveredEditsReplay";
447 TableName tableName = TableName.valueOf(method);
448 byte[] family = Bytes.toBytes("family");
449 this.region = initHRegion(tableName, method, CONF, family);
450 try {
451 Path regiondir = region.getRegionFileSystem().getRegionDir();
452 FileSystem fs = region.getRegionFileSystem().getFileSystem();
453 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
454
455 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
456
457 long maxSeqId = 1050;
458 long minSeqId = 1000;
459
460 for (long i = minSeqId; i <= maxSeqId; i += 10) {
461 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
462 fs.create(recoveredEdits);
463 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
464
465 long time = System.nanoTime();
466 WALEdit edit = new WALEdit();
467 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
468 .toBytes(i)));
469 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
470 HConstants.DEFAULT_CLUSTER_ID), edit));
471
472 writer.close();
473 }
474 MonitoredTask status = TaskMonitor.get().createStatus(method);
475 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
476 for (Store store : region.getStores().values()) {
477 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
478 }
479 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
480 assertEquals(maxSeqId, seqId);
481 Get get = new Get(row);
482 Result result = region.get(get);
483 for (long i = minSeqId; i <= maxSeqId; i += 10) {
484 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
485 assertEquals(1, kvs.size());
486 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
487 }
488 } finally {
489 HRegion.closeHRegion(this.region);
490 this.region = null;
491 }
492 }
493
494 @Test
495 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
496 String method = "testSkipRecoveredEditsReplaySomeIgnored";
497 TableName tableName = TableName.valueOf(method);
498 byte[] family = Bytes.toBytes("family");
499 this.region = initHRegion(tableName, method, CONF, family);
500 try {
501 Path regiondir = region.getRegionFileSystem().getRegionDir();
502 FileSystem fs = region.getRegionFileSystem().getFileSystem();
503 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
504
505 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
506
507 long maxSeqId = 1050;
508 long minSeqId = 1000;
509
510 for (long i = minSeqId; i <= maxSeqId; i += 10) {
511 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
512 fs.create(recoveredEdits);
513 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
514
515 long time = System.nanoTime();
516 WALEdit edit = new WALEdit();
517 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
518 .toBytes(i)));
519 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
520 HConstants.DEFAULT_CLUSTER_ID), edit));
521
522 writer.close();
523 }
524 long recoverSeqId = 1030;
525 MonitoredTask status = TaskMonitor.get().createStatus(method);
526 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
527 for (Store store : region.getStores().values()) {
528 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
529 }
530 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
531 assertEquals(maxSeqId, seqId);
532 Get get = new Get(row);
533 Result result = region.get(get);
534 for (long i = minSeqId; i <= maxSeqId; i += 10) {
535 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
536 if (i < recoverSeqId) {
537 assertEquals(0, kvs.size());
538 } else {
539 assertEquals(1, kvs.size());
540 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
541 }
542 }
543 } finally {
544 HRegion.closeHRegion(this.region);
545 this.region = null;
546 }
547 }
548
549 @Test
550 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
551 byte[] family = Bytes.toBytes("family");
552 this.region = initHRegion(tableName, method, CONF, family);
553 try {
554 Path regiondir = region.getRegionFileSystem().getRegionDir();
555 FileSystem fs = region.getRegionFileSystem().getFileSystem();
556
557 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
558 for (int i = 1000; i < 1050; i += 10) {
559 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
560 FSDataOutputStream dos = fs.create(recoveredEdits);
561 dos.writeInt(i);
562 dos.close();
563 }
564 long minSeqId = 2000;
565 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
566 FSDataOutputStream dos = fs.create(recoveredEdits);
567 dos.close();
568
569 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
570 for (Store store : region.getStores().values()) {
571 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
572 }
573 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
574 assertEquals(minSeqId, seqId);
575 } finally {
576 HRegion.closeHRegion(this.region);
577 this.region = null;
578 }
579 }
580
581 @Test
582 public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
583 String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
584 TableName tableName = TableName.valueOf(method);
585 byte[] family = Bytes.toBytes("family");
586 this.region = initHRegion(tableName, method, CONF, family);
587 try {
588 Path regiondir = region.getRegionFileSystem().getRegionDir();
589 FileSystem fs = region.getRegionFileSystem().getFileSystem();
590 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
591
592 assertEquals(0, region.getStoreFileList(
593 region.getStores().keySet().toArray(new byte[0][])).size());
594
595 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
596
597 long maxSeqId = 1050;
598 long minSeqId = 1000;
599
600 for (long i = minSeqId; i <= maxSeqId; i += 10) {
601 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
602 fs.create(recoveredEdits);
603 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
604
605 long time = System.nanoTime();
606 WALEdit edit = null;
607 if (i == maxSeqId) {
608 edit = WALEdit.createCompaction(CompactionDescriptor.newBuilder()
609 .setTableName(ByteString.copyFrom(tableName.getName()))
610 .setFamilyName(ByteString.copyFrom(regionName))
611 .setEncodedRegionName(ByteString.copyFrom(regionName))
612 .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
613 .build());
614 } else {
615 edit = new WALEdit();
616 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
617 .toBytes(i)));
618 }
619 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
620 HConstants.DEFAULT_CLUSTER_ID), edit));
621 writer.close();
622 }
623
624 long recoverSeqId = 1030;
625 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
626 MonitoredTask status = TaskMonitor.get().createStatus(method);
627 for (Store store : region.getStores().values()) {
628 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
629 }
630 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
631 assertEquals(maxSeqId, seqId);
632
633
634 assertEquals(1, region.getStoreFileList(
635 region.getStores().keySet().toArray(new byte[0][])).size());
636
637 } finally {
638 HRegion.closeHRegion(this.region);
639 this.region = null;
640 } }
641
642 @Test
643 public void testRecoveredEditsReplayCompaction() throws Exception {
644 String method = name.getMethodName();
645 TableName tableName = TableName.valueOf(method);
646 byte[] family = Bytes.toBytes("family");
647 this.region = initHRegion(tableName, method, CONF, family);
648 try {
649 Path regiondir = region.getRegionFileSystem().getRegionDir();
650 FileSystem fs = region.getRegionFileSystem().getFileSystem();
651 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
652
653 long maxSeqId = 3;
654 long minSeqId = 0;
655
656 for (long i = minSeqId; i < maxSeqId; i++) {
657 Put put = new Put(Bytes.toBytes(i));
658 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
659 region.put(put);
660 region.flushcache();
661 }
662
663
664 assertEquals(3, region.getStore(family).getStorefilesCount());
665 List<Path> storeFiles = new ArrayList<Path>(3);
666 for (StoreFile sf : region.getStore(family).getStorefiles()) {
667 storeFiles.add(sf.getPath());
668 }
669
670
671 CONF.setBoolean("hbase.hstore.compaction.complete", false);
672 region.compactStores();
673
674
675 assertEquals(3, region.getStore(family).getStorefilesCount());
676
677
678 Path tmpDir = region.getRegionFileSystem().getTempDir();
679 FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
680 String errorMsg = "Expected to find 1 file in the region temp directory "
681 + "from the compaction, could not find any";
682 assertNotNull(errorMsg, files);
683 assertEquals(errorMsg, 1, files.length);
684
685 Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
686 files[0].getPath());
687
688 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
689 .getRegionInfo(), family, storeFiles, Lists.newArrayList(newFile), region
690 .getRegionFileSystem().getStoreDir(Bytes.toString(family)));
691
692 HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
693 this.region.getRegionInfo(), compactionDescriptor, new AtomicLong(1));
694
695 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
696
697 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
698 fs.create(recoveredEdits);
699 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
700
701 long time = System.nanoTime();
702
703 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, 10, time,
704 HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(compactionDescriptor)));
705 writer.close();
706
707
708 region.getTableDesc();
709 region.getRegionInfo();
710 region.close();
711 region = HRegion.openHRegion(region, null);
712
713
714 Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
715 for (StoreFile sf : sfs) {
716 LOG.info(sf.getPath());
717 }
718 assertEquals(1, region.getStore(family).getStorefilesCount());
719 files = FSUtils.listStatus(fs, tmpDir);
720 assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
721
722 for (long i = minSeqId; i < maxSeqId; i++) {
723 Get get = new Get(Bytes.toBytes(i));
724 Result result = region.get(get);
725 byte[] value = result.getValue(family, Bytes.toBytes(i));
726 assertArrayEquals(Bytes.toBytes(i), value);
727 }
728 } finally {
729 HRegion.closeHRegion(this.region);
730 this.region = null;
731 }
732 }
733
734 @Test
735 public void testGetWhileRegionClose() throws IOException {
736 TableName tableName = TableName.valueOf(name.getMethodName());
737 Configuration hc = initSplit();
738 int numRows = 100;
739 byte[][] families = { fam1, fam2, fam3 };
740
741
742 String method = name.getMethodName();
743 this.region = initHRegion(tableName, method, hc, families);
744 try {
745
746 final int startRow = 100;
747 putData(startRow, numRows, qual1, families);
748 putData(startRow, numRows, qual2, families);
749 putData(startRow, numRows, qual3, families);
750 final AtomicBoolean done = new AtomicBoolean(false);
751 final AtomicInteger gets = new AtomicInteger(0);
752 GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
753 try {
754
755 for (int i = 0; i < threads.length / 2; i++) {
756 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
757 threads[i].setDaemon(true);
758 threads[i].start();
759 }
760
761
762 this.region.closing.set(true);
763 for (int i = threads.length / 2; i < threads.length; i++) {
764 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
765 threads[i].setDaemon(true);
766 threads[i].start();
767 }
768 } finally {
769 if (this.region != null) {
770 HRegion.closeHRegion(this.region);
771 }
772 }
773 done.set(true);
774 for (GetTillDoneOrException t : threads) {
775 try {
776 t.join();
777 } catch (InterruptedException e) {
778 e.printStackTrace();
779 }
780 if (t.e != null) {
781 LOG.info("Exception=" + t.e);
782 assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
783 }
784 }
785 } finally {
786 HRegion.closeHRegion(this.region);
787 this.region = null;
788 }
789 }
790
791
792
793
794
795 class GetTillDoneOrException extends Thread {
796 private final Get g;
797 private final AtomicBoolean done;
798 private final AtomicInteger count;
799 private Exception e;
800
801 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
802 super("getter." + i);
803 this.g = new Get(r);
804 this.done = d;
805 this.count = c;
806 }
807
808 @Override
809 public void run() {
810 while (!this.done.get()) {
811 try {
812 assertTrue(region.get(g).size() > 0);
813 this.count.incrementAndGet();
814 } catch (Exception e) {
815 this.e = e;
816 break;
817 }
818 }
819 }
820 }
821
822
823
824
825 @Test
826 public void testWeirdCacheBehaviour() throws Exception {
827 byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
828 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
829 Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
830 this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
831 try {
832 String value = "this is the value";
833 String value2 = "this is some other value";
834 String keyPrefix1 = "prefix1";
835 String keyPrefix2 = "prefix2";
836 String keyPrefix3 = "prefix3";
837 putRows(this.region, 3, value, keyPrefix1);
838 putRows(this.region, 3, value, keyPrefix2);
839 putRows(this.region, 3, value, keyPrefix3);
840 putRows(this.region, 3, value2, keyPrefix1);
841 putRows(this.region, 3, value2, keyPrefix2);
842 putRows(this.region, 3, value2, keyPrefix3);
843 System.out.println("Checking values for key: " + keyPrefix1);
844 assertEquals("Got back incorrect number of rows from scan", 3,
845 getNumberOfRows(keyPrefix1, value2, this.region));
846 System.out.println("Checking values for key: " + keyPrefix2);
847 assertEquals("Got back incorrect number of rows from scan", 3,
848 getNumberOfRows(keyPrefix2, value2, this.region));
849 System.out.println("Checking values for key: " + keyPrefix3);
850 assertEquals("Got back incorrect number of rows from scan", 3,
851 getNumberOfRows(keyPrefix3, value2, this.region));
852 deleteColumns(this.region, value2, keyPrefix1);
853 deleteColumns(this.region, value2, keyPrefix2);
854 deleteColumns(this.region, value2, keyPrefix3);
855 System.out.println("Starting important checks.....");
856 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
857 getNumberOfRows(keyPrefix1, value2, this.region));
858 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
859 getNumberOfRows(keyPrefix2, value2, this.region));
860 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
861 getNumberOfRows(keyPrefix3, value2, this.region));
862 } finally {
863 HRegion.closeHRegion(this.region);
864 this.region = null;
865 }
866 }
867
868 @Test
869 public void testAppendWithReadOnlyTable() throws Exception {
870 byte[] TABLE = Bytes.toBytes("readOnlyTable");
871 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
872 boolean exceptionCaught = false;
873 Append append = new Append(Bytes.toBytes("somerow"));
874 append.setDurability(Durability.SKIP_WAL);
875 append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
876 Bytes.toBytes("somevalue"));
877 try {
878 region.append(append);
879 } catch (IOException e) {
880 exceptionCaught = true;
881 } finally {
882 HRegion.closeHRegion(this.region);
883 this.region = null;
884 }
885 assertTrue(exceptionCaught == true);
886 }
887
888 @Test
889 public void testIncrWithReadOnlyTable() throws Exception {
890 byte[] TABLE = Bytes.toBytes("readOnlyTable");
891 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
892 boolean exceptionCaught = false;
893 Increment inc = new Increment(Bytes.toBytes("somerow"));
894 inc.setDurability(Durability.SKIP_WAL);
895 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
896 try {
897 region.increment(inc);
898 } catch (IOException e) {
899 exceptionCaught = true;
900 } finally {
901 HRegion.closeHRegion(this.region);
902 this.region = null;
903 }
904 assertTrue(exceptionCaught == true);
905 }
906
907 private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
908 InternalScanner scanner = buildScanner(keyPrefix, value, r);
909 int count = 0;
910 boolean more = false;
911 List<Cell> results = new ArrayList<Cell>();
912 do {
913 more = scanner.next(results);
914 if (results != null && !results.isEmpty())
915 count++;
916 else
917 break;
918 Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
919 delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
920 r.delete(delete);
921 results.clear();
922 } while (more);
923 assertEquals("Did not perform correct number of deletes", 3, count);
924 }
925
926 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
927 InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
928 int numberOfResults = 0;
929 List<Cell> results = new ArrayList<Cell>();
930 boolean more = false;
931 do {
932 more = resultScanner.next(results);
933 if (results != null && !results.isEmpty())
934 numberOfResults++;
935 else
936 break;
937 for (Cell kv : results) {
938 System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
939 }
940 results.clear();
941 } while (more);
942 return numberOfResults;
943 }
944
945 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
946 throws IOException {
947
948 FilterList allFilters = new FilterList();
949 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
950
951 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
952 Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
953 filter.setFilterIfMissing(true);
954 allFilters.addFilter(filter);
955 Scan scan = new Scan();
956 scan.addFamily(Bytes.toBytes("trans-blob"));
957 scan.addFamily(Bytes.toBytes("trans-type"));
958 scan.addFamily(Bytes.toBytes("trans-date"));
959 scan.addFamily(Bytes.toBytes("trans-tags"));
960 scan.addFamily(Bytes.toBytes("trans-group"));
961 scan.setFilter(allFilters);
962 return r.getScanner(scan);
963 }
964
965 private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
966 for (int i = 0; i < numRows; i++) {
967 String row = key + "_" + i
968 System.out.println(String.format("Saving row: %s, with value %s", row, value));
969 Put put = new Put(Bytes.toBytes(row));
970 put.setDurability(Durability.SKIP_WAL);
971 put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
972 put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
973 put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
974 put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
975 put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
976 r.put(put);
977 }
978 }
979
980 @Test
981 public void testFamilyWithAndWithoutColon() throws Exception {
982 byte[] b = Bytes.toBytes(getName());
983 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
984 this.region = initHRegion(b, getName(), CONF, cf);
985 try {
986 Put p = new Put(b);
987 byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
988 p.add(cfwithcolon, cfwithcolon, cfwithcolon);
989 boolean exception = false;
990 try {
991 this.region.put(p);
992 } catch (NoSuchColumnFamilyException e) {
993 exception = true;
994 }
995 assertTrue(exception);
996 } finally {
997 HRegion.closeHRegion(this.region);
998 this.region = null;
999 }
1000 }
1001
1002 @Test
1003 public void testBatchPut() throws Exception {
1004 byte[] b = Bytes.toBytes(getName());
1005 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1006 byte[] qual = Bytes.toBytes("qual");
1007 byte[] val = Bytes.toBytes("val");
1008 this.region = initHRegion(b, getName(), CONF, cf);
1009 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1010 try {
1011 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1012 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1013
1014 LOG.info("First a batch put with all valid puts");
1015 final Put[] puts = new Put[10];
1016 for (int i = 0; i < 10; i++) {
1017 puts[i] = new Put(Bytes.toBytes("row_" + i));
1018 puts[i].add(cf, qual, val);
1019 }
1020
1021 OperationStatus[] codes = this.region.batchMutate(puts);
1022 assertEquals(10, codes.length);
1023 for (int i = 0; i < 10; i++) {
1024 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1025 }
1026 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1027
1028 LOG.info("Next a batch put with one invalid family");
1029 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1030 codes = this.region.batchMutate(puts);
1031 assertEquals(10, codes.length);
1032 for (int i = 0; i < 10; i++) {
1033 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1034 codes[i].getOperationStatusCode());
1035 }
1036
1037 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1038
1039 LOG.info("Next a batch put that has to break into two batches to avoid a lock");
1040 RowLock rowLock = region.getRowLock(Bytes.toBytes("row_2"));
1041
1042 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1043 final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
1044 TestThread putter = new TestThread(ctx) {
1045 @Override
1046 public void doWork() throws IOException {
1047 retFromThread.set(region.batchMutate(puts));
1048 }
1049 };
1050 LOG.info("...starting put thread while holding lock");
1051 ctx.addThread(putter);
1052 ctx.startThreads();
1053
1054 LOG.info("...waiting for put thread to sync first time");
1055 long startWait = System.currentTimeMillis();
1056 while (metricsAssertHelper.getCounter("syncTimeNumOps", source) == syncs + 2) {
1057 Thread.sleep(100);
1058 if (System.currentTimeMillis() - startWait > 10000) {
1059 fail("Timed out waiting for thread to sync first minibatch");
1060 }
1061 }
1062 LOG.info("...releasing row lock, which should let put thread continue");
1063 rowLock.release();
1064 LOG.info("...joining on thread");
1065 ctx.stop();
1066 LOG.info("...checking that next batch was synced");
1067 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 4, source);
1068 codes = retFromThread.get();
1069 for (int i = 0; i < 10; i++) {
1070 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1071 codes[i].getOperationStatusCode());
1072 }
1073
1074 } finally {
1075 HRegion.closeHRegion(this.region);
1076 this.region = null;
1077 }
1078 }
1079
1080 @Test
1081 public void testBatchPutWithTsSlop() throws Exception {
1082 byte[] b = Bytes.toBytes(getName());
1083 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1084 byte[] qual = Bytes.toBytes("qual");
1085 byte[] val = Bytes.toBytes("val");
1086
1087
1088 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1089 this.region = initHRegion(b, getName(), CONF, cf);
1090
1091 try {
1092 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1093 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1094 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1095
1096 final Put[] puts = new Put[10];
1097 for (int i = 0; i < 10; i++) {
1098 puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
1099 puts[i].add(cf, qual, val);
1100 }
1101
1102 OperationStatus[] codes = this.region.batchMutate(puts);
1103 assertEquals(10, codes.length);
1104 for (int i = 0; i < 10; i++) {
1105 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1106 }
1107 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1108
1109 } finally {
1110 HRegion.closeHRegion(this.region);
1111 this.region = null;
1112 }
1113
1114 }
1115
1116
1117
1118
1119 @Test
1120 public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1121 byte[] row1 = Bytes.toBytes("row1");
1122 byte[] fam1 = Bytes.toBytes("fam1");
1123 byte[] qf1 = Bytes.toBytes("qualifier");
1124 byte[] emptyVal = new byte[] {};
1125 byte[] val1 = Bytes.toBytes("value1");
1126 byte[] val2 = Bytes.toBytes("value2");
1127
1128
1129 String method = this.getName();
1130 this.region = initHRegion(tableName, method, CONF, fam1);
1131 try {
1132
1133 Put put = new Put(row1);
1134 put.add(fam1, qf1, emptyVal);
1135
1136
1137 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1138 emptyVal), put, true);
1139 assertTrue(res);
1140
1141
1142 put = new Put(row1);
1143 put.add(fam1, qf1, val1);
1144
1145
1146 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1147 put, true);
1148 assertTrue(res);
1149
1150
1151 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1152 put, true);
1153 assertFalse(res);
1154
1155 Delete delete = new Delete(row1);
1156 delete.deleteColumn(fam1, qf1);
1157 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1158 delete, true);
1159 assertFalse(res);
1160
1161 put = new Put(row1);
1162 put.add(fam1, qf1, val2);
1163
1164 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1165 put, true);
1166 assertTrue(res);
1167
1168
1169 delete = new Delete(row1);
1170 delete.deleteColumn(fam1, qf1);
1171 delete.deleteColumn(fam1, qf1);
1172 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1173 delete, true);
1174 assertTrue(res);
1175
1176 delete = new Delete(row1);
1177 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1178 delete, true);
1179 assertTrue(res);
1180
1181
1182 put = new Put(row1);
1183 put.add(fam1, qf1, val1);
1184
1185 res = region
1186 .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
1187 assertTrue(res);
1188 } finally {
1189 HRegion.closeHRegion(this.region);
1190 this.region = null;
1191 }
1192 }
1193
1194 @Test
1195 public void testCheckAndMutate_WithWrongValue() throws IOException {
1196 byte[] row1 = Bytes.toBytes("row1");
1197 byte[] fam1 = Bytes.toBytes("fam1");
1198 byte[] qf1 = Bytes.toBytes("qualifier");
1199 byte[] val1 = Bytes.toBytes("value1");
1200 byte[] val2 = Bytes.toBytes("value2");
1201
1202
1203 String method = this.getName();
1204 this.region = initHRegion(tableName, method, CONF, fam1);
1205 try {
1206
1207 Put put = new Put(row1);
1208 put.add(fam1, qf1, val1);
1209 region.put(put);
1210
1211
1212 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1213 val2), put, true);
1214 assertEquals(false, res);
1215
1216
1217 Delete delete = new Delete(row1);
1218 delete.deleteFamily(fam1);
1219 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1220 delete, true);
1221 assertEquals(false, res);
1222 } finally {
1223 HRegion.closeHRegion(this.region);
1224 this.region = null;
1225 }
1226 }
1227
1228 @Test
1229 public void testCheckAndMutate_WithCorrectValue() throws IOException {
1230 byte[] row1 = Bytes.toBytes("row1");
1231 byte[] fam1 = Bytes.toBytes("fam1");
1232 byte[] qf1 = Bytes.toBytes("qualifier");
1233 byte[] val1 = Bytes.toBytes("value1");
1234
1235
1236 String method = this.getName();
1237 this.region = initHRegion(tableName, method, CONF, fam1);
1238 try {
1239
1240 Put put = new Put(row1);
1241 put.add(fam1, qf1, val1);
1242 region.put(put);
1243
1244
1245 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1246 val1), put, true);
1247 assertEquals(true, res);
1248
1249
1250 Delete delete = new Delete(row1);
1251 delete.deleteColumn(fam1, qf1);
1252 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1253 put, true);
1254 assertEquals(true, res);
1255 } finally {
1256 HRegion.closeHRegion(this.region);
1257 this.region = null;
1258 }
1259 }
1260
1261 @Test
1262 public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1263 byte[] row1 = Bytes.toBytes("row1");
1264 byte[] fam1 = Bytes.toBytes("fam1");
1265 byte[] qf1 = Bytes.toBytes("qualifier");
1266 byte[] val1 = Bytes.toBytes("value1");
1267 byte[] val2 = Bytes.toBytes("value2");
1268 byte[] val3 = Bytes.toBytes("value3");
1269 byte[] val4 = Bytes.toBytes("value4");
1270
1271
1272 String method = this.getName();
1273 this.region = initHRegion(tableName, method, CONF, fam1);
1274 try {
1275
1276 Put put = new Put(row1);
1277 put.add(fam1, qf1, val3);
1278 region.put(put);
1279
1280
1281 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1282 new BinaryComparator(val3), put, true);
1283 assertEquals(false, res);
1284
1285
1286 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1287 new BinaryComparator(val4), put, true);
1288 assertEquals(false, res);
1289
1290
1291
1292 put = new Put(row1);
1293 put.add(fam1, qf1, val2);
1294 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1295 new BinaryComparator(val2), put, true);
1296 assertEquals(true, res);
1297
1298
1299 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1300 new BinaryComparator(val3), put, true);
1301 assertEquals(false, res);
1302
1303
1304
1305 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1306 new BinaryComparator(val2), put, true);
1307 assertEquals(true, res);
1308
1309
1310
1311 put = new Put(row1);
1312 put.add(fam1, qf1, val3);
1313 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1314 new BinaryComparator(val1), put, true);
1315 assertEquals(true, res);
1316
1317
1318 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1319 new BinaryComparator(val3), put, true);
1320 assertEquals(false, res);
1321
1322
1323 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1324 new BinaryComparator(val2), put, true);
1325 assertEquals(false, res);
1326
1327
1328
1329 put = new Put(row1);
1330 put.add(fam1, qf1, val2);
1331 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1332 new BinaryComparator(val4), put, true);
1333 assertEquals(true, res);
1334
1335
1336 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1337 new BinaryComparator(val1), put, true);
1338 assertEquals(false, res);
1339
1340
1341
1342 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1343 new BinaryComparator(val2), put, true);
1344 assertEquals(true, res);
1345
1346
1347 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1348 new BinaryComparator(val3), put, true);
1349 assertEquals(true, res);
1350 } finally {
1351 HRegion.closeHRegion(this.region);
1352 this.region = null;
1353 }
1354 }
1355
1356 @Test
1357 public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1358 byte[] row1 = Bytes.toBytes("row1");
1359 byte[] fam1 = Bytes.toBytes("fam1");
1360 byte[] fam2 = Bytes.toBytes("fam2");
1361 byte[] qf1 = Bytes.toBytes("qualifier");
1362 byte[] val1 = Bytes.toBytes("value1");
1363 byte[] val2 = Bytes.toBytes("value2");
1364
1365 byte[][] families = { fam1, fam2 };
1366
1367
1368 String method = this.getName();
1369 this.region = initHRegion(tableName, method, CONF, families);
1370 try {
1371
1372 Put put = new Put(row1);
1373 put.add(fam1, qf1, val1);
1374 region.put(put);
1375
1376
1377 long ts = System.currentTimeMillis();
1378 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1379 put = new Put(row1);
1380 put.add(kv);
1381
1382
1383 HStore store = (HStore) region.getStore(fam1);
1384 store.memstore.kvset.size();
1385
1386 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1387 val1), put, true);
1388 assertEquals(true, res);
1389 store.memstore.kvset.size();
1390
1391 Get get = new Get(row1);
1392 get.addColumn(fam2, qf1);
1393 Cell[] actual = region.get(get).rawCells();
1394
1395 Cell[] expected = { kv };
1396
1397 assertEquals(expected.length, actual.length);
1398 for (int i = 0; i < actual.length; i++) {
1399 assertEquals(expected[i], actual[i]);
1400 }
1401 } finally {
1402 HRegion.closeHRegion(this.region);
1403 this.region = null;
1404 }
1405 }
1406
1407 @Test
1408 public void testCheckAndPut_wrongRowInPut() throws IOException {
1409 TableName tableName = TableName.valueOf(name.getMethodName());
1410 this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
1411 try {
1412 Put put = new Put(row2);
1413 put.add(fam1, qual1, value1);
1414 try {
1415 region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1416 new BinaryComparator(value2), put, false);
1417 fail();
1418 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1419
1420 }
1421 } finally {
1422 HRegion.closeHRegion(this.region);
1423 this.region = null;
1424 }
1425 }
1426
1427 @Test
1428 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1429 byte[] row1 = Bytes.toBytes("row1");
1430 byte[] fam1 = Bytes.toBytes("fam1");
1431 byte[] fam2 = Bytes.toBytes("fam2");
1432 byte[] qf1 = Bytes.toBytes("qualifier1");
1433 byte[] qf2 = Bytes.toBytes("qualifier2");
1434 byte[] qf3 = Bytes.toBytes("qualifier3");
1435 byte[] val1 = Bytes.toBytes("value1");
1436 byte[] val2 = Bytes.toBytes("value2");
1437 byte[] val3 = Bytes.toBytes("value3");
1438 byte[] emptyVal = new byte[] {};
1439
1440 byte[][] families = { fam1, fam2 };
1441
1442
1443 String method = this.getName();
1444 this.region = initHRegion(tableName, method, CONF, families);
1445 try {
1446
1447 Put put = new Put(row1);
1448 put.add(fam1, qf1, val1);
1449 region.put(put);
1450 Threads.sleep(2);
1451
1452 put = new Put(row1);
1453 put.add(fam1, qf1, val2);
1454 put.add(fam2, qf1, val3);
1455 put.add(fam2, qf2, val2);
1456 put.add(fam2, qf3, val1);
1457 put.add(fam1, qf3, val1);
1458 region.put(put);
1459
1460
1461 Delete delete = new Delete(row1);
1462 delete.deleteColumn(fam1, qf1);
1463 delete.deleteColumn(fam2, qf1);
1464 delete.deleteColumn(fam1, qf3);
1465 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1466 val2), delete, true);
1467 assertEquals(true, res);
1468
1469 Get get = new Get(row1);
1470 get.addColumn(fam1, qf1);
1471 get.addColumn(fam1, qf3);
1472 get.addColumn(fam2, qf2);
1473 Result r = region.get(get);
1474 assertEquals(2, r.size());
1475 assertArrayEquals(val1, r.getValue(fam1, qf1));
1476 assertArrayEquals(val2, r.getValue(fam2, qf2));
1477
1478
1479 delete = new Delete(row1);
1480 delete.deleteFamily(fam2);
1481 res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1482 delete, true);
1483 assertEquals(true, res);
1484
1485 get = new Get(row1);
1486 r = region.get(get);
1487 assertEquals(1, r.size());
1488 assertArrayEquals(val1, r.getValue(fam1, qf1));
1489
1490
1491 delete = new Delete(row1);
1492 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1493 delete, true);
1494 assertEquals(true, res);
1495 get = new Get(row1);
1496 r = region.get(get);
1497 assertEquals(0, r.size());
1498 } finally {
1499 HRegion.closeHRegion(this.region);
1500 this.region = null;
1501 }
1502 }
1503
1504
1505
1506
1507 @Test
1508 public void testDelete_multiDeleteColumn() throws IOException {
1509 byte[] row1 = Bytes.toBytes("row1");
1510 byte[] fam1 = Bytes.toBytes("fam1");
1511 byte[] qual = Bytes.toBytes("qualifier");
1512 byte[] value = Bytes.toBytes("value");
1513
1514 Put put = new Put(row1);
1515 put.add(fam1, qual, 1, value);
1516 put.add(fam1, qual, 2, value);
1517
1518 String method = this.getName();
1519 this.region = initHRegion(tableName, method, CONF, fam1);
1520 try {
1521 region.put(put);
1522
1523
1524 Delete delete = new Delete(row1);
1525 delete.deleteColumn(fam1, qual);
1526 delete.deleteColumn(fam1, qual);
1527 region.delete(delete);
1528
1529 Get get = new Get(row1);
1530 get.addFamily(fam1);
1531 Result r = region.get(get);
1532 assertEquals(0, r.size());
1533 } finally {
1534 HRegion.closeHRegion(this.region);
1535 this.region = null;
1536 }
1537 }
1538
1539 @Test
1540 public void testDelete_CheckFamily() throws IOException {
1541 byte[] row1 = Bytes.toBytes("row1");
1542 byte[] fam1 = Bytes.toBytes("fam1");
1543 byte[] fam2 = Bytes.toBytes("fam2");
1544 byte[] fam3 = Bytes.toBytes("fam3");
1545 byte[] fam4 = Bytes.toBytes("fam4");
1546
1547
1548 String method = this.getName();
1549 this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
1550 try {
1551 List<Cell> kvs = new ArrayList<Cell>();
1552 kvs.add(new KeyValue(row1, fam4, null, null));
1553
1554
1555 byte[] family = fam2;
1556 try {
1557 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1558 Bytes.BYTES_COMPARATOR);
1559 deleteMap.put(family, kvs);
1560 region.delete(deleteMap, Durability.SYNC_WAL);
1561 } catch (Exception e) {
1562 assertTrue("Family " + new String(family) + " does not exist", false);
1563 }
1564
1565
1566 boolean ok = false;
1567 family = fam4;
1568 try {
1569 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1570 Bytes.BYTES_COMPARATOR);
1571 deleteMap.put(family, kvs);
1572 region.delete(deleteMap, Durability.SYNC_WAL);
1573 } catch (Exception e) {
1574 ok = true;
1575 }
1576 assertEquals("Family " + new String(family) + " does exist", true, ok);
1577 } finally {
1578 HRegion.closeHRegion(this.region);
1579 this.region = null;
1580 }
1581 }
1582
1583 @Test
1584 public void testDelete_mixed() throws IOException, InterruptedException {
1585 byte[] fam = Bytes.toBytes("info");
1586 byte[][] families = { fam };
1587 String method = this.getName();
1588 this.region = initHRegion(tableName, method, CONF, families);
1589 try {
1590 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1591
1592 byte[] row = Bytes.toBytes("table_name");
1593
1594 byte[] serverinfo = Bytes.toBytes("serverinfo");
1595 byte[] splitA = Bytes.toBytes("splitA");
1596 byte[] splitB = Bytes.toBytes("splitB");
1597
1598
1599 Put put = new Put(row);
1600 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1601 region.put(put);
1602
1603 put = new Put(row);
1604 put.add(fam, splitB, Bytes.toBytes("reference_B"));
1605 region.put(put);
1606
1607 put = new Put(row);
1608 put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1609 region.put(put);
1610
1611
1612 Delete delete = new Delete(row);
1613 delete.deleteColumns(fam, splitA);
1614 region.delete(delete);
1615
1616
1617 Get get = new Get(row).addColumn(fam, serverinfo);
1618 Result result = region.get(get);
1619 assertEquals(1, result.size());
1620
1621 get = new Get(row).addColumn(fam, splitA);
1622 result = region.get(get);
1623 assertEquals(0, result.size());
1624
1625 get = new Get(row).addColumn(fam, splitB);
1626 result = region.get(get);
1627 assertEquals(1, result.size());
1628
1629
1630 put = new Put(row);
1631 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1632 region.put(put);
1633 get = new Get(row);
1634 result = region.get(get);
1635 assertEquals(3, result.size());
1636
1637
1638 delete = new Delete(row);
1639 region.delete(delete);
1640 assertEquals(0, region.get(get).size());
1641
1642 region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
1643 result = region.get(get);
1644 assertEquals(1, result.size());
1645 } finally {
1646 HRegion.closeHRegion(this.region);
1647 this.region = null;
1648 }
1649 }
1650
1651 @Test
1652 public void testDeleteRowWithFutureTs() throws IOException {
1653 byte[] fam = Bytes.toBytes("info");
1654 byte[][] families = { fam };
1655 String method = this.getName();
1656 this.region = initHRegion(tableName, method, CONF, families);
1657 try {
1658 byte[] row = Bytes.toBytes("table_name");
1659
1660 byte[] serverinfo = Bytes.toBytes("serverinfo");
1661
1662
1663 Put put = new Put(row);
1664 put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
1665 region.put(put);
1666
1667
1668 Delete delete = new Delete(row);
1669 region.delete(delete);
1670
1671
1672 Get get = new Get(row).addColumn(fam, serverinfo);
1673 Result result = region.get(get);
1674 assertEquals(1, result.size());
1675
1676
1677 delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
1678 region.delete(delete);
1679
1680
1681 get = new Get(row).addColumn(fam, serverinfo);
1682 result = region.get(get);
1683 assertEquals(0, result.size());
1684 } finally {
1685 HRegion.closeHRegion(this.region);
1686 this.region = null;
1687 }
1688 }
1689
1690
1691
1692
1693
1694 @Test
1695 public void testPutWithLatestTS() throws IOException {
1696 byte[] fam = Bytes.toBytes("info");
1697 byte[][] families = { fam };
1698 String method = this.getName();
1699 this.region = initHRegion(tableName, method, CONF, families);
1700 try {
1701 byte[] row = Bytes.toBytes("row1");
1702
1703 byte[] qual = Bytes.toBytes("qual");
1704
1705
1706 Put put = new Put(row);
1707 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1708 region.put(put);
1709
1710
1711 Get get = new Get(row).addColumn(fam, qual);
1712 Result result = region.get(get);
1713 assertEquals(1, result.size());
1714 Cell kv = result.rawCells()[0];
1715 LOG.info("Got: " + kv);
1716 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1717 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1718
1719
1720
1721 row = Bytes.toBytes("row2");
1722 put = new Put(row);
1723 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1724 region.put(put);
1725
1726
1727 get = new Get(row).addColumn(fam, qual);
1728 result = region.get(get);
1729 assertEquals(1, result.size());
1730 kv = result.rawCells()[0];
1731 LOG.info("Got: " + kv);
1732 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1733 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1734 } finally {
1735 HRegion.closeHRegion(this.region);
1736 this.region = null;
1737 }
1738
1739 }
1740
1741
1742
1743
1744
1745
1746 @Test
1747 public void testPutWithTsSlop() throws IOException {
1748 byte[] fam = Bytes.toBytes("info");
1749 byte[][] families = { fam };
1750 String method = this.getName();
1751
1752
1753 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1754 this.region = initHRegion(tableName, method, CONF, families);
1755 boolean caughtExcep = false;
1756 try {
1757 try {
1758
1759 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
1760
1761 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
1762 Bytes.toBytes("value")));
1763 fail("Expected IOE for TS out of configured timerange");
1764 } catch (FailedSanityCheckException ioe) {
1765 LOG.debug("Received expected exception", ioe);
1766 caughtExcep = true;
1767 }
1768 assertTrue("Should catch FailedSanityCheckException", caughtExcep);
1769 } finally {
1770 HRegion.closeHRegion(this.region);
1771 this.region = null;
1772 }
1773 }
1774
1775 @Test
1776 public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
1777 byte[] fam1 = Bytes.toBytes("columnA");
1778 byte[] fam2 = Bytes.toBytes("columnB");
1779 this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
1780 try {
1781 byte[] rowA = Bytes.toBytes("rowA");
1782 byte[] rowB = Bytes.toBytes("rowB");
1783
1784 byte[] value = Bytes.toBytes("value");
1785
1786 Delete delete = new Delete(rowA);
1787 delete.deleteFamily(fam1);
1788
1789 region.delete(delete);
1790
1791
1792 Put put = new Put(rowA);
1793 put.add(fam2, null, value);
1794 region.put(put);
1795
1796 put = new Put(rowB);
1797 put.add(fam1, null, value);
1798 put.add(fam2, null, value);
1799 region.put(put);
1800
1801 Scan scan = new Scan();
1802 scan.addFamily(fam1).addFamily(fam2);
1803 InternalScanner s = region.getScanner(scan);
1804 List<Cell> results = new ArrayList<Cell>();
1805 s.next(results);
1806 assertTrue(CellUtil.matchingRow(results.get(0), rowA));
1807
1808 results.clear();
1809 s.next(results);
1810 assertTrue(CellUtil.matchingRow(results.get(0), rowB));
1811 } finally {
1812 HRegion.closeHRegion(this.region);
1813 this.region = null;
1814 }
1815 }
1816
1817 @Test
1818 public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
1819 Delete delete = new Delete(row);
1820 delete.deleteColumns(fam1, qual1);
1821 doTestDelete_AndPostInsert(delete);
1822 }
1823
1824 @Test
1825 public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
1826 Delete delete = new Delete(row);
1827 delete.deleteFamily(fam1);
1828 doTestDelete_AndPostInsert(delete);
1829 }
1830
1831 public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
1832 TableName tableName = TableName.valueOf(name.getMethodName());
1833 this.region = initHRegion(tableName, getName(), CONF, fam1);
1834 try {
1835 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1836 Put put = new Put(row);
1837 put.add(fam1, qual1, value1);
1838 region.put(put);
1839
1840
1841 region.delete(delete);
1842
1843
1844 put = new Put(row);
1845 put.add(fam1, qual1, value2);
1846 region.put(put);
1847
1848
1849 Get get = new Get(row);
1850 get.addColumn(fam1, qual1);
1851
1852 Result r = region.get(get);
1853 assertEquals(1, r.size());
1854 assertArrayEquals(value2, r.getValue(fam1, qual1));
1855
1856
1857 Scan scan = new Scan(row);
1858 scan.addColumn(fam1, qual1);
1859 InternalScanner s = region.getScanner(scan);
1860
1861 List<Cell> results = new ArrayList<Cell>();
1862 assertEquals(false, s.next(results));
1863 assertEquals(1, results.size());
1864 Cell kv = results.get(0);
1865
1866 assertArrayEquals(value2, CellUtil.cloneValue(kv));
1867 assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
1868 assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
1869 assertArrayEquals(row, CellUtil.cloneRow(kv));
1870 } finally {
1871 HRegion.closeHRegion(this.region);
1872 this.region = null;
1873 }
1874 }
1875
1876 @Test
1877 public void testDelete_CheckTimestampUpdated() throws IOException {
1878 TableName tableName = TableName.valueOf(name.getMethodName());
1879 byte[] row1 = Bytes.toBytes("row1");
1880 byte[] col1 = Bytes.toBytes("col1");
1881 byte[] col2 = Bytes.toBytes("col2");
1882 byte[] col3 = Bytes.toBytes("col3");
1883
1884
1885 String method = this.getName();
1886 this.region = initHRegion(tableName, method, CONF, fam1);
1887 try {
1888
1889 List<Cell> kvs = new ArrayList<Cell>();
1890 kvs.add(new KeyValue(row1, fam1, col1, null));
1891 kvs.add(new KeyValue(row1, fam1, col2, null));
1892 kvs.add(new KeyValue(row1, fam1, col3, null));
1893
1894 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1895 Bytes.BYTES_COMPARATOR);
1896 deleteMap.put(fam1, kvs);
1897 region.delete(deleteMap, Durability.SYNC_WAL);
1898
1899
1900
1901 long now = System.currentTimeMillis();
1902 KeyValue firstKv = ((HStore) region.getStore(fam1)).memstore.kvset.first();
1903 assertTrue(firstKv.getTimestamp() <= now);
1904 now = firstKv.getTimestamp();
1905 for (KeyValue kv : ((HStore) region.getStore(fam1)).memstore.kvset) {
1906 assertTrue(kv.getTimestamp() <= now);
1907 now = kv.getTimestamp();
1908 }
1909 } finally {
1910 HRegion.closeHRegion(this.region);
1911 this.region = null;
1912 }
1913 }
1914
1915
1916
1917
1918 @Test
1919 public void testGet_FamilyChecker() throws IOException {
1920 byte[] row1 = Bytes.toBytes("row1");
1921 byte[] fam1 = Bytes.toBytes("fam1");
1922 byte[] fam2 = Bytes.toBytes("False");
1923 byte[] col1 = Bytes.toBytes("col1");
1924
1925
1926 String method = this.getName();
1927 this.region = initHRegion(tableName, method, CONF, fam1);
1928 try {
1929 Get get = new Get(row1);
1930 get.addColumn(fam2, col1);
1931
1932
1933 try {
1934 region.get(get);
1935 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
1936 assertFalse(false);
1937 return;
1938 }
1939 assertFalse(true);
1940 } finally {
1941 HRegion.closeHRegion(this.region);
1942 this.region = null;
1943 }
1944 }
1945
1946 @Test
1947 public void testGet_Basic() throws IOException {
1948 byte[] row1 = Bytes.toBytes("row1");
1949 byte[] fam1 = Bytes.toBytes("fam1");
1950 byte[] col1 = Bytes.toBytes("col1");
1951 byte[] col2 = Bytes.toBytes("col2");
1952 byte[] col3 = Bytes.toBytes("col3");
1953 byte[] col4 = Bytes.toBytes("col4");
1954 byte[] col5 = Bytes.toBytes("col5");
1955
1956
1957 String method = this.getName();
1958 this.region = initHRegion(tableName, method, CONF, fam1);
1959 try {
1960
1961 Put put = new Put(row1);
1962 put.add(fam1, col1, null);
1963 put.add(fam1, col2, null);
1964 put.add(fam1, col3, null);
1965 put.add(fam1, col4, null);
1966 put.add(fam1, col5, null);
1967 region.put(put);
1968
1969 Get get = new Get(row1);
1970 get.addColumn(fam1, col2);
1971 get.addColumn(fam1, col4);
1972
1973 KeyValue kv1 = new KeyValue(row1, fam1, col2);
1974 KeyValue kv2 = new KeyValue(row1, fam1, col4);
1975 KeyValue[] expected = { kv1, kv2 };
1976
1977
1978 Result res = region.get(get);
1979 assertEquals(expected.length, res.size());
1980 for (int i = 0; i < res.size(); i++) {
1981 assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
1982 assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
1983 assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
1984 }
1985
1986
1987 Get g = new Get(row1);
1988 final int count = 2;
1989 g.setFilter(new ColumnCountGetFilter(count));
1990 res = region.get(g);
1991 assertEquals(count, res.size());
1992 } finally {
1993 HRegion.closeHRegion(this.region);
1994 this.region = null;
1995 }
1996 }
1997
1998 @Test
1999 public void testGet_Empty() throws IOException {
2000 byte[] row = Bytes.toBytes("row");
2001 byte[] fam = Bytes.toBytes("fam");
2002
2003 String method = this.getName();
2004 this.region = initHRegion(tableName, method, CONF, fam);
2005 try {
2006 Get get = new Get(row);
2007 get.addFamily(fam);
2008 Result r = region.get(get);
2009
2010 assertTrue(r.isEmpty());
2011 } finally {
2012 HRegion.closeHRegion(this.region);
2013 this.region = null;
2014 }
2015 }
2016
2017
2018
2019
2020 @Test
2021 public void testMerge() throws IOException {
2022 byte[][] families = { fam1, fam2, fam3 };
2023 Configuration hc = initSplit();
2024
2025 String method = this.getName();
2026 this.region = initHRegion(tableName, method, hc, families);
2027 try {
2028 LOG.info("" + HBaseTestCase.addContent(region, fam3));
2029 region.flushcache();
2030 region.compactStores();
2031 byte[] splitRow = region.checkSplit();
2032 assertNotNull(splitRow);
2033 LOG.info("SplitRow: " + Bytes.toString(splitRow));
2034 HRegion[] subregions = splitRegion(region, splitRow);
2035 try {
2036
2037 for (int i = 0; i < subregions.length; i++) {
2038 HRegion.openHRegion(subregions[i], null);
2039 subregions[i].compactStores();
2040 }
2041 Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
2042 Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
2043 Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
2044 long startTime = System.currentTimeMillis();
2045 region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
2046 LOG.info("Merge regions elapsed time: "
2047 + ((System.currentTimeMillis() - startTime) / 1000.0));
2048 FILESYSTEM.delete(oldRegion1, true);
2049 FILESYSTEM.delete(oldRegion2, true);
2050 FILESYSTEM.delete(oldRegionPath, true);
2051 LOG.info("splitAndMerge completed.");
2052 } finally {
2053 for (int i = 0; i < subregions.length; i++) {
2054 try {
2055 HRegion.closeHRegion(subregions[i]);
2056 } catch (IOException e) {
2057
2058 }
2059 }
2060 }
2061 } finally {
2062 HRegion.closeHRegion(this.region);
2063 this.region = null;
2064 }
2065 }
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075 HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
2076 PairOfSameType<HRegion> result = null;
2077 SplitTransaction st = new SplitTransaction(parent, midkey);
2078
2079
2080 if (!st.prepare())
2081 return null;
2082 try {
2083 result = st.execute(null, null);
2084 } catch (IOException ioe) {
2085 try {
2086 LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; "
2087 + ioe.getMessage());
2088 st.rollback(null, null);
2089 LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString());
2090 return null;
2091 } catch (RuntimeException e) {
2092
2093 LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString()
2094 + " -- aborting server", e);
2095 }
2096 }
2097 return new HRegion[] { result.getFirst(), result.getSecond() };
2098 }
2099
2100
2101
2102
2103 @Test
2104 public void testGetScanner_WithOkFamilies() throws IOException {
2105 byte[] fam1 = Bytes.toBytes("fam1");
2106 byte[] fam2 = Bytes.toBytes("fam2");
2107
2108 byte[][] families = { fam1, fam2 };
2109
2110
2111 String method = this.getName();
2112 this.region = initHRegion(tableName, method, CONF, families);
2113 try {
2114 Scan scan = new Scan();
2115 scan.addFamily(fam1);
2116 scan.addFamily(fam2);
2117 try {
2118 region.getScanner(scan);
2119 } catch (Exception e) {
2120 assertTrue("Families could not be found in Region", false);
2121 }
2122 } finally {
2123 HRegion.closeHRegion(this.region);
2124 this.region = null;
2125 }
2126 }
2127
2128 @Test
2129 public void testGetScanner_WithNotOkFamilies() throws IOException {
2130 byte[] fam1 = Bytes.toBytes("fam1");
2131 byte[] fam2 = Bytes.toBytes("fam2");
2132
2133 byte[][] families = { fam1 };
2134
2135
2136 String method = this.getName();
2137 this.region = initHRegion(tableName, method, CONF, families);
2138 try {
2139 Scan scan = new Scan();
2140 scan.addFamily(fam2);
2141 boolean ok = false;
2142 try {
2143 region.getScanner(scan);
2144 } catch (Exception e) {
2145 ok = true;
2146 }
2147 assertTrue("Families could not be found in Region", ok);
2148 } finally {
2149 HRegion.closeHRegion(this.region);
2150 this.region = null;
2151 }
2152 }
2153
2154 @Test
2155 public void testGetScanner_WithNoFamilies() throws IOException {
2156 byte[] row1 = Bytes.toBytes("row1");
2157 byte[] fam1 = Bytes.toBytes("fam1");
2158 byte[] fam2 = Bytes.toBytes("fam2");
2159 byte[] fam3 = Bytes.toBytes("fam3");
2160 byte[] fam4 = Bytes.toBytes("fam4");
2161
2162 byte[][] families = { fam1, fam2, fam3, fam4 };
2163
2164
2165 String method = this.getName();
2166 this.region = initHRegion(tableName, method, CONF, families);
2167 try {
2168
2169
2170 Put put = new Put(row1);
2171 put.add(fam1, null, null);
2172 put.add(fam2, null, null);
2173 put.add(fam3, null, null);
2174 put.add(fam4, null, null);
2175 region.put(put);
2176
2177 Scan scan = null;
2178 HRegion.RegionScannerImpl is = null;
2179
2180
2181
2182
2183 scan = new Scan();
2184 scan.addFamily(fam2);
2185 scan.addFamily(fam4);
2186 is = (RegionScannerImpl) region.getScanner(scan);
2187 assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2188
2189 scan = new Scan();
2190 is = (RegionScannerImpl) region.getScanner(scan);
2191 assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2192 } finally {
2193 HRegion.closeHRegion(this.region);
2194 this.region = null;
2195 }
2196 }
2197
2198
2199
2200
2201
2202
2203 @Test
2204 public void testGetScanner_WithRegionClosed() throws IOException {
2205 byte[] fam1 = Bytes.toBytes("fam1");
2206 byte[] fam2 = Bytes.toBytes("fam2");
2207
2208 byte[][] families = { fam1, fam2 };
2209
2210
2211 String method = this.getName();
2212 try {
2213 this.region = initHRegion(tableName, method, CONF, families);
2214 } catch (IOException e) {
2215 e.printStackTrace();
2216 fail("Got IOException during initHRegion, " + e.getMessage());
2217 }
2218 try {
2219 region.closed.set(true);
2220 try {
2221 region.getScanner(null);
2222 fail("Expected to get an exception during getScanner on a region that is closed");
2223 } catch (NotServingRegionException e) {
2224
2225 } catch (IOException e) {
2226 fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2227 + e.getMessage());
2228 }
2229 } finally {
2230 HRegion.closeHRegion(this.region);
2231 this.region = null;
2232 }
2233 }
2234
2235 @Test
2236 public void testRegionScanner_Next() throws IOException {
2237 byte[] row1 = Bytes.toBytes("row1");
2238 byte[] row2 = Bytes.toBytes("row2");
2239 byte[] fam1 = Bytes.toBytes("fam1");
2240 byte[] fam2 = Bytes.toBytes("fam2");
2241 byte[] fam3 = Bytes.toBytes("fam3");
2242 byte[] fam4 = Bytes.toBytes("fam4");
2243
2244 byte[][] families = { fam1, fam2, fam3, fam4 };
2245 long ts = System.currentTimeMillis();
2246
2247
2248 String method = this.getName();
2249 this.region = initHRegion(tableName, method, CONF, families);
2250 try {
2251
2252 Put put = null;
2253 put = new Put(row1);
2254 put.add(fam1, (byte[]) null, ts, null);
2255 put.add(fam2, (byte[]) null, ts, null);
2256 put.add(fam3, (byte[]) null, ts, null);
2257 put.add(fam4, (byte[]) null, ts, null);
2258 region.put(put);
2259
2260 put = new Put(row2);
2261 put.add(fam1, (byte[]) null, ts, null);
2262 put.add(fam2, (byte[]) null, ts, null);
2263 put.add(fam3, (byte[]) null, ts, null);
2264 put.add(fam4, (byte[]) null, ts, null);
2265 region.put(put);
2266
2267 Scan scan = new Scan();
2268 scan.addFamily(fam2);
2269 scan.addFamily(fam4);
2270 InternalScanner is = region.getScanner(scan);
2271
2272 List<Cell> res = null;
2273
2274
2275 List<Cell> expected1 = new ArrayList<Cell>();
2276 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2277 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2278
2279 res = new ArrayList<Cell>();
2280 is.next(res);
2281 for (int i = 0; i < res.size(); i++) {
2282 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2283 }
2284
2285
2286 List<Cell> expected2 = new ArrayList<Cell>();
2287 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2288 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2289
2290 res = new ArrayList<Cell>();
2291 is.next(res);
2292 for (int i = 0; i < res.size(); i++) {
2293 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2294 }
2295 } finally {
2296 HRegion.closeHRegion(this.region);
2297 this.region = null;
2298 }
2299 }
2300
2301 @Test
2302 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2303 byte[] row1 = Bytes.toBytes("row1");
2304 byte[] qf1 = Bytes.toBytes("qualifier1");
2305 byte[] qf2 = Bytes.toBytes("qualifier2");
2306 byte[] fam1 = Bytes.toBytes("fam1");
2307 byte[][] families = { fam1 };
2308
2309 long ts1 = System.currentTimeMillis();
2310 long ts2 = ts1 + 1;
2311 long ts3 = ts1 + 2;
2312
2313
2314 String method = this.getName();
2315 this.region = initHRegion(tableName, method, CONF, families);
2316 try {
2317
2318 Put put = null;
2319 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2320 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2321 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2322
2323 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2324 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2325 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2326
2327 put = new Put(row1);
2328 put.add(kv13);
2329 put.add(kv12);
2330 put.add(kv11);
2331 put.add(kv23);
2332 put.add(kv22);
2333 put.add(kv21);
2334 region.put(put);
2335
2336
2337 List<Cell> expected = new ArrayList<Cell>();
2338 expected.add(kv13);
2339 expected.add(kv12);
2340
2341 Scan scan = new Scan(row1);
2342 scan.addColumn(fam1, qf1);
2343 scan.setMaxVersions(MAX_VERSIONS);
2344 List<Cell> actual = new ArrayList<Cell>();
2345 InternalScanner scanner = region.getScanner(scan);
2346
2347 boolean hasNext = scanner.next(actual);
2348 assertEquals(false, hasNext);
2349
2350
2351 for (int i = 0; i < expected.size(); i++) {
2352 assertEquals(expected.get(i), actual.get(i));
2353 }
2354 } finally {
2355 HRegion.closeHRegion(this.region);
2356 this.region = null;
2357 }
2358 }
2359
2360 @Test
2361 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2362 byte[] row1 = Bytes.toBytes("row1");
2363 byte[] qf1 = Bytes.toBytes("qualifier1");
2364 byte[] qf2 = Bytes.toBytes("qualifier2");
2365 byte[] fam1 = Bytes.toBytes("fam1");
2366 byte[][] families = { fam1 };
2367
2368 long ts1 = 1;
2369 long ts2 = ts1 + 1;
2370 long ts3 = ts1 + 2;
2371
2372
2373 String method = this.getName();
2374 this.region = initHRegion(tableName, method, CONF, families);
2375 try {
2376
2377 Put put = null;
2378 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2379 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2380 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2381
2382 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2383 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2384 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2385
2386 put = new Put(row1);
2387 put.add(kv13);
2388 put.add(kv12);
2389 put.add(kv11);
2390 put.add(kv23);
2391 put.add(kv22);
2392 put.add(kv21);
2393 region.put(put);
2394 region.flushcache();
2395
2396
2397 List<Cell> expected = new ArrayList<Cell>();
2398 expected.add(kv13);
2399 expected.add(kv12);
2400 expected.add(kv23);
2401 expected.add(kv22);
2402
2403 Scan scan = new Scan(row1);
2404 scan.addColumn(fam1, qf1);
2405 scan.addColumn(fam1, qf2);
2406 scan.setMaxVersions(MAX_VERSIONS);
2407 List<Cell> actual = new ArrayList<Cell>();
2408 InternalScanner scanner = region.getScanner(scan);
2409
2410 boolean hasNext = scanner.next(actual);
2411 assertEquals(false, hasNext);
2412
2413
2414 for (int i = 0; i < expected.size(); i++) {
2415 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2416 }
2417 } finally {
2418 HRegion.closeHRegion(this.region);
2419 this.region = null;
2420 }
2421 }
2422
2423 @Test
2424 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2425 byte[] row1 = Bytes.toBytes("row1");
2426 byte[] fam1 = Bytes.toBytes("fam1");
2427 byte[][] families = { fam1 };
2428 byte[] qf1 = Bytes.toBytes("qualifier1");
2429 byte[] qf2 = Bytes.toBytes("qualifier2");
2430
2431 long ts1 = 1;
2432 long ts2 = ts1 + 1;
2433 long ts3 = ts1 + 2;
2434 long ts4 = ts1 + 3;
2435
2436
2437 String method = this.getName();
2438 this.region = initHRegion(tableName, method, CONF, families);
2439 try {
2440
2441 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2442 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2443 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2444 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2445
2446 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2447 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2448 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2449 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2450
2451 Put put = null;
2452 put = new Put(row1);
2453 put.add(kv14);
2454 put.add(kv24);
2455 region.put(put);
2456 region.flushcache();
2457
2458 put = new Put(row1);
2459 put.add(kv23);
2460 put.add(kv13);
2461 region.put(put);
2462 region.flushcache();
2463
2464 put = new Put(row1);
2465 put.add(kv22);
2466 put.add(kv12);
2467 region.put(put);
2468 region.flushcache();
2469
2470 put = new Put(row1);
2471 put.add(kv21);
2472 put.add(kv11);
2473 region.put(put);
2474
2475
2476 List<Cell> expected = new ArrayList<Cell>();
2477 expected.add(kv14);
2478 expected.add(kv13);
2479 expected.add(kv12);
2480 expected.add(kv24);
2481 expected.add(kv23);
2482 expected.add(kv22);
2483
2484 Scan scan = new Scan(row1);
2485 scan.addColumn(fam1, qf1);
2486 scan.addColumn(fam1, qf2);
2487 int versions = 3;
2488 scan.setMaxVersions(versions);
2489 List<Cell> actual = new ArrayList<Cell>();
2490 InternalScanner scanner = region.getScanner(scan);
2491
2492 boolean hasNext = scanner.next(actual);
2493 assertEquals(false, hasNext);
2494
2495
2496 for (int i = 0; i < expected.size(); i++) {
2497 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2498 }
2499 } finally {
2500 HRegion.closeHRegion(this.region);
2501 this.region = null;
2502 }
2503 }
2504
2505 @Test
2506 public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2507 byte[] row1 = Bytes.toBytes("row1");
2508 byte[] qf1 = Bytes.toBytes("qualifier1");
2509 byte[] qf2 = Bytes.toBytes("qualifier2");
2510 byte[] fam1 = Bytes.toBytes("fam1");
2511 byte[][] families = { fam1 };
2512
2513 long ts1 = System.currentTimeMillis();
2514 long ts2 = ts1 + 1;
2515 long ts3 = ts1 + 2;
2516
2517
2518 String method = this.getName();
2519 this.region = initHRegion(tableName, method, CONF, families);
2520 try {
2521
2522 Put put = null;
2523 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2524 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2525 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2526
2527 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2528 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2529 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2530
2531 put = new Put(row1);
2532 put.add(kv13);
2533 put.add(kv12);
2534 put.add(kv11);
2535 put.add(kv23);
2536 put.add(kv22);
2537 put.add(kv21);
2538 region.put(put);
2539
2540
2541 List<Cell> expected = new ArrayList<Cell>();
2542 expected.add(kv13);
2543 expected.add(kv12);
2544 expected.add(kv23);
2545 expected.add(kv22);
2546
2547 Scan scan = new Scan(row1);
2548 scan.addFamily(fam1);
2549 scan.setMaxVersions(MAX_VERSIONS);
2550 List<Cell> actual = new ArrayList<Cell>();
2551 InternalScanner scanner = region.getScanner(scan);
2552
2553 boolean hasNext = scanner.next(actual);
2554 assertEquals(false, hasNext);
2555
2556
2557 for (int i = 0; i < expected.size(); i++) {
2558 assertEquals(expected.get(i), actual.get(i));
2559 }
2560 } finally {
2561 HRegion.closeHRegion(this.region);
2562 this.region = null;
2563 }
2564 }
2565
2566 @Test
2567 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
2568 byte[] row1 = Bytes.toBytes("row1");
2569 byte[] qf1 = Bytes.toBytes("qualifier1");
2570 byte[] qf2 = Bytes.toBytes("qualifier2");
2571 byte[] fam1 = Bytes.toBytes("fam1");
2572
2573 long ts1 = 1;
2574 long ts2 = ts1 + 1;
2575 long ts3 = ts1 + 2;
2576
2577
2578 String method = this.getName();
2579 this.region = initHRegion(tableName, method, CONF, fam1);
2580 try {
2581
2582 Put put = null;
2583 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2584 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2585 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2586
2587 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2588 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2589 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2590
2591 put = new Put(row1);
2592 put.add(kv13);
2593 put.add(kv12);
2594 put.add(kv11);
2595 put.add(kv23);
2596 put.add(kv22);
2597 put.add(kv21);
2598 region.put(put);
2599 region.flushcache();
2600
2601
2602 List<Cell> expected = new ArrayList<Cell>();
2603 expected.add(kv13);
2604 expected.add(kv12);
2605 expected.add(kv23);
2606 expected.add(kv22);
2607
2608 Scan scan = new Scan(row1);
2609 scan.addFamily(fam1);
2610 scan.setMaxVersions(MAX_VERSIONS);
2611 List<Cell> actual = new ArrayList<Cell>();
2612 InternalScanner scanner = region.getScanner(scan);
2613
2614 boolean hasNext = scanner.next(actual);
2615 assertEquals(false, hasNext);
2616
2617
2618 for (int i = 0; i < expected.size(); i++) {
2619 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2620 }
2621 } finally {
2622 HRegion.closeHRegion(this.region);
2623 this.region = null;
2624 }
2625 }
2626
2627 @Test
2628 public void testScanner_StopRow1542() throws IOException {
2629 byte[] family = Bytes.toBytes("testFamily");
2630 this.region = initHRegion(tableName, getName(), CONF, family);
2631 try {
2632 byte[] row1 = Bytes.toBytes("row111");
2633 byte[] row2 = Bytes.toBytes("row222");
2634 byte[] row3 = Bytes.toBytes("row333");
2635 byte[] row4 = Bytes.toBytes("row444");
2636 byte[] row5 = Bytes.toBytes("row555");
2637
2638 byte[] col1 = Bytes.toBytes("Pub111");
2639 byte[] col2 = Bytes.toBytes("Pub222");
2640
2641 Put put = new Put(row1);
2642 put.add(family, col1, Bytes.toBytes(10L));
2643 region.put(put);
2644
2645 put = new Put(row2);
2646 put.add(family, col1, Bytes.toBytes(15L));
2647 region.put(put);
2648
2649 put = new Put(row3);
2650 put.add(family, col2, Bytes.toBytes(20L));
2651 region.put(put);
2652
2653 put = new Put(row4);
2654 put.add(family, col2, Bytes.toBytes(30L));
2655 region.put(put);
2656
2657 put = new Put(row5);
2658 put.add(family, col1, Bytes.toBytes(40L));
2659 region.put(put);
2660
2661 Scan scan = new Scan(row3, row4);
2662 scan.setMaxVersions();
2663 scan.addColumn(family, col1);
2664 InternalScanner s = region.getScanner(scan);
2665
2666 List<Cell> results = new ArrayList<Cell>();
2667 assertEquals(false, s.next(results));
2668 assertEquals(0, results.size());
2669 } finally {
2670 HRegion.closeHRegion(this.region);
2671 this.region = null;
2672 }
2673 }
2674
2675 @Test
2676 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2677 byte[] row1 = Bytes.toBytes("row1");
2678 byte[] fam1 = Bytes.toBytes("fam1");
2679 byte[] qf1 = Bytes.toBytes("qualifier1");
2680 byte[] qf2 = Bytes.toBytes("quateslifier2");
2681
2682 long ts1 = 1;
2683 long ts2 = ts1 + 1;
2684 long ts3 = ts1 + 2;
2685 long ts4 = ts1 + 3;
2686
2687
2688 String method = this.getName();
2689 this.region = initHRegion(tableName, method, CONF, fam1);
2690 try {
2691
2692 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2693 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2694 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2695 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2696
2697 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2698 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2699 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2700 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2701
2702 Put put = null;
2703 put = new Put(row1);
2704 put.add(kv14);
2705 put.add(kv24);
2706 region.put(put);
2707 region.flushcache();
2708
2709 put = new Put(row1);
2710 put.add(kv23);
2711 put.add(kv13);
2712 region.put(put);
2713 region.flushcache();
2714
2715 put = new Put(row1);
2716 put.add(kv22);
2717 put.add(kv12);
2718 region.put(put);
2719 region.flushcache();
2720
2721 put = new Put(row1);
2722 put.add(kv21);
2723 put.add(kv11);
2724 region.put(put);
2725
2726
2727 List<KeyValue> expected = new ArrayList<KeyValue>();
2728 expected.add(kv14);
2729 expected.add(kv13);
2730 expected.add(kv12);
2731 expected.add(kv24);
2732 expected.add(kv23);
2733 expected.add(kv22);
2734
2735 Scan scan = new Scan(row1);
2736 int versions = 3;
2737 scan.setMaxVersions(versions);
2738 List<Cell> actual = new ArrayList<Cell>();
2739 InternalScanner scanner = region.getScanner(scan);
2740
2741 boolean hasNext = scanner.next(actual);
2742 assertEquals(false, hasNext);
2743
2744
2745 for (int i = 0; i < expected.size(); i++) {
2746 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2747 }
2748 } finally {
2749 HRegion.closeHRegion(this.region);
2750 this.region = null;
2751 }
2752 }
2753
2754
2755
2756
2757
2758
2759
2760 @Test
2761 public void testScanner_JoinedScanners() throws IOException {
2762 byte[] cf_essential = Bytes.toBytes("essential");
2763 byte[] cf_joined = Bytes.toBytes("joined");
2764 byte[] cf_alpha = Bytes.toBytes("alpha");
2765 this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
2766 try {
2767 byte[] row1 = Bytes.toBytes("row1");
2768 byte[] row2 = Bytes.toBytes("row2");
2769 byte[] row3 = Bytes.toBytes("row3");
2770
2771 byte[] col_normal = Bytes.toBytes("d");
2772 byte[] col_alpha = Bytes.toBytes("a");
2773
2774 byte[] filtered_val = Bytes.toBytes(3);
2775
2776 Put put = new Put(row1);
2777 put.add(cf_essential, col_normal, Bytes.toBytes(1));
2778 put.add(cf_joined, col_alpha, Bytes.toBytes(1));
2779 region.put(put);
2780
2781 put = new Put(row2);
2782 put.add(cf_essential, col_alpha, Bytes.toBytes(2));
2783 put.add(cf_joined, col_normal, Bytes.toBytes(2));
2784 put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
2785 region.put(put);
2786
2787 put = new Put(row3);
2788 put.add(cf_essential, col_normal, filtered_val);
2789 put.add(cf_joined, col_normal, filtered_val);
2790 region.put(put);
2791
2792
2793
2794
2795
2796 Scan scan = new Scan();
2797 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
2798 CompareOp.NOT_EQUAL, filtered_val);
2799 scan.setFilter(filter);
2800 scan.setLoadColumnFamiliesOnDemand(true);
2801 InternalScanner s = region.getScanner(scan);
2802
2803 List<Cell> results = new ArrayList<Cell>();
2804 assertTrue(s.next(results));
2805 assertEquals(results.size(), 1);
2806 results.clear();
2807
2808 assertTrue(s.next(results));
2809 assertEquals(results.size(), 3);
2810 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
2811 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
2812 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
2813 results.clear();
2814
2815 assertFalse(s.next(results));
2816 assertEquals(results.size(), 0);
2817 } finally {
2818 HRegion.closeHRegion(this.region);
2819 this.region = null;
2820 }
2821 }
2822
2823
2824
2825
2826
2827
2828 @Test
2829 public void testScanner_JoinedScannersWithLimits() throws IOException {
2830 final byte[] cf_first = Bytes.toBytes("first");
2831 final byte[] cf_second = Bytes.toBytes("second");
2832
2833 this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
2834 try {
2835 final byte[] col_a = Bytes.toBytes("a");
2836 final byte[] col_b = Bytes.toBytes("b");
2837
2838 Put put;
2839
2840 for (int i = 0; i < 10; i++) {
2841 put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
2842 put.add(cf_first, col_a, Bytes.toBytes(i));
2843 if (i < 5) {
2844 put.add(cf_first, col_b, Bytes.toBytes(i));
2845 put.add(cf_second, col_a, Bytes.toBytes(i));
2846 put.add(cf_second, col_b, Bytes.toBytes(i));
2847 }
2848 region.put(put);
2849 }
2850
2851 Scan scan = new Scan();
2852 scan.setLoadColumnFamiliesOnDemand(true);
2853 Filter bogusFilter = new FilterBase() {
2854 @Override
2855 public boolean isFamilyEssential(byte[] name) {
2856 return Bytes.equals(name, cf_first);
2857 }
2858 };
2859
2860 scan.setFilter(bogusFilter);
2861 InternalScanner s = region.getScanner(scan);
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892 List<Cell> results = new ArrayList<Cell>();
2893 int index = 0;
2894 while (true) {
2895 boolean more = s.next(results, 3);
2896 if ((index >> 1) < 5) {
2897 if (index % 2 == 0)
2898 assertEquals(results.size(), 3);
2899 else
2900 assertEquals(results.size(), 1);
2901 } else
2902 assertEquals(results.size(), 1);
2903 results.clear();
2904 index++;
2905 if (!more)
2906 break;
2907 }
2908 } finally {
2909 HRegion.closeHRegion(this.region);
2910 this.region = null;
2911 }
2912 }
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922 @Test
2923 public void testBasicSplit() throws Exception {
2924 byte[][] families = { fam1, fam2, fam3 };
2925
2926 Configuration hc = initSplit();
2927
2928 String method = this.getName();
2929 this.region = initHRegion(tableName, method, hc, families);
2930
2931 try {
2932 LOG.info("" + HBaseTestCase.addContent(region, fam3));
2933 region.flushcache();
2934 region.compactStores();
2935 byte[] splitRow = region.checkSplit();
2936 assertNotNull(splitRow);
2937 LOG.info("SplitRow: " + Bytes.toString(splitRow));
2938 HRegion[] regions = splitRegion(region, splitRow);
2939 try {
2940
2941
2942
2943 for (int i = 0; i < regions.length; i++) {
2944 regions[i] = HRegion.openHRegion(regions[i], null);
2945 }
2946
2947
2948 assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
2949 assertGet(regions[1], fam3, splitRow);
2950
2951 assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
2952 assertScan(regions[1], fam3, splitRow);
2953
2954 for (int i = 0; i < regions.length; i++) {
2955
2956
2957 for (int j = 0; j < 2; j++) {
2958 HBaseTestCase.addContent(regions[i], fam3);
2959 }
2960 HBaseTestCase.addContent(regions[i], fam2);
2961 HBaseTestCase.addContent(regions[i], fam1);
2962 regions[i].flushcache();
2963 }
2964
2965 byte[][] midkeys = new byte[regions.length][];
2966
2967 for (int i = 0; i < regions.length; i++) {
2968 regions[i].compactStores();
2969 midkeys[i] = regions[i].checkSplit();
2970 }
2971
2972 TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
2973
2974
2975 for (int i = 0; i < regions.length; i++) {
2976 HRegion[] rs = null;
2977 if (midkeys[i] != null) {
2978 rs = splitRegion(regions[i], midkeys[i]);
2979 for (int j = 0; j < rs.length; j++) {
2980 sortedMap.put(Bytes.toString(rs[j].getRegionName()), HRegion.openHRegion(rs[j], null));
2981 }
2982 }
2983 }
2984 LOG.info("Made 4 regions");
2985
2986
2987 int interval = (LAST_CHAR - FIRST_CHAR) / 3;
2988 byte[] b = Bytes.toBytes(START_KEY);
2989 for (HRegion r : sortedMap.values()) {
2990 assertGet(r, fam3, b);
2991 b[0] += interval;
2992 }
2993 } finally {
2994 for (int i = 0; i < regions.length; i++) {
2995 try {
2996 regions[i].close();
2997 } catch (IOException e) {
2998
2999 }
3000 }
3001 }
3002 } finally {
3003 HRegion.closeHRegion(this.region);
3004 this.region = null;
3005 }
3006 }
3007
3008 @Test
3009 public void testSplitRegion() throws IOException {
3010 byte[] qualifier = Bytes.toBytes("qualifier");
3011 Configuration hc = initSplit();
3012 int numRows = 10;
3013 byte[][] families = { fam1, fam3 };
3014
3015
3016 String method = this.getName();
3017 this.region = initHRegion(tableName, method, hc, families);
3018
3019
3020 int startRow = 100;
3021 putData(startRow, numRows, qualifier, families);
3022 int splitRow = startRow + numRows;
3023 putData(splitRow, numRows, qualifier, families);
3024 region.flushcache();
3025
3026 HRegion[] regions = null;
3027 try {
3028 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
3029
3030 for (int i = 0; i < regions.length; i++) {
3031 regions[i] = HRegion.openHRegion(regions[i], null);
3032 }
3033
3034 assertEquals(2, regions.length);
3035
3036
3037
3038 verifyData(regions[0], startRow, numRows, qualifier, families);
3039 verifyData(regions[1], splitRow, numRows, qualifier, families);
3040
3041 } finally {
3042 HRegion.closeHRegion(this.region);
3043 this.region = null;
3044 }
3045 }
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057 @Test
3058 public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3059 byte[] family = Bytes.toBytes("family");
3060 int numRows = 1000;
3061 int flushAndScanInterval = 10;
3062 int compactInterval = 10 * flushAndScanInterval;
3063
3064 String method = "testFlushCacheWhileScanning";
3065 this.region = initHRegion(tableName, method, CONF, family);
3066 try {
3067 FlushThread flushThread = new FlushThread();
3068 flushThread.start();
3069
3070 Scan scan = new Scan();
3071 scan.addFamily(family);
3072 scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
3073 new BinaryComparator(Bytes.toBytes(5L))));
3074
3075 int expectedCount = 0;
3076 List<Cell> res = new ArrayList<Cell>();
3077
3078 boolean toggle = true;
3079 for (long i = 0; i < numRows; i++) {
3080 Put put = new Put(Bytes.toBytes(i));
3081 put.setDurability(Durability.SKIP_WAL);
3082 put.add(family, qual1, Bytes.toBytes(i % 10));
3083 region.put(put);
3084
3085 if (i != 0 && i % compactInterval == 0) {
3086
3087 region.compactStores(true);
3088 }
3089
3090 if (i % 10 == 5L) {
3091 expectedCount++;
3092 }
3093
3094 if (i != 0 && i % flushAndScanInterval == 0) {
3095 res.clear();
3096 InternalScanner scanner = region.getScanner(scan);
3097 if (toggle) {
3098 flushThread.flush();
3099 }
3100 while (scanner.next(res))
3101 ;
3102 if (!toggle) {
3103 flushThread.flush();
3104 }
3105 assertEquals("i=" + i, expectedCount, res.size());
3106 toggle = !toggle;
3107 }
3108 }
3109
3110 flushThread.done();
3111 flushThread.join();
3112 flushThread.checkNoError();
3113 } finally {
3114 HRegion.closeHRegion(this.region);
3115 this.region = null;
3116 }
3117 }
3118
3119 protected class FlushThread extends Thread {
3120 private volatile boolean done;
3121 private Throwable error = null;
3122
3123 public void done() {
3124 done = true;
3125 synchronized (this) {
3126 interrupt();
3127 }
3128 }
3129
3130 public void checkNoError() {
3131 if (error != null) {
3132 assertNull(error);
3133 }
3134 }
3135
3136 @Override
3137 public void run() {
3138 done = false;
3139 while (!done) {
3140 synchronized (this) {
3141 try {
3142 wait();
3143 } catch (InterruptedException ignored) {
3144 if (done) {
3145 break;
3146 }
3147 }
3148 }
3149 try {
3150 region.flushcache();
3151 } catch (IOException e) {
3152 if (!done) {
3153 LOG.error("Error while flusing cache", e);
3154 error = e;
3155 }
3156 break;
3157 }
3158 }
3159
3160 }
3161
3162 public void flush() {
3163 synchronized (this) {
3164 notify();
3165 }
3166
3167 }
3168 }
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179 @Test
3180 public void testWritesWhileScanning() throws IOException, InterruptedException {
3181 int testCount = 100;
3182 int numRows = 1;
3183 int numFamilies = 10;
3184 int numQualifiers = 100;
3185 int flushInterval = 7;
3186 int compactInterval = 5 * flushInterval;
3187 byte[][] families = new byte[numFamilies][];
3188 for (int i = 0; i < numFamilies; i++) {
3189 families[i] = Bytes.toBytes("family" + i);
3190 }
3191 byte[][] qualifiers = new byte[numQualifiers][];
3192 for (int i = 0; i < numQualifiers; i++) {
3193 qualifiers[i] = Bytes.toBytes("qual" + i);
3194 }
3195
3196 String method = "testWritesWhileScanning";
3197 this.region = initHRegion(tableName, method, CONF, families);
3198 try {
3199 PutThread putThread = new PutThread(numRows, families, qualifiers);
3200 putThread.start();
3201 putThread.waitForFirstPut();
3202
3203 FlushThread flushThread = new FlushThread();
3204 flushThread.start();
3205
3206 Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3207
3208 int expectedCount = numFamilies * numQualifiers;
3209 List<Cell> res = new ArrayList<Cell>();
3210
3211 long prevTimestamp = 0L;
3212 for (int i = 0; i < testCount; i++) {
3213
3214 if (i != 0 && i % compactInterval == 0) {
3215 region.compactStores(true);
3216 }
3217
3218 if (i != 0 && i % flushInterval == 0) {
3219 flushThread.flush();
3220 }
3221
3222 boolean previousEmpty = res.isEmpty();
3223 res.clear();
3224 InternalScanner scanner = region.getScanner(scan);
3225 while (scanner.next(res))
3226 ;
3227 if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3228 assertEquals("i=" + i, expectedCount, res.size());
3229 long timestamp = res.get(0).getTimestamp();
3230 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3231 timestamp >= prevTimestamp);
3232 prevTimestamp = timestamp;
3233 }
3234 }
3235
3236 putThread.done();
3237
3238 region.flushcache();
3239
3240 putThread.join();
3241 putThread.checkNoError();
3242
3243 flushThread.done();
3244 flushThread.join();
3245 flushThread.checkNoError();
3246 } finally {
3247 try {
3248 HRegion.closeHRegion(this.region);
3249 } catch (DroppedSnapshotException dse) {
3250
3251
3252
3253
3254 }
3255 this.region = null;
3256 }
3257 }
3258
3259 protected class PutThread extends Thread {
3260 private volatile boolean done;
3261 private volatile int numPutsFinished = 0;
3262
3263 private Throwable error = null;
3264 private int numRows;
3265 private byte[][] families;
3266 private byte[][] qualifiers;
3267
3268 private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3269 this.numRows = numRows;
3270 this.families = families;
3271 this.qualifiers = qualifiers;
3272 }
3273
3274
3275
3276
3277 public void waitForFirstPut() throws InterruptedException {
3278
3279 while (numPutsFinished == 0) {
3280 checkNoError();
3281 Thread.sleep(50);
3282 }
3283 }
3284
3285 public void done() {
3286 done = true;
3287 synchronized (this) {
3288 interrupt();
3289 }
3290 }
3291
3292 public void checkNoError() {
3293 if (error != null) {
3294 assertNull(error);
3295 }
3296 }
3297
3298 @Override
3299 public void run() {
3300 done = false;
3301 while (!done) {
3302 try {
3303 for (int r = 0; r < numRows; r++) {
3304 byte[] row = Bytes.toBytes("row" + r);
3305 Put put = new Put(row);
3306 put.setDurability(Durability.SKIP_WAL);
3307 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3308 for (byte[] family : families) {
3309 for (byte[] qualifier : qualifiers) {
3310 put.add(family, qualifier, (long) numPutsFinished, value);
3311 }
3312 }
3313 region.put(put);
3314 numPutsFinished++;
3315 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3316 System.out.println("put iteration = " + numPutsFinished);
3317 Delete delete = new Delete(row, (long) numPutsFinished - 30);
3318 region.delete(delete);
3319 }
3320 numPutsFinished++;
3321 }
3322 } catch (InterruptedIOException e) {
3323
3324 } catch (IOException e) {
3325 LOG.error("error while putting records", e);
3326 error = e;
3327 break;
3328 }
3329 }
3330
3331 }
3332
3333 }
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344 @Test
3345 public void testWritesWhileGetting() throws Exception {
3346 int testCount = 100;
3347 int numRows = 1;
3348 int numFamilies = 10;
3349 int numQualifiers = 100;
3350 int compactInterval = 100;
3351 byte[][] families = new byte[numFamilies][];
3352 for (int i = 0; i < numFamilies; i++) {
3353 families[i] = Bytes.toBytes("family" + i);
3354 }
3355 byte[][] qualifiers = new byte[numQualifiers][];
3356 for (int i = 0; i < numQualifiers; i++) {
3357 qualifiers[i] = Bytes.toBytes("qual" + i);
3358 }
3359
3360
3361 String method = "testWritesWhileGetting";
3362
3363
3364
3365
3366
3367 Configuration conf = HBaseConfiguration.create(CONF);
3368 conf.setInt("hbase.hstore.compaction.min", 1);
3369 conf.setInt("hbase.hstore.compaction.max", 1000);
3370 this.region = initHRegion(tableName, method, conf, families);
3371 PutThread putThread = null;
3372 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3373 try {
3374 putThread = new PutThread(numRows, families, qualifiers);
3375 putThread.start();
3376 putThread.waitForFirstPut();
3377
3378
3379 ctx.addThread(new RepeatingTestThread(ctx) {
3380 private int flushesSinceCompact = 0;
3381 private final int maxFlushesSinceCompact = 20;
3382
3383 @Override
3384 public void doAnAction() throws Exception {
3385 if (region.flushcache().isCompactionNeeded()) {
3386 ++flushesSinceCompact;
3387 }
3388
3389
3390 if (flushesSinceCompact == maxFlushesSinceCompact) {
3391 region.compactStores(false);
3392 flushesSinceCompact = 0;
3393 }
3394 }
3395 });
3396 ctx.startThreads();
3397
3398 Get get = new Get(Bytes.toBytes("row0"));
3399 Result result = null;
3400
3401 int expectedCount = numFamilies * numQualifiers;
3402
3403 long prevTimestamp = 0L;
3404 for (int i = 0; i < testCount; i++) {
3405
3406 boolean previousEmpty = result == null || result.isEmpty();
3407 result = region.get(get);
3408 if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3409 assertEquals("i=" + i, expectedCount, result.size());
3410
3411
3412 long timestamp = 0;
3413 for (Cell kv : result.rawCells()) {
3414 if (CellUtil.matchingFamily(kv, families[0])
3415 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3416 timestamp = kv.getTimestamp();
3417 }
3418 }
3419 assertTrue(timestamp >= prevTimestamp);
3420 prevTimestamp = timestamp;
3421 Cell previousKV = null;
3422
3423 for (Cell kv : result.rawCells()) {
3424 byte[] thisValue = CellUtil.cloneValue(kv);
3425 if (previousKV != null) {
3426 if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3427 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3428 + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3429 + "(memStoreTS:" + kv.getMvccVersion() + ")");
3430 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3431 }
3432 }
3433 previousKV = kv;
3434 }
3435 }
3436 }
3437 } finally {
3438 if (putThread != null)
3439 putThread.done();
3440
3441 region.flushcache();
3442
3443 if (putThread != null) {
3444 putThread.join();
3445 putThread.checkNoError();
3446 }
3447
3448 ctx.stop();
3449 HRegion.closeHRegion(this.region);
3450 this.region = null;
3451 }
3452 }
3453
3454 @Test
3455 public void testHolesInMeta() throws Exception {
3456 byte[] family = Bytes.toBytes("family");
3457 this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF,
3458 false, family);
3459 try {
3460 byte[] rowNotServed = Bytes.toBytes("a");
3461 Get g = new Get(rowNotServed);
3462 try {
3463 region.get(g);
3464 fail();
3465 } catch (WrongRegionException x) {
3466
3467 }
3468 byte[] row = Bytes.toBytes("y");
3469 g = new Get(row);
3470 region.get(g);
3471 } finally {
3472 HRegion.closeHRegion(this.region);
3473 this.region = null;
3474 }
3475 }
3476
3477 @Test
3478 public void testIndexesScanWithOneDeletedRow() throws IOException {
3479 byte[] family = Bytes.toBytes("family");
3480
3481
3482 String method = "testIndexesScanWithOneDeletedRow";
3483 this.region = initHRegion(tableName, method, CONF, family);
3484 try {
3485 Put put = new Put(Bytes.toBytes(1L));
3486 put.add(family, qual1, 1L, Bytes.toBytes(1L));
3487 region.put(put);
3488
3489 region.flushcache();
3490
3491 Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3492 region.delete(delete);
3493
3494 put = new Put(Bytes.toBytes(2L));
3495 put.add(family, qual1, 2L, Bytes.toBytes(2L));
3496 region.put(put);
3497
3498 Scan idxScan = new Scan();
3499 idxScan.addFamily(family);
3500 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
3501 new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
3502 new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
3503 CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
3504 InternalScanner scanner = region.getScanner(idxScan);
3505 List<Cell> res = new ArrayList<Cell>();
3506
3507 while (scanner.next(res))
3508 ;
3509 assertEquals(1L, res.size());
3510 } finally {
3511 HRegion.closeHRegion(this.region);
3512 this.region = null;
3513 }
3514 }
3515
3516
3517
3518
3519 @Test
3520 public void testBloomFilterSize() throws IOException {
3521 byte[] fam1 = Bytes.toBytes("fam1");
3522 byte[] qf1 = Bytes.toBytes("col");
3523 byte[] val1 = Bytes.toBytes("value1");
3524
3525 HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
3526 .setBloomFilterType(BloomType.ROWCOL);
3527
3528 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3529 htd.addFamily(hcd);
3530 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3531 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3532 try {
3533 int num_unique_rows = 10;
3534 int duplicate_multiplier = 2;
3535 int num_storefiles = 4;
3536
3537 int version = 0;
3538 for (int f = 0; f < num_storefiles; f++) {
3539 for (int i = 0; i < duplicate_multiplier; i++) {
3540 for (int j = 0; j < num_unique_rows; j++) {
3541 Put put = new Put(Bytes.toBytes("row" + j));
3542 put.setDurability(Durability.SKIP_WAL);
3543 put.add(fam1, qf1, version++, val1);
3544 region.put(put);
3545 }
3546 }
3547 region.flushcache();
3548 }
3549
3550 HStore store = (HStore) region.getStore(fam1);
3551 Collection<StoreFile> storeFiles = store.getStorefiles();
3552 for (StoreFile storefile : storeFiles) {
3553 StoreFile.Reader reader = storefile.getReader();
3554 reader.loadFileInfo();
3555 reader.loadBloomfilter();
3556 assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
3557 assertEquals(num_unique_rows, reader.getFilterEntries());
3558 }
3559
3560 region.compactStores(true);
3561
3562
3563 storeFiles = store.getStorefiles();
3564 for (StoreFile storefile : storeFiles) {
3565 StoreFile.Reader reader = storefile.getReader();
3566 reader.loadFileInfo();
3567 reader.loadBloomfilter();
3568 assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
3569 assertEquals(num_unique_rows, reader.getFilterEntries());
3570 }
3571 } finally {
3572 HRegion.closeHRegion(this.region);
3573 this.region = null;
3574 }
3575 }
3576
3577 @Test
3578 public void testAllColumnsWithBloomFilter() throws IOException {
3579 byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
3580 byte[] FAMILY = Bytes.toBytes("family");
3581
3582
3583 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
3584 .setBloomFilterType(BloomType.ROWCOL);
3585 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
3586 htd.addFamily(hcd);
3587 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3588 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3589 try {
3590
3591 byte row[] = Bytes.toBytes("row:" + 0);
3592 byte column[] = Bytes.toBytes("column:" + 0);
3593 Put put = new Put(row);
3594 put.setDurability(Durability.SKIP_WAL);
3595 for (long idx = 1; idx <= 4; idx++) {
3596 put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
3597 }
3598 region.put(put);
3599
3600
3601 region.flushcache();
3602
3603
3604 Get get = new Get(row);
3605 get.setMaxVersions();
3606 Cell[] kvs = region.get(get).rawCells();
3607
3608
3609 assertEquals(4, kvs.length);
3610 checkOneCell(kvs[0], FAMILY, 0, 0, 4);
3611 checkOneCell(kvs[1], FAMILY, 0, 0, 3);
3612 checkOneCell(kvs[2], FAMILY, 0, 0, 2);
3613 checkOneCell(kvs[3], FAMILY, 0, 0, 1);
3614 } finally {
3615 HRegion.closeHRegion(this.region);
3616 this.region = null;
3617 }
3618 }
3619
3620
3621
3622
3623
3624
3625 @Test
3626 public void testDeleteRowWithBloomFilter() throws IOException {
3627 byte[] familyName = Bytes.toBytes("familyName");
3628
3629
3630 HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
3631 .setBloomFilterType(BloomType.ROWCOL);
3632
3633 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3634 htd.addFamily(hcd);
3635 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3636 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3637 try {
3638
3639 byte row[] = Bytes.toBytes("row1");
3640 byte col[] = Bytes.toBytes("col1");
3641
3642 Put put = new Put(row);
3643 put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
3644 region.put(put);
3645 region.flushcache();
3646
3647 Delete del = new Delete(row);
3648 region.delete(del);
3649 region.flushcache();
3650
3651
3652 Get get = new Get(row);
3653 get.addColumn(familyName, col);
3654
3655 Cell[] keyValues = region.get(get).rawCells();
3656 assertTrue(keyValues.length == 0);
3657 } finally {
3658 HRegion.closeHRegion(this.region);
3659 this.region = null;
3660 }
3661 }
3662
3663 @Test
3664 public void testgetHDFSBlocksDistribution() throws Exception {
3665 HBaseTestingUtility htu = new HBaseTestingUtility();
3666 final int DEFAULT_BLOCK_SIZE = 1024;
3667 htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
3668 htu.getConfiguration().setInt("dfs.replication", 2);
3669
3670
3671 MiniHBaseCluster cluster = null;
3672 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
3673 int regionServersCount = 3;
3674
3675 try {
3676 cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
3677 byte[][] families = { fam1, fam2 };
3678 HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);
3679
3680
3681 byte row[] = Bytes.toBytes("row1");
3682 byte col[] = Bytes.toBytes("col1");
3683
3684 Put put = new Put(row);
3685 put.add(fam1, col, 1, Bytes.toBytes("test1"));
3686 put.add(fam2, col, 1, Bytes.toBytes("test2"));
3687 ht.put(put);
3688
3689 HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
3690 .get(0);
3691 firstRegion.flushcache();
3692 HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
3693
3694
3695
3696
3697
3698 long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
3699
3700 String topHost = blocksDistribution1.getTopHosts().get(0);
3701 long topHostWeight = blocksDistribution1.getWeight(topHost);
3702 assertTrue(uniqueBlocksWeight1 == topHostWeight);
3703
3704
3705
3706 HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
3707 htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
3708 long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
3709
3710 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
3711
3712 ht.close();
3713 } finally {
3714 if (cluster != null) {
3715 htu.shutdownMiniCluster();
3716 }
3717 }
3718 }
3719
3720
3721
3722
3723
3724
3725
3726 @Test
3727 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
3728 TableName tableName = TableName.valueOf(name.getMethodName());
3729 HRegionInfo info = null;
3730 try {
3731 FileSystem fs = Mockito.mock(FileSystem.class);
3732 Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
3733 HTableDescriptor htd = new HTableDescriptor(tableName);
3734 htd.addFamily(new HColumnDescriptor("cf"));
3735 info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
3736 HConstants.EMPTY_BYTE_ARRAY, false);
3737 Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
3738 region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
3739
3740 region.initialize();
3741 fail("Region initialization should fail due to IOException");
3742 } catch (IOException io) {
3743 List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
3744 for (MonitoredTask monitoredTask : tasks) {
3745 if (!(monitoredTask instanceof MonitoredRPCHandler)
3746 && monitoredTask.getDescription().contains(region.toString())) {
3747 assertTrue("Region state should be ABORTED.",
3748 monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
3749 break;
3750 }
3751 }
3752 } finally {
3753 HRegion.closeHRegion(region);
3754 }
3755 }
3756
3757
3758
3759
3760
3761 @Test
3762 public void testRegionInfoFileCreation() throws IOException {
3763 Path rootDir = new Path(dir + "testRegionInfoFileCreation");
3764
3765 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
3766 htd.addFamily(new HColumnDescriptor("cf"));
3767
3768 HRegionInfo hri = new HRegionInfo(htd.getTableName());
3769
3770
3771 HRegion region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
3772
3773 Path regionDir = region.getRegionFileSystem().getRegionDir();
3774 FileSystem fs = region.getRegionFileSystem().getFileSystem();
3775 HRegion.closeHRegion(region);
3776
3777 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
3778
3779
3780 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3781 fs.exists(regionInfoFile));
3782
3783
3784 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3785 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3786 HRegion.closeHRegion(region);
3787
3788
3789 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3790 fs.exists(regionInfoFile));
3791
3792
3793 fs.delete(regionInfoFile);
3794 assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
3795 fs.exists(regionInfoFile));
3796
3797 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3798
3799 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3800 HRegion.closeHRegion(region);
3801
3802
3803 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3804 fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
3805 }
3806
3807
3808
3809
3810 private static class Incrementer implements Runnable {
3811 private HRegion region;
3812 private final static byte[] incRow = Bytes.toBytes("incRow");
3813 private final static byte[] family = Bytes.toBytes("family");
3814 private final static byte[] qualifier = Bytes.toBytes("qualifier");
3815 private final static long ONE = 1l;
3816 private int incCounter;
3817
3818 public Incrementer(HRegion region, int incCounter) {
3819 this.region = region;
3820 this.incCounter = incCounter;
3821 }
3822
3823 @Override
3824 public void run() {
3825 int count = 0;
3826 while (count < incCounter) {
3827 Increment inc = new Increment(incRow);
3828 inc.addColumn(family, qualifier, ONE);
3829 count++;
3830 try {
3831 region.increment(inc);
3832 } catch (IOException e) {
3833 e.printStackTrace();
3834 break;
3835 }
3836 }
3837 }
3838 }
3839
3840
3841
3842
3843
3844 @Test
3845 public void testParallelIncrementWithMemStoreFlush() throws Exception {
3846 byte[] family = Incrementer.family;
3847 this.region = initHRegion(tableName, method, CONF, family);
3848 final HRegion region = this.region;
3849 final AtomicBoolean incrementDone = new AtomicBoolean(false);
3850 Runnable flusher = new Runnable() {
3851 @Override
3852 public void run() {
3853 while (!incrementDone.get()) {
3854 try {
3855 region.flushcache();
3856 } catch (Exception e) {
3857 e.printStackTrace();
3858 }
3859 }
3860 }
3861 };
3862
3863
3864 int threadNum = 20;
3865 int incCounter = 100;
3866 long expected = threadNum * incCounter;
3867 Thread[] incrementers = new Thread[threadNum];
3868 Thread flushThread = new Thread(flusher);
3869 for (int i = 0; i < threadNum; i++) {
3870 incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
3871 incrementers[i].start();
3872 }
3873 flushThread.start();
3874 for (int i = 0; i < threadNum; i++) {
3875 incrementers[i].join();
3876 }
3877
3878 incrementDone.set(true);
3879 flushThread.join();
3880
3881 Get get = new Get(Incrementer.incRow);
3882 get.addColumn(Incrementer.family, Incrementer.qualifier);
3883 get.setMaxVersions(1);
3884 Result res = this.region.get(get);
3885 List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
3886
3887
3888 assertEquals(kvs.size(), 1);
3889 Cell kv = kvs.get(0);
3890 assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
3891 this.region = null;
3892 }
3893
3894
3895
3896
3897 private static class Appender implements Runnable {
3898 private HRegion region;
3899 private final static byte[] appendRow = Bytes.toBytes("appendRow");
3900 private final static byte[] family = Bytes.toBytes("family");
3901 private final static byte[] qualifier = Bytes.toBytes("qualifier");
3902 private final static byte[] CHAR = Bytes.toBytes("a");
3903 private int appendCounter;
3904
3905 public Appender(HRegion region, int appendCounter) {
3906 this.region = region;
3907 this.appendCounter = appendCounter;
3908 }
3909
3910 @Override
3911 public void run() {
3912 int count = 0;
3913 while (count < appendCounter) {
3914 Append app = new Append(appendRow);
3915 app.add(family, qualifier, CHAR);
3916 count++;
3917 try {
3918 region.append(app);
3919 } catch (IOException e) {
3920 e.printStackTrace();
3921 break;
3922 }
3923 }
3924 }
3925 }
3926
3927
3928
3929
3930
3931 @Test
3932 public void testParallelAppendWithMemStoreFlush() throws Exception {
3933 byte[] family = Appender.family;
3934 this.region = initHRegion(tableName, method, CONF, family);
3935 final HRegion region = this.region;
3936 final AtomicBoolean appendDone = new AtomicBoolean(false);
3937 Runnable flusher = new Runnable() {
3938 @Override
3939 public void run() {
3940 while (!appendDone.get()) {
3941 try {
3942 region.flushcache();
3943 } catch (Exception e) {
3944 e.printStackTrace();
3945 }
3946 }
3947 }
3948 };
3949
3950
3951
3952 int threadNum = 20;
3953 int appendCounter = 100;
3954 byte[] expected = new byte[threadNum * appendCounter];
3955 for (int i = 0; i < threadNum * appendCounter; i++) {
3956 System.arraycopy(Appender.CHAR, 0, expected, i, 1);
3957 }
3958 Thread[] appenders = new Thread[threadNum];
3959 Thread flushThread = new Thread(flusher);
3960 for (int i = 0; i < threadNum; i++) {
3961 appenders[i] = new Thread(new Appender(this.region, appendCounter));
3962 appenders[i].start();
3963 }
3964 flushThread.start();
3965 for (int i = 0; i < threadNum; i++) {
3966 appenders[i].join();
3967 }
3968
3969 appendDone.set(true);
3970 flushThread.join();
3971
3972 Get get = new Get(Appender.appendRow);
3973 get.addColumn(Appender.family, Appender.qualifier);
3974 get.setMaxVersions(1);
3975 Result res = this.region.get(get);
3976 List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
3977
3978
3979 assertEquals(kvs.size(), 1);
3980 Cell kv = kvs.get(0);
3981 byte[] appendResult = new byte[kv.getValueLength()];
3982 System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
3983 assertArrayEquals(expected, appendResult);
3984 this.region = null;
3985 }
3986
3987
3988
3989
3990
3991 @Test
3992 public void testPutWithMemStoreFlush() throws Exception {
3993 byte[] family = Bytes.toBytes("family");
3994 ;
3995 byte[] qualifier = Bytes.toBytes("qualifier");
3996 byte[] row = Bytes.toBytes("putRow");
3997 byte[] value = null;
3998 this.region = initHRegion(tableName, method, CONF, family);
3999 Put put = null;
4000 Get get = null;
4001 List<Cell> kvs = null;
4002 Result res = null;
4003
4004 put = new Put(row);
4005 value = Bytes.toBytes("value0");
4006 put.add(family, qualifier, 1234567l, value);
4007 region.put(put);
4008 get = new Get(row);
4009 get.addColumn(family, qualifier);
4010 get.setMaxVersions();
4011 res = this.region.get(get);
4012 kvs = res.getColumnCells(family, qualifier);
4013 assertEquals(1, kvs.size());
4014 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4015
4016 region.flushcache();
4017 get = new Get(row);
4018 get.addColumn(family, qualifier);
4019 get.setMaxVersions();
4020 res = this.region.get(get);
4021 kvs = res.getColumnCells(family, qualifier);
4022 assertEquals(1, kvs.size());
4023 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4024
4025 put = new Put(row);
4026 value = Bytes.toBytes("value1");
4027 put.add(family, qualifier, 1234567l, value);
4028 region.put(put);
4029 get = new Get(row);
4030 get.addColumn(family, qualifier);
4031 get.setMaxVersions();
4032 res = this.region.get(get);
4033 kvs = res.getColumnCells(family, qualifier);
4034 assertEquals(1, kvs.size());
4035 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4036
4037 region.flushcache();
4038 get = new Get(row);
4039 get.addColumn(family, qualifier);
4040 get.setMaxVersions();
4041 res = this.region.get(get);
4042 kvs = res.getColumnCells(family, qualifier);
4043 assertEquals(1, kvs.size());
4044 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4045 }
4046
4047 @Test
4048 public void testDurability() throws Exception {
4049 String method = "testDurability";
4050
4051
4052
4053
4054
4055 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4056 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4057 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4058
4059 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4060 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4061 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4062
4063 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4064 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4065
4066 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
4067 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4068
4069 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
4070 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
4071 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
4072
4073
4074 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4075 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4076 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4077 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4078 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
4079 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4080
4081 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4082 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4083 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4084 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4085 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
4086 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
4087
4088
4089 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4090 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4091 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4092 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
4093 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
4094 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
4095
4096 }
4097
4098 private void durabilityTest(String method, Durability tableDurability,
4099 Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
4100 final boolean expectSyncFromLogSyncer) throws Exception {
4101 Configuration conf = HBaseConfiguration.create(CONF);
4102 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
4103 TableName tableName = TableName.valueOf(method);
4104 byte[] family = Bytes.toBytes("family");
4105 Path logDir = new Path(new Path(dir + method), "log");
4106 HLog hlog = HLogFactory.createHLog(FILESYSTEM, logDir, UUID.randomUUID().toString(), conf);
4107 final HLog log = spy(hlog);
4108 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
4109 HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, log,
4110 new byte[][] { family });
4111
4112 Put put = new Put(Bytes.toBytes("r1"));
4113 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
4114 put.setDurability(mutationDurability);
4115 region.put(put);
4116
4117
4118 verify(log, expectAppend ? times(1) : never())
4119 .appendNoSync((HRegionInfo)any(), eq(tableName), (WALEdit)any(), (List<UUID>)any(),
4120 anyLong(), (HTableDescriptor)any(), (AtomicLong)any(), anyBoolean(), anyLong(), anyLong());
4121
4122
4123 if (expectSync || expectSyncFromLogSyncer) {
4124 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
4125 @Override
4126 public boolean evaluate() throws Exception {
4127 try {
4128 if (expectSync) {
4129 verify(log, times(1)).sync(anyLong());
4130 } else if (expectSyncFromLogSyncer) {
4131 verify(log, times(1)).sync();
4132 }
4133 } catch (Throwable ignore) {
4134 }
4135 return true;
4136 }
4137 });
4138 } else {
4139 verify(log, never()).sync(anyLong());
4140 verify(log, never()).sync();
4141 }
4142
4143 HRegion.closeHRegion(this.region);
4144 this.region = null;
4145 }
4146
4147 private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4148 for (int i = startRow; i < startRow + numRows; i++) {
4149 Put put = new Put(Bytes.toBytes("" + i));
4150 put.setDurability(Durability.SKIP_WAL);
4151 for (byte[] family : families) {
4152 put.add(family, qf, null);
4153 }
4154 region.put(put);
4155 }
4156 }
4157
4158 private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
4159 throws IOException {
4160 for (int i = startRow; i < startRow + numRows; i++) {
4161 byte[] row = Bytes.toBytes("" + i);
4162 Get get = new Get(row);
4163 for (byte[] family : families) {
4164 get.addColumn(family, qf);
4165 }
4166 Result result = newReg.get(get);
4167 Cell[] raw = result.rawCells();
4168 assertEquals(families.length, result.size());
4169 for (int j = 0; j < families.length; j++) {
4170 assertTrue(CellUtil.matchingRow(raw[j], row));
4171 assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
4172 assertTrue(CellUtil.matchingQualifier(raw[j], qf));
4173 }
4174 }
4175 }
4176
4177 private void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
4178
4179 Get get = new Get(k).addFamily(family).setMaxVersions();
4180 Cell[] results = r.get(get).rawCells();
4181 for (int j = 0; j < results.length; j++) {
4182 byte[] tmp = CellUtil.cloneValue(results[j]);
4183
4184 assertTrue(Bytes.equals(k, tmp));
4185 }
4186 }
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199 private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
4200 throws IOException {
4201 byte[][] families = { fs };
4202 Scan scan = new Scan();
4203 for (int i = 0; i < families.length; i++)
4204 scan.addFamily(families[i]);
4205 InternalScanner s = r.getScanner(scan);
4206 try {
4207 List<Cell> curVals = new ArrayList<Cell>();
4208 boolean first = true;
4209 OUTER_LOOP: while (s.next(curVals)) {
4210 for (Cell kv : curVals) {
4211 byte[] val = CellUtil.cloneValue(kv);
4212 byte[] curval = val;
4213 if (first) {
4214 first = false;
4215 assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4216 } else {
4217
4218 break OUTER_LOOP;
4219 }
4220 }
4221 }
4222 } finally {
4223 s.close();
4224 }
4225 }
4226
4227
4228
4229
4230
4231 @Test
4232 public void testFlushResult() throws IOException {
4233 String method = name.getMethodName();
4234 byte[] tableName = Bytes.toBytes(method);
4235 byte[] family = Bytes.toBytes("family");
4236
4237 this.region = initHRegion(tableName, method, family);
4238
4239
4240 HRegion.FlushResult fr = region.flushcache();
4241 assertFalse(fr.isFlushSucceeded());
4242 assertFalse(fr.isCompactionNeeded());
4243
4244
4245 for (int i = 0; i < 2; i++) {
4246 Put put = new Put(tableName).add(family, family, tableName);
4247 region.put(put);
4248 fr = region.flushcache();
4249 assertTrue(fr.isFlushSucceeded());
4250 assertFalse(fr.isCompactionNeeded());
4251 }
4252
4253
4254 for (int i = 0; i < 2; i++) {
4255 Put put = new Put(tableName).add(family, family, tableName);
4256 region.put(put);
4257 fr = region.flushcache();
4258 assertTrue(fr.isFlushSucceeded());
4259 assertTrue(fr.isCompactionNeeded());
4260 }
4261 }
4262
4263 private Configuration initSplit() {
4264
4265 CONF.setInt("hbase.hstore.compactionThreshold", 2);
4266
4267
4268 CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4269
4270 CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4271
4272
4273 CONF.setLong("hbase.client.pause", 15 * 1000);
4274
4275
4276
4277 CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4278 return CONF;
4279 }
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290 public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
4291 byte[]... families) throws IOException {
4292 return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
4293 }
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4305 byte[]... families) throws IOException {
4306 return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4307 }
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4320 boolean isReadOnly, byte[]... families) throws IOException {
4321 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4322 }
4323
4324 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4325 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4326 throws IOException {
4327 return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
4328 Durability.SYNC_WAL, null, families);
4329 }
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4344 String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4345 HLog hlog, byte[]... families) throws IOException {
4346 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly, durability, hlog, families);
4347 }
4348
4349
4350
4351
4352
4353 private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
4354 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4355 assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
4356 Bytes.toString(CellUtil.cloneRow(kv)));
4357 assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
4358 Bytes.toString(CellUtil.cloneFamily(kv)));
4359 assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
4360 Bytes.toString(CellUtil.cloneQualifier(kv)));
4361 assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
4362 assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
4363 Bytes.toString(CellUtil.cloneValue(kv)));
4364 }
4365
4366 public void testReverseScanner_FromMemStore_SingleCF_Normal()
4367 throws IOException {
4368 byte[] rowC = Bytes.toBytes("rowC");
4369 byte[] rowA = Bytes.toBytes("rowA");
4370 byte[] rowB = Bytes.toBytes("rowB");
4371 byte[] cf = Bytes.toBytes("CF");
4372 byte[][] families = { cf };
4373 byte[] col = Bytes.toBytes("C");
4374 long ts = 1;
4375 String method = this.getName();
4376 this.region = initHRegion(tableName, method, families);
4377 try {
4378 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4379 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4380 null);
4381 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4382 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4383 Put put = null;
4384 put = new Put(rowC);
4385 put.add(kv1);
4386 put.add(kv11);
4387 region.put(put);
4388 put = new Put(rowA);
4389 put.add(kv2);
4390 region.put(put);
4391 put = new Put(rowB);
4392 put.add(kv3);
4393 region.put(put);
4394
4395 Scan scan = new Scan(rowC);
4396 scan.setMaxVersions(5);
4397 scan.setReversed(true);
4398 InternalScanner scanner = region.getScanner(scan);
4399 List<Cell> currRow = new ArrayList<Cell>();
4400 boolean hasNext = scanner.next(currRow);
4401 assertEquals(2, currRow.size());
4402 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4403 assertTrue(hasNext);
4404 currRow.clear();
4405 hasNext = scanner.next(currRow);
4406 assertEquals(1, currRow.size());
4407 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4408 assertTrue(hasNext);
4409 currRow.clear();
4410 hasNext = scanner.next(currRow);
4411 assertEquals(1, currRow.size());
4412 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4413 assertFalse(hasNext);
4414 scanner.close();
4415 } finally {
4416 HRegion.closeHRegion(this.region);
4417 this.region = null;
4418 }
4419 }
4420
4421 public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
4422 throws IOException {
4423 byte[] rowC = Bytes.toBytes("rowC");
4424 byte[] rowA = Bytes.toBytes("rowA");
4425 byte[] rowB = Bytes.toBytes("rowB");
4426 byte[] rowD = Bytes.toBytes("rowD");
4427 byte[] cf = Bytes.toBytes("CF");
4428 byte[][] families = { cf };
4429 byte[] col = Bytes.toBytes("C");
4430 long ts = 1;
4431 String method = this.getName();
4432 this.region = initHRegion(tableName, method, families);
4433 try {
4434 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4435 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4436 null);
4437 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4438 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4439 Put put = null;
4440 put = new Put(rowC);
4441 put.add(kv1);
4442 put.add(kv11);
4443 region.put(put);
4444 put = new Put(rowA);
4445 put.add(kv2);
4446 region.put(put);
4447 put = new Put(rowB);
4448 put.add(kv3);
4449 region.put(put);
4450
4451 Scan scan = new Scan(rowD);
4452 List<Cell> currRow = new ArrayList<Cell>();
4453 scan.setReversed(true);
4454 scan.setMaxVersions(5);
4455 InternalScanner scanner = region.getScanner(scan);
4456 boolean hasNext = scanner.next(currRow);
4457 assertEquals(2, currRow.size());
4458 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4459 assertTrue(hasNext);
4460 currRow.clear();
4461 hasNext = scanner.next(currRow);
4462 assertEquals(1, currRow.size());
4463 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4464 assertTrue(hasNext);
4465 currRow.clear();
4466 hasNext = scanner.next(currRow);
4467 assertEquals(1, currRow.size());
4468 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4469 assertFalse(hasNext);
4470 scanner.close();
4471 } finally {
4472 HRegion.closeHRegion(this.region);
4473 this.region = null;
4474 }
4475 }
4476
4477 public void testReverseScanner_FromMemStore_SingleCF_FullScan()
4478 throws IOException {
4479 byte[] rowC = Bytes.toBytes("rowC");
4480 byte[] rowA = Bytes.toBytes("rowA");
4481 byte[] rowB = Bytes.toBytes("rowB");
4482 byte[] cf = Bytes.toBytes("CF");
4483 byte[][] families = { cf };
4484 byte[] col = Bytes.toBytes("C");
4485 long ts = 1;
4486 String method = this.getName();
4487 this.region = initHRegion(tableName, method, families);
4488 try {
4489 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4490 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4491 null);
4492 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4493 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4494 Put put = null;
4495 put = new Put(rowC);
4496 put.add(kv1);
4497 put.add(kv11);
4498 region.put(put);
4499 put = new Put(rowA);
4500 put.add(kv2);
4501 region.put(put);
4502 put = new Put(rowB);
4503 put.add(kv3);
4504 region.put(put);
4505 Scan scan = new Scan();
4506 List<Cell> currRow = new ArrayList<Cell>();
4507 scan.setReversed(true);
4508 InternalScanner scanner = region.getScanner(scan);
4509 boolean hasNext = scanner.next(currRow);
4510 assertEquals(1, currRow.size());
4511 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4512 assertTrue(hasNext);
4513 currRow.clear();
4514 hasNext = scanner.next(currRow);
4515 assertEquals(1, currRow.size());
4516 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4517 assertTrue(hasNext);
4518 currRow.clear();
4519 hasNext = scanner.next(currRow);
4520 assertEquals(1, currRow.size());
4521 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4522 assertFalse(hasNext);
4523 scanner.close();
4524 } finally {
4525 HRegion.closeHRegion(this.region);
4526 this.region = null;
4527 }
4528 }
4529
4530 public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
4531
4532 byte[] rowA = Bytes.toBytes("rowA");
4533 byte[] rowB = Bytes.toBytes("rowB");
4534 byte[] rowC = Bytes.toBytes("rowC");
4535 byte[] rowD = Bytes.toBytes("rowD");
4536 byte[] rowE = Bytes.toBytes("rowE");
4537 byte[] cf = Bytes.toBytes("CF");
4538 byte[][] families = { cf };
4539 byte[] col1 = Bytes.toBytes("col1");
4540 byte[] col2 = Bytes.toBytes("col2");
4541 long ts = 1;
4542 String method = this.getName();
4543 this.region = initHRegion(tableName, method, families);
4544 try {
4545 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4546 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4547 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4548 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4549 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4550 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4551 Put put = null;
4552 put = new Put(rowA);
4553 put.add(kv1);
4554 region.put(put);
4555 put = new Put(rowB);
4556 put.add(kv2);
4557 region.put(put);
4558 put = new Put(rowC);
4559 put.add(kv3);
4560 region.put(put);
4561 put = new Put(rowD);
4562 put.add(kv4_1);
4563 region.put(put);
4564 put = new Put(rowD);
4565 put.add(kv4_2);
4566 region.put(put);
4567 put = new Put(rowE);
4568 put.add(kv5);
4569 region.put(put);
4570 region.flushcache();
4571 Scan scan = new Scan(rowD, rowA);
4572 scan.addColumn(families[0], col1);
4573 scan.setReversed(true);
4574 List<Cell> currRow = new ArrayList<Cell>();
4575 InternalScanner scanner = region.getScanner(scan);
4576 boolean hasNext = scanner.next(currRow);
4577 assertEquals(1, currRow.size());
4578 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4579 assertTrue(hasNext);
4580 currRow.clear();
4581 hasNext = scanner.next(currRow);
4582 assertEquals(1, currRow.size());
4583 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4584 assertTrue(hasNext);
4585 currRow.clear();
4586 hasNext = scanner.next(currRow);
4587 assertEquals(1, currRow.size());
4588 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4589 assertFalse(hasNext);
4590 scanner.close();
4591
4592 scan = new Scan(rowD, rowA);
4593 scan.addColumn(families[0], col2);
4594 scan.setReversed(true);
4595 currRow.clear();
4596 scanner = region.getScanner(scan);
4597 hasNext = scanner.next(currRow);
4598 assertEquals(1, currRow.size());
4599 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4600 scanner.close();
4601 } finally {
4602 HRegion.closeHRegion(this.region);
4603 this.region = null;
4604 }
4605 }
4606
4607 public void testReverseScanner_smaller_blocksize() throws IOException {
4608
4609 byte[] rowA = Bytes.toBytes("rowA");
4610 byte[] rowB = Bytes.toBytes("rowB");
4611 byte[] rowC = Bytes.toBytes("rowC");
4612 byte[] rowD = Bytes.toBytes("rowD");
4613 byte[] rowE = Bytes.toBytes("rowE");
4614 byte[] cf = Bytes.toBytes("CF");
4615 byte[][] families = { cf };
4616 byte[] col1 = Bytes.toBytes("col1");
4617 byte[] col2 = Bytes.toBytes("col2");
4618 long ts = 1;
4619 String method = this.getName();
4620 HBaseConfiguration config = new HBaseConfiguration();
4621 config.setInt("test.block.size", 1);
4622 this.region = initHRegion(tableName, method, config, families);
4623 try {
4624 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4625 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4626 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4627 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4628 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4629 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4630 Put put = null;
4631 put = new Put(rowA);
4632 put.add(kv1);
4633 region.put(put);
4634 put = new Put(rowB);
4635 put.add(kv2);
4636 region.put(put);
4637 put = new Put(rowC);
4638 put.add(kv3);
4639 region.put(put);
4640 put = new Put(rowD);
4641 put.add(kv4_1);
4642 region.put(put);
4643 put = new Put(rowD);
4644 put.add(kv4_2);
4645 region.put(put);
4646 put = new Put(rowE);
4647 put.add(kv5);
4648 region.put(put);
4649 region.flushcache();
4650 Scan scan = new Scan(rowD, rowA);
4651 scan.addColumn(families[0], col1);
4652 scan.setReversed(true);
4653 List<Cell> currRow = new ArrayList<Cell>();
4654 InternalScanner scanner = region.getScanner(scan);
4655 boolean hasNext = scanner.next(currRow);
4656 assertEquals(1, currRow.size());
4657 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4658 assertTrue(hasNext);
4659 currRow.clear();
4660 hasNext = scanner.next(currRow);
4661 assertEquals(1, currRow.size());
4662 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4663 assertTrue(hasNext);
4664 currRow.clear();
4665 hasNext = scanner.next(currRow);
4666 assertEquals(1, currRow.size());
4667 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4668 assertFalse(hasNext);
4669 scanner.close();
4670
4671 scan = new Scan(rowD, rowA);
4672 scan.addColumn(families[0], col2);
4673 scan.setReversed(true);
4674 currRow.clear();
4675 scanner = region.getScanner(scan);
4676 hasNext = scanner.next(currRow);
4677 assertEquals(1, currRow.size());
4678 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4679 scanner.close();
4680 } finally {
4681 HRegion.closeHRegion(this.region);
4682 this.region = null;
4683 }
4684 }
4685
4686 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
4687 throws IOException {
4688 byte[] row0 = Bytes.toBytes("row0");
4689 byte[] row1 = Bytes.toBytes("row1");
4690 byte[] row2 = Bytes.toBytes("row2");
4691 byte[] row3 = Bytes.toBytes("row3");
4692 byte[] row4 = Bytes.toBytes("row4");
4693 byte[] row5 = Bytes.toBytes("row5");
4694 byte[] cf1 = Bytes.toBytes("CF1");
4695 byte[] cf2 = Bytes.toBytes("CF2");
4696 byte[] cf3 = Bytes.toBytes("CF3");
4697 byte[][] families = { cf1, cf2, cf3 };
4698 byte[] col = Bytes.toBytes("C");
4699 long ts = 1;
4700 String method = this.getName();
4701 HBaseConfiguration conf = new HBaseConfiguration();
4702
4703 conf.setInt("hbase.hstore.compactionThreshold", 10000);
4704 this.region = initHRegion(tableName, method, conf, families);
4705 try {
4706
4707 KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
4708 null);
4709 KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
4710 null);
4711 KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
4712 KeyValue.Type.Put, null);
4713 KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
4714 null);
4715 KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
4716 null);
4717 KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
4718 null);
4719 KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
4720 KeyValue.Type.Put, null);
4721 KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
4722 null);
4723 KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
4724 KeyValue.Type.Put, null);
4725 KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
4726 null);
4727 KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
4728 null);
4729 KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
4730 KeyValue.Type.Put, null);
4731 KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
4732 null);
4733 KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
4734 KeyValue.Type.Put, null);
4735 KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
4736 null);
4737 KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
4738 null);
4739
4740 Put put = null;
4741 put = new Put(row1);
4742 put.add(kv1_2_1);
4743 region.put(put);
4744 put = new Put(row2);
4745 put.add(kv2_4_1);
4746 region.put(put);
4747 put = new Put(row4);
4748 put.add(kv4_5_4);
4749 put.add(kv4_5_5);
4750 region.put(put);
4751 region.flushcache();
4752
4753 put = new Put(row4);
4754 put.add(kv4_5_1);
4755 put.add(kv4_5_3);
4756 region.put(put);
4757 put = new Put(row1);
4758 put.add(kv1_2_2);
4759 region.put(put);
4760 put = new Put(row2);
4761 put.add(kv2_4_4);
4762 region.put(put);
4763 region.flushcache();
4764
4765 put = new Put(row4);
4766 put.add(kv4_5_2);
4767 region.put(put);
4768 put = new Put(row2);
4769 put.add(kv2_4_2);
4770 put.add(kv2_4_3);
4771 region.put(put);
4772 put = new Put(row3);
4773 put.add(kv3_2_2);
4774 region.put(put);
4775 region.flushcache();
4776
4777
4778 put = new Put(row0);
4779 put.add(kv0_1_1);
4780 region.put(put);
4781 put = new Put(row3);
4782 put.add(kv3_2_1);
4783 region.put(put);
4784 put = new Put(row5);
4785 put.add(kv5_2_1);
4786 put.add(kv5_2_2);
4787 region.put(put);
4788
4789 Scan scan = new Scan(row4);
4790 scan.setMaxVersions(5);
4791 scan.setBatch(3);
4792 scan.setReversed(true);
4793 InternalScanner scanner = region.getScanner(scan);
4794 List<Cell> currRow = new ArrayList<Cell>();
4795 boolean hasNext = false;
4796
4797
4798
4799 hasNext = scanner.next(currRow);
4800 assertEquals(3, currRow.size());
4801 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4802 assertTrue(hasNext);
4803 currRow.clear();
4804 hasNext = scanner.next(currRow);
4805 assertEquals(2, currRow.size());
4806 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4807 assertTrue(hasNext);
4808
4809 currRow.clear();
4810 hasNext = scanner.next(currRow);
4811 assertEquals(2, currRow.size());
4812 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
4813 assertTrue(hasNext);
4814
4815
4816 currRow.clear();
4817 hasNext = scanner.next(currRow);
4818 assertEquals(3, currRow.size());
4819 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4820 assertTrue(hasNext);
4821 currRow.clear();
4822 hasNext = scanner.next(currRow);
4823 assertEquals(1, currRow.size());
4824 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4825 assertTrue(hasNext);
4826
4827 currRow.clear();
4828 hasNext = scanner.next(currRow);
4829 assertEquals(2, currRow.size());
4830 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
4831 assertTrue(hasNext);
4832
4833 currRow.clear();
4834 hasNext = scanner.next(currRow);
4835 assertEquals(1, currRow.size());
4836 assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
4837 assertFalse(hasNext);
4838
4839 scanner.close();
4840 } finally {
4841 HRegion.closeHRegion(this.region);
4842 this.region = null;
4843 }
4844 }
4845
4846 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
4847 throws IOException {
4848 byte[] row1 = Bytes.toBytes("row1");
4849 byte[] row2 = Bytes.toBytes("row2");
4850 byte[] row3 = Bytes.toBytes("row3");
4851 byte[] row4 = Bytes.toBytes("row4");
4852 byte[] cf1 = Bytes.toBytes("CF1");
4853 byte[] cf2 = Bytes.toBytes("CF2");
4854 byte[] cf3 = Bytes.toBytes("CF3");
4855 byte[] cf4 = Bytes.toBytes("CF4");
4856 byte[][] families = { cf1, cf2, cf3, cf4 };
4857 byte[] col = Bytes.toBytes("C");
4858 long ts = 1;
4859 String method = this.getName();
4860 HBaseConfiguration conf = new HBaseConfiguration();
4861
4862 conf.setInt("hbase.hstore.compactionThreshold", 10000);
4863 this.region = initHRegion(tableName, method, conf, families);
4864 try {
4865 KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
4866 KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
4867 KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
4868 KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
4869
4870 Put put = new Put(row1);
4871 put.add(kv1);
4872 region.put(put);
4873 region.flushcache();
4874
4875 put = new Put(row2);
4876 put.add(kv2);
4877 region.put(put);
4878 region.flushcache();
4879
4880 put = new Put(row3);
4881 put.add(kv3);
4882 region.put(put);
4883 region.flushcache();
4884
4885 put = new Put(row4);
4886 put.add(kv4);
4887 region.put(put);
4888
4889 Scan scan = new Scan(row4);
4890 scan.setReversed(true);
4891 scan.setBatch(10);
4892 InternalScanner scanner = region.getScanner(scan);
4893 List<Cell> currRow = new ArrayList<Cell>();
4894 boolean hasNext = scanner.next(currRow);
4895 assertEquals(1, currRow.size());
4896 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4897 assertTrue(hasNext);
4898 currRow.clear();
4899 hasNext = scanner.next(currRow);
4900 assertEquals(1, currRow.size());
4901 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
4902 assertTrue(hasNext);
4903 currRow.clear();
4904 hasNext = scanner.next(currRow);
4905 assertEquals(1, currRow.size());
4906 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4907 assertTrue(hasNext);
4908 currRow.clear();
4909 hasNext = scanner.next(currRow);
4910 assertEquals(1, currRow.size());
4911 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
4912 assertFalse(hasNext);
4913 } finally {
4914 HRegion.closeHRegion(this.region);
4915 this.region = null;
4916 }
4917 }
4918
4919 private static HRegion initHRegion(byte[] tableName, String callingMethod,
4920 byte[]... families) throws IOException {
4921 return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
4922 families);
4923 }
4924 }