1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21
22 import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23 import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24 import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25 import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29 import static org.junit.Assert.assertArrayEquals;
30 import static org.junit.Assert.assertEquals;
31 import static org.junit.Assert.assertFalse;
32 import static org.junit.Assert.assertNotNull;
33 import static org.junit.Assert.assertNull;
34 import static org.junit.Assert.assertTrue;
35 import static org.junit.Assert.fail;
36 import static org.mockito.Matchers.any;
37 import static org.mockito.Matchers.anyBoolean;
38 import static org.mockito.Matchers.anyLong;
39 import static org.mockito.Matchers.eq;
40 import static org.mockito.Mockito.never;
41 import static org.mockito.Mockito.spy;
42 import static org.mockito.Mockito.times;
43 import static org.mockito.Mockito.verify;
44
45 import java.io.IOException;
46 import java.io.InterruptedIOException;
47 import java.security.PrivilegedExceptionAction;
48 import java.util.ArrayList;
49 import java.util.Arrays;
50 import java.util.Collection;
51 import java.util.List;
52 import java.util.Map;
53 import java.util.NavigableMap;
54 import java.util.TreeMap;
55 import java.util.UUID;
56 import java.util.concurrent.atomic.AtomicBoolean;
57 import java.util.concurrent.atomic.AtomicInteger;
58 import java.util.concurrent.atomic.AtomicLong;
59 import java.util.concurrent.atomic.AtomicReference;
60
61 import org.apache.commons.logging.Log;
62 import org.apache.commons.logging.LogFactory;
63 import org.apache.hadoop.conf.Configuration;
64 import org.apache.hadoop.fs.FSDataOutputStream;
65 import org.apache.hadoop.fs.FileStatus;
66 import org.apache.hadoop.fs.FileSystem;
67 import org.apache.hadoop.fs.Path;
68 import org.apache.hadoop.hbase.Cell;
69 import org.apache.hadoop.hbase.CellComparator;
70 import org.apache.hadoop.hbase.CellUtil;
71 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
72 import org.apache.hadoop.hbase.DroppedSnapshotException;
73 import org.apache.hadoop.hbase.HBaseConfiguration;
74 import org.apache.hadoop.hbase.HBaseTestCase;
75 import org.apache.hadoop.hbase.HBaseTestingUtility;
76 import org.apache.hadoop.hbase.HColumnDescriptor;
77 import org.apache.hadoop.hbase.HConstants;
78 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
79 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
80 import org.apache.hadoop.hbase.HRegionInfo;
81 import org.apache.hadoop.hbase.HTableDescriptor;
82 import org.apache.hadoop.hbase.KeyValue;
83 import org.apache.hadoop.hbase.MediumTests;
84 import org.apache.hadoop.hbase.MiniHBaseCluster;
85 import org.apache.hadoop.hbase.MultithreadedTestUtil;
86 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
87 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
88 import org.apache.hadoop.hbase.NotServingRegionException;
89 import org.apache.hadoop.hbase.TableName;
90 import org.apache.hadoop.hbase.Waiter;
91 import org.apache.hadoop.hbase.client.Append;
92 import org.apache.hadoop.hbase.client.Delete;
93 import org.apache.hadoop.hbase.client.Durability;
94 import org.apache.hadoop.hbase.client.Get;
95 import org.apache.hadoop.hbase.client.HTable;
96 import org.apache.hadoop.hbase.client.Increment;
97 import org.apache.hadoop.hbase.client.Put;
98 import org.apache.hadoop.hbase.client.Result;
99 import org.apache.hadoop.hbase.client.Scan;
100 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
101 import org.apache.hadoop.hbase.filter.BinaryComparator;
102 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
103 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
104 import org.apache.hadoop.hbase.filter.Filter;
105 import org.apache.hadoop.hbase.filter.FilterBase;
106 import org.apache.hadoop.hbase.filter.FilterList;
107 import org.apache.hadoop.hbase.filter.NullComparator;
108 import org.apache.hadoop.hbase.filter.PrefixFilter;
109 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
110 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
111 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
112 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
113 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
114 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
115 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
116 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
117 import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
118 import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
119 import org.apache.hadoop.hbase.regionserver.wal.HLog;
120 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
121 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
122 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
123 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
124 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
125 import org.apache.hadoop.hbase.security.User;
126 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
127 import org.apache.hadoop.hbase.util.Bytes;
128 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
129 import org.apache.hadoop.hbase.util.FSUtils;
130 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
131 import org.apache.hadoop.hbase.util.PairOfSameType;
132 import org.apache.hadoop.hbase.util.Threads;
133 import org.junit.After;
134 import org.junit.Assert;
135 import org.junit.Before;
136 import org.junit.Rule;
137 import org.junit.Test;
138 import org.junit.experimental.categories.Category;
139 import org.junit.rules.TestName;
140 import org.mockito.Mockito;
141
142 import com.google.common.collect.Lists;
143 import com.google.protobuf.ByteString;
144
145
146
147
148
149
150
151 @Category(MediumTests.class)
152 @SuppressWarnings("deprecation")
153 public class TestHRegion {
154
155
156 static final Log LOG = LogFactory.getLog(TestHRegion.class);
157 @Rule public TestName name = new TestName();
158
159 private static final String COLUMN_FAMILY = "MyCF";
160 private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
161
162 HRegion region = null;
163
164 private static HBaseTestingUtility TEST_UTIL;
165 public static Configuration CONF ;
166 private String dir;
167 private static FileSystem FILESYSTEM;
168 private final int MAX_VERSIONS = 2;
169
170
171 protected byte[] tableName;
172 protected String method;
173 protected final byte[] qual1 = Bytes.toBytes("qual1");
174 protected final byte[] qual2 = Bytes.toBytes("qual2");
175 protected final byte[] qual3 = Bytes.toBytes("qual3");
176 protected final byte[] value1 = Bytes.toBytes("value1");
177 protected final byte[] value2 = Bytes.toBytes("value2");
178 protected final byte[] row = Bytes.toBytes("rowA");
179 protected final byte[] row2 = Bytes.toBytes("rowB");
180
181 protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
182 .getInstance(MetricsAssertHelper.class);
183
184 @Before
185 public void setup() throws IOException {
186 TEST_UTIL = HBaseTestingUtility.createLocalHTU();
187 FILESYSTEM = TEST_UTIL.getTestFileSystem();
188 CONF = TEST_UTIL.getConfiguration();
189 dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
190 method = name.getMethodName();
191 tableName = Bytes.toBytes(name.getMethodName());
192 }
193
194 @After
195 public void tearDown() throws Exception {
196 EnvironmentEdgeManagerTestHelper.reset();
197 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
198 TEST_UTIL.cleanupTestDir();
199 }
200
201 String getName() {
202 return name.getMethodName();
203 }
204
205
206
207
208
209
210
211
212
213
214
215 @Test (timeout=60000)
216 public void testCloseCarryingSnapshot() throws IOException {
217 HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
218 Store store = region.getStore(COLUMN_FAMILY_BYTES);
219
220 byte [] value = Bytes.toBytes(name.getMethodName());
221
222 Put put = new Put(value);
223 put.add(COLUMN_FAMILY_BYTES, null, value);
224
225 region.put(put);
226 StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
227 storeFlushCtx.prepare();
228
229 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
230 region.put(put);
231
232 region.close();
233 assertEquals(0, region.getMemstoreSize().get());
234 HRegion.closeHRegion(region);
235 }
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252 @Test (timeout=60000)
253 public void testFlushSizeAccounting() throws Exception {
254 final Configuration conf = HBaseConfiguration.create(CONF);
255
256 conf.setInt("hbase.hstore.flush.retries.number", 1);
257 final User user =
258 User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
259
260 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
261 user.runAs(new PrivilegedExceptionAction<Object>() {
262 @Override
263 public Object run() throws Exception {
264
265 FileSystem fs = FileSystem.get(conf);
266 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
267 FaultyFileSystem ffs = (FaultyFileSystem)fs;
268 HRegion region = null;
269 try {
270
271 region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
272 long size = region.getMemstoreSize().get();
273 Assert.assertEquals(0, size);
274
275 Put p1 = new Put(row);
276 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
277 region.put(p1);
278 final long sizeOfOnePut = region.getMemstoreSize().get();
279
280 try {
281 LOG.info("Flushing");
282 region.flushcache();
283 Assert.fail("Didn't bubble up IOE!");
284 } catch (DroppedSnapshotException dse) {
285
286 }
287
288 ffs.fault.set(false);
289
290 Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize().get());
291
292
293 Put p2 = new Put(row);
294 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
295 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
296 region.put(p2);
297 Assert.assertEquals(sizeOfOnePut * 3, region.getMemstoreSize().get());
298
299
300
301 region.flushcache();
302
303 Assert.assertEquals(sizeOfOnePut * 2, region.getMemstoreSize().get());
304 } finally {
305 HRegion.closeHRegion(region);
306 }
307 return null;
308 }
309 });
310 FileSystem.closeAllForUGI(user.getUGI());
311 }
312
313 @Test
314 public void testCompactionAffectedByScanners() throws Exception {
315 byte[] family = Bytes.toBytes("family");
316 this.region = initHRegion(tableName, method, CONF, family);
317
318 Put put = new Put(Bytes.toBytes("r1"));
319 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
320 region.put(put);
321 region.flushcache();
322
323 Scan scan = new Scan();
324 scan.setMaxVersions(3);
325
326 RegionScanner scanner1 = region.getScanner(scan);
327
328 Delete delete = new Delete(Bytes.toBytes("r1"));
329 region.delete(delete);
330 region.flushcache();
331
332
333 RegionScanner scanner2 = region.getScanner(scan);
334
335 List<Cell> results = new ArrayList<Cell>();
336
337 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
338
339
340 region.compactStores(true);
341
342
343 RegionScanner scanner3 = region.getScanner(scan);
344
345
346 scanner1.next(results);
347 System.out.println(results);
348 assertEquals(1, results.size());
349
350 results.clear();
351 scanner2.next(results);
352 System.out.println(results);
353 assertEquals(0, results.size());
354
355 results.clear();
356 scanner3.next(results);
357 System.out.println(results);
358 assertEquals(0, results.size());
359 }
360
361 @Test
362 public void testToShowNPEOnRegionScannerReseek() throws Exception {
363 byte[] family = Bytes.toBytes("family");
364 this.region = initHRegion(tableName, method, CONF, family);
365
366 Put put = new Put(Bytes.toBytes("r1"));
367 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
368 region.put(put);
369 put = new Put(Bytes.toBytes("r2"));
370 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
371 region.put(put);
372 region.flushcache();
373
374 Scan scan = new Scan();
375 scan.setMaxVersions(3);
376
377 RegionScanner scanner1 = region.getScanner(scan);
378
379 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
380
381 region.compactStores(true);
382
383 scanner1.reseek(Bytes.toBytes("r2"));
384 List<Cell> results = new ArrayList<Cell>();
385 scanner1.next(results);
386 Cell keyValue = results.get(0);
387 Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
388 scanner1.close();
389 }
390
391 @Test
392 public void testSkipRecoveredEditsReplay() throws Exception {
393 String method = "testSkipRecoveredEditsReplay";
394 TableName tableName = TableName.valueOf(method);
395 byte[] family = Bytes.toBytes("family");
396 this.region = initHRegion(tableName, method, CONF, family);
397 try {
398 Path regiondir = region.getRegionFileSystem().getRegionDir();
399 FileSystem fs = region.getRegionFileSystem().getFileSystem();
400 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
401
402 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
403
404 long maxSeqId = 1050;
405 long minSeqId = 1000;
406
407 for (long i = minSeqId; i <= maxSeqId; i += 10) {
408 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
409 fs.create(recoveredEdits);
410 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
411
412 long time = System.nanoTime();
413 WALEdit edit = new WALEdit();
414 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
415 .toBytes(i)));
416 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
417 HConstants.DEFAULT_CLUSTER_ID), edit));
418
419 writer.close();
420 }
421 MonitoredTask status = TaskMonitor.get().createStatus(method);
422 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
423 for (Store store : region.getStores().values()) {
424 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
425 }
426 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
427 assertEquals(maxSeqId, seqId);
428 Get get = new Get(row);
429 Result result = region.get(get);
430 for (long i = minSeqId; i <= maxSeqId; i += 10) {
431 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
432 assertEquals(1, kvs.size());
433 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
434 }
435 } finally {
436 HRegion.closeHRegion(this.region);
437 this.region = null;
438 }
439 }
440
441 @Test
442 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
443 String method = "testSkipRecoveredEditsReplaySomeIgnored";
444 TableName tableName = TableName.valueOf(method);
445 byte[] family = Bytes.toBytes("family");
446 this.region = initHRegion(tableName, method, CONF, family);
447 try {
448 Path regiondir = region.getRegionFileSystem().getRegionDir();
449 FileSystem fs = region.getRegionFileSystem().getFileSystem();
450 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
451
452 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
453
454 long maxSeqId = 1050;
455 long minSeqId = 1000;
456
457 for (long i = minSeqId; i <= maxSeqId; i += 10) {
458 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
459 fs.create(recoveredEdits);
460 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
461
462 long time = System.nanoTime();
463 WALEdit edit = new WALEdit();
464 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
465 .toBytes(i)));
466 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
467 HConstants.DEFAULT_CLUSTER_ID), edit));
468
469 writer.close();
470 }
471 long recoverSeqId = 1030;
472 MonitoredTask status = TaskMonitor.get().createStatus(method);
473 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
474 for (Store store : region.getStores().values()) {
475 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
476 }
477 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
478 assertEquals(maxSeqId, seqId);
479 Get get = new Get(row);
480 Result result = region.get(get);
481 for (long i = minSeqId; i <= maxSeqId; i += 10) {
482 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
483 if (i < recoverSeqId) {
484 assertEquals(0, kvs.size());
485 } else {
486 assertEquals(1, kvs.size());
487 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
488 }
489 }
490 } finally {
491 HRegion.closeHRegion(this.region);
492 this.region = null;
493 }
494 }
495
496 @Test
497 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
498 byte[] family = Bytes.toBytes("family");
499 this.region = initHRegion(tableName, method, CONF, family);
500 try {
501 Path regiondir = region.getRegionFileSystem().getRegionDir();
502 FileSystem fs = region.getRegionFileSystem().getFileSystem();
503
504 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
505 for (int i = 1000; i < 1050; i += 10) {
506 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
507 FSDataOutputStream dos = fs.create(recoveredEdits);
508 dos.writeInt(i);
509 dos.close();
510 }
511 long minSeqId = 2000;
512 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
513 FSDataOutputStream dos = fs.create(recoveredEdits);
514 dos.close();
515
516 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
517 for (Store store : region.getStores().values()) {
518 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
519 }
520 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
521 assertEquals(minSeqId, seqId);
522 } finally {
523 HRegion.closeHRegion(this.region);
524 this.region = null;
525 }
526 }
527
528 @Test
529 public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
530 String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
531 TableName tableName = TableName.valueOf(method);
532 byte[] family = Bytes.toBytes("family");
533 this.region = initHRegion(tableName, method, CONF, family);
534 try {
535 Path regiondir = region.getRegionFileSystem().getRegionDir();
536 FileSystem fs = region.getRegionFileSystem().getFileSystem();
537 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
538
539 assertEquals(0, region.getStoreFileList(
540 region.getStores().keySet().toArray(new byte[0][])).size());
541
542 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
543
544 long maxSeqId = 1050;
545 long minSeqId = 1000;
546
547 for (long i = minSeqId; i <= maxSeqId; i += 10) {
548 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
549 fs.create(recoveredEdits);
550 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
551
552 long time = System.nanoTime();
553 WALEdit edit = null;
554 if (i == maxSeqId) {
555 edit = WALEdit.createCompaction(CompactionDescriptor.newBuilder()
556 .setTableName(ByteString.copyFrom(tableName.getName()))
557 .setFamilyName(ByteString.copyFrom(regionName))
558 .setEncodedRegionName(ByteString.copyFrom(regionName))
559 .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
560 .build());
561 } else {
562 edit = new WALEdit();
563 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
564 .toBytes(i)));
565 }
566 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
567 HConstants.DEFAULT_CLUSTER_ID), edit));
568 writer.close();
569 }
570
571 long recoverSeqId = 1030;
572 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
573 MonitoredTask status = TaskMonitor.get().createStatus(method);
574 for (Store store : region.getStores().values()) {
575 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
576 }
577 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
578 assertEquals(maxSeqId, seqId);
579
580
581 assertEquals(1, region.getStoreFileList(
582 region.getStores().keySet().toArray(new byte[0][])).size());
583
584 } finally {
585 HRegion.closeHRegion(this.region);
586 this.region = null;
587 } }
588
589 @Test
590 public void testRecoveredEditsReplayCompaction() throws Exception {
591 String method = name.getMethodName();
592 TableName tableName = TableName.valueOf(method);
593 byte[] family = Bytes.toBytes("family");
594 this.region = initHRegion(tableName, method, CONF, family);
595 try {
596 Path regiondir = region.getRegionFileSystem().getRegionDir();
597 FileSystem fs = region.getRegionFileSystem().getFileSystem();
598 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
599
600 long maxSeqId = 3;
601 long minSeqId = 0;
602
603 for (long i = minSeqId; i < maxSeqId; i++) {
604 Put put = new Put(Bytes.toBytes(i));
605 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
606 region.put(put);
607 region.flushcache();
608 }
609
610
611 assertEquals(3, region.getStore(family).getStorefilesCount());
612 List<Path> storeFiles = new ArrayList<Path>(3);
613 for (StoreFile sf : region.getStore(family).getStorefiles()) {
614 storeFiles.add(sf.getPath());
615 }
616
617
618 CONF.setBoolean("hbase.hstore.compaction.complete", false);
619 region.compactStores();
620
621
622 assertEquals(3, region.getStore(family).getStorefilesCount());
623
624
625 Path tmpDir = region.getRegionFileSystem().getTempDir();
626 FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
627 String errorMsg = "Expected to find 1 file in the region temp directory "
628 + "from the compaction, could not find any";
629 assertNotNull(errorMsg, files);
630 assertEquals(errorMsg, 1, files.length);
631
632 Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
633 files[0].getPath());
634
635 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
636 .getRegionInfo(), family, storeFiles, Lists.newArrayList(newFile), region
637 .getRegionFileSystem().getStoreDir(Bytes.toString(family)));
638
639 HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
640 this.region.getRegionInfo(), compactionDescriptor, new AtomicLong(1));
641
642 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
643
644 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
645 fs.create(recoveredEdits);
646 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);
647
648 long time = System.nanoTime();
649
650 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, 10, time,
651 HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(compactionDescriptor)));
652 writer.close();
653
654
655 region.getTableDesc();
656 region.getRegionInfo();
657 region.close();
658 region = HRegion.openHRegion(region, null);
659
660
661 Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
662 for (StoreFile sf : sfs) {
663 LOG.info(sf.getPath());
664 }
665 assertEquals(1, region.getStore(family).getStorefilesCount());
666 files = FSUtils.listStatus(fs, tmpDir);
667 assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
668
669 for (long i = minSeqId; i < maxSeqId; i++) {
670 Get get = new Get(Bytes.toBytes(i));
671 Result result = region.get(get);
672 byte[] value = result.getValue(family, Bytes.toBytes(i));
673 assertArrayEquals(Bytes.toBytes(i), value);
674 }
675 } finally {
676 HRegion.closeHRegion(this.region);
677 this.region = null;
678 }
679 }
680
681 @Test
682 public void testGetWhileRegionClose() throws IOException {
683 TableName tableName = TableName.valueOf(name.getMethodName());
684 Configuration hc = initSplit();
685 int numRows = 100;
686 byte[][] families = { fam1, fam2, fam3 };
687
688
689 String method = name.getMethodName();
690 this.region = initHRegion(tableName, method, hc, families);
691 try {
692
693 final int startRow = 100;
694 putData(startRow, numRows, qual1, families);
695 putData(startRow, numRows, qual2, families);
696 putData(startRow, numRows, qual3, families);
697 final AtomicBoolean done = new AtomicBoolean(false);
698 final AtomicInteger gets = new AtomicInteger(0);
699 GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
700 try {
701
702 for (int i = 0; i < threads.length / 2; i++) {
703 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
704 threads[i].setDaemon(true);
705 threads[i].start();
706 }
707
708
709 this.region.closing.set(true);
710 for (int i = threads.length / 2; i < threads.length; i++) {
711 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
712 threads[i].setDaemon(true);
713 threads[i].start();
714 }
715 } finally {
716 if (this.region != null) {
717 HRegion.closeHRegion(this.region);
718 }
719 }
720 done.set(true);
721 for (GetTillDoneOrException t : threads) {
722 try {
723 t.join();
724 } catch (InterruptedException e) {
725 e.printStackTrace();
726 }
727 if (t.e != null) {
728 LOG.info("Exception=" + t.e);
729 assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
730 }
731 }
732 } finally {
733 HRegion.closeHRegion(this.region);
734 this.region = null;
735 }
736 }
737
738
739
740
741
742 class GetTillDoneOrException extends Thread {
743 private final Get g;
744 private final AtomicBoolean done;
745 private final AtomicInteger count;
746 private Exception e;
747
748 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
749 super("getter." + i);
750 this.g = new Get(r);
751 this.done = d;
752 this.count = c;
753 }
754
755 @Override
756 public void run() {
757 while (!this.done.get()) {
758 try {
759 assertTrue(region.get(g).size() > 0);
760 this.count.incrementAndGet();
761 } catch (Exception e) {
762 this.e = e;
763 break;
764 }
765 }
766 }
767 }
768
769
770
771
772 @Test
773 public void testWeirdCacheBehaviour() throws Exception {
774 byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
775 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
776 Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
777 this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
778 try {
779 String value = "this is the value";
780 String value2 = "this is some other value";
781 String keyPrefix1 = "prefix1";
782 String keyPrefix2 = "prefix2";
783 String keyPrefix3 = "prefix3";
784 putRows(this.region, 3, value, keyPrefix1);
785 putRows(this.region, 3, value, keyPrefix2);
786 putRows(this.region, 3, value, keyPrefix3);
787 putRows(this.region, 3, value2, keyPrefix1);
788 putRows(this.region, 3, value2, keyPrefix2);
789 putRows(this.region, 3, value2, keyPrefix3);
790 System.out.println("Checking values for key: " + keyPrefix1);
791 assertEquals("Got back incorrect number of rows from scan", 3,
792 getNumberOfRows(keyPrefix1, value2, this.region));
793 System.out.println("Checking values for key: " + keyPrefix2);
794 assertEquals("Got back incorrect number of rows from scan", 3,
795 getNumberOfRows(keyPrefix2, value2, this.region));
796 System.out.println("Checking values for key: " + keyPrefix3);
797 assertEquals("Got back incorrect number of rows from scan", 3,
798 getNumberOfRows(keyPrefix3, value2, this.region));
799 deleteColumns(this.region, value2, keyPrefix1);
800 deleteColumns(this.region, value2, keyPrefix2);
801 deleteColumns(this.region, value2, keyPrefix3);
802 System.out.println("Starting important checks.....");
803 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
804 getNumberOfRows(keyPrefix1, value2, this.region));
805 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
806 getNumberOfRows(keyPrefix2, value2, this.region));
807 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
808 getNumberOfRows(keyPrefix3, value2, this.region));
809 } finally {
810 HRegion.closeHRegion(this.region);
811 this.region = null;
812 }
813 }
814
815 @Test
816 public void testAppendWithReadOnlyTable() throws Exception {
817 byte[] TABLE = Bytes.toBytes("readOnlyTable");
818 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
819 boolean exceptionCaught = false;
820 Append append = new Append(Bytes.toBytes("somerow"));
821 append.setDurability(Durability.SKIP_WAL);
822 append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
823 Bytes.toBytes("somevalue"));
824 try {
825 region.append(append);
826 } catch (IOException e) {
827 exceptionCaught = true;
828 } finally {
829 HRegion.closeHRegion(this.region);
830 this.region = null;
831 }
832 assertTrue(exceptionCaught == true);
833 }
834
835 @Test
836 public void testIncrWithReadOnlyTable() throws Exception {
837 byte[] TABLE = Bytes.toBytes("readOnlyTable");
838 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
839 boolean exceptionCaught = false;
840 Increment inc = new Increment(Bytes.toBytes("somerow"));
841 inc.setDurability(Durability.SKIP_WAL);
842 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
843 try {
844 region.increment(inc);
845 } catch (IOException e) {
846 exceptionCaught = true;
847 } finally {
848 HRegion.closeHRegion(this.region);
849 this.region = null;
850 }
851 assertTrue(exceptionCaught == true);
852 }
853
854 private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
855 InternalScanner scanner = buildScanner(keyPrefix, value, r);
856 int count = 0;
857 boolean more = false;
858 List<Cell> results = new ArrayList<Cell>();
859 do {
860 more = scanner.next(results);
861 if (results != null && !results.isEmpty())
862 count++;
863 else
864 break;
865 Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
866 delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
867 r.delete(delete);
868 results.clear();
869 } while (more);
870 assertEquals("Did not perform correct number of deletes", 3, count);
871 }
872
873 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
874 InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
875 int numberOfResults = 0;
876 List<Cell> results = new ArrayList<Cell>();
877 boolean more = false;
878 do {
879 more = resultScanner.next(results);
880 if (results != null && !results.isEmpty())
881 numberOfResults++;
882 else
883 break;
884 for (Cell kv : results) {
885 System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
886 }
887 results.clear();
888 } while (more);
889 return numberOfResults;
890 }
891
892 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
893 throws IOException {
894
895 FilterList allFilters = new FilterList();
896 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
897
898 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
899 Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
900 filter.setFilterIfMissing(true);
901 allFilters.addFilter(filter);
902 Scan scan = new Scan();
903 scan.addFamily(Bytes.toBytes("trans-blob"));
904 scan.addFamily(Bytes.toBytes("trans-type"));
905 scan.addFamily(Bytes.toBytes("trans-date"));
906 scan.addFamily(Bytes.toBytes("trans-tags"));
907 scan.addFamily(Bytes.toBytes("trans-group"));
908 scan.setFilter(allFilters);
909 return r.getScanner(scan);
910 }
911
912 private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
913 for (int i = 0; i < numRows; i++) {
914 String row = key + "_" + i
915 System.out.println(String.format("Saving row: %s, with value %s", row, value));
916 Put put = new Put(Bytes.toBytes(row));
917 put.setDurability(Durability.SKIP_WAL);
918 put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
919 put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
920 put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
921 put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
922 put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
923 r.put(put);
924 }
925 }
926
927 @Test
928 public void testFamilyWithAndWithoutColon() throws Exception {
929 byte[] b = Bytes.toBytes(getName());
930 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
931 this.region = initHRegion(b, getName(), CONF, cf);
932 try {
933 Put p = new Put(b);
934 byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
935 p.add(cfwithcolon, cfwithcolon, cfwithcolon);
936 boolean exception = false;
937 try {
938 this.region.put(p);
939 } catch (NoSuchColumnFamilyException e) {
940 exception = true;
941 }
942 assertTrue(exception);
943 } finally {
944 HRegion.closeHRegion(this.region);
945 this.region = null;
946 }
947 }
948
949 @Test
950 public void testBatchPut() throws Exception {
951 byte[] b = Bytes.toBytes(getName());
952 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
953 byte[] qual = Bytes.toBytes("qual");
954 byte[] val = Bytes.toBytes("val");
955 this.region = initHRegion(b, getName(), CONF, cf);
956 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
957 try {
958 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
959 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
960
961 LOG.info("First a batch put with all valid puts");
962 final Put[] puts = new Put[10];
963 for (int i = 0; i < 10; i++) {
964 puts[i] = new Put(Bytes.toBytes("row_" + i));
965 puts[i].add(cf, qual, val);
966 }
967
968 OperationStatus[] codes = this.region.batchMutate(puts);
969 assertEquals(10, codes.length);
970 for (int i = 0; i < 10; i++) {
971 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
972 }
973 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
974
975 LOG.info("Next a batch put with one invalid family");
976 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
977 codes = this.region.batchMutate(puts);
978 assertEquals(10, codes.length);
979 for (int i = 0; i < 10; i++) {
980 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
981 codes[i].getOperationStatusCode());
982 }
983
984 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
985
986 LOG.info("Next a batch put that has to break into two batches to avoid a lock");
987 RowLock rowLock = region.getRowLock(Bytes.toBytes("row_2"));
988
989 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
990 final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
991 TestThread putter = new TestThread(ctx) {
992 @Override
993 public void doWork() throws IOException {
994 retFromThread.set(region.batchMutate(puts));
995 }
996 };
997 LOG.info("...starting put thread while holding lock");
998 ctx.addThread(putter);
999 ctx.startThreads();
1000
1001 LOG.info("...waiting for put thread to sync first time");
1002 long startWait = System.currentTimeMillis();
1003 while (metricsAssertHelper.getCounter("syncTimeNumOps", source) == syncs + 2) {
1004 Thread.sleep(100);
1005 if (System.currentTimeMillis() - startWait > 10000) {
1006 fail("Timed out waiting for thread to sync first minibatch");
1007 }
1008 }
1009 LOG.info("...releasing row lock, which should let put thread continue");
1010 rowLock.release();
1011 LOG.info("...joining on thread");
1012 ctx.stop();
1013 LOG.info("...checking that next batch was synced");
1014 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 4, source);
1015 codes = retFromThread.get();
1016 for (int i = 0; i < 10; i++) {
1017 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1018 codes[i].getOperationStatusCode());
1019 }
1020
1021 } finally {
1022 HRegion.closeHRegion(this.region);
1023 this.region = null;
1024 }
1025 }
1026
1027 @Test
1028 public void testBatchPutWithTsSlop() throws Exception {
1029 byte[] b = Bytes.toBytes(getName());
1030 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1031 byte[] qual = Bytes.toBytes("qual");
1032 byte[] val = Bytes.toBytes("val");
1033
1034
1035 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1036 this.region = initHRegion(b, getName(), CONF, cf);
1037
1038 try {
1039 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1040 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1041 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1042
1043 final Put[] puts = new Put[10];
1044 for (int i = 0; i < 10; i++) {
1045 puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
1046 puts[i].add(cf, qual, val);
1047 }
1048
1049 OperationStatus[] codes = this.region.batchMutate(puts);
1050 assertEquals(10, codes.length);
1051 for (int i = 0; i < 10; i++) {
1052 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1053 }
1054 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1055
1056 } finally {
1057 HRegion.closeHRegion(this.region);
1058 this.region = null;
1059 }
1060
1061 }
1062
1063
1064
1065
1066 @Test
1067 public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1068 byte[] row1 = Bytes.toBytes("row1");
1069 byte[] fam1 = Bytes.toBytes("fam1");
1070 byte[] qf1 = Bytes.toBytes("qualifier");
1071 byte[] emptyVal = new byte[] {};
1072 byte[] val1 = Bytes.toBytes("value1");
1073 byte[] val2 = Bytes.toBytes("value2");
1074
1075
1076 String method = this.getName();
1077 this.region = initHRegion(tableName, method, CONF, fam1);
1078 try {
1079
1080 Put put = new Put(row1);
1081 put.add(fam1, qf1, emptyVal);
1082
1083
1084 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1085 emptyVal), put, true);
1086 assertTrue(res);
1087
1088
1089 put = new Put(row1);
1090 put.add(fam1, qf1, val1);
1091
1092
1093 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1094 put, true);
1095 assertTrue(res);
1096
1097
1098 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1099 put, true);
1100 assertFalse(res);
1101
1102 Delete delete = new Delete(row1);
1103 delete.deleteColumn(fam1, qf1);
1104 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1105 delete, true);
1106 assertFalse(res);
1107
1108 put = new Put(row1);
1109 put.add(fam1, qf1, val2);
1110
1111 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1112 put, true);
1113 assertTrue(res);
1114
1115
1116 delete = new Delete(row1);
1117 delete.deleteColumn(fam1, qf1);
1118 delete.deleteColumn(fam1, qf1);
1119 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1120 delete, true);
1121 assertTrue(res);
1122
1123 delete = new Delete(row1);
1124 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1125 delete, true);
1126 assertTrue(res);
1127
1128
1129 put = new Put(row1);
1130 put.add(fam1, qf1, val1);
1131
1132 res = region
1133 .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
1134 assertTrue(res);
1135 } finally {
1136 HRegion.closeHRegion(this.region);
1137 this.region = null;
1138 }
1139 }
1140
1141 @Test
1142 public void testCheckAndMutate_WithWrongValue() throws IOException {
1143 byte[] row1 = Bytes.toBytes("row1");
1144 byte[] fam1 = Bytes.toBytes("fam1");
1145 byte[] qf1 = Bytes.toBytes("qualifier");
1146 byte[] val1 = Bytes.toBytes("value1");
1147 byte[] val2 = Bytes.toBytes("value2");
1148
1149
1150 String method = this.getName();
1151 this.region = initHRegion(tableName, method, CONF, fam1);
1152 try {
1153
1154 Put put = new Put(row1);
1155 put.add(fam1, qf1, val1);
1156 region.put(put);
1157
1158
1159 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1160 val2), put, true);
1161 assertEquals(false, res);
1162
1163
1164 Delete delete = new Delete(row1);
1165 delete.deleteFamily(fam1);
1166 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1167 delete, true);
1168 assertEquals(false, res);
1169 } finally {
1170 HRegion.closeHRegion(this.region);
1171 this.region = null;
1172 }
1173 }
1174
1175 @Test
1176 public void testCheckAndMutate_WithCorrectValue() throws IOException {
1177 byte[] row1 = Bytes.toBytes("row1");
1178 byte[] fam1 = Bytes.toBytes("fam1");
1179 byte[] qf1 = Bytes.toBytes("qualifier");
1180 byte[] val1 = Bytes.toBytes("value1");
1181
1182
1183 String method = this.getName();
1184 this.region = initHRegion(tableName, method, CONF, fam1);
1185 try {
1186
1187 Put put = new Put(row1);
1188 put.add(fam1, qf1, val1);
1189 region.put(put);
1190
1191
1192 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1193 val1), put, true);
1194 assertEquals(true, res);
1195
1196
1197 Delete delete = new Delete(row1);
1198 delete.deleteColumn(fam1, qf1);
1199 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1200 put, true);
1201 assertEquals(true, res);
1202 } finally {
1203 HRegion.closeHRegion(this.region);
1204 this.region = null;
1205 }
1206 }
1207
1208 @Test
1209 public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1210 byte[] row1 = Bytes.toBytes("row1");
1211 byte[] fam1 = Bytes.toBytes("fam1");
1212 byte[] qf1 = Bytes.toBytes("qualifier");
1213 byte[] val1 = Bytes.toBytes("value1");
1214 byte[] val2 = Bytes.toBytes("value2");
1215 byte[] val3 = Bytes.toBytes("value3");
1216 byte[] val4 = Bytes.toBytes("value4");
1217
1218
1219 String method = this.getName();
1220 this.region = initHRegion(tableName, method, CONF, fam1);
1221 try {
1222
1223 Put put = new Put(row1);
1224 put.add(fam1, qf1, val3);
1225 region.put(put);
1226
1227
1228 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1229 new BinaryComparator(val3), put, true);
1230 assertEquals(false, res);
1231
1232
1233 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1234 new BinaryComparator(val4), put, true);
1235 assertEquals(false, res);
1236
1237
1238
1239 put = new Put(row1);
1240 put.add(fam1, qf1, val2);
1241 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1242 new BinaryComparator(val2), put, true);
1243 assertEquals(true, res);
1244
1245
1246 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1247 new BinaryComparator(val3), put, true);
1248 assertEquals(false, res);
1249
1250
1251
1252 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1253 new BinaryComparator(val2), put, true);
1254 assertEquals(true, res);
1255
1256
1257
1258 put = new Put(row1);
1259 put.add(fam1, qf1, val3);
1260 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1261 new BinaryComparator(val1), put, true);
1262 assertEquals(true, res);
1263
1264
1265 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1266 new BinaryComparator(val3), put, true);
1267 assertEquals(false, res);
1268
1269
1270 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1271 new BinaryComparator(val2), put, true);
1272 assertEquals(false, res);
1273
1274
1275
1276 put = new Put(row1);
1277 put.add(fam1, qf1, val2);
1278 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1279 new BinaryComparator(val4), put, true);
1280 assertEquals(true, res);
1281
1282
1283 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1284 new BinaryComparator(val1), put, true);
1285 assertEquals(false, res);
1286
1287
1288
1289 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1290 new BinaryComparator(val2), put, true);
1291 assertEquals(true, res);
1292
1293
1294 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1295 new BinaryComparator(val3), put, true);
1296 assertEquals(true, res);
1297 } finally {
1298 HRegion.closeHRegion(this.region);
1299 this.region = null;
1300 }
1301 }
1302
1303 @Test
1304 public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1305 byte[] row1 = Bytes.toBytes("row1");
1306 byte[] fam1 = Bytes.toBytes("fam1");
1307 byte[] fam2 = Bytes.toBytes("fam2");
1308 byte[] qf1 = Bytes.toBytes("qualifier");
1309 byte[] val1 = Bytes.toBytes("value1");
1310 byte[] val2 = Bytes.toBytes("value2");
1311
1312 byte[][] families = { fam1, fam2 };
1313
1314
1315 String method = this.getName();
1316 this.region = initHRegion(tableName, method, CONF, families);
1317 try {
1318
1319 Put put = new Put(row1);
1320 put.add(fam1, qf1, val1);
1321 region.put(put);
1322
1323
1324 long ts = System.currentTimeMillis();
1325 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1326 put = new Put(row1);
1327 put.add(kv);
1328
1329
1330 HStore store = (HStore) region.getStore(fam1);
1331 store.memstore.kvset.size();
1332
1333 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1334 val1), put, true);
1335 assertEquals(true, res);
1336 store.memstore.kvset.size();
1337
1338 Get get = new Get(row1);
1339 get.addColumn(fam2, qf1);
1340 Cell[] actual = region.get(get).rawCells();
1341
1342 Cell[] expected = { kv };
1343
1344 assertEquals(expected.length, actual.length);
1345 for (int i = 0; i < actual.length; i++) {
1346 assertEquals(expected[i], actual[i]);
1347 }
1348 } finally {
1349 HRegion.closeHRegion(this.region);
1350 this.region = null;
1351 }
1352 }
1353
1354 @Test
1355 public void testCheckAndPut_wrongRowInPut() throws IOException {
1356 TableName tableName = TableName.valueOf(name.getMethodName());
1357 this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
1358 try {
1359 Put put = new Put(row2);
1360 put.add(fam1, qual1, value1);
1361 try {
1362 region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1363 new BinaryComparator(value2), put, false);
1364 fail();
1365 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1366
1367 }
1368 } finally {
1369 HRegion.closeHRegion(this.region);
1370 this.region = null;
1371 }
1372 }
1373
1374 @Test
1375 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1376 byte[] row1 = Bytes.toBytes("row1");
1377 byte[] fam1 = Bytes.toBytes("fam1");
1378 byte[] fam2 = Bytes.toBytes("fam2");
1379 byte[] qf1 = Bytes.toBytes("qualifier1");
1380 byte[] qf2 = Bytes.toBytes("qualifier2");
1381 byte[] qf3 = Bytes.toBytes("qualifier3");
1382 byte[] val1 = Bytes.toBytes("value1");
1383 byte[] val2 = Bytes.toBytes("value2");
1384 byte[] val3 = Bytes.toBytes("value3");
1385 byte[] emptyVal = new byte[] {};
1386
1387 byte[][] families = { fam1, fam2 };
1388
1389
1390 String method = this.getName();
1391 this.region = initHRegion(tableName, method, CONF, families);
1392 try {
1393
1394 Put put = new Put(row1);
1395 put.add(fam1, qf1, val1);
1396 region.put(put);
1397 Threads.sleep(2);
1398
1399 put = new Put(row1);
1400 put.add(fam1, qf1, val2);
1401 put.add(fam2, qf1, val3);
1402 put.add(fam2, qf2, val2);
1403 put.add(fam2, qf3, val1);
1404 put.add(fam1, qf3, val1);
1405 region.put(put);
1406
1407
1408 Delete delete = new Delete(row1);
1409 delete.deleteColumn(fam1, qf1);
1410 delete.deleteColumn(fam2, qf1);
1411 delete.deleteColumn(fam1, qf3);
1412 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1413 val2), delete, true);
1414 assertEquals(true, res);
1415
1416 Get get = new Get(row1);
1417 get.addColumn(fam1, qf1);
1418 get.addColumn(fam1, qf3);
1419 get.addColumn(fam2, qf2);
1420 Result r = region.get(get);
1421 assertEquals(2, r.size());
1422 assertArrayEquals(val1, r.getValue(fam1, qf1));
1423 assertArrayEquals(val2, r.getValue(fam2, qf2));
1424
1425
1426 delete = new Delete(row1);
1427 delete.deleteFamily(fam2);
1428 res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1429 delete, true);
1430 assertEquals(true, res);
1431
1432 get = new Get(row1);
1433 r = region.get(get);
1434 assertEquals(1, r.size());
1435 assertArrayEquals(val1, r.getValue(fam1, qf1));
1436
1437
1438 delete = new Delete(row1);
1439 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1440 delete, true);
1441 assertEquals(true, res);
1442 get = new Get(row1);
1443 r = region.get(get);
1444 assertEquals(0, r.size());
1445 } finally {
1446 HRegion.closeHRegion(this.region);
1447 this.region = null;
1448 }
1449 }
1450
1451
1452
1453
1454 @Test
1455 public void testDelete_multiDeleteColumn() throws IOException {
1456 byte[] row1 = Bytes.toBytes("row1");
1457 byte[] fam1 = Bytes.toBytes("fam1");
1458 byte[] qual = Bytes.toBytes("qualifier");
1459 byte[] value = Bytes.toBytes("value");
1460
1461 Put put = new Put(row1);
1462 put.add(fam1, qual, 1, value);
1463 put.add(fam1, qual, 2, value);
1464
1465 String method = this.getName();
1466 this.region = initHRegion(tableName, method, CONF, fam1);
1467 try {
1468 region.put(put);
1469
1470
1471 Delete delete = new Delete(row1);
1472 delete.deleteColumn(fam1, qual);
1473 delete.deleteColumn(fam1, qual);
1474 region.delete(delete);
1475
1476 Get get = new Get(row1);
1477 get.addFamily(fam1);
1478 Result r = region.get(get);
1479 assertEquals(0, r.size());
1480 } finally {
1481 HRegion.closeHRegion(this.region);
1482 this.region = null;
1483 }
1484 }
1485
1486 @Test
1487 public void testDelete_CheckFamily() throws IOException {
1488 byte[] row1 = Bytes.toBytes("row1");
1489 byte[] fam1 = Bytes.toBytes("fam1");
1490 byte[] fam2 = Bytes.toBytes("fam2");
1491 byte[] fam3 = Bytes.toBytes("fam3");
1492 byte[] fam4 = Bytes.toBytes("fam4");
1493
1494
1495 String method = this.getName();
1496 this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
1497 try {
1498 List<Cell> kvs = new ArrayList<Cell>();
1499 kvs.add(new KeyValue(row1, fam4, null, null));
1500
1501
1502 byte[] family = fam2;
1503 try {
1504 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1505 Bytes.BYTES_COMPARATOR);
1506 deleteMap.put(family, kvs);
1507 region.delete(deleteMap, Durability.SYNC_WAL);
1508 } catch (Exception e) {
1509 assertTrue("Family " + new String(family) + " does not exist", false);
1510 }
1511
1512
1513 boolean ok = false;
1514 family = fam4;
1515 try {
1516 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1517 Bytes.BYTES_COMPARATOR);
1518 deleteMap.put(family, kvs);
1519 region.delete(deleteMap, Durability.SYNC_WAL);
1520 } catch (Exception e) {
1521 ok = true;
1522 }
1523 assertEquals("Family " + new String(family) + " does exist", true, ok);
1524 } finally {
1525 HRegion.closeHRegion(this.region);
1526 this.region = null;
1527 }
1528 }
1529
1530 @Test
1531 public void testDelete_mixed() throws IOException, InterruptedException {
1532 byte[] fam = Bytes.toBytes("info");
1533 byte[][] families = { fam };
1534 String method = this.getName();
1535 this.region = initHRegion(tableName, method, CONF, families);
1536 try {
1537 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1538
1539 byte[] row = Bytes.toBytes("table_name");
1540
1541 byte[] serverinfo = Bytes.toBytes("serverinfo");
1542 byte[] splitA = Bytes.toBytes("splitA");
1543 byte[] splitB = Bytes.toBytes("splitB");
1544
1545
1546 Put put = new Put(row);
1547 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1548 region.put(put);
1549
1550 put = new Put(row);
1551 put.add(fam, splitB, Bytes.toBytes("reference_B"));
1552 region.put(put);
1553
1554 put = new Put(row);
1555 put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1556 region.put(put);
1557
1558
1559 Delete delete = new Delete(row);
1560 delete.deleteColumns(fam, splitA);
1561 region.delete(delete);
1562
1563
1564 Get get = new Get(row).addColumn(fam, serverinfo);
1565 Result result = region.get(get);
1566 assertEquals(1, result.size());
1567
1568 get = new Get(row).addColumn(fam, splitA);
1569 result = region.get(get);
1570 assertEquals(0, result.size());
1571
1572 get = new Get(row).addColumn(fam, splitB);
1573 result = region.get(get);
1574 assertEquals(1, result.size());
1575
1576
1577 put = new Put(row);
1578 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1579 region.put(put);
1580 get = new Get(row);
1581 result = region.get(get);
1582 assertEquals(3, result.size());
1583
1584
1585 delete = new Delete(row);
1586 region.delete(delete);
1587 assertEquals(0, region.get(get).size());
1588
1589 region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
1590 result = region.get(get);
1591 assertEquals(1, result.size());
1592 } finally {
1593 HRegion.closeHRegion(this.region);
1594 this.region = null;
1595 }
1596 }
1597
1598 @Test
1599 public void testDeleteRowWithFutureTs() throws IOException {
1600 byte[] fam = Bytes.toBytes("info");
1601 byte[][] families = { fam };
1602 String method = this.getName();
1603 this.region = initHRegion(tableName, method, CONF, families);
1604 try {
1605 byte[] row = Bytes.toBytes("table_name");
1606
1607 byte[] serverinfo = Bytes.toBytes("serverinfo");
1608
1609
1610 Put put = new Put(row);
1611 put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
1612 region.put(put);
1613
1614
1615 Delete delete = new Delete(row);
1616 region.delete(delete);
1617
1618
1619 Get get = new Get(row).addColumn(fam, serverinfo);
1620 Result result = region.get(get);
1621 assertEquals(1, result.size());
1622
1623
1624 delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
1625 region.delete(delete);
1626
1627
1628 get = new Get(row).addColumn(fam, serverinfo);
1629 result = region.get(get);
1630 assertEquals(0, result.size());
1631 } finally {
1632 HRegion.closeHRegion(this.region);
1633 this.region = null;
1634 }
1635 }
1636
1637
1638
1639
1640
1641 @Test
1642 public void testPutWithLatestTS() throws IOException {
1643 byte[] fam = Bytes.toBytes("info");
1644 byte[][] families = { fam };
1645 String method = this.getName();
1646 this.region = initHRegion(tableName, method, CONF, families);
1647 try {
1648 byte[] row = Bytes.toBytes("row1");
1649
1650 byte[] qual = Bytes.toBytes("qual");
1651
1652
1653 Put put = new Put(row);
1654 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1655 region.put(put);
1656
1657
1658 Get get = new Get(row).addColumn(fam, qual);
1659 Result result = region.get(get);
1660 assertEquals(1, result.size());
1661 Cell kv = result.rawCells()[0];
1662 LOG.info("Got: " + kv);
1663 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1664 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1665
1666
1667
1668 row = Bytes.toBytes("row2");
1669 put = new Put(row);
1670 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1671 region.put(put);
1672
1673
1674 get = new Get(row).addColumn(fam, qual);
1675 result = region.get(get);
1676 assertEquals(1, result.size());
1677 kv = result.rawCells()[0];
1678 LOG.info("Got: " + kv);
1679 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1680 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1681 } finally {
1682 HRegion.closeHRegion(this.region);
1683 this.region = null;
1684 }
1685
1686 }
1687
1688
1689
1690
1691
1692
1693 @Test
1694 public void testPutWithTsSlop() throws IOException {
1695 byte[] fam = Bytes.toBytes("info");
1696 byte[][] families = { fam };
1697 String method = this.getName();
1698
1699
1700 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1701 this.region = initHRegion(tableName, method, CONF, families);
1702 boolean caughtExcep = false;
1703 try {
1704 try {
1705
1706 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
1707
1708 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
1709 Bytes.toBytes("value")));
1710 fail("Expected IOE for TS out of configured timerange");
1711 } catch (FailedSanityCheckException ioe) {
1712 LOG.debug("Received expected exception", ioe);
1713 caughtExcep = true;
1714 }
1715 assertTrue("Should catch FailedSanityCheckException", caughtExcep);
1716 } finally {
1717 HRegion.closeHRegion(this.region);
1718 this.region = null;
1719 }
1720 }
1721
1722 @Test
1723 public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
1724 byte[] fam1 = Bytes.toBytes("columnA");
1725 byte[] fam2 = Bytes.toBytes("columnB");
1726 this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
1727 try {
1728 byte[] rowA = Bytes.toBytes("rowA");
1729 byte[] rowB = Bytes.toBytes("rowB");
1730
1731 byte[] value = Bytes.toBytes("value");
1732
1733 Delete delete = new Delete(rowA);
1734 delete.deleteFamily(fam1);
1735
1736 region.delete(delete);
1737
1738
1739 Put put = new Put(rowA);
1740 put.add(fam2, null, value);
1741 region.put(put);
1742
1743 put = new Put(rowB);
1744 put.add(fam1, null, value);
1745 put.add(fam2, null, value);
1746 region.put(put);
1747
1748 Scan scan = new Scan();
1749 scan.addFamily(fam1).addFamily(fam2);
1750 InternalScanner s = region.getScanner(scan);
1751 List<Cell> results = new ArrayList<Cell>();
1752 s.next(results);
1753 assertTrue(CellUtil.matchingRow(results.get(0), rowA));
1754
1755 results.clear();
1756 s.next(results);
1757 assertTrue(CellUtil.matchingRow(results.get(0), rowB));
1758 } finally {
1759 HRegion.closeHRegion(this.region);
1760 this.region = null;
1761 }
1762 }
1763
1764 @Test
1765 public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
1766 Delete delete = new Delete(row);
1767 delete.deleteColumns(fam1, qual1);
1768 doTestDelete_AndPostInsert(delete);
1769 }
1770
1771 @Test
1772 public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
1773 Delete delete = new Delete(row);
1774 delete.deleteFamily(fam1);
1775 doTestDelete_AndPostInsert(delete);
1776 }
1777
1778 public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
1779 TableName tableName = TableName.valueOf(name.getMethodName());
1780 this.region = initHRegion(tableName, getName(), CONF, fam1);
1781 try {
1782 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1783 Put put = new Put(row);
1784 put.add(fam1, qual1, value1);
1785 region.put(put);
1786
1787
1788 region.delete(delete);
1789
1790
1791 put = new Put(row);
1792 put.add(fam1, qual1, value2);
1793 region.put(put);
1794
1795
1796 Get get = new Get(row);
1797 get.addColumn(fam1, qual1);
1798
1799 Result r = region.get(get);
1800 assertEquals(1, r.size());
1801 assertArrayEquals(value2, r.getValue(fam1, qual1));
1802
1803
1804 Scan scan = new Scan(row);
1805 scan.addColumn(fam1, qual1);
1806 InternalScanner s = region.getScanner(scan);
1807
1808 List<Cell> results = new ArrayList<Cell>();
1809 assertEquals(false, s.next(results));
1810 assertEquals(1, results.size());
1811 Cell kv = results.get(0);
1812
1813 assertArrayEquals(value2, CellUtil.cloneValue(kv));
1814 assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
1815 assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
1816 assertArrayEquals(row, CellUtil.cloneRow(kv));
1817 } finally {
1818 HRegion.closeHRegion(this.region);
1819 this.region = null;
1820 }
1821 }
1822
1823 @Test
1824 public void testDelete_CheckTimestampUpdated() throws IOException {
1825 TableName tableName = TableName.valueOf(name.getMethodName());
1826 byte[] row1 = Bytes.toBytes("row1");
1827 byte[] col1 = Bytes.toBytes("col1");
1828 byte[] col2 = Bytes.toBytes("col2");
1829 byte[] col3 = Bytes.toBytes("col3");
1830
1831
1832 String method = this.getName();
1833 this.region = initHRegion(tableName, method, CONF, fam1);
1834 try {
1835
1836 List<Cell> kvs = new ArrayList<Cell>();
1837 kvs.add(new KeyValue(row1, fam1, col1, null));
1838 kvs.add(new KeyValue(row1, fam1, col2, null));
1839 kvs.add(new KeyValue(row1, fam1, col3, null));
1840
1841 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1842 Bytes.BYTES_COMPARATOR);
1843 deleteMap.put(fam1, kvs);
1844 region.delete(deleteMap, Durability.SYNC_WAL);
1845
1846
1847
1848 long now = System.currentTimeMillis();
1849 KeyValue firstKv = ((HStore) region.getStore(fam1)).memstore.kvset.first();
1850 assertTrue(firstKv.getTimestamp() <= now);
1851 now = firstKv.getTimestamp();
1852 for (KeyValue kv : ((HStore) region.getStore(fam1)).memstore.kvset) {
1853 assertTrue(kv.getTimestamp() <= now);
1854 now = kv.getTimestamp();
1855 }
1856 } finally {
1857 HRegion.closeHRegion(this.region);
1858 this.region = null;
1859 }
1860 }
1861
1862
1863
1864
1865 @Test
1866 public void testGet_FamilyChecker() throws IOException {
1867 byte[] row1 = Bytes.toBytes("row1");
1868 byte[] fam1 = Bytes.toBytes("fam1");
1869 byte[] fam2 = Bytes.toBytes("False");
1870 byte[] col1 = Bytes.toBytes("col1");
1871
1872
1873 String method = this.getName();
1874 this.region = initHRegion(tableName, method, CONF, fam1);
1875 try {
1876 Get get = new Get(row1);
1877 get.addColumn(fam2, col1);
1878
1879
1880 try {
1881 region.get(get);
1882 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
1883 assertFalse(false);
1884 return;
1885 }
1886 assertFalse(true);
1887 } finally {
1888 HRegion.closeHRegion(this.region);
1889 this.region = null;
1890 }
1891 }
1892
1893 @Test
1894 public void testGet_Basic() throws IOException {
1895 byte[] row1 = Bytes.toBytes("row1");
1896 byte[] fam1 = Bytes.toBytes("fam1");
1897 byte[] col1 = Bytes.toBytes("col1");
1898 byte[] col2 = Bytes.toBytes("col2");
1899 byte[] col3 = Bytes.toBytes("col3");
1900 byte[] col4 = Bytes.toBytes("col4");
1901 byte[] col5 = Bytes.toBytes("col5");
1902
1903
1904 String method = this.getName();
1905 this.region = initHRegion(tableName, method, CONF, fam1);
1906 try {
1907
1908 Put put = new Put(row1);
1909 put.add(fam1, col1, null);
1910 put.add(fam1, col2, null);
1911 put.add(fam1, col3, null);
1912 put.add(fam1, col4, null);
1913 put.add(fam1, col5, null);
1914 region.put(put);
1915
1916 Get get = new Get(row1);
1917 get.addColumn(fam1, col2);
1918 get.addColumn(fam1, col4);
1919
1920 KeyValue kv1 = new KeyValue(row1, fam1, col2);
1921 KeyValue kv2 = new KeyValue(row1, fam1, col4);
1922 KeyValue[] expected = { kv1, kv2 };
1923
1924
1925 Result res = region.get(get);
1926 assertEquals(expected.length, res.size());
1927 for (int i = 0; i < res.size(); i++) {
1928 assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
1929 assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
1930 assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
1931 }
1932
1933
1934 Get g = new Get(row1);
1935 final int count = 2;
1936 g.setFilter(new ColumnCountGetFilter(count));
1937 res = region.get(g);
1938 assertEquals(count, res.size());
1939 } finally {
1940 HRegion.closeHRegion(this.region);
1941 this.region = null;
1942 }
1943 }
1944
1945 @Test
1946 public void testGet_Empty() throws IOException {
1947 byte[] row = Bytes.toBytes("row");
1948 byte[] fam = Bytes.toBytes("fam");
1949
1950 String method = this.getName();
1951 this.region = initHRegion(tableName, method, CONF, fam);
1952 try {
1953 Get get = new Get(row);
1954 get.addFamily(fam);
1955 Result r = region.get(get);
1956
1957 assertTrue(r.isEmpty());
1958 } finally {
1959 HRegion.closeHRegion(this.region);
1960 this.region = null;
1961 }
1962 }
1963
1964
1965
1966
1967 @Test
1968 public void testMerge() throws IOException {
1969 byte[][] families = { fam1, fam2, fam3 };
1970 Configuration hc = initSplit();
1971
1972 String method = this.getName();
1973 this.region = initHRegion(tableName, method, hc, families);
1974 try {
1975 LOG.info("" + HBaseTestCase.addContent(region, fam3));
1976 region.flushcache();
1977 region.compactStores();
1978 byte[] splitRow = region.checkSplit();
1979 assertNotNull(splitRow);
1980 LOG.info("SplitRow: " + Bytes.toString(splitRow));
1981 HRegion[] subregions = splitRegion(region, splitRow);
1982 try {
1983
1984 for (int i = 0; i < subregions.length; i++) {
1985 HRegion.openHRegion(subregions[i], null);
1986 subregions[i].compactStores();
1987 }
1988 Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
1989 Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
1990 Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
1991 long startTime = System.currentTimeMillis();
1992 region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
1993 LOG.info("Merge regions elapsed time: "
1994 + ((System.currentTimeMillis() - startTime) / 1000.0));
1995 FILESYSTEM.delete(oldRegion1, true);
1996 FILESYSTEM.delete(oldRegion2, true);
1997 FILESYSTEM.delete(oldRegionPath, true);
1998 LOG.info("splitAndMerge completed.");
1999 } finally {
2000 for (int i = 0; i < subregions.length; i++) {
2001 try {
2002 HRegion.closeHRegion(subregions[i]);
2003 } catch (IOException e) {
2004
2005 }
2006 }
2007 }
2008 } finally {
2009 HRegion.closeHRegion(this.region);
2010 this.region = null;
2011 }
2012 }
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022 HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
2023 PairOfSameType<HRegion> result = null;
2024 SplitTransaction st = new SplitTransaction(parent, midkey);
2025
2026
2027 if (!st.prepare())
2028 return null;
2029 try {
2030 result = st.execute(null, null);
2031 } catch (IOException ioe) {
2032 try {
2033 LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; "
2034 + ioe.getMessage());
2035 st.rollback(null, null);
2036 LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString());
2037 return null;
2038 } catch (RuntimeException e) {
2039
2040 LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString()
2041 + " -- aborting server", e);
2042 }
2043 }
2044 return new HRegion[] { result.getFirst(), result.getSecond() };
2045 }
2046
2047
2048
2049
2050 @Test
2051 public void testGetScanner_WithOkFamilies() throws IOException {
2052 byte[] fam1 = Bytes.toBytes("fam1");
2053 byte[] fam2 = Bytes.toBytes("fam2");
2054
2055 byte[][] families = { fam1, fam2 };
2056
2057
2058 String method = this.getName();
2059 this.region = initHRegion(tableName, method, CONF, families);
2060 try {
2061 Scan scan = new Scan();
2062 scan.addFamily(fam1);
2063 scan.addFamily(fam2);
2064 try {
2065 region.getScanner(scan);
2066 } catch (Exception e) {
2067 assertTrue("Families could not be found in Region", false);
2068 }
2069 } finally {
2070 HRegion.closeHRegion(this.region);
2071 this.region = null;
2072 }
2073 }
2074
2075 @Test
2076 public void testGetScanner_WithNotOkFamilies() throws IOException {
2077 byte[] fam1 = Bytes.toBytes("fam1");
2078 byte[] fam2 = Bytes.toBytes("fam2");
2079
2080 byte[][] families = { fam1 };
2081
2082
2083 String method = this.getName();
2084 this.region = initHRegion(tableName, method, CONF, families);
2085 try {
2086 Scan scan = new Scan();
2087 scan.addFamily(fam2);
2088 boolean ok = false;
2089 try {
2090 region.getScanner(scan);
2091 } catch (Exception e) {
2092 ok = true;
2093 }
2094 assertTrue("Families could not be found in Region", ok);
2095 } finally {
2096 HRegion.closeHRegion(this.region);
2097 this.region = null;
2098 }
2099 }
2100
2101 @Test
2102 public void testGetScanner_WithNoFamilies() throws IOException {
2103 byte[] row1 = Bytes.toBytes("row1");
2104 byte[] fam1 = Bytes.toBytes("fam1");
2105 byte[] fam2 = Bytes.toBytes("fam2");
2106 byte[] fam3 = Bytes.toBytes("fam3");
2107 byte[] fam4 = Bytes.toBytes("fam4");
2108
2109 byte[][] families = { fam1, fam2, fam3, fam4 };
2110
2111
2112 String method = this.getName();
2113 this.region = initHRegion(tableName, method, CONF, families);
2114 try {
2115
2116
2117 Put put = new Put(row1);
2118 put.add(fam1, null, null);
2119 put.add(fam2, null, null);
2120 put.add(fam3, null, null);
2121 put.add(fam4, null, null);
2122 region.put(put);
2123
2124 Scan scan = null;
2125 HRegion.RegionScannerImpl is = null;
2126
2127
2128
2129
2130 scan = new Scan();
2131 scan.addFamily(fam2);
2132 scan.addFamily(fam4);
2133 is = (RegionScannerImpl) region.getScanner(scan);
2134 assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2135
2136 scan = new Scan();
2137 is = (RegionScannerImpl) region.getScanner(scan);
2138 assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2139 } finally {
2140 HRegion.closeHRegion(this.region);
2141 this.region = null;
2142 }
2143 }
2144
2145
2146
2147
2148
2149
2150 @Test
2151 public void testGetScanner_WithRegionClosed() throws IOException {
2152 byte[] fam1 = Bytes.toBytes("fam1");
2153 byte[] fam2 = Bytes.toBytes("fam2");
2154
2155 byte[][] families = { fam1, fam2 };
2156
2157
2158 String method = this.getName();
2159 try {
2160 this.region = initHRegion(tableName, method, CONF, families);
2161 } catch (IOException e) {
2162 e.printStackTrace();
2163 fail("Got IOException during initHRegion, " + e.getMessage());
2164 }
2165 try {
2166 region.closed.set(true);
2167 try {
2168 region.getScanner(null);
2169 fail("Expected to get an exception during getScanner on a region that is closed");
2170 } catch (NotServingRegionException e) {
2171
2172 } catch (IOException e) {
2173 fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2174 + e.getMessage());
2175 }
2176 } finally {
2177 HRegion.closeHRegion(this.region);
2178 this.region = null;
2179 }
2180 }
2181
2182 @Test
2183 public void testRegionScanner_Next() throws IOException {
2184 byte[] row1 = Bytes.toBytes("row1");
2185 byte[] row2 = Bytes.toBytes("row2");
2186 byte[] fam1 = Bytes.toBytes("fam1");
2187 byte[] fam2 = Bytes.toBytes("fam2");
2188 byte[] fam3 = Bytes.toBytes("fam3");
2189 byte[] fam4 = Bytes.toBytes("fam4");
2190
2191 byte[][] families = { fam1, fam2, fam3, fam4 };
2192 long ts = System.currentTimeMillis();
2193
2194
2195 String method = this.getName();
2196 this.region = initHRegion(tableName, method, CONF, families);
2197 try {
2198
2199 Put put = null;
2200 put = new Put(row1);
2201 put.add(fam1, (byte[]) null, ts, null);
2202 put.add(fam2, (byte[]) null, ts, null);
2203 put.add(fam3, (byte[]) null, ts, null);
2204 put.add(fam4, (byte[]) null, ts, null);
2205 region.put(put);
2206
2207 put = new Put(row2);
2208 put.add(fam1, (byte[]) null, ts, null);
2209 put.add(fam2, (byte[]) null, ts, null);
2210 put.add(fam3, (byte[]) null, ts, null);
2211 put.add(fam4, (byte[]) null, ts, null);
2212 region.put(put);
2213
2214 Scan scan = new Scan();
2215 scan.addFamily(fam2);
2216 scan.addFamily(fam4);
2217 InternalScanner is = region.getScanner(scan);
2218
2219 List<Cell> res = null;
2220
2221
2222 List<Cell> expected1 = new ArrayList<Cell>();
2223 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2224 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2225
2226 res = new ArrayList<Cell>();
2227 is.next(res);
2228 for (int i = 0; i < res.size(); i++) {
2229 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2230 }
2231
2232
2233 List<Cell> expected2 = new ArrayList<Cell>();
2234 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2235 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2236
2237 res = new ArrayList<Cell>();
2238 is.next(res);
2239 for (int i = 0; i < res.size(); i++) {
2240 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2241 }
2242 } finally {
2243 HRegion.closeHRegion(this.region);
2244 this.region = null;
2245 }
2246 }
2247
2248 @Test
2249 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2250 byte[] row1 = Bytes.toBytes("row1");
2251 byte[] qf1 = Bytes.toBytes("qualifier1");
2252 byte[] qf2 = Bytes.toBytes("qualifier2");
2253 byte[] fam1 = Bytes.toBytes("fam1");
2254 byte[][] families = { fam1 };
2255
2256 long ts1 = System.currentTimeMillis();
2257 long ts2 = ts1 + 1;
2258 long ts3 = ts1 + 2;
2259
2260
2261 String method = this.getName();
2262 this.region = initHRegion(tableName, method, CONF, families);
2263 try {
2264
2265 Put put = null;
2266 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2267 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2268 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2269
2270 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2271 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2272 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2273
2274 put = new Put(row1);
2275 put.add(kv13);
2276 put.add(kv12);
2277 put.add(kv11);
2278 put.add(kv23);
2279 put.add(kv22);
2280 put.add(kv21);
2281 region.put(put);
2282
2283
2284 List<Cell> expected = new ArrayList<Cell>();
2285 expected.add(kv13);
2286 expected.add(kv12);
2287
2288 Scan scan = new Scan(row1);
2289 scan.addColumn(fam1, qf1);
2290 scan.setMaxVersions(MAX_VERSIONS);
2291 List<Cell> actual = new ArrayList<Cell>();
2292 InternalScanner scanner = region.getScanner(scan);
2293
2294 boolean hasNext = scanner.next(actual);
2295 assertEquals(false, hasNext);
2296
2297
2298 for (int i = 0; i < expected.size(); i++) {
2299 assertEquals(expected.get(i), actual.get(i));
2300 }
2301 } finally {
2302 HRegion.closeHRegion(this.region);
2303 this.region = null;
2304 }
2305 }
2306
2307 @Test
2308 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2309 byte[] row1 = Bytes.toBytes("row1");
2310 byte[] qf1 = Bytes.toBytes("qualifier1");
2311 byte[] qf2 = Bytes.toBytes("qualifier2");
2312 byte[] fam1 = Bytes.toBytes("fam1");
2313 byte[][] families = { fam1 };
2314
2315 long ts1 = 1;
2316 long ts2 = ts1 + 1;
2317 long ts3 = ts1 + 2;
2318
2319
2320 String method = this.getName();
2321 this.region = initHRegion(tableName, method, CONF, families);
2322 try {
2323
2324 Put put = null;
2325 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2326 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2327 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2328
2329 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2330 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2331 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2332
2333 put = new Put(row1);
2334 put.add(kv13);
2335 put.add(kv12);
2336 put.add(kv11);
2337 put.add(kv23);
2338 put.add(kv22);
2339 put.add(kv21);
2340 region.put(put);
2341 region.flushcache();
2342
2343
2344 List<Cell> expected = new ArrayList<Cell>();
2345 expected.add(kv13);
2346 expected.add(kv12);
2347 expected.add(kv23);
2348 expected.add(kv22);
2349
2350 Scan scan = new Scan(row1);
2351 scan.addColumn(fam1, qf1);
2352 scan.addColumn(fam1, qf2);
2353 scan.setMaxVersions(MAX_VERSIONS);
2354 List<Cell> actual = new ArrayList<Cell>();
2355 InternalScanner scanner = region.getScanner(scan);
2356
2357 boolean hasNext = scanner.next(actual);
2358 assertEquals(false, hasNext);
2359
2360
2361 for (int i = 0; i < expected.size(); i++) {
2362 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2363 }
2364 } finally {
2365 HRegion.closeHRegion(this.region);
2366 this.region = null;
2367 }
2368 }
2369
2370 @Test
2371 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2372 byte[] row1 = Bytes.toBytes("row1");
2373 byte[] fam1 = Bytes.toBytes("fam1");
2374 byte[][] families = { fam1 };
2375 byte[] qf1 = Bytes.toBytes("qualifier1");
2376 byte[] qf2 = Bytes.toBytes("qualifier2");
2377
2378 long ts1 = 1;
2379 long ts2 = ts1 + 1;
2380 long ts3 = ts1 + 2;
2381 long ts4 = ts1 + 3;
2382
2383
2384 String method = this.getName();
2385 this.region = initHRegion(tableName, method, CONF, families);
2386 try {
2387
2388 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2389 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2390 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2391 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2392
2393 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2394 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2395 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2396 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2397
2398 Put put = null;
2399 put = new Put(row1);
2400 put.add(kv14);
2401 put.add(kv24);
2402 region.put(put);
2403 region.flushcache();
2404
2405 put = new Put(row1);
2406 put.add(kv23);
2407 put.add(kv13);
2408 region.put(put);
2409 region.flushcache();
2410
2411 put = new Put(row1);
2412 put.add(kv22);
2413 put.add(kv12);
2414 region.put(put);
2415 region.flushcache();
2416
2417 put = new Put(row1);
2418 put.add(kv21);
2419 put.add(kv11);
2420 region.put(put);
2421
2422
2423 List<Cell> expected = new ArrayList<Cell>();
2424 expected.add(kv14);
2425 expected.add(kv13);
2426 expected.add(kv12);
2427 expected.add(kv24);
2428 expected.add(kv23);
2429 expected.add(kv22);
2430
2431 Scan scan = new Scan(row1);
2432 scan.addColumn(fam1, qf1);
2433 scan.addColumn(fam1, qf2);
2434 int versions = 3;
2435 scan.setMaxVersions(versions);
2436 List<Cell> actual = new ArrayList<Cell>();
2437 InternalScanner scanner = region.getScanner(scan);
2438
2439 boolean hasNext = scanner.next(actual);
2440 assertEquals(false, hasNext);
2441
2442
2443 for (int i = 0; i < expected.size(); i++) {
2444 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2445 }
2446 } finally {
2447 HRegion.closeHRegion(this.region);
2448 this.region = null;
2449 }
2450 }
2451
2452 @Test
2453 public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2454 byte[] row1 = Bytes.toBytes("row1");
2455 byte[] qf1 = Bytes.toBytes("qualifier1");
2456 byte[] qf2 = Bytes.toBytes("qualifier2");
2457 byte[] fam1 = Bytes.toBytes("fam1");
2458 byte[][] families = { fam1 };
2459
2460 long ts1 = System.currentTimeMillis();
2461 long ts2 = ts1 + 1;
2462 long ts3 = ts1 + 2;
2463
2464
2465 String method = this.getName();
2466 this.region = initHRegion(tableName, method, CONF, families);
2467 try {
2468
2469 Put put = null;
2470 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2471 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2472 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2473
2474 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2475 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2476 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2477
2478 put = new Put(row1);
2479 put.add(kv13);
2480 put.add(kv12);
2481 put.add(kv11);
2482 put.add(kv23);
2483 put.add(kv22);
2484 put.add(kv21);
2485 region.put(put);
2486
2487
2488 List<Cell> expected = new ArrayList<Cell>();
2489 expected.add(kv13);
2490 expected.add(kv12);
2491 expected.add(kv23);
2492 expected.add(kv22);
2493
2494 Scan scan = new Scan(row1);
2495 scan.addFamily(fam1);
2496 scan.setMaxVersions(MAX_VERSIONS);
2497 List<Cell> actual = new ArrayList<Cell>();
2498 InternalScanner scanner = region.getScanner(scan);
2499
2500 boolean hasNext = scanner.next(actual);
2501 assertEquals(false, hasNext);
2502
2503
2504 for (int i = 0; i < expected.size(); i++) {
2505 assertEquals(expected.get(i), actual.get(i));
2506 }
2507 } finally {
2508 HRegion.closeHRegion(this.region);
2509 this.region = null;
2510 }
2511 }
2512
2513 @Test
2514 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
2515 byte[] row1 = Bytes.toBytes("row1");
2516 byte[] qf1 = Bytes.toBytes("qualifier1");
2517 byte[] qf2 = Bytes.toBytes("qualifier2");
2518 byte[] fam1 = Bytes.toBytes("fam1");
2519
2520 long ts1 = 1;
2521 long ts2 = ts1 + 1;
2522 long ts3 = ts1 + 2;
2523
2524
2525 String method = this.getName();
2526 this.region = initHRegion(tableName, method, CONF, fam1);
2527 try {
2528
2529 Put put = null;
2530 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2531 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2532 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2533
2534 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2535 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2536 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2537
2538 put = new Put(row1);
2539 put.add(kv13);
2540 put.add(kv12);
2541 put.add(kv11);
2542 put.add(kv23);
2543 put.add(kv22);
2544 put.add(kv21);
2545 region.put(put);
2546 region.flushcache();
2547
2548
2549 List<Cell> expected = new ArrayList<Cell>();
2550 expected.add(kv13);
2551 expected.add(kv12);
2552 expected.add(kv23);
2553 expected.add(kv22);
2554
2555 Scan scan = new Scan(row1);
2556 scan.addFamily(fam1);
2557 scan.setMaxVersions(MAX_VERSIONS);
2558 List<Cell> actual = new ArrayList<Cell>();
2559 InternalScanner scanner = region.getScanner(scan);
2560
2561 boolean hasNext = scanner.next(actual);
2562 assertEquals(false, hasNext);
2563
2564
2565 for (int i = 0; i < expected.size(); i++) {
2566 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2567 }
2568 } finally {
2569 HRegion.closeHRegion(this.region);
2570 this.region = null;
2571 }
2572 }
2573
2574 @Test
2575 public void testScanner_StopRow1542() throws IOException {
2576 byte[] family = Bytes.toBytes("testFamily");
2577 this.region = initHRegion(tableName, getName(), CONF, family);
2578 try {
2579 byte[] row1 = Bytes.toBytes("row111");
2580 byte[] row2 = Bytes.toBytes("row222");
2581 byte[] row3 = Bytes.toBytes("row333");
2582 byte[] row4 = Bytes.toBytes("row444");
2583 byte[] row5 = Bytes.toBytes("row555");
2584
2585 byte[] col1 = Bytes.toBytes("Pub111");
2586 byte[] col2 = Bytes.toBytes("Pub222");
2587
2588 Put put = new Put(row1);
2589 put.add(family, col1, Bytes.toBytes(10L));
2590 region.put(put);
2591
2592 put = new Put(row2);
2593 put.add(family, col1, Bytes.toBytes(15L));
2594 region.put(put);
2595
2596 put = new Put(row3);
2597 put.add(family, col2, Bytes.toBytes(20L));
2598 region.put(put);
2599
2600 put = new Put(row4);
2601 put.add(family, col2, Bytes.toBytes(30L));
2602 region.put(put);
2603
2604 put = new Put(row5);
2605 put.add(family, col1, Bytes.toBytes(40L));
2606 region.put(put);
2607
2608 Scan scan = new Scan(row3, row4);
2609 scan.setMaxVersions();
2610 scan.addColumn(family, col1);
2611 InternalScanner s = region.getScanner(scan);
2612
2613 List<Cell> results = new ArrayList<Cell>();
2614 assertEquals(false, s.next(results));
2615 assertEquals(0, results.size());
2616 } finally {
2617 HRegion.closeHRegion(this.region);
2618 this.region = null;
2619 }
2620 }
2621
2622 @Test
2623 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2624 byte[] row1 = Bytes.toBytes("row1");
2625 byte[] fam1 = Bytes.toBytes("fam1");
2626 byte[] qf1 = Bytes.toBytes("qualifier1");
2627 byte[] qf2 = Bytes.toBytes("quateslifier2");
2628
2629 long ts1 = 1;
2630 long ts2 = ts1 + 1;
2631 long ts3 = ts1 + 2;
2632 long ts4 = ts1 + 3;
2633
2634
2635 String method = this.getName();
2636 this.region = initHRegion(tableName, method, CONF, fam1);
2637 try {
2638
2639 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2640 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2641 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2642 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2643
2644 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2645 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2646 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2647 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2648
2649 Put put = null;
2650 put = new Put(row1);
2651 put.add(kv14);
2652 put.add(kv24);
2653 region.put(put);
2654 region.flushcache();
2655
2656 put = new Put(row1);
2657 put.add(kv23);
2658 put.add(kv13);
2659 region.put(put);
2660 region.flushcache();
2661
2662 put = new Put(row1);
2663 put.add(kv22);
2664 put.add(kv12);
2665 region.put(put);
2666 region.flushcache();
2667
2668 put = new Put(row1);
2669 put.add(kv21);
2670 put.add(kv11);
2671 region.put(put);
2672
2673
2674 List<KeyValue> expected = new ArrayList<KeyValue>();
2675 expected.add(kv14);
2676 expected.add(kv13);
2677 expected.add(kv12);
2678 expected.add(kv24);
2679 expected.add(kv23);
2680 expected.add(kv22);
2681
2682 Scan scan = new Scan(row1);
2683 int versions = 3;
2684 scan.setMaxVersions(versions);
2685 List<Cell> actual = new ArrayList<Cell>();
2686 InternalScanner scanner = region.getScanner(scan);
2687
2688 boolean hasNext = scanner.next(actual);
2689 assertEquals(false, hasNext);
2690
2691
2692 for (int i = 0; i < expected.size(); i++) {
2693 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2694 }
2695 } finally {
2696 HRegion.closeHRegion(this.region);
2697 this.region = null;
2698 }
2699 }
2700
2701
2702
2703
2704
2705
2706
2707 @Test
2708 public void testScanner_JoinedScanners() throws IOException {
2709 byte[] cf_essential = Bytes.toBytes("essential");
2710 byte[] cf_joined = Bytes.toBytes("joined");
2711 byte[] cf_alpha = Bytes.toBytes("alpha");
2712 this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
2713 try {
2714 byte[] row1 = Bytes.toBytes("row1");
2715 byte[] row2 = Bytes.toBytes("row2");
2716 byte[] row3 = Bytes.toBytes("row3");
2717
2718 byte[] col_normal = Bytes.toBytes("d");
2719 byte[] col_alpha = Bytes.toBytes("a");
2720
2721 byte[] filtered_val = Bytes.toBytes(3);
2722
2723 Put put = new Put(row1);
2724 put.add(cf_essential, col_normal, Bytes.toBytes(1));
2725 put.add(cf_joined, col_alpha, Bytes.toBytes(1));
2726 region.put(put);
2727
2728 put = new Put(row2);
2729 put.add(cf_essential, col_alpha, Bytes.toBytes(2));
2730 put.add(cf_joined, col_normal, Bytes.toBytes(2));
2731 put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
2732 region.put(put);
2733
2734 put = new Put(row3);
2735 put.add(cf_essential, col_normal, filtered_val);
2736 put.add(cf_joined, col_normal, filtered_val);
2737 region.put(put);
2738
2739
2740
2741
2742
2743 Scan scan = new Scan();
2744 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
2745 CompareOp.NOT_EQUAL, filtered_val);
2746 scan.setFilter(filter);
2747 scan.setLoadColumnFamiliesOnDemand(true);
2748 InternalScanner s = region.getScanner(scan);
2749
2750 List<Cell> results = new ArrayList<Cell>();
2751 assertTrue(s.next(results));
2752 assertEquals(results.size(), 1);
2753 results.clear();
2754
2755 assertTrue(s.next(results));
2756 assertEquals(results.size(), 3);
2757 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
2758 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
2759 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
2760 results.clear();
2761
2762 assertFalse(s.next(results));
2763 assertEquals(results.size(), 0);
2764 } finally {
2765 HRegion.closeHRegion(this.region);
2766 this.region = null;
2767 }
2768 }
2769
2770
2771
2772
2773
2774
2775 @Test
2776 public void testScanner_JoinedScannersWithLimits() throws IOException {
2777 final byte[] cf_first = Bytes.toBytes("first");
2778 final byte[] cf_second = Bytes.toBytes("second");
2779
2780 this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
2781 try {
2782 final byte[] col_a = Bytes.toBytes("a");
2783 final byte[] col_b = Bytes.toBytes("b");
2784
2785 Put put;
2786
2787 for (int i = 0; i < 10; i++) {
2788 put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
2789 put.add(cf_first, col_a, Bytes.toBytes(i));
2790 if (i < 5) {
2791 put.add(cf_first, col_b, Bytes.toBytes(i));
2792 put.add(cf_second, col_a, Bytes.toBytes(i));
2793 put.add(cf_second, col_b, Bytes.toBytes(i));
2794 }
2795 region.put(put);
2796 }
2797
2798 Scan scan = new Scan();
2799 scan.setLoadColumnFamiliesOnDemand(true);
2800 Filter bogusFilter = new FilterBase() {
2801 @Override
2802 public boolean isFamilyEssential(byte[] name) {
2803 return Bytes.equals(name, cf_first);
2804 }
2805 };
2806
2807 scan.setFilter(bogusFilter);
2808 InternalScanner s = region.getScanner(scan);
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839 List<Cell> results = new ArrayList<Cell>();
2840 int index = 0;
2841 while (true) {
2842 boolean more = s.next(results, 3);
2843 if ((index >> 1) < 5) {
2844 if (index % 2 == 0)
2845 assertEquals(results.size(), 3);
2846 else
2847 assertEquals(results.size(), 1);
2848 } else
2849 assertEquals(results.size(), 1);
2850 results.clear();
2851 index++;
2852 if (!more)
2853 break;
2854 }
2855 } finally {
2856 HRegion.closeHRegion(this.region);
2857 this.region = null;
2858 }
2859 }
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869 @Test
2870 public void testBasicSplit() throws Exception {
2871 byte[][] families = { fam1, fam2, fam3 };
2872
2873 Configuration hc = initSplit();
2874
2875 String method = this.getName();
2876 this.region = initHRegion(tableName, method, hc, families);
2877
2878 try {
2879 LOG.info("" + HBaseTestCase.addContent(region, fam3));
2880 region.flushcache();
2881 region.compactStores();
2882 byte[] splitRow = region.checkSplit();
2883 assertNotNull(splitRow);
2884 LOG.info("SplitRow: " + Bytes.toString(splitRow));
2885 HRegion[] regions = splitRegion(region, splitRow);
2886 try {
2887
2888
2889
2890 for (int i = 0; i < regions.length; i++) {
2891 regions[i] = HRegion.openHRegion(regions[i], null);
2892 }
2893
2894
2895 assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
2896 assertGet(regions[1], fam3, splitRow);
2897
2898 assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
2899 assertScan(regions[1], fam3, splitRow);
2900
2901 for (int i = 0; i < regions.length; i++) {
2902
2903
2904 for (int j = 0; j < 2; j++) {
2905 HBaseTestCase.addContent(regions[i], fam3);
2906 }
2907 HBaseTestCase.addContent(regions[i], fam2);
2908 HBaseTestCase.addContent(regions[i], fam1);
2909 regions[i].flushcache();
2910 }
2911
2912 byte[][] midkeys = new byte[regions.length][];
2913
2914 for (int i = 0; i < regions.length; i++) {
2915 regions[i].compactStores();
2916 midkeys[i] = regions[i].checkSplit();
2917 }
2918
2919 TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
2920
2921
2922 for (int i = 0; i < regions.length; i++) {
2923 HRegion[] rs = null;
2924 if (midkeys[i] != null) {
2925 rs = splitRegion(regions[i], midkeys[i]);
2926 for (int j = 0; j < rs.length; j++) {
2927 sortedMap.put(Bytes.toString(rs[j].getRegionName()), HRegion.openHRegion(rs[j], null));
2928 }
2929 }
2930 }
2931 LOG.info("Made 4 regions");
2932
2933
2934 int interval = (LAST_CHAR - FIRST_CHAR) / 3;
2935 byte[] b = Bytes.toBytes(START_KEY);
2936 for (HRegion r : sortedMap.values()) {
2937 assertGet(r, fam3, b);
2938 b[0] += interval;
2939 }
2940 } finally {
2941 for (int i = 0; i < regions.length; i++) {
2942 try {
2943 regions[i].close();
2944 } catch (IOException e) {
2945
2946 }
2947 }
2948 }
2949 } finally {
2950 HRegion.closeHRegion(this.region);
2951 this.region = null;
2952 }
2953 }
2954
2955 @Test
2956 public void testSplitRegion() throws IOException {
2957 byte[] qualifier = Bytes.toBytes("qualifier");
2958 Configuration hc = initSplit();
2959 int numRows = 10;
2960 byte[][] families = { fam1, fam3 };
2961
2962
2963 String method = this.getName();
2964 this.region = initHRegion(tableName, method, hc, families);
2965
2966
2967 int startRow = 100;
2968 putData(startRow, numRows, qualifier, families);
2969 int splitRow = startRow + numRows;
2970 putData(splitRow, numRows, qualifier, families);
2971 region.flushcache();
2972
2973 HRegion[] regions = null;
2974 try {
2975 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
2976
2977 for (int i = 0; i < regions.length; i++) {
2978 regions[i] = HRegion.openHRegion(regions[i], null);
2979 }
2980
2981 assertEquals(2, regions.length);
2982
2983
2984
2985 verifyData(regions[0], startRow, numRows, qualifier, families);
2986 verifyData(regions[1], splitRow, numRows, qualifier, families);
2987
2988 } finally {
2989 HRegion.closeHRegion(this.region);
2990 this.region = null;
2991 }
2992 }
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004 @Test
3005 public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3006 byte[] family = Bytes.toBytes("family");
3007 int numRows = 1000;
3008 int flushAndScanInterval = 10;
3009 int compactInterval = 10 * flushAndScanInterval;
3010
3011 String method = "testFlushCacheWhileScanning";
3012 this.region = initHRegion(tableName, method, CONF, family);
3013 try {
3014 FlushThread flushThread = new FlushThread();
3015 flushThread.start();
3016
3017 Scan scan = new Scan();
3018 scan.addFamily(family);
3019 scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
3020 new BinaryComparator(Bytes.toBytes(5L))));
3021
3022 int expectedCount = 0;
3023 List<Cell> res = new ArrayList<Cell>();
3024
3025 boolean toggle = true;
3026 for (long i = 0; i < numRows; i++) {
3027 Put put = new Put(Bytes.toBytes(i));
3028 put.setDurability(Durability.SKIP_WAL);
3029 put.add(family, qual1, Bytes.toBytes(i % 10));
3030 region.put(put);
3031
3032 if (i != 0 && i % compactInterval == 0) {
3033
3034 region.compactStores(true);
3035 }
3036
3037 if (i % 10 == 5L) {
3038 expectedCount++;
3039 }
3040
3041 if (i != 0 && i % flushAndScanInterval == 0) {
3042 res.clear();
3043 InternalScanner scanner = region.getScanner(scan);
3044 if (toggle) {
3045 flushThread.flush();
3046 }
3047 while (scanner.next(res))
3048 ;
3049 if (!toggle) {
3050 flushThread.flush();
3051 }
3052 assertEquals("i=" + i, expectedCount, res.size());
3053 toggle = !toggle;
3054 }
3055 }
3056
3057 flushThread.done();
3058 flushThread.join();
3059 flushThread.checkNoError();
3060 } finally {
3061 HRegion.closeHRegion(this.region);
3062 this.region = null;
3063 }
3064 }
3065
3066 protected class FlushThread extends Thread {
3067 private volatile boolean done;
3068 private Throwable error = null;
3069
3070 public void done() {
3071 done = true;
3072 synchronized (this) {
3073 interrupt();
3074 }
3075 }
3076
3077 public void checkNoError() {
3078 if (error != null) {
3079 assertNull(error);
3080 }
3081 }
3082
3083 @Override
3084 public void run() {
3085 done = false;
3086 while (!done) {
3087 synchronized (this) {
3088 try {
3089 wait();
3090 } catch (InterruptedException ignored) {
3091 if (done) {
3092 break;
3093 }
3094 }
3095 }
3096 try {
3097 region.flushcache();
3098 } catch (IOException e) {
3099 if (!done) {
3100 LOG.error("Error while flusing cache", e);
3101 error = e;
3102 }
3103 break;
3104 }
3105 }
3106
3107 }
3108
3109 public void flush() {
3110 synchronized (this) {
3111 notify();
3112 }
3113
3114 }
3115 }
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126 @Test
3127 public void testWritesWhileScanning() throws IOException, InterruptedException {
3128 int testCount = 100;
3129 int numRows = 1;
3130 int numFamilies = 10;
3131 int numQualifiers = 100;
3132 int flushInterval = 7;
3133 int compactInterval = 5 * flushInterval;
3134 byte[][] families = new byte[numFamilies][];
3135 for (int i = 0; i < numFamilies; i++) {
3136 families[i] = Bytes.toBytes("family" + i);
3137 }
3138 byte[][] qualifiers = new byte[numQualifiers][];
3139 for (int i = 0; i < numQualifiers; i++) {
3140 qualifiers[i] = Bytes.toBytes("qual" + i);
3141 }
3142
3143 String method = "testWritesWhileScanning";
3144 this.region = initHRegion(tableName, method, CONF, families);
3145 try {
3146 PutThread putThread = new PutThread(numRows, families, qualifiers);
3147 putThread.start();
3148 putThread.waitForFirstPut();
3149
3150 FlushThread flushThread = new FlushThread();
3151 flushThread.start();
3152
3153 Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3154
3155 int expectedCount = numFamilies * numQualifiers;
3156 List<Cell> res = new ArrayList<Cell>();
3157
3158 long prevTimestamp = 0L;
3159 for (int i = 0; i < testCount; i++) {
3160
3161 if (i != 0 && i % compactInterval == 0) {
3162 region.compactStores(true);
3163 }
3164
3165 if (i != 0 && i % flushInterval == 0) {
3166 flushThread.flush();
3167 }
3168
3169 boolean previousEmpty = res.isEmpty();
3170 res.clear();
3171 InternalScanner scanner = region.getScanner(scan);
3172 while (scanner.next(res))
3173 ;
3174 if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3175 assertEquals("i=" + i, expectedCount, res.size());
3176 long timestamp = res.get(0).getTimestamp();
3177 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3178 timestamp >= prevTimestamp);
3179 prevTimestamp = timestamp;
3180 }
3181 }
3182
3183 putThread.done();
3184
3185 region.flushcache();
3186
3187 putThread.join();
3188 putThread.checkNoError();
3189
3190 flushThread.done();
3191 flushThread.join();
3192 flushThread.checkNoError();
3193 } finally {
3194 try {
3195 HRegion.closeHRegion(this.region);
3196 } catch (DroppedSnapshotException dse) {
3197
3198
3199
3200
3201 }
3202 this.region = null;
3203 }
3204 }
3205
3206 protected class PutThread extends Thread {
3207 private volatile boolean done;
3208 private volatile int numPutsFinished = 0;
3209
3210 private Throwable error = null;
3211 private int numRows;
3212 private byte[][] families;
3213 private byte[][] qualifiers;
3214
3215 private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3216 this.numRows = numRows;
3217 this.families = families;
3218 this.qualifiers = qualifiers;
3219 }
3220
3221
3222
3223
3224 public void waitForFirstPut() throws InterruptedException {
3225
3226 while (numPutsFinished == 0) {
3227 checkNoError();
3228 Thread.sleep(50);
3229 }
3230 }
3231
3232 public void done() {
3233 done = true;
3234 synchronized (this) {
3235 interrupt();
3236 }
3237 }
3238
3239 public void checkNoError() {
3240 if (error != null) {
3241 assertNull(error);
3242 }
3243 }
3244
3245 @Override
3246 public void run() {
3247 done = false;
3248 while (!done) {
3249 try {
3250 for (int r = 0; r < numRows; r++) {
3251 byte[] row = Bytes.toBytes("row" + r);
3252 Put put = new Put(row);
3253 put.setDurability(Durability.SKIP_WAL);
3254 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3255 for (byte[] family : families) {
3256 for (byte[] qualifier : qualifiers) {
3257 put.add(family, qualifier, (long) numPutsFinished, value);
3258 }
3259 }
3260 region.put(put);
3261 numPutsFinished++;
3262 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3263 System.out.println("put iteration = " + numPutsFinished);
3264 Delete delete = new Delete(row, (long) numPutsFinished - 30);
3265 region.delete(delete);
3266 }
3267 numPutsFinished++;
3268 }
3269 } catch (InterruptedIOException e) {
3270
3271 } catch (IOException e) {
3272 LOG.error("error while putting records", e);
3273 error = e;
3274 break;
3275 }
3276 }
3277
3278 }
3279
3280 }
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291 @Test
3292 public void testWritesWhileGetting() throws Exception {
3293 int testCount = 100;
3294 int numRows = 1;
3295 int numFamilies = 10;
3296 int numQualifiers = 100;
3297 int compactInterval = 100;
3298 byte[][] families = new byte[numFamilies][];
3299 for (int i = 0; i < numFamilies; i++) {
3300 families[i] = Bytes.toBytes("family" + i);
3301 }
3302 byte[][] qualifiers = new byte[numQualifiers][];
3303 for (int i = 0; i < numQualifiers; i++) {
3304 qualifiers[i] = Bytes.toBytes("qual" + i);
3305 }
3306
3307
3308 String method = "testWritesWhileGetting";
3309
3310
3311
3312
3313
3314 Configuration conf = HBaseConfiguration.create(CONF);
3315 conf.setInt("hbase.hstore.compaction.min", 1);
3316 conf.setInt("hbase.hstore.compaction.max", 1000);
3317 this.region = initHRegion(tableName, method, conf, families);
3318 PutThread putThread = null;
3319 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3320 try {
3321 putThread = new PutThread(numRows, families, qualifiers);
3322 putThread.start();
3323 putThread.waitForFirstPut();
3324
3325
3326 ctx.addThread(new RepeatingTestThread(ctx) {
3327 private int flushesSinceCompact = 0;
3328 private final int maxFlushesSinceCompact = 20;
3329
3330 @Override
3331 public void doAnAction() throws Exception {
3332 if (region.flushcache()) {
3333 ++flushesSinceCompact;
3334 }
3335
3336
3337 if (flushesSinceCompact == maxFlushesSinceCompact) {
3338 region.compactStores(false);
3339 flushesSinceCompact = 0;
3340 }
3341 }
3342 });
3343 ctx.startThreads();
3344
3345 Get get = new Get(Bytes.toBytes("row0"));
3346 Result result = null;
3347
3348 int expectedCount = numFamilies * numQualifiers;
3349
3350 long prevTimestamp = 0L;
3351 for (int i = 0; i < testCount; i++) {
3352
3353 boolean previousEmpty = result == null || result.isEmpty();
3354 result = region.get(get);
3355 if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3356 assertEquals("i=" + i, expectedCount, result.size());
3357
3358
3359 long timestamp = 0;
3360 for (Cell kv : result.rawCells()) {
3361 if (CellUtil.matchingFamily(kv, families[0])
3362 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3363 timestamp = kv.getTimestamp();
3364 }
3365 }
3366 assertTrue(timestamp >= prevTimestamp);
3367 prevTimestamp = timestamp;
3368 Cell previousKV = null;
3369
3370 for (Cell kv : result.rawCells()) {
3371 byte[] thisValue = CellUtil.cloneValue(kv);
3372 if (previousKV != null) {
3373 if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3374 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3375 + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3376 + "(memStoreTS:" + kv.getMvccVersion() + ")");
3377 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3378 }
3379 }
3380 previousKV = kv;
3381 }
3382 }
3383 }
3384 } finally {
3385 if (putThread != null)
3386 putThread.done();
3387
3388 region.flushcache();
3389
3390 if (putThread != null) {
3391 putThread.join();
3392 putThread.checkNoError();
3393 }
3394
3395 ctx.stop();
3396 HRegion.closeHRegion(this.region);
3397 this.region = null;
3398 }
3399 }
3400
3401 @Test
3402 public void testHolesInMeta() throws Exception {
3403 byte[] family = Bytes.toBytes("family");
3404 this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF,
3405 false, family);
3406 try {
3407 byte[] rowNotServed = Bytes.toBytes("a");
3408 Get g = new Get(rowNotServed);
3409 try {
3410 region.get(g);
3411 fail();
3412 } catch (WrongRegionException x) {
3413
3414 }
3415 byte[] row = Bytes.toBytes("y");
3416 g = new Get(row);
3417 region.get(g);
3418 } finally {
3419 HRegion.closeHRegion(this.region);
3420 this.region = null;
3421 }
3422 }
3423
3424 @Test
3425 public void testIndexesScanWithOneDeletedRow() throws IOException {
3426 byte[] family = Bytes.toBytes("family");
3427
3428
3429 String method = "testIndexesScanWithOneDeletedRow";
3430 this.region = initHRegion(tableName, method, CONF, family);
3431 try {
3432 Put put = new Put(Bytes.toBytes(1L));
3433 put.add(family, qual1, 1L, Bytes.toBytes(1L));
3434 region.put(put);
3435
3436 region.flushcache();
3437
3438 Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3439 region.delete(delete);
3440
3441 put = new Put(Bytes.toBytes(2L));
3442 put.add(family, qual1, 2L, Bytes.toBytes(2L));
3443 region.put(put);
3444
3445 Scan idxScan = new Scan();
3446 idxScan.addFamily(family);
3447 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
3448 new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
3449 new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
3450 CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
3451 InternalScanner scanner = region.getScanner(idxScan);
3452 List<Cell> res = new ArrayList<Cell>();
3453
3454 while (scanner.next(res))
3455 ;
3456 assertEquals(1L, res.size());
3457 } finally {
3458 HRegion.closeHRegion(this.region);
3459 this.region = null;
3460 }
3461 }
3462
3463
3464
3465
3466 @Test
3467 public void testBloomFilterSize() throws IOException {
3468 byte[] fam1 = Bytes.toBytes("fam1");
3469 byte[] qf1 = Bytes.toBytes("col");
3470 byte[] val1 = Bytes.toBytes("value1");
3471
3472 HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
3473 .setBloomFilterType(BloomType.ROWCOL);
3474
3475 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3476 htd.addFamily(hcd);
3477 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3478 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3479 try {
3480 int num_unique_rows = 10;
3481 int duplicate_multiplier = 2;
3482 int num_storefiles = 4;
3483
3484 int version = 0;
3485 for (int f = 0; f < num_storefiles; f++) {
3486 for (int i = 0; i < duplicate_multiplier; i++) {
3487 for (int j = 0; j < num_unique_rows; j++) {
3488 Put put = new Put(Bytes.toBytes("row" + j));
3489 put.setDurability(Durability.SKIP_WAL);
3490 put.add(fam1, qf1, version++, val1);
3491 region.put(put);
3492 }
3493 }
3494 region.flushcache();
3495 }
3496
3497 HStore store = (HStore) region.getStore(fam1);
3498 Collection<StoreFile> storeFiles = store.getStorefiles();
3499 for (StoreFile storefile : storeFiles) {
3500 StoreFile.Reader reader = storefile.getReader();
3501 reader.loadFileInfo();
3502 reader.loadBloomfilter();
3503 assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
3504 assertEquals(num_unique_rows, reader.getFilterEntries());
3505 }
3506
3507 region.compactStores(true);
3508
3509
3510 storeFiles = store.getStorefiles();
3511 for (StoreFile storefile : storeFiles) {
3512 StoreFile.Reader reader = storefile.getReader();
3513 reader.loadFileInfo();
3514 reader.loadBloomfilter();
3515 assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
3516 assertEquals(num_unique_rows, reader.getFilterEntries());
3517 }
3518 } finally {
3519 HRegion.closeHRegion(this.region);
3520 this.region = null;
3521 }
3522 }
3523
3524 @Test
3525 public void testAllColumnsWithBloomFilter() throws IOException {
3526 byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
3527 byte[] FAMILY = Bytes.toBytes("family");
3528
3529
3530 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
3531 .setBloomFilterType(BloomType.ROWCOL);
3532 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
3533 htd.addFamily(hcd);
3534 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3535 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3536 try {
3537
3538 byte row[] = Bytes.toBytes("row:" + 0);
3539 byte column[] = Bytes.toBytes("column:" + 0);
3540 Put put = new Put(row);
3541 put.setDurability(Durability.SKIP_WAL);
3542 for (long idx = 1; idx <= 4; idx++) {
3543 put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
3544 }
3545 region.put(put);
3546
3547
3548 region.flushcache();
3549
3550
3551 Get get = new Get(row);
3552 get.setMaxVersions();
3553 Cell[] kvs = region.get(get).rawCells();
3554
3555
3556 assertEquals(4, kvs.length);
3557 checkOneCell(kvs[0], FAMILY, 0, 0, 4);
3558 checkOneCell(kvs[1], FAMILY, 0, 0, 3);
3559 checkOneCell(kvs[2], FAMILY, 0, 0, 2);
3560 checkOneCell(kvs[3], FAMILY, 0, 0, 1);
3561 } finally {
3562 HRegion.closeHRegion(this.region);
3563 this.region = null;
3564 }
3565 }
3566
3567
3568
3569
3570
3571
3572 @Test
3573 public void testDeleteRowWithBloomFilter() throws IOException {
3574 byte[] familyName = Bytes.toBytes("familyName");
3575
3576
3577 HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
3578 .setBloomFilterType(BloomType.ROWCOL);
3579
3580 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3581 htd.addFamily(hcd);
3582 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3583 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3584 try {
3585
3586 byte row[] = Bytes.toBytes("row1");
3587 byte col[] = Bytes.toBytes("col1");
3588
3589 Put put = new Put(row);
3590 put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
3591 region.put(put);
3592 region.flushcache();
3593
3594 Delete del = new Delete(row);
3595 region.delete(del);
3596 region.flushcache();
3597
3598
3599 Get get = new Get(row);
3600 get.addColumn(familyName, col);
3601
3602 Cell[] keyValues = region.get(get).rawCells();
3603 assertTrue(keyValues.length == 0);
3604 } finally {
3605 HRegion.closeHRegion(this.region);
3606 this.region = null;
3607 }
3608 }
3609
3610 @Test
3611 public void testgetHDFSBlocksDistribution() throws Exception {
3612 HBaseTestingUtility htu = new HBaseTestingUtility();
3613 final int DEFAULT_BLOCK_SIZE = 1024;
3614 htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
3615 htu.getConfiguration().setInt("dfs.replication", 2);
3616
3617
3618 MiniHBaseCluster cluster = null;
3619 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
3620 int regionServersCount = 3;
3621
3622 try {
3623 cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
3624 byte[][] families = { fam1, fam2 };
3625 HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);
3626
3627
3628 byte row[] = Bytes.toBytes("row1");
3629 byte col[] = Bytes.toBytes("col1");
3630
3631 Put put = new Put(row);
3632 put.add(fam1, col, 1, Bytes.toBytes("test1"));
3633 put.add(fam2, col, 1, Bytes.toBytes("test2"));
3634 ht.put(put);
3635
3636 HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
3637 .get(0);
3638 firstRegion.flushcache();
3639 HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
3640
3641
3642
3643
3644
3645 long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
3646
3647 String topHost = blocksDistribution1.getTopHosts().get(0);
3648 long topHostWeight = blocksDistribution1.getWeight(topHost);
3649 assertTrue(uniqueBlocksWeight1 == topHostWeight);
3650
3651
3652
3653 HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
3654 htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
3655 long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
3656
3657 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
3658
3659 ht.close();
3660 } finally {
3661 if (cluster != null) {
3662 htu.shutdownMiniCluster();
3663 }
3664 }
3665 }
3666
3667
3668
3669
3670
3671
3672
3673 @Test
3674 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
3675 TableName tableName = TableName.valueOf(name.getMethodName());
3676 HRegionInfo info = null;
3677 try {
3678 FileSystem fs = Mockito.mock(FileSystem.class);
3679 Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
3680 HTableDescriptor htd = new HTableDescriptor(tableName);
3681 htd.addFamily(new HColumnDescriptor("cf"));
3682 info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
3683 HConstants.EMPTY_BYTE_ARRAY, false);
3684 Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
3685 region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
3686
3687 region.initialize();
3688 fail("Region initialization should fail due to IOException");
3689 } catch (IOException io) {
3690 List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
3691 for (MonitoredTask monitoredTask : tasks) {
3692 if (!(monitoredTask instanceof MonitoredRPCHandler)
3693 && monitoredTask.getDescription().contains(region.toString())) {
3694 assertTrue("Region state should be ABORTED.",
3695 monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
3696 break;
3697 }
3698 }
3699 } finally {
3700 HRegion.closeHRegion(region);
3701 }
3702 }
3703
3704
3705
3706
3707
3708 @Test
3709 public void testRegionInfoFileCreation() throws IOException {
3710 Path rootDir = new Path(dir + "testRegionInfoFileCreation");
3711
3712 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
3713 htd.addFamily(new HColumnDescriptor("cf"));
3714
3715 HRegionInfo hri = new HRegionInfo(htd.getTableName());
3716
3717
3718 HRegion region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
3719
3720 Path regionDir = region.getRegionFileSystem().getRegionDir();
3721 FileSystem fs = region.getRegionFileSystem().getFileSystem();
3722 HRegion.closeHRegion(region);
3723
3724 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
3725
3726
3727 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3728 fs.exists(regionInfoFile));
3729
3730
3731 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3732 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3733 HRegion.closeHRegion(region);
3734
3735
3736 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3737 fs.exists(regionInfoFile));
3738
3739
3740 fs.delete(regionInfoFile);
3741 assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
3742 fs.exists(regionInfoFile));
3743
3744 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
3745
3746 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3747 HRegion.closeHRegion(region);
3748
3749
3750 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3751 fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
3752 }
3753
3754
3755
3756
3757 private static class Incrementer implements Runnable {
3758 private HRegion region;
3759 private final static byte[] incRow = Bytes.toBytes("incRow");
3760 private final static byte[] family = Bytes.toBytes("family");
3761 private final static byte[] qualifier = Bytes.toBytes("qualifier");
3762 private final static long ONE = 1l;
3763 private int incCounter;
3764
3765 public Incrementer(HRegion region, int incCounter) {
3766 this.region = region;
3767 this.incCounter = incCounter;
3768 }
3769
3770 @Override
3771 public void run() {
3772 int count = 0;
3773 while (count < incCounter) {
3774 Increment inc = new Increment(incRow);
3775 inc.addColumn(family, qualifier, ONE);
3776 count++;
3777 try {
3778 region.increment(inc);
3779 } catch (IOException e) {
3780 e.printStackTrace();
3781 break;
3782 }
3783 }
3784 }
3785 }
3786
3787
3788
3789
3790
3791 @Test
3792 public void testParallelIncrementWithMemStoreFlush() throws Exception {
3793 byte[] family = Incrementer.family;
3794 this.region = initHRegion(tableName, method, CONF, family);
3795 final HRegion region = this.region;
3796 final AtomicBoolean incrementDone = new AtomicBoolean(false);
3797 Runnable flusher = new Runnable() {
3798 @Override
3799 public void run() {
3800 while (!incrementDone.get()) {
3801 try {
3802 region.flushcache();
3803 } catch (Exception e) {
3804 e.printStackTrace();
3805 }
3806 }
3807 }
3808 };
3809
3810
3811 int threadNum = 20;
3812 int incCounter = 100;
3813 long expected = threadNum * incCounter;
3814 Thread[] incrementers = new Thread[threadNum];
3815 Thread flushThread = new Thread(flusher);
3816 for (int i = 0; i < threadNum; i++) {
3817 incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
3818 incrementers[i].start();
3819 }
3820 flushThread.start();
3821 for (int i = 0; i < threadNum; i++) {
3822 incrementers[i].join();
3823 }
3824
3825 incrementDone.set(true);
3826 flushThread.join();
3827
3828 Get get = new Get(Incrementer.incRow);
3829 get.addColumn(Incrementer.family, Incrementer.qualifier);
3830 get.setMaxVersions(1);
3831 Result res = this.region.get(get);
3832 List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
3833
3834
3835 assertEquals(kvs.size(), 1);
3836 Cell kv = kvs.get(0);
3837 assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
3838 this.region = null;
3839 }
3840
3841
3842
3843
3844 private static class Appender implements Runnable {
3845 private HRegion region;
3846 private final static byte[] appendRow = Bytes.toBytes("appendRow");
3847 private final static byte[] family = Bytes.toBytes("family");
3848 private final static byte[] qualifier = Bytes.toBytes("qualifier");
3849 private final static byte[] CHAR = Bytes.toBytes("a");
3850 private int appendCounter;
3851
3852 public Appender(HRegion region, int appendCounter) {
3853 this.region = region;
3854 this.appendCounter = appendCounter;
3855 }
3856
3857 @Override
3858 public void run() {
3859 int count = 0;
3860 while (count < appendCounter) {
3861 Append app = new Append(appendRow);
3862 app.add(family, qualifier, CHAR);
3863 count++;
3864 try {
3865 region.append(app);
3866 } catch (IOException e) {
3867 e.printStackTrace();
3868 break;
3869 }
3870 }
3871 }
3872 }
3873
3874
3875
3876
3877
3878 @Test
3879 public void testParallelAppendWithMemStoreFlush() throws Exception {
3880 byte[] family = Appender.family;
3881 this.region = initHRegion(tableName, method, CONF, family);
3882 final HRegion region = this.region;
3883 final AtomicBoolean appendDone = new AtomicBoolean(false);
3884 Runnable flusher = new Runnable() {
3885 @Override
3886 public void run() {
3887 while (!appendDone.get()) {
3888 try {
3889 region.flushcache();
3890 } catch (Exception e) {
3891 e.printStackTrace();
3892 }
3893 }
3894 }
3895 };
3896
3897
3898
3899 int threadNum = 20;
3900 int appendCounter = 100;
3901 byte[] expected = new byte[threadNum * appendCounter];
3902 for (int i = 0; i < threadNum * appendCounter; i++) {
3903 System.arraycopy(Appender.CHAR, 0, expected, i, 1);
3904 }
3905 Thread[] appenders = new Thread[threadNum];
3906 Thread flushThread = new Thread(flusher);
3907 for (int i = 0; i < threadNum; i++) {
3908 appenders[i] = new Thread(new Appender(this.region, appendCounter));
3909 appenders[i].start();
3910 }
3911 flushThread.start();
3912 for (int i = 0; i < threadNum; i++) {
3913 appenders[i].join();
3914 }
3915
3916 appendDone.set(true);
3917 flushThread.join();
3918
3919 Get get = new Get(Appender.appendRow);
3920 get.addColumn(Appender.family, Appender.qualifier);
3921 get.setMaxVersions(1);
3922 Result res = this.region.get(get);
3923 List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
3924
3925
3926 assertEquals(kvs.size(), 1);
3927 Cell kv = kvs.get(0);
3928 byte[] appendResult = new byte[kv.getValueLength()];
3929 System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
3930 assertArrayEquals(expected, appendResult);
3931 this.region = null;
3932 }
3933
3934
3935
3936
3937
3938 @Test
3939 public void testPutWithMemStoreFlush() throws Exception {
3940 byte[] family = Bytes.toBytes("family");
3941 ;
3942 byte[] qualifier = Bytes.toBytes("qualifier");
3943 byte[] row = Bytes.toBytes("putRow");
3944 byte[] value = null;
3945 this.region = initHRegion(tableName, method, CONF, family);
3946 Put put = null;
3947 Get get = null;
3948 List<Cell> kvs = null;
3949 Result res = null;
3950
3951 put = new Put(row);
3952 value = Bytes.toBytes("value0");
3953 put.add(family, qualifier, 1234567l, value);
3954 region.put(put);
3955 get = new Get(row);
3956 get.addColumn(family, qualifier);
3957 get.setMaxVersions();
3958 res = this.region.get(get);
3959 kvs = res.getColumnCells(family, qualifier);
3960 assertEquals(1, kvs.size());
3961 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
3962
3963 region.flushcache();
3964 get = new Get(row);
3965 get.addColumn(family, qualifier);
3966 get.setMaxVersions();
3967 res = this.region.get(get);
3968 kvs = res.getColumnCells(family, qualifier);
3969 assertEquals(1, kvs.size());
3970 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
3971
3972 put = new Put(row);
3973 value = Bytes.toBytes("value1");
3974 put.add(family, qualifier, 1234567l, value);
3975 region.put(put);
3976 get = new Get(row);
3977 get.addColumn(family, qualifier);
3978 get.setMaxVersions();
3979 res = this.region.get(get);
3980 kvs = res.getColumnCells(family, qualifier);
3981 assertEquals(1, kvs.size());
3982 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
3983
3984 region.flushcache();
3985 get = new Get(row);
3986 get.addColumn(family, qualifier);
3987 get.setMaxVersions();
3988 res = this.region.get(get);
3989 kvs = res.getColumnCells(family, qualifier);
3990 assertEquals(1, kvs.size());
3991 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
3992 }
3993
3994 @Test
3995 public void testDurability() throws Exception {
3996 String method = "testDurability";
3997
3998
3999
4000
4001
4002 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4003 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4004 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4005
4006 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4007 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4008 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4009
4010 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4011 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4012
4013 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
4014 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4015
4016 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
4017 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
4018 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
4019
4020
4021 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4022 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4023 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4024 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4025 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
4026 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4027
4028 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4029 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4030 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4031 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4032 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
4033 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
4034
4035
4036 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4037 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4038 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
4039 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
4040 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
4041 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
4042
4043 }
4044
4045 private void durabilityTest(String method, Durability tableDurability,
4046 Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
4047 final boolean expectSyncFromLogSyncer) throws Exception {
4048 Configuration conf = HBaseConfiguration.create(CONF);
4049 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
4050 TableName tableName = TableName.valueOf(method);
4051 byte[] family = Bytes.toBytes("family");
4052 Path logDir = new Path(new Path(dir + method), "log");
4053 HLog hlog = HLogFactory.createHLog(FILESYSTEM, logDir, UUID.randomUUID().toString(), conf);
4054 final HLog log = spy(hlog);
4055 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
4056 HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, log,
4057 new byte[][] { family });
4058
4059 Put put = new Put(Bytes.toBytes("r1"));
4060 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
4061 put.setDurability(mutationDurability);
4062 region.put(put);
4063
4064
4065 verify(log, expectAppend ? times(1) : never())
4066 .appendNoSync((HRegionInfo)any(), eq(tableName), (WALEdit)any(), (List<UUID>)any(),
4067 anyLong(), (HTableDescriptor)any(), (AtomicLong)any(), anyBoolean(), anyLong(), anyLong());
4068
4069
4070 if (expectSync || expectSyncFromLogSyncer) {
4071 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
4072 @Override
4073 public boolean evaluate() throws Exception {
4074 try {
4075 if (expectSync) {
4076 verify(log, times(1)).sync(anyLong());
4077 } else if (expectSyncFromLogSyncer) {
4078 verify(log, times(1)).sync();
4079 }
4080 } catch (Throwable ignore) {
4081 }
4082 return true;
4083 }
4084 });
4085 } else {
4086 verify(log, never()).sync(anyLong());
4087 verify(log, never()).sync();
4088 }
4089
4090 HRegion.closeHRegion(this.region);
4091 this.region = null;
4092 }
4093
4094 private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4095 for (int i = startRow; i < startRow + numRows; i++) {
4096 Put put = new Put(Bytes.toBytes("" + i));
4097 put.setDurability(Durability.SKIP_WAL);
4098 for (byte[] family : families) {
4099 put.add(family, qf, null);
4100 }
4101 region.put(put);
4102 }
4103 }
4104
4105 private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
4106 throws IOException {
4107 for (int i = startRow; i < startRow + numRows; i++) {
4108 byte[] row = Bytes.toBytes("" + i);
4109 Get get = new Get(row);
4110 for (byte[] family : families) {
4111 get.addColumn(family, qf);
4112 }
4113 Result result = newReg.get(get);
4114 Cell[] raw = result.rawCells();
4115 assertEquals(families.length, result.size());
4116 for (int j = 0; j < families.length; j++) {
4117 assertTrue(CellUtil.matchingRow(raw[j], row));
4118 assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
4119 assertTrue(CellUtil.matchingQualifier(raw[j], qf));
4120 }
4121 }
4122 }
4123
4124 private void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
4125
4126 Get get = new Get(k).addFamily(family).setMaxVersions();
4127 Cell[] results = r.get(get).rawCells();
4128 for (int j = 0; j < results.length; j++) {
4129 byte[] tmp = CellUtil.cloneValue(results[j]);
4130
4131 assertTrue(Bytes.equals(k, tmp));
4132 }
4133 }
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146 private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
4147 throws IOException {
4148 byte[][] families = { fs };
4149 Scan scan = new Scan();
4150 for (int i = 0; i < families.length; i++)
4151 scan.addFamily(families[i]);
4152 InternalScanner s = r.getScanner(scan);
4153 try {
4154 List<Cell> curVals = new ArrayList<Cell>();
4155 boolean first = true;
4156 OUTER_LOOP: while (s.next(curVals)) {
4157 for (Cell kv : curVals) {
4158 byte[] val = CellUtil.cloneValue(kv);
4159 byte[] curval = val;
4160 if (first) {
4161 first = false;
4162 assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4163 } else {
4164
4165 break OUTER_LOOP;
4166 }
4167 }
4168 }
4169 } finally {
4170 s.close();
4171 }
4172 }
4173
4174 private Configuration initSplit() {
4175
4176 CONF.setInt("hbase.hstore.compactionThreshold", 2);
4177
4178
4179 CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4180
4181 CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4182
4183
4184 CONF.setLong("hbase.client.pause", 15 * 1000);
4185
4186
4187
4188 CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4189 return CONF;
4190 }
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201 public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
4202 byte[]... families) throws IOException {
4203 return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
4204 }
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4216 byte[]... families) throws IOException {
4217 return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4218 }
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4231 boolean isReadOnly, byte[]... families) throws IOException {
4232 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4233 }
4234
4235 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4236 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4237 throws IOException {
4238 return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
4239 Durability.SYNC_WAL, null, families);
4240 }
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4255 String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4256 HLog hlog, byte[]... families) throws IOException {
4257 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly, durability, hlog, families);
4258 }
4259
4260
4261
4262
4263
4264 private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
4265 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4266 assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
4267 Bytes.toString(CellUtil.cloneRow(kv)));
4268 assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
4269 Bytes.toString(CellUtil.cloneFamily(kv)));
4270 assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
4271 Bytes.toString(CellUtil.cloneQualifier(kv)));
4272 assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
4273 assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
4274 Bytes.toString(CellUtil.cloneValue(kv)));
4275 }
4276
4277 public void testReverseScanner_FromMemStore_SingleCF_Normal()
4278 throws IOException {
4279 byte[] rowC = Bytes.toBytes("rowC");
4280 byte[] rowA = Bytes.toBytes("rowA");
4281 byte[] rowB = Bytes.toBytes("rowB");
4282 byte[] cf = Bytes.toBytes("CF");
4283 byte[][] families = { cf };
4284 byte[] col = Bytes.toBytes("C");
4285 long ts = 1;
4286 String method = this.getName();
4287 this.region = initHRegion(tableName, method, families);
4288 try {
4289 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4290 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4291 null);
4292 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4293 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4294 Put put = null;
4295 put = new Put(rowC);
4296 put.add(kv1);
4297 put.add(kv11);
4298 region.put(put);
4299 put = new Put(rowA);
4300 put.add(kv2);
4301 region.put(put);
4302 put = new Put(rowB);
4303 put.add(kv3);
4304 region.put(put);
4305
4306 Scan scan = new Scan(rowC);
4307 scan.setMaxVersions(5);
4308 scan.setReversed(true);
4309 InternalScanner scanner = region.getScanner(scan);
4310 List<Cell> currRow = new ArrayList<Cell>();
4311 boolean hasNext = scanner.next(currRow);
4312 assertEquals(2, currRow.size());
4313 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4314 assertTrue(hasNext);
4315 currRow.clear();
4316 hasNext = scanner.next(currRow);
4317 assertEquals(1, currRow.size());
4318 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4319 assertTrue(hasNext);
4320 currRow.clear();
4321 hasNext = scanner.next(currRow);
4322 assertEquals(1, currRow.size());
4323 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4324 assertFalse(hasNext);
4325 scanner.close();
4326 } finally {
4327 HRegion.closeHRegion(this.region);
4328 this.region = null;
4329 }
4330 }
4331
4332 public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
4333 throws IOException {
4334 byte[] rowC = Bytes.toBytes("rowC");
4335 byte[] rowA = Bytes.toBytes("rowA");
4336 byte[] rowB = Bytes.toBytes("rowB");
4337 byte[] rowD = Bytes.toBytes("rowD");
4338 byte[] cf = Bytes.toBytes("CF");
4339 byte[][] families = { cf };
4340 byte[] col = Bytes.toBytes("C");
4341 long ts = 1;
4342 String method = this.getName();
4343 this.region = initHRegion(tableName, method, families);
4344 try {
4345 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4346 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4347 null);
4348 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4349 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4350 Put put = null;
4351 put = new Put(rowC);
4352 put.add(kv1);
4353 put.add(kv11);
4354 region.put(put);
4355 put = new Put(rowA);
4356 put.add(kv2);
4357 region.put(put);
4358 put = new Put(rowB);
4359 put.add(kv3);
4360 region.put(put);
4361
4362 Scan scan = new Scan(rowD);
4363 List<Cell> currRow = new ArrayList<Cell>();
4364 scan.setReversed(true);
4365 scan.setMaxVersions(5);
4366 InternalScanner scanner = region.getScanner(scan);
4367 boolean hasNext = scanner.next(currRow);
4368 assertEquals(2, currRow.size());
4369 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4370 assertTrue(hasNext);
4371 currRow.clear();
4372 hasNext = scanner.next(currRow);
4373 assertEquals(1, currRow.size());
4374 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4375 assertTrue(hasNext);
4376 currRow.clear();
4377 hasNext = scanner.next(currRow);
4378 assertEquals(1, currRow.size());
4379 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4380 assertFalse(hasNext);
4381 scanner.close();
4382 } finally {
4383 HRegion.closeHRegion(this.region);
4384 this.region = null;
4385 }
4386 }
4387
4388 public void testReverseScanner_FromMemStore_SingleCF_FullScan()
4389 throws IOException {
4390 byte[] rowC = Bytes.toBytes("rowC");
4391 byte[] rowA = Bytes.toBytes("rowA");
4392 byte[] rowB = Bytes.toBytes("rowB");
4393 byte[] cf = Bytes.toBytes("CF");
4394 byte[][] families = { cf };
4395 byte[] col = Bytes.toBytes("C");
4396 long ts = 1;
4397 String method = this.getName();
4398 this.region = initHRegion(tableName, method, families);
4399 try {
4400 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4401 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4402 null);
4403 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4404 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4405 Put put = null;
4406 put = new Put(rowC);
4407 put.add(kv1);
4408 put.add(kv11);
4409 region.put(put);
4410 put = new Put(rowA);
4411 put.add(kv2);
4412 region.put(put);
4413 put = new Put(rowB);
4414 put.add(kv3);
4415 region.put(put);
4416 Scan scan = new Scan();
4417 List<Cell> currRow = new ArrayList<Cell>();
4418 scan.setReversed(true);
4419 InternalScanner scanner = region.getScanner(scan);
4420 boolean hasNext = scanner.next(currRow);
4421 assertEquals(1, currRow.size());
4422 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4423 assertTrue(hasNext);
4424 currRow.clear();
4425 hasNext = scanner.next(currRow);
4426 assertEquals(1, currRow.size());
4427 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4428 assertTrue(hasNext);
4429 currRow.clear();
4430 hasNext = scanner.next(currRow);
4431 assertEquals(1, currRow.size());
4432 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4433 assertFalse(hasNext);
4434 scanner.close();
4435 } finally {
4436 HRegion.closeHRegion(this.region);
4437 this.region = null;
4438 }
4439 }
4440
4441 public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
4442
4443 byte[] rowA = Bytes.toBytes("rowA");
4444 byte[] rowB = Bytes.toBytes("rowB");
4445 byte[] rowC = Bytes.toBytes("rowC");
4446 byte[] rowD = Bytes.toBytes("rowD");
4447 byte[] rowE = Bytes.toBytes("rowE");
4448 byte[] cf = Bytes.toBytes("CF");
4449 byte[][] families = { cf };
4450 byte[] col1 = Bytes.toBytes("col1");
4451 byte[] col2 = Bytes.toBytes("col2");
4452 long ts = 1;
4453 String method = this.getName();
4454 this.region = initHRegion(tableName, method, families);
4455 try {
4456 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4457 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4458 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4459 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4460 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4461 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4462 Put put = null;
4463 put = new Put(rowA);
4464 put.add(kv1);
4465 region.put(put);
4466 put = new Put(rowB);
4467 put.add(kv2);
4468 region.put(put);
4469 put = new Put(rowC);
4470 put.add(kv3);
4471 region.put(put);
4472 put = new Put(rowD);
4473 put.add(kv4_1);
4474 region.put(put);
4475 put = new Put(rowD);
4476 put.add(kv4_2);
4477 region.put(put);
4478 put = new Put(rowE);
4479 put.add(kv5);
4480 region.put(put);
4481 region.flushcache();
4482 Scan scan = new Scan(rowD, rowA);
4483 scan.addColumn(families[0], col1);
4484 scan.setReversed(true);
4485 List<Cell> currRow = new ArrayList<Cell>();
4486 InternalScanner scanner = region.getScanner(scan);
4487 boolean hasNext = scanner.next(currRow);
4488 assertEquals(1, currRow.size());
4489 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4490 assertTrue(hasNext);
4491 currRow.clear();
4492 hasNext = scanner.next(currRow);
4493 assertEquals(1, currRow.size());
4494 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4495 assertTrue(hasNext);
4496 currRow.clear();
4497 hasNext = scanner.next(currRow);
4498 assertEquals(1, currRow.size());
4499 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4500 assertFalse(hasNext);
4501 scanner.close();
4502
4503 scan = new Scan(rowD, rowA);
4504 scan.addColumn(families[0], col2);
4505 scan.setReversed(true);
4506 currRow.clear();
4507 scanner = region.getScanner(scan);
4508 hasNext = scanner.next(currRow);
4509 assertEquals(1, currRow.size());
4510 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4511 scanner.close();
4512 } finally {
4513 HRegion.closeHRegion(this.region);
4514 this.region = null;
4515 }
4516 }
4517
4518 public void testReverseScanner_smaller_blocksize() throws IOException {
4519
4520 byte[] rowA = Bytes.toBytes("rowA");
4521 byte[] rowB = Bytes.toBytes("rowB");
4522 byte[] rowC = Bytes.toBytes("rowC");
4523 byte[] rowD = Bytes.toBytes("rowD");
4524 byte[] rowE = Bytes.toBytes("rowE");
4525 byte[] cf = Bytes.toBytes("CF");
4526 byte[][] families = { cf };
4527 byte[] col1 = Bytes.toBytes("col1");
4528 byte[] col2 = Bytes.toBytes("col2");
4529 long ts = 1;
4530 String method = this.getName();
4531 HBaseConfiguration config = new HBaseConfiguration();
4532 config.setInt("test.block.size", 1);
4533 this.region = initHRegion(tableName, method, config, families);
4534 try {
4535 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4536 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4537 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4538 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4539 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4540 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4541 Put put = null;
4542 put = new Put(rowA);
4543 put.add(kv1);
4544 region.put(put);
4545 put = new Put(rowB);
4546 put.add(kv2);
4547 region.put(put);
4548 put = new Put(rowC);
4549 put.add(kv3);
4550 region.put(put);
4551 put = new Put(rowD);
4552 put.add(kv4_1);
4553 region.put(put);
4554 put = new Put(rowD);
4555 put.add(kv4_2);
4556 region.put(put);
4557 put = new Put(rowE);
4558 put.add(kv5);
4559 region.put(put);
4560 region.flushcache();
4561 Scan scan = new Scan(rowD, rowA);
4562 scan.addColumn(families[0], col1);
4563 scan.setReversed(true);
4564 List<Cell> currRow = new ArrayList<Cell>();
4565 InternalScanner scanner = region.getScanner(scan);
4566 boolean hasNext = scanner.next(currRow);
4567 assertEquals(1, currRow.size());
4568 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4569 assertTrue(hasNext);
4570 currRow.clear();
4571 hasNext = scanner.next(currRow);
4572 assertEquals(1, currRow.size());
4573 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4574 assertTrue(hasNext);
4575 currRow.clear();
4576 hasNext = scanner.next(currRow);
4577 assertEquals(1, currRow.size());
4578 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4579 assertFalse(hasNext);
4580 scanner.close();
4581
4582 scan = new Scan(rowD, rowA);
4583 scan.addColumn(families[0], col2);
4584 scan.setReversed(true);
4585 currRow.clear();
4586 scanner = region.getScanner(scan);
4587 hasNext = scanner.next(currRow);
4588 assertEquals(1, currRow.size());
4589 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4590 scanner.close();
4591 } finally {
4592 HRegion.closeHRegion(this.region);
4593 this.region = null;
4594 }
4595 }
4596
4597 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
4598 throws IOException {
4599 byte[] row0 = Bytes.toBytes("row0");
4600 byte[] row1 = Bytes.toBytes("row1");
4601 byte[] row2 = Bytes.toBytes("row2");
4602 byte[] row3 = Bytes.toBytes("row3");
4603 byte[] row4 = Bytes.toBytes("row4");
4604 byte[] row5 = Bytes.toBytes("row5");
4605 byte[] cf1 = Bytes.toBytes("CF1");
4606 byte[] cf2 = Bytes.toBytes("CF2");
4607 byte[] cf3 = Bytes.toBytes("CF3");
4608 byte[][] families = { cf1, cf2, cf3 };
4609 byte[] col = Bytes.toBytes("C");
4610 long ts = 1;
4611 String method = this.getName();
4612 HBaseConfiguration conf = new HBaseConfiguration();
4613
4614 conf.setInt("hbase.hstore.compactionThreshold", 10000);
4615 this.region = initHRegion(tableName, method, conf, families);
4616 try {
4617
4618 KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
4619 null);
4620 KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
4621 null);
4622 KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
4623 KeyValue.Type.Put, null);
4624 KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
4625 null);
4626 KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
4627 null);
4628 KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
4629 null);
4630 KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
4631 KeyValue.Type.Put, null);
4632 KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
4633 null);
4634 KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
4635 KeyValue.Type.Put, null);
4636 KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
4637 null);
4638 KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
4639 null);
4640 KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
4641 KeyValue.Type.Put, null);
4642 KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
4643 null);
4644 KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
4645 KeyValue.Type.Put, null);
4646 KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
4647 null);
4648 KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
4649 null);
4650
4651 Put put = null;
4652 put = new Put(row1);
4653 put.add(kv1_2_1);
4654 region.put(put);
4655 put = new Put(row2);
4656 put.add(kv2_4_1);
4657 region.put(put);
4658 put = new Put(row4);
4659 put.add(kv4_5_4);
4660 put.add(kv4_5_5);
4661 region.put(put);
4662 region.flushcache();
4663
4664 put = new Put(row4);
4665 put.add(kv4_5_1);
4666 put.add(kv4_5_3);
4667 region.put(put);
4668 put = new Put(row1);
4669 put.add(kv1_2_2);
4670 region.put(put);
4671 put = new Put(row2);
4672 put.add(kv2_4_4);
4673 region.put(put);
4674 region.flushcache();
4675
4676 put = new Put(row4);
4677 put.add(kv4_5_2);
4678 region.put(put);
4679 put = new Put(row2);
4680 put.add(kv2_4_2);
4681 put.add(kv2_4_3);
4682 region.put(put);
4683 put = new Put(row3);
4684 put.add(kv3_2_2);
4685 region.put(put);
4686 region.flushcache();
4687
4688
4689 put = new Put(row0);
4690 put.add(kv0_1_1);
4691 region.put(put);
4692 put = new Put(row3);
4693 put.add(kv3_2_1);
4694 region.put(put);
4695 put = new Put(row5);
4696 put.add(kv5_2_1);
4697 put.add(kv5_2_2);
4698 region.put(put);
4699
4700 Scan scan = new Scan(row4);
4701 scan.setMaxVersions(5);
4702 scan.setBatch(3);
4703 scan.setReversed(true);
4704 InternalScanner scanner = region.getScanner(scan);
4705 List<Cell> currRow = new ArrayList<Cell>();
4706 boolean hasNext = false;
4707
4708
4709
4710 hasNext = scanner.next(currRow);
4711 assertEquals(3, currRow.size());
4712 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4713 assertTrue(hasNext);
4714 currRow.clear();
4715 hasNext = scanner.next(currRow);
4716 assertEquals(2, currRow.size());
4717 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4718 assertTrue(hasNext);
4719
4720 currRow.clear();
4721 hasNext = scanner.next(currRow);
4722 assertEquals(2, currRow.size());
4723 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
4724 assertTrue(hasNext);
4725
4726
4727 currRow.clear();
4728 hasNext = scanner.next(currRow);
4729 assertEquals(3, currRow.size());
4730 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4731 assertTrue(hasNext);
4732 currRow.clear();
4733 hasNext = scanner.next(currRow);
4734 assertEquals(1, currRow.size());
4735 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4736 assertTrue(hasNext);
4737
4738 currRow.clear();
4739 hasNext = scanner.next(currRow);
4740 assertEquals(2, currRow.size());
4741 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
4742 assertTrue(hasNext);
4743
4744 currRow.clear();
4745 hasNext = scanner.next(currRow);
4746 assertEquals(1, currRow.size());
4747 assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
4748 assertFalse(hasNext);
4749
4750 scanner.close();
4751 } finally {
4752 HRegion.closeHRegion(this.region);
4753 this.region = null;
4754 }
4755 }
4756
4757 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
4758 throws IOException {
4759 byte[] row1 = Bytes.toBytes("row1");
4760 byte[] row2 = Bytes.toBytes("row2");
4761 byte[] row3 = Bytes.toBytes("row3");
4762 byte[] row4 = Bytes.toBytes("row4");
4763 byte[] cf1 = Bytes.toBytes("CF1");
4764 byte[] cf2 = Bytes.toBytes("CF2");
4765 byte[] cf3 = Bytes.toBytes("CF3");
4766 byte[] cf4 = Bytes.toBytes("CF4");
4767 byte[][] families = { cf1, cf2, cf3, cf4 };
4768 byte[] col = Bytes.toBytes("C");
4769 long ts = 1;
4770 String method = this.getName();
4771 HBaseConfiguration conf = new HBaseConfiguration();
4772
4773 conf.setInt("hbase.hstore.compactionThreshold", 10000);
4774 this.region = initHRegion(tableName, method, conf, families);
4775 try {
4776 KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
4777 KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
4778 KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
4779 KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
4780
4781 Put put = new Put(row1);
4782 put.add(kv1);
4783 region.put(put);
4784 region.flushcache();
4785
4786 put = new Put(row2);
4787 put.add(kv2);
4788 region.put(put);
4789 region.flushcache();
4790
4791 put = new Put(row3);
4792 put.add(kv3);
4793 region.put(put);
4794 region.flushcache();
4795
4796 put = new Put(row4);
4797 put.add(kv4);
4798 region.put(put);
4799
4800 Scan scan = new Scan(row4);
4801 scan.setReversed(true);
4802 scan.setBatch(10);
4803 InternalScanner scanner = region.getScanner(scan);
4804 List<Cell> currRow = new ArrayList<Cell>();
4805 boolean hasNext = scanner.next(currRow);
4806 assertEquals(1, currRow.size());
4807 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4808 assertTrue(hasNext);
4809 currRow.clear();
4810 hasNext = scanner.next(currRow);
4811 assertEquals(1, currRow.size());
4812 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
4813 assertTrue(hasNext);
4814 currRow.clear();
4815 hasNext = scanner.next(currRow);
4816 assertEquals(1, currRow.size());
4817 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4818 assertTrue(hasNext);
4819 currRow.clear();
4820 hasNext = scanner.next(currRow);
4821 assertEquals(1, currRow.size());
4822 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
4823 assertFalse(hasNext);
4824 } finally {
4825 HRegion.closeHRegion(this.region);
4826 this.region = null;
4827 }
4828 }
4829
4830 private static HRegion initHRegion(byte[] tableName, String callingMethod,
4831 byte[]... families) throws IOException {
4832 return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
4833 families);
4834 }
4835 }