1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21
22 import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23 import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24 import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25 import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29 import static org.junit.Assert.assertArrayEquals;
30 import static org.junit.Assert.assertEquals;
31 import static org.junit.Assert.assertFalse;
32 import static org.junit.Assert.assertNotNull;
33 import static org.junit.Assert.assertNull;
34 import static org.junit.Assert.assertTrue;
35 import static org.junit.Assert.fail;
36 import static org.mockito.Matchers.*;
37 import static org.mockito.Mockito.never;
38 import static org.mockito.Mockito.spy;
39 import static org.mockito.Mockito.times;
40 import static org.mockito.Mockito.verify;
41
42 import java.io.IOException;
43 import java.io.InterruptedIOException;
44 import java.util.ArrayList;
45 import java.util.Arrays;
46 import java.util.Collection;
47 import java.util.List;
48 import java.util.Map;
49 import java.util.NavigableMap;
50 import java.util.TreeMap;
51 import java.util.UUID;
52 import java.util.concurrent.atomic.AtomicBoolean;
53 import java.util.concurrent.atomic.AtomicInteger;
54 import java.util.concurrent.atomic.AtomicLong;
55 import java.util.concurrent.atomic.AtomicReference;
56
57 import org.apache.commons.logging.Log;
58 import org.apache.commons.logging.LogFactory;
59 import org.apache.hadoop.conf.Configuration;
60 import org.apache.hadoop.fs.FSDataOutputStream;
61 import org.apache.hadoop.fs.FileStatus;
62 import org.apache.hadoop.fs.FileSystem;
63 import org.apache.hadoop.fs.Path;
64 import org.apache.hadoop.hbase.Cell;
65 import org.apache.hadoop.hbase.CellComparator;
66 import org.apache.hadoop.hbase.CellUtil;
67 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
68 import org.apache.hadoop.hbase.HBaseConfiguration;
69 import org.apache.hadoop.hbase.HBaseTestCase;
70 import org.apache.hadoop.hbase.HBaseTestingUtility;
71 import org.apache.hadoop.hbase.HColumnDescriptor;
72 import org.apache.hadoop.hbase.HConstants;
73 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
74 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
75 import org.apache.hadoop.hbase.HRegionInfo;
76 import org.apache.hadoop.hbase.HTableDescriptor;
77 import org.apache.hadoop.hbase.KeyValue;
78 import org.apache.hadoop.hbase.MediumTests;
79 import org.apache.hadoop.hbase.MiniHBaseCluster;
80 import org.apache.hadoop.hbase.MultithreadedTestUtil;
81 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
82 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
83 import org.apache.hadoop.hbase.NotServingRegionException;
84 import org.apache.hadoop.hbase.TableName;
85 import org.apache.hadoop.hbase.Waiter;
86 import org.apache.hadoop.hbase.client.Append;
87 import org.apache.hadoop.hbase.client.Delete;
88 import org.apache.hadoop.hbase.client.Durability;
89 import org.apache.hadoop.hbase.client.Get;
90 import org.apache.hadoop.hbase.client.HTable;
91 import org.apache.hadoop.hbase.client.Increment;
92 import org.apache.hadoop.hbase.client.Put;
93 import org.apache.hadoop.hbase.client.Result;
94 import org.apache.hadoop.hbase.client.Scan;
95 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
96 import org.apache.hadoop.hbase.filter.BinaryComparator;
97 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
98 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
99 import org.apache.hadoop.hbase.filter.Filter;
100 import org.apache.hadoop.hbase.filter.FilterBase;
101 import org.apache.hadoop.hbase.filter.FilterList;
102 import org.apache.hadoop.hbase.filter.NullComparator;
103 import org.apache.hadoop.hbase.filter.PrefixFilter;
104 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
105 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
106 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
107 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
108 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
109 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
110 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
111 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
112 import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
113 import org.apache.hadoop.hbase.regionserver.wal.HLog;
114 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
115 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
116 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
117 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
118 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
119 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
120 import org.apache.hadoop.hbase.util.Bytes;
121 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
122 import org.apache.hadoop.hbase.util.FSUtils;
123 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
124 import org.apache.hadoop.hbase.util.PairOfSameType;
125 import org.apache.hadoop.hbase.util.Threads;
126 import org.junit.After;
127 import org.junit.Assert;
128 import org.junit.Before;
129 import org.junit.Rule;
130 import org.junit.Test;
131 import org.junit.experimental.categories.Category;
132 import org.junit.rules.TestName;
133 import org.mockito.Mockito;
134
135 import com.google.common.collect.Lists;
136
137
138
139
140
141
142
143 @Category(MediumTests.class)
144 @SuppressWarnings("deprecation")
145 public class TestHRegion {
146
147
148 static final Log LOG = LogFactory.getLog(TestHRegion.class);
149 @Rule public TestName name = new TestName();
150
151 private static final String COLUMN_FAMILY = "MyCF";
152
153 HRegion region = null;
154 private static HBaseTestingUtility TEST_UTIL;
155 public static Configuration conf ;
156 private String DIR;
157 private static FileSystem fs;
158 private final int MAX_VERSIONS = 2;
159
160
161 protected byte[] tableName;
162 protected String method;
163 protected final byte[] qual1 = Bytes.toBytes("qual1");
164 protected final byte[] qual2 = Bytes.toBytes("qual2");
165 protected final byte[] qual3 = Bytes.toBytes("qual3");
166 protected final byte[] value1 = Bytes.toBytes("value1");
167 protected final byte[] value2 = Bytes.toBytes("value2");
168 protected final byte[] row = Bytes.toBytes("rowA");
169 protected final byte[] row2 = Bytes.toBytes("rowB");
170
171 protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
172 .getInstance(MetricsAssertHelper.class);
173
174 @Before
175 public void setup() throws IOException {
176 this.TEST_UTIL = HBaseTestingUtility.createLocalHTU();
177 this.fs = TEST_UTIL.getTestFileSystem();
178 this.conf = TEST_UTIL.getConfiguration();
179 this.DIR = TEST_UTIL.getDataTestDir("TestHRegion").toString();
180 method = name.getMethodName();
181 tableName = Bytes.toBytes(name.getMethodName());
182 }
183
184 @After
185 public void tearDown() throws Exception {
186 EnvironmentEdgeManagerTestHelper.reset();
187 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
188 TEST_UTIL.cleanupTestDir();
189 }
190
191 String getName() {
192 return name.getMethodName();
193 }
194
195
196
197
198
199
200
201 @Test
202 public void testCompactionAffectedByScanners() throws Exception {
203 byte[] family = Bytes.toBytes("family");
204 this.region = initHRegion(tableName, method, conf, family);
205
206 Put put = new Put(Bytes.toBytes("r1"));
207 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
208 region.put(put);
209 region.flushcache();
210
211 Scan scan = new Scan();
212 scan.setMaxVersions(3);
213
214 RegionScanner scanner1 = region.getScanner(scan);
215
216 Delete delete = new Delete(Bytes.toBytes("r1"));
217 region.delete(delete);
218 region.flushcache();
219
220
221 RegionScanner scanner2 = region.getScanner(scan);
222
223 List<Cell> results = new ArrayList<Cell>();
224
225 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
226
227
228 region.compactStores(true);
229
230
231 RegionScanner scanner3 = region.getScanner(scan);
232
233
234 scanner1.next(results);
235 System.out.println(results);
236 assertEquals(1, results.size());
237
238 results.clear();
239 scanner2.next(results);
240 System.out.println(results);
241 assertEquals(0, results.size());
242
243 results.clear();
244 scanner3.next(results);
245 System.out.println(results);
246 assertEquals(0, results.size());
247 }
248
249 @Test
250 public void testToShowNPEOnRegionScannerReseek() throws Exception {
251 byte[] family = Bytes.toBytes("family");
252 this.region = initHRegion(tableName, method, conf, family);
253
254 Put put = new Put(Bytes.toBytes("r1"));
255 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
256 region.put(put);
257 put = new Put(Bytes.toBytes("r2"));
258 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
259 region.put(put);
260 region.flushcache();
261
262 Scan scan = new Scan();
263 scan.setMaxVersions(3);
264
265 RegionScanner scanner1 = region.getScanner(scan);
266
267 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
268
269 region.compactStores(true);
270
271 scanner1.reseek(Bytes.toBytes("r2"));
272 List<Cell> results = new ArrayList<Cell>();
273 scanner1.next(results);
274 Cell keyValue = results.get(0);
275 Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
276 scanner1.close();
277 }
278
279 @Test
280 public void testSkipRecoveredEditsReplay() throws Exception {
281 String method = "testSkipRecoveredEditsReplay";
282 TableName tableName = TableName.valueOf(method);
283 byte[] family = Bytes.toBytes("family");
284 this.region = initHRegion(tableName, method, conf, family);
285 try {
286 Path regiondir = region.getRegionFileSystem().getRegionDir();
287 FileSystem fs = region.getRegionFileSystem().getFileSystem();
288 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
289
290 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
291
292 long maxSeqId = 1050;
293 long minSeqId = 1000;
294
295 for (long i = minSeqId; i <= maxSeqId; i += 10) {
296 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
297 fs.create(recoveredEdits);
298 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, conf);
299
300 long time = System.nanoTime();
301 WALEdit edit = new WALEdit();
302 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
303 .toBytes(i)));
304 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
305 HConstants.DEFAULT_CLUSTER_ID), edit));
306
307 writer.close();
308 }
309 MonitoredTask status = TaskMonitor.get().createStatus(method);
310 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
311 for (Store store : region.getStores().values()) {
312 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
313 }
314 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
315 assertEquals(maxSeqId, seqId);
316 Get get = new Get(row);
317 Result result = region.get(get);
318 for (long i = minSeqId; i <= maxSeqId; i += 10) {
319 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
320 assertEquals(1, kvs.size());
321 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
322 }
323 } finally {
324 HRegion.closeHRegion(this.region);
325 this.region = null;
326 }
327 }
328
329 @Test
330 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
331 String method = "testSkipRecoveredEditsReplaySomeIgnored";
332 TableName tableName = TableName.valueOf(method);
333 byte[] family = Bytes.toBytes("family");
334 this.region = initHRegion(tableName, method, conf, family);
335 try {
336 Path regiondir = region.getRegionFileSystem().getRegionDir();
337 FileSystem fs = region.getRegionFileSystem().getFileSystem();
338 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
339
340 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
341
342 long maxSeqId = 1050;
343 long minSeqId = 1000;
344
345 for (long i = minSeqId; i <= maxSeqId; i += 10) {
346 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
347 fs.create(recoveredEdits);
348 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, conf);
349
350 long time = System.nanoTime();
351 WALEdit edit = new WALEdit();
352 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
353 .toBytes(i)));
354 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
355 HConstants.DEFAULT_CLUSTER_ID), edit));
356
357 writer.close();
358 }
359 long recoverSeqId = 1030;
360 MonitoredTask status = TaskMonitor.get().createStatus(method);
361 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
362 for (Store store : region.getStores().values()) {
363 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
364 }
365 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
366 assertEquals(maxSeqId, seqId);
367 Get get = new Get(row);
368 Result result = region.get(get);
369 for (long i = minSeqId; i <= maxSeqId; i += 10) {
370 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
371 if (i < recoverSeqId) {
372 assertEquals(0, kvs.size());
373 } else {
374 assertEquals(1, kvs.size());
375 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
376 }
377 }
378 } finally {
379 HRegion.closeHRegion(this.region);
380 this.region = null;
381 }
382 }
383
384 @Test
385 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
386 byte[] family = Bytes.toBytes("family");
387 this.region = initHRegion(tableName, method, conf, family);
388 try {
389 Path regiondir = region.getRegionFileSystem().getRegionDir();
390 FileSystem fs = region.getRegionFileSystem().getFileSystem();
391
392 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
393 for (int i = 1000; i < 1050; i += 10) {
394 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
395 FSDataOutputStream dos = fs.create(recoveredEdits);
396 dos.writeInt(i);
397 dos.close();
398 }
399 long minSeqId = 2000;
400 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
401 FSDataOutputStream dos = fs.create(recoveredEdits);
402 dos.close();
403
404 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
405 for (Store store : region.getStores().values()) {
406 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
407 }
408 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
409 assertEquals(minSeqId, seqId);
410 } finally {
411 HRegion.closeHRegion(this.region);
412 this.region = null;
413 }
414 }
415
416 @Test
417 public void testRecoveredEditsReplayCompaction() throws Exception {
418 String method = name.getMethodName();
419 TableName tableName = TableName.valueOf(method);
420 byte[] family = Bytes.toBytes("family");
421 this.region = initHRegion(tableName, method, conf, family);
422 try {
423 Path regiondir = region.getRegionFileSystem().getRegionDir();
424 FileSystem fs = region.getRegionFileSystem().getFileSystem();
425 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
426
427 long maxSeqId = 3;
428 long minSeqId = 0;
429
430 for (long i = minSeqId; i < maxSeqId; i++) {
431 Put put = new Put(Bytes.toBytes(i));
432 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
433 region.put(put);
434 region.flushcache();
435 }
436
437
438 assertEquals(3, region.getStore(family).getStorefilesCount());
439 List<Path> storeFiles = new ArrayList<Path>(3);
440 for (StoreFile sf : region.getStore(family).getStorefiles()) {
441 storeFiles.add(sf.getPath());
442 }
443
444
445 conf.setBoolean("hbase.hstore.compaction.complete", false);
446 region.compactStores();
447
448
449 assertEquals(3, region.getStore(family).getStorefilesCount());
450
451
452 Path tmpDir = region.getRegionFileSystem().getTempDir();
453 FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
454 String errorMsg = "Expected to find 1 file in the region temp directory "
455 + "from the compaction, could not find any";
456 assertNotNull(errorMsg, files);
457 assertEquals(errorMsg, 1, files.length);
458
459 Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
460 files[0].getPath());
461
462 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
463 .getRegionInfo(), family, storeFiles, Lists.newArrayList(newFile), region
464 .getRegionFileSystem().getStoreDir(Bytes.toString(family)));
465
466 HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
467 this.region.getRegionInfo(), compactionDescriptor, new AtomicLong(1));
468
469 Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
470
471 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
472 fs.create(recoveredEdits);
473 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, conf);
474
475 long time = System.nanoTime();
476
477 writer.append(new HLog.Entry(new HLogKey(regionName, tableName, 10, time,
478 HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(compactionDescriptor)));
479 writer.close();
480
481
482 HTableDescriptor htd = region.getTableDesc();
483 HRegionInfo info = region.getRegionInfo();
484 region.close();
485 region = HRegion.openHRegion(region, null);
486
487
488 Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
489 for (StoreFile sf : sfs) {
490 LOG.info(sf.getPath());
491 }
492 assertEquals(1, region.getStore(family).getStorefilesCount());
493 files = FSUtils.listStatus(fs, tmpDir);
494 assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
495
496 for (long i = minSeqId; i < maxSeqId; i++) {
497 Get get = new Get(Bytes.toBytes(i));
498 Result result = region.get(get);
499 byte[] value = result.getValue(family, Bytes.toBytes(i));
500 assertArrayEquals(Bytes.toBytes(i), value);
501 }
502 } finally {
503 HRegion.closeHRegion(this.region);
504 this.region = null;
505 }
506 }
507
508 @Test
509 public void testGetWhileRegionClose() throws IOException {
510 TableName tableName = TableName.valueOf(name.getMethodName());
511 Configuration hc = initSplit();
512 int numRows = 100;
513 byte[][] families = { fam1, fam2, fam3 };
514
515
516 String method = name.getMethodName();
517 this.region = initHRegion(tableName, method, hc, families);
518 try {
519
520 final int startRow = 100;
521 putData(startRow, numRows, qual1, families);
522 putData(startRow, numRows, qual2, families);
523 putData(startRow, numRows, qual3, families);
524 final AtomicBoolean done = new AtomicBoolean(false);
525 final AtomicInteger gets = new AtomicInteger(0);
526 GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
527 try {
528
529 for (int i = 0; i < threads.length / 2; i++) {
530 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
531 threads[i].setDaemon(true);
532 threads[i].start();
533 }
534
535
536 this.region.closing.set(true);
537 for (int i = threads.length / 2; i < threads.length; i++) {
538 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
539 threads[i].setDaemon(true);
540 threads[i].start();
541 }
542 } finally {
543 if (this.region != null) {
544 HRegion.closeHRegion(this.region);
545 }
546 }
547 done.set(true);
548 for (GetTillDoneOrException t : threads) {
549 try {
550 t.join();
551 } catch (InterruptedException e) {
552 e.printStackTrace();
553 }
554 if (t.e != null) {
555 LOG.info("Exception=" + t.e);
556 assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
557 }
558 }
559 } finally {
560 HRegion.closeHRegion(this.region);
561 this.region = null;
562 }
563 }
564
565
566
567
568
569 class GetTillDoneOrException extends Thread {
570 private final Get g;
571 private final AtomicBoolean done;
572 private final AtomicInteger count;
573 private Exception e;
574
575 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
576 super("getter." + i);
577 this.g = new Get(r);
578 this.done = d;
579 this.count = c;
580 }
581
582 @Override
583 public void run() {
584 while (!this.done.get()) {
585 try {
586 assertTrue(region.get(g).size() > 0);
587 this.count.incrementAndGet();
588 } catch (Exception e) {
589 this.e = e;
590 break;
591 }
592 }
593 }
594 }
595
596
597
598
599 @Test
600 public void testWeirdCacheBehaviour() throws Exception {
601 byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
602 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
603 Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
604 this.region = initHRegion(TABLE, getName(), conf, FAMILIES);
605 try {
606 String value = "this is the value";
607 String value2 = "this is some other value";
608 String keyPrefix1 = "prefix1";
609 String keyPrefix2 = "prefix2";
610 String keyPrefix3 = "prefix3";
611 putRows(this.region, 3, value, keyPrefix1);
612 putRows(this.region, 3, value, keyPrefix2);
613 putRows(this.region, 3, value, keyPrefix3);
614 putRows(this.region, 3, value2, keyPrefix1);
615 putRows(this.region, 3, value2, keyPrefix2);
616 putRows(this.region, 3, value2, keyPrefix3);
617 System.out.println("Checking values for key: " + keyPrefix1);
618 assertEquals("Got back incorrect number of rows from scan", 3,
619 getNumberOfRows(keyPrefix1, value2, this.region));
620 System.out.println("Checking values for key: " + keyPrefix2);
621 assertEquals("Got back incorrect number of rows from scan", 3,
622 getNumberOfRows(keyPrefix2, value2, this.region));
623 System.out.println("Checking values for key: " + keyPrefix3);
624 assertEquals("Got back incorrect number of rows from scan", 3,
625 getNumberOfRows(keyPrefix3, value2, this.region));
626 deleteColumns(this.region, value2, keyPrefix1);
627 deleteColumns(this.region, value2, keyPrefix2);
628 deleteColumns(this.region, value2, keyPrefix3);
629 System.out.println("Starting important checks.....");
630 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
631 getNumberOfRows(keyPrefix1, value2, this.region));
632 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
633 getNumberOfRows(keyPrefix2, value2, this.region));
634 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
635 getNumberOfRows(keyPrefix3, value2, this.region));
636 } finally {
637 HRegion.closeHRegion(this.region);
638 this.region = null;
639 }
640 }
641
642 @Test
643 public void testAppendWithReadOnlyTable() throws Exception {
644 byte[] TABLE = Bytes.toBytes("readOnlyTable");
645 this.region = initHRegion(TABLE, getName(), conf, true, Bytes.toBytes("somefamily"));
646 boolean exceptionCaught = false;
647 Append append = new Append(Bytes.toBytes("somerow"));
648 append.setDurability(Durability.SKIP_WAL);
649 append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
650 Bytes.toBytes("somevalue"));
651 try {
652 region.append(append);
653 } catch (IOException e) {
654 exceptionCaught = true;
655 } finally {
656 HRegion.closeHRegion(this.region);
657 this.region = null;
658 }
659 assertTrue(exceptionCaught == true);
660 }
661
662 @Test
663 public void testIncrWithReadOnlyTable() throws Exception {
664 byte[] TABLE = Bytes.toBytes("readOnlyTable");
665 this.region = initHRegion(TABLE, getName(), conf, true, Bytes.toBytes("somefamily"));
666 boolean exceptionCaught = false;
667 Increment inc = new Increment(Bytes.toBytes("somerow"));
668 inc.setDurability(Durability.SKIP_WAL);
669 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
670 try {
671 region.increment(inc);
672 } catch (IOException e) {
673 exceptionCaught = true;
674 } finally {
675 HRegion.closeHRegion(this.region);
676 this.region = null;
677 }
678 assertTrue(exceptionCaught == true);
679 }
680
681 private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
682 InternalScanner scanner = buildScanner(keyPrefix, value, r);
683 int count = 0;
684 boolean more = false;
685 List<Cell> results = new ArrayList<Cell>();
686 do {
687 more = scanner.next(results);
688 if (results != null && !results.isEmpty())
689 count++;
690 else
691 break;
692 Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
693 delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
694 r.delete(delete);
695 results.clear();
696 } while (more);
697 assertEquals("Did not perform correct number of deletes", 3, count);
698 }
699
700 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
701 InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
702 int numberOfResults = 0;
703 List<Cell> results = new ArrayList<Cell>();
704 boolean more = false;
705 do {
706 more = resultScanner.next(results);
707 if (results != null && !results.isEmpty())
708 numberOfResults++;
709 else
710 break;
711 for (Cell kv : results) {
712 System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
713 }
714 results.clear();
715 } while (more);
716 return numberOfResults;
717 }
718
719 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
720 throws IOException {
721
722 FilterList allFilters = new FilterList();
723 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
724
725 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
726 Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
727 filter.setFilterIfMissing(true);
728 allFilters.addFilter(filter);
729 Scan scan = new Scan();
730 scan.addFamily(Bytes.toBytes("trans-blob"));
731 scan.addFamily(Bytes.toBytes("trans-type"));
732 scan.addFamily(Bytes.toBytes("trans-date"));
733 scan.addFamily(Bytes.toBytes("trans-tags"));
734 scan.addFamily(Bytes.toBytes("trans-group"));
735 scan.setFilter(allFilters);
736 return r.getScanner(scan);
737 }
738
739 private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
740 for (int i = 0; i < numRows; i++) {
741 String row = key + "_" + i
742 System.out.println(String.format("Saving row: %s, with value %s", row, value));
743 Put put = new Put(Bytes.toBytes(row));
744 put.setDurability(Durability.SKIP_WAL);
745 put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
746 put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
747 put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
748 put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
749 put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
750 r.put(put);
751 }
752 }
753
754 @Test
755 public void testFamilyWithAndWithoutColon() throws Exception {
756 byte[] b = Bytes.toBytes(getName());
757 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
758 this.region = initHRegion(b, getName(), conf, cf);
759 try {
760 Put p = new Put(b);
761 byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
762 p.add(cfwithcolon, cfwithcolon, cfwithcolon);
763 boolean exception = false;
764 try {
765 this.region.put(p);
766 } catch (NoSuchColumnFamilyException e) {
767 exception = true;
768 }
769 assertTrue(exception);
770 } finally {
771 HRegion.closeHRegion(this.region);
772 this.region = null;
773 }
774 }
775
776 @Test
777 public void testBatchPut() throws Exception {
778 byte[] b = Bytes.toBytes(getName());
779 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
780 byte[] qual = Bytes.toBytes("qual");
781 byte[] val = Bytes.toBytes("val");
782 this.region = initHRegion(b, getName(), conf, cf);
783 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
784 try {
785 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
786 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
787
788 LOG.info("First a batch put with all valid puts");
789 final Put[] puts = new Put[10];
790 for (int i = 0; i < 10; i++) {
791 puts[i] = new Put(Bytes.toBytes("row_" + i));
792 puts[i].add(cf, qual, val);
793 }
794
795 OperationStatus[] codes = this.region.batchMutate(puts);
796 assertEquals(10, codes.length);
797 for (int i = 0; i < 10; i++) {
798 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
799 }
800 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
801
802 LOG.info("Next a batch put with one invalid family");
803 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
804 codes = this.region.batchMutate(puts);
805 assertEquals(10, codes.length);
806 for (int i = 0; i < 10; i++) {
807 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
808 codes[i].getOperationStatusCode());
809 }
810
811 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
812
813 LOG.info("Next a batch put that has to break into two batches to avoid a lock");
814 RowLock rowLock = region.getRowLock(Bytes.toBytes("row_2"));
815
816 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
817 final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
818 TestThread putter = new TestThread(ctx) {
819 @Override
820 public void doWork() throws IOException {
821 retFromThread.set(region.batchMutate(puts));
822 }
823 };
824 LOG.info("...starting put thread while holding lock");
825 ctx.addThread(putter);
826 ctx.startThreads();
827
828 LOG.info("...waiting for put thread to sync first time");
829 long startWait = System.currentTimeMillis();
830 while (metricsAssertHelper.getCounter("syncTimeNumOps", source) == syncs + 2) {
831 Thread.sleep(100);
832 if (System.currentTimeMillis() - startWait > 10000) {
833 fail("Timed out waiting for thread to sync first minibatch");
834 }
835 }
836 LOG.info("...releasing row lock, which should let put thread continue");
837 rowLock.release();
838 LOG.info("...joining on thread");
839 ctx.stop();
840 LOG.info("...checking that next batch was synced");
841 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 4, source);
842 codes = retFromThread.get();
843 for (int i = 0; i < 10; i++) {
844 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
845 codes[i].getOperationStatusCode());
846 }
847
848 } finally {
849 HRegion.closeHRegion(this.region);
850 this.region = null;
851 }
852 }
853
854 @Test
855 public void testBatchPutWithTsSlop() throws Exception {
856 byte[] b = Bytes.toBytes(getName());
857 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
858 byte[] qual = Bytes.toBytes("qual");
859 byte[] val = Bytes.toBytes("val");
860
861
862 conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
863 this.region = initHRegion(b, getName(), conf, cf);
864
865 try {
866 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
867 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
868 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
869
870 final Put[] puts = new Put[10];
871 for (int i = 0; i < 10; i++) {
872 puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
873 puts[i].add(cf, qual, val);
874 }
875
876 OperationStatus[] codes = this.region.batchMutate(puts);
877 assertEquals(10, codes.length);
878 for (int i = 0; i < 10; i++) {
879 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
880 }
881 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
882
883 } finally {
884 HRegion.closeHRegion(this.region);
885 this.region = null;
886 }
887
888 }
889
890
891
892
893 @Test
894 public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
895 byte[] row1 = Bytes.toBytes("row1");
896 byte[] fam1 = Bytes.toBytes("fam1");
897 byte[] qf1 = Bytes.toBytes("qualifier");
898 byte[] emptyVal = new byte[] {};
899 byte[] val1 = Bytes.toBytes("value1");
900 byte[] val2 = Bytes.toBytes("value2");
901
902
903 String method = this.getName();
904 this.region = initHRegion(tableName, method, conf, fam1);
905 try {
906
907 Put put = new Put(row1);
908 put.add(fam1, qf1, emptyVal);
909
910
911 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
912 emptyVal), put, true);
913 assertTrue(res);
914
915
916 put = new Put(row1);
917 put.add(fam1, qf1, val1);
918
919
920 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
921 put, true);
922 assertTrue(res);
923
924
925 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
926 put, true);
927 assertFalse(res);
928
929 Delete delete = new Delete(row1);
930 delete.deleteColumn(fam1, qf1);
931 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
932 delete, true);
933 assertFalse(res);
934
935 put = new Put(row1);
936 put.add(fam1, qf1, val2);
937
938 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
939 put, true);
940 assertTrue(res);
941
942
943 delete = new Delete(row1);
944 delete.deleteColumn(fam1, qf1);
945 delete.deleteColumn(fam1, qf1);
946 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
947 delete, true);
948 assertTrue(res);
949
950 delete = new Delete(row1);
951 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
952 delete, true);
953 assertTrue(res);
954
955
956 put = new Put(row1);
957 put.add(fam1, qf1, val1);
958
959 res = region
960 .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
961 assertTrue(res);
962 } finally {
963 HRegion.closeHRegion(this.region);
964 this.region = null;
965 }
966 }
967
968 @Test
969 public void testCheckAndMutate_WithWrongValue() throws IOException {
970 byte[] row1 = Bytes.toBytes("row1");
971 byte[] fam1 = Bytes.toBytes("fam1");
972 byte[] qf1 = Bytes.toBytes("qualifier");
973 byte[] val1 = Bytes.toBytes("value1");
974 byte[] val2 = Bytes.toBytes("value2");
975
976
977 String method = this.getName();
978 this.region = initHRegion(tableName, method, conf, fam1);
979 try {
980
981 Put put = new Put(row1);
982 put.add(fam1, qf1, val1);
983 region.put(put);
984
985
986 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
987 val2), put, true);
988 assertEquals(false, res);
989
990
991 Delete delete = new Delete(row1);
992 delete.deleteFamily(fam1);
993 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
994 delete, true);
995 assertEquals(false, res);
996 } finally {
997 HRegion.closeHRegion(this.region);
998 this.region = null;
999 }
1000 }
1001
1002 @Test
1003 public void testCheckAndMutate_WithCorrectValue() throws IOException {
1004 byte[] row1 = Bytes.toBytes("row1");
1005 byte[] fam1 = Bytes.toBytes("fam1");
1006 byte[] qf1 = Bytes.toBytes("qualifier");
1007 byte[] val1 = Bytes.toBytes("value1");
1008
1009
1010 String method = this.getName();
1011 this.region = initHRegion(tableName, method, conf, fam1);
1012 try {
1013
1014 Put put = new Put(row1);
1015 put.add(fam1, qf1, val1);
1016 region.put(put);
1017
1018
1019 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1020 val1), put, true);
1021 assertEquals(true, res);
1022
1023
1024 Delete delete = new Delete(row1);
1025 delete.deleteColumn(fam1, qf1);
1026 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1027 put, true);
1028 assertEquals(true, res);
1029 } finally {
1030 HRegion.closeHRegion(this.region);
1031 this.region = null;
1032 }
1033 }
1034
1035 @Test
1036 public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1037 byte[] row1 = Bytes.toBytes("row1");
1038 byte[] fam1 = Bytes.toBytes("fam1");
1039 byte[] qf1 = Bytes.toBytes("qualifier");
1040 byte[] val1 = Bytes.toBytes("value1");
1041 byte[] val2 = Bytes.toBytes("value2");
1042 byte[] val3 = Bytes.toBytes("value3");
1043 byte[] val4 = Bytes.toBytes("value4");
1044
1045
1046 String method = this.getName();
1047 this.region = initHRegion(tableName, method, conf, fam1);
1048 try {
1049
1050 Put put = new Put(row1);
1051 put.add(fam1, qf1, val3);
1052 region.put(put);
1053
1054
1055 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1056 new BinaryComparator(val3), put, true);
1057 assertEquals(false, res);
1058
1059
1060 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1061 new BinaryComparator(val4), put, true);
1062 assertEquals(false, res);
1063
1064
1065
1066 put = new Put(row1);
1067 put.add(fam1, qf1, val2);
1068 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1069 new BinaryComparator(val2), put, true);
1070 assertEquals(true, res);
1071
1072
1073 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1074 new BinaryComparator(val3), put, true);
1075 assertEquals(false, res);
1076
1077
1078
1079 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1080 new BinaryComparator(val2), put, true);
1081 assertEquals(true, res);
1082
1083
1084
1085 put = new Put(row1);
1086 put.add(fam1, qf1, val3);
1087 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1088 new BinaryComparator(val1), put, true);
1089 assertEquals(true, res);
1090
1091
1092 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1093 new BinaryComparator(val3), put, true);
1094 assertEquals(false, res);
1095
1096
1097 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1098 new BinaryComparator(val2), put, true);
1099 assertEquals(false, res);
1100
1101
1102
1103 put = new Put(row1);
1104 put.add(fam1, qf1, val2);
1105 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1106 new BinaryComparator(val4), put, true);
1107 assertEquals(true, res);
1108
1109
1110 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1111 new BinaryComparator(val1), put, true);
1112 assertEquals(false, res);
1113
1114
1115
1116 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1117 new BinaryComparator(val2), put, true);
1118 assertEquals(true, res);
1119
1120
1121 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1122 new BinaryComparator(val3), put, true);
1123 assertEquals(true, res);
1124 } finally {
1125 HRegion.closeHRegion(this.region);
1126 this.region = null;
1127 }
1128 }
1129
1130 @Test
1131 public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1132 byte[] row1 = Bytes.toBytes("row1");
1133 byte[] fam1 = Bytes.toBytes("fam1");
1134 byte[] fam2 = Bytes.toBytes("fam2");
1135 byte[] qf1 = Bytes.toBytes("qualifier");
1136 byte[] val1 = Bytes.toBytes("value1");
1137 byte[] val2 = Bytes.toBytes("value2");
1138
1139 byte[][] families = { fam1, fam2 };
1140
1141
1142 String method = this.getName();
1143 this.region = initHRegion(tableName, method, conf, families);
1144 try {
1145
1146 Put put = new Put(row1);
1147 put.add(fam1, qf1, val1);
1148 region.put(put);
1149
1150
1151 long ts = System.currentTimeMillis();
1152 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1153 put = new Put(row1);
1154 put.add(kv);
1155
1156
1157 HStore store = (HStore) region.getStore(fam1);
1158 store.memstore.kvset.size();
1159
1160 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1161 val1), put, true);
1162 assertEquals(true, res);
1163 store.memstore.kvset.size();
1164
1165 Get get = new Get(row1);
1166 get.addColumn(fam2, qf1);
1167 Cell[] actual = region.get(get).rawCells();
1168
1169 Cell[] expected = { kv };
1170
1171 assertEquals(expected.length, actual.length);
1172 for (int i = 0; i < actual.length; i++) {
1173 assertEquals(expected[i], actual[i]);
1174 }
1175 } finally {
1176 HRegion.closeHRegion(this.region);
1177 this.region = null;
1178 }
1179 }
1180
1181 @Test
1182 public void testCheckAndPut_wrongRowInPut() throws IOException {
1183 TableName tableName = TableName.valueOf(name.getMethodName());
1184 this.region = initHRegion(tableName, this.getName(), conf, COLUMNS);
1185 try {
1186 Put put = new Put(row2);
1187 put.add(fam1, qual1, value1);
1188 try {
1189 boolean res = region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1190 new BinaryComparator(value2), put, false);
1191 fail();
1192 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1193
1194 }
1195 } finally {
1196 HRegion.closeHRegion(this.region);
1197 this.region = null;
1198 }
1199 }
1200
1201 @Test
1202 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1203 byte[] row1 = Bytes.toBytes("row1");
1204 byte[] fam1 = Bytes.toBytes("fam1");
1205 byte[] fam2 = Bytes.toBytes("fam2");
1206 byte[] qf1 = Bytes.toBytes("qualifier1");
1207 byte[] qf2 = Bytes.toBytes("qualifier2");
1208 byte[] qf3 = Bytes.toBytes("qualifier3");
1209 byte[] val1 = Bytes.toBytes("value1");
1210 byte[] val2 = Bytes.toBytes("value2");
1211 byte[] val3 = Bytes.toBytes("value3");
1212 byte[] emptyVal = new byte[] {};
1213
1214 byte[][] families = { fam1, fam2 };
1215
1216
1217 String method = this.getName();
1218 this.region = initHRegion(tableName, method, conf, families);
1219 try {
1220
1221 Put put = new Put(row1);
1222 put.add(fam1, qf1, val1);
1223 region.put(put);
1224 Threads.sleep(2);
1225
1226 put = new Put(row1);
1227 put.add(fam1, qf1, val2);
1228 put.add(fam2, qf1, val3);
1229 put.add(fam2, qf2, val2);
1230 put.add(fam2, qf3, val1);
1231 put.add(fam1, qf3, val1);
1232 region.put(put);
1233
1234
1235 Delete delete = new Delete(row1);
1236 delete.deleteColumn(fam1, qf1);
1237 delete.deleteColumn(fam2, qf1);
1238 delete.deleteColumn(fam1, qf3);
1239 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1240 val2), delete, true);
1241 assertEquals(true, res);
1242
1243 Get get = new Get(row1);
1244 get.addColumn(fam1, qf1);
1245 get.addColumn(fam1, qf3);
1246 get.addColumn(fam2, qf2);
1247 Result r = region.get(get);
1248 assertEquals(2, r.size());
1249 assertArrayEquals(val1, r.getValue(fam1, qf1));
1250 assertArrayEquals(val2, r.getValue(fam2, qf2));
1251
1252
1253 delete = new Delete(row1);
1254 delete.deleteFamily(fam2);
1255 res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1256 delete, true);
1257 assertEquals(true, res);
1258
1259 get = new Get(row1);
1260 r = region.get(get);
1261 assertEquals(1, r.size());
1262 assertArrayEquals(val1, r.getValue(fam1, qf1));
1263
1264
1265 delete = new Delete(row1);
1266 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1267 delete, true);
1268 assertEquals(true, res);
1269 get = new Get(row1);
1270 r = region.get(get);
1271 assertEquals(0, r.size());
1272 } finally {
1273 HRegion.closeHRegion(this.region);
1274 this.region = null;
1275 }
1276 }
1277
1278
1279
1280
1281 @Test
1282 public void testDelete_multiDeleteColumn() throws IOException {
1283 byte[] row1 = Bytes.toBytes("row1");
1284 byte[] fam1 = Bytes.toBytes("fam1");
1285 byte[] qual = Bytes.toBytes("qualifier");
1286 byte[] value = Bytes.toBytes("value");
1287
1288 Put put = new Put(row1);
1289 put.add(fam1, qual, 1, value);
1290 put.add(fam1, qual, 2, value);
1291
1292 String method = this.getName();
1293 this.region = initHRegion(tableName, method, conf, fam1);
1294 try {
1295 region.put(put);
1296
1297
1298 Delete delete = new Delete(row1);
1299 delete.deleteColumn(fam1, qual);
1300 delete.deleteColumn(fam1, qual);
1301 region.delete(delete);
1302
1303 Get get = new Get(row1);
1304 get.addFamily(fam1);
1305 Result r = region.get(get);
1306 assertEquals(0, r.size());
1307 } finally {
1308 HRegion.closeHRegion(this.region);
1309 this.region = null;
1310 }
1311 }
1312
1313 @Test
1314 public void testDelete_CheckFamily() throws IOException {
1315 byte[] row1 = Bytes.toBytes("row1");
1316 byte[] fam1 = Bytes.toBytes("fam1");
1317 byte[] fam2 = Bytes.toBytes("fam2");
1318 byte[] fam3 = Bytes.toBytes("fam3");
1319 byte[] fam4 = Bytes.toBytes("fam4");
1320
1321
1322 String method = this.getName();
1323 this.region = initHRegion(tableName, method, conf, fam1, fam2, fam3);
1324 try {
1325 List<Cell> kvs = new ArrayList<Cell>();
1326 kvs.add(new KeyValue(row1, fam4, null, null));
1327
1328
1329 byte[] family = fam2;
1330 try {
1331 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1332 Bytes.BYTES_COMPARATOR);
1333 deleteMap.put(family, kvs);
1334 region.delete(deleteMap, Durability.SYNC_WAL);
1335 } catch (Exception e) {
1336 assertTrue("Family " + new String(family) + " does not exist", false);
1337 }
1338
1339
1340 boolean ok = false;
1341 family = fam4;
1342 try {
1343 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1344 Bytes.BYTES_COMPARATOR);
1345 deleteMap.put(family, kvs);
1346 region.delete(deleteMap, Durability.SYNC_WAL);
1347 } catch (Exception e) {
1348 ok = true;
1349 }
1350 assertEquals("Family " + new String(family) + " does exist", true, ok);
1351 } finally {
1352 HRegion.closeHRegion(this.region);
1353 this.region = null;
1354 }
1355 }
1356
1357 @Test
1358 public void testDelete_mixed() throws IOException, InterruptedException {
1359 byte[] fam = Bytes.toBytes("info");
1360 byte[][] families = { fam };
1361 String method = this.getName();
1362 this.region = initHRegion(tableName, method, conf, families);
1363 try {
1364 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1365
1366 byte[] row = Bytes.toBytes("table_name");
1367
1368 byte[] serverinfo = Bytes.toBytes("serverinfo");
1369 byte[] splitA = Bytes.toBytes("splitA");
1370 byte[] splitB = Bytes.toBytes("splitB");
1371
1372
1373 Put put = new Put(row);
1374 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1375 region.put(put);
1376
1377 put = new Put(row);
1378 put.add(fam, splitB, Bytes.toBytes("reference_B"));
1379 region.put(put);
1380
1381 put = new Put(row);
1382 put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1383 region.put(put);
1384
1385
1386 Delete delete = new Delete(row);
1387 delete.deleteColumns(fam, splitA);
1388 region.delete(delete);
1389
1390
1391 Get get = new Get(row).addColumn(fam, serverinfo);
1392 Result result = region.get(get);
1393 assertEquals(1, result.size());
1394
1395 get = new Get(row).addColumn(fam, splitA);
1396 result = region.get(get);
1397 assertEquals(0, result.size());
1398
1399 get = new Get(row).addColumn(fam, splitB);
1400 result = region.get(get);
1401 assertEquals(1, result.size());
1402
1403
1404 put = new Put(row);
1405 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1406 region.put(put);
1407 get = new Get(row);
1408 result = region.get(get);
1409 assertEquals(3, result.size());
1410
1411
1412 delete = new Delete(row);
1413 region.delete(delete);
1414 assertEquals(0, region.get(get).size());
1415
1416 region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
1417 result = region.get(get);
1418 assertEquals(1, result.size());
1419 } finally {
1420 HRegion.closeHRegion(this.region);
1421 this.region = null;
1422 }
1423 }
1424
1425 @Test
1426 public void testDeleteRowWithFutureTs() throws IOException {
1427 byte[] fam = Bytes.toBytes("info");
1428 byte[][] families = { fam };
1429 String method = this.getName();
1430 this.region = initHRegion(tableName, method, conf, families);
1431 try {
1432 byte[] row = Bytes.toBytes("table_name");
1433
1434 byte[] serverinfo = Bytes.toBytes("serverinfo");
1435
1436
1437 Put put = new Put(row);
1438 put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
1439 region.put(put);
1440
1441
1442 Delete delete = new Delete(row);
1443 region.delete(delete);
1444
1445
1446 Get get = new Get(row).addColumn(fam, serverinfo);
1447 Result result = region.get(get);
1448 assertEquals(1, result.size());
1449
1450
1451 delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
1452 region.delete(delete);
1453
1454
1455 get = new Get(row).addColumn(fam, serverinfo);
1456 result = region.get(get);
1457 assertEquals(0, result.size());
1458 } finally {
1459 HRegion.closeHRegion(this.region);
1460 this.region = null;
1461 }
1462 }
1463
1464
1465
1466
1467
1468 @Test
1469 public void testPutWithLatestTS() throws IOException {
1470 byte[] fam = Bytes.toBytes("info");
1471 byte[][] families = { fam };
1472 String method = this.getName();
1473 this.region = initHRegion(tableName, method, conf, families);
1474 try {
1475 byte[] row = Bytes.toBytes("row1");
1476
1477 byte[] qual = Bytes.toBytes("qual");
1478
1479
1480 Put put = new Put(row);
1481 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1482 region.put(put);
1483
1484
1485 Get get = new Get(row).addColumn(fam, qual);
1486 Result result = region.get(get);
1487 assertEquals(1, result.size());
1488 Cell kv = result.rawCells()[0];
1489 LOG.info("Got: " + kv);
1490 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1491 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1492
1493
1494
1495 row = Bytes.toBytes("row2");
1496 put = new Put(row);
1497 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1498 region.put(put);
1499
1500
1501 get = new Get(row).addColumn(fam, qual);
1502 result = region.get(get);
1503 assertEquals(1, result.size());
1504 kv = result.rawCells()[0];
1505 LOG.info("Got: " + kv);
1506 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1507 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1508 } finally {
1509 HRegion.closeHRegion(this.region);
1510 this.region = null;
1511 }
1512
1513 }
1514
1515
1516
1517
1518
1519
1520 @Test
1521 public void testPutWithTsSlop() throws IOException {
1522 byte[] fam = Bytes.toBytes("info");
1523 byte[][] families = { fam };
1524 String method = this.getName();
1525
1526
1527 conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1528 this.region = initHRegion(tableName, method, conf, families);
1529 boolean caughtExcep = false;
1530 try {
1531 try {
1532
1533 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
1534
1535 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
1536 Bytes.toBytes("value")));
1537 fail("Expected IOE for TS out of configured timerange");
1538 } catch (FailedSanityCheckException ioe) {
1539 LOG.debug("Received expected exception", ioe);
1540 caughtExcep = true;
1541 }
1542 assertTrue("Should catch FailedSanityCheckException", caughtExcep);
1543 } finally {
1544 HRegion.closeHRegion(this.region);
1545 this.region = null;
1546 }
1547 }
1548
1549 @Test
1550 public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
1551 byte[] fam1 = Bytes.toBytes("columnA");
1552 byte[] fam2 = Bytes.toBytes("columnB");
1553 this.region = initHRegion(tableName, getName(), conf, fam1, fam2);
1554 try {
1555 byte[] rowA = Bytes.toBytes("rowA");
1556 byte[] rowB = Bytes.toBytes("rowB");
1557
1558 byte[] value = Bytes.toBytes("value");
1559
1560 Delete delete = new Delete(rowA);
1561 delete.deleteFamily(fam1);
1562
1563 region.delete(delete);
1564
1565
1566 Put put = new Put(rowA);
1567 put.add(fam2, null, value);
1568 region.put(put);
1569
1570 put = new Put(rowB);
1571 put.add(fam1, null, value);
1572 put.add(fam2, null, value);
1573 region.put(put);
1574
1575 Scan scan = new Scan();
1576 scan.addFamily(fam1).addFamily(fam2);
1577 InternalScanner s = region.getScanner(scan);
1578 List<Cell> results = new ArrayList<Cell>();
1579 s.next(results);
1580 assertTrue(CellUtil.matchingRow(results.get(0), rowA));
1581
1582 results.clear();
1583 s.next(results);
1584 assertTrue(CellUtil.matchingRow(results.get(0), rowB));
1585 } finally {
1586 HRegion.closeHRegion(this.region);
1587 this.region = null;
1588 }
1589 }
1590
1591 @Test
1592 public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
1593 Delete delete = new Delete(row);
1594 delete.deleteColumns(fam1, qual1);
1595 doTestDelete_AndPostInsert(delete);
1596 }
1597
1598 @Test
1599 public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
1600 Delete delete = new Delete(row);
1601 delete.deleteFamily(fam1);
1602 doTestDelete_AndPostInsert(delete);
1603 }
1604
1605 public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
1606 TableName tableName = TableName.valueOf(name.getMethodName());
1607 this.region = initHRegion(tableName, getName(), conf, fam1);
1608 try {
1609 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1610 Put put = new Put(row);
1611 put.add(fam1, qual1, value1);
1612 region.put(put);
1613
1614
1615 region.delete(delete);
1616
1617
1618 put = new Put(row);
1619 put.add(fam1, qual1, value2);
1620 region.put(put);
1621
1622
1623 Get get = new Get(row);
1624 get.addColumn(fam1, qual1);
1625
1626 Result r = region.get(get);
1627 assertEquals(1, r.size());
1628 assertArrayEquals(value2, r.getValue(fam1, qual1));
1629
1630
1631 Scan scan = new Scan(row);
1632 scan.addColumn(fam1, qual1);
1633 InternalScanner s = region.getScanner(scan);
1634
1635 List<Cell> results = new ArrayList<Cell>();
1636 assertEquals(false, s.next(results));
1637 assertEquals(1, results.size());
1638 Cell kv = results.get(0);
1639
1640 assertArrayEquals(value2, CellUtil.cloneValue(kv));
1641 assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
1642 assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
1643 assertArrayEquals(row, CellUtil.cloneRow(kv));
1644 } finally {
1645 HRegion.closeHRegion(this.region);
1646 this.region = null;
1647 }
1648 }
1649
1650 @Test
1651 public void testDelete_CheckTimestampUpdated() throws IOException {
1652 TableName tableName = TableName.valueOf(name.getMethodName());
1653 byte[] row1 = Bytes.toBytes("row1");
1654 byte[] col1 = Bytes.toBytes("col1");
1655 byte[] col2 = Bytes.toBytes("col2");
1656 byte[] col3 = Bytes.toBytes("col3");
1657
1658
1659 String method = this.getName();
1660 this.region = initHRegion(tableName, method, conf, fam1);
1661 try {
1662
1663 List<Cell> kvs = new ArrayList<Cell>();
1664 kvs.add(new KeyValue(row1, fam1, col1, null));
1665 kvs.add(new KeyValue(row1, fam1, col2, null));
1666 kvs.add(new KeyValue(row1, fam1, col3, null));
1667
1668 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
1669 Bytes.BYTES_COMPARATOR);
1670 deleteMap.put(fam1, kvs);
1671 region.delete(deleteMap, Durability.SYNC_WAL);
1672
1673
1674
1675 long now = System.currentTimeMillis();
1676 KeyValue firstKv = ((HStore) region.getStore(fam1)).memstore.kvset.first();
1677 assertTrue(firstKv.getTimestamp() <= now);
1678 now = firstKv.getTimestamp();
1679 for (KeyValue kv : ((HStore) region.getStore(fam1)).memstore.kvset) {
1680 assertTrue(kv.getTimestamp() <= now);
1681 now = kv.getTimestamp();
1682 }
1683 } finally {
1684 HRegion.closeHRegion(this.region);
1685 this.region = null;
1686 }
1687 }
1688
1689
1690
1691
1692 @Test
1693 public void testGet_FamilyChecker() throws IOException {
1694 byte[] row1 = Bytes.toBytes("row1");
1695 byte[] fam1 = Bytes.toBytes("fam1");
1696 byte[] fam2 = Bytes.toBytes("False");
1697 byte[] col1 = Bytes.toBytes("col1");
1698
1699
1700 String method = this.getName();
1701 this.region = initHRegion(tableName, method, conf, fam1);
1702 try {
1703 Get get = new Get(row1);
1704 get.addColumn(fam2, col1);
1705
1706
1707 try {
1708 region.get(get);
1709 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
1710 assertFalse(false);
1711 return;
1712 }
1713 assertFalse(true);
1714 } finally {
1715 HRegion.closeHRegion(this.region);
1716 this.region = null;
1717 }
1718 }
1719
1720 @Test
1721 public void testGet_Basic() throws IOException {
1722 byte[] row1 = Bytes.toBytes("row1");
1723 byte[] fam1 = Bytes.toBytes("fam1");
1724 byte[] col1 = Bytes.toBytes("col1");
1725 byte[] col2 = Bytes.toBytes("col2");
1726 byte[] col3 = Bytes.toBytes("col3");
1727 byte[] col4 = Bytes.toBytes("col4");
1728 byte[] col5 = Bytes.toBytes("col5");
1729
1730
1731 String method = this.getName();
1732 this.region = initHRegion(tableName, method, conf, fam1);
1733 try {
1734
1735 Put put = new Put(row1);
1736 put.add(fam1, col1, null);
1737 put.add(fam1, col2, null);
1738 put.add(fam1, col3, null);
1739 put.add(fam1, col4, null);
1740 put.add(fam1, col5, null);
1741 region.put(put);
1742
1743 Get get = new Get(row1);
1744 get.addColumn(fam1, col2);
1745 get.addColumn(fam1, col4);
1746
1747 KeyValue kv1 = new KeyValue(row1, fam1, col2);
1748 KeyValue kv2 = new KeyValue(row1, fam1, col4);
1749 KeyValue[] expected = { kv1, kv2 };
1750
1751
1752 Result res = region.get(get);
1753 assertEquals(expected.length, res.size());
1754 for (int i = 0; i < res.size(); i++) {
1755 assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
1756 assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
1757 assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
1758 }
1759
1760
1761 Get g = new Get(row1);
1762 final int count = 2;
1763 g.setFilter(new ColumnCountGetFilter(count));
1764 res = region.get(g);
1765 assertEquals(count, res.size());
1766 } finally {
1767 HRegion.closeHRegion(this.region);
1768 this.region = null;
1769 }
1770 }
1771
1772 @Test
1773 public void testGet_Empty() throws IOException {
1774 byte[] row = Bytes.toBytes("row");
1775 byte[] fam = Bytes.toBytes("fam");
1776
1777 String method = this.getName();
1778 this.region = initHRegion(tableName, method, conf, fam);
1779 try {
1780 Get get = new Get(row);
1781 get.addFamily(fam);
1782 Result r = region.get(get);
1783
1784 assertTrue(r.isEmpty());
1785 } finally {
1786 HRegion.closeHRegion(this.region);
1787 this.region = null;
1788 }
1789 }
1790
1791
1792
1793
1794 @Test
1795 public void testMerge() throws IOException {
1796 byte[][] families = { fam1, fam2, fam3 };
1797 Configuration hc = initSplit();
1798
1799 String method = this.getName();
1800 this.region = initHRegion(tableName, method, hc, families);
1801 try {
1802 LOG.info("" + HBaseTestCase.addContent(region, fam3));
1803 region.flushcache();
1804 region.compactStores();
1805 byte[] splitRow = region.checkSplit();
1806 assertNotNull(splitRow);
1807 LOG.info("SplitRow: " + Bytes.toString(splitRow));
1808 HRegion[] subregions = splitRegion(region, splitRow);
1809 try {
1810
1811 for (int i = 0; i < subregions.length; i++) {
1812 HRegion.openHRegion(subregions[i], null);
1813 subregions[i].compactStores();
1814 }
1815 Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
1816 Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
1817 Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
1818 long startTime = System.currentTimeMillis();
1819 region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
1820 LOG.info("Merge regions elapsed time: "
1821 + ((System.currentTimeMillis() - startTime) / 1000.0));
1822 fs.delete(oldRegion1, true);
1823 fs.delete(oldRegion2, true);
1824 fs.delete(oldRegionPath, true);
1825 LOG.info("splitAndMerge completed.");
1826 } finally {
1827 for (int i = 0; i < subregions.length; i++) {
1828 try {
1829 HRegion.closeHRegion(subregions[i]);
1830 } catch (IOException e) {
1831
1832 }
1833 }
1834 }
1835 } finally {
1836 HRegion.closeHRegion(this.region);
1837 this.region = null;
1838 }
1839 }
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849 HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
1850 PairOfSameType<HRegion> result = null;
1851 SplitTransaction st = new SplitTransaction(parent, midkey);
1852
1853
1854 if (!st.prepare())
1855 return null;
1856 try {
1857 result = st.execute(null, null);
1858 } catch (IOException ioe) {
1859 try {
1860 LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; "
1861 + ioe.getMessage());
1862 st.rollback(null, null);
1863 LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString());
1864 return null;
1865 } catch (RuntimeException e) {
1866
1867 LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString()
1868 + " -- aborting server", e);
1869 }
1870 }
1871 return new HRegion[] { result.getFirst(), result.getSecond() };
1872 }
1873
1874
1875
1876
1877 @Test
1878 public void testGetScanner_WithOkFamilies() throws IOException {
1879 byte[] fam1 = Bytes.toBytes("fam1");
1880 byte[] fam2 = Bytes.toBytes("fam2");
1881
1882 byte[][] families = { fam1, fam2 };
1883
1884
1885 String method = this.getName();
1886 this.region = initHRegion(tableName, method, conf, families);
1887 try {
1888 Scan scan = new Scan();
1889 scan.addFamily(fam1);
1890 scan.addFamily(fam2);
1891 try {
1892 region.getScanner(scan);
1893 } catch (Exception e) {
1894 assertTrue("Families could not be found in Region", false);
1895 }
1896 } finally {
1897 HRegion.closeHRegion(this.region);
1898 this.region = null;
1899 }
1900 }
1901
1902 @Test
1903 public void testGetScanner_WithNotOkFamilies() throws IOException {
1904 byte[] fam1 = Bytes.toBytes("fam1");
1905 byte[] fam2 = Bytes.toBytes("fam2");
1906
1907 byte[][] families = { fam1 };
1908
1909
1910 String method = this.getName();
1911 this.region = initHRegion(tableName, method, conf, families);
1912 try {
1913 Scan scan = new Scan();
1914 scan.addFamily(fam2);
1915 boolean ok = false;
1916 try {
1917 region.getScanner(scan);
1918 } catch (Exception e) {
1919 ok = true;
1920 }
1921 assertTrue("Families could not be found in Region", ok);
1922 } finally {
1923 HRegion.closeHRegion(this.region);
1924 this.region = null;
1925 }
1926 }
1927
1928 @Test
1929 public void testGetScanner_WithNoFamilies() throws IOException {
1930 byte[] row1 = Bytes.toBytes("row1");
1931 byte[] fam1 = Bytes.toBytes("fam1");
1932 byte[] fam2 = Bytes.toBytes("fam2");
1933 byte[] fam3 = Bytes.toBytes("fam3");
1934 byte[] fam4 = Bytes.toBytes("fam4");
1935
1936 byte[][] families = { fam1, fam2, fam3, fam4 };
1937
1938
1939 String method = this.getName();
1940 this.region = initHRegion(tableName, method, conf, families);
1941 try {
1942
1943
1944 Put put = new Put(row1);
1945 put.add(fam1, null, null);
1946 put.add(fam2, null, null);
1947 put.add(fam3, null, null);
1948 put.add(fam4, null, null);
1949 region.put(put);
1950
1951 Scan scan = null;
1952 HRegion.RegionScannerImpl is = null;
1953
1954
1955
1956
1957 scan = new Scan();
1958 scan.addFamily(fam2);
1959 scan.addFamily(fam4);
1960 is = (RegionScannerImpl) region.getScanner(scan);
1961 assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
1962
1963 scan = new Scan();
1964 is = (RegionScannerImpl) region.getScanner(scan);
1965 assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
1966 } finally {
1967 HRegion.closeHRegion(this.region);
1968 this.region = null;
1969 }
1970 }
1971
1972
1973
1974
1975
1976
1977 @Test
1978 public void testGetScanner_WithRegionClosed() throws IOException {
1979 byte[] fam1 = Bytes.toBytes("fam1");
1980 byte[] fam2 = Bytes.toBytes("fam2");
1981
1982 byte[][] families = { fam1, fam2 };
1983
1984
1985 String method = this.getName();
1986 try {
1987 this.region = initHRegion(tableName, method, conf, families);
1988 } catch (IOException e) {
1989 e.printStackTrace();
1990 fail("Got IOException during initHRegion, " + e.getMessage());
1991 }
1992 try {
1993 region.closed.set(true);
1994 try {
1995 region.getScanner(null);
1996 fail("Expected to get an exception during getScanner on a region that is closed");
1997 } catch (NotServingRegionException e) {
1998
1999 } catch (IOException e) {
2000 fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2001 + e.getMessage());
2002 }
2003 } finally {
2004 HRegion.closeHRegion(this.region);
2005 this.region = null;
2006 }
2007 }
2008
2009 @Test
2010 public void testRegionScanner_Next() throws IOException {
2011 byte[] row1 = Bytes.toBytes("row1");
2012 byte[] row2 = Bytes.toBytes("row2");
2013 byte[] fam1 = Bytes.toBytes("fam1");
2014 byte[] fam2 = Bytes.toBytes("fam2");
2015 byte[] fam3 = Bytes.toBytes("fam3");
2016 byte[] fam4 = Bytes.toBytes("fam4");
2017
2018 byte[][] families = { fam1, fam2, fam3, fam4 };
2019 long ts = System.currentTimeMillis();
2020
2021
2022 String method = this.getName();
2023 this.region = initHRegion(tableName, method, conf, families);
2024 try {
2025
2026 Put put = null;
2027 put = new Put(row1);
2028 put.add(fam1, (byte[]) null, ts, null);
2029 put.add(fam2, (byte[]) null, ts, null);
2030 put.add(fam3, (byte[]) null, ts, null);
2031 put.add(fam4, (byte[]) null, ts, null);
2032 region.put(put);
2033
2034 put = new Put(row2);
2035 put.add(fam1, (byte[]) null, ts, null);
2036 put.add(fam2, (byte[]) null, ts, null);
2037 put.add(fam3, (byte[]) null, ts, null);
2038 put.add(fam4, (byte[]) null, ts, null);
2039 region.put(put);
2040
2041 Scan scan = new Scan();
2042 scan.addFamily(fam2);
2043 scan.addFamily(fam4);
2044 InternalScanner is = region.getScanner(scan);
2045
2046 List<Cell> res = null;
2047
2048
2049 List<Cell> expected1 = new ArrayList<Cell>();
2050 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2051 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2052
2053 res = new ArrayList<Cell>();
2054 is.next(res);
2055 for (int i = 0; i < res.size(); i++) {
2056 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2057 }
2058
2059
2060 List<Cell> expected2 = new ArrayList<Cell>();
2061 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2062 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2063
2064 res = new ArrayList<Cell>();
2065 is.next(res);
2066 for (int i = 0; i < res.size(); i++) {
2067 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2068 }
2069 } finally {
2070 HRegion.closeHRegion(this.region);
2071 this.region = null;
2072 }
2073 }
2074
2075 @Test
2076 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2077 byte[] row1 = Bytes.toBytes("row1");
2078 byte[] qf1 = Bytes.toBytes("qualifier1");
2079 byte[] qf2 = Bytes.toBytes("qualifier2");
2080 byte[] fam1 = Bytes.toBytes("fam1");
2081 byte[][] families = { fam1 };
2082
2083 long ts1 = System.currentTimeMillis();
2084 long ts2 = ts1 + 1;
2085 long ts3 = ts1 + 2;
2086
2087
2088 String method = this.getName();
2089 this.region = initHRegion(tableName, method, conf, families);
2090 try {
2091
2092 Put put = null;
2093 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2094 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2095 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2096
2097 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2098 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2099 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2100
2101 put = new Put(row1);
2102 put.add(kv13);
2103 put.add(kv12);
2104 put.add(kv11);
2105 put.add(kv23);
2106 put.add(kv22);
2107 put.add(kv21);
2108 region.put(put);
2109
2110
2111 List<Cell> expected = new ArrayList<Cell>();
2112 expected.add(kv13);
2113 expected.add(kv12);
2114
2115 Scan scan = new Scan(row1);
2116 scan.addColumn(fam1, qf1);
2117 scan.setMaxVersions(MAX_VERSIONS);
2118 List<Cell> actual = new ArrayList<Cell>();
2119 InternalScanner scanner = region.getScanner(scan);
2120
2121 boolean hasNext = scanner.next(actual);
2122 assertEquals(false, hasNext);
2123
2124
2125 for (int i = 0; i < expected.size(); i++) {
2126 assertEquals(expected.get(i), actual.get(i));
2127 }
2128 } finally {
2129 HRegion.closeHRegion(this.region);
2130 this.region = null;
2131 }
2132 }
2133
2134 @Test
2135 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2136 byte[] row1 = Bytes.toBytes("row1");
2137 byte[] qf1 = Bytes.toBytes("qualifier1");
2138 byte[] qf2 = Bytes.toBytes("qualifier2");
2139 byte[] fam1 = Bytes.toBytes("fam1");
2140 byte[][] families = { fam1 };
2141
2142 long ts1 = 1;
2143 long ts2 = ts1 + 1;
2144 long ts3 = ts1 + 2;
2145
2146
2147 String method = this.getName();
2148 this.region = initHRegion(tableName, method, conf, families);
2149 try {
2150
2151 Put put = null;
2152 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2153 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2154 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2155
2156 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2157 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2158 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2159
2160 put = new Put(row1);
2161 put.add(kv13);
2162 put.add(kv12);
2163 put.add(kv11);
2164 put.add(kv23);
2165 put.add(kv22);
2166 put.add(kv21);
2167 region.put(put);
2168 region.flushcache();
2169
2170
2171 List<Cell> expected = new ArrayList<Cell>();
2172 expected.add(kv13);
2173 expected.add(kv12);
2174 expected.add(kv23);
2175 expected.add(kv22);
2176
2177 Scan scan = new Scan(row1);
2178 scan.addColumn(fam1, qf1);
2179 scan.addColumn(fam1, qf2);
2180 scan.setMaxVersions(MAX_VERSIONS);
2181 List<Cell> actual = new ArrayList<Cell>();
2182 InternalScanner scanner = region.getScanner(scan);
2183
2184 boolean hasNext = scanner.next(actual);
2185 assertEquals(false, hasNext);
2186
2187
2188 for (int i = 0; i < expected.size(); i++) {
2189 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2190 }
2191 } finally {
2192 HRegion.closeHRegion(this.region);
2193 this.region = null;
2194 }
2195 }
2196
2197 @Test
2198 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2199 byte[] row1 = Bytes.toBytes("row1");
2200 byte[] fam1 = Bytes.toBytes("fam1");
2201 byte[][] families = { fam1 };
2202 byte[] qf1 = Bytes.toBytes("qualifier1");
2203 byte[] qf2 = Bytes.toBytes("qualifier2");
2204
2205 long ts1 = 1;
2206 long ts2 = ts1 + 1;
2207 long ts3 = ts1 + 2;
2208 long ts4 = ts1 + 3;
2209
2210
2211 String method = this.getName();
2212 this.region = initHRegion(tableName, method, conf, families);
2213 try {
2214
2215 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2216 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2217 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2218 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2219
2220 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2221 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2222 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2223 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2224
2225 Put put = null;
2226 put = new Put(row1);
2227 put.add(kv14);
2228 put.add(kv24);
2229 region.put(put);
2230 region.flushcache();
2231
2232 put = new Put(row1);
2233 put.add(kv23);
2234 put.add(kv13);
2235 region.put(put);
2236 region.flushcache();
2237
2238 put = new Put(row1);
2239 put.add(kv22);
2240 put.add(kv12);
2241 region.put(put);
2242 region.flushcache();
2243
2244 put = new Put(row1);
2245 put.add(kv21);
2246 put.add(kv11);
2247 region.put(put);
2248
2249
2250 List<Cell> expected = new ArrayList<Cell>();
2251 expected.add(kv14);
2252 expected.add(kv13);
2253 expected.add(kv12);
2254 expected.add(kv24);
2255 expected.add(kv23);
2256 expected.add(kv22);
2257
2258 Scan scan = new Scan(row1);
2259 scan.addColumn(fam1, qf1);
2260 scan.addColumn(fam1, qf2);
2261 int versions = 3;
2262 scan.setMaxVersions(versions);
2263 List<Cell> actual = new ArrayList<Cell>();
2264 InternalScanner scanner = region.getScanner(scan);
2265
2266 boolean hasNext = scanner.next(actual);
2267 assertEquals(false, hasNext);
2268
2269
2270 for (int i = 0; i < expected.size(); i++) {
2271 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2272 }
2273 } finally {
2274 HRegion.closeHRegion(this.region);
2275 this.region = null;
2276 }
2277 }
2278
2279 @Test
2280 public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2281 byte[] row1 = Bytes.toBytes("row1");
2282 byte[] qf1 = Bytes.toBytes("qualifier1");
2283 byte[] qf2 = Bytes.toBytes("qualifier2");
2284 byte[] fam1 = Bytes.toBytes("fam1");
2285 byte[][] families = { fam1 };
2286
2287 long ts1 = System.currentTimeMillis();
2288 long ts2 = ts1 + 1;
2289 long ts3 = ts1 + 2;
2290
2291
2292 String method = this.getName();
2293 this.region = initHRegion(tableName, method, conf, families);
2294 try {
2295
2296 Put put = null;
2297 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2298 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2299 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2300
2301 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2302 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2303 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2304
2305 put = new Put(row1);
2306 put.add(kv13);
2307 put.add(kv12);
2308 put.add(kv11);
2309 put.add(kv23);
2310 put.add(kv22);
2311 put.add(kv21);
2312 region.put(put);
2313
2314
2315 List<Cell> expected = new ArrayList<Cell>();
2316 expected.add(kv13);
2317 expected.add(kv12);
2318 expected.add(kv23);
2319 expected.add(kv22);
2320
2321 Scan scan = new Scan(row1);
2322 scan.addFamily(fam1);
2323 scan.setMaxVersions(MAX_VERSIONS);
2324 List<Cell> actual = new ArrayList<Cell>();
2325 InternalScanner scanner = region.getScanner(scan);
2326
2327 boolean hasNext = scanner.next(actual);
2328 assertEquals(false, hasNext);
2329
2330
2331 for (int i = 0; i < expected.size(); i++) {
2332 assertEquals(expected.get(i), actual.get(i));
2333 }
2334 } finally {
2335 HRegion.closeHRegion(this.region);
2336 this.region = null;
2337 }
2338 }
2339
2340 @Test
2341 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
2342 byte[] row1 = Bytes.toBytes("row1");
2343 byte[] qf1 = Bytes.toBytes("qualifier1");
2344 byte[] qf2 = Bytes.toBytes("qualifier2");
2345 byte[] fam1 = Bytes.toBytes("fam1");
2346
2347 long ts1 = 1;
2348 long ts2 = ts1 + 1;
2349 long ts3 = ts1 + 2;
2350
2351
2352 String method = this.getName();
2353 this.region = initHRegion(tableName, method, conf, fam1);
2354 try {
2355
2356 Put put = null;
2357 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2358 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2359 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2360
2361 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2362 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2363 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2364
2365 put = new Put(row1);
2366 put.add(kv13);
2367 put.add(kv12);
2368 put.add(kv11);
2369 put.add(kv23);
2370 put.add(kv22);
2371 put.add(kv21);
2372 region.put(put);
2373 region.flushcache();
2374
2375
2376 List<Cell> expected = new ArrayList<Cell>();
2377 expected.add(kv13);
2378 expected.add(kv12);
2379 expected.add(kv23);
2380 expected.add(kv22);
2381
2382 Scan scan = new Scan(row1);
2383 scan.addFamily(fam1);
2384 scan.setMaxVersions(MAX_VERSIONS);
2385 List<Cell> actual = new ArrayList<Cell>();
2386 InternalScanner scanner = region.getScanner(scan);
2387
2388 boolean hasNext = scanner.next(actual);
2389 assertEquals(false, hasNext);
2390
2391
2392 for (int i = 0; i < expected.size(); i++) {
2393 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2394 }
2395 } finally {
2396 HRegion.closeHRegion(this.region);
2397 this.region = null;
2398 }
2399 }
2400
2401 @Test
2402 public void testScanner_StopRow1542() throws IOException {
2403 byte[] family = Bytes.toBytes("testFamily");
2404 this.region = initHRegion(tableName, getName(), conf, family);
2405 try {
2406 byte[] row1 = Bytes.toBytes("row111");
2407 byte[] row2 = Bytes.toBytes("row222");
2408 byte[] row3 = Bytes.toBytes("row333");
2409 byte[] row4 = Bytes.toBytes("row444");
2410 byte[] row5 = Bytes.toBytes("row555");
2411
2412 byte[] col1 = Bytes.toBytes("Pub111");
2413 byte[] col2 = Bytes.toBytes("Pub222");
2414
2415 Put put = new Put(row1);
2416 put.add(family, col1, Bytes.toBytes(10L));
2417 region.put(put);
2418
2419 put = new Put(row2);
2420 put.add(family, col1, Bytes.toBytes(15L));
2421 region.put(put);
2422
2423 put = new Put(row3);
2424 put.add(family, col2, Bytes.toBytes(20L));
2425 region.put(put);
2426
2427 put = new Put(row4);
2428 put.add(family, col2, Bytes.toBytes(30L));
2429 region.put(put);
2430
2431 put = new Put(row5);
2432 put.add(family, col1, Bytes.toBytes(40L));
2433 region.put(put);
2434
2435 Scan scan = new Scan(row3, row4);
2436 scan.setMaxVersions();
2437 scan.addColumn(family, col1);
2438 InternalScanner s = region.getScanner(scan);
2439
2440 List<Cell> results = new ArrayList<Cell>();
2441 assertEquals(false, s.next(results));
2442 assertEquals(0, results.size());
2443 } finally {
2444 HRegion.closeHRegion(this.region);
2445 this.region = null;
2446 }
2447 }
2448
2449 private void assertICV(byte[] row, byte[] familiy, byte[] qualifier, int amount)
2450 throws IOException {
2451
2452 Get get = new Get(row);
2453 get.addColumn(familiy, qualifier);
2454 Result result = region.get(get);
2455 assertEquals(1, result.size());
2456
2457 Cell kv = result.rawCells()[0];
2458 int r = Bytes.toInt(CellUtil.cloneValue(kv));
2459 assertEquals(amount, r);
2460 }
2461
2462 @Test
2463 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2464 byte[] row1 = Bytes.toBytes("row1");
2465 byte[] fam1 = Bytes.toBytes("fam1");
2466 byte[] qf1 = Bytes.toBytes("qualifier1");
2467 byte[] qf2 = Bytes.toBytes("quateslifier2");
2468
2469 long ts1 = 1;
2470 long ts2 = ts1 + 1;
2471 long ts3 = ts1 + 2;
2472 long ts4 = ts1 + 3;
2473
2474
2475 String method = this.getName();
2476 this.region = initHRegion(tableName, method, conf, fam1);
2477 try {
2478
2479 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2480 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2481 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2482 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2483
2484 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2485 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2486 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2487 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2488
2489 Put put = null;
2490 put = new Put(row1);
2491 put.add(kv14);
2492 put.add(kv24);
2493 region.put(put);
2494 region.flushcache();
2495
2496 put = new Put(row1);
2497 put.add(kv23);
2498 put.add(kv13);
2499 region.put(put);
2500 region.flushcache();
2501
2502 put = new Put(row1);
2503 put.add(kv22);
2504 put.add(kv12);
2505 region.put(put);
2506 region.flushcache();
2507
2508 put = new Put(row1);
2509 put.add(kv21);
2510 put.add(kv11);
2511 region.put(put);
2512
2513
2514 List<KeyValue> expected = new ArrayList<KeyValue>();
2515 expected.add(kv14);
2516 expected.add(kv13);
2517 expected.add(kv12);
2518 expected.add(kv24);
2519 expected.add(kv23);
2520 expected.add(kv22);
2521
2522 Scan scan = new Scan(row1);
2523 int versions = 3;
2524 scan.setMaxVersions(versions);
2525 List<Cell> actual = new ArrayList<Cell>();
2526 InternalScanner scanner = region.getScanner(scan);
2527
2528 boolean hasNext = scanner.next(actual);
2529 assertEquals(false, hasNext);
2530
2531
2532 for (int i = 0; i < expected.size(); i++) {
2533 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2534 }
2535 } finally {
2536 HRegion.closeHRegion(this.region);
2537 this.region = null;
2538 }
2539 }
2540
2541
2542
2543
2544
2545
2546
2547 @Test
2548 public void testScanner_JoinedScanners() throws IOException {
2549 byte[] cf_essential = Bytes.toBytes("essential");
2550 byte[] cf_joined = Bytes.toBytes("joined");
2551 byte[] cf_alpha = Bytes.toBytes("alpha");
2552 this.region = initHRegion(tableName, getName(), conf, cf_essential, cf_joined, cf_alpha);
2553 try {
2554 byte[] row1 = Bytes.toBytes("row1");
2555 byte[] row2 = Bytes.toBytes("row2");
2556 byte[] row3 = Bytes.toBytes("row3");
2557
2558 byte[] col_normal = Bytes.toBytes("d");
2559 byte[] col_alpha = Bytes.toBytes("a");
2560
2561 byte[] filtered_val = Bytes.toBytes(3);
2562
2563 Put put = new Put(row1);
2564 put.add(cf_essential, col_normal, Bytes.toBytes(1));
2565 put.add(cf_joined, col_alpha, Bytes.toBytes(1));
2566 region.put(put);
2567
2568 put = new Put(row2);
2569 put.add(cf_essential, col_alpha, Bytes.toBytes(2));
2570 put.add(cf_joined, col_normal, Bytes.toBytes(2));
2571 put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
2572 region.put(put);
2573
2574 put = new Put(row3);
2575 put.add(cf_essential, col_normal, filtered_val);
2576 put.add(cf_joined, col_normal, filtered_val);
2577 region.put(put);
2578
2579
2580
2581
2582
2583 Scan scan = new Scan();
2584 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
2585 CompareOp.NOT_EQUAL, filtered_val);
2586 scan.setFilter(filter);
2587 scan.setLoadColumnFamiliesOnDemand(true);
2588 InternalScanner s = region.getScanner(scan);
2589
2590 List<Cell> results = new ArrayList<Cell>();
2591 assertTrue(s.next(results));
2592 assertEquals(results.size(), 1);
2593 results.clear();
2594
2595 assertTrue(s.next(results));
2596 assertEquals(results.size(), 3);
2597 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
2598 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
2599 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
2600 results.clear();
2601
2602 assertFalse(s.next(results));
2603 assertEquals(results.size(), 0);
2604 } finally {
2605 HRegion.closeHRegion(this.region);
2606 this.region = null;
2607 }
2608 }
2609
2610
2611
2612
2613
2614
2615 @Test
2616 public void testScanner_JoinedScannersWithLimits() throws IOException {
2617 final byte[] cf_first = Bytes.toBytes("first");
2618 final byte[] cf_second = Bytes.toBytes("second");
2619
2620 this.region = initHRegion(tableName, getName(), conf, cf_first, cf_second);
2621 try {
2622 final byte[] col_a = Bytes.toBytes("a");
2623 final byte[] col_b = Bytes.toBytes("b");
2624
2625 Put put;
2626
2627 for (int i = 0; i < 10; i++) {
2628 put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
2629 put.add(cf_first, col_a, Bytes.toBytes(i));
2630 if (i < 5) {
2631 put.add(cf_first, col_b, Bytes.toBytes(i));
2632 put.add(cf_second, col_a, Bytes.toBytes(i));
2633 put.add(cf_second, col_b, Bytes.toBytes(i));
2634 }
2635 region.put(put);
2636 }
2637
2638 Scan scan = new Scan();
2639 scan.setLoadColumnFamiliesOnDemand(true);
2640 Filter bogusFilter = new FilterBase() {
2641 @Override
2642 public boolean isFamilyEssential(byte[] name) {
2643 return Bytes.equals(name, cf_first);
2644 }
2645 };
2646
2647 scan.setFilter(bogusFilter);
2648 InternalScanner s = region.getScanner(scan);
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679 List<Cell> results = new ArrayList<Cell>();
2680 int index = 0;
2681 while (true) {
2682 boolean more = s.next(results, 3);
2683 if ((index >> 1) < 5) {
2684 if (index % 2 == 0)
2685 assertEquals(results.size(), 3);
2686 else
2687 assertEquals(results.size(), 1);
2688 } else
2689 assertEquals(results.size(), 1);
2690 results.clear();
2691 index++;
2692 if (!more)
2693 break;
2694 }
2695 } finally {
2696 HRegion.closeHRegion(this.region);
2697 this.region = null;
2698 }
2699 }
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709 @Test
2710 public void testBasicSplit() throws Exception {
2711 byte[][] families = { fam1, fam2, fam3 };
2712
2713 Configuration hc = initSplit();
2714
2715 String method = this.getName();
2716 this.region = initHRegion(tableName, method, hc, families);
2717
2718 try {
2719 LOG.info("" + HBaseTestCase.addContent(region, fam3));
2720 region.flushcache();
2721 region.compactStores();
2722 byte[] splitRow = region.checkSplit();
2723 assertNotNull(splitRow);
2724 LOG.info("SplitRow: " + Bytes.toString(splitRow));
2725 HRegion[] regions = splitRegion(region, splitRow);
2726 try {
2727
2728
2729
2730 for (int i = 0; i < regions.length; i++) {
2731 regions[i] = HRegion.openHRegion(regions[i], null);
2732 }
2733
2734
2735 assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
2736 assertGet(regions[1], fam3, splitRow);
2737
2738 assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
2739 assertScan(regions[1], fam3, splitRow);
2740
2741 for (int i = 0; i < regions.length; i++) {
2742
2743
2744 for (int j = 0; j < 2; j++) {
2745 HBaseTestCase.addContent(regions[i], fam3);
2746 }
2747 HBaseTestCase.addContent(regions[i], fam2);
2748 HBaseTestCase.addContent(regions[i], fam1);
2749 regions[i].flushcache();
2750 }
2751
2752 byte[][] midkeys = new byte[regions.length][];
2753
2754 for (int i = 0; i < regions.length; i++) {
2755 regions[i].compactStores();
2756 midkeys[i] = regions[i].checkSplit();
2757 }
2758
2759 TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
2760
2761
2762 for (int i = 0; i < regions.length; i++) {
2763 HRegion[] rs = null;
2764 if (midkeys[i] != null) {
2765 rs = splitRegion(regions[i], midkeys[i]);
2766 for (int j = 0; j < rs.length; j++) {
2767 sortedMap.put(Bytes.toString(rs[j].getRegionName()), HRegion.openHRegion(rs[j], null));
2768 }
2769 }
2770 }
2771 LOG.info("Made 4 regions");
2772
2773
2774 int interval = (LAST_CHAR - FIRST_CHAR) / 3;
2775 byte[] b = Bytes.toBytes(START_KEY);
2776 for (HRegion r : sortedMap.values()) {
2777 assertGet(r, fam3, b);
2778 b[0] += interval;
2779 }
2780 } finally {
2781 for (int i = 0; i < regions.length; i++) {
2782 try {
2783 regions[i].close();
2784 } catch (IOException e) {
2785
2786 }
2787 }
2788 }
2789 } finally {
2790 HRegion.closeHRegion(this.region);
2791 this.region = null;
2792 }
2793 }
2794
2795 @Test
2796 public void testSplitRegion() throws IOException {
2797 byte[] qualifier = Bytes.toBytes("qualifier");
2798 Configuration hc = initSplit();
2799 int numRows = 10;
2800 byte[][] families = { fam1, fam3 };
2801
2802
2803 String method = this.getName();
2804 this.region = initHRegion(tableName, method, hc, families);
2805
2806
2807 int startRow = 100;
2808 putData(startRow, numRows, qualifier, families);
2809 int splitRow = startRow + numRows;
2810 putData(splitRow, numRows, qualifier, families);
2811 region.flushcache();
2812
2813 HRegion[] regions = null;
2814 try {
2815 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
2816
2817 for (int i = 0; i < regions.length; i++) {
2818 regions[i] = HRegion.openHRegion(regions[i], null);
2819 }
2820
2821 assertEquals(2, regions.length);
2822
2823
2824
2825 verifyData(regions[0], startRow, numRows, qualifier, families);
2826 verifyData(regions[1], splitRow, numRows, qualifier, families);
2827
2828 } finally {
2829 HRegion.closeHRegion(this.region);
2830 this.region = null;
2831 }
2832 }
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844 @Test
2845 public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
2846 byte[] family = Bytes.toBytes("family");
2847 int numRows = 1000;
2848 int flushAndScanInterval = 10;
2849 int compactInterval = 10 * flushAndScanInterval;
2850
2851 String method = "testFlushCacheWhileScanning";
2852 this.region = initHRegion(tableName, method, conf, family);
2853 try {
2854 FlushThread flushThread = new FlushThread();
2855 flushThread.start();
2856
2857 Scan scan = new Scan();
2858 scan.addFamily(family);
2859 scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
2860 new BinaryComparator(Bytes.toBytes(5L))));
2861
2862 int expectedCount = 0;
2863 List<Cell> res = new ArrayList<Cell>();
2864
2865 boolean toggle = true;
2866 for (long i = 0; i < numRows; i++) {
2867 Put put = new Put(Bytes.toBytes(i));
2868 put.setDurability(Durability.SKIP_WAL);
2869 put.add(family, qual1, Bytes.toBytes(i % 10));
2870 region.put(put);
2871
2872 if (i != 0 && i % compactInterval == 0) {
2873
2874 region.compactStores(true);
2875 }
2876
2877 if (i % 10 == 5L) {
2878 expectedCount++;
2879 }
2880
2881 if (i != 0 && i % flushAndScanInterval == 0) {
2882 res.clear();
2883 InternalScanner scanner = region.getScanner(scan);
2884 if (toggle) {
2885 flushThread.flush();
2886 }
2887 while (scanner.next(res))
2888 ;
2889 if (!toggle) {
2890 flushThread.flush();
2891 }
2892 assertEquals("i=" + i, expectedCount, res.size());
2893 toggle = !toggle;
2894 }
2895 }
2896
2897 flushThread.done();
2898 flushThread.join();
2899 flushThread.checkNoError();
2900 } finally {
2901 HRegion.closeHRegion(this.region);
2902 this.region = null;
2903 }
2904 }
2905
2906 protected class FlushThread extends Thread {
2907 private volatile boolean done;
2908 private Throwable error = null;
2909
2910 public void done() {
2911 done = true;
2912 synchronized (this) {
2913 interrupt();
2914 }
2915 }
2916
2917 public void checkNoError() {
2918 if (error != null) {
2919 assertNull(error);
2920 }
2921 }
2922
2923 @Override
2924 public void run() {
2925 done = false;
2926 while (!done) {
2927 synchronized (this) {
2928 try {
2929 wait();
2930 } catch (InterruptedException ignored) {
2931 if (done) {
2932 break;
2933 }
2934 }
2935 }
2936 try {
2937 region.flushcache();
2938 } catch (IOException e) {
2939 if (!done) {
2940 LOG.error("Error while flusing cache", e);
2941 error = e;
2942 }
2943 break;
2944 }
2945 }
2946
2947 }
2948
2949 public void flush() {
2950 synchronized (this) {
2951 notify();
2952 }
2953
2954 }
2955 }
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966 @Test
2967 public void testWritesWhileScanning() throws IOException, InterruptedException {
2968 int testCount = 100;
2969 int numRows = 1;
2970 int numFamilies = 10;
2971 int numQualifiers = 100;
2972 int flushInterval = 7;
2973 int compactInterval = 5 * flushInterval;
2974 byte[][] families = new byte[numFamilies][];
2975 for (int i = 0; i < numFamilies; i++) {
2976 families[i] = Bytes.toBytes("family" + i);
2977 }
2978 byte[][] qualifiers = new byte[numQualifiers][];
2979 for (int i = 0; i < numQualifiers; i++) {
2980 qualifiers[i] = Bytes.toBytes("qual" + i);
2981 }
2982
2983 String method = "testWritesWhileScanning";
2984 this.region = initHRegion(tableName, method, conf, families);
2985 try {
2986 PutThread putThread = new PutThread(numRows, families, qualifiers);
2987 putThread.start();
2988 putThread.waitForFirstPut();
2989
2990 FlushThread flushThread = new FlushThread();
2991 flushThread.start();
2992
2993 Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
2994
2995 int expectedCount = numFamilies * numQualifiers;
2996 List<Cell> res = new ArrayList<Cell>();
2997
2998 long prevTimestamp = 0L;
2999 for (int i = 0; i < testCount; i++) {
3000
3001 if (i != 0 && i % compactInterval == 0) {
3002 region.compactStores(true);
3003 }
3004
3005 if (i != 0 && i % flushInterval == 0) {
3006 flushThread.flush();
3007 }
3008
3009 boolean previousEmpty = res.isEmpty();
3010 res.clear();
3011 InternalScanner scanner = region.getScanner(scan);
3012 while (scanner.next(res))
3013 ;
3014 if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3015 assertEquals("i=" + i, expectedCount, res.size());
3016 long timestamp = res.get(0).getTimestamp();
3017 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3018 timestamp >= prevTimestamp);
3019 prevTimestamp = timestamp;
3020 }
3021 }
3022
3023 putThread.done();
3024
3025 region.flushcache();
3026
3027 putThread.join();
3028 putThread.checkNoError();
3029
3030 flushThread.done();
3031 flushThread.join();
3032 flushThread.checkNoError();
3033 } finally {
3034 HRegion.closeHRegion(this.region);
3035 this.region = null;
3036 }
3037 }
3038
3039 protected class PutThread extends Thread {
3040 private volatile boolean done;
3041 private volatile int numPutsFinished = 0;
3042
3043 private Throwable error = null;
3044 private int numRows;
3045 private byte[][] families;
3046 private byte[][] qualifiers;
3047
3048 private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3049 this.numRows = numRows;
3050 this.families = families;
3051 this.qualifiers = qualifiers;
3052 }
3053
3054
3055
3056
3057 public void waitForFirstPut() throws InterruptedException {
3058
3059 while (numPutsFinished == 0) {
3060 checkNoError();
3061 Thread.sleep(50);
3062 }
3063 }
3064
3065 public void done() {
3066 done = true;
3067 synchronized (this) {
3068 interrupt();
3069 }
3070 }
3071
3072 public void checkNoError() {
3073 if (error != null) {
3074 assertNull(error);
3075 }
3076 }
3077
3078 @Override
3079 public void run() {
3080 done = false;
3081 while (!done) {
3082 try {
3083 for (int r = 0; r < numRows; r++) {
3084 byte[] row = Bytes.toBytes("row" + r);
3085 Put put = new Put(row);
3086 put.setDurability(Durability.SKIP_WAL);
3087 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3088 for (byte[] family : families) {
3089 for (byte[] qualifier : qualifiers) {
3090 put.add(family, qualifier, (long) numPutsFinished, value);
3091 }
3092 }
3093 region.put(put);
3094 numPutsFinished++;
3095 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3096 System.out.println("put iteration = " + numPutsFinished);
3097 Delete delete = new Delete(row, (long) numPutsFinished - 30);
3098 region.delete(delete);
3099 }
3100 numPutsFinished++;
3101 }
3102 } catch (InterruptedIOException e) {
3103
3104 } catch (IOException e) {
3105 LOG.error("error while putting records", e);
3106 error = e;
3107 break;
3108 }
3109 }
3110
3111 }
3112
3113 }
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124 @Test
3125 public void testWritesWhileGetting() throws Exception {
3126 int testCount = 100;
3127 int numRows = 1;
3128 int numFamilies = 10;
3129 int numQualifiers = 100;
3130 int compactInterval = 100;
3131 byte[][] families = new byte[numFamilies][];
3132 for (int i = 0; i < numFamilies; i++) {
3133 families[i] = Bytes.toBytes("family" + i);
3134 }
3135 byte[][] qualifiers = new byte[numQualifiers][];
3136 for (int i = 0; i < numQualifiers; i++) {
3137 qualifiers[i] = Bytes.toBytes("qual" + i);
3138 }
3139
3140
3141 String method = "testWritesWhileGetting";
3142
3143
3144
3145
3146
3147 conf.setInt("hbase.hstore.compaction.min", 1);
3148 conf.setInt("hbase.hstore.compaction.max", 1000);
3149 this.region = initHRegion(tableName, method, conf, families);
3150 PutThread putThread = null;
3151 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3152 try {
3153 putThread = new PutThread(numRows, families, qualifiers);
3154 putThread.start();
3155 putThread.waitForFirstPut();
3156
3157
3158 ctx.addThread(new RepeatingTestThread(ctx) {
3159 private int flushesSinceCompact = 0;
3160 private final int maxFlushesSinceCompact = 20;
3161
3162 @Override
3163 public void doAnAction() throws Exception {
3164 if (region.flushcache()) {
3165 ++flushesSinceCompact;
3166 }
3167
3168
3169 if (flushesSinceCompact == maxFlushesSinceCompact) {
3170 region.compactStores(false);
3171 flushesSinceCompact = 0;
3172 }
3173 }
3174 });
3175 ctx.startThreads();
3176
3177 Get get = new Get(Bytes.toBytes("row0"));
3178 Result result = null;
3179
3180 int expectedCount = numFamilies * numQualifiers;
3181
3182 long prevTimestamp = 0L;
3183 for (int i = 0; i < testCount; i++) {
3184
3185 boolean previousEmpty = result == null || result.isEmpty();
3186 result = region.get(get);
3187 if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3188 assertEquals("i=" + i, expectedCount, result.size());
3189
3190
3191 long timestamp = 0;
3192 for (Cell kv : result.rawCells()) {
3193 if (CellUtil.matchingFamily(kv, families[0])
3194 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3195 timestamp = kv.getTimestamp();
3196 }
3197 }
3198 assertTrue(timestamp >= prevTimestamp);
3199 prevTimestamp = timestamp;
3200 Cell previousKV = null;
3201
3202 for (Cell kv : result.rawCells()) {
3203 byte[] thisValue = CellUtil.cloneValue(kv);
3204 if (previousKV != null) {
3205 if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3206 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3207 + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3208 + "(memStoreTS:" + kv.getMvccVersion() + ")");
3209 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3210 }
3211 }
3212 previousKV = kv;
3213 }
3214 }
3215 }
3216 } finally {
3217 if (putThread != null)
3218 putThread.done();
3219
3220 region.flushcache();
3221
3222 if (putThread != null) {
3223 putThread.join();
3224 putThread.checkNoError();
3225 }
3226
3227 ctx.stop();
3228 HRegion.closeHRegion(this.region);
3229 this.region = null;
3230 }
3231 }
3232
3233 @Test
3234 public void testHolesInMeta() throws Exception {
3235 byte[] family = Bytes.toBytes("family");
3236 this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, conf,
3237 false, family);
3238 try {
3239 byte[] rowNotServed = Bytes.toBytes("a");
3240 Get g = new Get(rowNotServed);
3241 try {
3242 region.get(g);
3243 fail();
3244 } catch (WrongRegionException x) {
3245
3246 }
3247 byte[] row = Bytes.toBytes("y");
3248 g = new Get(row);
3249 region.get(g);
3250 } finally {
3251 HRegion.closeHRegion(this.region);
3252 this.region = null;
3253 }
3254 }
3255
3256 @Test
3257 public void testIndexesScanWithOneDeletedRow() throws IOException {
3258 byte[] family = Bytes.toBytes("family");
3259
3260
3261 String method = "testIndexesScanWithOneDeletedRow";
3262 this.region = initHRegion(tableName, method, conf, family);
3263 try {
3264 Put put = new Put(Bytes.toBytes(1L));
3265 put.add(family, qual1, 1L, Bytes.toBytes(1L));
3266 region.put(put);
3267
3268 region.flushcache();
3269
3270 Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3271 region.delete(delete);
3272
3273 put = new Put(Bytes.toBytes(2L));
3274 put.add(family, qual1, 2L, Bytes.toBytes(2L));
3275 region.put(put);
3276
3277 Scan idxScan = new Scan();
3278 idxScan.addFamily(family);
3279 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
3280 new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
3281 new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
3282 CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
3283 InternalScanner scanner = region.getScanner(idxScan);
3284 List<Cell> res = new ArrayList<Cell>();
3285
3286 while (scanner.next(res))
3287 ;
3288 assertEquals(1L, res.size());
3289 } finally {
3290 HRegion.closeHRegion(this.region);
3291 this.region = null;
3292 }
3293 }
3294
3295
3296
3297
3298 @Test
3299 public void testBloomFilterSize() throws IOException {
3300 byte[] row1 = Bytes.toBytes("row1");
3301 byte[] fam1 = Bytes.toBytes("fam1");
3302 byte[] qf1 = Bytes.toBytes("col");
3303 byte[] val1 = Bytes.toBytes("value1");
3304
3305 HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
3306 .setBloomFilterType(BloomType.ROWCOL);
3307
3308 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3309 htd.addFamily(hcd);
3310 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3311 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3312 try {
3313 int num_unique_rows = 10;
3314 int duplicate_multiplier = 2;
3315 int num_storefiles = 4;
3316
3317 int version = 0;
3318 for (int f = 0; f < num_storefiles; f++) {
3319 for (int i = 0; i < duplicate_multiplier; i++) {
3320 for (int j = 0; j < num_unique_rows; j++) {
3321 Put put = new Put(Bytes.toBytes("row" + j));
3322 put.setDurability(Durability.SKIP_WAL);
3323 put.add(fam1, qf1, version++, val1);
3324 region.put(put);
3325 }
3326 }
3327 region.flushcache();
3328 }
3329
3330 HStore store = (HStore) region.getStore(fam1);
3331 Collection<StoreFile> storeFiles = store.getStorefiles();
3332 for (StoreFile storefile : storeFiles) {
3333 StoreFile.Reader reader = storefile.getReader();
3334 reader.loadFileInfo();
3335 reader.loadBloomfilter();
3336 assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
3337 assertEquals(num_unique_rows, reader.getFilterEntries());
3338 }
3339
3340 region.compactStores(true);
3341
3342
3343 storeFiles = store.getStorefiles();
3344 for (StoreFile storefile : storeFiles) {
3345 StoreFile.Reader reader = storefile.getReader();
3346 reader.loadFileInfo();
3347 reader.loadBloomfilter();
3348 assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
3349 assertEquals(num_unique_rows, reader.getFilterEntries());
3350 }
3351 } finally {
3352 HRegion.closeHRegion(this.region);
3353 this.region = null;
3354 }
3355 }
3356
3357 @Test
3358 public void testAllColumnsWithBloomFilter() throws IOException {
3359 byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
3360 byte[] FAMILY = Bytes.toBytes("family");
3361
3362
3363 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
3364 .setBloomFilterType(BloomType.ROWCOL);
3365 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
3366 htd.addFamily(hcd);
3367 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3368 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3369 try {
3370
3371 byte row[] = Bytes.toBytes("row:" + 0);
3372 byte column[] = Bytes.toBytes("column:" + 0);
3373 Put put = new Put(row);
3374 put.setDurability(Durability.SKIP_WAL);
3375 for (long idx = 1; idx <= 4; idx++) {
3376 put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
3377 }
3378 region.put(put);
3379
3380
3381 region.flushcache();
3382
3383
3384 Get get = new Get(row);
3385 get.setMaxVersions();
3386 Cell[] kvs = region.get(get).rawCells();
3387
3388
3389 assertEquals(4, kvs.length);
3390 checkOneCell(kvs[0], FAMILY, 0, 0, 4);
3391 checkOneCell(kvs[1], FAMILY, 0, 0, 3);
3392 checkOneCell(kvs[2], FAMILY, 0, 0, 2);
3393 checkOneCell(kvs[3], FAMILY, 0, 0, 1);
3394 } finally {
3395 HRegion.closeHRegion(this.region);
3396 this.region = null;
3397 }
3398 }
3399
3400
3401
3402
3403
3404
3405 @Test
3406 public void testDeleteRowWithBloomFilter() throws IOException {
3407 byte[] familyName = Bytes.toBytes("familyName");
3408
3409
3410 HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
3411 .setBloomFilterType(BloomType.ROWCOL);
3412
3413 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3414 htd.addFamily(hcd);
3415 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3416 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3417 try {
3418
3419 byte row[] = Bytes.toBytes("row1");
3420 byte col[] = Bytes.toBytes("col1");
3421
3422 Put put = new Put(row);
3423 put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
3424 region.put(put);
3425 region.flushcache();
3426
3427 Delete del = new Delete(row);
3428 region.delete(del);
3429 region.flushcache();
3430
3431
3432 Get get = new Get(row);
3433 get.addColumn(familyName, col);
3434
3435 Cell[] keyValues = region.get(get).rawCells();
3436 assertTrue(keyValues.length == 0);
3437 } finally {
3438 HRegion.closeHRegion(this.region);
3439 this.region = null;
3440 }
3441 }
3442
3443 @Test
3444 public void testgetHDFSBlocksDistribution() throws Exception {
3445 HBaseTestingUtility htu = new HBaseTestingUtility();
3446 final int DEFAULT_BLOCK_SIZE = 1024;
3447 htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
3448 htu.getConfiguration().setInt("dfs.replication", 2);
3449
3450
3451 MiniHBaseCluster cluster = null;
3452 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
3453 int regionServersCount = 3;
3454
3455 try {
3456 cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
3457 byte[][] families = { fam1, fam2 };
3458 HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);
3459
3460
3461 byte row[] = Bytes.toBytes("row1");
3462 byte col[] = Bytes.toBytes("col1");
3463
3464 Put put = new Put(row);
3465 put.add(fam1, col, 1, Bytes.toBytes("test1"));
3466 put.add(fam2, col, 1, Bytes.toBytes("test2"));
3467 ht.put(put);
3468
3469 HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
3470 .get(0);
3471 firstRegion.flushcache();
3472 HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
3473
3474
3475
3476
3477
3478 long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
3479
3480 String topHost = blocksDistribution1.getTopHosts().get(0);
3481 long topHostWeight = blocksDistribution1.getWeight(topHost);
3482 assertTrue(uniqueBlocksWeight1 == topHostWeight);
3483
3484
3485
3486 HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
3487 htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
3488 long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
3489
3490 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
3491
3492 ht.close();
3493 } finally {
3494 if (cluster != null) {
3495 htu.shutdownMiniCluster();
3496 }
3497 }
3498 }
3499
3500
3501
3502
3503
3504
3505
3506 @Test
3507 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
3508 TableName tableName = TableName.valueOf(name.getMethodName());
3509 HRegionInfo info = null;
3510 try {
3511 FileSystem fs = Mockito.mock(FileSystem.class);
3512 Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
3513 HTableDescriptor htd = new HTableDescriptor(tableName);
3514 htd.addFamily(new HColumnDescriptor("cf"));
3515 info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
3516 HConstants.EMPTY_BYTE_ARRAY, false);
3517 Path path = new Path(DIR + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
3518 region = HRegion.newHRegion(path, null, fs, conf, info, htd, null);
3519
3520 region.initialize();
3521 fail("Region initialization should fail due to IOException");
3522 } catch (IOException io) {
3523 List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
3524 for (MonitoredTask monitoredTask : tasks) {
3525 if (!(monitoredTask instanceof MonitoredRPCHandler)
3526 && monitoredTask.getDescription().contains(region.toString())) {
3527 assertTrue("Region state should be ABORTED.",
3528 monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
3529 break;
3530 }
3531 }
3532 } finally {
3533 HRegion.closeHRegion(region);
3534 }
3535 }
3536
3537
3538
3539
3540
3541 @Test
3542 public void testRegionInfoFileCreation() throws IOException {
3543 Path rootDir = new Path(DIR + "testRegionInfoFileCreation");
3544
3545 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
3546 htd.addFamily(new HColumnDescriptor("cf"));
3547
3548 HRegionInfo hri = new HRegionInfo(htd.getTableName());
3549
3550
3551 HRegion region = HRegion.createHRegion(hri, rootDir, conf, htd, null, false, true);
3552
3553 Path regionDir = region.getRegionFileSystem().getRegionDir();
3554 FileSystem fs = region.getRegionFileSystem().getFileSystem();
3555 HRegion.closeHRegion(region);
3556
3557 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
3558
3559
3560 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3561 fs.exists(regionInfoFile));
3562
3563
3564 region = HRegion.openHRegion(rootDir, hri, htd, null, conf);
3565 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3566 HRegion.closeHRegion(region);
3567
3568
3569 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3570 fs.exists(regionInfoFile));
3571
3572
3573 fs.delete(regionInfoFile);
3574 assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
3575 fs.exists(regionInfoFile));
3576
3577 region = HRegion.openHRegion(rootDir, hri, htd, null, conf);
3578
3579 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
3580 HRegion.closeHRegion(region);
3581
3582
3583 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
3584 fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
3585 }
3586
3587
3588
3589
3590 private static class Incrementer implements Runnable {
3591 private HRegion region;
3592 private final static byte[] incRow = Bytes.toBytes("incRow");
3593 private final static byte[] family = Bytes.toBytes("family");
3594 private final static byte[] qualifier = Bytes.toBytes("qualifier");
3595 private final static long ONE = 1l;
3596 private int incCounter;
3597
3598 public Incrementer(HRegion region, int incCounter) {
3599 this.region = region;
3600 this.incCounter = incCounter;
3601 }
3602
3603 @Override
3604 public void run() {
3605 int count = 0;
3606 while (count < incCounter) {
3607 Increment inc = new Increment(incRow);
3608 inc.addColumn(family, qualifier, ONE);
3609 count++;
3610 try {
3611 region.increment(inc);
3612 } catch (IOException e) {
3613 e.printStackTrace();
3614 break;
3615 }
3616 }
3617 }
3618 }
3619
3620
3621
3622
3623
3624 @Test
3625 public void testParallelIncrementWithMemStoreFlush() throws Exception {
3626 byte[] family = Incrementer.family;
3627 this.region = initHRegion(tableName, method, conf, family);
3628 final HRegion region = this.region;
3629 final AtomicBoolean incrementDone = new AtomicBoolean(false);
3630 Runnable flusher = new Runnable() {
3631 @Override
3632 public void run() {
3633 while (!incrementDone.get()) {
3634 try {
3635 region.flushcache();
3636 } catch (Exception e) {
3637 e.printStackTrace();
3638 }
3639 }
3640 }
3641 };
3642
3643
3644 int threadNum = 20;
3645 int incCounter = 100;
3646 long expected = threadNum * incCounter;
3647 Thread[] incrementers = new Thread[threadNum];
3648 Thread flushThread = new Thread(flusher);
3649 for (int i = 0; i < threadNum; i++) {
3650 incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
3651 incrementers[i].start();
3652 }
3653 flushThread.start();
3654 for (int i = 0; i < threadNum; i++) {
3655 incrementers[i].join();
3656 }
3657
3658 incrementDone.set(true);
3659 flushThread.join();
3660
3661 Get get = new Get(Incrementer.incRow);
3662 get.addColumn(Incrementer.family, Incrementer.qualifier);
3663 get.setMaxVersions(1);
3664 Result res = this.region.get(get);
3665 List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
3666
3667
3668 assertEquals(kvs.size(), 1);
3669 Cell kv = kvs.get(0);
3670 assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
3671 this.region = null;
3672 }
3673
3674
3675
3676
3677 private static class Appender implements Runnable {
3678 private HRegion region;
3679 private final static byte[] appendRow = Bytes.toBytes("appendRow");
3680 private final static byte[] family = Bytes.toBytes("family");
3681 private final static byte[] qualifier = Bytes.toBytes("qualifier");
3682 private final static byte[] CHAR = Bytes.toBytes("a");
3683 private int appendCounter;
3684
3685 public Appender(HRegion region, int appendCounter) {
3686 this.region = region;
3687 this.appendCounter = appendCounter;
3688 }
3689
3690 @Override
3691 public void run() {
3692 int count = 0;
3693 while (count < appendCounter) {
3694 Append app = new Append(appendRow);
3695 app.add(family, qualifier, CHAR);
3696 count++;
3697 try {
3698 region.append(app);
3699 } catch (IOException e) {
3700 e.printStackTrace();
3701 break;
3702 }
3703 }
3704 }
3705 }
3706
3707
3708
3709
3710
3711 @Test
3712 public void testParallelAppendWithMemStoreFlush() throws Exception {
3713 byte[] family = Appender.family;
3714 this.region = initHRegion(tableName, method, conf, family);
3715 final HRegion region = this.region;
3716 final AtomicBoolean appendDone = new AtomicBoolean(false);
3717 Runnable flusher = new Runnable() {
3718 @Override
3719 public void run() {
3720 while (!appendDone.get()) {
3721 try {
3722 region.flushcache();
3723 } catch (Exception e) {
3724 e.printStackTrace();
3725 }
3726 }
3727 }
3728 };
3729
3730
3731
3732 int threadNum = 20;
3733 int appendCounter = 100;
3734 byte[] expected = new byte[threadNum * appendCounter];
3735 for (int i = 0; i < threadNum * appendCounter; i++) {
3736 System.arraycopy(Appender.CHAR, 0, expected, i, 1);
3737 }
3738 Thread[] appenders = new Thread[threadNum];
3739 Thread flushThread = new Thread(flusher);
3740 for (int i = 0; i < threadNum; i++) {
3741 appenders[i] = new Thread(new Appender(this.region, appendCounter));
3742 appenders[i].start();
3743 }
3744 flushThread.start();
3745 for (int i = 0; i < threadNum; i++) {
3746 appenders[i].join();
3747 }
3748
3749 appendDone.set(true);
3750 flushThread.join();
3751
3752 Get get = new Get(Appender.appendRow);
3753 get.addColumn(Appender.family, Appender.qualifier);
3754 get.setMaxVersions(1);
3755 Result res = this.region.get(get);
3756 List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
3757
3758
3759 assertEquals(kvs.size(), 1);
3760 Cell kv = kvs.get(0);
3761 byte[] appendResult = new byte[kv.getValueLength()];
3762 System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
3763 assertArrayEquals(expected, appendResult);
3764 this.region = null;
3765 }
3766
3767
3768
3769
3770
3771 @Test
3772 public void testPutWithMemStoreFlush() throws Exception {
3773 byte[] family = Bytes.toBytes("family");
3774 ;
3775 byte[] qualifier = Bytes.toBytes("qualifier");
3776 byte[] row = Bytes.toBytes("putRow");
3777 byte[] value = null;
3778 this.region = initHRegion(tableName, method, conf, family);
3779 Put put = null;
3780 Get get = null;
3781 List<Cell> kvs = null;
3782 Result res = null;
3783
3784 put = new Put(row);
3785 value = Bytes.toBytes("value0");
3786 put.add(family, qualifier, 1234567l, value);
3787 region.put(put);
3788 get = new Get(row);
3789 get.addColumn(family, qualifier);
3790 get.setMaxVersions();
3791 res = this.region.get(get);
3792 kvs = res.getColumnCells(family, qualifier);
3793 assertEquals(1, kvs.size());
3794 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
3795
3796 region.flushcache();
3797 get = new Get(row);
3798 get.addColumn(family, qualifier);
3799 get.setMaxVersions();
3800 res = this.region.get(get);
3801 kvs = res.getColumnCells(family, qualifier);
3802 assertEquals(1, kvs.size());
3803 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
3804
3805 put = new Put(row);
3806 value = Bytes.toBytes("value1");
3807 put.add(family, qualifier, 1234567l, value);
3808 region.put(put);
3809 get = new Get(row);
3810 get.addColumn(family, qualifier);
3811 get.setMaxVersions();
3812 res = this.region.get(get);
3813 kvs = res.getColumnCells(family, qualifier);
3814 assertEquals(1, kvs.size());
3815 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
3816
3817 region.flushcache();
3818 get = new Get(row);
3819 get.addColumn(family, qualifier);
3820 get.setMaxVersions();
3821 res = this.region.get(get);
3822 kvs = res.getColumnCells(family, qualifier);
3823 assertEquals(1, kvs.size());
3824 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
3825 }
3826
3827 @Test
3828 public void testDurability() throws Exception {
3829 String method = "testDurability";
3830
3831
3832
3833
3834
3835 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
3836 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
3837 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
3838
3839 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
3840 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
3841 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
3842
3843 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
3844 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
3845
3846 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
3847 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
3848
3849 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
3850 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
3851 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
3852
3853
3854 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
3855 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
3856 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
3857 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
3858 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
3859 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
3860
3861 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
3862 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
3863 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
3864 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
3865 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
3866 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
3867
3868
3869 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
3870 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
3871 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
3872 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
3873 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
3874 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
3875
3876 }
3877
3878 private void durabilityTest(String method, Durability tableDurability,
3879 Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
3880 final boolean expectSyncFromLogSyncer) throws Exception {
3881 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
3882 TableName tableName = TableName.valueOf(method);
3883 byte[] family = Bytes.toBytes("family");
3884 Path logDir = new Path(new Path(DIR + method), "log");
3885 HLog hlog = HLogFactory.createHLog(fs, logDir, UUID.randomUUID().toString(), conf);
3886 final HLog log = spy(hlog);
3887 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
3888 HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, log,
3889 new byte[][] { family });
3890
3891 Put put = new Put(Bytes.toBytes("r1"));
3892 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
3893 put.setDurability(mutationDurability);
3894 region.put(put);
3895
3896
3897 verify(log, expectAppend ? times(1) : never())
3898 .appendNoSync((HRegionInfo)any(), eq(tableName), (WALEdit)any(), (List<UUID>)any(),
3899 anyLong(), (HTableDescriptor)any(), (AtomicLong)any(), anyBoolean(), anyLong(), anyLong());
3900
3901
3902 if (expectSync || expectSyncFromLogSyncer) {
3903 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
3904 @Override
3905 public boolean evaluate() throws Exception {
3906 try {
3907 if (expectSync) {
3908 verify(log, times(1)).sync(anyLong());
3909 } else if (expectSyncFromLogSyncer) {
3910 verify(log, times(1)).sync();
3911 }
3912 } catch (Throwable ignore) {
3913 }
3914 return true;
3915 }
3916 });
3917 } else {
3918 verify(log, never()).sync(anyLong());
3919 verify(log, never()).sync();
3920 }
3921
3922 hlog.close();
3923 region.close();
3924 }
3925
3926 private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
3927 for (int i = startRow; i < startRow + numRows; i++) {
3928 Put put = new Put(Bytes.toBytes("" + i));
3929 put.setDurability(Durability.SKIP_WAL);
3930 for (byte[] family : families) {
3931 put.add(family, qf, null);
3932 }
3933 region.put(put);
3934 }
3935 }
3936
3937 private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
3938 throws IOException {
3939 for (int i = startRow; i < startRow + numRows; i++) {
3940 byte[] row = Bytes.toBytes("" + i);
3941 Get get = new Get(row);
3942 for (byte[] family : families) {
3943 get.addColumn(family, qf);
3944 }
3945 Result result = newReg.get(get);
3946 Cell[] raw = result.rawCells();
3947 assertEquals(families.length, result.size());
3948 for (int j = 0; j < families.length; j++) {
3949 assertTrue(CellUtil.matchingRow(raw[j], row));
3950 assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
3951 assertTrue(CellUtil.matchingQualifier(raw[j], qf));
3952 }
3953 }
3954 }
3955
3956 private void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
3957
3958 Get get = new Get(k).addFamily(family).setMaxVersions();
3959 Cell[] results = r.get(get).rawCells();
3960 for (int j = 0; j < results.length; j++) {
3961 byte[] tmp = CellUtil.cloneValue(results[j]);
3962
3963 assertTrue(Bytes.equals(k, tmp));
3964 }
3965 }
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978 private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
3979 throws IOException {
3980 byte[][] families = { fs };
3981 Scan scan = new Scan();
3982 for (int i = 0; i < families.length; i++)
3983 scan.addFamily(families[i]);
3984 InternalScanner s = r.getScanner(scan);
3985 try {
3986 List<Cell> curVals = new ArrayList<Cell>();
3987 boolean first = true;
3988 OUTER_LOOP: while (s.next(curVals)) {
3989 for (Cell kv : curVals) {
3990 byte[] val = CellUtil.cloneValue(kv);
3991 byte[] curval = val;
3992 if (first) {
3993 first = false;
3994 assertTrue(Bytes.compareTo(curval, firstValue) == 0);
3995 } else {
3996
3997 break OUTER_LOOP;
3998 }
3999 }
4000 }
4001 } finally {
4002 s.close();
4003 }
4004 }
4005
4006 private Configuration initSplit() {
4007
4008 conf.setInt("hbase.hstore.compactionThreshold", 2);
4009
4010
4011 conf.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4012
4013 conf.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4014
4015
4016 conf.setLong("hbase.client.pause", 15 * 1000);
4017
4018
4019
4020 conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4021 return conf;
4022 }
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033 public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
4034 byte[]... families) throws IOException {
4035 return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
4036 }
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4048 byte[]... families) throws IOException {
4049 return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4050 }
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4063 boolean isReadOnly, byte[]... families) throws IOException {
4064 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4065 }
4066
4067 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4068 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4069 throws IOException {
4070 return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
4071 Durability.SYNC_WAL, null, families);
4072 }
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4087 String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4088 HLog hlog, byte[]... families) throws IOException {
4089 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly, durability, hlog, families);
4090 }
4091
4092
4093
4094
4095
4096 private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
4097 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4098 assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
4099 Bytes.toString(CellUtil.cloneRow(kv)));
4100 assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
4101 Bytes.toString(CellUtil.cloneFamily(kv)));
4102 assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
4103 Bytes.toString(CellUtil.cloneQualifier(kv)));
4104 assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
4105 assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
4106 Bytes.toString(CellUtil.cloneValue(kv)));
4107 }
4108
4109 public void testReverseScanner_FromMemStore_SingleCF_Normal()
4110 throws IOException {
4111 byte[] rowC = Bytes.toBytes("rowC");
4112 byte[] rowA = Bytes.toBytes("rowA");
4113 byte[] rowB = Bytes.toBytes("rowB");
4114 byte[] cf = Bytes.toBytes("CF");
4115 byte[][] families = { cf };
4116 byte[] col = Bytes.toBytes("C");
4117 long ts = 1;
4118 String method = this.getName();
4119 this.region = initHRegion(tableName, method, families);
4120 try {
4121 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4122 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4123 null);
4124 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4125 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4126 Put put = null;
4127 put = new Put(rowC);
4128 put.add(kv1);
4129 put.add(kv11);
4130 region.put(put);
4131 put = new Put(rowA);
4132 put.add(kv2);
4133 region.put(put);
4134 put = new Put(rowB);
4135 put.add(kv3);
4136 region.put(put);
4137
4138 Scan scan = new Scan(rowC);
4139 scan.setMaxVersions(5);
4140 scan.setReversed(true);
4141 InternalScanner scanner = region.getScanner(scan);
4142 List<Cell> currRow = new ArrayList<Cell>();
4143 boolean hasNext = scanner.next(currRow);
4144 assertEquals(2, currRow.size());
4145 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4146 assertTrue(hasNext);
4147 currRow.clear();
4148 hasNext = scanner.next(currRow);
4149 assertEquals(1, currRow.size());
4150 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4151 assertTrue(hasNext);
4152 currRow.clear();
4153 hasNext = scanner.next(currRow);
4154 assertEquals(1, currRow.size());
4155 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4156 assertFalse(hasNext);
4157 scanner.close();
4158 } finally {
4159 HRegion.closeHRegion(this.region);
4160 this.region = null;
4161 }
4162 }
4163
4164 public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
4165 throws IOException {
4166 byte[] rowC = Bytes.toBytes("rowC");
4167 byte[] rowA = Bytes.toBytes("rowA");
4168 byte[] rowB = Bytes.toBytes("rowB");
4169 byte[] rowD = Bytes.toBytes("rowD");
4170 byte[] cf = Bytes.toBytes("CF");
4171 byte[][] families = { cf };
4172 byte[] col = Bytes.toBytes("C");
4173 long ts = 1;
4174 String method = this.getName();
4175 this.region = initHRegion(tableName, method, families);
4176 try {
4177 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4178 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4179 null);
4180 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4181 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4182 Put put = null;
4183 put = new Put(rowC);
4184 put.add(kv1);
4185 put.add(kv11);
4186 region.put(put);
4187 put = new Put(rowA);
4188 put.add(kv2);
4189 region.put(put);
4190 put = new Put(rowB);
4191 put.add(kv3);
4192 region.put(put);
4193
4194 Scan scan = new Scan(rowD);
4195 List<Cell> currRow = new ArrayList<Cell>();
4196 scan.setReversed(true);
4197 scan.setMaxVersions(5);
4198 InternalScanner scanner = region.getScanner(scan);
4199 boolean hasNext = scanner.next(currRow);
4200 assertEquals(2, currRow.size());
4201 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4202 assertTrue(hasNext);
4203 currRow.clear();
4204 hasNext = scanner.next(currRow);
4205 assertEquals(1, currRow.size());
4206 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4207 assertTrue(hasNext);
4208 currRow.clear();
4209 hasNext = scanner.next(currRow);
4210 assertEquals(1, currRow.size());
4211 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4212 assertFalse(hasNext);
4213 scanner.close();
4214 } finally {
4215 HRegion.closeHRegion(this.region);
4216 this.region = null;
4217 }
4218 }
4219
4220 public void testReverseScanner_FromMemStore_SingleCF_FullScan()
4221 throws IOException {
4222 byte[] rowC = Bytes.toBytes("rowC");
4223 byte[] rowA = Bytes.toBytes("rowA");
4224 byte[] rowB = Bytes.toBytes("rowB");
4225 byte[] cf = Bytes.toBytes("CF");
4226 byte[][] families = { cf };
4227 byte[] col = Bytes.toBytes("C");
4228 long ts = 1;
4229 String method = this.getName();
4230 this.region = initHRegion(tableName, method, families);
4231 try {
4232 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
4233 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
4234 null);
4235 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
4236 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
4237 Put put = null;
4238 put = new Put(rowC);
4239 put.add(kv1);
4240 put.add(kv11);
4241 region.put(put);
4242 put = new Put(rowA);
4243 put.add(kv2);
4244 region.put(put);
4245 put = new Put(rowB);
4246 put.add(kv3);
4247 region.put(put);
4248 Scan scan = new Scan();
4249 List<Cell> currRow = new ArrayList<Cell>();
4250 scan.setReversed(true);
4251 InternalScanner scanner = region.getScanner(scan);
4252 boolean hasNext = scanner.next(currRow);
4253 assertEquals(1, currRow.size());
4254 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4255 assertTrue(hasNext);
4256 currRow.clear();
4257 hasNext = scanner.next(currRow);
4258 assertEquals(1, currRow.size());
4259 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4260 assertTrue(hasNext);
4261 currRow.clear();
4262 hasNext = scanner.next(currRow);
4263 assertEquals(1, currRow.size());
4264 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
4265 assertFalse(hasNext);
4266 scanner.close();
4267 } finally {
4268 HRegion.closeHRegion(this.region);
4269 this.region = null;
4270 }
4271 }
4272
4273 public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
4274
4275 byte[] rowA = Bytes.toBytes("rowA");
4276 byte[] rowB = Bytes.toBytes("rowB");
4277 byte[] rowC = Bytes.toBytes("rowC");
4278 byte[] rowD = Bytes.toBytes("rowD");
4279 byte[] rowE = Bytes.toBytes("rowE");
4280 byte[] cf = Bytes.toBytes("CF");
4281 byte[][] families = { cf };
4282 byte[] col1 = Bytes.toBytes("col1");
4283 byte[] col2 = Bytes.toBytes("col2");
4284 long ts = 1;
4285 String method = this.getName();
4286 this.region = initHRegion(tableName, method, families);
4287 try {
4288 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4289 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4290 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4291 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4292 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4293 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4294 Put put = null;
4295 put = new Put(rowA);
4296 put.add(kv1);
4297 region.put(put);
4298 put = new Put(rowB);
4299 put.add(kv2);
4300 region.put(put);
4301 put = new Put(rowC);
4302 put.add(kv3);
4303 region.put(put);
4304 put = new Put(rowD);
4305 put.add(kv4_1);
4306 region.put(put);
4307 put = new Put(rowD);
4308 put.add(kv4_2);
4309 region.put(put);
4310 put = new Put(rowE);
4311 put.add(kv5);
4312 region.put(put);
4313 region.flushcache();
4314 Scan scan = new Scan(rowD, rowA);
4315 scan.addColumn(families[0], col1);
4316 scan.setReversed(true);
4317 List<Cell> currRow = new ArrayList<Cell>();
4318 InternalScanner scanner = region.getScanner(scan);
4319 boolean hasNext = scanner.next(currRow);
4320 assertEquals(1, currRow.size());
4321 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4322 assertTrue(hasNext);
4323 currRow.clear();
4324 hasNext = scanner.next(currRow);
4325 assertEquals(1, currRow.size());
4326 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4327 assertTrue(hasNext);
4328 currRow.clear();
4329 hasNext = scanner.next(currRow);
4330 assertEquals(1, currRow.size());
4331 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4332 assertFalse(hasNext);
4333 scanner.close();
4334
4335 scan = new Scan(rowD, rowA);
4336 scan.addColumn(families[0], col2);
4337 scan.setReversed(true);
4338 currRow.clear();
4339 scanner = region.getScanner(scan);
4340 hasNext = scanner.next(currRow);
4341 assertEquals(1, currRow.size());
4342 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4343 scanner.close();
4344 } finally {
4345 HRegion.closeHRegion(this.region);
4346 this.region = null;
4347 }
4348 }
4349
4350 public void testReverseScanner_smaller_blocksize() throws IOException {
4351
4352 byte[] rowA = Bytes.toBytes("rowA");
4353 byte[] rowB = Bytes.toBytes("rowB");
4354 byte[] rowC = Bytes.toBytes("rowC");
4355 byte[] rowD = Bytes.toBytes("rowD");
4356 byte[] rowE = Bytes.toBytes("rowE");
4357 byte[] cf = Bytes.toBytes("CF");
4358 byte[][] families = { cf };
4359 byte[] col1 = Bytes.toBytes("col1");
4360 byte[] col2 = Bytes.toBytes("col2");
4361 long ts = 1;
4362 String method = this.getName();
4363 HBaseConfiguration config = new HBaseConfiguration();
4364 config.setInt("test.block.size", 1);
4365 this.region = initHRegion(tableName, method, config, families);
4366 try {
4367 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
4368 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
4369 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
4370 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
4371 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
4372 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
4373 Put put = null;
4374 put = new Put(rowA);
4375 put.add(kv1);
4376 region.put(put);
4377 put = new Put(rowB);
4378 put.add(kv2);
4379 region.put(put);
4380 put = new Put(rowC);
4381 put.add(kv3);
4382 region.put(put);
4383 put = new Put(rowD);
4384 put.add(kv4_1);
4385 region.put(put);
4386 put = new Put(rowD);
4387 put.add(kv4_2);
4388 region.put(put);
4389 put = new Put(rowE);
4390 put.add(kv5);
4391 region.put(put);
4392 region.flushcache();
4393 Scan scan = new Scan(rowD, rowA);
4394 scan.addColumn(families[0], col1);
4395 scan.setReversed(true);
4396 List<Cell> currRow = new ArrayList<Cell>();
4397 InternalScanner scanner = region.getScanner(scan);
4398 boolean hasNext = scanner.next(currRow);
4399 assertEquals(1, currRow.size());
4400 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4401 assertTrue(hasNext);
4402 currRow.clear();
4403 hasNext = scanner.next(currRow);
4404 assertEquals(1, currRow.size());
4405 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
4406 assertTrue(hasNext);
4407 currRow.clear();
4408 hasNext = scanner.next(currRow);
4409 assertEquals(1, currRow.size());
4410 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
4411 assertFalse(hasNext);
4412 scanner.close();
4413
4414 scan = new Scan(rowD, rowA);
4415 scan.addColumn(families[0], col2);
4416 scan.setReversed(true);
4417 currRow.clear();
4418 scanner = region.getScanner(scan);
4419 hasNext = scanner.next(currRow);
4420 assertEquals(1, currRow.size());
4421 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
4422 scanner.close();
4423 } finally {
4424 HRegion.closeHRegion(this.region);
4425 this.region = null;
4426 }
4427 }
4428
4429 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
4430 throws IOException {
4431 byte[] row0 = Bytes.toBytes("row0");
4432 byte[] row1 = Bytes.toBytes("row1");
4433 byte[] row2 = Bytes.toBytes("row2");
4434 byte[] row3 = Bytes.toBytes("row3");
4435 byte[] row4 = Bytes.toBytes("row4");
4436 byte[] row5 = Bytes.toBytes("row5");
4437 byte[] cf1 = Bytes.toBytes("CF1");
4438 byte[] cf2 = Bytes.toBytes("CF2");
4439 byte[] cf3 = Bytes.toBytes("CF3");
4440 byte[][] families = { cf1, cf2, cf3 };
4441 byte[] col = Bytes.toBytes("C");
4442 long ts = 1;
4443 String method = this.getName();
4444 HBaseConfiguration conf = new HBaseConfiguration();
4445
4446 conf.setInt("hbase.hstore.compactionThreshold", 10000);
4447 this.region = initHRegion(tableName, method, conf, families);
4448 try {
4449
4450 KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
4451 null);
4452 KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
4453 null);
4454 KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
4455 KeyValue.Type.Put, null);
4456 KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
4457 null);
4458 KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
4459 null);
4460 KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
4461 null);
4462 KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
4463 KeyValue.Type.Put, null);
4464 KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
4465 null);
4466 KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
4467 KeyValue.Type.Put, null);
4468 KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
4469 null);
4470 KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
4471 null);
4472 KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
4473 KeyValue.Type.Put, null);
4474 KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
4475 null);
4476 KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
4477 KeyValue.Type.Put, null);
4478 KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
4479 null);
4480 KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
4481 null);
4482
4483 Put put = null;
4484 put = new Put(row1);
4485 put.add(kv1_2_1);
4486 region.put(put);
4487 put = new Put(row2);
4488 put.add(kv2_4_1);
4489 region.put(put);
4490 put = new Put(row4);
4491 put.add(kv4_5_4);
4492 put.add(kv4_5_5);
4493 region.put(put);
4494 region.flushcache();
4495
4496 put = new Put(row4);
4497 put.add(kv4_5_1);
4498 put.add(kv4_5_3);
4499 region.put(put);
4500 put = new Put(row1);
4501 put.add(kv1_2_2);
4502 region.put(put);
4503 put = new Put(row2);
4504 put.add(kv2_4_4);
4505 region.put(put);
4506 region.flushcache();
4507
4508 put = new Put(row4);
4509 put.add(kv4_5_2);
4510 region.put(put);
4511 put = new Put(row2);
4512 put.add(kv2_4_2);
4513 put.add(kv2_4_3);
4514 region.put(put);
4515 put = new Put(row3);
4516 put.add(kv3_2_2);
4517 region.put(put);
4518 region.flushcache();
4519
4520
4521 put = new Put(row0);
4522 put.add(kv0_1_1);
4523 region.put(put);
4524 put = new Put(row3);
4525 put.add(kv3_2_1);
4526 region.put(put);
4527 put = new Put(row5);
4528 put.add(kv5_2_1);
4529 put.add(kv5_2_2);
4530 region.put(put);
4531
4532 Scan scan = new Scan(row4);
4533 scan.setMaxVersions(5);
4534 scan.setBatch(3);
4535 scan.setReversed(true);
4536 InternalScanner scanner = region.getScanner(scan);
4537 List<Cell> currRow = new ArrayList<Cell>();
4538 boolean hasNext = false;
4539
4540
4541
4542 hasNext = scanner.next(currRow);
4543 assertEquals(3, currRow.size());
4544 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4545 assertTrue(hasNext);
4546 currRow.clear();
4547 hasNext = scanner.next(currRow);
4548 assertEquals(2, currRow.size());
4549 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4550 assertTrue(hasNext);
4551
4552 currRow.clear();
4553 hasNext = scanner.next(currRow);
4554 assertEquals(2, currRow.size());
4555 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
4556 assertTrue(hasNext);
4557
4558
4559 currRow.clear();
4560 hasNext = scanner.next(currRow);
4561 assertEquals(3, currRow.size());
4562 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4563 assertTrue(hasNext);
4564 currRow.clear();
4565 hasNext = scanner.next(currRow);
4566 assertEquals(1, currRow.size());
4567 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4568 assertTrue(hasNext);
4569
4570 currRow.clear();
4571 hasNext = scanner.next(currRow);
4572 assertEquals(2, currRow.size());
4573 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
4574 assertTrue(hasNext);
4575
4576 currRow.clear();
4577 hasNext = scanner.next(currRow);
4578 assertEquals(1, currRow.size());
4579 assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
4580 assertFalse(hasNext);
4581
4582 scanner.close();
4583 } finally {
4584 HRegion.closeHRegion(this.region);
4585 this.region = null;
4586 }
4587 }
4588
4589 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
4590 throws IOException {
4591 byte[] row1 = Bytes.toBytes("row1");
4592 byte[] row2 = Bytes.toBytes("row2");
4593 byte[] row3 = Bytes.toBytes("row3");
4594 byte[] row4 = Bytes.toBytes("row4");
4595 byte[] cf1 = Bytes.toBytes("CF1");
4596 byte[] cf2 = Bytes.toBytes("CF2");
4597 byte[] cf3 = Bytes.toBytes("CF3");
4598 byte[] cf4 = Bytes.toBytes("CF4");
4599 byte[][] families = { cf1, cf2, cf3, cf4 };
4600 byte[] col = Bytes.toBytes("C");
4601 long ts = 1;
4602 String method = this.getName();
4603 HBaseConfiguration conf = new HBaseConfiguration();
4604
4605 conf.setInt("hbase.hstore.compactionThreshold", 10000);
4606 this.region = initHRegion(tableName, method, conf, families);
4607 try {
4608 KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
4609 KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
4610 KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
4611 KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
4612
4613 Put put = new Put(row1);
4614 put.add(kv1);
4615 region.put(put);
4616 region.flushcache();
4617
4618 put = new Put(row2);
4619 put.add(kv2);
4620 region.put(put);
4621 region.flushcache();
4622
4623 put = new Put(row3);
4624 put.add(kv3);
4625 region.put(put);
4626 region.flushcache();
4627
4628 put = new Put(row4);
4629 put.add(kv4);
4630 region.put(put);
4631
4632 Scan scan = new Scan(row4);
4633 scan.setReversed(true);
4634 scan.setBatch(10);
4635 InternalScanner scanner = region.getScanner(scan);
4636 List<Cell> currRow = new ArrayList<Cell>();
4637 boolean hasNext = scanner.next(currRow);
4638 assertEquals(1, currRow.size());
4639 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
4640 assertTrue(hasNext);
4641 currRow.clear();
4642 hasNext = scanner.next(currRow);
4643 assertEquals(1, currRow.size());
4644 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
4645 assertTrue(hasNext);
4646 currRow.clear();
4647 hasNext = scanner.next(currRow);
4648 assertEquals(1, currRow.size());
4649 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
4650 assertTrue(hasNext);
4651 currRow.clear();
4652 hasNext = scanner.next(currRow);
4653 assertEquals(1, currRow.size());
4654 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
4655 assertFalse(hasNext);
4656 } finally {
4657 HRegion.closeHRegion(this.region);
4658 this.region = null;
4659 }
4660 }
4661
4662 private static HRegion initHRegion(byte[] tableName, String callingMethod,
4663 byte[]... families) throws IOException {
4664 return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
4665 families);
4666 }
4667 }