1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.Arrays;
25 import java.util.HashMap;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.TreeMap;
29 import java.util.concurrent.atomic.AtomicBoolean;
30 import java.util.concurrent.atomic.AtomicInteger;
31 import java.util.concurrent.atomic.AtomicReference;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hbase.DoNotRetryIOException;
38 import org.apache.hadoop.hbase.HBaseConfiguration;
39 import org.apache.hadoop.hbase.HBaseTestCase;
40 import org.apache.hadoop.hbase.HBaseTestingUtility;
41 import org.apache.hadoop.hbase.HColumnDescriptor;
42 import org.apache.hadoop.hbase.HConstants;
43 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
44 import org.apache.hadoop.hbase.HRegionInfo;
45 import org.apache.hadoop.hbase.HTableDescriptor;
46 import org.apache.hadoop.hbase.KeyValue;
47 import org.apache.hadoop.hbase.MultithreadedTestUtil;
48 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
49 import org.apache.hadoop.hbase.client.Delete;
50 import org.apache.hadoop.hbase.client.Get;
51 import org.apache.hadoop.hbase.client.Put;
52 import org.apache.hadoop.hbase.client.Result;
53 import org.apache.hadoop.hbase.client.Scan;
54 import org.apache.hadoop.hbase.filter.BinaryComparator;
55 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
56 import org.apache.hadoop.hbase.filter.CompareFilter;
57 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
58 import org.apache.hadoop.hbase.filter.Filter;
59 import org.apache.hadoop.hbase.filter.FilterList;
60 import org.apache.hadoop.hbase.filter.PrefixFilter;
61 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
62 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScanner;
63 import org.apache.hadoop.hbase.regionserver.wal.HLog;
64 import org.apache.hadoop.hbase.util.Bytes;
65 import org.apache.hadoop.hbase.util.EnvironmentEdge;
66 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
67 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
68 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
69 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
70 import org.apache.hadoop.hbase.util.Pair;
71 import org.apache.hadoop.hbase.util.PairOfSameType;
72 import org.apache.hadoop.hbase.util.Threads;
73
74 import com.google.common.collect.Lists;
75
76
77
78
79
80
81
82
83 public class TestHRegion extends HBaseTestCase {
84 static final Log LOG = LogFactory.getLog(TestHRegion.class);
85
86 HRegion region = null;
87 private final String DIR = HBaseTestingUtility.getTestDir() +
88 "/TestHRegion/";
89
90 private final int MAX_VERSIONS = 2;
91
92
93 protected final byte[] tableName = Bytes.toBytes("testtable");;
94 protected final byte[] qual1 = Bytes.toBytes("qual1");
95 protected final byte[] qual2 = Bytes.toBytes("qual2");
96 protected final byte[] qual3 = Bytes.toBytes("qual3");
97 protected final byte[] value1 = Bytes.toBytes("value1");
98 protected final byte[] value2 = Bytes.toBytes("value2");
99 protected final byte [] row = Bytes.toBytes("rowA");
100 protected final byte [] row2 = Bytes.toBytes("rowB");
101
102
103
104
105
106 @Override
107 protected void setUp() throws Exception {
108 super.setUp();
109 }
110
111 @Override
112 protected void tearDown() throws Exception {
113 super.tearDown();
114 EnvironmentEdgeManagerTestHelper.reset();
115 }
116
117
118
119
120
121
122
123 public void testGetWhileRegionClose() throws IOException {
124 Configuration hc = initSplit();
125 int numRows = 100;
126 byte [][] families = {fam1, fam2, fam3};
127
128
129 String method = this.getName();
130 initHRegion(tableName, method, hc, families);
131
132
133 final int startRow = 100;
134 putData(startRow, numRows, qual1, families);
135 putData(startRow, numRows, qual2, families);
136 putData(startRow, numRows, qual3, families);
137
138 final AtomicBoolean done = new AtomicBoolean(false);
139 final AtomicInteger gets = new AtomicInteger(0);
140 GetTillDoneOrException [] threads = new GetTillDoneOrException[10];
141 try {
142
143 for (int i = 0; i < threads.length / 2; i++) {
144 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow),
145 done, gets);
146 threads[i].setDaemon(true);
147 threads[i].start();
148 }
149
150
151 this.region.closing.set(true);
152 for (int i = threads.length / 2; i < threads.length; i++) {
153 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow),
154 done, gets);
155 threads[i].setDaemon(true);
156 threads[i].start();
157 }
158 } finally {
159 if (this.region != null) {
160 this.region.close();
161 this.region.getLog().closeAndDelete();
162 }
163 }
164 done.set(true);
165 for (GetTillDoneOrException t: threads) {
166 try {
167 t.join();
168 } catch (InterruptedException e) {
169 e.printStackTrace();
170 }
171 if (t.e != null) {
172 LOG.info("Exception=" + t.e);
173 assertFalse("Found a NPE in " + t.getName(),
174 t.e instanceof NullPointerException);
175 }
176 }
177 }
178
179
180
181
182
183 class GetTillDoneOrException extends Thread {
184 private final Get g;
185 private final AtomicBoolean done;
186 private final AtomicInteger count;
187 private Exception e;
188
189 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d,
190 final AtomicInteger c) {
191 super("getter." + i);
192 this.g = new Get(r);
193 this.done = d;
194 this.count = c;
195 }
196
197 @Override
198 public void run() {
199 while (!this.done.get()) {
200 try {
201 assertTrue(region.get(g, null).size() > 0);
202 this.count.incrementAndGet();
203 } catch (Exception e) {
204 this.e = e;
205 break;
206 }
207 }
208 }
209 }
210
211
212
213
214 public void testWeirdCacheBehaviour() throws Exception {
215 byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
216 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
217 Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
218 Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
219 initHRegion(TABLE, getName(), FAMILIES);
220 String value = "this is the value";
221 String value2 = "this is some other value";
222 String keyPrefix1 = "prefix1";
223 String keyPrefix2 = "prefix2";
224 String keyPrefix3 = "prefix3";
225 putRows(this.region, 3, value, keyPrefix1);
226 putRows(this.region, 3, value, keyPrefix2);
227 putRows(this.region, 3, value, keyPrefix3);
228
229 putRows(this.region, 3, value2, keyPrefix1);
230 putRows(this.region, 3, value2, keyPrefix2);
231 putRows(this.region, 3, value2, keyPrefix3);
232 System.out.println("Checking values for key: " + keyPrefix1);
233 assertEquals("Got back incorrect number of rows from scan", 3,
234 getNumberOfRows(keyPrefix1, value2, this.region));
235 System.out.println("Checking values for key: " + keyPrefix2);
236 assertEquals("Got back incorrect number of rows from scan", 3,
237 getNumberOfRows(keyPrefix2, value2, this.region));
238 System.out.println("Checking values for key: " + keyPrefix3);
239 assertEquals("Got back incorrect number of rows from scan", 3,
240 getNumberOfRows(keyPrefix3, value2, this.region));
241 deleteColumns(this.region, value2, keyPrefix1);
242 deleteColumns(this.region, value2, keyPrefix2);
243 deleteColumns(this.region, value2, keyPrefix3);
244 System.out.println("Starting important checks.....");
245 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,
246 0, getNumberOfRows(keyPrefix1, value2, this.region));
247 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,
248 0, getNumberOfRows(keyPrefix2, value2, this.region));
249 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,
250 0, getNumberOfRows(keyPrefix3, value2, this.region));
251 }
252
253 private void deleteColumns(HRegion r, String value, String keyPrefix)
254 throws IOException {
255 InternalScanner scanner = buildScanner(keyPrefix, value, r);
256 int count = 0;
257 boolean more = false;
258 List<KeyValue> results = new ArrayList<KeyValue>();
259 do {
260 more = scanner.next(results);
261 if (results != null && !results.isEmpty())
262 count++;
263 else
264 break;
265 Delete delete = new Delete(results.get(0).getRow());
266 delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
267 r.delete(delete, null, false);
268 results.clear();
269 } while (more);
270 assertEquals("Did not perform correct number of deletes", 3, count);
271 }
272
273 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
274 InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
275 int numberOfResults = 0;
276 List<KeyValue> results = new ArrayList<KeyValue>();
277 boolean more = false;
278 do {
279 more = resultScanner.next(results);
280 if (results != null && !results.isEmpty()) numberOfResults++;
281 else break;
282 for (KeyValue kv: results) {
283 System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(kv.getValue()));
284 }
285 results.clear();
286 } while(more);
287 return numberOfResults;
288 }
289
290 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
291 throws IOException {
292
293 FilterList allFilters = new FilterList();
294 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
295
296 SingleColumnValueFilter filter =
297 new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
298 Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
299 filter.setFilterIfMissing(true);
300 allFilters.addFilter(filter);
301 Scan scan = new Scan();
302 scan.addFamily(Bytes.toBytes("trans-blob"));
303 scan.addFamily(Bytes.toBytes("trans-type"));
304 scan.addFamily(Bytes.toBytes("trans-date"));
305 scan.addFamily(Bytes.toBytes("trans-tags"));
306 scan.addFamily(Bytes.toBytes("trans-group"));
307 scan.setFilter(allFilters);
308 return r.getScanner(scan);
309 }
310
311 private void putRows(HRegion r, int numRows, String value, String key)
312 throws IOException {
313 for (int i = 0; i < numRows; i++) {
314 String row = key + "_" + i
315 System.out.println(String.format("Saving row: %s, with value %s", row,
316 value));
317 Put put = new Put(Bytes.toBytes(row));
318 put.add(Bytes.toBytes("trans-blob"), null,
319 Bytes.toBytes("value for blob"));
320 put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
321 put.add(Bytes.toBytes("trans-date"), null,
322 Bytes.toBytes("20090921010101999"));
323 put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"),
324 Bytes.toBytes(value));
325 put.add(Bytes.toBytes("trans-group"), null,
326 Bytes.toBytes("adhocTransactionGroupId"));
327 r.put(put);
328 }
329 }
330
331 public void testFamilyWithAndWithoutColon() throws Exception {
332 byte [] b = Bytes.toBytes(getName());
333 byte [] cf = Bytes.toBytes("cf");
334 initHRegion(b, getName(), cf);
335 Put p = new Put(b);
336 byte [] cfwithcolon = Bytes.toBytes("cf:");
337 p.add(cfwithcolon, cfwithcolon, cfwithcolon);
338 boolean exception = false;
339 try {
340 this.region.put(p);
341 } catch (NoSuchColumnFamilyException e) {
342 exception = true;
343 }
344 assertTrue(exception);
345 }
346
347 @SuppressWarnings("unchecked")
348 public void testBatchPut() throws Exception {
349 byte[] b = Bytes.toBytes(getName());
350 byte[] cf = Bytes.toBytes("cf");
351 byte[] qual = Bytes.toBytes("qual");
352 byte[] val = Bytes.toBytes("val");
353 initHRegion(b, getName(), cf);
354
355 HLog.getSyncOps();
356 assertEquals(0, HLog.getSyncOps());
357
358 LOG.info("First a batch put with all valid puts");
359 final Put[] puts = new Put[10];
360 for (int i = 0; i < 10; i++) {
361 puts[i] = new Put(Bytes.toBytes("row_" + i));
362 puts[i].add(cf, qual, val);
363 }
364
365 OperationStatusCode[] codes = this.region.put(puts);
366 assertEquals(10, codes.length);
367 for (int i = 0; i < 10; i++) {
368 assertEquals(OperationStatusCode.SUCCESS, codes[i]);
369 }
370 assertEquals(1, HLog.getSyncOps());
371
372 LOG.info("Next a batch put with one invalid family");
373 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
374 codes = this.region.put(puts);
375 assertEquals(10, codes.length);
376 for (int i = 0; i < 10; i++) {
377 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY :
378 OperationStatusCode.SUCCESS, codes[i]);
379 }
380 assertEquals(1, HLog.getSyncOps());
381
382 LOG.info("Next a batch put that has to break into two batches to avoid a lock");
383 Integer lockedRow = region.obtainRowLock(Bytes.toBytes("row_2"));
384
385 MultithreadedTestUtil.TestContext ctx =
386 new MultithreadedTestUtil.TestContext(HBaseConfiguration.create());
387 final AtomicReference<OperationStatusCode[]> retFromThread =
388 new AtomicReference<OperationStatusCode[]>();
389 TestThread putter = new TestThread(ctx) {
390 @Override
391 public void doWork() throws IOException {
392 retFromThread.set(region.put(puts));
393 }
394 };
395 LOG.info("...starting put thread while holding lock");
396 ctx.addThread(putter);
397 ctx.startThreads();
398
399 LOG.info("...waiting for put thread to sync first time");
400 long startWait = System.currentTimeMillis();
401 while (HLog.getSyncOps() == 0) {
402 Thread.sleep(100);
403 if (System.currentTimeMillis() - startWait > 10000) {
404 fail("Timed out waiting for thread to sync first minibatch");
405 }
406 }
407 LOG.info("...releasing row lock, which should let put thread continue");
408 region.releaseRowLock(lockedRow);
409 LOG.info("...joining on thread");
410 ctx.stop();
411 LOG.info("...checking that next batch was synced");
412 assertEquals(1, HLog.getSyncOps());
413 codes = retFromThread.get();
414 for (int i = 0; i < 10; i++) {
415 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY :
416 OperationStatusCode.SUCCESS, codes[i]);
417 }
418
419 LOG.info("Nexta, a batch put which uses an already-held lock");
420 lockedRow = region.obtainRowLock(Bytes.toBytes("row_2"));
421 LOG.info("...obtained row lock");
422 List<Pair<Put, Integer>> putsAndLocks = Lists.newArrayList();
423 for (int i = 0; i < 10; i++) {
424 Pair<Put, Integer> pair = new Pair<Put, Integer>(puts[i], null);
425 if (i == 2) pair.setSecond(lockedRow);
426 putsAndLocks.add(pair);
427 }
428
429 codes = region.put(putsAndLocks.toArray(new Pair[0]));
430 LOG.info("...performed put");
431 for (int i = 0; i < 10; i++) {
432 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY :
433 OperationStatusCode.SUCCESS, codes[i]);
434 }
435
436 assertEquals(1, HLog.getSyncOps());
437
438
439 assertTrue(region.isRowLocked(lockedRow));
440 LOG.info("...releasing lock");
441 region.releaseRowLock(lockedRow);
442 }
443
444
445
446
447 public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
448 byte [] tableName = Bytes.toBytes("testtable");
449 byte [] row1 = Bytes.toBytes("row1");
450 byte [] fam1 = Bytes.toBytes("fam1");
451 byte [] qf1 = Bytes.toBytes("qualifier");
452 byte [] emptyVal = new byte[] {};
453 byte [] val1 = Bytes.toBytes("value1");
454 byte [] val2 = Bytes.toBytes("value2");
455 Integer lockId = null;
456
457
458 String method = this.getName();
459 initHRegion(tableName, method, fam1);
460
461 Put put = new Put(row1);
462 put.add(fam1, qf1, val1);
463
464
465 boolean res = region.checkAndMutate(row1, fam1, qf1, emptyVal, put, lockId,
466 true);
467 assertTrue(res);
468
469
470 res = region.checkAndMutate(row1, fam1, qf1, emptyVal, put, lockId, true);
471 assertFalse(res);
472
473 Delete delete = new Delete(row1);
474 delete.deleteColumn(fam1, qf1);
475 res = region.checkAndMutate(row1, fam1, qf1, emptyVal, delete, lockId,
476 true);
477 assertFalse(res);
478
479 put = new Put(row1);
480 put.add(fam1, qf1, val2);
481
482 res = region.checkAndMutate(row1, fam1, qf1, val1, put, lockId, true);
483 assertTrue(res);
484
485
486 delete = new Delete(row1);
487 delete.deleteColumn(fam1, qf1);
488 delete.deleteColumn(fam1, qf1);
489 res = region.checkAndMutate(row1, fam1, qf1, val2, delete, lockId, true);
490 assertTrue(res);
491
492 delete = new Delete(row1);
493 res = region.checkAndMutate(row1, fam1, qf1, emptyVal, delete, lockId,
494 true);
495 assertTrue(res);
496
497
498 put = new Put(row1);
499 put.add(fam1, qf1, val1);
500
501 res = region.checkAndMutate(row1, fam1, qf1, null, put, lockId, true);
502 assertTrue(res);
503
504 }
505
506 public void testCheckAndMutate_WithWrongValue() throws IOException{
507 byte [] tableName = Bytes.toBytes("testtable");
508 byte [] row1 = Bytes.toBytes("row1");
509 byte [] fam1 = Bytes.toBytes("fam1");
510 byte [] qf1 = Bytes.toBytes("qualifier");
511 byte [] val1 = Bytes.toBytes("value1");
512 byte [] val2 = Bytes.toBytes("value2");
513 Integer lockId = null;
514
515
516 String method = this.getName();
517 initHRegion(tableName, method, fam1);
518
519
520 Put put = new Put(row1);
521 put.add(fam1, qf1, val1);
522 region.put(put);
523
524
525 boolean res = region.checkAndMutate(row1, fam1, qf1, val2, put, lockId, true);
526 assertEquals(false, res);
527
528
529 Delete delete = new Delete(row1);
530 delete.deleteFamily(fam1);
531 res = region.checkAndMutate(row1, fam1, qf1, val2, delete, lockId, true);
532 assertEquals(false, res);
533 }
534
535 public void testCheckAndMutate_WithCorrectValue() throws IOException{
536 byte [] tableName = Bytes.toBytes("testtable");
537 byte [] row1 = Bytes.toBytes("row1");
538 byte [] fam1 = Bytes.toBytes("fam1");
539 byte [] qf1 = Bytes.toBytes("qualifier");
540 byte [] val1 = Bytes.toBytes("value1");
541 Integer lockId = null;
542
543
544 String method = this.getName();
545 initHRegion(tableName, method, fam1);
546
547
548 Put put = new Put(row1);
549 put.add(fam1, qf1, val1);
550 region.put(put);
551
552
553 boolean res = region.checkAndMutate(row1, fam1, qf1, val1, put, lockId, true);
554 assertEquals(true, res);
555
556
557 Delete delete = new Delete(row1);
558 delete.deleteColumn(fam1, qf1);
559 res = region.checkAndMutate(row1, fam1, qf1, val1, put, lockId, true);
560 assertEquals(true, res);
561 }
562
563 public void testCheckAndPut_ThatPutWasWritten() throws IOException{
564 byte [] tableName = Bytes.toBytes("testtable");
565 byte [] row1 = Bytes.toBytes("row1");
566 byte [] fam1 = Bytes.toBytes("fam1");
567 byte [] fam2 = Bytes.toBytes("fam2");
568 byte [] qf1 = Bytes.toBytes("qualifier");
569 byte [] val1 = Bytes.toBytes("value1");
570 byte [] val2 = Bytes.toBytes("value2");
571 Integer lockId = null;
572
573 byte [][] families = {fam1, fam2};
574
575
576 String method = this.getName();
577 initHRegion(tableName, method, families);
578
579
580 Put put = new Put(row1);
581 put.add(fam1, qf1, val1);
582 region.put(put);
583
584
585 long ts = System.currentTimeMillis();
586 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
587 put = new Put(row1);
588 put.add(kv);
589
590
591 Store store = region.getStore(fam1);
592 store.memstore.kvset.size();
593
594 boolean res = region.checkAndMutate(row1, fam1, qf1, val1, put, lockId, true);
595 assertEquals(true, res);
596 store.memstore.kvset.size();
597
598 Get get = new Get(row1);
599 get.addColumn(fam2, qf1);
600 KeyValue [] actual = region.get(get, null).raw();
601
602 KeyValue [] expected = {kv};
603
604 assertEquals(expected.length, actual.length);
605 for(int i=0; i<actual.length; i++) {
606 assertEquals(expected[i], actual[i]);
607 }
608
609 }
610
611 public void testCheckAndPut_wrongRowInPut() throws IOException {
612 initHRegion(tableName, this.getName(), COLUMNS);
613
614 Put put = new Put(row2);
615 put.add(fam1, qual1, value1);
616 try {
617 boolean res = region.checkAndMutate(row,
618 fam1, qual1, value2, put, null, false);
619 fail();
620 } catch (DoNotRetryIOException expected) {
621
622 }
623 }
624
625 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException{
626 byte [] tableName = Bytes.toBytes("testtable");
627 byte [] row1 = Bytes.toBytes("row1");
628 byte [] fam1 = Bytes.toBytes("fam1");
629 byte [] fam2 = Bytes.toBytes("fam2");
630 byte [] qf1 = Bytes.toBytes("qualifier1");
631 byte [] qf2 = Bytes.toBytes("qualifier2");
632 byte [] qf3 = Bytes.toBytes("qualifier3");
633 byte [] val1 = Bytes.toBytes("value1");
634 byte [] val2 = Bytes.toBytes("value2");
635 byte [] val3 = Bytes.toBytes("value3");
636 byte[] emptyVal = new byte[] { };
637 Integer lockId = null;
638
639 byte [][] families = {fam1, fam2};
640
641
642 String method = this.getName();
643 initHRegion(tableName, method, families);
644
645
646 Put put = new Put(row1);
647 put.add(fam1, qf1, val1);
648 region.put(put);
649 Threads.sleep(2);
650
651 put = new Put(row1);
652 put.add(fam1, qf1, val2);
653 put.add(fam2, qf1, val3);
654 put.add(fam2, qf2, val2);
655 put.add(fam2, qf3, val1);
656 put.add(fam1, qf3, val1);
657 region.put(put);
658
659
660 Delete delete = new Delete(row1);
661 delete.deleteColumn(fam1, qf1);
662 delete.deleteColumn(fam2, qf1);
663 delete.deleteColumn(fam1, qf3);
664 boolean res = region.checkAndMutate(row1, fam1, qf1, val2, delete, lockId,
665 true);
666 assertEquals(true, res);
667
668 Get get = new Get(row1);
669 get.addColumn(fam1, qf1);
670 get.addColumn(fam1, qf3);
671 get.addColumn(fam2, qf2);
672 Result r = region.get(get, null);
673 assertEquals(2, r.size());
674 assertEquals(val1, r.getValue(fam1, qf1));
675 assertEquals(val2, r.getValue(fam2, qf2));
676
677
678 delete = new Delete(row1);
679 delete.deleteFamily(fam2);
680 res = region.checkAndMutate(row1, fam2, qf1, emptyVal, delete, lockId,
681 true);
682 assertEquals(true, res);
683
684 get = new Get(row1);
685 r = region.get(get, null);
686 assertEquals(1, r.size());
687 assertEquals(val1, r.getValue(fam1, qf1));
688
689
690 delete = new Delete(row1);
691 res = region.checkAndMutate(row1, fam1, qf1, val1, delete, lockId,
692 true);
693 assertEquals(true, res);
694 get = new Get(row1);
695 r = region.get(get, null);
696 assertEquals(0, r.size());
697 }
698
699
700
701
702 public void testDelete_multiDeleteColumn() throws IOException {
703 byte [] tableName = Bytes.toBytes("testtable");
704 byte [] row1 = Bytes.toBytes("row1");
705 byte [] fam1 = Bytes.toBytes("fam1");
706 byte [] qual = Bytes.toBytes("qualifier");
707 byte [] value = Bytes.toBytes("value");
708
709 Put put = new Put(row1);
710 put.add(fam1, qual, 1, value);
711 put.add(fam1, qual, 2, value);
712
713 String method = this.getName();
714 initHRegion(tableName, method, fam1);
715
716 region.put(put);
717
718
719 Delete delete = new Delete(row1);
720 delete.deleteColumn(fam1, qual);
721 delete.deleteColumn(fam1, qual);
722 region.delete(delete, null, false);
723
724 Get get = new Get(row1);
725 get.addFamily(fam1);
726 Result r = region.get(get, null);
727 assertEquals(0, r.size());
728 }
729
730 public void testDelete_CheckFamily() throws IOException {
731 byte [] tableName = Bytes.toBytes("testtable");
732 byte [] row1 = Bytes.toBytes("row1");
733 byte [] fam1 = Bytes.toBytes("fam1");
734 byte [] fam2 = Bytes.toBytes("fam2");
735 byte [] fam3 = Bytes.toBytes("fam3");
736 byte [] fam4 = Bytes.toBytes("fam4");
737
738
739 String method = this.getName();
740 initHRegion(tableName, method, fam1, fam2, fam3);
741
742 List<KeyValue> kvs = new ArrayList<KeyValue>();
743 kvs.add(new KeyValue(row1, fam4, null, null));
744
745
746
747 byte [] family = fam2;
748 try {
749 Map<byte[], List<KeyValue>> deleteMap = new HashMap<byte[], List<KeyValue>>();
750 deleteMap.put(family, kvs);
751 region.delete(deleteMap, true);
752 } catch (Exception e) {
753 assertTrue("Family " +new String(family)+ " does not exist", false);
754 }
755
756
757 boolean ok = false;
758 family = fam4;
759 try {
760 Map<byte[], List<KeyValue>> deleteMap = new HashMap<byte[], List<KeyValue>>();
761 deleteMap.put(family, kvs);
762 region.delete(deleteMap, true);
763 } catch (Exception e) {
764 ok = true;
765 }
766 assertEquals("Family " +new String(family)+ " does exist", true, ok);
767 }
768
769 public void testDelete_mixed() throws IOException, InterruptedException {
770 byte [] tableName = Bytes.toBytes("testtable");
771 byte [] fam = Bytes.toBytes("info");
772 byte [][] families = {fam};
773 String method = this.getName();
774 initHRegion(tableName, method, families);
775 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
776
777 byte [] row = Bytes.toBytes("table_name");
778
779 byte [] serverinfo = Bytes.toBytes("serverinfo");
780 byte [] splitA = Bytes.toBytes("splitA");
781 byte [] splitB = Bytes.toBytes("splitB");
782
783
784 Put put = new Put(row);
785 put.add(fam, splitA, Bytes.toBytes("reference_A"));
786 region.put(put);
787
788 put = new Put(row);
789 put.add(fam, splitB, Bytes.toBytes("reference_B"));
790 region.put(put);
791
792 put = new Put(row);
793 put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
794 region.put(put);
795
796
797 Delete delete = new Delete(row);
798 delete.deleteColumns(fam, splitA);
799 region.delete(delete, null, true);
800
801
802 Get get = new Get(row).addColumn(fam, serverinfo);
803 Result result = region.get(get, null);
804 assertEquals(1, result.size());
805
806 get = new Get(row).addColumn(fam, splitA);
807 result = region.get(get, null);
808 assertEquals(0, result.size());
809
810 get = new Get(row).addColumn(fam, splitB);
811 result = region.get(get, null);
812 assertEquals(1, result.size());
813
814
815 put = new Put(row);
816 put.add(fam, splitA, Bytes.toBytes("reference_A"));
817 region.put(put);
818 get = new Get(row);
819 result = region.get(get, null);
820 assertEquals(3, result.size());
821
822
823 delete = new Delete(row);
824 region.delete(delete, null, false);
825 assertEquals(0, region.get(get, null).size());
826
827 region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
828 result = region.get(get, null);
829 assertEquals(1, result.size());
830 }
831
832 public void testDeleteRowWithFutureTs() throws IOException {
833 byte [] tableName = Bytes.toBytes("testtable");
834 byte [] fam = Bytes.toBytes("info");
835 byte [][] families = {fam};
836 String method = this.getName();
837 initHRegion(tableName, method, families);
838
839 byte [] row = Bytes.toBytes("table_name");
840
841 byte [] serverinfo = Bytes.toBytes("serverinfo");
842
843
844 Put put = new Put(row);
845 put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP-5,Bytes.toBytes("value"));
846 region.put(put);
847
848
849 Delete delete = new Delete(row);
850 region.delete(delete, null, true);
851
852
853 Get get = new Get(row).addColumn(fam, serverinfo);
854 Result result = region.get(get, null);
855 assertEquals(1, result.size());
856
857
858 delete = new Delete(row,HConstants.LATEST_TIMESTAMP-3,null);
859 region.delete(delete, null, true);
860
861
862 get = new Get(row).addColumn(fam, serverinfo);
863 result = region.get(get, null);
864 assertEquals(0, result.size());
865 }
866
867
868
869
870
871 public void testPutWithLatestTS() throws IOException {
872 byte [] tableName = Bytes.toBytes("testtable");
873 byte [] fam = Bytes.toBytes("info");
874 byte [][] families = {fam};
875 String method = this.getName();
876 initHRegion(tableName, method, families);
877
878 byte [] row = Bytes.toBytes("row1");
879
880 byte [] qual = Bytes.toBytes("qual");
881
882
883 Put put = new Put(row);
884 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
885 region.put(put, false);
886
887
888 Get get = new Get(row).addColumn(fam, qual);
889 Result result = region.get(get, null);
890 assertEquals(1, result.size());
891 KeyValue kv = result.raw()[0];
892 LOG.info("Got: " + kv);
893 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
894 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
895
896
897
898 row = Bytes.toBytes("row2");
899 put = new Put(row);
900 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
901 region.put(put, true);
902
903
904 get = new Get(row).addColumn(fam, qual);
905 result = region.get(get, null);
906 assertEquals(1, result.size());
907 kv = result.raw()[0];
908 LOG.info("Got: " + kv);
909 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
910 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
911
912 }
913
914 public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
915 byte [] tableName = Bytes.toBytes("test_table");
916 byte [] fam1 = Bytes.toBytes("columnA");
917 byte [] fam2 = Bytes.toBytes("columnB");
918 initHRegion(tableName, getName(), fam1, fam2);
919
920 byte [] rowA = Bytes.toBytes("rowA");
921 byte [] rowB = Bytes.toBytes("rowB");
922
923 byte [] value = Bytes.toBytes("value");
924
925 Delete delete = new Delete(rowA);
926 delete.deleteFamily(fam1);
927
928 region.delete(delete, null, true);
929
930
931 Put put = new Put(rowA);
932 put.add(fam2, null, value);
933 region.put(put);
934
935 put = new Put(rowB);
936 put.add(fam1, null, value);
937 put.add(fam2, null, value);
938 region.put(put);
939
940 Scan scan = new Scan();
941 scan.addFamily(fam1).addFamily(fam2);
942 InternalScanner s = region.getScanner(scan);
943 List<KeyValue> results = new ArrayList<KeyValue>();
944 s.next(results);
945 assertTrue(Bytes.equals(rowA, results.get(0).getRow()));
946
947 results.clear();
948 s.next(results);
949 assertTrue(Bytes.equals(rowB, results.get(0).getRow()));
950
951 }
952
953 public void testDeleteColumns_PostInsert() throws IOException,
954 InterruptedException {
955 Delete delete = new Delete(row);
956 delete.deleteColumns(fam1, qual1);
957 doTestDelete_AndPostInsert(delete);
958 }
959
960 public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
961 Delete delete = new Delete(row);
962 delete.deleteFamily(fam1);
963 doTestDelete_AndPostInsert(delete);
964 }
965
966 public void doTestDelete_AndPostInsert(Delete delete)
967 throws IOException, InterruptedException {
968 initHRegion(tableName, getName(), fam1);
969 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
970 Put put = new Put(row);
971 put.add(fam1, qual1, value1);
972 region.put(put);
973
974
975 region.delete(delete, null, true);
976
977
978
979 put = new Put(row);
980 put.add(fam1, qual1, value2);
981 region.put(put);
982
983
984 Get get = new Get(row);
985 get.addColumn(fam1, qual1);
986
987 Result r = region.get(get, null);
988 assertEquals(1, r.size());
989 assertByteEquals(value2, r.getValue(fam1, qual1));
990
991
992 Scan scan = new Scan(row);
993 scan.addColumn(fam1, qual1);
994 InternalScanner s = region.getScanner(scan);
995
996 List<KeyValue> results = new ArrayList<KeyValue>();
997 assertEquals(false, s.next(results));
998 assertEquals(1, results.size());
999 KeyValue kv = results.get(0);
1000
1001 assertByteEquals(value2, kv.getValue());
1002 assertByteEquals(fam1, kv.getFamily());
1003 assertByteEquals(qual1, kv.getQualifier());
1004 assertByteEquals(row, kv.getRow());
1005 }
1006
1007
1008
1009 public void testDelete_CheckTimestampUpdated()
1010 throws IOException {
1011 byte [] row1 = Bytes.toBytes("row1");
1012 byte [] col1 = Bytes.toBytes("col1");
1013 byte [] col2 = Bytes.toBytes("col2");
1014 byte [] col3 = Bytes.toBytes("col3");
1015
1016
1017 String method = this.getName();
1018 initHRegion(tableName, method, fam1);
1019
1020
1021 List<KeyValue> kvs = new ArrayList<KeyValue>();
1022 kvs.add(new KeyValue(row1, fam1, col1, null));
1023 kvs.add(new KeyValue(row1, fam1, col2, null));
1024 kvs.add(new KeyValue(row1, fam1, col3, null));
1025
1026 Map<byte[], List<KeyValue>> deleteMap = new HashMap<byte[], List<KeyValue>>();
1027 deleteMap.put(fam1, kvs);
1028 region.delete(deleteMap, true);
1029
1030
1031
1032 long now = System.currentTimeMillis();
1033 KeyValue firstKv = region.getStore(fam1).memstore.kvset.first();
1034 assertTrue(firstKv.getTimestamp() <= now);
1035 now = firstKv.getTimestamp();
1036 for (KeyValue kv: region.getStore(fam1).memstore.kvset) {
1037 assertTrue(kv.getTimestamp() <= now);
1038 now = kv.getTimestamp();
1039 }
1040 }
1041
1042
1043
1044
1045 public void testGet_FamilyChecker() throws IOException {
1046 byte [] tableName = Bytes.toBytes("testtable");
1047 byte [] row1 = Bytes.toBytes("row1");
1048 byte [] fam1 = Bytes.toBytes("fam1");
1049 byte [] fam2 = Bytes.toBytes("False");
1050 byte [] col1 = Bytes.toBytes("col1");
1051
1052
1053 String method = this.getName();
1054 initHRegion(tableName, method, fam1);
1055
1056 Get get = new Get(row1);
1057 get.addColumn(fam2, col1);
1058
1059
1060 try {
1061 region.get(get, null);
1062 } catch (NoSuchColumnFamilyException e){
1063 assertFalse(false);
1064 return;
1065 }
1066 assertFalse(true);
1067 }
1068
1069 public void testGet_Basic() throws IOException {
1070 byte [] tableName = Bytes.toBytes("testtable");
1071 byte [] row1 = Bytes.toBytes("row1");
1072 byte [] fam1 = Bytes.toBytes("fam1");
1073 byte [] col1 = Bytes.toBytes("col1");
1074 byte [] col2 = Bytes.toBytes("col2");
1075 byte [] col3 = Bytes.toBytes("col3");
1076 byte [] col4 = Bytes.toBytes("col4");
1077 byte [] col5 = Bytes.toBytes("col5");
1078
1079
1080 String method = this.getName();
1081 initHRegion(tableName, method, fam1);
1082
1083
1084 Put put = new Put(row1);
1085 put.add(fam1, col1, null);
1086 put.add(fam1, col2, null);
1087 put.add(fam1, col3, null);
1088 put.add(fam1, col4, null);
1089 put.add(fam1, col5, null);
1090 region.put(put);
1091
1092 Get get = new Get(row1);
1093 get.addColumn(fam1, col2);
1094 get.addColumn(fam1, col4);
1095
1096 KeyValue kv1 = new KeyValue(row1, fam1, col2);
1097 KeyValue kv2 = new KeyValue(row1, fam1, col4);
1098 KeyValue [] expected = {kv1, kv2};
1099
1100
1101 Result res = region.get(get, null);
1102 assertEquals(expected.length, res.size());
1103 for(int i=0; i<res.size(); i++){
1104 assertEquals(0,
1105 Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
1106 assertEquals(0,
1107 Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
1108 assertEquals(0,
1109 Bytes.compareTo(
1110 expected[i].getQualifier(), res.raw()[i].getQualifier()));
1111 }
1112
1113
1114 Get g = new Get(row1);
1115 final int count = 2;
1116 g.setFilter(new ColumnCountGetFilter(count));
1117 res = region.get(g, null);
1118 assertEquals(count, res.size());
1119 }
1120
1121 public void testGet_Empty() throws IOException {
1122 byte [] tableName = Bytes.toBytes("emptytable");
1123 byte [] row = Bytes.toBytes("row");
1124 byte [] fam = Bytes.toBytes("fam");
1125
1126 String method = this.getName();
1127 initHRegion(tableName, method, fam);
1128
1129 Get get = new Get(row);
1130 get.addFamily(fam);
1131 Result r = region.get(get, null);
1132
1133 assertTrue(r.isEmpty());
1134 }
1135
1136
1137
1138
1139
1140 public void stestGet_Root() throws IOException {
1141
1142 String method = this.getName();
1143 initHRegion(HConstants.ROOT_TABLE_NAME, method, HConstants.CATALOG_FAMILY);
1144
1145
1146 Put put = new Put(HConstants.EMPTY_START_ROW);
1147 put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, null);
1148 region.put(put);
1149
1150 Get get = new Get(HConstants.EMPTY_START_ROW);
1151 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
1152
1153
1154 KeyValue kv1 = new KeyValue(HConstants.EMPTY_START_ROW,
1155 HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
1156 KeyValue [] expected = {kv1};
1157
1158
1159 Result res = region.get(get, null);
1160
1161 assertEquals(expected.length, res.size());
1162 for(int i=0; i<res.size(); i++){
1163 assertEquals(0,
1164 Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
1165 assertEquals(0,
1166 Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
1167 assertEquals(0,
1168 Bytes.compareTo(
1169 expected[i].getQualifier(), res.raw()[i].getQualifier()));
1170 }
1171
1172
1173 region.flushcache();
1174
1175
1176 res = region.get(get, null);
1177
1178 assertEquals(expected.length, res.size());
1179 for(int i=0; i<res.size(); i++){
1180 assertEquals(0,
1181 Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
1182 assertEquals(0,
1183 Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
1184 assertEquals(0,
1185 Bytes.compareTo(
1186 expected[i].getQualifier(), res.raw()[i].getQualifier()));
1187 }
1188
1189
1190 Scan scan = new Scan();
1191 scan.addFamily(HConstants.CATALOG_FAMILY);
1192 InternalScanner s = region.getScanner(scan);
1193 List<KeyValue> result = new ArrayList<KeyValue>();
1194 s.next(result);
1195
1196 assertEquals(expected.length, result.size());
1197 for(int i=0; i<res.size(); i++){
1198 assertEquals(0,
1199 Bytes.compareTo(expected[i].getRow(), result.get(i).getRow()));
1200 assertEquals(0,
1201 Bytes.compareTo(expected[i].getFamily(), result.get(i).getFamily()));
1202 assertEquals(0,
1203 Bytes.compareTo(
1204 expected[i].getQualifier(), result.get(i).getQualifier()));
1205 }
1206 }
1207
1208
1209
1210
1211 public void testLocks() throws IOException{
1212 byte [] tableName = Bytes.toBytes("testtable");
1213 byte [][] families = {fam1, fam2, fam3};
1214
1215 Configuration hc = initSplit();
1216
1217 String method = this.getName();
1218 initHRegion(tableName, method, hc, families);
1219
1220 final int threadCount = 10;
1221 final int lockCount = 10;
1222
1223 List<Thread>threads = new ArrayList<Thread>(threadCount);
1224 for (int i = 0; i < threadCount; i++) {
1225 threads.add(new Thread(Integer.toString(i)) {
1226 @Override
1227 public void run() {
1228 Integer [] lockids = new Integer[lockCount];
1229
1230 for (int i = 0; i < lockCount; i++) {
1231 try {
1232 byte [] rowid = Bytes.toBytes(Integer.toString(i));
1233 lockids[i] = region.obtainRowLock(rowid);
1234 assertEquals(rowid, region.getRowFromLock(lockids[i]));
1235 LOG.debug(getName() + " locked " + Bytes.toString(rowid));
1236 } catch (IOException e) {
1237 e.printStackTrace();
1238 }
1239 }
1240 LOG.debug(getName() + " set " +
1241 Integer.toString(lockCount) + " locks");
1242
1243
1244 for (int i = lockCount - 1; i >= 0; i--) {
1245 region.releaseRowLock(lockids[i]);
1246 LOG.debug(getName() + " unlocked " + i);
1247 }
1248 LOG.debug(getName() + " released " +
1249 Integer.toString(lockCount) + " locks");
1250 }
1251 });
1252 }
1253
1254
1255 for (Thread t : threads) {
1256 t.start();
1257 }
1258
1259
1260 for (Thread t: threads) {
1261 while (t.isAlive()) {
1262 try {
1263 Thread.sleep(1);
1264 } catch (InterruptedException e) {
1265
1266 }
1267 }
1268 }
1269 LOG.info("locks completed.");
1270 }
1271
1272
1273
1274
1275 public void testMerge() throws IOException {
1276 byte [] tableName = Bytes.toBytes("testtable");
1277 byte [][] families = {fam1, fam2, fam3};
1278 Configuration hc = initSplit();
1279
1280 String method = this.getName();
1281 initHRegion(tableName, method, hc, families);
1282 try {
1283 LOG.info("" + addContent(region, fam3));
1284 region.flushcache();
1285 byte [] splitRow = region.compactStores();
1286 assertNotNull(splitRow);
1287 LOG.info("SplitRow: " + Bytes.toString(splitRow));
1288 HRegion [] subregions = splitRegion(region, splitRow);
1289 try {
1290
1291 for (int i = 0; i < subregions.length; i++) {
1292 openClosedRegion(subregions[i]);
1293 subregions[i].compactStores();
1294 }
1295 Path oldRegionPath = region.getRegionDir();
1296 Path oldRegion1 = subregions[0].getRegionDir();
1297 Path oldRegion2 = subregions[1].getRegionDir();
1298 long startTime = System.currentTimeMillis();
1299 region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
1300 LOG.info("Merge regions elapsed time: " +
1301 ((System.currentTimeMillis() - startTime) / 1000.0));
1302 fs.delete(oldRegion1, true);
1303 fs.delete(oldRegion2, true);
1304 fs.delete(oldRegionPath, true);
1305 LOG.info("splitAndMerge completed.");
1306 } finally {
1307 if (subregions != null) {
1308 for (int i = 0; i < subregions.length; i++) {
1309 try {
1310 subregions[i].close();
1311 } catch (IOException e) {
1312
1313 }
1314 }
1315 }
1316 }
1317 } finally {
1318 if (region != null) {
1319 region.close();
1320 region.getLog().closeAndDelete();
1321 }
1322 }
1323 }
1324
1325
1326
1327
1328
1329
1330
1331 HRegion [] splitRegion(final HRegion parent, final byte [] midkey)
1332 throws IOException {
1333 PairOfSameType<HRegion> result = null;
1334 SplitTransaction st = new SplitTransaction(parent, midkey);
1335
1336
1337 if (!st.prepare()) return null;
1338 try {
1339 result = st.execute(null, null);
1340 } catch (IOException ioe) {
1341 try {
1342 LOG.info("Running rollback of failed split of " +
1343 parent.getRegionNameAsString() + "; " + ioe.getMessage());
1344 st.rollback(null, null);
1345 LOG.info("Successful rollback of failed split of " +
1346 parent.getRegionNameAsString());
1347 return null;
1348 } catch (RuntimeException e) {
1349
1350 LOG.info("Failed rollback of failed split of " +
1351 parent.getRegionNameAsString() + " -- aborting server", e);
1352 }
1353 }
1354 return new HRegion [] {result.getFirst(), result.getSecond()};
1355 }
1356
1357
1358
1359
1360 public void testGetScanner_WithOkFamilies() throws IOException {
1361 byte [] tableName = Bytes.toBytes("testtable");
1362 byte [] fam1 = Bytes.toBytes("fam1");
1363 byte [] fam2 = Bytes.toBytes("fam2");
1364
1365 byte [][] families = {fam1, fam2};
1366
1367
1368 String method = this.getName();
1369 initHRegion(tableName, method, families);
1370
1371 Scan scan = new Scan();
1372 scan.addFamily(fam1);
1373 scan.addFamily(fam2);
1374 try {
1375 region.getScanner(scan);
1376 } catch (Exception e) {
1377 assertTrue("Families could not be found in Region", false);
1378 }
1379 }
1380
1381 public void testGetScanner_WithNotOkFamilies() throws IOException {
1382 byte [] tableName = Bytes.toBytes("testtable");
1383 byte [] fam1 = Bytes.toBytes("fam1");
1384 byte [] fam2 = Bytes.toBytes("fam2");
1385
1386 byte [][] families = {fam1};
1387
1388
1389 String method = this.getName();
1390 initHRegion(tableName, method, families);
1391
1392 Scan scan = new Scan();
1393 scan.addFamily(fam2);
1394 boolean ok = false;
1395 try {
1396 region.getScanner(scan);
1397 } catch (Exception e) {
1398 ok = true;
1399 }
1400 assertTrue("Families could not be found in Region", ok);
1401 }
1402
1403 public void testGetScanner_WithNoFamilies() throws IOException {
1404 byte [] tableName = Bytes.toBytes("testtable");
1405 byte [] row1 = Bytes.toBytes("row1");
1406 byte [] fam1 = Bytes.toBytes("fam1");
1407 byte [] fam2 = Bytes.toBytes("fam2");
1408 byte [] fam3 = Bytes.toBytes("fam3");
1409 byte [] fam4 = Bytes.toBytes("fam4");
1410
1411 byte [][] families = {fam1, fam2, fam3, fam4};
1412
1413
1414 String method = this.getName();
1415 initHRegion(tableName, method, families);
1416
1417
1418
1419 Put put = new Put(row1);
1420 put.add(fam1, null, null);
1421 put.add(fam2, null, null);
1422 put.add(fam3, null, null);
1423 put.add(fam4, null, null);
1424 region.put(put);
1425
1426 Scan scan = null;
1427 HRegion.RegionScanner is = null;
1428
1429
1430
1431 scan = new Scan();
1432 scan.addFamily(fam2);
1433 scan.addFamily(fam4);
1434 is = (RegionScanner) region.getScanner(scan);
1435 ReadWriteConsistencyControl.resetThreadReadPoint(region.getRWCC());
1436 assertEquals(1, ((RegionScanner)is).storeHeap.getHeap().size());
1437
1438 scan = new Scan();
1439 is = (RegionScanner) region.getScanner(scan);
1440 ReadWriteConsistencyControl.resetThreadReadPoint(region.getRWCC());
1441 assertEquals(families.length -1,
1442 ((RegionScanner)is).storeHeap.getHeap().size());
1443 }
1444
1445
1446
1447
1448 public void testGetScanner_WithRegionClosed() {
1449 byte[] tableName = Bytes.toBytes("testtable");
1450 byte[] fam1 = Bytes.toBytes("fam1");
1451 byte[] fam2 = Bytes.toBytes("fam2");
1452
1453 byte[][] families = {fam1, fam2};
1454
1455
1456 String method = this.getName();
1457 try {
1458 initHRegion(tableName, method, families);
1459 } catch (IOException e) {
1460 e.printStackTrace();
1461 fail("Got IOException during initHRegion, " + e.getMessage());
1462 }
1463 region.closed.set(true);
1464 try {
1465 region.getScanner(null);
1466 fail("Expected to get an exception during getScanner on a region that is closed");
1467 } catch (org.apache.hadoop.hbase.NotServingRegionException e) {
1468
1469 } catch (IOException e) {
1470 fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
1471 + e.getMessage());
1472 }
1473 }
1474
1475 public void testRegionScanner_Next() throws IOException {
1476 byte [] tableName = Bytes.toBytes("testtable");
1477 byte [] row1 = Bytes.toBytes("row1");
1478 byte [] row2 = Bytes.toBytes("row2");
1479 byte [] fam1 = Bytes.toBytes("fam1");
1480 byte [] fam2 = Bytes.toBytes("fam2");
1481 byte [] fam3 = Bytes.toBytes("fam3");
1482 byte [] fam4 = Bytes.toBytes("fam4");
1483
1484 byte [][] families = {fam1, fam2, fam3, fam4};
1485 long ts = System.currentTimeMillis();
1486
1487
1488 String method = this.getName();
1489 initHRegion(tableName, method, families);
1490
1491
1492 Put put = null;
1493 put = new Put(row1);
1494 put.add(fam1, null, ts, null);
1495 put.add(fam2, null, ts, null);
1496 put.add(fam3, null, ts, null);
1497 put.add(fam4, null, ts, null);
1498 region.put(put);
1499
1500 put = new Put(row2);
1501 put.add(fam1, null, ts, null);
1502 put.add(fam2, null, ts, null);
1503 put.add(fam3, null, ts, null);
1504 put.add(fam4, null, ts, null);
1505 region.put(put);
1506
1507 Scan scan = new Scan();
1508 scan.addFamily(fam2);
1509 scan.addFamily(fam4);
1510 InternalScanner is = region.getScanner(scan);
1511
1512 List<KeyValue> res = null;
1513
1514
1515 List<KeyValue> expected1 = new ArrayList<KeyValue>();
1516 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
1517 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
1518
1519 res = new ArrayList<KeyValue>();
1520 is.next(res);
1521 for(int i=0; i<res.size(); i++) {
1522 assertEquals(expected1.get(i), res.get(i));
1523 }
1524
1525
1526 List<KeyValue> expected2 = new ArrayList<KeyValue>();
1527 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
1528 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
1529
1530 res = new ArrayList<KeyValue>();
1531 is.next(res);
1532 for(int i=0; i<res.size(); i++) {
1533 assertEquals(expected2.get(i), res.get(i));
1534 }
1535
1536 }
1537
1538 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions()
1539 throws IOException {
1540 byte [] tableName = Bytes.toBytes("testtable");
1541 byte [] row1 = Bytes.toBytes("row1");
1542 byte [] qf1 = Bytes.toBytes("qualifier1");
1543 byte [] qf2 = Bytes.toBytes("qualifier2");
1544 byte [] fam1 = Bytes.toBytes("fam1");
1545 byte [][] families = {fam1};
1546
1547 long ts1 = System.currentTimeMillis();
1548 long ts2 = ts1 + 1;
1549 long ts3 = ts1 + 2;
1550
1551
1552 String method = this.getName();
1553 initHRegion(tableName, method, families);
1554
1555
1556 Put put = null;
1557 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
1558 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
1559 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
1560
1561 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
1562 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
1563 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
1564
1565 put = new Put(row1);
1566 put.add(kv13);
1567 put.add(kv12);
1568 put.add(kv11);
1569 put.add(kv23);
1570 put.add(kv22);
1571 put.add(kv21);
1572 region.put(put);
1573
1574
1575 List<KeyValue> expected = new ArrayList<KeyValue>();
1576 expected.add(kv13);
1577 expected.add(kv12);
1578
1579 Scan scan = new Scan(row1);
1580 scan.addColumn(fam1, qf1);
1581 scan.setMaxVersions(MAX_VERSIONS);
1582 List<KeyValue> actual = new ArrayList<KeyValue>();
1583 InternalScanner scanner = region.getScanner(scan);
1584
1585 boolean hasNext = scanner.next(actual);
1586 assertEquals(false, hasNext);
1587
1588
1589 for(int i=0; i<expected.size(); i++) {
1590 assertEquals(expected.get(i), actual.get(i));
1591 }
1592 }
1593
1594 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions()
1595 throws IOException{
1596 byte [] tableName = Bytes.toBytes("testtable");
1597 byte [] row1 = Bytes.toBytes("row1");
1598 byte [] qf1 = Bytes.toBytes("qualifier1");
1599 byte [] qf2 = Bytes.toBytes("qualifier2");
1600 byte [] fam1 = Bytes.toBytes("fam1");
1601 byte [][] families = {fam1};
1602
1603 long ts1 = 1;
1604 long ts2 = ts1 + 1;
1605 long ts3 = ts1 + 2;
1606
1607
1608 String method = this.getName();
1609 initHRegion(tableName, method, families);
1610
1611
1612 Put put = null;
1613 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
1614 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
1615 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
1616
1617 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
1618 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
1619 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
1620
1621 put = new Put(row1);
1622 put.add(kv13);
1623 put.add(kv12);
1624 put.add(kv11);
1625 put.add(kv23);
1626 put.add(kv22);
1627 put.add(kv21);
1628 region.put(put);
1629 region.flushcache();
1630
1631
1632 List<KeyValue> expected = new ArrayList<KeyValue>();
1633 expected.add(kv13);
1634 expected.add(kv12);
1635 expected.add(kv23);
1636 expected.add(kv22);
1637
1638 Scan scan = new Scan(row1);
1639 scan.addColumn(fam1, qf1);
1640 scan.addColumn(fam1, qf2);
1641 scan.setMaxVersions(MAX_VERSIONS);
1642 List<KeyValue> actual = new ArrayList<KeyValue>();
1643 InternalScanner scanner = region.getScanner(scan);
1644
1645 boolean hasNext = scanner.next(actual);
1646 assertEquals(false, hasNext);
1647
1648
1649 for(int i=0; i<expected.size(); i++) {
1650 assertEquals(expected.get(i), actual.get(i));
1651 }
1652 }
1653
1654 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions()
1655 throws IOException {
1656 byte [] tableName = Bytes.toBytes("testtable");
1657 byte [] row1 = Bytes.toBytes("row1");
1658 byte [] fam1 = Bytes.toBytes("fam1");
1659 byte [][] families = {fam1};
1660 byte [] qf1 = Bytes.toBytes("qualifier1");
1661 byte [] qf2 = Bytes.toBytes("qualifier2");
1662
1663 long ts1 = 1;
1664 long ts2 = ts1 + 1;
1665 long ts3 = ts1 + 2;
1666 long ts4 = ts1 + 3;
1667
1668
1669 String method = this.getName();
1670 initHRegion(tableName, method, families);
1671
1672
1673 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
1674 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
1675 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
1676 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
1677
1678 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
1679 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
1680 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
1681 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
1682
1683 Put put = null;
1684 put = new Put(row1);
1685 put.add(kv14);
1686 put.add(kv24);
1687 region.put(put);
1688 region.flushcache();
1689
1690 put = new Put(row1);
1691 put.add(kv23);
1692 put.add(kv13);
1693 region.put(put);
1694 region.flushcache();
1695
1696 put = new Put(row1);
1697 put.add(kv22);
1698 put.add(kv12);
1699 region.put(put);
1700 region.flushcache();
1701
1702 put = new Put(row1);
1703 put.add(kv21);
1704 put.add(kv11);
1705 region.put(put);
1706
1707
1708 List<KeyValue> expected = new ArrayList<KeyValue>();
1709 expected.add(kv14);
1710 expected.add(kv13);
1711 expected.add(kv12);
1712 expected.add(kv24);
1713 expected.add(kv23);
1714 expected.add(kv22);
1715
1716 Scan scan = new Scan(row1);
1717 scan.addColumn(fam1, qf1);
1718 scan.addColumn(fam1, qf2);
1719 int versions = 3;
1720 scan.setMaxVersions(versions);
1721 List<KeyValue> actual = new ArrayList<KeyValue>();
1722 InternalScanner scanner = region.getScanner(scan);
1723
1724 boolean hasNext = scanner.next(actual);
1725 assertEquals(false, hasNext);
1726
1727
1728 for(int i=0; i<expected.size(); i++) {
1729 assertEquals(expected.get(i), actual.get(i));
1730 }
1731 }
1732
1733 public void testScanner_Wildcard_FromMemStore_EnforceVersions()
1734 throws IOException {
1735 byte [] tableName = Bytes.toBytes("testtable");
1736 byte [] row1 = Bytes.toBytes("row1");
1737 byte [] qf1 = Bytes.toBytes("qualifier1");
1738 byte [] qf2 = Bytes.toBytes("qualifier2");
1739 byte [] fam1 = Bytes.toBytes("fam1");
1740 byte [][] families = {fam1};
1741
1742 long ts1 = System.currentTimeMillis();
1743 long ts2 = ts1 + 1;
1744 long ts3 = ts1 + 2;
1745
1746
1747 String method = this.getName();
1748 initHRegion(tableName, method, families);
1749
1750
1751 Put put = null;
1752 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
1753 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
1754 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
1755
1756 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
1757 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
1758 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
1759
1760 put = new Put(row1);
1761 put.add(kv13);
1762 put.add(kv12);
1763 put.add(kv11);
1764 put.add(kv23);
1765 put.add(kv22);
1766 put.add(kv21);
1767 region.put(put);
1768
1769
1770 List<KeyValue> expected = new ArrayList<KeyValue>();
1771 expected.add(kv13);
1772 expected.add(kv12);
1773 expected.add(kv23);
1774 expected.add(kv22);
1775
1776 Scan scan = new Scan(row1);
1777 scan.addFamily(fam1);
1778 scan.setMaxVersions(MAX_VERSIONS);
1779 List<KeyValue> actual = new ArrayList<KeyValue>();
1780 InternalScanner scanner = region.getScanner(scan);
1781
1782 boolean hasNext = scanner.next(actual);
1783 assertEquals(false, hasNext);
1784
1785
1786 for(int i=0; i<expected.size(); i++) {
1787 assertEquals(expected.get(i), actual.get(i));
1788 }
1789 }
1790
1791 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions()
1792 throws IOException{
1793 byte [] tableName = Bytes.toBytes("testtable");
1794 byte [] row1 = Bytes.toBytes("row1");
1795 byte [] qf1 = Bytes.toBytes("qualifier1");
1796 byte [] qf2 = Bytes.toBytes("qualifier2");
1797 byte [] fam1 = Bytes.toBytes("fam1");
1798
1799 long ts1 = 1;
1800 long ts2 = ts1 + 1;
1801 long ts3 = ts1 + 2;
1802
1803
1804 String method = this.getName();
1805 initHRegion(tableName, method, fam1);
1806
1807
1808 Put put = null;
1809 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
1810 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
1811 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
1812
1813 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
1814 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
1815 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
1816
1817 put = new Put(row1);
1818 put.add(kv13);
1819 put.add(kv12);
1820 put.add(kv11);
1821 put.add(kv23);
1822 put.add(kv22);
1823 put.add(kv21);
1824 region.put(put);
1825 region.flushcache();
1826
1827
1828 List<KeyValue> expected = new ArrayList<KeyValue>();
1829 expected.add(kv13);
1830 expected.add(kv12);
1831 expected.add(kv23);
1832 expected.add(kv22);
1833
1834 Scan scan = new Scan(row1);
1835 scan.addFamily(fam1);
1836 scan.setMaxVersions(MAX_VERSIONS);
1837 List<KeyValue> actual = new ArrayList<KeyValue>();
1838 InternalScanner scanner = region.getScanner(scan);
1839
1840 boolean hasNext = scanner.next(actual);
1841 assertEquals(false, hasNext);
1842
1843
1844 for(int i=0; i<expected.size(); i++) {
1845 assertEquals(expected.get(i), actual.get(i));
1846 }
1847 }
1848
1849 public void testScanner_StopRow1542() throws IOException {
1850 byte [] tableName = Bytes.toBytes("test_table");
1851 byte [] family = Bytes.toBytes("testFamily");
1852 initHRegion(tableName, getName(), family);
1853
1854 byte [] row1 = Bytes.toBytes("row111");
1855 byte [] row2 = Bytes.toBytes("row222");
1856 byte [] row3 = Bytes.toBytes("row333");
1857 byte [] row4 = Bytes.toBytes("row444");
1858 byte [] row5 = Bytes.toBytes("row555");
1859
1860 byte [] col1 = Bytes.toBytes("Pub111");
1861 byte [] col2 = Bytes.toBytes("Pub222");
1862
1863
1864 Put put = new Put(row1);
1865 put.add(family, col1, Bytes.toBytes(10L));
1866 region.put(put);
1867
1868 put = new Put(row2);
1869 put.add(family, col1, Bytes.toBytes(15L));
1870 region.put(put);
1871
1872 put = new Put(row3);
1873 put.add(family, col2, Bytes.toBytes(20L));
1874 region.put(put);
1875
1876 put = new Put(row4);
1877 put.add(family, col2, Bytes.toBytes(30L));
1878 region.put(put);
1879
1880 put = new Put(row5);
1881 put.add(family, col1, Bytes.toBytes(40L));
1882 region.put(put);
1883
1884 Scan scan = new Scan(row3, row4);
1885 scan.setMaxVersions();
1886 scan.addColumn(family, col1);
1887 InternalScanner s = region.getScanner(scan);
1888
1889 List<KeyValue> results = new ArrayList<KeyValue>();
1890 assertEquals(false, s.next(results));
1891 assertEquals(0, results.size());
1892 }
1893
1894 public void testIncrementColumnValue_UpdatingInPlace() throws IOException {
1895 initHRegion(tableName, getName(), fam1);
1896
1897 long value = 1L;
1898 long amount = 3L;
1899
1900 Put put = new Put(row);
1901 put.add(fam1, qual1, Bytes.toBytes(value));
1902 region.put(put);
1903
1904 long result = region.incrementColumnValue(row, fam1, qual1, amount, true);
1905
1906 assertEquals(value+amount, result);
1907
1908 Store store = region.getStore(fam1);
1909
1910 assertEquals(1, store.memstore.kvset.size());
1911 assertTrue(store.memstore.snapshot.isEmpty());
1912
1913 assertICV(row, fam1, qual1, value+amount);
1914 }
1915
1916 public void testIncrementColumnValue_BumpSnapshot() throws IOException {
1917 ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
1918 EnvironmentEdgeManagerTestHelper.injectEdge(mee);
1919 initHRegion(tableName, getName(), fam1);
1920
1921 long value = 42L;
1922 long incr = 44L;
1923
1924
1925 Put put = new Put(row);
1926 put.add(fam1, qual1, Bytes.toBytes(value));
1927 region.put(put);
1928
1929
1930 Store s = region.getStore(fam1);
1931 s.snapshot();
1932
1933
1934 long newVal = region.incrementColumnValue(row, fam1, qual1,
1935 incr, false);
1936
1937 assertEquals(value+incr, newVal);
1938
1939
1940 Get get = new Get(row);
1941 get.setMaxVersions();
1942 get.addColumn(fam1,qual1);
1943
1944 Result r = region.get(get, null);
1945 assertEquals(2, r.size());
1946 KeyValue first = r.raw()[0];
1947 KeyValue second = r.raw()[1];
1948
1949 assertTrue("ICV failed to upgrade timestamp",
1950 first.getTimestamp() != second.getTimestamp());
1951 }
1952
1953 public void testIncrementColumnValue_ConcurrentFlush() throws IOException {
1954 initHRegion(tableName, getName(), fam1);
1955
1956 long value = 1L;
1957 long amount = 3L;
1958
1959 Put put = new Put(row);
1960 put.add(fam1, qual1, Bytes.toBytes(value));
1961 region.put(put);
1962
1963
1964 Thread t = new Thread() {
1965 public void run() {
1966 try {
1967 region.flushcache();
1968 } catch (IOException e) {
1969 LOG.info("test ICV, got IOE during flushcache()");
1970 }
1971 }
1972 };
1973 t.start();
1974 long r = region.incrementColumnValue(row, fam1, qual1, amount, true);
1975 assertEquals(value+amount, r);
1976
1977
1978 assertICV(row, fam1, qual1, value+amount);
1979 }
1980
1981 public void testIncrementColumnValue_heapSize() throws IOException {
1982 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1983
1984 initHRegion(tableName, getName(), fam1);
1985
1986 long byAmount = 1L;
1987 long size;
1988
1989 for( int i = 0; i < 1000 ; i++) {
1990 region.incrementColumnValue(row, fam1, qual1, byAmount, true);
1991
1992 size = region.memstoreSize.get();
1993 assertTrue("memstore size: " + size, size >= 0);
1994 }
1995 }
1996
1997 public void testIncrementColumnValue_UpdatingInPlace_Negative()
1998 throws IOException {
1999 initHRegion(tableName, getName(), fam1);
2000
2001 long value = 3L;
2002 long amount = -1L;
2003
2004 Put put = new Put(row);
2005 put.add(fam1, qual1, Bytes.toBytes(value));
2006 region.put(put);
2007
2008 long result = region.incrementColumnValue(row, fam1, qual1, amount, true);
2009 assertEquals(value+amount, result);
2010
2011 assertICV(row, fam1, qual1, value+amount);
2012 }
2013
2014 public void testIncrementColumnValue_AddingNew()
2015 throws IOException {
2016 initHRegion(tableName, getName(), fam1);
2017
2018 long value = 1L;
2019 long amount = 3L;
2020
2021 Put put = new Put(row);
2022 put.add(fam1, qual1, Bytes.toBytes(value));
2023 put.add(fam1, qual2, Bytes.toBytes(value));
2024 region.put(put);
2025
2026 long result = region.incrementColumnValue(row, fam1, qual3, amount, true);
2027 assertEquals(amount, result);
2028
2029 Get get = new Get(row);
2030 get.addColumn(fam1, qual3);
2031 Result rr = region.get(get, null);
2032 assertEquals(1, rr.size());
2033
2034
2035 assertICV(row, fam1, qual1, value);
2036 assertICV(row, fam1, qual2, value);
2037 assertICV(row, fam1, qual3, amount);
2038 }
2039
2040 public void testIncrementColumnValue_UpdatingFromSF() throws IOException {
2041 initHRegion(tableName, getName(), fam1);
2042
2043 long value = 1L;
2044 long amount = 3L;
2045
2046 Put put = new Put(row);
2047 put.add(fam1, qual1, Bytes.toBytes(value));
2048 put.add(fam1, qual2, Bytes.toBytes(value));
2049 region.put(put);
2050
2051
2052 region.flushcache();
2053
2054 Store store = region.getStore(fam1);
2055 assertEquals(0, store.memstore.kvset.size());
2056
2057 long r = region.incrementColumnValue(row, fam1, qual1, amount, true);
2058 assertEquals(value+amount, r);
2059
2060 assertICV(row, fam1, qual1, value+amount);
2061 }
2062
2063 public void testIncrementColumnValue_AddingNewAfterSFCheck()
2064 throws IOException {
2065 initHRegion(tableName, getName(), fam1);
2066
2067 long value = 1L;
2068 long amount = 3L;
2069
2070 Put put = new Put(row);
2071 put.add(fam1, qual1, Bytes.toBytes(value));
2072 put.add(fam1, qual2, Bytes.toBytes(value));
2073 region.put(put);
2074 region.flushcache();
2075
2076 Store store = region.getStore(fam1);
2077 assertEquals(0, store.memstore.kvset.size());
2078
2079 long r = region.incrementColumnValue(row, fam1, qual3, amount, true);
2080 assertEquals(amount, r);
2081
2082 assertICV(row, fam1, qual3, amount);
2083
2084 region.flushcache();
2085
2086
2087 assertICV(row, fam1, qual3, amount);
2088 }
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099 public void testIncrementColumnValue_UpdatingInPlace_TimestampClobber() throws IOException {
2100 initHRegion(tableName, getName(), fam1);
2101
2102 long value = 1L;
2103 long amount = 3L;
2104 long now = EnvironmentEdgeManager.currentTimeMillis();
2105 ManualEnvironmentEdge mock = new ManualEnvironmentEdge();
2106 mock.setValue(now);
2107 EnvironmentEdgeManagerTestHelper.injectEdge(mock);
2108
2109
2110 Put put = new Put(row);
2111 put.add(fam1, qual1, now, Bytes.toBytes(value));
2112 region.put(put);
2113
2114 long result = region.incrementColumnValue(row, fam1, qual1, amount, true);
2115
2116 assertEquals(value+amount, result);
2117
2118 Store store = region.getStore(fam1);
2119
2120 assertEquals(1, store.memstore.kvset.size());
2121 assertTrue(store.memstore.snapshot.isEmpty());
2122
2123 assertICV(row, fam1, qual1, value+amount);
2124
2125
2126 put = new Put(row);
2127 put.add(fam1, qual2, now+1, Bytes.toBytes(value));
2128 region.put(put);
2129
2130 result = region.incrementColumnValue(row, fam1, qual2, amount, true);
2131
2132 assertEquals(value+amount, result);
2133
2134 store = region.getStore(fam1);
2135
2136 assertEquals(2, store.memstore.kvset.size());
2137 assertTrue(store.memstore.snapshot.isEmpty());
2138
2139 assertICV(row, fam1, qual2, value+amount);
2140 EnvironmentEdgeManagerTestHelper.reset();
2141 }
2142
2143 private void assertICV(byte [] row,
2144 byte [] familiy,
2145 byte[] qualifier,
2146 long amount) throws IOException {
2147
2148 Get get = new Get(row);
2149 get.addColumn(familiy, qualifier);
2150 Result result = region.get(get, null);
2151 assertEquals(1, result.size());
2152
2153 KeyValue kv = result.raw()[0];
2154 long r = Bytes.toLong(kv.getValue());
2155 assertEquals(amount, r);
2156 }
2157
2158
2159
2160 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions()
2161 throws IOException {
2162 byte [] tableName = Bytes.toBytes("testtable");
2163 byte [] row1 = Bytes.toBytes("row1");
2164 byte [] fam1 = Bytes.toBytes("fam1");
2165 byte [] qf1 = Bytes.toBytes("qualifier1");
2166 byte [] qf2 = Bytes.toBytes("quateslifier2");
2167
2168 long ts1 = 1;
2169 long ts2 = ts1 + 1;
2170 long ts3 = ts1 + 2;
2171 long ts4 = ts1 + 3;
2172
2173
2174 String method = this.getName();
2175 initHRegion(tableName, method, fam1);
2176
2177
2178 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2179 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2180 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2181 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2182
2183 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2184 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2185 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2186 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2187
2188 Put put = null;
2189 put = new Put(row1);
2190 put.add(kv14);
2191 put.add(kv24);
2192 region.put(put);
2193 region.flushcache();
2194
2195 put = new Put(row1);
2196 put.add(kv23);
2197 put.add(kv13);
2198 region.put(put);
2199 region.flushcache();
2200
2201 put = new Put(row1);
2202 put.add(kv22);
2203 put.add(kv12);
2204 region.put(put);
2205 region.flushcache();
2206
2207 put = new Put(row1);
2208 put.add(kv21);
2209 put.add(kv11);
2210 region.put(put);
2211
2212
2213 List<KeyValue> expected = new ArrayList<KeyValue>();
2214 expected.add(kv14);
2215 expected.add(kv13);
2216 expected.add(kv12);
2217 expected.add(kv24);
2218 expected.add(kv23);
2219 expected.add(kv22);
2220
2221 Scan scan = new Scan(row1);
2222 int versions = 3;
2223 scan.setMaxVersions(versions);
2224 List<KeyValue> actual = new ArrayList<KeyValue>();
2225 InternalScanner scanner = region.getScanner(scan);
2226
2227 boolean hasNext = scanner.next(actual);
2228 assertEquals(false, hasNext);
2229
2230
2231 for(int i=0; i<expected.size(); i++) {
2232 assertEquals(expected.get(i), actual.get(i));
2233 }
2234 }
2235
2236
2237
2238
2239
2240
2241
2242
2243 public void testBasicSplit() throws Exception {
2244 byte [] tableName = Bytes.toBytes("testtable");
2245 byte [][] families = {fam1, fam2, fam3};
2246
2247 Configuration hc = initSplit();
2248
2249 String method = this.getName();
2250 initHRegion(tableName, method, hc, families);
2251
2252 try {
2253 LOG.info("" + addContent(region, fam3));
2254 region.flushcache();
2255 byte [] splitRow = region.compactStores();
2256 assertNotNull(splitRow);
2257 LOG.info("SplitRow: " + Bytes.toString(splitRow));
2258 HRegion [] regions = splitRegion(region, splitRow);
2259 try {
2260
2261
2262
2263 for (int i = 0; i < regions.length; i++) {
2264 regions[i] = openClosedRegion(regions[i]);
2265 }
2266
2267
2268 assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
2269 assertGet(regions[1], fam3, splitRow);
2270
2271 assertScan(regions[0], fam3,
2272 Bytes.toBytes(START_KEY));
2273 assertScan(regions[1], fam3, splitRow);
2274
2275 for (int i = 0; i < regions.length; i++) {
2276
2277
2278 for (int j = 0; j < 2; j++) {
2279 addContent(regions[i], fam3);
2280 }
2281 addContent(regions[i], fam2);
2282 addContent(regions[i], fam1);
2283 regions[i].flushcache();
2284 }
2285
2286 byte [][] midkeys = new byte [regions.length][];
2287
2288 for (int i = 0; i < regions.length; i++) {
2289 midkeys[i] = regions[i].compactStores();
2290 }
2291
2292 TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
2293
2294
2295 for (int i = 0; i < regions.length; i++) {
2296 HRegion[] rs = null;
2297 if (midkeys[i] != null) {
2298 rs = splitRegion(regions[i], midkeys[i]);
2299 for (int j = 0; j < rs.length; j++) {
2300 sortedMap.put(Bytes.toString(rs[j].getRegionName()),
2301 openClosedRegion(rs[j]));
2302 }
2303 }
2304 }
2305 LOG.info("Made 4 regions");
2306
2307
2308 int interval = (LAST_CHAR - FIRST_CHAR) / 3;
2309 byte[] b = Bytes.toBytes(START_KEY);
2310 for (HRegion r : sortedMap.values()) {
2311 assertGet(r, fam3, b);
2312 b[0] += interval;
2313 }
2314 } finally {
2315 for (int i = 0; i < regions.length; i++) {
2316 try {
2317 regions[i].close();
2318 } catch (IOException e) {
2319
2320 }
2321 }
2322 }
2323 } finally {
2324 if (region != null) {
2325 region.close();
2326 region.getLog().closeAndDelete();
2327 }
2328 }
2329 }
2330
2331 public void testSplitRegion() throws IOException {
2332 byte [] tableName = Bytes.toBytes("testtable");
2333 byte [] qualifier = Bytes.toBytes("qualifier");
2334 Configuration hc = initSplit();
2335 int numRows = 10;
2336 byte [][] families = {fam1, fam3};
2337
2338
2339 String method = this.getName();
2340 initHRegion(tableName, method, hc, families);
2341
2342
2343 int startRow = 100;
2344 putData(startRow, numRows, qualifier, families);
2345 int splitRow = startRow + numRows;
2346 putData(splitRow, numRows, qualifier, families);
2347 region.flushcache();
2348
2349 HRegion [] regions = null;
2350 try {
2351 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
2352
2353 for (int i = 0; i < regions.length; i++) {
2354 regions[i] = openClosedRegion(regions[i]);
2355 }
2356
2357 assertEquals(2, regions.length);
2358
2359
2360
2361 verifyData(regions[0], startRow, numRows, qualifier, families);
2362 verifyData(regions[1], splitRow, numRows, qualifier, families);
2363
2364 } finally {
2365 if (region != null) {
2366 region.close();
2367 region.getLog().closeAndDelete();
2368 }
2369 }
2370 }
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380 public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
2381 byte[] tableName = Bytes.toBytes("testFlushCacheWhileScanning");
2382 byte[] family = Bytes.toBytes("family");
2383 int numRows = 1000;
2384 int flushAndScanInterval = 10;
2385 int compactInterval = 10 * flushAndScanInterval;
2386
2387 String method = "testFlushCacheWhileScanning";
2388 initHRegion(tableName,method, family);
2389 FlushThread flushThread = new FlushThread();
2390 flushThread.start();
2391
2392 Scan scan = new Scan();
2393 scan.addFamily(family);
2394 scan.setFilter(new SingleColumnValueFilter(family, qual1,
2395 CompareFilter.CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(5L))));
2396
2397 int expectedCount = 0;
2398 List<KeyValue> res = new ArrayList<KeyValue>();
2399
2400 boolean toggle=true;
2401 for (long i = 0; i < numRows; i++) {
2402 Put put = new Put(Bytes.toBytes(i));
2403 put.add(family, qual1, Bytes.toBytes(i % 10));
2404 region.put(put);
2405
2406 if (i != 0 && i % compactInterval == 0) {
2407
2408 region.compactStores(true);
2409 }
2410
2411 if (i % 10 == 5L) {
2412 expectedCount++;
2413 }
2414
2415 if (i != 0 && i % flushAndScanInterval == 0) {
2416 res.clear();
2417 InternalScanner scanner = region.getScanner(scan);
2418 if (toggle) {
2419 flushThread.flush();
2420 }
2421 while (scanner.next(res)) ;
2422 if (!toggle) {
2423 flushThread.flush();
2424 }
2425 assertEquals("i=" + i, expectedCount, res.size());
2426 toggle = !toggle;
2427 }
2428 }
2429
2430 flushThread.done();
2431 flushThread.join();
2432 flushThread.checkNoError();
2433 }
2434
2435 protected class FlushThread extends Thread {
2436 private volatile boolean done;
2437 private Throwable error = null;
2438
2439 public void done() {
2440 done = true;
2441 synchronized (this) {
2442 interrupt();
2443 }
2444 }
2445
2446 public void checkNoError() {
2447 if (error != null) {
2448 assertNull(error);
2449 }
2450 }
2451
2452 @Override
2453 public void run() {
2454 done = false;
2455 while (!done) {
2456 synchronized (this) {
2457 try {
2458 wait();
2459 } catch (InterruptedException ignored) {
2460 if (done) {
2461 break;
2462 }
2463 }
2464 }
2465 try {
2466 region.flushcache();
2467 } catch (IOException e) {
2468 if (!done) {
2469 LOG.error("Error while flusing cache", e);
2470 error = e;
2471 }
2472 break;
2473 }
2474 }
2475
2476 }
2477
2478 public void flush() {
2479 synchronized (this) {
2480 notify();
2481 }
2482
2483 }
2484 }
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494 public void testWritesWhileScanning()
2495 throws IOException, InterruptedException {
2496 byte[] tableName = Bytes.toBytes("testWritesWhileScanning");
2497 int testCount = 100;
2498 int numRows = 1;
2499 int numFamilies = 10;
2500 int numQualifiers = 100;
2501 int flushInterval = 7;
2502 int compactInterval = 5 * flushInterval;
2503 byte[][] families = new byte[numFamilies][];
2504 for (int i = 0; i < numFamilies; i++) {
2505 families[i] = Bytes.toBytes("family" + i);
2506 }
2507 byte[][] qualifiers = new byte[numQualifiers][];
2508 for (int i = 0; i < numQualifiers; i++) {
2509 qualifiers[i] = Bytes.toBytes("qual" + i);
2510 }
2511
2512 String method = "testWritesWhileScanning";
2513 initHRegion(tableName, method, families);
2514 PutThread putThread = new PutThread(numRows, families, qualifiers);
2515 putThread.start();
2516 putThread.waitForFirstPut();
2517
2518 FlushThread flushThread = new FlushThread();
2519 flushThread.start();
2520
2521 Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
2522
2523
2524
2525 int expectedCount = numFamilies * numQualifiers;
2526 List<KeyValue> res = new ArrayList<KeyValue>();
2527
2528 long prevTimestamp = 0L;
2529 for (int i = 0; i < testCount; i++) {
2530
2531 if (i != 0 && i % compactInterval == 0) {
2532 region.compactStores(true);
2533 }
2534
2535 if (i != 0 && i % flushInterval == 0) {
2536
2537 flushThread.flush();
2538 }
2539
2540 boolean previousEmpty = res.isEmpty();
2541 res.clear();
2542 InternalScanner scanner = region.getScanner(scan);
2543 while (scanner.next(res)) ;
2544 if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
2545 assertEquals("i=" + i, expectedCount, res.size());
2546 long timestamp = res.get(0).getTimestamp();
2547 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
2548 timestamp >= prevTimestamp);
2549 prevTimestamp = timestamp;
2550 }
2551 }
2552
2553 putThread.done();
2554
2555 region.flushcache();
2556
2557 putThread.join();
2558 putThread.checkNoError();
2559
2560 flushThread.done();
2561 flushThread.join();
2562 flushThread.checkNoError();
2563 }
2564
2565 protected class PutThread extends Thread {
2566 private volatile boolean done;
2567 private volatile int numPutsFinished = 0;
2568
2569 private Throwable error = null;
2570 private int numRows;
2571 private byte[][] families;
2572 private byte[][] qualifiers;
2573
2574 private PutThread(int numRows, byte[][] families,
2575 byte[][] qualifiers) {
2576 this.numRows = numRows;
2577 this.families = families;
2578 this.qualifiers = qualifiers;
2579 }
2580
2581
2582
2583
2584 public void waitForFirstPut() throws InterruptedException {
2585
2586 while (numPutsFinished == 0) {
2587 checkNoError();
2588 Thread.sleep(50);
2589 }
2590 }
2591
2592 public void done() {
2593 done = true;
2594 synchronized (this) {
2595 interrupt();
2596 }
2597 }
2598
2599 public void checkNoError() {
2600 if (error != null) {
2601 assertNull(error);
2602 }
2603 }
2604
2605 @Override
2606 public void run() {
2607 done = false;
2608 while (!done) {
2609 try {
2610 for (int r = 0; r < numRows; r++) {
2611 byte[] row = Bytes.toBytes("row" + r);
2612 Put put = new Put(row);
2613 for (byte[] family : families) {
2614 for (byte[] qualifier : qualifiers) {
2615 put.add(family, qualifier, (long) numPutsFinished,
2616 Bytes.toBytes(numPutsFinished));
2617 }
2618 }
2619
2620 region.put(put);
2621 numPutsFinished++;
2622 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
2623 System.out.println("put iteration = " + numPutsFinished);
2624 Delete delete = new Delete(row, (long)numPutsFinished-30, null);
2625 region.delete(delete, null, true);
2626 }
2627 numPutsFinished++;
2628 }
2629 } catch (IOException e) {
2630 LOG.error("error while putting records", e);
2631 error = e;
2632 break;
2633 }
2634 }
2635
2636 }
2637
2638 }
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649 public void testWritesWhileGetting()
2650 throws IOException, InterruptedException {
2651 byte[] tableName = Bytes.toBytes("testWritesWhileGetting");
2652 int testCount = 100;
2653 int numRows = 1;
2654 int numFamilies = 10;
2655 int numQualifiers = 100;
2656 int flushInterval = 10;
2657 int compactInterval = 10 * flushInterval;
2658 byte[][] families = new byte[numFamilies][];
2659 for (int i = 0; i < numFamilies; i++) {
2660 families[i] = Bytes.toBytes("family" + i);
2661 }
2662 byte[][] qualifiers = new byte[numQualifiers][];
2663 for (int i = 0; i < numQualifiers; i++) {
2664 qualifiers[i] = Bytes.toBytes("qual" + i);
2665 }
2666
2667 String method = "testWritesWhileGetting";
2668 initHRegion(tableName, method, families);
2669 PutThread putThread = new PutThread(numRows, families, qualifiers);
2670 putThread.start();
2671 putThread.waitForFirstPut();
2672
2673 FlushThread flushThread = new FlushThread();
2674 flushThread.start();
2675
2676 Get get = new Get(Bytes.toBytes("row0"));
2677 Result result = null;
2678
2679 int expectedCount = numFamilies * numQualifiers;
2680
2681 long prevTimestamp = 0L;
2682 for (int i = 0; i < testCount; i++) {
2683
2684 if (i != 0 && i % compactInterval == 0) {
2685 region.compactStores(true);
2686 }
2687
2688 if (i != 0 && i % flushInterval == 0) {
2689
2690 flushThread.flush();
2691 }
2692
2693 boolean previousEmpty = result == null || result.isEmpty();
2694 result = region.get(get, null);
2695 if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
2696 assertEquals("i=" + i, expectedCount, result.size());
2697
2698
2699 long timestamp = 0;
2700 for (KeyValue kv : result.sorted()) {
2701 if (Bytes.equals(kv.getFamily(), families[0])
2702 && Bytes.equals(kv.getQualifier(), qualifiers[0])) {
2703 timestamp = kv.getTimestamp();
2704 }
2705 }
2706 assertTrue(timestamp >= prevTimestamp);
2707 prevTimestamp = timestamp;
2708
2709 byte [] gotValue = null;
2710 for (KeyValue kv : result.raw()) {
2711 byte [] thisValue = kv.getValue();
2712 if (gotValue != null) {
2713 assertEquals(gotValue, thisValue);
2714 }
2715 gotValue = thisValue;
2716 }
2717 }
2718 }
2719
2720 putThread.done();
2721
2722 region.flushcache();
2723
2724 putThread.join();
2725 putThread.checkNoError();
2726
2727 flushThread.done();
2728 flushThread.join();
2729 flushThread.checkNoError();
2730 }
2731
2732
2733 public void testIndexesScanWithOneDeletedRow() throws IOException {
2734 byte[] tableName = Bytes.toBytes("testIndexesScanWithOneDeletedRow");
2735 byte[] family = Bytes.toBytes("family");
2736
2737
2738 String method = "testIndexesScanWithOneDeletedRow";
2739 initHRegion(tableName, method, HBaseConfiguration.create(), family);
2740
2741 Put put = new Put(Bytes.toBytes(1L));
2742 put.add(family, qual1, 1L, Bytes.toBytes(1L));
2743 region.put(put);
2744
2745 region.flushcache();
2746
2747 Delete delete = new Delete(Bytes.toBytes(1L), 1L, null);
2748
2749 region.delete(delete, null, true);
2750
2751 put = new Put(Bytes.toBytes(2L));
2752 put.add(family, qual1, 2L, Bytes.toBytes(2L));
2753 region.put(put);
2754
2755 Scan idxScan = new Scan();
2756 idxScan.addFamily(family);
2757 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,
2758 Arrays.<Filter>asList(new SingleColumnValueFilter(family, qual1,
2759 CompareFilter.CompareOp.GREATER_OR_EQUAL,
2760 new BinaryComparator(Bytes.toBytes(0L))),
2761 new SingleColumnValueFilter(family, qual1,
2762 CompareFilter.CompareOp.LESS_OR_EQUAL,
2763 new BinaryComparator(Bytes.toBytes(3L)))
2764 )));
2765 InternalScanner scanner = region.getScanner(idxScan);
2766 List<KeyValue> res = new ArrayList<KeyValue>();
2767
2768
2769 while (scanner.next(res)) ;
2770
2771
2772 assertEquals(1L, res.size());
2773
2774 }
2775
2776
2777
2778
2779 public void testBloomFilterSize() throws IOException {
2780 byte [] tableName = Bytes.toBytes("testBloomFilterSize");
2781 byte [] row1 = Bytes.toBytes("row1");
2782 byte [] fam1 = Bytes.toBytes("fam1");
2783 byte [] qf1 = Bytes.toBytes("col");
2784 byte [] val1 = Bytes.toBytes("value1");
2785
2786 HColumnDescriptor hcd = new HColumnDescriptor(fam1, Integer.MAX_VALUE,
2787 HColumnDescriptor.DEFAULT_COMPRESSION, false, true,
2788 HColumnDescriptor.DEFAULT_TTL, "rowcol");
2789
2790 HTableDescriptor htd = new HTableDescriptor(tableName);
2791 htd.addFamily(hcd);
2792 HRegionInfo info = new HRegionInfo(htd, null, null, false);
2793 Path path = new Path(DIR + "testBloomFilterSize");
2794 region = HRegion.createHRegion(info, path, conf);
2795
2796 int num_unique_rows = 10;
2797 int duplicate_multiplier =2;
2798 int num_storefiles = 4;
2799
2800 int version = 0;
2801 for (int f =0 ; f < num_storefiles; f++) {
2802 for (int i = 0; i < duplicate_multiplier; i ++) {
2803 for (int j = 0; j < num_unique_rows; j++) {
2804 Put put = new Put(Bytes.toBytes("row" + j));
2805 put.add(fam1, qf1, version++, val1);
2806 region.put(put);
2807 }
2808 }
2809 region.flushcache();
2810 }
2811
2812 Store store = region.getStore(fam1);
2813 List<StoreFile> storeFiles = store.getStorefiles();
2814 for (StoreFile storefile : storeFiles) {
2815 StoreFile.Reader reader = storefile.getReader();
2816 reader.loadFileInfo();
2817 reader.loadBloomfilter();
2818 assertEquals(num_unique_rows*duplicate_multiplier, reader.getEntries());
2819 assertEquals(num_unique_rows, reader.getFilterEntries());
2820 }
2821
2822 region.compactStores(true);
2823
2824
2825 storeFiles = store.getStorefiles();
2826 for (StoreFile storefile : storeFiles) {
2827 StoreFile.Reader reader = storefile.getReader();
2828 reader.loadFileInfo();
2829 reader.loadBloomfilter();
2830 assertEquals(num_unique_rows*duplicate_multiplier*num_storefiles,
2831 reader.getEntries());
2832 assertEquals(num_unique_rows, reader.getFilterEntries());
2833 }
2834 }
2835
2836 public void testAllColumnsWithBloomFilter() throws IOException {
2837 byte [] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
2838 byte [] FAMILY = Bytes.toBytes("family");
2839
2840
2841 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY, Integer.MAX_VALUE,
2842 HColumnDescriptor.DEFAULT_COMPRESSION,
2843 HColumnDescriptor.DEFAULT_IN_MEMORY,
2844 HColumnDescriptor.DEFAULT_BLOCKCACHE,
2845 Integer.MAX_VALUE, HColumnDescriptor.DEFAULT_TTL,
2846 "rowcol",
2847 HColumnDescriptor.DEFAULT_REPLICATION_SCOPE);
2848 HTableDescriptor htd = new HTableDescriptor(TABLE);
2849 htd.addFamily(hcd);
2850 HRegionInfo info = new HRegionInfo(htd, null, null, false);
2851 Path path = new Path(DIR + "testAllColumnsWithBloomFilter");
2852 region = HRegion.createHRegion(info, path, conf);
2853
2854
2855 byte row[] = Bytes.toBytes("row:" + 0);
2856 byte column[] = Bytes.toBytes("column:" + 0);
2857 Put put = new Put(row);
2858 for (long idx = 1; idx <= 4; idx++) {
2859 put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
2860 }
2861 region.put(put);
2862
2863
2864 region.flushcache();
2865
2866
2867 Get get = new Get(row);
2868 get.setMaxVersions();
2869 KeyValue[] kvs = region.get(get, null).raw();
2870
2871
2872 assertEquals(4, kvs.length);
2873 checkOneCell(kvs[0], FAMILY, 0, 0, 4);
2874 checkOneCell(kvs[1], FAMILY, 0, 0, 3);
2875 checkOneCell(kvs[2], FAMILY, 0, 0, 2);
2876 checkOneCell(kvs[3], FAMILY, 0, 0, 1);
2877 }
2878
2879
2880
2881
2882
2883
2884 public void testDeleteRowWithBloomFilter() throws IOException {
2885 byte [] tableName = Bytes.toBytes("testDeleteRowWithBloomFilter");
2886 byte [] familyName = Bytes.toBytes("familyName");
2887
2888
2889 HColumnDescriptor hcd = new HColumnDescriptor(familyName, Integer.MAX_VALUE,
2890 HColumnDescriptor.DEFAULT_COMPRESSION, false, true,
2891 HColumnDescriptor.DEFAULT_TTL, "rowcol");
2892
2893 HTableDescriptor htd = new HTableDescriptor(tableName);
2894 htd.addFamily(hcd);
2895 HRegionInfo info = new HRegionInfo(htd, null, null, false);
2896 Path path = new Path(DIR + "TestDeleteRowWithBloomFilter");
2897 region = HRegion.createHRegion(info, path, conf);
2898
2899
2900 byte row[] = Bytes.toBytes("row1");
2901 byte col[] = Bytes.toBytes("col1");
2902
2903 Put put = new Put(row);
2904 put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
2905 region.put(put);
2906 region.flushcache();
2907
2908 Delete del = new Delete(row);
2909 region.delete(del, null, true);
2910 region.flushcache();
2911
2912
2913 Get get = new Get(row);
2914 get.addColumn(familyName, col);
2915
2916 KeyValue[] keyValues = region.get(get, null).raw();
2917 assertTrue(keyValues.length == 0);
2918 }
2919
2920 private void putData(int startRow, int numRows, byte [] qf,
2921 byte [] ...families)
2922 throws IOException {
2923 for(int i=startRow; i<startRow+numRows; i++) {
2924 Put put = new Put(Bytes.toBytes("" + i));
2925 for(byte [] family : families) {
2926 put.add(family, qf, null);
2927 }
2928 region.put(put);
2929 }
2930 }
2931
2932 private void verifyData(HRegion newReg, int startRow, int numRows, byte [] qf,
2933 byte [] ... families)
2934 throws IOException {
2935 for(int i=startRow; i<startRow + numRows; i++) {
2936 byte [] row = Bytes.toBytes("" + i);
2937 Get get = new Get(row);
2938 for(byte [] family : families) {
2939 get.addColumn(family, qf);
2940 }
2941 Result result = newReg.get(get, null);
2942 KeyValue [] raw = result.sorted();
2943 assertEquals(families.length, result.size());
2944 for(int j=0; j<families.length; j++) {
2945 assertEquals(0, Bytes.compareTo(row, raw[j].getRow()));
2946 assertEquals(0, Bytes.compareTo(families[j], raw[j].getFamily()));
2947 assertEquals(0, Bytes.compareTo(qf, raw[j].getQualifier()));
2948 }
2949 }
2950 }
2951
2952 private void assertGet(final HRegion r, final byte [] family, final byte [] k)
2953 throws IOException {
2954
2955 Get get = new Get(k).addFamily(family).setMaxVersions();
2956 KeyValue [] results = r.get(get, null).raw();
2957 for (int j = 0; j < results.length; j++) {
2958 byte [] tmp = results[j].getValue();
2959
2960 assertTrue(Bytes.equals(k, tmp));
2961 }
2962 }
2963
2964
2965
2966
2967
2968
2969
2970
2971 private void assertScan(final HRegion r, final byte [] fs,
2972 final byte [] firstValue)
2973 throws IOException {
2974 byte [][] families = {fs};
2975 Scan scan = new Scan();
2976 for (int i = 0; i < families.length; i++) scan.addFamily(families[i]);
2977 InternalScanner s = r.getScanner(scan);
2978 try {
2979 List<KeyValue> curVals = new ArrayList<KeyValue>();
2980 boolean first = true;
2981 OUTER_LOOP: while(s.next(curVals)) {
2982 for (KeyValue kv: curVals) {
2983 byte [] val = kv.getValue();
2984 byte [] curval = val;
2985 if (first) {
2986 first = false;
2987 assertTrue(Bytes.compareTo(curval, firstValue) == 0);
2988 } else {
2989
2990 break OUTER_LOOP;
2991 }
2992 }
2993 }
2994 } finally {
2995 s.close();
2996 }
2997 }
2998
2999 private Configuration initSplit() {
3000 Configuration conf = HBaseConfiguration.create();
3001
3002 conf.setInt("hbase.hstore.compactionThreshold", 2);
3003
3004
3005 conf.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
3006
3007 conf.setInt(HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, 10 * 1000);
3008
3009
3010 conf.setLong("hbase.client.pause", 15 * 1000);
3011
3012
3013
3014 conf.setLong("hbase.hregion.max.filesize", 1024 * 128);
3015 return conf;
3016 }
3017
3018 private void initHRegion (byte [] tableName, String callingMethod,
3019 byte[] ... families)
3020 throws IOException {
3021 initHRegion(tableName, callingMethod, HBaseConfiguration.create(), families);
3022 }
3023
3024 private void initHRegion (byte [] tableName, String callingMethod,
3025 Configuration conf, byte [] ... families)
3026 throws IOException{
3027 HTableDescriptor htd = new HTableDescriptor(tableName);
3028 for(byte [] family : families) {
3029 htd.addFamily(new HColumnDescriptor(family));
3030 }
3031 HRegionInfo info = new HRegionInfo(htd, null, null, false);
3032 Path path = new Path(DIR + callingMethod);
3033 if (fs.exists(path)) {
3034 if (!fs.delete(path, true)) {
3035 throw new IOException("Failed delete of " + path);
3036 }
3037 }
3038 region = HRegion.createHRegion(info, path, conf);
3039 }
3040
3041
3042
3043
3044
3045 private void checkOneCell(KeyValue kv, byte[] cf,
3046 int rowIdx, int colIdx, long ts) {
3047 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
3048 assertEquals("Row mismatch which checking: " + ctx,
3049 "row:"+ rowIdx, Bytes.toString(kv.getRow()));
3050 assertEquals("ColumnFamily mismatch while checking: " + ctx,
3051 Bytes.toString(cf), Bytes.toString(kv.getFamily()));
3052 assertEquals("Column qualifier mismatch while checking: " + ctx,
3053 "column:" + colIdx, Bytes.toString(kv.getQualifier()));
3054 assertEquals("Timestamp mismatch while checking: " + ctx,
3055 ts, kv.getTimestamp());
3056 assertEquals("Value mismatch while checking: " + ctx,
3057 "value-version-" + ts, Bytes.toString(kv.getValue()));
3058 }
3059
3060 }