1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver;
19
20 import java.io.IOException;
21 import java.util.ArrayList;
22 import java.util.List;
23
24 import org.apache.hadoop.hbase.HBaseTestCase;
25 import org.apache.hadoop.hbase.HConstants;
26 import org.apache.hadoop.hbase.HTableDescriptor;
27 import org.apache.hadoop.hbase.KeyValue;
28 import org.apache.hadoop.hbase.SmallTests;
29 import org.apache.hadoop.hbase.client.Delete;
30 import org.apache.hadoop.hbase.client.Get;
31 import org.apache.hadoop.hbase.client.Put;
32 import org.apache.hadoop.hbase.client.Result;
33 import org.apache.hadoop.hbase.client.Scan;
34 import org.apache.hadoop.hbase.util.Bytes;
35 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
36 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
37 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
38 import org.junit.experimental.categories.Category;
39
40 @Category(SmallTests.class)
41 public class TestKeepDeletes extends HBaseTestCase {
42 private final byte[] T0 = Bytes.toBytes("0");
43 private final byte[] T1 = Bytes.toBytes("1");
44 private final byte[] T2 = Bytes.toBytes("2");
45 private final byte[] T3 = Bytes.toBytes("3");
46 private final byte[] T4 = Bytes.toBytes("4");
47 private final byte[] T5 = Bytes.toBytes("5");
48 private final byte[] T6 = Bytes.toBytes("6");
49
50 private final byte[] c0 = COLUMNS[0];
51 private final byte[] c1 = COLUMNS[1];
52
53 @Override
54 protected void setUp() throws Exception {
55 super.setUp();
56
57
58
59
60
61
62
63
64
65
66 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
67 }
68
69 @Override
70 protected void tearDown() throws Exception {
71 super.tearDown();
72 EnvironmentEdgeManager.reset();
73 }
74
75
76
77
78
79
80
81 public void testBasicScenario() throws Exception {
82
83 HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
84 HConstants.FOREVER, true);
85 HRegion region = createNewHRegion(htd, null, null);
86
87 long ts = EnvironmentEdgeManager.currentTimeMillis();
88 Put p = new Put(T1, ts);
89 p.add(c0, c0, T1);
90 region.put(p);
91 p = new Put(T1, ts+1);
92 p.add(c0, c0, T2);
93 region.put(p);
94 p = new Put(T1, ts+2);
95 p.add(c0, c0, T3);
96 region.put(p);
97 p = new Put(T1, ts+4);
98 p.add(c0, c0, T4);
99 region.put(p);
100
101
102 Delete d = new Delete(T1, ts+2);
103 region.delete(d);
104
105
106
107 assertEquals(3, countDeleteMarkers(region));
108
109
110 Get g = new Get(T1);
111 g.setMaxVersions();
112 g.setTimeRange(0L, ts+2);
113 Result r = region.get(g);
114 checkResult(r, c0, c0, T2,T1);
115
116
117 region.flushcache();
118
119
120 r = region.get(g);
121 checkResult(r, c0, c0, T2);
122
123
124 region.compactStores(true);
125 region.compactStores(true);
126
127
128
129 assertEquals(1, countDeleteMarkers(region));
130
131
132 r = region.get(g);
133 checkResult(r, c0, c0, T2);
134
135
136 g.setTimeRange(0L, ts+4);
137 r = region.get(g);
138 assertTrue(r.isEmpty());
139
140
141 p = new Put(T1, ts+5);
142 p.add(c0, c0, T5);
143 region.put(p);
144 p = new Put(T1, ts+6);
145 p.add(c0, c0, T6);
146 region.put(p);
147
148
149
150 p = new Put(T1, ts);
151 p.add(c0, c0, T1);
152 region.put(p);
153 r = region.get(g);
154 assertTrue(r.isEmpty());
155
156 region.flushcache();
157 region.compactStores(true);
158 region.compactStores(true);
159
160
161 region.put(p);
162 r = region.get(g);
163 checkResult(r, c0, c0, T1);
164 assertEquals(0, countDeleteMarkers(region));
165
166 HRegion.closeHRegion(region);
167 }
168
169
170
171
172
173
174
175
176
177 public void testRawScanWithoutKeepingDeletes() throws Exception {
178
179 HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
180 HConstants.FOREVER, false);
181 HRegion region = createNewHRegion(htd, null, null);
182
183 long ts = EnvironmentEdgeManager.currentTimeMillis();
184 Put p = new Put(T1, ts);
185 p.add(c0, c0, T1);
186 region.put(p);
187
188 Delete d = new Delete(T1, ts);
189 d.deleteColumn(c0, c0, ts);
190 region.delete(d);
191
192
193 Scan s = new Scan();
194 s.setRaw(true);
195 s.setMaxVersions();
196 InternalScanner scan = region.getScanner(s);
197 List<KeyValue> kvs = new ArrayList<KeyValue>();
198 scan.next(kvs);
199 assertEquals(2, kvs.size());
200
201 region.flushcache();
202 region.compactStores(true);
203
204
205
206
207 s = new Scan();
208 s.setRaw(true);
209 s.setMaxVersions();
210 scan = region.getScanner(s);
211 kvs = new ArrayList<KeyValue>();
212 scan.next(kvs);
213 assertTrue(kvs.isEmpty());
214
215 HRegion.closeHRegion(region);
216 }
217
218
219
220
221 public void testWithoutKeepingDeletes() throws Exception {
222
223 HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
224 HConstants.FOREVER, false);
225 HRegion region = createNewHRegion(htd, null, null);
226
227 long ts = EnvironmentEdgeManager.currentTimeMillis();
228 Put p = new Put(T1, ts);
229 p.add(c0, c0, T1);
230 region.put(p);
231 Delete d = new Delete(T1, ts+2);
232 d.deleteColumn(c0, c0, ts);
233 region.delete(d);
234
235
236 Get g = new Get(T1);
237 g.setMaxVersions();
238 g.setTimeRange(0L, ts+1);
239 Result r = region.get(g);
240 assertTrue(r.isEmpty());
241
242
243 Scan s = new Scan();
244 s.setMaxVersions();
245 s.setTimeRange(0L, ts+1);
246 InternalScanner scanner = region.getScanner(s);
247 List<KeyValue> kvs = new ArrayList<KeyValue>();
248 while(scanner.next(kvs));
249 assertTrue(kvs.isEmpty());
250
251
252 region.flushcache();
253 region.compactStores();
254 assertEquals(1, countDeleteMarkers(region));
255 region.compactStores(true);
256
257 assertEquals(0, countDeleteMarkers(region));
258
259 HRegion.closeHRegion(region);
260 }
261
262
263
264
265 public void testRawScanWithColumns() throws Exception {
266 HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
267 HConstants.FOREVER, true);
268 HRegion region = createNewHRegion(htd, null, null);
269
270 Scan s = new Scan();
271 s.setRaw(true);
272 s.setMaxVersions();
273 s.addColumn(c0, c0);
274
275 try {
276 InternalScanner scan = region.getScanner(s);
277 fail("raw scanner with columns should have failed");
278 } catch (org.apache.hadoop.hbase.DoNotRetryIOException dnre) {
279
280 }
281
282 HRegion.closeHRegion(region);
283 }
284
285
286
287
288 public void testRawScan() throws Exception {
289 HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
290 HConstants.FOREVER, true);
291 HRegion region = createNewHRegion(htd, null, null);
292
293 long ts = EnvironmentEdgeManager.currentTimeMillis();
294 Put p = new Put(T1, ts);
295 p.add(c0, c0, T1);
296 region.put(p);
297 p = new Put(T1, ts+2);
298 p.add(c0, c0, T2);
299 region.put(p);
300 p = new Put(T1, ts+4);
301 p.add(c0, c0, T3);
302 region.put(p);
303
304 Delete d = new Delete(T1, ts+1);
305 region.delete(d);
306
307 d = new Delete(T1, ts+2);
308 d.deleteColumn(c0, c0, ts+2);
309 region.delete(d);
310
311 d = new Delete(T1, ts+3);
312 d.deleteColumns(c0, c0, ts+3);
313 region.delete(d);
314
315 Scan s = new Scan();
316 s.setRaw(true);
317 s.setMaxVersions();
318 InternalScanner scan = region.getScanner(s);
319 List<KeyValue> kvs = new ArrayList<KeyValue>();
320 scan.next(kvs);
321 assertEquals(8, kvs.size());
322 assertTrue(kvs.get(0).isDeleteFamily());
323 assertEquals(kvs.get(1).getValue(), T3);
324 assertTrue(kvs.get(2).isDelete());
325 assertTrue(kvs.get(3).isDeleteType());
326 assertEquals(kvs.get(4).getValue(), T2);
327 assertEquals(kvs.get(5).getValue(), T1);
328
329 assertTrue(kvs.get(6).isDeleteFamily());
330 assertTrue(kvs.get(7).isDeleteFamily());
331
332
333 s = new Scan();
334 s.setRaw(true);
335 s.setMaxVersions();
336 s.setTimeRange(0, 1);
337 scan = region.getScanner(s);
338 kvs = new ArrayList<KeyValue>();
339 scan.next(kvs);
340
341 assertTrue(kvs.isEmpty());
342
343
344 s = new Scan();
345 s.setRaw(true);
346 s.setMaxVersions();
347 s.setTimeRange(0, ts+2);
348 scan = region.getScanner(s);
349 kvs = new ArrayList<KeyValue>();
350 scan.next(kvs);
351 assertEquals(4, kvs.size());
352 assertTrue(kvs.get(0).isDeleteFamily());
353 assertEquals(kvs.get(1).getValue(), T1);
354
355 assertTrue(kvs.get(2).isDeleteFamily());
356 assertTrue(kvs.get(3).isDeleteFamily());
357
358
359 s = new Scan();
360 s.setRaw(true);
361 s.setMaxVersions();
362 s.setTimeRange(ts+3, ts+5);
363 scan = region.getScanner(s);
364 kvs = new ArrayList<KeyValue>();
365 scan.next(kvs);
366 assertEquals(2, kvs.size());
367 assertEquals(kvs.get(0).getValue(), T3);
368 assertTrue(kvs.get(1).isDelete());
369
370
371 HRegion.closeHRegion(region);
372 }
373
374
375
376
377 public void testDeleteMarkerExpirationEmptyStore() throws Exception {
378 HTableDescriptor htd = createTableDescriptor(getName(), 0, 1,
379 HConstants.FOREVER, true);
380 HRegion region = createNewHRegion(htd, null, null);
381
382 long ts = EnvironmentEdgeManager.currentTimeMillis();
383
384 Delete d = new Delete(T1, ts);
385 d.deleteColumns(c0, c0, ts);
386 region.delete(d);
387
388 d = new Delete(T1, ts);
389 d.deleteFamily(c0);
390 region.delete(d);
391
392 d = new Delete(T1, ts);
393 d.deleteColumn(c0, c0, ts+1);
394 region.delete(d);
395
396 d = new Delete(T1, ts);
397 d.deleteColumn(c0, c0, ts+2);
398 region.delete(d);
399
400
401 assertEquals(4, countDeleteMarkers(region));
402
403
404 region.flushcache();
405 assertEquals(4, countDeleteMarkers(region));
406 region.compactStores(false);
407 assertEquals(4, countDeleteMarkers(region));
408
409
410 region.compactStores(true);
411 assertEquals(0, countDeleteMarkers(region));
412
413 HRegion.closeHRegion(region);
414 }
415
416
417
418
419 public void testDeleteMarkerExpiration() throws Exception {
420 HTableDescriptor htd = createTableDescriptor(getName(), 0, 1,
421 HConstants.FOREVER, true);
422 HRegion region = createNewHRegion(htd, null, null);
423
424 long ts = EnvironmentEdgeManager.currentTimeMillis();
425
426 Put p = new Put(T1, ts);
427 p.add(c0, c0, T1);
428 region.put(p);
429
430
431 p = new Put(T1, ts-10);
432 p.add(c1, c0, T1);
433 region.put(p);
434
435
436 Delete d = new Delete(T1, ts);
437 d.deleteColumns(c0, c0, ts);
438 region.delete(d);
439
440 d = new Delete(T1, ts);
441 d.deleteFamily(c0, ts);
442 region.delete(d);
443
444 d = new Delete(T1, ts);
445 d.deleteColumn(c0, c0, ts+1);
446 region.delete(d);
447
448 d = new Delete(T1, ts);
449 d.deleteColumn(c0, c0, ts+2);
450 region.delete(d);
451
452
453 assertEquals(4, countDeleteMarkers(region));
454
455 region.flushcache();
456 assertEquals(4, countDeleteMarkers(region));
457 region.compactStores(false);
458 assertEquals(4, countDeleteMarkers(region));
459
460
461 p = new Put(T1, ts+3);
462 p.add(c0, c0, T1);
463 region.put(p);
464
465 region.flushcache();
466
467 region.compactStores(true);
468 assertEquals(4, countDeleteMarkers(region));
469
470
471
472 region.compactStores(true);
473 assertEquals(0, countDeleteMarkers(region));
474
475 HRegion.closeHRegion(region);
476 }
477
478
479
480
481 public void testRanges() throws Exception {
482 HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
483 HConstants.FOREVER, true);
484 HRegion region = createNewHRegion(htd, null, null);
485
486 long ts = EnvironmentEdgeManager.currentTimeMillis();
487 Put p = new Put(T1, ts);
488 p.add(c0, c0, T1);
489 p.add(c0, c1, T1);
490 p.add(c1, c0, T1);
491 p.add(c1, c1, T1);
492 region.put(p);
493
494 p = new Put(T2, ts);
495 p.add(c0, c0, T1);
496 p.add(c0, c1, T1);
497 p.add(c1, c0, T1);
498 p.add(c1, c1, T1);
499 region.put(p);
500
501 p = new Put(T1, ts+1);
502 p.add(c0, c0, T2);
503 p.add(c0, c1, T2);
504 p.add(c1, c0, T2);
505 p.add(c1, c1, T2);
506 region.put(p);
507
508 p = new Put(T2, ts+1);
509 p.add(c0, c0, T2);
510 p.add(c0, c1, T2);
511 p.add(c1, c0, T2);
512 p.add(c1, c1, T2);
513 region.put(p);
514
515 Delete d = new Delete(T1, ts+2);
516 d.deleteColumns(c0, c0, ts+2);
517 region.delete(d);
518
519 d = new Delete(T1, ts+2);
520 d.deleteFamily(c1, ts+2);
521 region.delete(d);
522
523 d = new Delete(T2, ts+2);
524 d.deleteFamily(c0, ts+2);
525 region.delete(d);
526
527
528 d = new Delete(T1, ts-10);
529 d.deleteFamily(c1, ts-10);
530 region.delete(d);
531
532
533 checkGet(region, T1, c0, c0, ts+2, T2, T1);
534 checkGet(region, T1, c0, c1, ts+2, T2, T1);
535 checkGet(region, T1, c1, c0, ts+2, T2, T1);
536 checkGet(region, T1, c1, c1, ts+2, T2, T1);
537
538 checkGet(region, T2, c0, c0, ts+2, T2, T1);
539 checkGet(region, T2, c0, c1, ts+2, T2, T1);
540 checkGet(region, T2, c1, c0, ts+2, T2, T1);
541 checkGet(region, T2, c1, c1, ts+2, T2, T1);
542
543
544 checkGet(region, T1, c0, c0, ts+3);
545 checkGet(region, T1, c0, c1, ts+3, T2, T1);
546 checkGet(region, T1, c1, c0, ts+3);
547 checkGet(region, T1, c1, c1, ts+3);
548
549 checkGet(region, T2, c0, c0, ts+3);
550 checkGet(region, T2, c0, c1, ts+3);
551 checkGet(region, T2, c1, c0, ts+3, T2, T1);
552 checkGet(region, T2, c1, c1, ts+3, T2, T1);
553
554 HRegion.closeHRegion(region);
555 }
556
557
558
559
560
561
562 public void testDeleteMarkerVersioning() throws Exception {
563 HTableDescriptor htd = createTableDescriptor(getName(), 0, 1,
564 HConstants.FOREVER, true);
565 HRegion region = createNewHRegion(htd, null, null);
566
567 long ts = EnvironmentEdgeManager.currentTimeMillis();
568 Put p = new Put(T1, ts);
569 p.add(c0, c0, T1);
570 region.put(p);
571
572
573
574 p = new Put(T1, ts-10);
575 p.add(c0, c1, T1);
576 region.put(p);
577
578 Delete d = new Delete(T1, ts);
579
580 d.deleteColumns(c0, c0, ts);
581 region.delete(d);
582
583 d = new Delete(T1, ts+1);
584 d.deleteColumn(c0, c0, ts+1);
585 region.delete(d);
586
587 d = new Delete(T1, ts+3);
588 d.deleteColumn(c0, c0, ts+3);
589 region.delete(d);
590
591 region.flushcache();
592 region.compactStores(true);
593 region.compactStores(true);
594 assertEquals(3, countDeleteMarkers(region));
595
596
597
598
599 p = new Put(T1, ts+2);
600 p.add(c0, c0, T2);
601 region.put(p);
602
603
604 assertEquals(3, countDeleteMarkers(region));
605
606 p = new Put(T1, ts+3);
607 p.add(c0, c0, T3);
608 region.put(p);
609
610
611
612
613
614
615
616
617
618
619
620 assertEquals(1, countDeleteMarkers(region));
621
622
623 region.flushcache();
624
625
626
627
628
629
630 assertEquals(3, countDeleteMarkers(region));
631
632 region.compactStores(true);
633 assertEquals(3, countDeleteMarkers(region));
634
635
636 p = new Put(T1, ts+4);
637 p.add(c0, c0, T4);
638 region.put(p);
639
640 region.flushcache();
641
642
643 assertEquals(1, countDeleteMarkers(region));
644 region.compactStores(true);
645 region.compactStores(true);
646 assertEquals(1, countDeleteMarkers(region));
647
648 HRegion.closeHRegion(region);
649 }
650
651
652
653
654 public void testWithMixedCFs() throws Exception {
655 HTableDescriptor htd = createTableDescriptor(getName(), 0, 1,
656 HConstants.FOREVER, true);
657 HRegion region = createNewHRegion(htd, null, null);
658
659 long ts = EnvironmentEdgeManager.currentTimeMillis();
660
661 Put p = new Put(T1, ts);
662 p.add(c0, c0, T1);
663 p.add(c0, c1, T1);
664 p.add(c1, c0, T1);
665 p.add(c1, c1, T1);
666 region.put(p);
667
668 p = new Put(T2, ts+1);
669 p.add(c0, c0, T2);
670 p.add(c0, c1, T2);
671 p.add(c1, c0, T2);
672 p.add(c1, c1, T2);
673 region.put(p);
674
675
676 Delete d = new Delete(T1, ts+1);
677 region.delete(d);
678
679 d = new Delete(T2, ts+2);
680 region.delete(d);
681
682 Scan s = new Scan(T1);
683 s.setTimeRange(0, ts+1);
684 InternalScanner scanner = region.getScanner(s);
685 List<KeyValue> kvs = new ArrayList<KeyValue>();
686 scanner.next(kvs);
687 assertEquals(4, kvs.size());
688 scanner.close();
689
690 s = new Scan(T2);
691 s.setTimeRange(0, ts+2);
692 scanner = region.getScanner(s);
693 kvs = new ArrayList<KeyValue>();
694 scanner.next(kvs);
695 assertEquals(4, kvs.size());
696 scanner.close();
697
698 HRegion.closeHRegion(region);
699 }
700
701
702
703
704
705 public void testWithMinVersions() throws Exception {
706 HTableDescriptor htd = createTableDescriptor(getName(), 3, 1000, 1, true);
707 HRegion region = createNewHRegion(htd, null, null);
708
709 long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
710
711 Put p = new Put(T1, ts);
712 p.add(c0, c0, T3);
713 region.put(p);
714 p = new Put(T1, ts-1);
715 p.add(c0, c0, T2);
716 region.put(p);
717 p = new Put(T1, ts-3);
718 p.add(c0, c0, T1);
719 region.put(p);
720 p = new Put(T1, ts-4);
721 p.add(c0, c0, T0);
722 region.put(p);
723
724
725
726
727 Delete d = new Delete(T1, ts-1);
728 region.delete(d);
729
730 d = new Delete(T1, ts-2);
731 d.deleteColumns(c0, c0, ts-1);
732 region.delete(d);
733
734 Get g = new Get(T1);
735 g.setMaxVersions();
736 g.setTimeRange(0L, ts-2);
737 Result r = region.get(g);
738 checkResult(r, c0, c0, T1,T0);
739
740
741 assertEquals(4, countDeleteMarkers(region));
742
743 region.flushcache();
744
745 assertEquals(4, countDeleteMarkers(region));
746
747 r = region.get(g);
748 checkResult(r, c0, c0, T1);
749 p = new Put(T1, ts+1);
750 p.add(c0, c0, T4);
751 region.put(p);
752 region.flushcache();
753
754 assertEquals(4, countDeleteMarkers(region));
755
756 r = region.get(g);
757 checkResult(r, c0, c0, T1);
758
759
760
761 p = new Put(T1, ts+2);
762 p.add(c0, c0, T5);
763 region.put(p);
764
765 region.flushcache();
766 region.compactStores(true);
767
768 assertEquals(2, countDeleteMarkers(region));
769
770
771
772 region.compactStores(true);
773 assertEquals(0, countDeleteMarkers(region));
774
775 HRegion.closeHRegion(region);
776 }
777
778 private void checkGet(HRegion region, byte[] row, byte[] fam, byte[] col,
779 long time, byte[]... vals) throws IOException {
780 Get g = new Get(row);
781 g.addColumn(fam, col);
782 g.setMaxVersions();
783 g.setTimeRange(0L, time);
784 Result r = region.get(g);
785 checkResult(r, fam, col, vals);
786
787 }
788
789 private int countDeleteMarkers(HRegion region) throws IOException {
790 Scan s = new Scan();
791 s.setRaw(true);
792
793 s.setMaxVersions(region.getStores().values().iterator().next().getScanInfo().getMaxVersions());
794 InternalScanner scan = region.getScanner(s);
795 List<KeyValue> kvs = new ArrayList<KeyValue>();
796 int res = 0;
797 boolean hasMore;
798 do {
799 hasMore = scan.next(kvs);
800 for (KeyValue kv : kvs) {
801 if(kv.isDelete()) res++;
802 }
803 kvs.clear();
804 } while (hasMore);
805 scan.close();
806 return res;
807 }
808
809 private void checkResult(Result r, byte[] fam, byte[] col, byte[] ... vals) {
810 assertEquals(r.size(), vals.length);
811 List<KeyValue> kvs = r.getColumn(fam, col);
812 assertEquals(kvs.size(), vals.length);
813 for (int i=0;i<vals.length;i++) {
814 assertEquals(kvs.get(i).getValue(), vals[i]);
815 }
816 }
817
818
819 }
820