1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver;
19
20 import java.io.IOException;
21 import java.util.ArrayList;
22 import java.util.List;
23
24 import org.apache.hadoop.hbase.*;
25 import org.apache.hadoop.hbase.client.Delete;
26 import org.apache.hadoop.hbase.client.Get;
27 import org.apache.hadoop.hbase.client.Put;
28 import org.apache.hadoop.hbase.client.Result;
29 import org.apache.hadoop.hbase.client.Scan;
30 import org.apache.hadoop.hbase.util.Bytes;
31 import org.junit.experimental.categories.Category;
32
33 @Category(SmallTests.class)
34 public class TestKeepDeletes extends HBaseTestCase {
35 private final byte[] T0 = Bytes.toBytes("0");
36 private final byte[] T1 = Bytes.toBytes("1");
37 private final byte[] T2 = Bytes.toBytes("2");
38 private final byte[] T3 = Bytes.toBytes("3");
39 private final byte[] T4 = Bytes.toBytes("4");
40 private final byte[] T5 = Bytes.toBytes("5");
41 private final byte[] T6 = Bytes.toBytes("6");
42
43 private final byte[] c0 = COLUMNS[0];
44 private final byte[] c1 = COLUMNS[1];
45
46
47
48
49
50
51
52 public void testBasicScenario() throws Exception {
53
54 HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
55 HConstants.FOREVER, true);
56 HRegion region = createNewHRegion(htd, null, null);
57
58 long ts = System.currentTimeMillis();
59 Put p = new Put(T1, ts);
60 p.add(c0, c0, T1);
61 region.put(p);
62 p = new Put(T1, ts+1);
63 p.add(c0, c0, T2);
64 region.put(p);
65 p = new Put(T1, ts+2);
66 p.add(c0, c0, T3);
67 region.put(p);
68 p = new Put(T1, ts+4);
69 p.add(c0, c0, T4);
70 region.put(p);
71
72
73 Delete d = new Delete(T1, ts+2, null);
74 region.delete(d, null, true);
75
76
77
78 assertEquals(3, countDeleteMarkers(region));
79
80
81 Get g = new Get(T1);
82 g.setMaxVersions();
83 g.setTimeRange(0L, ts+2);
84 Result r = region.get(g, null);
85 checkResult(r, c0, c0, T2,T1);
86
87
88 region.flushcache();
89
90
91 r = region.get(g, null);
92 checkResult(r, c0, c0, T2);
93
94
95 region.compactStores(true);
96 region.compactStores(true);
97
98
99
100 assertEquals(1, countDeleteMarkers(region));
101
102
103 r = region.get(g, null);
104 checkResult(r, c0, c0, T2);
105
106
107 g.setTimeRange(0L, ts+4);
108 r = region.get(g, null);
109 assertTrue(r.isEmpty());
110
111
112 p = new Put(T1, ts+5);
113 p.add(c0, c0, T5);
114 region.put(p);
115 p = new Put(T1, ts+6);
116 p.add(c0, c0, T6);
117 region.put(p);
118
119
120
121 p = new Put(T1, ts);
122 p.add(c0, c0, T1);
123 region.put(p);
124 r = region.get(g, null);
125 assertTrue(r.isEmpty());
126
127 region.flushcache();
128 region.compactStores(true);
129 region.compactStores(true);
130
131
132 region.put(p);
133 r = region.get(g, null);
134 checkResult(r, c0, c0, T1);
135 assertEquals(0, countDeleteMarkers(region));
136
137 region.close();
138 region.getLog().closeAndDelete();
139 }
140
141
142
143
144
145
146
147
148
149 public void testRawScanWithoutKeepingDeletes() throws Exception {
150
151 HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
152 HConstants.FOREVER, false);
153 HRegion region = createNewHRegion(htd, null, null);
154
155 long ts = System.currentTimeMillis();
156 Put p = new Put(T1, ts);
157 p.add(c0, c0, T1);
158 region.put(p);
159
160 Delete d = new Delete(T1, ts, null);
161 d.deleteColumn(c0, c0, ts);
162 region.delete(d, null, true);
163
164
165 Scan s = new Scan();
166 s.setRaw(true);
167 s.setMaxVersions();
168 InternalScanner scan = region.getScanner(s);
169 List<KeyValue> kvs = new ArrayList<KeyValue>();
170 scan.next(kvs);
171 assertEquals(2, kvs.size());
172
173 region.flushcache();
174 region.compactStores(true);
175
176
177
178
179 s = new Scan();
180 s.setRaw(true);
181 s.setMaxVersions();
182 scan = region.getScanner(s);
183 kvs = new ArrayList<KeyValue>();
184 scan.next(kvs);
185 assertTrue(kvs.isEmpty());
186
187 region.close();
188 region.getLog().closeAndDelete();
189 }
190
191
192
193
194 public void testWithoutKeepingDeletes() throws Exception {
195
196 HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
197 HConstants.FOREVER, false);
198 HRegion region = createNewHRegion(htd, null, null);
199
200 long ts = System.currentTimeMillis();
201 Put p = new Put(T1, ts);
202 p.add(c0, c0, T1);
203 region.put(p);
204 Delete d = new Delete(T1, ts+2, null);
205 d.deleteColumn(c0, c0, ts);
206 region.delete(d, null, true);
207
208
209 Get g = new Get(T1);
210 g.setMaxVersions();
211 g.setTimeRange(0L, ts+1);
212 Result r = region.get(g, null);
213 assertTrue(r.isEmpty());
214
215
216 Scan s = new Scan();
217 s.setMaxVersions();
218 s.setTimeRange(0L, ts+1);
219 InternalScanner scanner = region.getScanner(s);
220 List<KeyValue> kvs = new ArrayList<KeyValue>();
221 while(scanner.next(kvs));
222 assertTrue(kvs.isEmpty());
223
224
225 region.flushcache();
226 region.compactStores();
227 assertEquals(1, countDeleteMarkers(region));
228 region.compactStores(true);
229
230 assertEquals(0, countDeleteMarkers(region));
231
232 region.close();
233 region.getLog().closeAndDelete();
234 }
235
236
237
238
239 public void testRawScanWithColumns() throws Exception {
240 HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
241 HConstants.FOREVER, true);
242 HRegion region = createNewHRegion(htd, null, null);
243
244 Scan s = new Scan();
245 s.setRaw(true);
246 s.setMaxVersions();
247 s.addColumn(c0, c0);
248
249 try {
250 InternalScanner scan = region.getScanner(s);
251 fail("raw scanner with columns should have failed");
252 } catch (DoNotRetryIOException dnre) {
253
254 }
255
256 region.close();
257 region.getLog().closeAndDelete();
258 }
259
260
261
262
263 public void testRawScan() throws Exception {
264 HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
265 HConstants.FOREVER, true);
266 HRegion region = createNewHRegion(htd, null, null);
267
268 long ts = System.currentTimeMillis();
269 Put p = new Put(T1, ts);
270 p.add(c0, c0, T1);
271 region.put(p);
272 p = new Put(T1, ts+2);
273 p.add(c0, c0, T2);
274 region.put(p);
275 p = new Put(T1, ts+4);
276 p.add(c0, c0, T3);
277 region.put(p);
278
279 Delete d = new Delete(T1, ts+1);
280 region.delete(d, true);
281
282 d = new Delete(T1, ts+2);
283 d.deleteColumn(c0, c0, ts+2);
284 region.delete(d, true);
285
286 d = new Delete(T1, ts+3);
287 d.deleteColumns(c0, c0, ts+3);
288 region.delete(d, true);
289
290 Scan s = new Scan();
291 s.setRaw(true);
292 s.setMaxVersions();
293 InternalScanner scan = region.getScanner(s);
294 List<KeyValue> kvs = new ArrayList<KeyValue>();
295 scan.next(kvs);
296 assertEquals(8, kvs.size());
297 assertTrue(kvs.get(0).isDeleteFamily());
298 assertEquals(kvs.get(1).getValue(), T3);
299 assertTrue(kvs.get(2).isDelete());
300 assertTrue(kvs.get(3).isDeleteType());
301 assertEquals(kvs.get(4).getValue(), T2);
302 assertEquals(kvs.get(5).getValue(), T1);
303
304 assertTrue(kvs.get(6).isDeleteFamily());
305 assertTrue(kvs.get(7).isDeleteFamily());
306
307
308 s = new Scan();
309 s.setRaw(true);
310 s.setMaxVersions();
311 s.setTimeRange(0, 1);
312 scan = region.getScanner(s);
313 kvs = new ArrayList<KeyValue>();
314 scan.next(kvs);
315
316 assertTrue(kvs.isEmpty());
317
318
319 s = new Scan();
320 s.setRaw(true);
321 s.setMaxVersions();
322 s.setTimeRange(0, ts+2);
323 scan = region.getScanner(s);
324 kvs = new ArrayList<KeyValue>();
325 scan.next(kvs);
326 assertEquals(4, kvs.size());
327 assertTrue(kvs.get(0).isDeleteFamily());
328 assertEquals(kvs.get(1).getValue(), T1);
329
330 assertTrue(kvs.get(2).isDeleteFamily());
331 assertTrue(kvs.get(3).isDeleteFamily());
332
333
334 s = new Scan();
335 s.setRaw(true);
336 s.setMaxVersions();
337 s.setTimeRange(ts+3, ts+5);
338 scan = region.getScanner(s);
339 kvs = new ArrayList<KeyValue>();
340 scan.next(kvs);
341 assertEquals(2, kvs.size());
342 assertEquals(kvs.get(0).getValue(), T3);
343 assertTrue(kvs.get(1).isDelete());
344
345 region.close();
346 region.getLog().closeAndDelete();
347 }
348
349
350
351
352 public void testDeleteMarkerExpirationEmptyStore() throws Exception {
353 HTableDescriptor htd = createTableDescriptor(getName(), 0, 1,
354 HConstants.FOREVER, true);
355 HRegion region = createNewHRegion(htd, null, null);
356
357 long ts = System.currentTimeMillis();
358
359 Delete d = new Delete(T1, ts, null);
360 d.deleteColumns(c0, c0, ts);
361 region.delete(d, null, true);
362
363 d = new Delete(T1, ts, null);
364 d.deleteFamily(c0);
365 region.delete(d, null, true);
366
367 d = new Delete(T1, ts, null);
368 d.deleteColumn(c0, c0, ts+1);
369 region.delete(d, null, true);
370
371 d = new Delete(T1, ts, null);
372 d.deleteColumn(c0, c0, ts+2);
373 region.delete(d, null, true);
374
375
376 assertEquals(4, countDeleteMarkers(region));
377
378
379 region.flushcache();
380 assertEquals(4, countDeleteMarkers(region));
381 region.compactStores(false);
382 assertEquals(4, countDeleteMarkers(region));
383
384
385 region.compactStores(true);
386 assertEquals(0, countDeleteMarkers(region));
387
388 region.close();
389 region.getLog().closeAndDelete();
390 }
391
392
393
394
395 public void testDeleteMarkerExpiration() throws Exception {
396 HTableDescriptor htd = createTableDescriptor(getName(), 0, 1,
397 HConstants.FOREVER, true);
398 HRegion region = createNewHRegion(htd, null, null);
399
400 long ts = System.currentTimeMillis();
401
402 Put p = new Put(T1, ts);
403 p.add(c0, c0, T1);
404 region.put(p);
405
406
407 p = new Put(T1, ts-10);
408 p.add(c1, c0, T1);
409 region.put(p);
410
411
412 Delete d = new Delete(T1, ts, null);
413 d.deleteColumns(c0, c0, ts);
414 region.delete(d, null, true);
415
416 d = new Delete(T1, ts, null);
417 d.deleteFamily(c0, ts);
418 region.delete(d, null, true);
419
420 d = new Delete(T1, ts, null);
421 d.deleteColumn(c0, c0, ts+1);
422 region.delete(d, null, true);
423
424 d = new Delete(T1, ts, null);
425 d.deleteColumn(c0, c0, ts+2);
426 region.delete(d, null, true);
427
428
429 assertEquals(4, countDeleteMarkers(region));
430
431 region.flushcache();
432 assertEquals(4, countDeleteMarkers(region));
433 region.compactStores(false);
434 assertEquals(4, countDeleteMarkers(region));
435
436
437 p = new Put(T1, ts+3);
438 p.add(c0, c0, T1);
439 region.put(p);
440
441 region.flushcache();
442
443 region.compactStores(true);
444 assertEquals(4, countDeleteMarkers(region));
445
446
447
448 region.compactStores(true);
449 assertEquals(0, countDeleteMarkers(region));
450
451 region.close();
452 region.getLog().closeAndDelete();
453 }
454
455
456
457
458 public void testRanges() throws Exception {
459 HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
460 HConstants.FOREVER, true);
461 HRegion region = createNewHRegion(htd, null, null);
462
463 long ts = System.currentTimeMillis();
464 Put p = new Put(T1, ts);
465 p.add(c0, c0, T1);
466 p.add(c0, c1, T1);
467 p.add(c1, c0, T1);
468 p.add(c1, c1, T1);
469 region.put(p);
470
471 p = new Put(T2, ts);
472 p.add(c0, c0, T1);
473 p.add(c0, c1, T1);
474 p.add(c1, c0, T1);
475 p.add(c1, c1, T1);
476 region.put(p);
477
478 p = new Put(T1, ts+1);
479 p.add(c0, c0, T2);
480 p.add(c0, c1, T2);
481 p.add(c1, c0, T2);
482 p.add(c1, c1, T2);
483 region.put(p);
484
485 p = new Put(T2, ts+1);
486 p.add(c0, c0, T2);
487 p.add(c0, c1, T2);
488 p.add(c1, c0, T2);
489 p.add(c1, c1, T2);
490 region.put(p);
491
492 Delete d = new Delete(T1, ts+2, null);
493 d.deleteColumns(c0, c0, ts+2);
494 region.delete(d, null, true);
495
496 d = new Delete(T1, ts+2, null);
497 d.deleteFamily(c1, ts+2);
498 region.delete(d, null, true);
499
500 d = new Delete(T2, ts+2, null);
501 d.deleteFamily(c0, ts+2);
502 region.delete(d, null, true);
503
504
505 d = new Delete(T1, ts-10, null);
506 d.deleteFamily(c1, ts-10);
507 region.delete(d, null, true);
508
509
510 checkGet(region, T1, c0, c0, ts+2, T2, T1);
511 checkGet(region, T1, c0, c1, ts+2, T2, T1);
512 checkGet(region, T1, c1, c0, ts+2, T2, T1);
513 checkGet(region, T1, c1, c1, ts+2, T2, T1);
514
515 checkGet(region, T2, c0, c0, ts+2, T2, T1);
516 checkGet(region, T2, c0, c1, ts+2, T2, T1);
517 checkGet(region, T2, c1, c0, ts+2, T2, T1);
518 checkGet(region, T2, c1, c1, ts+2, T2, T1);
519
520
521 checkGet(region, T1, c0, c0, ts+3);
522 checkGet(region, T1, c0, c1, ts+3, T2, T1);
523 checkGet(region, T1, c1, c0, ts+3);
524 checkGet(region, T1, c1, c1, ts+3);
525
526 checkGet(region, T2, c0, c0, ts+3);
527 checkGet(region, T2, c0, c1, ts+3);
528 checkGet(region, T2, c1, c0, ts+3, T2, T1);
529 checkGet(region, T2, c1, c1, ts+3, T2, T1);
530
531 region.close();
532 region.getLog().closeAndDelete();
533 }
534
535
536
537
538
539
540 public void testDeleteMarkerVersioning() throws Exception {
541 HTableDescriptor htd = createTableDescriptor(getName(), 0, 1,
542 HConstants.FOREVER, true);
543 HRegion region = createNewHRegion(htd, null, null);
544
545 long ts = System.currentTimeMillis();
546 Put p = new Put(T1, ts);
547 p.add(c0, c0, T1);
548 region.put(p);
549
550
551
552 p = new Put(T1, ts-10);
553 p.add(c0, c1, T1);
554 region.put(p);
555
556 Delete d = new Delete(T1, ts, null);
557
558 d.deleteColumns(c0, c0, ts);
559 region.delete(d, null, true);
560
561 d = new Delete(T1, ts+1, null);
562 d.deleteColumn(c0, c0, ts+1);
563 region.delete(d, null, true);
564
565 d = new Delete(T1, ts+3, null);
566 d.deleteColumn(c0, c0, ts+3);
567 region.delete(d, null, true);
568
569 region.flushcache();
570 region.compactStores(true);
571 region.compactStores(true);
572 assertEquals(3, countDeleteMarkers(region));
573
574
575
576
577 p = new Put(T1, ts+2);
578 p.add(c0, c0, T2);
579 region.put(p);
580
581
582 assertEquals(3, countDeleteMarkers(region));
583
584 p = new Put(T1, ts+3);
585 p.add(c0, c0, T3);
586 region.put(p);
587
588
589
590
591
592
593
594
595
596
597
598 assertEquals(1, countDeleteMarkers(region));
599
600
601 region.flushcache();
602
603
604
605
606
607
608 assertEquals(3, countDeleteMarkers(region));
609
610 region.compactStores(true);
611 assertEquals(3, countDeleteMarkers(region));
612
613
614 p = new Put(T1, ts+4);
615 p.add(c0, c0, T4);
616 region.put(p);
617
618 region.flushcache();
619
620
621 assertEquals(1, countDeleteMarkers(region));
622 region.compactStores(true);
623 region.compactStores(true);
624 assertEquals(1, countDeleteMarkers(region));
625
626 region.close();
627 region.getLog().closeAndDelete();
628 }
629
630
631
632
633 public void testWithMixedCFs() throws Exception {
634 HTableDescriptor htd = createTableDescriptor(getName(), 0, 1,
635 HConstants.FOREVER, true);
636 HRegion region = createNewHRegion(htd, null, null);
637
638 long ts = System.currentTimeMillis();
639
640 Put p = new Put(T1, ts);
641 p.add(c0, c0, T1);
642 p.add(c0, c1, T1);
643 p.add(c1, c0, T1);
644 p.add(c1, c1, T1);
645 region.put(p);
646
647 p = new Put(T2, ts+1);
648 p.add(c0, c0, T2);
649 p.add(c0, c1, T2);
650 p.add(c1, c0, T2);
651 p.add(c1, c1, T2);
652 region.put(p);
653
654
655 Delete d = new Delete(T1, ts+1, null);
656 region.delete(d, null, true);
657
658 d = new Delete(T2, ts+2, null);
659 region.delete(d, null, true);
660
661 Scan s = new Scan(T1);
662 s.setTimeRange(0, ts+1);
663 InternalScanner scanner = region.getScanner(s);
664 List<KeyValue> kvs = new ArrayList<KeyValue>();
665 scanner.next(kvs);
666 assertEquals(4, kvs.size());
667 scanner.close();
668
669 s = new Scan(T2);
670 s.setTimeRange(0, ts+2);
671 scanner = region.getScanner(s);
672 kvs = new ArrayList<KeyValue>();
673 scanner.next(kvs);
674 assertEquals(4, kvs.size());
675 scanner.close();
676
677 region.close();
678 region.getLog().closeAndDelete();
679 }
680
681
682
683
684
685 public void testWithMinVersions() throws Exception {
686 HTableDescriptor htd = createTableDescriptor(getName(), 3, 1000, 1, true);
687 HRegion region = createNewHRegion(htd, null, null);
688
689 long ts = System.currentTimeMillis() - 2000;
690
691 Put p = new Put(T1, ts);
692 p.add(c0, c0, T3);
693 region.put(p);
694 p = new Put(T1, ts-1);
695 p.add(c0, c0, T2);
696 region.put(p);
697 p = new Put(T1, ts-3);
698 p.add(c0, c0, T1);
699 region.put(p);
700 p = new Put(T1, ts-4);
701 p.add(c0, c0, T0);
702 region.put(p);
703
704
705
706
707 Delete d = new Delete(T1, ts-1, null);
708 region.delete(d, null, true);
709
710 d = new Delete(T1, ts-2, null);
711 d.deleteColumns(c0, c0, ts-1);
712 region.delete(d, null, true);
713
714 Get g = new Get(T1);
715 g.setMaxVersions();
716 g.setTimeRange(0L, ts-2);
717 Result r = region.get(g, null);
718 checkResult(r, c0, c0, T1,T0);
719
720
721 assertEquals(4, countDeleteMarkers(region));
722
723 region.flushcache();
724
725 assertEquals(4, countDeleteMarkers(region));
726
727 r = region.get(g, null);
728 checkResult(r, c0, c0, T1);
729 p = new Put(T1, ts+1);
730 p.add(c0, c0, T4);
731 region.put(p);
732 region.flushcache();
733
734 assertEquals(4, countDeleteMarkers(region));
735
736 r = region.get(g, null);
737 checkResult(r, c0, c0, T1);
738
739
740
741 p = new Put(T1, ts+2);
742 p.add(c0, c0, T5);
743 region.put(p);
744
745 region.flushcache();
746 region.compactStores(true);
747
748 assertEquals(2, countDeleteMarkers(region));
749
750
751
752 region.compactStores(true);
753 assertEquals(0, countDeleteMarkers(region));
754
755 region.close();
756 region.getLog().closeAndDelete();
757 }
758
759 private void checkGet(HRegion region, byte[] row, byte[] fam, byte[] col,
760 long time, byte[]... vals) throws IOException {
761 Get g = new Get(row);
762 g.addColumn(fam, col);
763 g.setMaxVersions();
764 g.setTimeRange(0L, time);
765 Result r = region.get(g, null);
766 checkResult(r, fam, col, vals);
767
768 }
769
770 private int countDeleteMarkers(HRegion region) throws IOException {
771 Scan s = new Scan();
772 s.setRaw(true);
773
774 s.setMaxVersions(region.getStores().values().iterator().next().getScanInfo().getMaxVersions());
775 InternalScanner scan = region.getScanner(s);
776 List<KeyValue> kvs = new ArrayList<KeyValue>();
777 int res = 0;
778 boolean hasMore;
779 do {
780 hasMore = scan.next(kvs);
781 for (KeyValue kv : kvs) {
782 if(kv.isDelete()) res++;
783 }
784 kvs.clear();
785 } while (hasMore);
786 scan.close();
787 return res;
788 }
789
790 private void checkResult(Result r, byte[] fam, byte[] col, byte[] ... vals) {
791 assertEquals(r.size(), vals.length);
792 List<KeyValue> kvs = r.getColumn(fam, col);
793 assertEquals(kvs.size(), vals.length);
794 for (int i=0;i<vals.length;i++) {
795 assertEquals(kvs.get(i).getValue(), vals[i]);
796 }
797 }
798
799
800 @org.junit.Rule
801 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
802 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
803 }
804