1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver;
19
20 import java.io.IOException;
21 import java.util.ArrayList;
22 import java.util.List;
23
24 import org.apache.hadoop.hbase.*;
25 import org.apache.hadoop.hbase.client.Delete;
26 import org.apache.hadoop.hbase.client.Get;
27 import org.apache.hadoop.hbase.client.Put;
28 import org.apache.hadoop.hbase.client.Result;
29 import org.apache.hadoop.hbase.client.Scan;
30 import org.apache.hadoop.hbase.util.Bytes;
31 import org.junit.experimental.categories.Category;
32
33 @Category(SmallTests.class)
34 public class TestKeepDeletes extends HBaseTestCase {
35 private final byte[] T0 = Bytes.toBytes("0");
36 private final byte[] T1 = Bytes.toBytes("1");
37 private final byte[] T2 = Bytes.toBytes("2");
38 private final byte[] T3 = Bytes.toBytes("3");
39 private final byte[] T4 = Bytes.toBytes("4");
40 private final byte[] T5 = Bytes.toBytes("5");
41 private final byte[] T6 = Bytes.toBytes("6");
42
43 private final byte[] c0 = COLUMNS[0];
44 private final byte[] c1 = COLUMNS[1];
45
46
47
48
49
50
51
52 public void testBasicScenario() throws Exception {
53
54 HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
55 HConstants.FOREVER, true);
56 HRegion region = createNewHRegion(htd, null, null);
57
58 long ts = System.currentTimeMillis();
59 Put p = new Put(T1, ts);
60 p.add(c0, c0, T1);
61 region.put(p);
62 p = new Put(T1, ts+1);
63 p.add(c0, c0, T2);
64 region.put(p);
65 p = new Put(T1, ts+2);
66 p.add(c0, c0, T3);
67 region.put(p);
68 p = new Put(T1, ts+4);
69 p.add(c0, c0, T4);
70 region.put(p);
71
72
73 Delete d = new Delete(T1, ts+2, null);
74 region.delete(d, null, true);
75
76
77
78 assertEquals(3, countDeleteMarkers(region));
79
80
81 Get g = new Get(T1);
82 g.setMaxVersions();
83 g.setTimeRange(0L, ts+2);
84 Result r = region.get(g, null);
85 checkResult(r, c0, c0, T2,T1);
86
87
88 region.flushcache();
89
90
91 r = region.get(g, null);
92 checkResult(r, c0, c0, T2);
93
94
95 region.compactStores(true);
96 region.compactStores(true);
97
98
99
100 assertEquals(1, countDeleteMarkers(region));
101
102
103 r = region.get(g, null);
104 checkResult(r, c0, c0, T2);
105
106
107 g.setTimeRange(0L, ts+4);
108 r = region.get(g, null);
109 assertTrue(r.isEmpty());
110
111
112 p = new Put(T1, ts+5);
113 p.add(c0, c0, T5);
114 region.put(p);
115 p = new Put(T1, ts+6);
116 p.add(c0, c0, T6);
117 region.put(p);
118
119
120
121 p = new Put(T1, ts);
122 p.add(c0, c0, T1);
123 region.put(p);
124 r = region.get(g, null);
125 assertTrue(r.isEmpty());
126
127 region.flushcache();
128 region.compactStores(true);
129 region.compactStores(true);
130
131
132 region.put(p);
133 r = region.get(g, null);
134 checkResult(r, c0, c0, T1);
135 assertEquals(0, countDeleteMarkers(region));
136
137 region.close();
138 region.getLog().closeAndDelete();
139 }
140
141
142
143
144
145
146
147
148
149 public void testRawScanWithoutKeepingDeletes() throws Exception {
150
151 HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
152 HConstants.FOREVER, false);
153 HRegion region = createNewHRegion(htd, null, null);
154
155 long ts = System.currentTimeMillis();
156 Put p = new Put(T1, ts);
157 p.add(c0, c0, T1);
158 region.put(p);
159
160 Delete d = new Delete(T1, ts, null);
161 d.deleteColumn(c0, c0, ts);
162 region.delete(d, null, true);
163
164
165 Scan s = new Scan();
166 s.setRaw(true);
167 s.setMaxVersions();
168 InternalScanner scan = region.getScanner(s);
169 List<KeyValue> kvs = new ArrayList<KeyValue>();
170 scan.next(kvs);
171 assertEquals(2, kvs.size());
172
173 region.flushcache();
174 region.compactStores(true);
175
176
177
178
179 s = new Scan();
180 s.setRaw(true);
181 s.setMaxVersions();
182 scan = region.getScanner(s);
183 kvs = new ArrayList<KeyValue>();
184 scan.next(kvs);
185 assertTrue(kvs.isEmpty());
186
187 region.close();
188 region.getLog().closeAndDelete();
189 }
190
191
192
193
194 public void testWithoutKeepingDeletes() throws Exception {
195
196 HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
197 HConstants.FOREVER, false);
198 HRegion region = createNewHRegion(htd, null, null);
199
200 long ts = System.currentTimeMillis();
201 Put p = new Put(T1, ts);
202 p.add(c0, c0, T1);
203 region.put(p);
204 Delete d = new Delete(T1, ts+2, null);
205 d.deleteColumn(c0, c0, ts);
206 region.delete(d, null, true);
207
208
209 Get g = new Get(T1);
210 g.setMaxVersions();
211 g.setTimeRange(0L, ts+1);
212 Result r = region.get(g, null);
213 assertTrue(r.isEmpty());
214
215
216 Scan s = new Scan();
217 s.setMaxVersions();
218 s.setTimeRange(0L, ts+1);
219 InternalScanner scanner = region.getScanner(s);
220 List<KeyValue> kvs = new ArrayList<KeyValue>();
221 while(scanner.next(kvs));
222 assertTrue(kvs.isEmpty());
223
224
225 region.flushcache();
226 region.compactStores();
227 assertEquals(1, countDeleteMarkers(region));
228 region.compactStores(true);
229
230 assertEquals(0, countDeleteMarkers(region));
231
232 region.close();
233 region.getLog().closeAndDelete();
234 }
235
236
237
238
239 public void testRawScanWithColumns() throws Exception {
240 HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
241 HConstants.FOREVER, true);
242 HRegion region = createNewHRegion(htd, null, null);
243
244 Scan s = new Scan();
245 s.setRaw(true);
246 s.setMaxVersions();
247 s.addColumn(c0, c0);
248
249 try {
250 InternalScanner scan = region.getScanner(s);
251 fail("raw scanner with columns should have failed");
252 } catch (DoNotRetryIOException dnre) {
253
254 }
255
256 region.close();
257 region.getLog().closeAndDelete();
258 }
259
260
261
262
263 public void testRawScan() throws Exception {
264 HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
265 HConstants.FOREVER, true);
266 HRegion region = createNewHRegion(htd, null, null);
267
268 long ts = System.currentTimeMillis();
269 Put p = new Put(T1, ts);
270 p.add(c0, c0, T1);
271 region.put(p);
272 p = new Put(T1, ts+2);
273 p.add(c0, c0, T2);
274 region.put(p);
275 p = new Put(T1, ts+4);
276 p.add(c0, c0, T3);
277 region.put(p);
278
279 Delete d = new Delete(T1, ts+1, null);
280 region.delete(d, null, true);
281
282 d = new Delete(T1, ts+2, null);
283 d.deleteColumn(c0, c0, ts+2);
284 region.delete(d, null, true);
285
286 d = new Delete(T1, ts+3, null);
287 d.deleteColumns(c0, c0, ts+3);
288 region.delete(d, null, true);
289
290 Scan s = new Scan();
291 s.setRaw(true);
292 s.setMaxVersions();
293 InternalScanner scan = region.getScanner(s);
294 List<KeyValue> kvs = new ArrayList<KeyValue>();
295 scan.next(kvs);
296 assertTrue(kvs.get(0).isDeleteFamily());
297 assertEquals(kvs.get(1).getValue(), T3);
298 assertTrue(kvs.get(2).isDelete());
299 assertTrue(kvs.get(3).isDeleteType());
300 assertEquals(kvs.get(4).getValue(), T2);
301 assertEquals(kvs.get(5).getValue(), T1);
302
303 region.close();
304 region.getLog().closeAndDelete();
305 }
306
307
308
309
310 public void testDeleteMarkerExpirationEmptyStore() throws Exception {
311 HTableDescriptor htd = createTableDescriptor(getName(), 0, 1,
312 HConstants.FOREVER, true);
313 HRegion region = createNewHRegion(htd, null, null);
314
315 long ts = System.currentTimeMillis();
316
317 Delete d = new Delete(T1, ts, null);
318 d.deleteColumns(c0, c0, ts);
319 region.delete(d, null, true);
320
321 d = new Delete(T1, ts, null);
322 d.deleteFamily(c0);
323 region.delete(d, null, true);
324
325 d = new Delete(T1, ts, null);
326 d.deleteColumn(c0, c0, ts+1);
327 region.delete(d, null, true);
328
329 d = new Delete(T1, ts, null);
330 d.deleteColumn(c0, c0, ts+2);
331 region.delete(d, null, true);
332
333
334 assertEquals(4, countDeleteMarkers(region));
335
336
337 region.flushcache();
338 assertEquals(4, countDeleteMarkers(region));
339 region.compactStores(false);
340 assertEquals(4, countDeleteMarkers(region));
341
342
343 region.compactStores(true);
344 assertEquals(0, countDeleteMarkers(region));
345
346 region.close();
347 region.getLog().closeAndDelete();
348 }
349
350
351
352
353 public void testDeleteMarkerExpiration() throws Exception {
354 HTableDescriptor htd = createTableDescriptor(getName(), 0, 1,
355 HConstants.FOREVER, true);
356 HRegion region = createNewHRegion(htd, null, null);
357
358 long ts = System.currentTimeMillis();
359
360 Put p = new Put(T1, ts);
361 p.add(c0, c0, T1);
362 region.put(p);
363
364
365 p = new Put(T1, ts-10);
366 p.add(c1, c0, T1);
367 region.put(p);
368
369
370 Delete d = new Delete(T1, ts, null);
371 d.deleteColumns(c0, c0, ts);
372 region.delete(d, null, true);
373
374 d = new Delete(T1, ts, null);
375 d.deleteFamily(c0, ts);
376 region.delete(d, null, true);
377
378 d = new Delete(T1, ts, null);
379 d.deleteColumn(c0, c0, ts+1);
380 region.delete(d, null, true);
381
382 d = new Delete(T1, ts, null);
383 d.deleteColumn(c0, c0, ts+2);
384 region.delete(d, null, true);
385
386
387 assertEquals(4, countDeleteMarkers(region));
388
389 region.flushcache();
390 assertEquals(4, countDeleteMarkers(region));
391 region.compactStores(false);
392 assertEquals(4, countDeleteMarkers(region));
393
394
395 p = new Put(T1, ts+3);
396 p.add(c0, c0, T1);
397 region.put(p);
398
399 region.flushcache();
400
401 region.compactStores(true);
402 assertEquals(4, countDeleteMarkers(region));
403
404
405
406 region.compactStores(true);
407 assertEquals(0, countDeleteMarkers(region));
408
409 region.close();
410 region.getLog().closeAndDelete();
411 }
412
413
414
415
416 public void testRanges() throws Exception {
417 HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
418 HConstants.FOREVER, true);
419 HRegion region = createNewHRegion(htd, null, null);
420
421 long ts = System.currentTimeMillis();
422 Put p = new Put(T1, ts);
423 p.add(c0, c0, T1);
424 p.add(c0, c1, T1);
425 p.add(c1, c0, T1);
426 p.add(c1, c1, T1);
427 region.put(p);
428
429 p = new Put(T2, ts);
430 p.add(c0, c0, T1);
431 p.add(c0, c1, T1);
432 p.add(c1, c0, T1);
433 p.add(c1, c1, T1);
434 region.put(p);
435
436 p = new Put(T1, ts+1);
437 p.add(c0, c0, T2);
438 p.add(c0, c1, T2);
439 p.add(c1, c0, T2);
440 p.add(c1, c1, T2);
441 region.put(p);
442
443 p = new Put(T2, ts+1);
444 p.add(c0, c0, T2);
445 p.add(c0, c1, T2);
446 p.add(c1, c0, T2);
447 p.add(c1, c1, T2);
448 region.put(p);
449
450 Delete d = new Delete(T1, ts+2, null);
451 d.deleteColumns(c0, c0, ts+2);
452 region.delete(d, null, true);
453
454 d = new Delete(T1, ts+2, null);
455 d.deleteFamily(c1, ts+2);
456 region.delete(d, null, true);
457
458 d = new Delete(T2, ts+2, null);
459 d.deleteFamily(c0, ts+2);
460 region.delete(d, null, true);
461
462
463 d = new Delete(T1, ts-10, null);
464 d.deleteFamily(c1, ts-10);
465 region.delete(d, null, true);
466
467
468 checkGet(region, T1, c0, c0, ts+2, T2, T1);
469 checkGet(region, T1, c0, c1, ts+2, T2, T1);
470 checkGet(region, T1, c1, c0, ts+2, T2, T1);
471 checkGet(region, T1, c1, c1, ts+2, T2, T1);
472
473 checkGet(region, T2, c0, c0, ts+2, T2, T1);
474 checkGet(region, T2, c0, c1, ts+2, T2, T1);
475 checkGet(region, T2, c1, c0, ts+2, T2, T1);
476 checkGet(region, T2, c1, c1, ts+2, T2, T1);
477
478
479 checkGet(region, T1, c0, c0, ts+3);
480 checkGet(region, T1, c0, c1, ts+3, T2, T1);
481 checkGet(region, T1, c1, c0, ts+3);
482 checkGet(region, T1, c1, c1, ts+3);
483
484 checkGet(region, T2, c0, c0, ts+3);
485 checkGet(region, T2, c0, c1, ts+3);
486 checkGet(region, T2, c1, c0, ts+3, T2, T1);
487 checkGet(region, T2, c1, c1, ts+3, T2, T1);
488
489 region.close();
490 region.getLog().closeAndDelete();
491 }
492
493
494
495
496
497
498 public void testDeleteMarkerVersioning() throws Exception {
499 HTableDescriptor htd = createTableDescriptor(getName(), 0, 1,
500 HConstants.FOREVER, true);
501 HRegion region = createNewHRegion(htd, null, null);
502
503 long ts = System.currentTimeMillis();
504 Put p = new Put(T1, ts);
505 p.add(c0, c0, T1);
506 region.put(p);
507
508
509
510 p = new Put(T1, ts-10);
511 p.add(c0, c1, T1);
512 region.put(p);
513
514 Delete d = new Delete(T1, ts, null);
515
516 d.deleteColumns(c0, c0, ts);
517 region.delete(d, null, true);
518
519 d = new Delete(T1, ts+1, null);
520 d.deleteColumn(c0, c0, ts+1);
521 region.delete(d, null, true);
522
523 d = new Delete(T1, ts+3, null);
524 d.deleteColumn(c0, c0, ts+3);
525 region.delete(d, null, true);
526
527 region.flushcache();
528 region.compactStores(true);
529 region.compactStores(true);
530 assertEquals(3, countDeleteMarkers(region));
531
532
533
534
535 p = new Put(T1, ts+2);
536 p.add(c0, c0, T2);
537 region.put(p);
538
539
540 assertEquals(3, countDeleteMarkers(region));
541
542 p = new Put(T1, ts+3);
543 p.add(c0, c0, T3);
544 region.put(p);
545
546
547
548
549
550
551
552
553
554
555
556 assertEquals(1, countDeleteMarkers(region));
557
558
559 region.flushcache();
560
561
562
563
564
565
566 assertEquals(3, countDeleteMarkers(region));
567
568 region.compactStores(true);
569 assertEquals(3, countDeleteMarkers(region));
570
571
572 p = new Put(T1, ts+4);
573 p.add(c0, c0, T4);
574 region.put(p);
575
576 region.flushcache();
577
578
579 assertEquals(1, countDeleteMarkers(region));
580 region.compactStores(true);
581 region.compactStores(true);
582 assertEquals(1, countDeleteMarkers(region));
583
584 region.close();
585 region.getLog().closeAndDelete();
586 }
587
588
589
590
591 public void testWithMixedCFs() throws Exception {
592 HTableDescriptor htd = createTableDescriptor(getName(), 0, 1,
593 HConstants.FOREVER, true);
594 HRegion region = createNewHRegion(htd, null, null);
595
596 long ts = System.currentTimeMillis();
597
598 Put p = new Put(T1, ts);
599 p.add(c0, c0, T1);
600 p.add(c0, c1, T1);
601 p.add(c1, c0, T1);
602 p.add(c1, c1, T1);
603 region.put(p);
604
605 p = new Put(T2, ts+1);
606 p.add(c0, c0, T2);
607 p.add(c0, c1, T2);
608 p.add(c1, c0, T2);
609 p.add(c1, c1, T2);
610 region.put(p);
611
612
613 Delete d = new Delete(T1, ts+1, null);
614 region.delete(d, null, true);
615
616 d = new Delete(T2, ts+2, null);
617 region.delete(d, null, true);
618
619 Scan s = new Scan(T1);
620 s.setTimeRange(0, ts+1);
621 InternalScanner scanner = region.getScanner(s);
622 List<KeyValue> kvs = new ArrayList<KeyValue>();
623 scanner.next(kvs);
624 assertEquals(4, kvs.size());
625 scanner.close();
626
627 s = new Scan(T2);
628 s.setTimeRange(0, ts+2);
629 scanner = region.getScanner(s);
630 kvs = new ArrayList<KeyValue>();
631 scanner.next(kvs);
632 assertEquals(4, kvs.size());
633 scanner.close();
634
635 region.close();
636 region.getLog().closeAndDelete();
637 }
638
639
640
641
642
643 public void testWithMinVersions() throws Exception {
644 HTableDescriptor htd = createTableDescriptor(getName(), 3, 1000, 1, true);
645 HRegion region = createNewHRegion(htd, null, null);
646
647 long ts = System.currentTimeMillis() - 2000;
648
649 Put p = new Put(T1, ts);
650 p.add(c0, c0, T3);
651 region.put(p);
652 p = new Put(T1, ts-1);
653 p.add(c0, c0, T2);
654 region.put(p);
655 p = new Put(T1, ts-3);
656 p.add(c0, c0, T1);
657 region.put(p);
658 p = new Put(T1, ts-4);
659 p.add(c0, c0, T0);
660 region.put(p);
661
662
663
664
665 Delete d = new Delete(T1, ts-1, null);
666 region.delete(d, null, true);
667
668 d = new Delete(T1, ts-2, null);
669 d.deleteColumns(c0, c0, ts-1);
670 region.delete(d, null, true);
671
672 Get g = new Get(T1);
673 g.setMaxVersions();
674 g.setTimeRange(0L, ts-2);
675 Result r = region.get(g, null);
676 checkResult(r, c0, c0, T1,T0);
677
678
679 assertEquals(4, countDeleteMarkers(region));
680
681 region.flushcache();
682
683 assertEquals(4, countDeleteMarkers(region));
684
685 r = region.get(g, null);
686 checkResult(r, c0, c0, T1);
687 p = new Put(T1, ts+1);
688 p.add(c0, c0, T4);
689 region.put(p);
690 region.flushcache();
691
692 assertEquals(4, countDeleteMarkers(region));
693
694 r = region.get(g, null);
695 checkResult(r, c0, c0, T1);
696
697
698
699 p = new Put(T1, ts+2);
700 p.add(c0, c0, T5);
701 region.put(p);
702
703 region.flushcache();
704 region.compactStores(true);
705
706 assertEquals(2, countDeleteMarkers(region));
707
708
709
710 region.compactStores(true);
711 assertEquals(0, countDeleteMarkers(region));
712
713 region.close();
714 region.getLog().closeAndDelete();
715 }
716
717 private void checkGet(HRegion region, byte[] row, byte[] fam, byte[] col,
718 long time, byte[]... vals) throws IOException {
719 Get g = new Get(row);
720 g.addColumn(fam, col);
721 g.setMaxVersions();
722 g.setTimeRange(0L, time);
723 Result r = region.get(g, null);
724 checkResult(r, fam, col, vals);
725
726 }
727
728 private int countDeleteMarkers(HRegion region) throws IOException {
729 Scan s = new Scan();
730 s.setRaw(true);
731 s.setMaxVersions();
732 InternalScanner scan = region.getScanner(s);
733 List<KeyValue> kvs = new ArrayList<KeyValue>();
734 int res = 0;
735 boolean hasMore;
736 do {
737 hasMore = scan.next(kvs);
738 for (KeyValue kv : kvs) {
739 if(kv.isDelete()) res++;
740 }
741 kvs.clear();
742 } while (hasMore);
743 scan.close();
744 return res;
745 }
746
747 private void checkResult(Result r, byte[] fam, byte[] col, byte[] ... vals) {
748 assertEquals(r.size(), vals.length);
749 List<KeyValue> kvs = r.getColumn(fam, col);
750 assertEquals(kvs.size(), vals.length);
751 for (int i=0;i<vals.length;i++) {
752 assertEquals(kvs.get(i).getValue(), vals[i]);
753 }
754 }
755
756
757 @org.junit.Rule
758 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
759 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
760 }
761