1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import java.io.IOException;
21  import java.util.ArrayList;
22  import java.util.List;
23  
24  import org.apache.hadoop.hbase.*;
25  import org.apache.hadoop.hbase.client.Delete;
26  import org.apache.hadoop.hbase.client.Get;
27  import org.apache.hadoop.hbase.client.Put;
28  import org.apache.hadoop.hbase.client.Result;
29  import org.apache.hadoop.hbase.client.Scan;
30  import org.apache.hadoop.hbase.util.Bytes;
31  import org.junit.experimental.categories.Category;
32  
33  @Category(SmallTests.class)
34  public class TestKeepDeletes extends HBaseTestCase {
35    private final byte[] T0 = Bytes.toBytes("0");
36    private final byte[] T1 = Bytes.toBytes("1");
37    private final byte[] T2 = Bytes.toBytes("2");
38    private final byte[] T3 = Bytes.toBytes("3");
39    private final byte[] T4 = Bytes.toBytes("4");
40    private final byte[] T5 = Bytes.toBytes("5");
41    private final byte[] T6 = Bytes.toBytes("6");
42  
43    private final byte[] c0 = COLUMNS[0];
44    private final byte[] c1 = COLUMNS[1];
45  
46    /**
47     * Make sure that deleted rows are retained.
48     * Family delete markers are deleted.
49     * Column Delete markers are versioned
50     * Time range scan of deleted rows are possible
51     */
52    public void testBasicScenario() throws Exception {
53      // keep 3 versions, rows do not expire
54      HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
55          HConstants.FOREVER, true);
56      HRegion region = createNewHRegion(htd, null, null);
57  
58      long ts = System.currentTimeMillis();
59      Put p = new Put(T1, ts);
60      p.add(c0, c0, T1);
61      region.put(p);
62      p = new Put(T1, ts+1);
63      p.add(c0, c0, T2);
64      region.put(p);
65      p = new Put(T1, ts+2);
66      p.add(c0, c0, T3);
67      region.put(p);
68      p = new Put(T1, ts+4);
69      p.add(c0, c0, T4);
70      region.put(p);
71  
72      // now place a delete marker at ts+2
73      Delete d = new Delete(T1, ts+2, null);
74      region.delete(d, null, true);
75  
76      // a raw scan can see the delete markers
77      // (one for each column family)
78      assertEquals(3, countDeleteMarkers(region));
79  
80      // get something *before* the delete marker
81      Get g = new Get(T1);
82      g.setMaxVersions();
83      g.setTimeRange(0L, ts+2);
84      Result r = region.get(g, null);
85      checkResult(r, c0, c0, T2,T1);
86  
87      // flush
88      region.flushcache();
89  
90      // yep, T2 still there, T1 gone
91      r = region.get(g, null);
92      checkResult(r, c0, c0, T2);
93  
94      // major compact
95      region.compactStores(true);
96      region.compactStores(true);
97  
98      // one delete marker left (the others did not
99      // have older puts)
100     assertEquals(1, countDeleteMarkers(region));
101 
102     // still there (even after multiple compactions)
103     r = region.get(g, null);
104     checkResult(r, c0, c0, T2);
105 
106     // a timerange that includes the delete marker won't see past rows
107     g.setTimeRange(0L, ts+4);
108     r = region.get(g, null);
109     assertTrue(r.isEmpty());
110 
111     // two more puts, this will expire the older puts.
112     p = new Put(T1, ts+5);
113     p.add(c0, c0, T5);
114     region.put(p);
115     p = new Put(T1, ts+6);
116     p.add(c0, c0, T6);
117     region.put(p);
118 
119     // also add an old put again
120     // (which is past the max versions)
121     p = new Put(T1, ts);
122     p.add(c0, c0, T1);
123     region.put(p);
124     r = region.get(g, null);
125     assertTrue(r.isEmpty());
126 
127     region.flushcache();
128     region.compactStores(true);
129     region.compactStores(true);
130 
131     // verify that the delete marker itself was collected
132     region.put(p);
133     r = region.get(g, null);
134     checkResult(r, c0, c0, T1);
135     assertEquals(0, countDeleteMarkers(region));
136 
137     region.close();
138     region.getLog().closeAndDelete();
139   }
140 
141   /**
142    * Even when the store does not keep deletes a "raw" scan will 
143    * return everything it can find (unless discarding cells is guaranteed
144    * to have no effect).
145    * Assuming this the desired behavior. Could also disallow "raw" scanning
146    * if the store does not have KEEP_DELETED_CELLS enabled.
147    * (can be changed easily)
148    */
149   public void testRawScanWithoutKeepingDeletes() throws Exception {
150     // KEEP_DELETED_CELLS is NOT enabled
151     HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
152         HConstants.FOREVER, false);
153     HRegion region = createNewHRegion(htd, null, null);
154 
155     long ts = System.currentTimeMillis();
156     Put p = new Put(T1, ts);
157     p.add(c0, c0, T1);
158     region.put(p);
159 
160     Delete d = new Delete(T1, ts, null);
161     d.deleteColumn(c0, c0, ts);
162     region.delete(d, null, true);
163 
164     // scan still returns delete markers and deletes rows
165     Scan s = new Scan();
166     s.setRaw(true);
167     s.setMaxVersions();
168     InternalScanner scan = region.getScanner(s);
169     List<KeyValue> kvs = new ArrayList<KeyValue>();
170     scan.next(kvs);
171     assertEquals(2, kvs.size());
172 
173     region.flushcache();
174     region.compactStores(true);
175 
176     // after compaction they are gone
177     // (note that this a test with a Store without
178     //  KEEP_DELETED_CELLS)
179     s = new Scan();
180     s.setRaw(true);
181     s.setMaxVersions();
182     scan = region.getScanner(s);
183     kvs = new ArrayList<KeyValue>();
184     scan.next(kvs);
185     assertTrue(kvs.isEmpty());
186 
187     region.close();
188     region.getLog().closeAndDelete();
189   }
190 
191   /**
192    * basic verification of existing behavior
193    */
194   public void testWithoutKeepingDeletes() throws Exception {
195     // KEEP_DELETED_CELLS is NOT enabled
196     HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
197         HConstants.FOREVER, false);
198     HRegion region = createNewHRegion(htd, null, null);
199 
200     long ts = System.currentTimeMillis();  
201     Put p = new Put(T1, ts);
202     p.add(c0, c0, T1);
203     region.put(p);
204     Delete d = new Delete(T1, ts+2, null);
205     d.deleteColumn(c0, c0, ts);
206     region.delete(d, null, true);
207 
208     // "past" get does not see rows behind delete marker
209     Get g = new Get(T1);
210     g.setMaxVersions();
211     g.setTimeRange(0L, ts+1);
212     Result r = region.get(g, null);
213     assertTrue(r.isEmpty());
214 
215     // "past" scan does not see rows behind delete marker
216     Scan s = new Scan();
217     s.setMaxVersions();
218     s.setTimeRange(0L, ts+1);
219     InternalScanner scanner = region.getScanner(s);
220     List<KeyValue> kvs = new ArrayList<KeyValue>();
221     while(scanner.next(kvs));
222     assertTrue(kvs.isEmpty());
223 
224     // flushing and minor compaction keep delete markers
225     region.flushcache();
226     region.compactStores();
227     assertEquals(1, countDeleteMarkers(region));
228     region.compactStores(true);
229     // major compaction deleted it
230     assertEquals(0, countDeleteMarkers(region));
231 
232     region.close();
233     region.getLog().closeAndDelete();
234   }
235 
236   /**
237    * The ExplicitColumnTracker does not support "raw" scanning.
238    */
239   public void testRawScanWithColumns() throws Exception {
240     HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
241         HConstants.FOREVER, true);
242     HRegion region = createNewHRegion(htd, null, null);
243 
244     Scan s = new Scan();
245     s.setRaw(true);
246     s.setMaxVersions();
247     s.addColumn(c0, c0);
248     
249     try {
250       InternalScanner scan = region.getScanner(s);
251       fail("raw scanner with columns should have failed");
252     } catch (DoNotRetryIOException dnre) {
253       // ok!
254     }
255 
256     region.close();
257     region.getLog().closeAndDelete();
258   }
259 
260   /**
261    * Verify that "raw" scanning mode return delete markers and deletes rows.
262    */
263   public void testRawScan() throws Exception {
264     HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
265         HConstants.FOREVER, true);
266     HRegion region = createNewHRegion(htd, null, null);
267 
268     long ts = System.currentTimeMillis();
269     Put p = new Put(T1, ts);
270     p.add(c0, c0, T1);
271     region.put(p);
272     p = new Put(T1, ts+2);
273     p.add(c0, c0, T2);
274     region.put(p);
275     p = new Put(T1, ts+4);
276     p.add(c0, c0, T3);
277     region.put(p);
278 
279     Delete d = new Delete(T1, ts+1, null);
280     region.delete(d, null, true);
281 
282     d = new Delete(T1, ts+2, null);
283     d.deleteColumn(c0, c0, ts+2);
284     region.delete(d, null, true);
285 
286     d = new Delete(T1, ts+3, null);
287     d.deleteColumns(c0, c0, ts+3);
288     region.delete(d, null, true);
289 
290     Scan s = new Scan();
291     s.setRaw(true);
292     s.setMaxVersions();
293     InternalScanner scan = region.getScanner(s);
294     List<KeyValue> kvs = new ArrayList<KeyValue>();
295     scan.next(kvs);
296     assertTrue(kvs.get(0).isDeleteFamily());
297     assertEquals(kvs.get(1).getValue(), T3);
298     assertTrue(kvs.get(2).isDelete());
299     assertTrue(kvs.get(3).isDeleteType());
300     assertEquals(kvs.get(4).getValue(), T2);
301     assertEquals(kvs.get(5).getValue(), T1);
302 
303     region.close();
304     region.getLog().closeAndDelete();
305   }
306 
307   /**
308    * Verify that delete markers are removed from an otherwise empty store.
309    */
310   public void testDeleteMarkerExpirationEmptyStore() throws Exception {
311     HTableDescriptor htd = createTableDescriptor(getName(), 0, 1,
312         HConstants.FOREVER, true);
313     HRegion region = createNewHRegion(htd, null, null);
314 
315     long ts = System.currentTimeMillis();
316 
317     Delete d = new Delete(T1, ts, null);
318     d.deleteColumns(c0, c0, ts);
319     region.delete(d, null, true);
320 
321     d = new Delete(T1, ts, null);
322     d.deleteFamily(c0);
323     region.delete(d, null, true);
324 
325     d = new Delete(T1, ts, null);
326     d.deleteColumn(c0, c0, ts+1);
327     region.delete(d, null, true);
328     
329     d = new Delete(T1, ts, null);
330     d.deleteColumn(c0, c0, ts+2);
331     region.delete(d, null, true);
332 
333     // 1 family marker, 1 column marker, 2 version markers
334     assertEquals(4, countDeleteMarkers(region));
335 
336     // neither flush nor minor compaction removes any marker
337     region.flushcache();
338     assertEquals(4, countDeleteMarkers(region));
339     region.compactStores(false);
340     assertEquals(4, countDeleteMarkers(region));
341 
342     // major compaction removes all, since there are no puts they affect
343     region.compactStores(true);
344     assertEquals(0, countDeleteMarkers(region));
345 
346     region.close();
347     region.getLog().closeAndDelete();
348   }
349 
350   /**
351    * Test delete marker removal from store files.
352    */
353   public void testDeleteMarkerExpiration() throws Exception {
354     HTableDescriptor htd = createTableDescriptor(getName(), 0, 1,
355         HConstants.FOREVER, true);
356     HRegion region = createNewHRegion(htd, null, null);
357 
358     long ts = System.currentTimeMillis();
359 
360     Put p = new Put(T1, ts);
361     p.add(c0, c0, T1);
362     region.put(p);
363 
364     // a put into another store (CF) should have no effect
365     p = new Put(T1, ts-10);
366     p.add(c1, c0, T1);
367     region.put(p);
368 
369     // all the following deletes affect the put
370     Delete d = new Delete(T1, ts, null);
371     d.deleteColumns(c0, c0, ts);
372     region.delete(d, null, true);
373 
374     d = new Delete(T1, ts, null);
375     d.deleteFamily(c0, ts);
376     region.delete(d, null, true);
377 
378     d = new Delete(T1, ts, null);
379     d.deleteColumn(c0, c0, ts+1);
380     region.delete(d, null, true);
381     
382     d = new Delete(T1, ts, null);
383     d.deleteColumn(c0, c0, ts+2);
384     region.delete(d, null, true);
385 
386     // 1 family marker, 1 column marker, 2 version markers
387     assertEquals(4, countDeleteMarkers(region));
388 
389     region.flushcache();
390     assertEquals(4, countDeleteMarkers(region));
391     region.compactStores(false);
392     assertEquals(4, countDeleteMarkers(region));
393 
394     // another put will push out the earlier put...
395     p = new Put(T1, ts+3);
396     p.add(c0, c0, T1);
397     region.put(p);
398 
399     region.flushcache();
400     // no markers are collected, since there is an affected put
401     region.compactStores(true);
402     assertEquals(4, countDeleteMarkers(region));
403 
404     // the last collections collected the earlier put
405     // so after this collection all markers
406     region.compactStores(true);
407     assertEquals(0, countDeleteMarkers(region));
408 
409     region.close();
410     region.getLog().closeAndDelete();
411   }
412 
413   /**
414    * Verify correct range demarcation
415    */
416   public void testRanges() throws Exception {
417     HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
418         HConstants.FOREVER, true);
419     HRegion region = createNewHRegion(htd, null, null);
420 
421     long ts = System.currentTimeMillis();
422     Put p = new Put(T1, ts);
423     p.add(c0, c0, T1);
424     p.add(c0, c1, T1);
425     p.add(c1, c0, T1);
426     p.add(c1, c1, T1);
427     region.put(p);
428 
429     p = new Put(T2, ts);
430     p.add(c0, c0, T1);
431     p.add(c0, c1, T1);
432     p.add(c1, c0, T1);
433     p.add(c1, c1, T1);
434     region.put(p);
435 
436     p = new Put(T1, ts+1);
437     p.add(c0, c0, T2);
438     p.add(c0, c1, T2);
439     p.add(c1, c0, T2);
440     p.add(c1, c1, T2);
441     region.put(p);
442 
443     p = new Put(T2, ts+1);
444     p.add(c0, c0, T2);
445     p.add(c0, c1, T2);
446     p.add(c1, c0, T2);
447     p.add(c1, c1, T2);
448     region.put(p);
449 
450     Delete d = new Delete(T1, ts+2, null);
451     d.deleteColumns(c0, c0, ts+2);
452     region.delete(d, null, true);
453 
454     d = new Delete(T1, ts+2, null);
455     d.deleteFamily(c1, ts+2);
456     region.delete(d, null, true);
457 
458     d = new Delete(T2, ts+2, null);
459     d.deleteFamily(c0, ts+2);
460     region.delete(d, null, true);
461 
462     // add an older delete, to make sure it is filtered
463     d = new Delete(T1, ts-10, null);
464     d.deleteFamily(c1, ts-10);
465     region.delete(d, null, true);
466 
467     // ts + 2 does NOT include the delete at ts+2
468     checkGet(region, T1, c0, c0, ts+2, T2, T1);
469     checkGet(region, T1, c0, c1, ts+2, T2, T1);
470     checkGet(region, T1, c1, c0, ts+2, T2, T1);
471     checkGet(region, T1, c1, c1, ts+2, T2, T1);
472 
473     checkGet(region, T2, c0, c0, ts+2, T2, T1);
474     checkGet(region, T2, c0, c1, ts+2, T2, T1);
475     checkGet(region, T2, c1, c0, ts+2, T2, T1);
476     checkGet(region, T2, c1, c1, ts+2, T2, T1);
477 
478     // ts + 3 does
479     checkGet(region, T1, c0, c0, ts+3);
480     checkGet(region, T1, c0, c1, ts+3, T2, T1);
481     checkGet(region, T1, c1, c0, ts+3);
482     checkGet(region, T1, c1, c1, ts+3);
483 
484     checkGet(region, T2, c0, c0, ts+3);
485     checkGet(region, T2, c0, c1, ts+3);
486     checkGet(region, T2, c1, c0, ts+3, T2, T1);
487     checkGet(region, T2, c1, c1, ts+3, T2, T1);
488 
489     region.close();
490     region.getLog().closeAndDelete();
491   }
492 
493   /**
494    * Verify that column/version delete makers are sorted
495    * with their respective puts and removed correctly by
496    * versioning (i.e. not relying on the store earliestPutTS).
497    */
498   public void testDeleteMarkerVersioning() throws Exception {
499     HTableDescriptor htd = createTableDescriptor(getName(), 0, 1,
500         HConstants.FOREVER, true);
501     HRegion region = createNewHRegion(htd, null, null);
502 
503     long ts = System.currentTimeMillis();
504     Put p = new Put(T1, ts);
505     p.add(c0, c0, T1);
506     region.put(p);
507 
508     // this prevents marker collection based on earliestPut
509     // (cannot keep earliest put per column in the store file)
510     p = new Put(T1, ts-10);
511     p.add(c0, c1, T1);
512     region.put(p);
513     
514     Delete d = new Delete(T1, ts, null);
515     // test corner case (Put and Delete have same TS)
516     d.deleteColumns(c0, c0, ts);
517     region.delete(d, null, true);
518 
519     d = new Delete(T1, ts+1, null);
520     d.deleteColumn(c0, c0, ts+1);
521     region.delete(d, null, true);
522     
523     d = new Delete(T1, ts+3, null);
524     d.deleteColumn(c0, c0, ts+3);
525     region.delete(d, null, true);
526 
527     region.flushcache();
528     region.compactStores(true);
529     region.compactStores(true);
530     assertEquals(3, countDeleteMarkers(region));
531 
532     // add two more puts, since max version is 1
533     // the 2nd put (and all delete markers following)
534     // will be removed.
535     p = new Put(T1, ts+2);
536     p.add(c0, c0, T2);
537     region.put(p);
538     
539     // delete, put, delete, delete, put
540     assertEquals(3, countDeleteMarkers(region));
541 
542     p = new Put(T1, ts+3);
543     p.add(c0, c0, T3);
544     region.put(p);
545 
546     // This is potentially questionable behavior.
547     // This could be changed by not letting the ScanQueryMatcher
548     // return SEEK_NEXT_COL if a put is past VERSIONS, but instead
549     // return SKIP if the store has KEEP_DELETED_CELLS set.
550     //
551     // As it stands, the 1 here is correct here.
552     // There are two puts, VERSIONS is one, so after the 1st put the scanner
553     // knows that there can be no more KVs (put or delete) that have any effect.
554     //
555     // delete, put, put | delete, delete
556     assertEquals(1, countDeleteMarkers(region));
557 
558     // flush cache only sees what is in the memstore
559     region.flushcache();
560 
561     // Here we have the three markers again, because the flush above
562     // removed the 2nd put before the file is written.
563     // So there's only one put, and hence the deletes already in the store
564     // files cannot be removed safely.
565     // delete, put, delete, delete
566     assertEquals(3, countDeleteMarkers(region));
567 
568     region.compactStores(true);
569     assertEquals(3, countDeleteMarkers(region));
570 
571     // add one more put
572     p = new Put(T1, ts+4);
573     p.add(c0, c0, T4);
574     region.put(p);
575 
576     region.flushcache();
577     // one trailing delete marker remains (but only one)
578     // because delete markers do not increase the version count
579     assertEquals(1, countDeleteMarkers(region));
580     region.compactStores(true);
581     region.compactStores(true);
582     assertEquals(1, countDeleteMarkers(region));
583 
584     region.close();
585     region.getLog().closeAndDelete();
586   }
587 
588   /**
589    * Verify scenarios with multiple CFs and columns
590    */
591   public void testWithMixedCFs() throws Exception {
592     HTableDescriptor htd = createTableDescriptor(getName(), 0, 1,
593         HConstants.FOREVER, true);
594     HRegion region = createNewHRegion(htd, null, null);
595 
596     long ts = System.currentTimeMillis();
597 
598     Put p = new Put(T1, ts);
599     p.add(c0, c0, T1);
600     p.add(c0, c1, T1);
601     p.add(c1, c0, T1);
602     p.add(c1, c1, T1);
603     region.put(p);
604 
605     p = new Put(T2, ts+1);
606     p.add(c0, c0, T2);
607     p.add(c0, c1, T2);
608     p.add(c1, c0, T2);
609     p.add(c1, c1, T2);
610     region.put(p);
611 
612     // family markers are each family
613     Delete d = new Delete(T1, ts+1, null);
614     region.delete(d, null, true);
615 
616     d = new Delete(T2, ts+2, null);
617     region.delete(d, null, true);
618 
619     Scan s = new Scan(T1);
620     s.setTimeRange(0, ts+1);
621     InternalScanner scanner = region.getScanner(s);
622     List<KeyValue> kvs = new ArrayList<KeyValue>();
623     scanner.next(kvs);
624     assertEquals(4, kvs.size());
625     scanner.close();
626 
627     s = new Scan(T2);
628     s.setTimeRange(0, ts+2);
629     scanner = region.getScanner(s);
630     kvs = new ArrayList<KeyValue>();
631     scanner.next(kvs);
632     assertEquals(4, kvs.size());
633     scanner.close();
634 
635     region.close();
636     region.getLog().closeAndDelete();
637   }
638 
639   /**
640    * Test keeping deleted rows together with min versions set
641    * @throws Exception
642    */
643   public void testWithMinVersions() throws Exception {
644     HTableDescriptor htd = createTableDescriptor(getName(), 3, 1000, 1, true);
645     HRegion region = createNewHRegion(htd, null, null);
646 
647     long ts = System.currentTimeMillis() - 2000; // 2s in the past
648 
649     Put p = new Put(T1, ts);
650     p.add(c0, c0, T3);
651     region.put(p);
652     p = new Put(T1, ts-1);
653     p.add(c0, c0, T2);
654     region.put(p);
655     p = new Put(T1, ts-3);
656     p.add(c0, c0, T1);
657     region.put(p);
658     p = new Put(T1, ts-4);
659     p.add(c0, c0, T0);
660     region.put(p);
661 
662     // all puts now are just retained because of min versions = 3
663 
664     // place a family delete marker
665     Delete d = new Delete(T1, ts-1, null);
666     region.delete(d, null, true);
667     // and a column delete marker
668     d = new Delete(T1, ts-2, null);
669     d.deleteColumns(c0, c0, ts-1);
670     region.delete(d, null, true);
671 
672     Get g = new Get(T1);
673     g.setMaxVersions();
674     g.setTimeRange(0L, ts-2);
675     Result r = region.get(g, null);
676     checkResult(r, c0, c0, T1,T0);
677 
678     // 3 families, one column delete marker
679     assertEquals(4, countDeleteMarkers(region));
680 
681     region.flushcache();
682     // no delete marker removes by the flush
683     assertEquals(4, countDeleteMarkers(region));
684 
685     r = region.get(g, null);
686     checkResult(r, c0, c0, T1);
687     p = new Put(T1, ts+1);
688     p.add(c0, c0, T4);
689     region.put(p);
690     region.flushcache();
691 
692     assertEquals(4, countDeleteMarkers(region));
693 
694     r = region.get(g, null);
695     checkResult(r, c0, c0, T1);
696 
697     // this will push out the last put before
698     // family delete marker
699     p = new Put(T1, ts+2);
700     p.add(c0, c0, T5);
701     region.put(p);
702 
703     region.flushcache();
704     region.compactStores(true);
705     // the two family markers without puts are gone
706     assertEquals(2, countDeleteMarkers(region));
707 
708     // the last compactStores updated the earliestPutTs,
709     // so after the next compaction the last family delete marker is also gone
710     region.compactStores(true);
711     assertEquals(0, countDeleteMarkers(region));
712 
713     region.close();
714     region.getLog().closeAndDelete();
715   }
716 
717   private void checkGet(HRegion region, byte[] row, byte[] fam, byte[] col,
718       long time, byte[]... vals) throws IOException {
719     Get g = new Get(row);
720     g.addColumn(fam, col);
721     g.setMaxVersions();
722     g.setTimeRange(0L, time);
723     Result r = region.get(g, null);
724     checkResult(r, fam, col, vals);
725 
726   }
727 
728   private int countDeleteMarkers(HRegion region) throws IOException {
729     Scan s = new Scan();
730     s.setRaw(true);
731     s.setMaxVersions();
732     InternalScanner scan = region.getScanner(s);
733     List<KeyValue> kvs = new ArrayList<KeyValue>();
734     int res = 0;
735     boolean hasMore;
736     do {
737       hasMore = scan.next(kvs);
738       for (KeyValue kv : kvs) {
739         if(kv.isDelete()) res++;
740       }
741       kvs.clear();
742     } while (hasMore);
743     scan.close();
744     return res;
745   }
746 
747   private void checkResult(Result r, byte[] fam, byte[] col, byte[] ... vals) {
748     assertEquals(r.size(), vals.length);
749     List<KeyValue> kvs = r.getColumn(fam, col);
750     assertEquals(kvs.size(), vals.length);
751     for (int i=0;i<vals.length;i++) {
752       assertEquals(kvs.get(i).getValue(), vals[i]);
753     }
754   }
755 
756 
757   @org.junit.Rule
758   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
759     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
760 }
761