1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.coprocessor.example;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.List;
26  import java.util.Map;
27  
28  import org.apache.hadoop.hbase.HBaseTestingUtility;
29  import org.apache.hadoop.hbase.HColumnDescriptor;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.HTableDescriptor;
32  import org.apache.hadoop.hbase.KeyValue;
33  import org.apache.hadoop.hbase.MediumTests;
34  import org.apache.hadoop.hbase.client.HTable;
35  import org.apache.hadoop.hbase.client.Put;
36  import org.apache.hadoop.hbase.client.Result;
37  import org.apache.hadoop.hbase.client.Scan;
38  import org.apache.hadoop.hbase.client.coprocessor.Batch;
39  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
40  import org.apache.hadoop.hbase.coprocessor.example.BulkDeleteProtocol.DeleteType;
41  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
42  import org.apache.hadoop.hbase.filter.FilterList;
43  import org.apache.hadoop.hbase.filter.FilterList.Operator;
44  import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
45  import org.apache.hadoop.hbase.util.Bytes;
46  import org.junit.AfterClass;
47  import org.junit.BeforeClass;
48  import org.junit.Test;
49  import org.junit.experimental.categories.Category;
50  
51  @Category(MediumTests.class)
52  public class TestBulkDeleteProtocol {
53    private static final byte[] FAMILY1 = Bytes.toBytes("cf1");
54    private static final byte[] FAMILY2 = Bytes.toBytes("cf2");
55    private static final byte[] QUALIFIER1 = Bytes.toBytes("c1");
56    private static final byte[] QUALIFIER2 = Bytes.toBytes("c2");
57    private static final byte[] QUALIFIER3 = Bytes.toBytes("c3");
58    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
59  
60    @BeforeClass
61    public static void setupBeforeClass() throws Exception {
62      TEST_UTIL.getConfiguration().set(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
63          BulkDeleteEndpoint.class.getName());
64      TEST_UTIL.startMiniCluster(2);
65    }
66  
67    @AfterClass
68    public static void tearDownAfterClass() throws Exception {
69      TEST_UTIL.shutdownMiniCluster();
70    }
71  
72    @Test
73    public void testBulkDeleteEndpoint() throws Throwable {
74      byte[] tableName = Bytes.toBytes("testBulkDeleteEndpoint");
75      HTable ht = createTable(tableName);
76      List<Put> puts = new ArrayList<Put>(100);
77      for (int j = 0; j < 100; j++) {
78        byte[] rowkey = Bytes.toBytes(j);
79        puts.add(createPut(rowkey, "v1"));
80      }
81      ht.put(puts);
82      // Deleting all the rows.
83      long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, new Scan(), 500, DeleteType.ROW,
84          null);
85      assertEquals(100, noOfRowsDeleted);
86  
87      int rows = 0;
88      for (Result result : ht.getScanner(new Scan())) {
89        rows++;
90      }
91      assertEquals(0, rows);
92    }
93  
94    @Test
95    public void testBulkDeleteEndpointWhenRowBatchSizeLessThanRowsToDeleteFromARegion()
96        throws Throwable {
97      byte[] tableName = Bytes
98          .toBytes("testBulkDeleteEndpointWhenRowBatchSizeLessThanRowsToDeleteFromARegion");
99      HTable ht = createTable(tableName);
100     List<Put> puts = new ArrayList<Put>(100);
101     for (int j = 0; j < 100; j++) {
102       byte[] rowkey = Bytes.toBytes(j);
103       puts.add(createPut(rowkey, "v1"));
104     }
105     ht.put(puts);
106     // Deleting all the rows.
107     long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, new Scan(), 10, DeleteType.ROW, null);
108     assertEquals(100, noOfRowsDeleted);
109 
110     int rows = 0;
111     for (Result result : ht.getScanner(new Scan())) {
112       rows++;
113     }
114     assertEquals(0, rows);
115   }
116   
117   private long invokeBulkDeleteProtocol(byte[] tableName, final Scan scan, final int rowBatchSize,
118       final byte deleteType, final Long timeStamp) throws Throwable {
119     HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
120     long noOfDeletedRows = 0L;
121     Batch.Call<BulkDeleteProtocol, BulkDeleteResponse> callable = 
122         new Batch.Call<BulkDeleteProtocol, BulkDeleteResponse>() {
123       public BulkDeleteResponse call(BulkDeleteProtocol instance) throws IOException {
124         return instance.delete(scan, deleteType, timeStamp, rowBatchSize);
125       }
126     };
127     Map<byte[], BulkDeleteResponse> result = ht.coprocessorExec(BulkDeleteProtocol.class,
128         scan.getStartRow(), scan.getStopRow(), callable);
129     for (BulkDeleteResponse response : result.values()) {
130       noOfDeletedRows += response.getRowsDeleted();
131     }
132     return noOfDeletedRows;
133   }
134 
135   @Test
136   public void testBulkDeleteWithConditionBasedDelete() throws Throwable {
137     byte[] tableName = Bytes.toBytes("testBulkDeleteWithConditionBasedDelete");
138     HTable ht = createTable(tableName);
139     List<Put> puts = new ArrayList<Put>(100);
140     for (int j = 0; j < 100; j++) {
141       byte[] rowkey = Bytes.toBytes(j);
142       String value = (j % 10 == 0) ? "v1" : "v2";
143       puts.add(createPut(rowkey, value));
144     }
145     ht.put(puts);
146     Scan scan = new Scan();
147     FilterList fl = new FilterList(Operator.MUST_PASS_ALL);
148     SingleColumnValueFilter scvf = new SingleColumnValueFilter(FAMILY1, QUALIFIER3,
149         CompareOp.EQUAL, Bytes.toBytes("v1"));
150     //fl.addFilter(new FirstKeyOnlyFilter());
151     fl.addFilter(scvf);
152     scan.setFilter(fl);
153     // Deleting all the rows where cf1:c1=v1
154     long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.ROW, null);
155     assertEquals(10, noOfRowsDeleted);
156 
157     int rows = 0;
158     for (Result result : ht.getScanner(new Scan())) {
159       rows++;
160     }
161     assertEquals(90, rows);
162   }
163 
164   @Test
165   public void testBulkDeleteColumn() throws Throwable {
166     byte[] tableName = Bytes.toBytes("testBulkDeleteColumn");
167     HTable ht = createTable(tableName);
168     List<Put> puts = new ArrayList<Put>(100);
169     for (int j = 0; j < 100; j++) {
170       byte[] rowkey = Bytes.toBytes(j);
171       String value = (j % 10 == 0) ? "v1" : "v2";
172       puts.add(createPut(rowkey, value));
173     }
174     ht.put(puts);
175     Scan scan = new Scan ();
176     scan.addColumn(FAMILY1, QUALIFIER2);
177     // Delete the column cf1:col2
178     long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.COLUMN, null);
179     assertEquals(100, noOfRowsDeleted);
180 
181     int rows = 0;
182     for (Result result : ht.getScanner(new Scan())) {
183       assertEquals(2, result.getFamilyMap(FAMILY1).size());
184       assertTrue(result.getColumn(FAMILY1, QUALIFIER2).isEmpty());
185       assertEquals(1, result.getColumn(FAMILY1, QUALIFIER1).size());
186       assertEquals(1, result.getColumn(FAMILY1, QUALIFIER3).size());
187       rows++;
188     }
189     assertEquals(100, rows);
190   }
191   
192   @Test
193   public void testBulkDeleteFamily() throws Throwable {
194     byte[] tableName = Bytes.toBytes("testBulkDeleteFamily");
195     HTableDescriptor htd = new HTableDescriptor(tableName);
196     htd.addFamily(new HColumnDescriptor(FAMILY1));
197     htd.addFamily(new HColumnDescriptor(FAMILY2));
198     TEST_UTIL.getHBaseAdmin().createTable(htd, Bytes.toBytes(0), Bytes.toBytes(120), 5);
199     HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
200     List<Put> puts = new ArrayList<Put>(100);
201     for (int j = 0; j < 100; j++) {
202       Put put = new Put(Bytes.toBytes(j));
203       put.add(FAMILY1, QUALIFIER1, "v1".getBytes());
204       put.add(FAMILY2, QUALIFIER2, "v2".getBytes());
205       puts.add(put);
206     }
207     ht.put(puts);
208     Scan scan = new Scan ();
209     scan.addFamily(FAMILY1);
210     // Delete the column family cf1
211     long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.FAMILY, null);
212     assertEquals(100, noOfRowsDeleted);
213     int rows = 0;
214     for (Result result : ht.getScanner(new Scan())) {
215       assertTrue(result.getFamilyMap(FAMILY1).isEmpty());
216       assertEquals(1, result.getColumn(FAMILY2, QUALIFIER2).size());
217       rows++;
218     }
219     assertEquals(100, rows);
220   }
221   
222   @Test
223   public void testBulkDeleteColumnVersion() throws Throwable {
224     byte[] tableName = Bytes.toBytes("testBulkDeleteColumnVersion");
225     HTable ht = createTable(tableName);
226     List<Put> puts = new ArrayList<Put>(100);
227     for (int j = 0; j < 100; j++) {
228       Put put = new Put(Bytes.toBytes(j));
229       byte[] value = "v1".getBytes();
230       put.add(FAMILY1, QUALIFIER1, 1234L, value);
231       put.add(FAMILY1, QUALIFIER2, 1234L, value);
232       put.add(FAMILY1, QUALIFIER3, 1234L, value);
233       // Latest version values
234       value = "v2".getBytes();
235       put.add(FAMILY1, QUALIFIER1, value);
236       put.add(FAMILY1, QUALIFIER2, value);
237       put.add(FAMILY1, QUALIFIER3, value);
238       put.add(FAMILY1, null, value);
239       puts.add(put);
240     }
241     ht.put(puts);
242     Scan scan = new Scan ();
243     scan.addFamily(FAMILY1);
244     // Delete the latest version values of all the columns in family cf1.
245     long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.VERSION,
246         HConstants.LATEST_TIMESTAMP);
247     assertEquals(100, noOfRowsDeleted);
248     int rows = 0;
249     scan = new Scan ();
250     scan.setMaxVersions();
251     for (Result result : ht.getScanner(scan)) {
252       assertEquals(3, result.getFamilyMap(FAMILY1).size());
253       List<KeyValue> column = result.getColumn(FAMILY1, QUALIFIER1);
254       assertEquals(1, column.size());
255       assertTrue(Bytes.equals("v1".getBytes(), column.get(0).getValue()));
256       
257       column = result.getColumn(FAMILY1, QUALIFIER2);
258       assertEquals(1, column.size());
259       assertTrue(Bytes.equals("v1".getBytes(), column.get(0).getValue()));
260       
261       column = result.getColumn(FAMILY1, QUALIFIER3);
262       assertEquals(1, column.size());
263       assertTrue(Bytes.equals("v1".getBytes(), column.get(0).getValue()));
264       rows++;
265     }
266     assertEquals(100, rows);
267   }
268   
269   @Test
270   public void testBulkDeleteColumnVersionBasedOnTS() throws Throwable {
271     byte[] tableName = Bytes.toBytes("testBulkDeleteColumnVersionBasedOnTS");
272     HTable ht = createTable(tableName);
273     List<Put> puts = new ArrayList<Put>(100);
274     for (int j = 0; j < 100; j++) {
275       Put put = new Put(Bytes.toBytes(j));
276       // TS = 1000L
277       byte[] value = "v1".getBytes();
278       put.add(FAMILY1, QUALIFIER1, 1000L, value);
279       put.add(FAMILY1, QUALIFIER2, 1000L, value);
280       put.add(FAMILY1, QUALIFIER3, 1000L, value);
281       // TS = 1234L
282       value = "v2".getBytes();
283       put.add(FAMILY1, QUALIFIER1, 1234L, value);
284       put.add(FAMILY1, QUALIFIER2, 1234L, value);
285       put.add(FAMILY1, QUALIFIER3, 1234L, value);
286       // Latest version values
287       value = "v3".getBytes();
288       put.add(FAMILY1, QUALIFIER1, value);
289       put.add(FAMILY1, QUALIFIER2, value);
290       put.add(FAMILY1, QUALIFIER3, value);
291       puts.add(put);
292     }
293     ht.put(puts);
294     Scan scan = new Scan ();
295     scan.addColumn(FAMILY1, QUALIFIER3);
296     // Delete the column cf1:c3's one version at TS=1234 
297     long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.VERSION, 1234L);
298     assertEquals(100, noOfRowsDeleted);
299     int rows = 0;
300     scan = new Scan ();
301     scan.setMaxVersions();
302     for (Result result : ht.getScanner(scan)) {
303       assertEquals(3, result.getFamilyMap(FAMILY1).size());
304       assertEquals(3, result.getColumn(FAMILY1, QUALIFIER1).size());
305       assertEquals(3, result.getColumn(FAMILY1, QUALIFIER2).size());
306       List<KeyValue> column = result.getColumn(FAMILY1, QUALIFIER3);
307       assertEquals(2, column.size());
308       assertTrue(Bytes.equals("v3".getBytes(), column.get(0).getValue()));
309       assertTrue(Bytes.equals("v1".getBytes(), column.get(1).getValue()));
310       rows++;
311     }
312     assertEquals(100, rows);
313   }
314   
315   @Test
316   public void testBulkDeleteWithNumberOfVersions() throws Throwable {
317     byte[] tableName = Bytes.toBytes("testBulkDeleteWithNumberOfVersions");
318     HTable ht = createTable(tableName);
319     List<Put> puts = new ArrayList<Put>(100);
320     for (int j = 0; j < 100; j++) {
321       Put put = new Put(Bytes.toBytes(j));
322       // TS = 1000L
323       byte[] value = "v1".getBytes();
324       put.add(FAMILY1, QUALIFIER1, 1000L, value);
325       put.add(FAMILY1, QUALIFIER2, 1000L, value);
326       put.add(FAMILY1, QUALIFIER3, 1000L, value);
327       // TS = 1234L
328       value = "v2".getBytes();
329       put.add(FAMILY1, QUALIFIER1, 1234L, value);
330       put.add(FAMILY1, QUALIFIER2, 1234L, value);
331       put.add(FAMILY1, QUALIFIER3, 1234L, value);
332       // TS = 2000L
333       value = "v3".getBytes();
334       put.add(FAMILY1, QUALIFIER1, 2000L, value);
335       put.add(FAMILY1, QUALIFIER2, 2000L, value);
336       put.add(FAMILY1, QUALIFIER3, 2000L, value);
337       // Latest version values
338       value = "v4".getBytes();
339       put.add(FAMILY1, QUALIFIER1, value);
340       put.add(FAMILY1, QUALIFIER2, value);
341       put.add(FAMILY1, QUALIFIER3, value);
342       puts.add(put);
343     }
344     ht.put(puts);
345     
346     // Delete all the versions of columns cf1:c1 and cf1:c2 falling with the time range
347     // [1000,2000)
348     final Scan scan = new Scan();
349     scan.addColumn(FAMILY1, QUALIFIER1);
350     scan.addColumn(FAMILY1, QUALIFIER2);
351     scan.setTimeRange(1000L, 2000L);
352     scan.setMaxVersions();
353     
354     long noOfDeletedRows = 0L;
355     long noOfVersionsDeleted = 0L;
356     Batch.Call<BulkDeleteProtocol, BulkDeleteResponse> callable = 
357         new Batch.Call<BulkDeleteProtocol, BulkDeleteResponse>() {
358       public BulkDeleteResponse call(BulkDeleteProtocol instance) throws IOException {
359         return instance.delete(scan, DeleteType.VERSION, null, 500);
360       }
361     };
362     Map<byte[], BulkDeleteResponse> result = ht.coprocessorExec(BulkDeleteProtocol.class,
363         scan.getStartRow(), scan.getStopRow(), callable);
364     for (BulkDeleteResponse response : result.values()) {
365       noOfDeletedRows += response.getRowsDeleted();
366       noOfVersionsDeleted += response.getVersionsDeleted();
367     }
368     assertEquals(100, noOfDeletedRows);
369     assertEquals(400, noOfVersionsDeleted);
370     
371     int rows = 0;
372     Scan scan1 = new Scan ();
373     scan1.setMaxVersions();
374     for (Result res : ht.getScanner(scan1)) {
375       assertEquals(3, res.getFamilyMap(FAMILY1).size());
376       List<KeyValue> column = res.getColumn(FAMILY1, QUALIFIER1);
377       assertEquals(2, column.size());
378       assertTrue(Bytes.equals("v4".getBytes(), column.get(0).getValue()));
379       assertTrue(Bytes.equals("v3".getBytes(), column.get(1).getValue()));
380       column = res.getColumn(FAMILY1, QUALIFIER2);
381       assertEquals(2, column.size());
382       assertTrue(Bytes.equals("v4".getBytes(), column.get(0).getValue()));
383       assertTrue(Bytes.equals("v3".getBytes(), column.get(1).getValue()));
384       assertEquals(4, res.getColumn(FAMILY1, QUALIFIER3).size());
385       rows++;
386     }
387     assertEquals(100, rows);
388   }
389   
390   private HTable createTable(byte[] tableName) throws IOException {
391     HTableDescriptor htd = new HTableDescriptor(tableName);
392     HColumnDescriptor hcd = new HColumnDescriptor(FAMILY1);
393     hcd.setMaxVersions(10);// Just setting 10 as I am not testing with more than 10 versions here
394     htd.addFamily(hcd);
395     TEST_UTIL.getHBaseAdmin().createTable(htd, Bytes.toBytes(0), Bytes.toBytes(120), 5);
396     HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
397     return ht;
398   }
399 
400   private Put createPut(byte[] rowkey, String value) throws IOException {
401     Put put = new Put(rowkey);
402     put.add(FAMILY1, QUALIFIER1, value.getBytes());
403     put.add(FAMILY1, QUALIFIER2, value.getBytes());
404     put.add(FAMILY1, QUALIFIER3, value.getBytes());
405     return put;
406   }
407 }