View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.coprocessor.example;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.List;
26  import java.util.Map;
27  
28  import org.apache.hadoop.hbase.HBaseTestingUtility;
29  import org.apache.hadoop.hbase.HColumnDescriptor;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.HTableDescriptor;
32  import org.apache.hadoop.hbase.KeyValue;
33  import org.apache.hadoop.hbase.MediumTests;
34  import org.apache.hadoop.hbase.TableName;
35  import org.apache.hadoop.hbase.client.HTable;
36  import org.apache.hadoop.hbase.client.Put;
37  import org.apache.hadoop.hbase.client.Result;
38  import org.apache.hadoop.hbase.client.Scan;
39  import org.apache.hadoop.hbase.client.coprocessor.Batch;
40  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
41  import org.apache.hadoop.hbase.coprocessor.example.generated.BulkDeleteProtos.BulkDeleteRequest;
42  import org.apache.hadoop.hbase.coprocessor.example.generated.BulkDeleteProtos.BulkDeleteResponse;
43  import org.apache.hadoop.hbase.coprocessor.example.generated.BulkDeleteProtos.BulkDeleteService;
44  import org.apache.hadoop.hbase.coprocessor.example.generated.BulkDeleteProtos.BulkDeleteRequest.Builder;
45  import org.apache.hadoop.hbase.coprocessor.example.generated.BulkDeleteProtos.BulkDeleteRequest.DeleteType;
46  import org.apache.hadoop.hbase.filter.FilterList;
47  import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
48  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
49  import org.apache.hadoop.hbase.filter.FilterList.Operator;
50  import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
51  import org.apache.hadoop.hbase.ipc.ServerRpcController;
52  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
53  import org.apache.hadoop.hbase.util.Bytes;
54  import org.junit.AfterClass;
55  import org.junit.BeforeClass;
56  import org.junit.Test;
57  import org.junit.Ignore;
58  import org.junit.experimental.categories.Category;
59  
60  @Category(MediumTests.class)
61  public class TestBulkDeleteProtocol {
62    private static final byte[] FAMILY1 = Bytes.toBytes("cf1");
63    private static final byte[] FAMILY2 = Bytes.toBytes("cf2");
64    private static final byte[] QUALIFIER1 = Bytes.toBytes("c1");
65    private static final byte[] QUALIFIER2 = Bytes.toBytes("c2");
66    private static final byte[] QUALIFIER3 = Bytes.toBytes("c3");
67    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
68  
69    // @Ignore @BeforeClass
70    public static void setupBeforeClass() throws Exception {
71      TEST_UTIL.getConfiguration().set(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
72          BulkDeleteEndpoint.class.getName());
73      TEST_UTIL.startMiniCluster(2);
74    }
75  
76    // @Ignore @AfterClass
77    public static void tearDownAfterClass() throws Exception {
78      TEST_UTIL.shutdownMiniCluster();
79    }
80  
81    // @Ignore @Test
82    public void testBulkDeleteEndpoint() throws Throwable {
83      byte[] tableName = Bytes.toBytes("testBulkDeleteEndpoint");
84      HTable ht = createTable(tableName);
85      List<Put> puts = new ArrayList<Put>(100);
86      for (int j = 0; j < 100; j++) {
87        byte[] rowkey = Bytes.toBytes(j);
88        puts.add(createPut(rowkey, "v1"));
89      }
90      ht.put(puts);
91      // Deleting all the rows.
92      long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, new Scan(), 5, DeleteType.ROW, null);
93      assertEquals(100, noOfRowsDeleted);
94  
95      int rows = 0;
96      for (Result result : ht.getScanner(new Scan())) {
97        rows++;
98      }
99      assertEquals(0, rows);
100     ht.close();
101   }
102 
103   // @Ignore @Test
104   public void testBulkDeleteEndpointWhenRowBatchSizeLessThanRowsToDeleteFromARegion()
105       throws Throwable {
106     byte[] tableName = Bytes
107         .toBytes("testBulkDeleteEndpointWhenRowBatchSizeLessThanRowsToDeleteFromARegion");
108     HTable ht = createTable(tableName);
109     List<Put> puts = new ArrayList<Put>(100);
110     for (int j = 0; j < 100; j++) {
111       byte[] rowkey = Bytes.toBytes(j);
112       puts.add(createPut(rowkey, "v1"));
113     }
114     ht.put(puts);
115     // Deleting all the rows.
116     long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, new Scan(), 10, DeleteType.ROW, null);
117     assertEquals(100, noOfRowsDeleted);
118 
119     int rows = 0;
120     for (Result result : ht.getScanner(new Scan())) {
121       rows++;
122     }
123     assertEquals(0, rows);
124     ht.close();
125   }
126 
127   private long invokeBulkDeleteProtocol(byte[] tableName, final Scan scan, final int rowBatchSize,
128       final DeleteType deleteType, final Long timeStamp) throws Throwable {
129     HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
130     long noOfDeletedRows = 0L;
131     Batch.Call<BulkDeleteService, BulkDeleteResponse> callable =
132       new Batch.Call<BulkDeleteService, BulkDeleteResponse>() {
133       ServerRpcController controller = new ServerRpcController();
134       BlockingRpcCallback<BulkDeleteResponse> rpcCallback =
135         new BlockingRpcCallback<BulkDeleteResponse>();
136 
137       public BulkDeleteResponse call(BulkDeleteService service) throws IOException {
138         Builder builder = BulkDeleteRequest.newBuilder();
139         builder.setScan(ProtobufUtil.toScan(scan));
140         builder.setDeleteType(deleteType);
141         builder.setRowBatchSize(rowBatchSize);
142         if (timeStamp != null) {
143           builder.setTimestamp(timeStamp);
144         }
145         service.delete(controller, builder.build(), rpcCallback);
146         return rpcCallback.get();
147       }
148     };
149     Map<byte[], BulkDeleteResponse> result = ht.coprocessorService(BulkDeleteService.class, scan
150         .getStartRow(), scan.getStopRow(), callable);
151     for (BulkDeleteResponse response : result.values()) {
152       noOfDeletedRows += response.getRowsDeleted();
153     }
154     ht.close();
155     return noOfDeletedRows;
156   }
157 
158   // @Ignore @Test
159   public void testBulkDeleteWithConditionBasedDelete() throws Throwable {
160     byte[] tableName = Bytes.toBytes("testBulkDeleteWithConditionBasedDelete");
161     HTable ht = createTable(tableName);
162     List<Put> puts = new ArrayList<Put>(100);
163     for (int j = 0; j < 100; j++) {
164       byte[] rowkey = Bytes.toBytes(j);
165       String value = (j % 10 == 0) ? "v1" : "v2";
166       puts.add(createPut(rowkey, value));
167     }
168     ht.put(puts);
169     Scan scan = new Scan();
170     FilterList fl = new FilterList(Operator.MUST_PASS_ALL);
171     SingleColumnValueFilter scvf = new SingleColumnValueFilter(FAMILY1, QUALIFIER3,
172         CompareOp.EQUAL, Bytes.toBytes("v1"));
173     // fl.addFilter(new FirstKeyOnlyFilter());
174     fl.addFilter(scvf);
175     scan.setFilter(fl);
176     // Deleting all the rows where cf1:c1=v1
177     long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.ROW, null);
178     assertEquals(10, noOfRowsDeleted);
179 
180     int rows = 0;
181     for (Result result : ht.getScanner(new Scan())) {
182       rows++;
183     }
184     assertEquals(90, rows);
185     ht.close();
186   }
187 
188   // @Ignore @Test
189   public void testBulkDeleteColumn() throws Throwable {
190     byte[] tableName = Bytes.toBytes("testBulkDeleteColumn");
191     HTable ht = createTable(tableName);
192     List<Put> puts = new ArrayList<Put>(100);
193     for (int j = 0; j < 100; j++) {
194       byte[] rowkey = Bytes.toBytes(j);
195       String value = (j % 10 == 0) ? "v1" : "v2";
196       puts.add(createPut(rowkey, value));
197     }
198     ht.put(puts);
199     Scan scan = new Scan();
200     scan.addColumn(FAMILY1, QUALIFIER2);
201     // Delete the column cf1:col2
202     long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.COLUMN, null);
203     assertEquals(100, noOfRowsDeleted);
204 
205     int rows = 0;
206     for (Result result : ht.getScanner(new Scan())) {
207       assertEquals(2, result.getFamilyMap(FAMILY1).size());
208       assertTrue(result.getColumn(FAMILY1, QUALIFIER2).isEmpty());
209       assertEquals(1, result.getColumn(FAMILY1, QUALIFIER1).size());
210       assertEquals(1, result.getColumn(FAMILY1, QUALIFIER3).size());
211       rows++;
212     }
213     assertEquals(100, rows);
214     ht.close();
215   }
216 
217   // @Ignore @Test
218   public void testBulkDeleteFamily() throws Throwable {
219     byte[] tableName = Bytes.toBytes("testBulkDeleteFamily");
220     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
221     htd.addFamily(new HColumnDescriptor(FAMILY1));
222     htd.addFamily(new HColumnDescriptor(FAMILY2));
223     TEST_UTIL.getHBaseAdmin().createTable(htd, Bytes.toBytes(0), Bytes.toBytes(120), 5);
224     HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
225     List<Put> puts = new ArrayList<Put>(100);
226     for (int j = 0; j < 100; j++) {
227       Put put = new Put(Bytes.toBytes(j));
228       put.add(FAMILY1, QUALIFIER1, "v1".getBytes());
229       put.add(FAMILY2, QUALIFIER2, "v2".getBytes());
230       puts.add(put);
231     }
232     ht.put(puts);
233     Scan scan = new Scan();
234     scan.addFamily(FAMILY1);
235     // Delete the column family cf1
236     long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.FAMILY, null);
237     assertEquals(100, noOfRowsDeleted);
238     int rows = 0;
239     for (Result result : ht.getScanner(new Scan())) {
240       assertTrue(result.getFamilyMap(FAMILY1).isEmpty());
241       assertEquals(1, result.getColumn(FAMILY2, QUALIFIER2).size());
242       rows++;
243     }
244     assertEquals(100, rows);
245     ht.close();
246   }
247 
248   // @Ignore @Test
249   public void testBulkDeleteColumnVersion() throws Throwable {
250     byte[] tableName = Bytes.toBytes("testBulkDeleteColumnVersion");
251     HTable ht = createTable(tableName);
252     List<Put> puts = new ArrayList<Put>(100);
253     for (int j = 0; j < 100; j++) {
254       Put put = new Put(Bytes.toBytes(j));
255       byte[] value = "v1".getBytes();
256       put.add(FAMILY1, QUALIFIER1, 1234L, value);
257       put.add(FAMILY1, QUALIFIER2, 1234L, value);
258       put.add(FAMILY1, QUALIFIER3, 1234L, value);
259       // Latest version values
260       value = "v2".getBytes();
261       put.add(FAMILY1, QUALIFIER1, value);
262       put.add(FAMILY1, QUALIFIER2, value);
263       put.add(FAMILY1, QUALIFIER3, value);
264       put.add(FAMILY1, null, value);
265       puts.add(put);
266     }
267     ht.put(puts);
268     Scan scan = new Scan();
269     scan.addFamily(FAMILY1);
270     // Delete the latest version values of all the columns in family cf1.
271     long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.VERSION,
272         HConstants.LATEST_TIMESTAMP);
273     assertEquals(100, noOfRowsDeleted);
274     int rows = 0;
275     scan = new Scan();
276     scan.setMaxVersions();
277     for (Result result : ht.getScanner(scan)) {
278       assertEquals(3, result.getFamilyMap(FAMILY1).size());
279       List<KeyValue> column = result.getColumn(FAMILY1, QUALIFIER1);
280       assertEquals(1, column.size());
281       assertTrue(Bytes.equals("v1".getBytes(), column.get(0).getValue()));
282 
283       column = result.getColumn(FAMILY1, QUALIFIER2);
284       assertEquals(1, column.size());
285       assertTrue(Bytes.equals("v1".getBytes(), column.get(0).getValue()));
286 
287       column = result.getColumn(FAMILY1, QUALIFIER3);
288       assertEquals(1, column.size());
289       assertTrue(Bytes.equals("v1".getBytes(), column.get(0).getValue()));
290       rows++;
291     }
292     assertEquals(100, rows);
293     ht.close();
294   }
295 
296   // @Ignore @Test
297   public void testBulkDeleteColumnVersionBasedOnTS() throws Throwable {
298     byte[] tableName = Bytes.toBytes("testBulkDeleteColumnVersionBasedOnTS");
299     HTable ht = createTable(tableName);
300     List<Put> puts = new ArrayList<Put>(100);
301     for (int j = 0; j < 100; j++) {
302       Put put = new Put(Bytes.toBytes(j));
303       // TS = 1000L
304       byte[] value = "v1".getBytes();
305       put.add(FAMILY1, QUALIFIER1, 1000L, value);
306       put.add(FAMILY1, QUALIFIER2, 1000L, value);
307       put.add(FAMILY1, QUALIFIER3, 1000L, value);
308       // TS = 1234L
309       value = "v2".getBytes();
310       put.add(FAMILY1, QUALIFIER1, 1234L, value);
311       put.add(FAMILY1, QUALIFIER2, 1234L, value);
312       put.add(FAMILY1, QUALIFIER3, 1234L, value);
313       // Latest version values
314       value = "v3".getBytes();
315       put.add(FAMILY1, QUALIFIER1, value);
316       put.add(FAMILY1, QUALIFIER2, value);
317       put.add(FAMILY1, QUALIFIER3, value);
318       puts.add(put);
319     }
320     ht.put(puts);
321     Scan scan = new Scan();
322     scan.addColumn(FAMILY1, QUALIFIER3);
323     // Delete the column cf1:c3's one version at TS=1234
324     long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.VERSION, 1234L);
325     assertEquals(100, noOfRowsDeleted);
326     int rows = 0;
327     scan = new Scan();
328     scan.setMaxVersions();
329     for (Result result : ht.getScanner(scan)) {
330       assertEquals(3, result.getFamilyMap(FAMILY1).size());
331       assertEquals(3, result.getColumn(FAMILY1, QUALIFIER1).size());
332       assertEquals(3, result.getColumn(FAMILY1, QUALIFIER2).size());
333       List<KeyValue> column = result.getColumn(FAMILY1, QUALIFIER3);
334       assertEquals(2, column.size());
335       assertTrue(Bytes.equals("v3".getBytes(), column.get(0).getValue()));
336       assertTrue(Bytes.equals("v1".getBytes(), column.get(1).getValue()));
337       rows++;
338     }
339     assertEquals(100, rows);
340     ht.close();
341   }
342 
343   // @Ignore @Test
344   public void testBulkDeleteWithNumberOfVersions() throws Throwable {
345     byte[] tableName = Bytes.toBytes("testBulkDeleteWithNumberOfVersions");
346     HTable ht = createTable(tableName);
347     List<Put> puts = new ArrayList<Put>(100);
348     for (int j = 0; j < 100; j++) {
349       Put put = new Put(Bytes.toBytes(j));
350       // TS = 1000L
351       byte[] value = "v1".getBytes();
352       put.add(FAMILY1, QUALIFIER1, 1000L, value);
353       put.add(FAMILY1, QUALIFIER2, 1000L, value);
354       put.add(FAMILY1, QUALIFIER3, 1000L, value);
355       // TS = 1234L
356       value = "v2".getBytes();
357       put.add(FAMILY1, QUALIFIER1, 1234L, value);
358       put.add(FAMILY1, QUALIFIER2, 1234L, value);
359       put.add(FAMILY1, QUALIFIER3, 1234L, value);
360       // TS = 2000L
361       value = "v3".getBytes();
362       put.add(FAMILY1, QUALIFIER1, 2000L, value);
363       put.add(FAMILY1, QUALIFIER2, 2000L, value);
364       put.add(FAMILY1, QUALIFIER3, 2000L, value);
365       // Latest version values
366       value = "v4".getBytes();
367       put.add(FAMILY1, QUALIFIER1, value);
368       put.add(FAMILY1, QUALIFIER2, value);
369       put.add(FAMILY1, QUALIFIER3, value);
370       puts.add(put);
371     }
372     ht.put(puts);
373 
374     // Delete all the versions of columns cf1:c1 and cf1:c2 falling with the time range
375     // [1000,2000)
376     final Scan scan = new Scan();
377     scan.addColumn(FAMILY1, QUALIFIER1);
378     scan.addColumn(FAMILY1, QUALIFIER2);
379     scan.setTimeRange(1000L, 2000L);
380     scan.setMaxVersions();
381 
382     long noOfDeletedRows = 0L;
383     long noOfVersionsDeleted = 0L;
384     Batch.Call<BulkDeleteService, BulkDeleteResponse> callable =
385       new Batch.Call<BulkDeleteService, BulkDeleteResponse>() {
386       ServerRpcController controller = new ServerRpcController();
387       BlockingRpcCallback<BulkDeleteResponse> rpcCallback =
388         new BlockingRpcCallback<BulkDeleteResponse>();
389 
390       public BulkDeleteResponse call(BulkDeleteService service) throws IOException {
391         Builder builder = BulkDeleteRequest.newBuilder();
392         builder.setScan(ProtobufUtil.toScan(scan));
393         builder.setDeleteType(DeleteType.VERSION);
394         builder.setRowBatchSize(500);
395         service.delete(controller, builder.build(), rpcCallback);
396         return rpcCallback.get();
397       }
398     };
399     Map<byte[], BulkDeleteResponse> result = ht.coprocessorService(BulkDeleteService.class, scan
400         .getStartRow(), scan.getStopRow(), callable);
401     for (BulkDeleteResponse response : result.values()) {
402       noOfDeletedRows += response.getRowsDeleted();
403       noOfVersionsDeleted += response.getVersionsDeleted();
404     }
405     assertEquals(100, noOfDeletedRows);
406     assertEquals(400, noOfVersionsDeleted);
407 
408     int rows = 0;
409     Scan scan1 = new Scan();
410     scan1.setMaxVersions();
411     for (Result res : ht.getScanner(scan1)) {
412       assertEquals(3, res.getFamilyMap(FAMILY1).size());
413       List<KeyValue> column = res.getColumn(FAMILY1, QUALIFIER1);
414       assertEquals(2, column.size());
415       assertTrue(Bytes.equals("v4".getBytes(), column.get(0).getValue()));
416       assertTrue(Bytes.equals("v3".getBytes(), column.get(1).getValue()));
417       column = res.getColumn(FAMILY1, QUALIFIER2);
418       assertEquals(2, column.size());
419       assertTrue(Bytes.equals("v4".getBytes(), column.get(0).getValue()));
420       assertTrue(Bytes.equals("v3".getBytes(), column.get(1).getValue()));
421       assertEquals(4, res.getColumn(FAMILY1, QUALIFIER3).size());
422       rows++;
423     }
424     assertEquals(100, rows);
425     ht.close();
426   }
427 
428   private HTable createTable(byte[] tableName) throws IOException {
429     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
430     HColumnDescriptor hcd = new HColumnDescriptor(FAMILY1);
431     hcd.setMaxVersions(10);// Just setting 10 as I am not testing with more than 10 versions here
432     htd.addFamily(hcd);
433     TEST_UTIL.getHBaseAdmin().createTable(htd, Bytes.toBytes(0), Bytes.toBytes(120), 5);
434     HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
435     return ht;
436   }
437 
438   private Put createPut(byte[] rowkey, String value) throws IOException {
439     Put put = new Put(rowkey);
440     put.add(FAMILY1, QUALIFIER1, value.getBytes());
441     put.add(FAMILY1, QUALIFIER2, value.getBytes());
442     put.add(FAMILY1, QUALIFIER3, value.getBytes());
443     return put;
444   }
445 }