1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.coprocessor.example;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.List;
26 import java.util.Map;
27
28 import org.apache.hadoop.hbase.HBaseTestingUtility;
29 import org.apache.hadoop.hbase.HColumnDescriptor;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.HTableDescriptor;
32 import org.apache.hadoop.hbase.KeyValue;
33 import org.apache.hadoop.hbase.MediumTests;
34 import org.apache.hadoop.hbase.TableName;
35 import org.apache.hadoop.hbase.client.HTable;
36 import org.apache.hadoop.hbase.client.Put;
37 import org.apache.hadoop.hbase.client.Result;
38 import org.apache.hadoop.hbase.client.Scan;
39 import org.apache.hadoop.hbase.client.coprocessor.Batch;
40 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
41 import org.apache.hadoop.hbase.coprocessor.example.generated.BulkDeleteProtos.BulkDeleteRequest;
42 import org.apache.hadoop.hbase.coprocessor.example.generated.BulkDeleteProtos.BulkDeleteResponse;
43 import org.apache.hadoop.hbase.coprocessor.example.generated.BulkDeleteProtos.BulkDeleteService;
44 import org.apache.hadoop.hbase.coprocessor.example.generated.BulkDeleteProtos.BulkDeleteRequest.Builder;
45 import org.apache.hadoop.hbase.coprocessor.example.generated.BulkDeleteProtos.BulkDeleteRequest.DeleteType;
46 import org.apache.hadoop.hbase.filter.FilterList;
47 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
48 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
49 import org.apache.hadoop.hbase.filter.FilterList.Operator;
50 import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
51 import org.apache.hadoop.hbase.ipc.ServerRpcController;
52 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
53 import org.apache.hadoop.hbase.util.Bytes;
54 import org.junit.AfterClass;
55 import org.junit.BeforeClass;
56 import org.junit.Test;
57 import org.junit.Ignore;
58 import org.junit.experimental.categories.Category;
59
60 @Category(MediumTests.class)
61 public class TestBulkDeleteProtocol {
62 private static final byte[] FAMILY1 = Bytes.toBytes("cf1");
63 private static final byte[] FAMILY2 = Bytes.toBytes("cf2");
64 private static final byte[] QUALIFIER1 = Bytes.toBytes("c1");
65 private static final byte[] QUALIFIER2 = Bytes.toBytes("c2");
66 private static final byte[] QUALIFIER3 = Bytes.toBytes("c3");
67 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
68
69
70 public static void setupBeforeClass() throws Exception {
71 TEST_UTIL.getConfiguration().set(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
72 BulkDeleteEndpoint.class.getName());
73 TEST_UTIL.startMiniCluster(2);
74 }
75
76
77 public static void tearDownAfterClass() throws Exception {
78 TEST_UTIL.shutdownMiniCluster();
79 }
80
81
82 public void testBulkDeleteEndpoint() throws Throwable {
83 byte[] tableName = Bytes.toBytes("testBulkDeleteEndpoint");
84 HTable ht = createTable(tableName);
85 List<Put> puts = new ArrayList<Put>(100);
86 for (int j = 0; j < 100; j++) {
87 byte[] rowkey = Bytes.toBytes(j);
88 puts.add(createPut(rowkey, "v1"));
89 }
90 ht.put(puts);
91
92 long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, new Scan(), 5, DeleteType.ROW, null);
93 assertEquals(100, noOfRowsDeleted);
94
95 int rows = 0;
96 for (Result result : ht.getScanner(new Scan())) {
97 rows++;
98 }
99 assertEquals(0, rows);
100 ht.close();
101 }
102
103
104 public void testBulkDeleteEndpointWhenRowBatchSizeLessThanRowsToDeleteFromARegion()
105 throws Throwable {
106 byte[] tableName = Bytes
107 .toBytes("testBulkDeleteEndpointWhenRowBatchSizeLessThanRowsToDeleteFromARegion");
108 HTable ht = createTable(tableName);
109 List<Put> puts = new ArrayList<Put>(100);
110 for (int j = 0; j < 100; j++) {
111 byte[] rowkey = Bytes.toBytes(j);
112 puts.add(createPut(rowkey, "v1"));
113 }
114 ht.put(puts);
115
116 long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, new Scan(), 10, DeleteType.ROW, null);
117 assertEquals(100, noOfRowsDeleted);
118
119 int rows = 0;
120 for (Result result : ht.getScanner(new Scan())) {
121 rows++;
122 }
123 assertEquals(0, rows);
124 ht.close();
125 }
126
127 private long invokeBulkDeleteProtocol(byte[] tableName, final Scan scan, final int rowBatchSize,
128 final DeleteType deleteType, final Long timeStamp) throws Throwable {
129 HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
130 long noOfDeletedRows = 0L;
131 Batch.Call<BulkDeleteService, BulkDeleteResponse> callable =
132 new Batch.Call<BulkDeleteService, BulkDeleteResponse>() {
133 ServerRpcController controller = new ServerRpcController();
134 BlockingRpcCallback<BulkDeleteResponse> rpcCallback =
135 new BlockingRpcCallback<BulkDeleteResponse>();
136
137 public BulkDeleteResponse call(BulkDeleteService service) throws IOException {
138 Builder builder = BulkDeleteRequest.newBuilder();
139 builder.setScan(ProtobufUtil.toScan(scan));
140 builder.setDeleteType(deleteType);
141 builder.setRowBatchSize(rowBatchSize);
142 if (timeStamp != null) {
143 builder.setTimestamp(timeStamp);
144 }
145 service.delete(controller, builder.build(), rpcCallback);
146 return rpcCallback.get();
147 }
148 };
149 Map<byte[], BulkDeleteResponse> result = ht.coprocessorService(BulkDeleteService.class, scan
150 .getStartRow(), scan.getStopRow(), callable);
151 for (BulkDeleteResponse response : result.values()) {
152 noOfDeletedRows += response.getRowsDeleted();
153 }
154 ht.close();
155 return noOfDeletedRows;
156 }
157
158
159 public void testBulkDeleteWithConditionBasedDelete() throws Throwable {
160 byte[] tableName = Bytes.toBytes("testBulkDeleteWithConditionBasedDelete");
161 HTable ht = createTable(tableName);
162 List<Put> puts = new ArrayList<Put>(100);
163 for (int j = 0; j < 100; j++) {
164 byte[] rowkey = Bytes.toBytes(j);
165 String value = (j % 10 == 0) ? "v1" : "v2";
166 puts.add(createPut(rowkey, value));
167 }
168 ht.put(puts);
169 Scan scan = new Scan();
170 FilterList fl = new FilterList(Operator.MUST_PASS_ALL);
171 SingleColumnValueFilter scvf = new SingleColumnValueFilter(FAMILY1, QUALIFIER3,
172 CompareOp.EQUAL, Bytes.toBytes("v1"));
173
174 fl.addFilter(scvf);
175 scan.setFilter(fl);
176
177 long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.ROW, null);
178 assertEquals(10, noOfRowsDeleted);
179
180 int rows = 0;
181 for (Result result : ht.getScanner(new Scan())) {
182 rows++;
183 }
184 assertEquals(90, rows);
185 ht.close();
186 }
187
188
189 public void testBulkDeleteColumn() throws Throwable {
190 byte[] tableName = Bytes.toBytes("testBulkDeleteColumn");
191 HTable ht = createTable(tableName);
192 List<Put> puts = new ArrayList<Put>(100);
193 for (int j = 0; j < 100; j++) {
194 byte[] rowkey = Bytes.toBytes(j);
195 String value = (j % 10 == 0) ? "v1" : "v2";
196 puts.add(createPut(rowkey, value));
197 }
198 ht.put(puts);
199 Scan scan = new Scan();
200 scan.addColumn(FAMILY1, QUALIFIER2);
201
202 long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.COLUMN, null);
203 assertEquals(100, noOfRowsDeleted);
204
205 int rows = 0;
206 for (Result result : ht.getScanner(new Scan())) {
207 assertEquals(2, result.getFamilyMap(FAMILY1).size());
208 assertTrue(result.getColumn(FAMILY1, QUALIFIER2).isEmpty());
209 assertEquals(1, result.getColumn(FAMILY1, QUALIFIER1).size());
210 assertEquals(1, result.getColumn(FAMILY1, QUALIFIER3).size());
211 rows++;
212 }
213 assertEquals(100, rows);
214 ht.close();
215 }
216
217
218 public void testBulkDeleteFamily() throws Throwable {
219 byte[] tableName = Bytes.toBytes("testBulkDeleteFamily");
220 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
221 htd.addFamily(new HColumnDescriptor(FAMILY1));
222 htd.addFamily(new HColumnDescriptor(FAMILY2));
223 TEST_UTIL.getHBaseAdmin().createTable(htd, Bytes.toBytes(0), Bytes.toBytes(120), 5);
224 HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
225 List<Put> puts = new ArrayList<Put>(100);
226 for (int j = 0; j < 100; j++) {
227 Put put = new Put(Bytes.toBytes(j));
228 put.add(FAMILY1, QUALIFIER1, "v1".getBytes());
229 put.add(FAMILY2, QUALIFIER2, "v2".getBytes());
230 puts.add(put);
231 }
232 ht.put(puts);
233 Scan scan = new Scan();
234 scan.addFamily(FAMILY1);
235
236 long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.FAMILY, null);
237 assertEquals(100, noOfRowsDeleted);
238 int rows = 0;
239 for (Result result : ht.getScanner(new Scan())) {
240 assertTrue(result.getFamilyMap(FAMILY1).isEmpty());
241 assertEquals(1, result.getColumn(FAMILY2, QUALIFIER2).size());
242 rows++;
243 }
244 assertEquals(100, rows);
245 ht.close();
246 }
247
248
249 public void testBulkDeleteColumnVersion() throws Throwable {
250 byte[] tableName = Bytes.toBytes("testBulkDeleteColumnVersion");
251 HTable ht = createTable(tableName);
252 List<Put> puts = new ArrayList<Put>(100);
253 for (int j = 0; j < 100; j++) {
254 Put put = new Put(Bytes.toBytes(j));
255 byte[] value = "v1".getBytes();
256 put.add(FAMILY1, QUALIFIER1, 1234L, value);
257 put.add(FAMILY1, QUALIFIER2, 1234L, value);
258 put.add(FAMILY1, QUALIFIER3, 1234L, value);
259
260 value = "v2".getBytes();
261 put.add(FAMILY1, QUALIFIER1, value);
262 put.add(FAMILY1, QUALIFIER2, value);
263 put.add(FAMILY1, QUALIFIER3, value);
264 put.add(FAMILY1, null, value);
265 puts.add(put);
266 }
267 ht.put(puts);
268 Scan scan = new Scan();
269 scan.addFamily(FAMILY1);
270
271 long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.VERSION,
272 HConstants.LATEST_TIMESTAMP);
273 assertEquals(100, noOfRowsDeleted);
274 int rows = 0;
275 scan = new Scan();
276 scan.setMaxVersions();
277 for (Result result : ht.getScanner(scan)) {
278 assertEquals(3, result.getFamilyMap(FAMILY1).size());
279 List<KeyValue> column = result.getColumn(FAMILY1, QUALIFIER1);
280 assertEquals(1, column.size());
281 assertTrue(Bytes.equals("v1".getBytes(), column.get(0).getValue()));
282
283 column = result.getColumn(FAMILY1, QUALIFIER2);
284 assertEquals(1, column.size());
285 assertTrue(Bytes.equals("v1".getBytes(), column.get(0).getValue()));
286
287 column = result.getColumn(FAMILY1, QUALIFIER3);
288 assertEquals(1, column.size());
289 assertTrue(Bytes.equals("v1".getBytes(), column.get(0).getValue()));
290 rows++;
291 }
292 assertEquals(100, rows);
293 ht.close();
294 }
295
296
297 public void testBulkDeleteColumnVersionBasedOnTS() throws Throwable {
298 byte[] tableName = Bytes.toBytes("testBulkDeleteColumnVersionBasedOnTS");
299 HTable ht = createTable(tableName);
300 List<Put> puts = new ArrayList<Put>(100);
301 for (int j = 0; j < 100; j++) {
302 Put put = new Put(Bytes.toBytes(j));
303
304 byte[] value = "v1".getBytes();
305 put.add(FAMILY1, QUALIFIER1, 1000L, value);
306 put.add(FAMILY1, QUALIFIER2, 1000L, value);
307 put.add(FAMILY1, QUALIFIER3, 1000L, value);
308
309 value = "v2".getBytes();
310 put.add(FAMILY1, QUALIFIER1, 1234L, value);
311 put.add(FAMILY1, QUALIFIER2, 1234L, value);
312 put.add(FAMILY1, QUALIFIER3, 1234L, value);
313
314 value = "v3".getBytes();
315 put.add(FAMILY1, QUALIFIER1, value);
316 put.add(FAMILY1, QUALIFIER2, value);
317 put.add(FAMILY1, QUALIFIER3, value);
318 puts.add(put);
319 }
320 ht.put(puts);
321 Scan scan = new Scan();
322 scan.addColumn(FAMILY1, QUALIFIER3);
323
324 long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.VERSION, 1234L);
325 assertEquals(100, noOfRowsDeleted);
326 int rows = 0;
327 scan = new Scan();
328 scan.setMaxVersions();
329 for (Result result : ht.getScanner(scan)) {
330 assertEquals(3, result.getFamilyMap(FAMILY1).size());
331 assertEquals(3, result.getColumn(FAMILY1, QUALIFIER1).size());
332 assertEquals(3, result.getColumn(FAMILY1, QUALIFIER2).size());
333 List<KeyValue> column = result.getColumn(FAMILY1, QUALIFIER3);
334 assertEquals(2, column.size());
335 assertTrue(Bytes.equals("v3".getBytes(), column.get(0).getValue()));
336 assertTrue(Bytes.equals("v1".getBytes(), column.get(1).getValue()));
337 rows++;
338 }
339 assertEquals(100, rows);
340 ht.close();
341 }
342
343
344 public void testBulkDeleteWithNumberOfVersions() throws Throwable {
345 byte[] tableName = Bytes.toBytes("testBulkDeleteWithNumberOfVersions");
346 HTable ht = createTable(tableName);
347 List<Put> puts = new ArrayList<Put>(100);
348 for (int j = 0; j < 100; j++) {
349 Put put = new Put(Bytes.toBytes(j));
350
351 byte[] value = "v1".getBytes();
352 put.add(FAMILY1, QUALIFIER1, 1000L, value);
353 put.add(FAMILY1, QUALIFIER2, 1000L, value);
354 put.add(FAMILY1, QUALIFIER3, 1000L, value);
355
356 value = "v2".getBytes();
357 put.add(FAMILY1, QUALIFIER1, 1234L, value);
358 put.add(FAMILY1, QUALIFIER2, 1234L, value);
359 put.add(FAMILY1, QUALIFIER3, 1234L, value);
360
361 value = "v3".getBytes();
362 put.add(FAMILY1, QUALIFIER1, 2000L, value);
363 put.add(FAMILY1, QUALIFIER2, 2000L, value);
364 put.add(FAMILY1, QUALIFIER3, 2000L, value);
365
366 value = "v4".getBytes();
367 put.add(FAMILY1, QUALIFIER1, value);
368 put.add(FAMILY1, QUALIFIER2, value);
369 put.add(FAMILY1, QUALIFIER3, value);
370 puts.add(put);
371 }
372 ht.put(puts);
373
374
375
376 final Scan scan = new Scan();
377 scan.addColumn(FAMILY1, QUALIFIER1);
378 scan.addColumn(FAMILY1, QUALIFIER2);
379 scan.setTimeRange(1000L, 2000L);
380 scan.setMaxVersions();
381
382 long noOfDeletedRows = 0L;
383 long noOfVersionsDeleted = 0L;
384 Batch.Call<BulkDeleteService, BulkDeleteResponse> callable =
385 new Batch.Call<BulkDeleteService, BulkDeleteResponse>() {
386 ServerRpcController controller = new ServerRpcController();
387 BlockingRpcCallback<BulkDeleteResponse> rpcCallback =
388 new BlockingRpcCallback<BulkDeleteResponse>();
389
390 public BulkDeleteResponse call(BulkDeleteService service) throws IOException {
391 Builder builder = BulkDeleteRequest.newBuilder();
392 builder.setScan(ProtobufUtil.toScan(scan));
393 builder.setDeleteType(DeleteType.VERSION);
394 builder.setRowBatchSize(500);
395 service.delete(controller, builder.build(), rpcCallback);
396 return rpcCallback.get();
397 }
398 };
399 Map<byte[], BulkDeleteResponse> result = ht.coprocessorService(BulkDeleteService.class, scan
400 .getStartRow(), scan.getStopRow(), callable);
401 for (BulkDeleteResponse response : result.values()) {
402 noOfDeletedRows += response.getRowsDeleted();
403 noOfVersionsDeleted += response.getVersionsDeleted();
404 }
405 assertEquals(100, noOfDeletedRows);
406 assertEquals(400, noOfVersionsDeleted);
407
408 int rows = 0;
409 Scan scan1 = new Scan();
410 scan1.setMaxVersions();
411 for (Result res : ht.getScanner(scan1)) {
412 assertEquals(3, res.getFamilyMap(FAMILY1).size());
413 List<KeyValue> column = res.getColumn(FAMILY1, QUALIFIER1);
414 assertEquals(2, column.size());
415 assertTrue(Bytes.equals("v4".getBytes(), column.get(0).getValue()));
416 assertTrue(Bytes.equals("v3".getBytes(), column.get(1).getValue()));
417 column = res.getColumn(FAMILY1, QUALIFIER2);
418 assertEquals(2, column.size());
419 assertTrue(Bytes.equals("v4".getBytes(), column.get(0).getValue()));
420 assertTrue(Bytes.equals("v3".getBytes(), column.get(1).getValue()));
421 assertEquals(4, res.getColumn(FAMILY1, QUALIFIER3).size());
422 rows++;
423 }
424 assertEquals(100, rows);
425 ht.close();
426 }
427
428 private HTable createTable(byte[] tableName) throws IOException {
429 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
430 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY1);
431 hcd.setMaxVersions(10);
432 htd.addFamily(hcd);
433 TEST_UTIL.getHBaseAdmin().createTable(htd, Bytes.toBytes(0), Bytes.toBytes(120), 5);
434 HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
435 return ht;
436 }
437
438 private Put createPut(byte[] rowkey, String value) throws IOException {
439 Put put = new Put(rowkey);
440 put.add(FAMILY1, QUALIFIER1, value.getBytes());
441 put.add(FAMILY1, QUALIFIER2, value.getBytes());
442 put.add(FAMILY1, QUALIFIER3, value.getBytes());
443 return put;
444 }
445 }