1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.coprocessor.example;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.List;
26 import java.util.Map;
27
28 import org.apache.hadoop.hbase.HBaseTestingUtility;
29 import org.apache.hadoop.hbase.HColumnDescriptor;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.HTableDescriptor;
32 import org.apache.hadoop.hbase.KeyValue;
33 import org.apache.hadoop.hbase.MediumTests;
34 import org.apache.hadoop.hbase.client.HTable;
35 import org.apache.hadoop.hbase.client.Put;
36 import org.apache.hadoop.hbase.client.Result;
37 import org.apache.hadoop.hbase.client.Scan;
38 import org.apache.hadoop.hbase.client.coprocessor.Batch;
39 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
40 import org.apache.hadoop.hbase.coprocessor.example.BulkDeleteProtocol.DeleteType;
41 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
42 import org.apache.hadoop.hbase.filter.FilterList;
43 import org.apache.hadoop.hbase.filter.FilterList.Operator;
44 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.junit.AfterClass;
47 import org.junit.BeforeClass;
48 import org.junit.Test;
49 import org.junit.experimental.categories.Category;
50
51 @Category(MediumTests.class)
52 public class TestBulkDeleteProtocol {
53 private static final byte[] FAMILY1 = Bytes.toBytes("cf1");
54 private static final byte[] FAMILY2 = Bytes.toBytes("cf2");
55 private static final byte[] QUALIFIER1 = Bytes.toBytes("c1");
56 private static final byte[] QUALIFIER2 = Bytes.toBytes("c2");
57 private static final byte[] QUALIFIER3 = Bytes.toBytes("c3");
58 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
59
60 @BeforeClass
61 public static void setupBeforeClass() throws Exception {
62 TEST_UTIL.getConfiguration().set(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
63 BulkDeleteEndpoint.class.getName());
64 TEST_UTIL.startMiniCluster(2);
65 }
66
67 @AfterClass
68 public static void tearDownAfterClass() throws Exception {
69 TEST_UTIL.shutdownMiniCluster();
70 }
71
72 @Test
73 public void testBulkDeleteEndpoint() throws Throwable {
74 byte[] tableName = Bytes.toBytes("testBulkDeleteEndpoint");
75 HTable ht = createTable(tableName);
76 List<Put> puts = new ArrayList<Put>(100);
77 for (int j = 0; j < 100; j++) {
78 byte[] rowkey = Bytes.toBytes(j);
79 puts.add(createPut(rowkey, "v1"));
80 }
81 ht.put(puts);
82
83 long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, new Scan(), 500, DeleteType.ROW,
84 null);
85 assertEquals(100, noOfRowsDeleted);
86
87 int rows = 0;
88 for (Result result : ht.getScanner(new Scan())) {
89 rows++;
90 }
91 assertEquals(0, rows);
92 }
93
94 @Test
95 public void testBulkDeleteEndpointWhenRowBatchSizeLessThanRowsToDeleteFromARegion()
96 throws Throwable {
97 byte[] tableName = Bytes
98 .toBytes("testBulkDeleteEndpointWhenRowBatchSizeLessThanRowsToDeleteFromARegion");
99 HTable ht = createTable(tableName);
100 List<Put> puts = new ArrayList<Put>(100);
101 for (int j = 0; j < 100; j++) {
102 byte[] rowkey = Bytes.toBytes(j);
103 puts.add(createPut(rowkey, "v1"));
104 }
105 ht.put(puts);
106
107 long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, new Scan(), 10, DeleteType.ROW, null);
108 assertEquals(100, noOfRowsDeleted);
109
110 int rows = 0;
111 for (Result result : ht.getScanner(new Scan())) {
112 rows++;
113 }
114 assertEquals(0, rows);
115 }
116
117 private long invokeBulkDeleteProtocol(byte[] tableName, final Scan scan, final int rowBatchSize,
118 final byte deleteType, final Long timeStamp) throws Throwable {
119 HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
120 long noOfDeletedRows = 0L;
121 Batch.Call<BulkDeleteProtocol, BulkDeleteResponse> callable =
122 new Batch.Call<BulkDeleteProtocol, BulkDeleteResponse>() {
123 public BulkDeleteResponse call(BulkDeleteProtocol instance) throws IOException {
124 return instance.delete(scan, deleteType, timeStamp, rowBatchSize);
125 }
126 };
127 Map<byte[], BulkDeleteResponse> result = ht.coprocessorExec(BulkDeleteProtocol.class,
128 scan.getStartRow(), scan.getStopRow(), callable);
129 for (BulkDeleteResponse response : result.values()) {
130 noOfDeletedRows += response.getRowsDeleted();
131 }
132 return noOfDeletedRows;
133 }
134
135 @Test
136 public void testBulkDeleteWithConditionBasedDelete() throws Throwable {
137 byte[] tableName = Bytes.toBytes("testBulkDeleteWithConditionBasedDelete");
138 HTable ht = createTable(tableName);
139 List<Put> puts = new ArrayList<Put>(100);
140 for (int j = 0; j < 100; j++) {
141 byte[] rowkey = Bytes.toBytes(j);
142 String value = (j % 10 == 0) ? "v1" : "v2";
143 puts.add(createPut(rowkey, value));
144 }
145 ht.put(puts);
146 Scan scan = new Scan();
147 FilterList fl = new FilterList(Operator.MUST_PASS_ALL);
148 SingleColumnValueFilter scvf = new SingleColumnValueFilter(FAMILY1, QUALIFIER3,
149 CompareOp.EQUAL, Bytes.toBytes("v1"));
150
151 fl.addFilter(scvf);
152 scan.setFilter(fl);
153
154 long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.ROW, null);
155 assertEquals(10, noOfRowsDeleted);
156
157 int rows = 0;
158 for (Result result : ht.getScanner(new Scan())) {
159 rows++;
160 }
161 assertEquals(90, rows);
162 }
163
164 @Test
165 public void testBulkDeleteColumn() throws Throwable {
166 byte[] tableName = Bytes.toBytes("testBulkDeleteColumn");
167 HTable ht = createTable(tableName);
168 List<Put> puts = new ArrayList<Put>(100);
169 for (int j = 0; j < 100; j++) {
170 byte[] rowkey = Bytes.toBytes(j);
171 String value = (j % 10 == 0) ? "v1" : "v2";
172 puts.add(createPut(rowkey, value));
173 }
174 ht.put(puts);
175 Scan scan = new Scan ();
176 scan.addColumn(FAMILY1, QUALIFIER2);
177
178 long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.COLUMN, null);
179 assertEquals(100, noOfRowsDeleted);
180
181 int rows = 0;
182 for (Result result : ht.getScanner(new Scan())) {
183 assertEquals(2, result.getFamilyMap(FAMILY1).size());
184 assertTrue(result.getColumn(FAMILY1, QUALIFIER2).isEmpty());
185 assertEquals(1, result.getColumn(FAMILY1, QUALIFIER1).size());
186 assertEquals(1, result.getColumn(FAMILY1, QUALIFIER3).size());
187 rows++;
188 }
189 assertEquals(100, rows);
190 }
191
192 @Test
193 public void testBulkDeleteFamily() throws Throwable {
194 byte[] tableName = Bytes.toBytes("testBulkDeleteFamily");
195 HTableDescriptor htd = new HTableDescriptor(tableName);
196 htd.addFamily(new HColumnDescriptor(FAMILY1));
197 htd.addFamily(new HColumnDescriptor(FAMILY2));
198 TEST_UTIL.getHBaseAdmin().createTable(htd, Bytes.toBytes(0), Bytes.toBytes(120), 5);
199 HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
200 List<Put> puts = new ArrayList<Put>(100);
201 for (int j = 0; j < 100; j++) {
202 Put put = new Put(Bytes.toBytes(j));
203 put.add(FAMILY1, QUALIFIER1, "v1".getBytes());
204 put.add(FAMILY2, QUALIFIER2, "v2".getBytes());
205 puts.add(put);
206 }
207 ht.put(puts);
208 Scan scan = new Scan ();
209 scan.addFamily(FAMILY1);
210
211 long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.FAMILY, null);
212 assertEquals(100, noOfRowsDeleted);
213 int rows = 0;
214 for (Result result : ht.getScanner(new Scan())) {
215 assertTrue(result.getFamilyMap(FAMILY1).isEmpty());
216 assertEquals(1, result.getColumn(FAMILY2, QUALIFIER2).size());
217 rows++;
218 }
219 assertEquals(100, rows);
220 }
221
222 @Test
223 public void testBulkDeleteColumnVersion() throws Throwable {
224 byte[] tableName = Bytes.toBytes("testBulkDeleteColumnVersion");
225 HTable ht = createTable(tableName);
226 List<Put> puts = new ArrayList<Put>(100);
227 for (int j = 0; j < 100; j++) {
228 Put put = new Put(Bytes.toBytes(j));
229 byte[] value = "v1".getBytes();
230 put.add(FAMILY1, QUALIFIER1, 1234L, value);
231 put.add(FAMILY1, QUALIFIER2, 1234L, value);
232 put.add(FAMILY1, QUALIFIER3, 1234L, value);
233
234 value = "v2".getBytes();
235 put.add(FAMILY1, QUALIFIER1, value);
236 put.add(FAMILY1, QUALIFIER2, value);
237 put.add(FAMILY1, QUALIFIER3, value);
238 put.add(FAMILY1, null, value);
239 puts.add(put);
240 }
241 ht.put(puts);
242 Scan scan = new Scan ();
243 scan.addFamily(FAMILY1);
244
245 long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.VERSION,
246 HConstants.LATEST_TIMESTAMP);
247 assertEquals(100, noOfRowsDeleted);
248 int rows = 0;
249 scan = new Scan ();
250 scan.setMaxVersions();
251 for (Result result : ht.getScanner(scan)) {
252 assertEquals(3, result.getFamilyMap(FAMILY1).size());
253 List<KeyValue> column = result.getColumn(FAMILY1, QUALIFIER1);
254 assertEquals(1, column.size());
255 assertTrue(Bytes.equals("v1".getBytes(), column.get(0).getValue()));
256
257 column = result.getColumn(FAMILY1, QUALIFIER2);
258 assertEquals(1, column.size());
259 assertTrue(Bytes.equals("v1".getBytes(), column.get(0).getValue()));
260
261 column = result.getColumn(FAMILY1, QUALIFIER3);
262 assertEquals(1, column.size());
263 assertTrue(Bytes.equals("v1".getBytes(), column.get(0).getValue()));
264 rows++;
265 }
266 assertEquals(100, rows);
267 }
268
269 @Test
270 public void testBulkDeleteColumnVersionBasedOnTS() throws Throwable {
271 byte[] tableName = Bytes.toBytes("testBulkDeleteColumnVersionBasedOnTS");
272 HTable ht = createTable(tableName);
273 List<Put> puts = new ArrayList<Put>(100);
274 for (int j = 0; j < 100; j++) {
275 Put put = new Put(Bytes.toBytes(j));
276
277 byte[] value = "v1".getBytes();
278 put.add(FAMILY1, QUALIFIER1, 1000L, value);
279 put.add(FAMILY1, QUALIFIER2, 1000L, value);
280 put.add(FAMILY1, QUALIFIER3, 1000L, value);
281
282 value = "v2".getBytes();
283 put.add(FAMILY1, QUALIFIER1, 1234L, value);
284 put.add(FAMILY1, QUALIFIER2, 1234L, value);
285 put.add(FAMILY1, QUALIFIER3, 1234L, value);
286
287 value = "v3".getBytes();
288 put.add(FAMILY1, QUALIFIER1, value);
289 put.add(FAMILY1, QUALIFIER2, value);
290 put.add(FAMILY1, QUALIFIER3, value);
291 puts.add(put);
292 }
293 ht.put(puts);
294 Scan scan = new Scan ();
295 scan.addColumn(FAMILY1, QUALIFIER3);
296
297 long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.VERSION, 1234L);
298 assertEquals(100, noOfRowsDeleted);
299 int rows = 0;
300 scan = new Scan ();
301 scan.setMaxVersions();
302 for (Result result : ht.getScanner(scan)) {
303 assertEquals(3, result.getFamilyMap(FAMILY1).size());
304 assertEquals(3, result.getColumn(FAMILY1, QUALIFIER1).size());
305 assertEquals(3, result.getColumn(FAMILY1, QUALIFIER2).size());
306 List<KeyValue> column = result.getColumn(FAMILY1, QUALIFIER3);
307 assertEquals(2, column.size());
308 assertTrue(Bytes.equals("v3".getBytes(), column.get(0).getValue()));
309 assertTrue(Bytes.equals("v1".getBytes(), column.get(1).getValue()));
310 rows++;
311 }
312 assertEquals(100, rows);
313 }
314
315 @Test
316 public void testBulkDeleteWithNumberOfVersions() throws Throwable {
317 byte[] tableName = Bytes.toBytes("testBulkDeleteWithNumberOfVersions");
318 HTable ht = createTable(tableName);
319 List<Put> puts = new ArrayList<Put>(100);
320 for (int j = 0; j < 100; j++) {
321 Put put = new Put(Bytes.toBytes(j));
322
323 byte[] value = "v1".getBytes();
324 put.add(FAMILY1, QUALIFIER1, 1000L, value);
325 put.add(FAMILY1, QUALIFIER2, 1000L, value);
326 put.add(FAMILY1, QUALIFIER3, 1000L, value);
327
328 value = "v2".getBytes();
329 put.add(FAMILY1, QUALIFIER1, 1234L, value);
330 put.add(FAMILY1, QUALIFIER2, 1234L, value);
331 put.add(FAMILY1, QUALIFIER3, 1234L, value);
332
333 value = "v3".getBytes();
334 put.add(FAMILY1, QUALIFIER1, 2000L, value);
335 put.add(FAMILY1, QUALIFIER2, 2000L, value);
336 put.add(FAMILY1, QUALIFIER3, 2000L, value);
337
338 value = "v4".getBytes();
339 put.add(FAMILY1, QUALIFIER1, value);
340 put.add(FAMILY1, QUALIFIER2, value);
341 put.add(FAMILY1, QUALIFIER3, value);
342 puts.add(put);
343 }
344 ht.put(puts);
345
346
347
348 final Scan scan = new Scan();
349 scan.addColumn(FAMILY1, QUALIFIER1);
350 scan.addColumn(FAMILY1, QUALIFIER2);
351 scan.setTimeRange(1000L, 2000L);
352 scan.setMaxVersions();
353
354 long noOfDeletedRows = 0L;
355 long noOfVersionsDeleted = 0L;
356 Batch.Call<BulkDeleteProtocol, BulkDeleteResponse> callable =
357 new Batch.Call<BulkDeleteProtocol, BulkDeleteResponse>() {
358 public BulkDeleteResponse call(BulkDeleteProtocol instance) throws IOException {
359 return instance.delete(scan, DeleteType.VERSION, null, 500);
360 }
361 };
362 Map<byte[], BulkDeleteResponse> result = ht.coprocessorExec(BulkDeleteProtocol.class,
363 scan.getStartRow(), scan.getStopRow(), callable);
364 for (BulkDeleteResponse response : result.values()) {
365 noOfDeletedRows += response.getRowsDeleted();
366 noOfVersionsDeleted += response.getVersionsDeleted();
367 }
368 assertEquals(100, noOfDeletedRows);
369 assertEquals(400, noOfVersionsDeleted);
370
371 int rows = 0;
372 Scan scan1 = new Scan ();
373 scan1.setMaxVersions();
374 for (Result res : ht.getScanner(scan1)) {
375 assertEquals(3, res.getFamilyMap(FAMILY1).size());
376 List<KeyValue> column = res.getColumn(FAMILY1, QUALIFIER1);
377 assertEquals(2, column.size());
378 assertTrue(Bytes.equals("v4".getBytes(), column.get(0).getValue()));
379 assertTrue(Bytes.equals("v3".getBytes(), column.get(1).getValue()));
380 column = res.getColumn(FAMILY1, QUALIFIER2);
381 assertEquals(2, column.size());
382 assertTrue(Bytes.equals("v4".getBytes(), column.get(0).getValue()));
383 assertTrue(Bytes.equals("v3".getBytes(), column.get(1).getValue()));
384 assertEquals(4, res.getColumn(FAMILY1, QUALIFIER3).size());
385 rows++;
386 }
387 assertEquals(100, rows);
388 }
389
390 private HTable createTable(byte[] tableName) throws IOException {
391 HTableDescriptor htd = new HTableDescriptor(tableName);
392 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY1);
393 hcd.setMaxVersions(10);
394 htd.addFamily(hcd);
395 TEST_UTIL.getHBaseAdmin().createTable(htd, Bytes.toBytes(0), Bytes.toBytes(120), 5);
396 HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
397 return ht;
398 }
399
400 private Put createPut(byte[] rowkey, String value) throws IOException {
401 Put put = new Put(rowkey);
402 put.add(FAMILY1, QUALIFIER1, value.getBytes());
403 put.add(FAMILY1, QUALIFIER2, value.getBytes());
404 put.add(FAMILY1, QUALIFIER3, value.getBytes());
405 return put;
406 }
407 }