1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.filter;
20
21 import java.util.ArrayList;
22 import java.util.Arrays;
23 import java.util.List;
24
25 import static org.junit.Assert.assertEquals;
26 import static org.junit.Assert.assertFalse;
27 import static org.junit.Assert.assertTrue;
28 import static org.junit.Assert.assertNull;
29
30 import org.apache.hadoop.hbase.Cell;
31 import org.apache.hadoop.hbase.KeyValue;
32 import org.apache.hadoop.hbase.KeyValueUtil;
33 import org.apache.hadoop.hbase.SmallTests;
34 import org.apache.hadoop.hbase.exceptions.DeserializationException;
35 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
36 import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
37 import org.apache.hadoop.hbase.filter.FilterList.Operator;
38 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
39 import org.apache.hadoop.hbase.util.Bytes;
40 import org.junit.Test;
41 import org.junit.experimental.categories.Category;
42
43 import com.google.common.collect.Lists;
44
45
46
47
48
49 @Category(SmallTests.class)
50 public class TestFilterList {
51 static final int MAX_PAGES = 2;
52 static final char FIRST_CHAR = 'a';
53 static final char LAST_CHAR = 'e';
54 static byte[] GOOD_BYTES = Bytes.toBytes("abc");
55 static byte[] BAD_BYTES = Bytes.toBytes("def");
56
57
58 @Test
59 public void testAddFilter() throws Exception {
60 Filter filter1 = new FirstKeyOnlyFilter();
61 Filter filter2 = new FirstKeyOnlyFilter();
62
63 FilterList filterList = new FilterList(filter1, filter2);
64 filterList.addFilter(new FirstKeyOnlyFilter());
65
66 filterList = new FilterList(Arrays.asList(filter1, filter2));
67 filterList.addFilter(new FirstKeyOnlyFilter());
68
69 filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
70 filterList.addFilter(new FirstKeyOnlyFilter());
71
72 filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList(filter1, filter2));
73 filterList.addFilter(new FirstKeyOnlyFilter());
74
75 }
76
77
78
79
80
81
82 @Test
83 public void testMPONE() throws Exception {
84 mpOneTest(getFilterMPONE());
85 }
86
87 private Filter getFilterMPONE() {
88 List<Filter> filters = new ArrayList<Filter>();
89 filters.add(new PageFilter(MAX_PAGES));
90 filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy"))));
91 Filter filterMPONE =
92 new FilterList(FilterList.Operator.MUST_PASS_ONE, filters);
93 return filterMPONE;
94 }
95
96 private void mpOneTest(Filter filterMPONE) throws Exception {
97
98
99
100
101
102
103
104
105
106
107
108
109 filterMPONE.reset();
110 assertFalse(filterMPONE.filterAllRemaining());
111
112
113 byte [] rowkey = Bytes.toBytes("yyyyyyyyy");
114 for (int i = 0; i < MAX_PAGES - 1; i++) {
115 assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
116 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i),
117 Bytes.toBytes(i));
118 assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
119 assertFalse(filterMPONE.filterRow());
120 }
121
122
123 rowkey = Bytes.toBytes("z");
124 assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
125 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(0),
126 Bytes.toBytes(0));
127 assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
128 assertFalse(filterMPONE.filterRow());
129
130
131 rowkey = Bytes.toBytes("yyy");
132 assertTrue(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
133 kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(0),
134 Bytes.toBytes(0));
135 assertFalse(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
136 assertFalse(filterMPONE.filterRow());
137
138
139 rowkey = Bytes.toBytes("z");
140 assertTrue(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
141 assertTrue(filterMPONE.filterAllRemaining());
142 }
143
144
145
146
147
148 @Test
149 public void testMPALL() throws Exception {
150 mpAllTest(getMPALLFilter());
151 }
152
153 private Filter getMPALLFilter() {
154 List<Filter> filters = new ArrayList<Filter>();
155 filters.add(new PageFilter(MAX_PAGES));
156 filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy"))));
157 Filter filterMPALL =
158 new FilterList(FilterList.Operator.MUST_PASS_ALL, filters);
159 return filterMPALL;
160 }
161
162 private void mpAllTest(Filter filterMPALL) throws Exception {
163
164
165
166
167
168
169
170
171
172
173
174
175 filterMPALL.reset();
176 assertFalse(filterMPALL.filterAllRemaining());
177 byte [] rowkey = Bytes.toBytes("yyyyyyyyy");
178 for (int i = 0; i < MAX_PAGES - 1; i++) {
179 assertFalse(filterMPALL.filterRowKey(rowkey, 0, rowkey.length));
180 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i),
181 Bytes.toBytes(i));
182 assertTrue(Filter.ReturnCode.INCLUDE == filterMPALL.filterKeyValue(kv));
183 }
184 filterMPALL.reset();
185 rowkey = Bytes.toBytes("z");
186 assertTrue(filterMPALL.filterRowKey(rowkey, 0, rowkey.length));
187
188 KeyValue kv = new KeyValue(rowkey, rowkey, rowkey, rowkey);
189 assertTrue(Filter.ReturnCode.NEXT_ROW == filterMPALL.filterKeyValue(kv));
190 }
191
192
193
194
195
196 @Test
197 public void testOrdering() throws Exception {
198 orderingTest(getOrderingFilter());
199 }
200
201 public Filter getOrderingFilter() {
202 List<Filter> filters = new ArrayList<Filter>();
203 filters.add(new PrefixFilter(Bytes.toBytes("yyy")));
204 filters.add(new PageFilter(MAX_PAGES));
205 Filter filterMPONE =
206 new FilterList(FilterList.Operator.MUST_PASS_ONE, filters);
207 return filterMPONE;
208 }
209
210 public void orderingTest(Filter filterMPONE) throws Exception {
211
212
213
214
215
216
217
218
219
220
221
222
223 filterMPONE.reset();
224 assertFalse(filterMPONE.filterAllRemaining());
225
226
227 byte [] rowkey = Bytes.toBytes("yyyyyyyy");
228 for (int i = 0; i < MAX_PAGES; i++) {
229 assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
230 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i),
231 Bytes.toBytes(i));
232 assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
233 assertFalse(filterMPONE.filterRow());
234 }
235
236
237 rowkey = Bytes.toBytes("xxxxxxx");
238 for (int i = 0; i < MAX_PAGES; i++) {
239 assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
240 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i),
241 Bytes.toBytes(i));
242 assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
243 assertFalse(filterMPONE.filterRow());
244 }
245
246
247 rowkey = Bytes.toBytes("yyy");
248 for (int i = 0; i < MAX_PAGES; i++) {
249 assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length));
250 KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i),
251 Bytes.toBytes(i));
252 assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv));
253 assertFalse(filterMPONE.filterRow());
254 }
255 }
256
257
258
259
260
261
262 public void testFilterListTwoFiltersMustPassOne() throws Exception {
263 byte[] r1 = Bytes.toBytes("Row1");
264 byte[] r11 = Bytes.toBytes("Row11");
265 byte[] r2 = Bytes.toBytes("Row2");
266
267 FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
268 flist.addFilter(new PrefixFilter(r1));
269 flist.filterRowKey(r1, 0, r1.length);
270 assertEquals(flist.filterKeyValue(new KeyValue(r1,r1,r1)), ReturnCode.INCLUDE);
271 assertEquals(flist.filterKeyValue(new KeyValue(r11,r11,r11)), ReturnCode.INCLUDE);
272
273 flist.reset();
274 flist.filterRowKey(r2, 0, r2.length);
275 assertEquals(flist.filterKeyValue(new KeyValue(r2,r2,r2)), ReturnCode.SKIP);
276
277 flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
278 flist.addFilter(new AlwaysNextColFilter());
279 flist.addFilter(new PrefixFilter(r1));
280 flist.filterRowKey(r1, 0, r1.length);
281 assertEquals(flist.filterKeyValue(new KeyValue(r1,r1,r1)), ReturnCode.INCLUDE);
282 assertEquals(flist.filterKeyValue(new KeyValue(r11,r11,r11)), ReturnCode.INCLUDE);
283
284 flist.reset();
285 flist.filterRowKey(r2, 0, r2.length);
286 assertEquals(flist.filterKeyValue(new KeyValue(r2,r2,r2)), ReturnCode.SKIP);
287 }
288
289
290
291
292
293 @Test
294 public void testSerialization() throws Exception {
295 List<Filter> filters = new ArrayList<Filter>();
296 filters.add(new PageFilter(MAX_PAGES));
297 filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy"))));
298 Filter filterMPALL =
299 new FilterList(FilterList.Operator.MUST_PASS_ALL, filters);
300
301
302 byte[] buffer = filterMPALL.toByteArray();
303
304
305 FilterList newFilter = FilterList.parseFrom(buffer);
306
307
308 mpOneTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getFilterMPONE())));
309 mpAllTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getMPALLFilter())));
310 orderingTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getOrderingFilter())));
311 }
312
313
314
315
316
317
318 public void testFilterListWithInclusiveStopFilteMustPassOne() throws Exception {
319 byte[] r1 = Bytes.toBytes("Row1");
320 byte[] r11 = Bytes.toBytes("Row11");
321 byte[] r2 = Bytes.toBytes("Row2");
322
323 FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
324 flist.addFilter(new AlwaysNextColFilter());
325 flist.addFilter(new InclusiveStopFilter(r1));
326 flist.filterRowKey(r1, 0, r1.length);
327 assertEquals(flist.filterKeyValue(new KeyValue(r1,r1,r1)), ReturnCode.INCLUDE);
328 assertEquals(flist.filterKeyValue(new KeyValue(r11,r11,r11)), ReturnCode.INCLUDE);
329
330 flist.reset();
331 flist.filterRowKey(r2, 0, r2.length);
332 assertEquals(flist.filterKeyValue(new KeyValue(r2,r2,r2)), ReturnCode.SKIP);
333 }
334
335 private static class AlwaysNextColFilter extends FilterBase {
336 public AlwaysNextColFilter() {
337 super();
338 }
339 @Override
340 public ReturnCode filterKeyValue(Cell v) {
341 return ReturnCode.NEXT_COL;
342 }
343 public static AlwaysNextColFilter parseFrom(final byte [] pbBytes)
344 throws DeserializationException {
345 return new AlwaysNextColFilter();
346 }
347 }
348
349
350
351
352
353
354 public void testFilterKeyValue() throws Exception {
355 Filter includeFilter = new FilterBase() {
356 @Override
357 public Filter.ReturnCode filterKeyValue(Cell v) {
358 return Filter.ReturnCode.INCLUDE;
359 }
360 };
361
362 Filter alternateFilter = new FilterBase() {
363 boolean returnInclude = true;
364
365 @Override
366 public Filter.ReturnCode filterKeyValue(Cell v) {
367 Filter.ReturnCode returnCode = returnInclude ? Filter.ReturnCode.INCLUDE :
368 Filter.ReturnCode.SKIP;
369 returnInclude = !returnInclude;
370 return returnCode;
371 }
372 };
373
374 Filter alternateIncludeFilter = new FilterBase() {
375 boolean returnIncludeOnly = false;
376
377 @Override
378 public Filter.ReturnCode filterKeyValue(Cell v) {
379 Filter.ReturnCode returnCode = returnIncludeOnly ? Filter.ReturnCode.INCLUDE :
380 Filter.ReturnCode.INCLUDE_AND_NEXT_COL;
381 returnIncludeOnly = !returnIncludeOnly;
382 return returnCode;
383 }
384 };
385
386
387 FilterList mpOnefilterList = new FilterList(Operator.MUST_PASS_ONE,
388 Arrays.asList(new Filter[] { includeFilter, alternateIncludeFilter, alternateFilter }));
389
390 assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, mpOnefilterList.filterKeyValue(null));
391
392 assertEquals(Filter.ReturnCode.INCLUDE, mpOnefilterList.filterKeyValue(null));
393
394
395 FilterList mpAllfilterList = new FilterList(Operator.MUST_PASS_ALL,
396 Arrays.asList(new Filter[] { includeFilter, alternateIncludeFilter, alternateFilter }));
397
398 assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, mpAllfilterList.filterKeyValue(null));
399
400 assertEquals(Filter.ReturnCode.SKIP, mpAllfilterList.filterKeyValue(null));
401 }
402
403
404
405
406 @Test
407 public void testHintPassThru() throws Exception {
408
409 final KeyValue minKeyValue = new KeyValue(Bytes.toBytes(0L), null, null);
410 final KeyValue maxKeyValue = new KeyValue(Bytes.toBytes(Long.MAX_VALUE),
411 null, null);
412
413 Filter filterNoHint = new FilterBase() {
414 @Override
415 public byte [] toByteArray() {return null;}
416 };
417
418 Filter filterMinHint = new FilterBase() {
419 @Override
420 public ReturnCode filterKeyValue(Cell ignored) {
421 return ReturnCode.SEEK_NEXT_USING_HINT;
422 }
423
424 @Override
425 public Cell getNextCellHint(Cell currentKV) {
426 return minKeyValue;
427 }
428
429 @Override
430 public byte [] toByteArray() {return null;}
431 };
432
433 Filter filterMaxHint = new FilterBase() {
434 @Override
435 public ReturnCode filterKeyValue(Cell ignored) {
436 return ReturnCode.SEEK_NEXT_USING_HINT;
437 }
438
439 @Override
440 public Cell getNextCellHint(Cell currentKV) {
441 return new KeyValue(Bytes.toBytes(Long.MAX_VALUE), null, null);
442 }
443
444 @Override
445 public byte [] toByteArray() {return null;}
446 };
447
448
449
450
451 FilterList filterList = new FilterList(Operator.MUST_PASS_ONE,
452 Arrays.asList(new Filter [] { filterMinHint, filterMaxHint } ));
453 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
454 minKeyValue));
455
456
457 filterList = new FilterList(Operator.MUST_PASS_ONE,
458 Arrays.asList(
459 new Filter [] { filterMinHint, filterMaxHint, filterNoHint } ));
460 assertNull(filterList.getNextKeyHint(null));
461 filterList = new FilterList(Operator.MUST_PASS_ONE,
462 Arrays.asList(new Filter [] { filterNoHint, filterMaxHint } ));
463 assertNull(filterList.getNextKeyHint(null));
464
465
466 filterList = new FilterList(Operator.MUST_PASS_ONE,
467 Arrays.asList(new Filter [] { filterMaxHint, filterMaxHint } ));
468 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
469 maxKeyValue));
470
471
472
473
474 filterList = new FilterList(Operator.MUST_PASS_ALL,
475 Arrays.asList(new Filter [] { filterMinHint, filterMaxHint } ));
476 filterList.filterKeyValue(null);
477 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
478 minKeyValue));
479
480 filterList = new FilterList(Operator.MUST_PASS_ALL,
481 Arrays.asList(new Filter [] { filterMaxHint, filterMinHint } ));
482 filterList.filterKeyValue(null);
483 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
484 maxKeyValue));
485
486
487 filterList = new FilterList(Operator.MUST_PASS_ALL,
488 Arrays.asList(
489 new Filter [] { filterNoHint, filterMinHint, filterMaxHint } ));
490 filterList.filterKeyValue(null);
491 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
492 minKeyValue));
493 filterList = new FilterList(Operator.MUST_PASS_ALL,
494 Arrays.asList(new Filter [] { filterNoHint, filterMaxHint } ));
495 filterList.filterKeyValue(null);
496 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
497 maxKeyValue));
498 filterList = new FilterList(Operator.MUST_PASS_ALL,
499 Arrays.asList(new Filter [] { filterNoHint, filterMinHint } ));
500 filterList.filterKeyValue(null);
501 assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null),
502 minKeyValue));
503 }
504
505
506
507
508
509
510
511 @Test
512 public void testTransformMPO() throws Exception {
513
514
515
516 final FilterList flist = new FilterList(Operator.MUST_PASS_ONE, Lists.<Filter>newArrayList(
517 new FilterList(Operator.MUST_PASS_ALL, Lists.<Filter>newArrayList(
518 new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))),
519 new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("qual1"))),
520 new KeyOnlyFilter())),
521 new FilterList(Operator.MUST_PASS_ALL, Lists.<Filter>newArrayList(
522 new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))),
523 new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("qual2")))))));
524
525 final KeyValue kvQual1 = new KeyValue(
526 Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual1"), Bytes.toBytes("value"));
527 final KeyValue kvQual2 = new KeyValue(
528 Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual2"), Bytes.toBytes("value"));
529 final KeyValue kvQual3 = new KeyValue(
530 Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual3"), Bytes.toBytes("value"));
531
532
533 assertEquals(Filter.ReturnCode.INCLUDE, flist.filterKeyValue(kvQual1));
534 final KeyValue transformedQual1 = KeyValueUtil.ensureKeyValue(flist.transform(kvQual1));
535 assertEquals(0, transformedQual1.getValue().length);
536
537
538 assertEquals(Filter.ReturnCode.INCLUDE, flist.filterKeyValue(kvQual2));
539 final KeyValue transformedQual2 = KeyValueUtil.ensureKeyValue(flist.transform(kvQual2));
540 assertEquals("value", Bytes.toString(transformedQual2.getValue()));
541
542
543 assertEquals(Filter.ReturnCode.SKIP, flist.filterKeyValue(kvQual3));
544 }
545
546 }
547