1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.filter;
19
20 import static org.junit.Assert.assertEquals;
21
22 import java.io.IOException;
23 import java.nio.ByteBuffer;
24 import java.util.ArrayList;
25 import java.util.List;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.hbase.Cell;
31 import org.apache.hadoop.hbase.CellUtil;
32 import org.apache.hadoop.hbase.HBaseTestingUtility;
33 import org.apache.hadoop.hbase.HConstants;
34 import org.apache.hadoop.hbase.TableName;
35 import org.apache.hadoop.hbase.client.Durability;
36 import org.apache.hadoop.hbase.client.Put;
37 import org.apache.hadoop.hbase.client.Result;
38 import org.apache.hadoop.hbase.client.ResultScanner;
39 import org.apache.hadoop.hbase.client.Scan;
40 import org.apache.hadoop.hbase.client.HTable;
41 import org.apache.hadoop.hbase.filter.FilterList.Operator;
42 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
43 import org.apache.hadoop.hbase.regionserver.HRegion;
44 import org.apache.hadoop.hbase.regionserver.RegionScanner;
45 import org.apache.hadoop.hbase.testclassification.MediumTests;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hbase.util.Pair;
48 import org.junit.After;
49 import org.junit.AfterClass;
50 import org.junit.Before;
51 import org.junit.BeforeClass;
52 import org.junit.Test;
53 import org.junit.experimental.categories.Category;
54
55 import com.google.common.collect.Lists;
56
57
58
59 @Category(MediumTests.class)
60 public class TestFuzzyRowFilterEndToEnd {
61 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
62 private final static byte fuzzyValue = (byte) 63;
63 private static final Log LOG = LogFactory.getLog(TestFuzzyRowFilterEndToEnd.class);
64
65 private static int firstPartCardinality = 50;
66 private static int secondPartCardinality = 50;
67 private static int thirdPartCardinality = 50;
68 private static int colQualifiersTotal = 5;
69 private static int totalFuzzyKeys = thirdPartCardinality / 2;
70
71 private static String table = "TestFuzzyRowFilterEndToEnd";
72
73
74
75
76 @BeforeClass
77 public static void setUpBeforeClass() throws Exception {
78 Configuration conf = TEST_UTIL.getConfiguration();
79 conf.setInt("hbase.client.scanner.caching", 1000);
80 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
81 ConstantSizeRegionSplitPolicy.class.getName());
82
83 conf.setLong(HConstants.HREGION_MAX_FILESIZE, ((long) 1024) * 1024 * 1024 * 10);
84
85 TEST_UTIL.startMiniCluster();
86 }
87
88
89
90
91 @AfterClass
92 public static void tearDownAfterClass() throws Exception {
93 TEST_UTIL.shutdownMiniCluster();
94 }
95
96
97
98
99 @Before
100 public void setUp() throws Exception {
101
102 }
103
104
105
106
107 @After
108 public void tearDown() throws Exception {
109
110 }
111
112 @Test
113 public void testHBASE14782() throws IOException
114 {
115 String cf = "f";
116 String cq = "q";
117 String table = "HBASE14872";
118
119 HTable ht =
120 TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
121
122 String[] rows = new String[]{
123 "\\x9C\\x00\\x044\\x00\\x00\\x00\\x00",
124 "\\x9C\\x00\\x044\\x01\\x00\\x00\\x00",
125 "\\x9C\\x00\\x044\\x00\\x01\\x00\\x00",
126 "\\x9C\\x00\\x044\\x00\\x00\\x01\\x00",
127 "\\x9C\\x00\\x044\\x00\\x01\\x00\\x01",
128 "\\x9B\\x00\\x044e\\xBB\\xB2\\xBB",
129 };
130
131 String badRow = "\\x9C\\x00\\x03\\xE9e\\xBB{X\\x1Fwts\\x1F\\x15vRX";
132
133 for(int i=0; i < rows.length; i++){
134 Put p = new Put(Bytes.toBytesBinary(rows[i]));
135 p.add(cf.getBytes(), cq.getBytes(), "value".getBytes());
136 ht.put(p);
137 }
138
139 Put p = new Put(Bytes.toBytesBinary(badRow));
140 p.add(cf.getBytes(), cq.getBytes(), "value".getBytes());
141 ht.put(p);
142
143 TEST_UTIL.flush();
144
145 List<Pair<byte[], byte[]>> data = new ArrayList<Pair<byte[], byte[]>>();
146 byte[] fuzzyKey = Bytes.toBytesBinary("\\x00\\x00\\x044");
147 byte[] mask = new byte[] { 1,0,0,0};
148 data.add(new Pair<byte[], byte[]>(fuzzyKey, mask));
149 FuzzyRowFilter filter = new FuzzyRowFilter(data);
150
151 Scan scan = new Scan();
152 scan.setFilter(filter);
153
154 ResultScanner scanner = ht.getScanner(scan);
155 int total = 0;
156 while(scanner.next() != null){
157 total++;
158 }
159 assertEquals(rows.length, total);
160 TEST_UTIL.deleteTable(TableName.valueOf(table));
161 }
162
163 @Test
164 public void testEndToEnd() throws Exception {
165 String cf = "f";
166
167 HTable ht =
168 TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
169
170
171
172
173
174 for (int i0 = 0; i0 < firstPartCardinality; i0++) {
175
176 for (int i1 = 0; i1 < secondPartCardinality; i1++) {
177
178 for (int i2 = 0; i2 < thirdPartCardinality; i2++) {
179 byte[] rk = new byte[10];
180
181 ByteBuffer buf = ByteBuffer.wrap(rk);
182 buf.clear();
183 buf.putShort((short) i0);
184 buf.putInt(i1);
185 buf.putInt(i2);
186 for (int c = 0; c < colQualifiersTotal; c++) {
187 byte[] cq = new byte[4];
188 Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
189
190 Put p = new Put(rk);
191 p.setDurability(Durability.SKIP_WAL);
192 p.add(cf.getBytes(), cq, Bytes.toBytes(c));
193 ht.put(p);
194 }
195 }
196 }
197 }
198
199 TEST_UTIL.flush();
200
201
202 runTest1(ht);
203 runTest2(ht);
204
205 }
206
207 private void runTest1(HTable hTable) throws IOException {
208
209
210 byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
211
212 List<Pair<byte[], byte[]>> list = new ArrayList<Pair<byte[], byte[]>>();
213 for (int i = 0; i < totalFuzzyKeys; i++) {
214 byte[] fuzzyKey = new byte[10];
215 ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
216 buf.clear();
217 buf.putShort((short) 2);
218 for (int j = 0; j < 4; j++) {
219 buf.put(fuzzyValue);
220 }
221 buf.putInt(i);
222
223 Pair<byte[], byte[]> pair = new Pair<byte[], byte[]>(fuzzyKey, mask);
224 list.add(pair);
225 }
226
227 int expectedSize = secondPartCardinality * totalFuzzyKeys * colQualifiersTotal;
228 FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
229
230 FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
231
232
233 runScanner(hTable, expectedSize, fuzzyRowFilter0);
234
235 runScanner(hTable, expectedSize, fuzzyRowFilter1);
236
237 }
238
239 private void runTest2(HTable hTable) throws IOException {
240
241
242 byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
243
244 List<Pair<byte[], byte[]>> list = new ArrayList<Pair<byte[], byte[]>>();
245
246 for (int i = 0; i < totalFuzzyKeys; i++) {
247 byte[] fuzzyKey = new byte[10];
248 ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
249 buf.clear();
250 buf.putShort((short) (i * 2));
251 for (int j = 0; j < 4; j++) {
252 buf.put(fuzzyValue);
253 }
254 buf.putInt(i * 2);
255
256 Pair<byte[], byte[]> pair = new Pair<byte[], byte[]>(fuzzyKey, mask);
257 list.add(pair);
258 }
259
260 int expectedSize = totalFuzzyKeys * secondPartCardinality * colQualifiersTotal;
261
262 FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
263
264 FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
265
266
267 runScanner(hTable, expectedSize, fuzzyRowFilter0);
268
269 runScanner(hTable, expectedSize, fuzzyRowFilter1);
270
271 }
272
273 private void runScanner(HTable hTable, int expectedSize, Filter filter) throws IOException {
274
275 String cf = "f";
276 Scan scan = new Scan();
277 scan.addFamily(cf.getBytes());
278 scan.setFilter(filter);
279 List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes());
280 HRegion first = regions.get(0);
281 first.getScanner(scan);
282 RegionScanner scanner = first.getScanner(scan);
283 List<Cell> results = new ArrayList<Cell>();
284
285 long timeBeforeScan = System.currentTimeMillis();
286 int found = 0;
287 while (scanner.next(results)) {
288 found += results.size();
289 results.clear();
290 }
291 found += results.size();
292 long scanTime = System.currentTimeMillis() - timeBeforeScan;
293 scanner.close();
294
295 LOG.info("\nscan time = " + scanTime + "ms");
296 LOG.info("found " + found + " results\n");
297
298 assertEquals(expectedSize, found);
299 }
300
301 @SuppressWarnings("deprecation")
302 @Test
303 public void testFilterList() throws Exception {
304 String cf = "f";
305 String table = "TestFuzzyRowFiltersInFilterList";
306 HTable ht =
307 TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
308
309
310
311
312
313 for (int i1 = 0; i1 < 5; i1++) {
314 for (int i2 = 0; i2 < 5; i2++) {
315 byte[] rk = new byte[10];
316
317 ByteBuffer buf = ByteBuffer.wrap(rk);
318 buf.clear();
319 buf.putShort((short) 2);
320 buf.putInt(i1);
321 buf.putInt(i2);
322
323
324 for (int c = 0; c < 5; c++) {
325 byte[] cq = new byte[4];
326 Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
327
328 Put p = new Put(rk);
329 p.setDurability(Durability.SKIP_WAL);
330 p.add(cf.getBytes(), cq, Bytes.toBytes(c));
331 ht.put(p);
332 LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: "
333 + Bytes.toStringBinary(cq));
334 }
335 }
336 }
337
338 TEST_UTIL.flush();
339
340
341 runTest(ht, 5);
342
343 }
344
345 @SuppressWarnings("unchecked")
346 private void runTest(HTable hTable, int expectedSize) throws IOException {
347
348 byte[] fuzzyKey1 = new byte[10];
349 ByteBuffer buf = ByteBuffer.wrap(fuzzyKey1);
350 buf.clear();
351 buf.putShort((short) 2);
352 for (int i = 0; i < 4; i++)
353 buf.put(fuzzyValue);
354 buf.putInt((short) 1);
355 byte[] mask1 = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
356
357 byte[] fuzzyKey2 = new byte[10];
358 buf = ByteBuffer.wrap(fuzzyKey2);
359 buf.clear();
360 buf.putShort((short) 2);
361 buf.putInt((short) 2);
362 for (int i = 0; i < 4; i++)
363 buf.put(fuzzyValue);
364
365 byte[] mask2 = new byte[] { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1 };
366
367 Pair<byte[], byte[]> pair1 = new Pair<byte[], byte[]>(fuzzyKey1, mask1);
368 Pair<byte[], byte[]> pair2 = new Pair<byte[], byte[]>(fuzzyKey2, mask2);
369
370 FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(Lists.newArrayList(pair1));
371 FuzzyRowFilter fuzzyRowFilter2 = new FuzzyRowFilter(Lists.newArrayList(pair2));
372
373 runScanner(hTable, expectedSize, fuzzyRowFilter1, fuzzyRowFilter2);
374 }
375
376 private void runScanner(HTable hTable, int expectedSize, Filter filter1, Filter filter2)
377 throws IOException {
378 String cf = "f";
379 Scan scan = new Scan();
380 scan.addFamily(cf.getBytes());
381 FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
382 scan.setFilter(filterList);
383
384 ResultScanner scanner = hTable.getScanner(scan);
385 List<Cell> results = new ArrayList<Cell>();
386 Result result;
387 long timeBeforeScan = System.currentTimeMillis();
388 while ((result = scanner.next()) != null) {
389 for (Cell kv : result.listCells()) {
390 LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
391 + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
392 results.add(kv);
393 }
394 }
395 long scanTime = System.currentTimeMillis() - timeBeforeScan;
396 scanner.close();
397
398 LOG.info("scan time = " + scanTime + "ms");
399 LOG.info("found " + results.size() + " results");
400
401 assertEquals(expectedSize, results.size());
402 }
403 }