1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.filter;
19
20 import static org.junit.Assert.assertEquals;
21
22 import java.io.IOException;
23 import java.nio.ByteBuffer;
24 import java.util.ArrayList;
25 import java.util.List;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.hbase.Cell;
31 import org.apache.hadoop.hbase.CellUtil;
32 import org.apache.hadoop.hbase.HBaseTestingUtility;
33 import org.apache.hadoop.hbase.HConstants;
34 import org.apache.hadoop.hbase.TableName;
35 import org.apache.hadoop.hbase.client.Durability;
36 import org.apache.hadoop.hbase.client.Put;
37 import org.apache.hadoop.hbase.client.Result;
38 import org.apache.hadoop.hbase.client.ResultScanner;
39 import org.apache.hadoop.hbase.client.Scan;
40 import org.apache.hadoop.hbase.client.HTable;
41 import org.apache.hadoop.hbase.filter.FilterList.Operator;
42 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
43 import org.apache.hadoop.hbase.regionserver.HRegion;
44 import org.apache.hadoop.hbase.regionserver.RegionScanner;
45 import org.apache.hadoop.hbase.testclassification.MediumTests;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hbase.util.Pair;
48 import org.junit.After;
49 import org.junit.AfterClass;
50 import org.junit.Before;
51 import org.junit.BeforeClass;
52 import org.junit.Test;
53 import org.junit.experimental.categories.Category;
54
55 import com.google.common.collect.Lists;
56
57
58
59 @Category(MediumTests.class)
60 public class TestFuzzyRowFilterEndToEnd {
61 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
62 private final static byte fuzzyValue = (byte) 63;
63 private static final Log LOG = LogFactory.getLog(TestFuzzyRowFilterEndToEnd.class);
64
65 private static int firstPartCardinality = 50;
66 private static int secondPartCardinality = 50;
67 private static int thirdPartCardinality = 50;
68 private static int colQualifiersTotal = 5;
69 private static int totalFuzzyKeys = thirdPartCardinality / 2;
70
71 private static String table = "TestFuzzyRowFilterEndToEnd";
72
73
74
75
76 @BeforeClass
77 public static void setUpBeforeClass() throws Exception {
78 Configuration conf = TEST_UTIL.getConfiguration();
79 conf.setInt("hbase.client.scanner.caching", 1000);
80 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
81 ConstantSizeRegionSplitPolicy.class.getName());
82
83 conf.setLong(HConstants.HREGION_MAX_FILESIZE, ((long) 1024) * 1024 * 1024 * 10);
84
85 TEST_UTIL.startMiniCluster();
86 }
87
88
89
90
91 @AfterClass
92 public static void tearDownAfterClass() throws Exception {
93 TEST_UTIL.shutdownMiniCluster();
94 }
95
96
97
98
99 @Before
100 public void setUp() throws Exception {
101
102 }
103
104
105
106
107 @After
108 public void tearDown() throws Exception {
109
110 }
111
112 @Test
113 public void testEndToEnd() throws Exception {
114 String cf = "f";
115
116 HTable ht =
117 TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
118
119
120
121
122
123 for (int i0 = 0; i0 < firstPartCardinality; i0++) {
124
125 for (int i1 = 0; i1 < secondPartCardinality; i1++) {
126
127 for (int i2 = 0; i2 < thirdPartCardinality; i2++) {
128 byte[] rk = new byte[10];
129
130 ByteBuffer buf = ByteBuffer.wrap(rk);
131 buf.clear();
132 buf.putShort((short) i0);
133 buf.putInt(i1);
134 buf.putInt(i2);
135 for (int c = 0; c < colQualifiersTotal; c++) {
136 byte[] cq = new byte[4];
137 Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
138
139 Put p = new Put(rk);
140 p.setDurability(Durability.SKIP_WAL);
141 p.add(cf.getBytes(), cq, Bytes.toBytes(c));
142 ht.put(p);
143 }
144 }
145 }
146 }
147
148 TEST_UTIL.flush();
149
150
151 runTest1(ht);
152 runTest2(ht);
153
154 }
155
156 private void runTest1(HTable hTable) throws IOException {
157
158
159 byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
160
161 List<Pair<byte[], byte[]>> list = new ArrayList<Pair<byte[], byte[]>>();
162 for (int i = 0; i < totalFuzzyKeys; i++) {
163 byte[] fuzzyKey = new byte[10];
164 ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
165 buf.clear();
166 buf.putShort((short) 2);
167 for (int j = 0; j < 4; j++) {
168 buf.put(fuzzyValue);
169 }
170 buf.putInt(i);
171
172 Pair<byte[], byte[]> pair = new Pair<byte[], byte[]>(fuzzyKey, mask);
173 list.add(pair);
174 }
175
176 int expectedSize = secondPartCardinality * totalFuzzyKeys * colQualifiersTotal;
177 FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
178
179 FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
180
181
182 runScanner(hTable, expectedSize, fuzzyRowFilter0);
183
184 runScanner(hTable, expectedSize, fuzzyRowFilter1);
185
186 }
187
188 private void runTest2(HTable hTable) throws IOException {
189
190
191 byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
192
193 List<Pair<byte[], byte[]>> list = new ArrayList<Pair<byte[], byte[]>>();
194
195 for (int i = 0; i < totalFuzzyKeys; i++) {
196 byte[] fuzzyKey = new byte[10];
197 ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
198 buf.clear();
199 buf.putShort((short) (i * 2));
200 for (int j = 0; j < 4; j++) {
201 buf.put(fuzzyValue);
202 }
203 buf.putInt(i * 2);
204
205 Pair<byte[], byte[]> pair = new Pair<byte[], byte[]>(fuzzyKey, mask);
206 list.add(pair);
207 }
208
209 int expectedSize = totalFuzzyKeys * secondPartCardinality * colQualifiersTotal;
210
211 FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
212
213 FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
214
215
216 runScanner(hTable, expectedSize, fuzzyRowFilter0);
217
218 runScanner(hTable, expectedSize, fuzzyRowFilter1);
219
220 }
221
222 private void runScanner(HTable hTable, int expectedSize, Filter filter) throws IOException {
223
224 String cf = "f";
225 Scan scan = new Scan();
226 scan.addFamily(cf.getBytes());
227 scan.setFilter(filter);
228 List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes());
229 HRegion first = regions.get(0);
230 first.getScanner(scan);
231 RegionScanner scanner = first.getScanner(scan);
232 List<Cell> results = new ArrayList<Cell>();
233
234 long timeBeforeScan = System.currentTimeMillis();
235 int found = 0;
236 while (scanner.next(results)) {
237 found += results.size();
238 results.clear();
239 }
240 found += results.size();
241 long scanTime = System.currentTimeMillis() - timeBeforeScan;
242 scanner.close();
243
244 LOG.info("\nscan time = " + scanTime + "ms");
245 LOG.info("found " + found + " results\n");
246
247 assertEquals(expectedSize, found);
248 }
249
250 @SuppressWarnings("deprecation")
251 @Test
252 public void testFilterList() throws Exception {
253 String cf = "f";
254 String table = "TestFuzzyRowFiltersInFilterList";
255 HTable ht =
256 TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
257
258
259
260
261
262 for (int i1 = 0; i1 < 5; i1++) {
263 for (int i2 = 0; i2 < 5; i2++) {
264 byte[] rk = new byte[10];
265
266 ByteBuffer buf = ByteBuffer.wrap(rk);
267 buf.clear();
268 buf.putShort((short) 2);
269 buf.putInt(i1);
270 buf.putInt(i2);
271
272
273 for (int c = 0; c < 5; c++) {
274 byte[] cq = new byte[4];
275 Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
276
277 Put p = new Put(rk);
278 p.setDurability(Durability.SKIP_WAL);
279 p.add(cf.getBytes(), cq, Bytes.toBytes(c));
280 ht.put(p);
281 LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: "
282 + Bytes.toStringBinary(cq));
283 }
284 }
285 }
286
287 TEST_UTIL.flush();
288
289
290 runTest(ht, 5);
291
292 }
293
294 @SuppressWarnings("unchecked")
295 private void runTest(HTable hTable, int expectedSize) throws IOException {
296
297 byte[] fuzzyKey1 = new byte[10];
298 ByteBuffer buf = ByteBuffer.wrap(fuzzyKey1);
299 buf.clear();
300 buf.putShort((short) 2);
301 for (int i = 0; i < 4; i++)
302 buf.put(fuzzyValue);
303 buf.putInt((short) 1);
304 byte[] mask1 = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
305
306 byte[] fuzzyKey2 = new byte[10];
307 buf = ByteBuffer.wrap(fuzzyKey2);
308 buf.clear();
309 buf.putShort((short) 2);
310 buf.putInt((short) 2);
311 for (int i = 0; i < 4; i++)
312 buf.put(fuzzyValue);
313
314 byte[] mask2 = new byte[] { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1 };
315
316 Pair<byte[], byte[]> pair1 = new Pair<byte[], byte[]>(fuzzyKey1, mask1);
317 Pair<byte[], byte[]> pair2 = new Pair<byte[], byte[]>(fuzzyKey2, mask2);
318
319 FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(Lists.newArrayList(pair1));
320 FuzzyRowFilter fuzzyRowFilter2 = new FuzzyRowFilter(Lists.newArrayList(pair2));
321
322 runScanner(hTable, expectedSize, fuzzyRowFilter1, fuzzyRowFilter2);
323 }
324
325 private void runScanner(HTable hTable, int expectedSize, Filter filter1, Filter filter2)
326 throws IOException {
327 String cf = "f";
328 Scan scan = new Scan();
329 scan.addFamily(cf.getBytes());
330 FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
331 scan.setFilter(filterList);
332
333 ResultScanner scanner = hTable.getScanner(scan);
334 List<Cell> results = new ArrayList<Cell>();
335 Result result;
336 long timeBeforeScan = System.currentTimeMillis();
337 while ((result = scanner.next()) != null) {
338 for (Cell kv : result.listCells()) {
339 LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
340 + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
341 results.add(kv);
342 }
343 }
344 long scanTime = System.currentTimeMillis() - timeBeforeScan;
345 scanner.close();
346
347 LOG.info("scan time = " + scanTime + "ms");
348 LOG.info("found " + results.size() + " results");
349
350 assertEquals(expectedSize, results.size());
351 }
352 }