1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.filter;
19
20 import static org.junit.Assert.assertEquals;
21
22 import java.io.IOException;
23 import java.nio.ByteBuffer;
24 import java.util.ArrayList;
25 import java.util.List;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.hbase.Cell;
31 import org.apache.hadoop.hbase.CellUtil;
32 import org.apache.hadoop.hbase.HBaseTestingUtility;
33 import org.apache.hadoop.hbase.HConstants;
34 import org.apache.hadoop.hbase.TableName;
35 import org.apache.hadoop.hbase.client.Durability;
36 import org.apache.hadoop.hbase.client.Put;
37 import org.apache.hadoop.hbase.client.Result;
38 import org.apache.hadoop.hbase.client.ResultScanner;
39 import org.apache.hadoop.hbase.client.Scan;
40 import org.apache.hadoop.hbase.client.HTable;
41 import org.apache.hadoop.hbase.filter.FilterList.Operator;
42 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
43 import org.apache.hadoop.hbase.regionserver.HRegion;
44 import org.apache.hadoop.hbase.regionserver.RegionScanner;
45 import org.apache.hadoop.hbase.testclassification.MediumTests;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hbase.util.Pair;
48 import org.junit.After;
49 import org.junit.AfterClass;
50 import org.junit.Before;
51 import org.junit.BeforeClass;
52 import org.junit.Test;
53 import org.junit.experimental.categories.Category;
54
55 import com.google.common.collect.Lists;
56
57
58
59 @Category(MediumTests.class)
60 public class TestFuzzyRowFilterEndToEnd {
61 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
62 private static final Log LOG = LogFactory.getLog(TestFuzzyRowFilterEndToEnd.class);
63
64 private static int firstPartCardinality = 50;
65 private static int secondPartCardinality = 40;
66 private static int colQualifiersTotal = 50;
67 private static int totalFuzzyKeys = secondPartCardinality / 2;
68
69 private static String table = "TestFuzzyRowFilterEndToEnd";
70
71
72
73
74 @BeforeClass
75 public static void setUpBeforeClass() throws Exception {
76 Configuration conf = TEST_UTIL.getConfiguration();
77 conf.setInt("hbase.client.scanner.caching", 1000);
78 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
79 ConstantSizeRegionSplitPolicy.class.getName());
80
81 conf.setLong(HConstants.HREGION_MAX_FILESIZE, ((long) 1024) * 1024 * 1024 * 10);
82
83 TEST_UTIL.startMiniCluster();
84 }
85
86
87
88
89 @AfterClass
90 public static void tearDownAfterClass() throws Exception {
91 TEST_UTIL.shutdownMiniCluster();
92 }
93
94
95
96
97 @Before
98 public void setUp() throws Exception {
99
100 }
101
102
103
104
105 @After
106 public void tearDown() throws Exception {
107
108 }
109
110 @Test
111 public void testEndToEnd() throws Exception {
112 String cf = "f";
113
114 HTable ht =
115 TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
116
117
118
119
120
121 for (int i1 = 0; i1 < firstPartCardinality; i1++) {
122 if ((i1 % 1000) == 0) LOG.info("put " + i1);
123
124 for (int i2 = 0; i2 < secondPartCardinality; i2++) {
125 byte[] rk = new byte[10];
126
127 ByteBuffer buf = ByteBuffer.wrap(rk);
128 buf.clear();
129 buf.putShort((short) 2);
130 buf.putInt(i1);
131 buf.putInt(i2);
132 for (int c = 0; c < colQualifiersTotal; c++) {
133 byte[] cq = new byte[4];
134 Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
135
136 Put p = new Put(rk);
137 p.setDurability(Durability.SKIP_WAL);
138 p.add(cf.getBytes(), cq, Bytes.toBytes(c));
139 ht.put(p);
140 }
141 }
142 }
143
144 TEST_UTIL.flush();
145
146
147 runTest(ht);
148
149 }
150
151 private void runTest(HTable hTable) throws IOException {
152
153
154 byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
155
156 List<Pair<byte[], byte[]>> list = new ArrayList<Pair<byte[], byte[]>>();
157 for (int i = 0; i < totalFuzzyKeys; i++) {
158 byte[] fuzzyKey = new byte[10];
159 ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
160 buf.clear();
161 buf.putShort((short) 2);
162 for (int j = 0; j < 4; j++) {
163 buf.put((byte) 63);
164 }
165 buf.putInt(i);
166
167 Pair<byte[], byte[]> pair = new Pair<byte[], byte[]>(fuzzyKey, mask);
168 list.add(pair);
169 }
170
171 int expectedSize = firstPartCardinality * totalFuzzyKeys * colQualifiersTotal;
172 FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
173
174 FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
175
176
177 runScanner(hTable, expectedSize, fuzzyRowFilter0);
178
179 runScanner(hTable, expectedSize, fuzzyRowFilter1);
180
181 }
182
183 private void runScanner(HTable hTable, int expectedSize, Filter filter) throws IOException {
184
185 String cf = "f";
186 Scan scan = new Scan();
187 scan.addFamily(cf.getBytes());
188 scan.setFilter(filter);
189 List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes());
190 HRegion first = regions.get(0);
191 first.getScanner(scan);
192 RegionScanner scanner = first.getScanner(scan);
193 List<Cell> results = new ArrayList<Cell>();
194
195 long timeBeforeScan = System.currentTimeMillis();
196 int found = 0;
197 while (scanner.next(results)) {
198 found += results.size();
199 results.clear();
200 }
201 found += results.size();
202 long scanTime = System.currentTimeMillis() - timeBeforeScan;
203 scanner.close();
204
205 LOG.info("\nscan time = " + scanTime + "ms");
206 LOG.info("found " + found + " results\n");
207
208 assertEquals(expectedSize, found);
209 }
210
211 @SuppressWarnings("deprecation")
212 @Test
213 public void testFilterList() throws Exception {
214 String cf = "f";
215 String table = "TestFuzzyRowFiltersInFilterList";
216 HTable ht =
217 TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
218
219
220
221
222
223 for (int i1 = 0; i1 < 5; i1++) {
224 for (int i2 = 0; i2 < 5; i2++) {
225 byte[] rk = new byte[10];
226
227 ByteBuffer buf = ByteBuffer.wrap(rk);
228 buf.clear();
229 buf.putShort((short) 2);
230 buf.putInt(i1);
231 buf.putInt(i2);
232
233
234 for (int c = 0; c < 5; c++) {
235 byte[] cq = new byte[4];
236 Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
237
238 Put p = new Put(rk);
239 p.setDurability(Durability.SKIP_WAL);
240 p.add(cf.getBytes(), cq, Bytes.toBytes(c));
241 ht.put(p);
242 LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: "
243 + Bytes.toStringBinary(cq));
244 }
245 }
246 }
247
248 TEST_UTIL.flush();
249
250
251 runTest(ht, 5);
252
253 }
254
255 @SuppressWarnings("unchecked")
256 private void runTest(HTable hTable, int expectedSize) throws IOException {
257
258 byte[] fuzzyKey1 = new byte[10];
259 ByteBuffer buf = ByteBuffer.wrap(fuzzyKey1);
260 buf.clear();
261 buf.putShort((short) 2);
262 for (int i = 0; i < 4; i++)
263 buf.put((byte) 63);
264 buf.putInt((short) 1);
265 byte[] mask1 = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
266
267 byte[] fuzzyKey2 = new byte[10];
268 buf = ByteBuffer.wrap(fuzzyKey2);
269 buf.clear();
270 buf.putShort((short) 2);
271 buf.putInt((short) 2);
272 for (int i = 0; i < 4; i++)
273 buf.put((byte) 63);
274
275 byte[] mask2 = new byte[] { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1 };
276
277 Pair<byte[], byte[]> pair1 = new Pair<byte[], byte[]>(fuzzyKey1, mask1);
278 Pair<byte[], byte[]> pair2 = new Pair<byte[], byte[]>(fuzzyKey2, mask2);
279
280 FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(Lists.newArrayList(pair1));
281 FuzzyRowFilter fuzzyRowFilter2 = new FuzzyRowFilter(Lists.newArrayList(pair2));
282
283 runScanner(hTable, expectedSize, fuzzyRowFilter1, fuzzyRowFilter2);
284 }
285
286 private void runScanner(HTable hTable, int expectedSize, Filter filter1, Filter filter2) throws IOException {
287 String cf = "f";
288 Scan scan = new Scan();
289 scan.addFamily(cf.getBytes());
290 FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
291 scan.setFilter(filterList);
292
293 ResultScanner scanner = hTable.getScanner(scan);
294 List<Cell> results = new ArrayList<Cell>();
295 Result result;
296 long timeBeforeScan = System.currentTimeMillis();
297 while ((result = scanner.next()) != null) {
298 for (Cell kv : result.listCells()) {
299 LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
300 + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
301 results.add(kv);
302 }
303 }
304 long scanTime = System.currentTimeMillis() - timeBeforeScan;
305 scanner.close();
306
307 LOG.info("scan time = " + scanTime + "ms");
308 LOG.info("found " + results.size() + " results");
309
310 assertEquals(expectedSize, results.size());
311 }
312 }