View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.filter;
19  
20  import static org.junit.Assert.assertEquals;
21  
22  import java.io.IOException;
23  import java.nio.ByteBuffer;
24  import java.util.ArrayList;
25  import java.util.List;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.hbase.Cell;
31  import org.apache.hadoop.hbase.CellUtil;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.TableName;
35  import org.apache.hadoop.hbase.client.Durability;
36  import org.apache.hadoop.hbase.client.Put;
37  import org.apache.hadoop.hbase.client.Result;
38  import org.apache.hadoop.hbase.client.ResultScanner;
39  import org.apache.hadoop.hbase.client.Scan;
40  import org.apache.hadoop.hbase.client.HTable;
41  import org.apache.hadoop.hbase.filter.FilterList.Operator;
42  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
43  import org.apache.hadoop.hbase.regionserver.HRegion;
44  import org.apache.hadoop.hbase.regionserver.RegionScanner;
45  import org.apache.hadoop.hbase.testclassification.MediumTests;
46  import org.apache.hadoop.hbase.util.Bytes;
47  import org.apache.hadoop.hbase.util.Pair;
48  import org.junit.After;
49  import org.junit.AfterClass;
50  import org.junit.Before;
51  import org.junit.BeforeClass;
52  import org.junit.Test;
53  import org.junit.experimental.categories.Category;
54  
55  import com.google.common.collect.Lists;
56  
57  /**
58   */
59  @Category(MediumTests.class)
60  public class TestFuzzyRowFilterEndToEnd {
61    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
62    private final static byte fuzzyValue = (byte) 63;
63    private static final Log LOG = LogFactory.getLog(TestFuzzyRowFilterEndToEnd.class);
64  
65    private static int firstPartCardinality = 50;
66    private static int secondPartCardinality = 50;
67    private static int thirdPartCardinality = 50;
68    private static int colQualifiersTotal = 5;
69    private static int totalFuzzyKeys = thirdPartCardinality / 2;
70  
71    private static String table = "TestFuzzyRowFilterEndToEnd";
72  
73    /**
74     * @throws java.lang.Exception
75     */
76    @BeforeClass
77    public static void setUpBeforeClass() throws Exception {
78      Configuration conf = TEST_UTIL.getConfiguration();
79      conf.setInt("hbase.client.scanner.caching", 1000);
80      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
81        ConstantSizeRegionSplitPolicy.class.getName());
82      // set no splits
83      conf.setLong(HConstants.HREGION_MAX_FILESIZE, ((long) 1024) * 1024 * 1024 * 10);
84  
85      TEST_UTIL.startMiniCluster();
86    }
87  
88    /**
89     * @throws java.lang.Exception
90     */
91    @AfterClass
92    public static void tearDownAfterClass() throws Exception {
93      TEST_UTIL.shutdownMiniCluster();
94    }
95  
96    /**
97     * @throws java.lang.Exception
98     */
99    @Before
100   public void setUp() throws Exception {
101     // Nothing to do.
102   }
103 
104   /**
105    * @throws java.lang.Exception
106    */
107   @After
108   public void tearDown() throws Exception {
109     // Nothing to do.
110   }
111 
112   @Test  
113   public void testHBASE14782() throws IOException
114   {
115     String cf = "f";
116     String cq = "q";
117     String table = "HBASE14872";
118 
119     HTable ht =
120         TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
121     // Load data
122     String[] rows = new String[]{
123         "\\x9C\\x00\\x044\\x00\\x00\\x00\\x00",
124         "\\x9C\\x00\\x044\\x01\\x00\\x00\\x00", 
125         "\\x9C\\x00\\x044\\x00\\x01\\x00\\x00",
126         "\\x9C\\x00\\x044\\x00\\x00\\x01\\x00",
127         "\\x9C\\x00\\x044\\x00\\x01\\x00\\x01", 
128         "\\x9B\\x00\\x044e\\xBB\\xB2\\xBB", 
129     };
130     
131     String badRow = "\\x9C\\x00\\x03\\xE9e\\xBB{X\\x1Fwts\\x1F\\x15vRX";
132     
133     for(int i=0; i < rows.length; i++){
134       Put p = new Put(Bytes.toBytesBinary(rows[i]));
135       p.add(cf.getBytes(), cq.getBytes(), "value".getBytes());
136       ht.put(p);            
137     }
138     
139     Put p = new Put(Bytes.toBytesBinary(badRow));
140     p.add(cf.getBytes(), cq.getBytes(), "value".getBytes());
141     ht.put(p);            
142 
143     TEST_UTIL.flush();
144 
145     List<Pair<byte[], byte[]>> data =  new ArrayList<Pair<byte[], byte[]>>();
146     byte[] fuzzyKey = Bytes.toBytesBinary("\\x00\\x00\\x044");
147     byte[] mask = new byte[] { 1,0,0,0};
148     data.add(new Pair<byte[], byte[]>(fuzzyKey, mask));
149     FuzzyRowFilter filter = new FuzzyRowFilter(data);
150     
151     Scan scan = new Scan();
152     scan.setFilter(filter);
153     
154     ResultScanner scanner = ht.getScanner(scan);
155     int total = 0;
156     while(scanner.next() != null){
157       total++;
158     }    
159     assertEquals(rows.length, total);
160     TEST_UTIL.deleteTable(TableName.valueOf(table));
161   }
162   
163   @Test
164   public void testEndToEnd() throws Exception {
165     String cf = "f";
166 
167     HTable ht =
168         TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
169 
170     // 10 byte row key - (2 bytes 4 bytes 4 bytes)
171     // 4 byte qualifier
172     // 4 byte value
173 
174     for (int i0 = 0; i0 < firstPartCardinality; i0++) {
175 
176       for (int i1 = 0; i1 < secondPartCardinality; i1++) {
177 
178         for (int i2 = 0; i2 < thirdPartCardinality; i2++) {
179           byte[] rk = new byte[10];
180 
181           ByteBuffer buf = ByteBuffer.wrap(rk);
182           buf.clear();
183           buf.putShort((short) i0);
184           buf.putInt(i1);
185           buf.putInt(i2);
186           for (int c = 0; c < colQualifiersTotal; c++) {
187             byte[] cq = new byte[4];
188             Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
189 
190             Put p = new Put(rk);
191             p.setDurability(Durability.SKIP_WAL);
192             p.add(cf.getBytes(), cq, Bytes.toBytes(c));
193             ht.put(p);
194           }
195         }
196       }
197     }
198 
199     TEST_UTIL.flush();
200 
201     // test passes
202     runTest1(ht);
203     runTest2(ht);
204 
205   }
206 
207   private void runTest1(HTable hTable) throws IOException {
208     // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
209 
210     byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
211 
212     List<Pair<byte[], byte[]>> list = new ArrayList<Pair<byte[], byte[]>>();
213     for (int i = 0; i < totalFuzzyKeys; i++) {
214       byte[] fuzzyKey = new byte[10];
215       ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
216       buf.clear();
217       buf.putShort((short) 2);
218       for (int j = 0; j < 4; j++) {
219         buf.put(fuzzyValue);
220       }
221       buf.putInt(i);
222 
223       Pair<byte[], byte[]> pair = new Pair<byte[], byte[]>(fuzzyKey, mask);
224       list.add(pair);
225     }
226 
227     int expectedSize = secondPartCardinality * totalFuzzyKeys * colQualifiersTotal;
228     FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
229     // Filters are not stateless - we can't reuse them
230     FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
231 
232     // regular test
233     runScanner(hTable, expectedSize, fuzzyRowFilter0);
234     // optimized from block cache
235     runScanner(hTable, expectedSize, fuzzyRowFilter1);
236 
237   }
238 
239   private void runTest2(HTable hTable) throws IOException {
240     // [0, 0, ?, ?, ?, ?, 0, 0, 0, 0] , [0, 1, ?, ?, ?, ?, 0, 0, 0, 1]...
241 
242     byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
243 
244     List<Pair<byte[], byte[]>> list = new ArrayList<Pair<byte[], byte[]>>();
245 
246     for (int i = 0; i < totalFuzzyKeys; i++) {
247       byte[] fuzzyKey = new byte[10];
248       ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
249       buf.clear();
250       buf.putShort((short) (i * 2));
251       for (int j = 0; j < 4; j++) {
252         buf.put(fuzzyValue);
253       }
254       buf.putInt(i * 2);
255 
256       Pair<byte[], byte[]> pair = new Pair<byte[], byte[]>(fuzzyKey, mask);
257       list.add(pair);
258     }
259 
260     int expectedSize = totalFuzzyKeys * secondPartCardinality * colQualifiersTotal;
261 
262     FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
263     // Filters are not stateless - we can't reuse them
264     FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
265 
266     // regular test
267     runScanner(hTable, expectedSize, fuzzyRowFilter0);
268     // optimized from block cache
269     runScanner(hTable, expectedSize, fuzzyRowFilter1);
270 
271   }
272 
273   private void runScanner(HTable hTable, int expectedSize, Filter filter) throws IOException {
274 
275     String cf = "f";
276     Scan scan = new Scan();
277     scan.addFamily(cf.getBytes());
278     scan.setFilter(filter);
279     List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes());
280     HRegion first = regions.get(0);
281     first.getScanner(scan);
282     RegionScanner scanner = first.getScanner(scan);
283     List<Cell> results = new ArrayList<Cell>();
284     // Result result;
285     long timeBeforeScan = System.currentTimeMillis();
286     int found = 0;
287     while (scanner.next(results)) {
288       found += results.size();
289       results.clear();
290     }
291     found += results.size();
292     long scanTime = System.currentTimeMillis() - timeBeforeScan;
293     scanner.close();
294 
295     LOG.info("\nscan time = " + scanTime + "ms");
296     LOG.info("found " + found + " results\n");
297 
298     assertEquals(expectedSize, found);
299   }
300 
301   @SuppressWarnings("deprecation")
302   @Test
303   public void testFilterList() throws Exception {
304     String cf = "f";
305     String table = "TestFuzzyRowFiltersInFilterList";
306     HTable ht =
307         TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
308 
309     // 10 byte row key - (2 bytes 4 bytes 4 bytes)
310     // 4 byte qualifier
311     // 4 byte value
312 
313     for (int i1 = 0; i1 < 5; i1++) {
314       for (int i2 = 0; i2 < 5; i2++) {
315         byte[] rk = new byte[10];
316 
317         ByteBuffer buf = ByteBuffer.wrap(rk);
318         buf.clear();
319         buf.putShort((short) 2);
320         buf.putInt(i1);
321         buf.putInt(i2);
322 
323         // Each row contains 5 columns
324         for (int c = 0; c < 5; c++) {
325           byte[] cq = new byte[4];
326           Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
327 
328           Put p = new Put(rk);
329           p.setDurability(Durability.SKIP_WAL);
330           p.add(cf.getBytes(), cq, Bytes.toBytes(c));
331           ht.put(p);
332           LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: "
333               + Bytes.toStringBinary(cq));
334         }
335       }
336     }
337 
338     TEST_UTIL.flush();
339 
340     // test passes if we get back 5 KV's (1 row)
341     runTest(ht, 5);
342 
343   }
344 
345   @SuppressWarnings("unchecked")
346   private void runTest(HTable hTable, int expectedSize) throws IOException {
347     // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
348     byte[] fuzzyKey1 = new byte[10];
349     ByteBuffer buf = ByteBuffer.wrap(fuzzyKey1);
350     buf.clear();
351     buf.putShort((short) 2);
352     for (int i = 0; i < 4; i++)
353       buf.put(fuzzyValue);
354     buf.putInt((short) 1);
355     byte[] mask1 = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
356 
357     byte[] fuzzyKey2 = new byte[10];
358     buf = ByteBuffer.wrap(fuzzyKey2);
359     buf.clear();
360     buf.putShort((short) 2);
361     buf.putInt((short) 2);
362     for (int i = 0; i < 4; i++)
363       buf.put(fuzzyValue);
364 
365     byte[] mask2 = new byte[] { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1 };
366 
367     Pair<byte[], byte[]> pair1 = new Pair<byte[], byte[]>(fuzzyKey1, mask1);
368     Pair<byte[], byte[]> pair2 = new Pair<byte[], byte[]>(fuzzyKey2, mask2);
369 
370     FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(Lists.newArrayList(pair1));
371     FuzzyRowFilter fuzzyRowFilter2 = new FuzzyRowFilter(Lists.newArrayList(pair2));
372     // regular test - we expect 1 row back (5 KVs)
373     runScanner(hTable, expectedSize, fuzzyRowFilter1, fuzzyRowFilter2);
374   }
375 
376   private void runScanner(HTable hTable, int expectedSize, Filter filter1, Filter filter2)
377       throws IOException {
378     String cf = "f";
379     Scan scan = new Scan();
380     scan.addFamily(cf.getBytes());
381     FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
382     scan.setFilter(filterList);
383 
384     ResultScanner scanner = hTable.getScanner(scan);
385     List<Cell> results = new ArrayList<Cell>();
386     Result result;
387     long timeBeforeScan = System.currentTimeMillis();
388     while ((result = scanner.next()) != null) {
389       for (Cell kv : result.listCells()) {
390         LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
391             + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
392         results.add(kv);
393       }
394     }
395     long scanTime = System.currentTimeMillis() - timeBeforeScan;
396     scanner.close();
397 
398     LOG.info("scan time = " + scanTime + "ms");
399     LOG.info("found " + results.size() + " results");
400 
401     assertEquals(expectedSize, results.size());
402   }
403 }