View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.filter;
19  
20  import static org.junit.Assert.assertEquals;
21  
22  import java.io.IOException;
23  import java.nio.ByteBuffer;
24  import java.util.ArrayList;
25  import java.util.List;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.hbase.Cell;
31  import org.apache.hadoop.hbase.CellUtil;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.TableName;
35  import org.apache.hadoop.hbase.client.Durability;
36  import org.apache.hadoop.hbase.client.Put;
37  import org.apache.hadoop.hbase.client.Result;
38  import org.apache.hadoop.hbase.client.ResultScanner;
39  import org.apache.hadoop.hbase.client.Scan;
40  import org.apache.hadoop.hbase.client.HTable;
41  import org.apache.hadoop.hbase.filter.FilterList.Operator;
42  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
43  import org.apache.hadoop.hbase.regionserver.HRegion;
44  import org.apache.hadoop.hbase.regionserver.RegionScanner;
45  import org.apache.hadoop.hbase.testclassification.MediumTests;
46  import org.apache.hadoop.hbase.util.Bytes;
47  import org.apache.hadoop.hbase.util.Pair;
48  import org.junit.After;
49  import org.junit.AfterClass;
50  import org.junit.Before;
51  import org.junit.BeforeClass;
52  import org.junit.Test;
53  import org.junit.experimental.categories.Category;
54  
55  import com.google.common.collect.Lists;
56  
57  /**
58   */
59  @Category(MediumTests.class)
60  public class TestFuzzyRowFilterEndToEnd {
61    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
62    private final static byte fuzzyValue = (byte) 63;
63    private static final Log LOG = LogFactory.getLog(TestFuzzyRowFilterEndToEnd.class);
64  
65    private static int firstPartCardinality = 50;
66    private static int secondPartCardinality = 50;
67    private static int thirdPartCardinality = 50;
68    private static int colQualifiersTotal = 5;
69    private static int totalFuzzyKeys = thirdPartCardinality / 2;
70  
71    private static String table = "TestFuzzyRowFilterEndToEnd";
72  
73    /**
74     * @throws java.lang.Exception
75     */
76    @BeforeClass
77    public static void setUpBeforeClass() throws Exception {
78      Configuration conf = TEST_UTIL.getConfiguration();
79      conf.setInt("hbase.client.scanner.caching", 1000);
80      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
81        ConstantSizeRegionSplitPolicy.class.getName());
82      // set no splits
83      conf.setLong(HConstants.HREGION_MAX_FILESIZE, ((long) 1024) * 1024 * 1024 * 10);
84  
85      TEST_UTIL.startMiniCluster();
86    }
87  
88    /**
89     * @throws java.lang.Exception
90     */
91    @AfterClass
92    public static void tearDownAfterClass() throws Exception {
93      TEST_UTIL.shutdownMiniCluster();
94    }
95  
96    /**
97     * @throws java.lang.Exception
98     */
99    @Before
100   public void setUp() throws Exception {
101     // Nothing to do.
102   }
103 
104   /**
105    * @throws java.lang.Exception
106    */
107   @After
108   public void tearDown() throws Exception {
109     // Nothing to do.
110   }
111 
112   @Test
113   public void testEndToEnd() throws Exception {
114     String cf = "f";
115 
116     HTable ht =
117         TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
118 
119     // 10 byte row key - (2 bytes 4 bytes 4 bytes)
120     // 4 byte qualifier
121     // 4 byte value
122 
123     for (int i0 = 0; i0 < firstPartCardinality; i0++) {
124 
125       for (int i1 = 0; i1 < secondPartCardinality; i1++) {
126 
127         for (int i2 = 0; i2 < thirdPartCardinality; i2++) {
128           byte[] rk = new byte[10];
129 
130           ByteBuffer buf = ByteBuffer.wrap(rk);
131           buf.clear();
132           buf.putShort((short) i0);
133           buf.putInt(i1);
134           buf.putInt(i2);
135           for (int c = 0; c < colQualifiersTotal; c++) {
136             byte[] cq = new byte[4];
137             Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
138 
139             Put p = new Put(rk);
140             p.setDurability(Durability.SKIP_WAL);
141             p.add(cf.getBytes(), cq, Bytes.toBytes(c));
142             ht.put(p);
143           }
144         }
145       }
146     }
147 
148     TEST_UTIL.flush();
149 
150     // test passes
151     runTest1(ht);
152     runTest2(ht);
153 
154   }
155 
156   private void runTest1(HTable hTable) throws IOException {
157     // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
158 
159     byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
160 
161     List<Pair<byte[], byte[]>> list = new ArrayList<Pair<byte[], byte[]>>();
162     for (int i = 0; i < totalFuzzyKeys; i++) {
163       byte[] fuzzyKey = new byte[10];
164       ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
165       buf.clear();
166       buf.putShort((short) 2);
167       for (int j = 0; j < 4; j++) {
168         buf.put(fuzzyValue);
169       }
170       buf.putInt(i);
171 
172       Pair<byte[], byte[]> pair = new Pair<byte[], byte[]>(fuzzyKey, mask);
173       list.add(pair);
174     }
175 
176     int expectedSize = secondPartCardinality * totalFuzzyKeys * colQualifiersTotal;
177     FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
178     // Filters are not stateless - we can't reuse them
179     FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
180 
181     // regular test
182     runScanner(hTable, expectedSize, fuzzyRowFilter0);
183     // optimized from block cache
184     runScanner(hTable, expectedSize, fuzzyRowFilter1);
185 
186   }
187 
188   private void runTest2(HTable hTable) throws IOException {
189     // [0, 0, ?, ?, ?, ?, 0, 0, 0, 0] , [0, 1, ?, ?, ?, ?, 0, 0, 0, 1]...
190 
191     byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
192 
193     List<Pair<byte[], byte[]>> list = new ArrayList<Pair<byte[], byte[]>>();
194 
195     for (int i = 0; i < totalFuzzyKeys; i++) {
196       byte[] fuzzyKey = new byte[10];
197       ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
198       buf.clear();
199       buf.putShort((short) (i * 2));
200       for (int j = 0; j < 4; j++) {
201         buf.put(fuzzyValue);
202       }
203       buf.putInt(i * 2);
204 
205       Pair<byte[], byte[]> pair = new Pair<byte[], byte[]>(fuzzyKey, mask);
206       list.add(pair);
207     }
208 
209     int expectedSize = totalFuzzyKeys * secondPartCardinality * colQualifiersTotal;
210 
211     FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
212     // Filters are not stateless - we can't reuse them
213     FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
214 
215     // regular test
216     runScanner(hTable, expectedSize, fuzzyRowFilter0);
217     // optimized from block cache
218     runScanner(hTable, expectedSize, fuzzyRowFilter1);
219 
220   }
221 
222   private void runScanner(HTable hTable, int expectedSize, Filter filter) throws IOException {
223 
224     String cf = "f";
225     Scan scan = new Scan();
226     scan.addFamily(cf.getBytes());
227     scan.setFilter(filter);
228     List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes());
229     HRegion first = regions.get(0);
230     first.getScanner(scan);
231     RegionScanner scanner = first.getScanner(scan);
232     List<Cell> results = new ArrayList<Cell>();
233     // Result result;
234     long timeBeforeScan = System.currentTimeMillis();
235     int found = 0;
236     while (scanner.next(results)) {
237       found += results.size();
238       results.clear();
239     }
240     found += results.size();
241     long scanTime = System.currentTimeMillis() - timeBeforeScan;
242     scanner.close();
243 
244     LOG.info("\nscan time = " + scanTime + "ms");
245     LOG.info("found " + found + " results\n");
246 
247     assertEquals(expectedSize, found);
248   }
249 
250   @SuppressWarnings("deprecation")
251   @Test
252   public void testFilterList() throws Exception {
253     String cf = "f";
254     String table = "TestFuzzyRowFiltersInFilterList";
255     HTable ht =
256         TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
257 
258     // 10 byte row key - (2 bytes 4 bytes 4 bytes)
259     // 4 byte qualifier
260     // 4 byte value
261 
262     for (int i1 = 0; i1 < 5; i1++) {
263       for (int i2 = 0; i2 < 5; i2++) {
264         byte[] rk = new byte[10];
265 
266         ByteBuffer buf = ByteBuffer.wrap(rk);
267         buf.clear();
268         buf.putShort((short) 2);
269         buf.putInt(i1);
270         buf.putInt(i2);
271 
272         // Each row contains 5 columns
273         for (int c = 0; c < 5; c++) {
274           byte[] cq = new byte[4];
275           Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
276 
277           Put p = new Put(rk);
278           p.setDurability(Durability.SKIP_WAL);
279           p.add(cf.getBytes(), cq, Bytes.toBytes(c));
280           ht.put(p);
281           LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: "
282               + Bytes.toStringBinary(cq));
283         }
284       }
285     }
286 
287     TEST_UTIL.flush();
288 
289     // test passes if we get back 5 KV's (1 row)
290     runTest(ht, 5);
291 
292   }
293 
294   @SuppressWarnings("unchecked")
295   private void runTest(HTable hTable, int expectedSize) throws IOException {
296     // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
297     byte[] fuzzyKey1 = new byte[10];
298     ByteBuffer buf = ByteBuffer.wrap(fuzzyKey1);
299     buf.clear();
300     buf.putShort((short) 2);
301     for (int i = 0; i < 4; i++)
302       buf.put(fuzzyValue);
303     buf.putInt((short) 1);
304     byte[] mask1 = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
305 
306     byte[] fuzzyKey2 = new byte[10];
307     buf = ByteBuffer.wrap(fuzzyKey2);
308     buf.clear();
309     buf.putShort((short) 2);
310     buf.putInt((short) 2);
311     for (int i = 0; i < 4; i++)
312       buf.put(fuzzyValue);
313 
314     byte[] mask2 = new byte[] { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1 };
315 
316     Pair<byte[], byte[]> pair1 = new Pair<byte[], byte[]>(fuzzyKey1, mask1);
317     Pair<byte[], byte[]> pair2 = new Pair<byte[], byte[]>(fuzzyKey2, mask2);
318 
319     FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(Lists.newArrayList(pair1));
320     FuzzyRowFilter fuzzyRowFilter2 = new FuzzyRowFilter(Lists.newArrayList(pair2));
321     // regular test - we expect 1 row back (5 KVs)
322     runScanner(hTable, expectedSize, fuzzyRowFilter1, fuzzyRowFilter2);
323   }
324 
325   private void runScanner(HTable hTable, int expectedSize, Filter filter1, Filter filter2)
326       throws IOException {
327     String cf = "f";
328     Scan scan = new Scan();
329     scan.addFamily(cf.getBytes());
330     FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
331     scan.setFilter(filterList);
332 
333     ResultScanner scanner = hTable.getScanner(scan);
334     List<Cell> results = new ArrayList<Cell>();
335     Result result;
336     long timeBeforeScan = System.currentTimeMillis();
337     while ((result = scanner.next()) != null) {
338       for (Cell kv : result.listCells()) {
339         LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
340             + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
341         results.add(kv);
342       }
343     }
344     long scanTime = System.currentTimeMillis() - timeBeforeScan;
345     scanner.close();
346 
347     LOG.info("scan time = " + scanTime + "ms");
348     LOG.info("found " + results.size() + " results");
349 
350     assertEquals(expectedSize, results.size());
351   }
352 }