View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.filter;
19  
20  import static org.junit.Assert.assertEquals;
21  
22  import java.io.IOException;
23  import java.nio.ByteBuffer;
24  import java.util.ArrayList;
25  import java.util.List;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.hbase.Cell;
31  import org.apache.hadoop.hbase.CellUtil;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.TableName;
35  import org.apache.hadoop.hbase.client.Durability;
36  import org.apache.hadoop.hbase.client.Put;
37  import org.apache.hadoop.hbase.client.Result;
38  import org.apache.hadoop.hbase.client.ResultScanner;
39  import org.apache.hadoop.hbase.client.Scan;
40  import org.apache.hadoop.hbase.client.HTable;
41  import org.apache.hadoop.hbase.filter.FilterList.Operator;
42  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
43  import org.apache.hadoop.hbase.regionserver.HRegion;
44  import org.apache.hadoop.hbase.regionserver.RegionScanner;
45  import org.apache.hadoop.hbase.testclassification.MediumTests;
46  import org.apache.hadoop.hbase.util.Bytes;
47  import org.apache.hadoop.hbase.util.Pair;
48  import org.junit.After;
49  import org.junit.AfterClass;
50  import org.junit.Before;
51  import org.junit.BeforeClass;
52  import org.junit.Test;
53  import org.junit.experimental.categories.Category;
54  
55  import com.google.common.collect.Lists;
56  
57  /**
58   */
59  @Category(MediumTests.class)
60  public class TestFuzzyRowFilterEndToEnd {
61    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
62    private static final Log LOG = LogFactory.getLog(TestFuzzyRowFilterEndToEnd.class);
63  
64    private static int firstPartCardinality = 50;
65    private static int secondPartCardinality = 40;
66    private static int colQualifiersTotal = 50;
67    private static int totalFuzzyKeys = secondPartCardinality / 2;
68  
69    private static String table = "TestFuzzyRowFilterEndToEnd";
70  
71    /**
72     * @throws java.lang.Exception
73     */
74    @BeforeClass
75    public static void setUpBeforeClass() throws Exception {
76      Configuration conf = TEST_UTIL.getConfiguration();
77      conf.setInt("hbase.client.scanner.caching", 1000);
78      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
79        ConstantSizeRegionSplitPolicy.class.getName());
80      // set no splits
81      conf.setLong(HConstants.HREGION_MAX_FILESIZE, ((long) 1024) * 1024 * 1024 * 10);
82  
83      TEST_UTIL.startMiniCluster();
84    }
85  
86    /**
87     * @throws java.lang.Exception
88     */
89    @AfterClass
90    public static void tearDownAfterClass() throws Exception {
91      TEST_UTIL.shutdownMiniCluster();
92    }
93  
94    /**
95     * @throws java.lang.Exception
96     */
97    @Before
98    public void setUp() throws Exception {
99      // Nothing to do.
100   }
101 
102   /**
103    * @throws java.lang.Exception
104    */
105   @After
106   public void tearDown() throws Exception {
107     // Nothing to do.
108   }
109 
110   @Test
111   public void testEndToEnd() throws Exception {
112     String cf = "f";
113 
114     HTable ht =
115         TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
116 
117     // 10 byte row key - (2 bytes 4 bytes 4 bytes)
118     // 4 byte qualifier
119     // 4 byte value
120 
121     for (int i1 = 0; i1 < firstPartCardinality; i1++) {
122       if ((i1 % 1000) == 0) LOG.info("put " + i1);
123 
124       for (int i2 = 0; i2 < secondPartCardinality; i2++) {
125         byte[] rk = new byte[10];
126 
127         ByteBuffer buf = ByteBuffer.wrap(rk);
128         buf.clear();
129         buf.putShort((short) 2);
130         buf.putInt(i1);
131         buf.putInt(i2);
132         for (int c = 0; c < colQualifiersTotal; c++) {
133           byte[] cq = new byte[4];
134           Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
135 
136           Put p = new Put(rk);
137           p.setDurability(Durability.SKIP_WAL);
138           p.add(cf.getBytes(), cq, Bytes.toBytes(c));
139           ht.put(p);
140         }
141       }
142     }
143 
144     TEST_UTIL.flush();
145 
146     // test passes
147     runTest(ht);
148 
149   }
150 
151   private void runTest(HTable hTable) throws IOException {
152     // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
153 
154     byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
155 
156     List<Pair<byte[], byte[]>> list = new ArrayList<Pair<byte[], byte[]>>();
157     for (int i = 0; i < totalFuzzyKeys; i++) {
158       byte[] fuzzyKey = new byte[10];
159       ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
160       buf.clear();
161       buf.putShort((short) 2);
162       for (int j = 0; j < 4; j++) {
163         buf.put((byte) 63);
164       }
165       buf.putInt(i);
166 
167       Pair<byte[], byte[]> pair = new Pair<byte[], byte[]>(fuzzyKey, mask);
168       list.add(pair);
169     }
170 
171     int expectedSize = firstPartCardinality * totalFuzzyKeys * colQualifiersTotal;
172     FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
173     // Filters are not stateless - we can't reuse them
174     FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);
175 
176     // regular test
177     runScanner(hTable, expectedSize, fuzzyRowFilter0);
178     // optimized from block cache
179     runScanner(hTable, expectedSize, fuzzyRowFilter1);
180 
181   }
182 
183   private void runScanner(HTable hTable, int expectedSize, Filter filter) throws IOException {
184 
185     String cf = "f";
186     Scan scan = new Scan();
187     scan.addFamily(cf.getBytes());
188     scan.setFilter(filter);
189     List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes());
190     HRegion first = regions.get(0);
191     first.getScanner(scan);
192     RegionScanner scanner = first.getScanner(scan);
193     List<Cell> results = new ArrayList<Cell>();
194     // Result result;
195     long timeBeforeScan = System.currentTimeMillis();
196     int found = 0;
197     while (scanner.next(results)) {
198       found += results.size();
199       results.clear();
200     }
201     found += results.size();
202     long scanTime = System.currentTimeMillis() - timeBeforeScan;
203     scanner.close();
204 
205     LOG.info("\nscan time = " + scanTime + "ms");
206     LOG.info("found " + found + " results\n");
207 
208     assertEquals(expectedSize, found);
209   }
210   
211   @SuppressWarnings("deprecation")
212   @Test
213   public void testFilterList() throws Exception {
214     String cf = "f";
215     String table = "TestFuzzyRowFiltersInFilterList";
216     HTable ht =
217         TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
218 
219     // 10 byte row key - (2 bytes 4 bytes 4 bytes)
220     // 4 byte qualifier
221     // 4 byte value
222 
223     for (int i1 = 0; i1 < 5; i1++) {
224       for (int i2 = 0; i2 < 5; i2++) {
225         byte[] rk = new byte[10];
226 
227         ByteBuffer buf = ByteBuffer.wrap(rk);
228         buf.clear();
229         buf.putShort((short) 2);
230         buf.putInt(i1);
231         buf.putInt(i2);
232 
233         // Each row contains 5 columns
234         for (int c = 0; c < 5; c++) {
235           byte[] cq = new byte[4];
236           Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
237 
238           Put p = new Put(rk);
239           p.setDurability(Durability.SKIP_WAL);
240           p.add(cf.getBytes(), cq, Bytes.toBytes(c));
241           ht.put(p);
242           LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: "
243               + Bytes.toStringBinary(cq));
244         }
245       }
246     }
247 
248     TEST_UTIL.flush();
249 
250     // test passes if we get back 5 KV's (1 row)
251     runTest(ht, 5);
252 
253   }
254 
255   @SuppressWarnings("unchecked")
256   private void runTest(HTable hTable, int expectedSize) throws IOException {
257     // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
258     byte[] fuzzyKey1 = new byte[10];
259     ByteBuffer buf = ByteBuffer.wrap(fuzzyKey1);
260     buf.clear();
261     buf.putShort((short) 2);
262     for (int i = 0; i < 4; i++)
263       buf.put((byte) 63);
264     buf.putInt((short) 1);
265     byte[] mask1 = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
266 
267     byte[] fuzzyKey2 = new byte[10];
268     buf = ByteBuffer.wrap(fuzzyKey2);
269     buf.clear();
270     buf.putShort((short) 2);
271     buf.putInt((short) 2);
272     for (int i = 0; i < 4; i++)
273       buf.put((byte) 63);
274 
275     byte[] mask2 = new byte[] { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1 };
276 
277     Pair<byte[], byte[]> pair1 = new Pair<byte[], byte[]>(fuzzyKey1, mask1);
278     Pair<byte[], byte[]> pair2 = new Pair<byte[], byte[]>(fuzzyKey2, mask2);
279 
280     FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(Lists.newArrayList(pair1));
281     FuzzyRowFilter fuzzyRowFilter2 = new FuzzyRowFilter(Lists.newArrayList(pair2));
282     // regular test - we expect 1 row back (5 KVs)
283     runScanner(hTable, expectedSize, fuzzyRowFilter1, fuzzyRowFilter2);
284   }
285 
286   private void runScanner(HTable hTable, int expectedSize, Filter filter1, Filter filter2) throws IOException {
287     String cf = "f";
288     Scan scan = new Scan();
289     scan.addFamily(cf.getBytes());
290     FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
291     scan.setFilter(filterList);
292 
293     ResultScanner scanner = hTable.getScanner(scan);
294     List<Cell> results = new ArrayList<Cell>();
295     Result result;
296     long timeBeforeScan = System.currentTimeMillis();
297     while ((result = scanner.next()) != null) {
298       for (Cell kv : result.listCells()) {
299         LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
300             + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
301         results.add(kv);
302       }
303     }
304     long scanTime = System.currentTimeMillis() - timeBeforeScan;
305     scanner.close();
306 
307     LOG.info("scan time = " + scanTime + "ms");
308     LOG.info("found " + results.size() + " results");
309 
310     assertEquals(expectedSize, results.size());
311   }
312 }