View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.List;
25  import java.util.NavigableSet;
26  
27  import static org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode.*;
28  
29  import org.apache.hadoop.hbase.HBaseTestCase;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.KeyValue;
32  import org.apache.hadoop.hbase.KeyValue.KVComparator;
33  import org.apache.hadoop.hbase.KeyValue.Type;
34  import org.apache.hadoop.hbase.SmallTests;
35  import org.apache.hadoop.hbase.client.Get;
36  import org.apache.hadoop.hbase.client.Scan;
37  import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
40  import org.junit.experimental.categories.Category;
41  
42  @Category(SmallTests.class)
43  public class TestQueryMatcher extends HBaseTestCase {
44    private static final boolean PRINT = false;
45  
46    private byte[] row1;
47    private byte[] row2;
48    private byte[] row3;
49    private byte[] fam1;
50    private byte[] fam2;
51    private byte[] col1;
52    private byte[] col2;
53    private byte[] col3;
54    private byte[] col4;
55    private byte[] col5;
56  
57    private byte[] data;
58  
59    private Get get;
60  
61    long ttl = Long.MAX_VALUE;
62    KVComparator rowComparator;
63    private Scan scan;
64  
65    public void setUp() throws Exception {
66      super.setUp();
67      row1 = Bytes.toBytes("row1");
68      row2 = Bytes.toBytes("row2");
69      row3 = Bytes.toBytes("row3");
70      fam1 = Bytes.toBytes("fam1");
71      fam2 = Bytes.toBytes("fam2");
72      col1 = Bytes.toBytes("col1");
73      col2 = Bytes.toBytes("col2");
74      col3 = Bytes.toBytes("col3");
75      col4 = Bytes.toBytes("col4");
76      col5 = Bytes.toBytes("col5");
77  
78      data = Bytes.toBytes("data");
79  
80      //Create Get
81      get = new Get(row1);
82      get.addFamily(fam1);
83      get.addColumn(fam2, col2);
84      get.addColumn(fam2, col4);
85      get.addColumn(fam2, col5);
86      this.scan = new Scan(get);
87  
88      rowComparator = KeyValue.COMPARATOR;
89  
90    }
91  
92      private void _testMatch_ExplicitColumns(Scan scan, List<MatchCode> expected) throws IOException {
93      // 2,4,5    
94      ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
95          0, 1, ttl, false, 0, rowComparator), get.getFamilyMap().get(fam2),
96          EnvironmentEdgeManager.currentTimeMillis() - ttl);
97  
98      List<KeyValue> memstore = new ArrayList<KeyValue>();
99      memstore.add(new KeyValue(row1, fam2, col1, 1, data));
100     memstore.add(new KeyValue(row1, fam2, col2, 1, data));
101     memstore.add(new KeyValue(row1, fam2, col3, 1, data));
102     memstore.add(new KeyValue(row1, fam2, col4, 1, data));
103     memstore.add(new KeyValue(row1, fam2, col5, 1, data));
104 
105     memstore.add(new KeyValue(row2, fam1, col1, data));
106 
107     List<ScanQueryMatcher.MatchCode> actual = new ArrayList<ScanQueryMatcher.MatchCode>();
108     KeyValue k = memstore.get(0);
109     qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
110 
111     for (KeyValue kv : memstore){
112       actual.add(qm.match(kv));
113     }
114 
115     assertEquals(expected.size(), actual.size());
116     for(int i=0; i< expected.size(); i++){
117       assertEquals(expected.get(i), actual.get(i));
118       if(PRINT){
119         System.out.println("expected "+expected.get(i)+
120             ", actual " +actual.get(i));
121       }
122     }
123   }
124 
125   public void testMatch_ExplicitColumns()
126   throws IOException {
127     //Moving up from the Tracker by using Gets and List<KeyValue> instead
128     //of just byte []
129 
130     //Expected result
131     List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
132     expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
133     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
134     expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
135     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
136     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
137     expected.add(ScanQueryMatcher.MatchCode.DONE);
138 
139     _testMatch_ExplicitColumns(scan, expected);
140   }
141 
142   public void testMatch_ExplicitColumnsWithLookAhead()
143   throws IOException {
144     //Moving up from the Tracker by using Gets and List<KeyValue> instead
145     //of just byte []
146 
147     //Expected result
148     List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
149     expected.add(ScanQueryMatcher.MatchCode.SKIP);
150     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
151     expected.add(ScanQueryMatcher.MatchCode.SKIP);
152     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
153     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
154     expected.add(ScanQueryMatcher.MatchCode.DONE);
155 
156     Scan s = new Scan(scan);
157     s.setAttribute(Scan.HINT_LOOKAHEAD, Bytes.toBytes(2));
158     _testMatch_ExplicitColumns(s, expected);
159   }
160 
161 
162   public void testMatch_Wildcard()
163   throws IOException {
164     //Moving up from the Tracker by using Gets and List<KeyValue> instead
165     //of just byte []
166 
167     //Expected result
168     List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
169     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
170     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
171     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
172     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
173     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
174     expected.add(ScanQueryMatcher.MatchCode.DONE);
175 
176     ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
177         0, 1, ttl, false, 0, rowComparator), null,
178         EnvironmentEdgeManager.currentTimeMillis() - ttl);
179 
180     List<KeyValue> memstore = new ArrayList<KeyValue>();
181     memstore.add(new KeyValue(row1, fam2, col1, 1, data));
182     memstore.add(new KeyValue(row1, fam2, col2, 1, data));
183     memstore.add(new KeyValue(row1, fam2, col3, 1, data));
184     memstore.add(new KeyValue(row1, fam2, col4, 1, data));
185     memstore.add(new KeyValue(row1, fam2, col5, 1, data));
186     memstore.add(new KeyValue(row2, fam1, col1, 1, data));
187 
188     List<ScanQueryMatcher.MatchCode> actual = new ArrayList<ScanQueryMatcher.MatchCode>();
189 
190     KeyValue k = memstore.get(0);
191     qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
192 
193     for(KeyValue kv : memstore) {
194       actual.add(qm.match(kv));
195     }
196 
197     assertEquals(expected.size(), actual.size());
198     for(int i=0; i< expected.size(); i++){
199       assertEquals(expected.get(i), actual.get(i));
200       if(PRINT){
201         System.out.println("expected "+expected.get(i)+
202             ", actual " +actual.get(i));
203       }
204     }
205   }
206 
207 
208   /**
209    * Verify that {@link ScanQueryMatcher} only skips expired KeyValue
210    * instances and does not exit early from the row (skipping
211    * later non-expired KeyValues).  This version mimics a Get with
212    * explicitly specified column qualifiers.
213    *
214    * @throws IOException
215    */
216   public void testMatch_ExpiredExplicit()
217   throws IOException {
218 
219     long testTTL = 1000;
220     MatchCode [] expected = new MatchCode[] {
221         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
222         ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL,
223         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
224         ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL,
225         ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW,
226         ScanQueryMatcher.MatchCode.DONE
227     };
228 
229     long now = EnvironmentEdgeManager.currentTimeMillis();
230     ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
231         0, 1, testTTL, false, 0, rowComparator), get.getFamilyMap().get(fam2),
232         now - testTTL);
233 
234     KeyValue [] kvs = new KeyValue[] {
235         new KeyValue(row1, fam2, col1, now-100, data),
236         new KeyValue(row1, fam2, col2, now-50, data),
237         new KeyValue(row1, fam2, col3, now-5000, data),
238         new KeyValue(row1, fam2, col4, now-500, data),
239         new KeyValue(row1, fam2, col5, now-10000, data),
240         new KeyValue(row2, fam1, col1, now-10, data)
241     };
242 
243     KeyValue k = kvs[0];
244     qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
245 
246     List<MatchCode> actual = new ArrayList<MatchCode>(kvs.length);
247     for (KeyValue kv : kvs) {
248       actual.add( qm.match(kv) );
249     }
250 
251     assertEquals(expected.length, actual.size());
252     for (int i=0; i<expected.length; i++) {
253       if(PRINT){
254         System.out.println("expected "+expected[i]+
255             ", actual " +actual.get(i));
256       }
257       assertEquals(expected[i], actual.get(i));
258     }
259   }
260 
261 
262   /**
263    * Verify that {@link ScanQueryMatcher} only skips expired KeyValue
264    * instances and does not exit early from the row (skipping
265    * later non-expired KeyValues).  This version mimics a Get with
266    * wildcard-inferred column qualifiers.
267    *
268    * @throws IOException
269    */
270   public void testMatch_ExpiredWildcard()
271   throws IOException {
272 
273     long testTTL = 1000;
274     MatchCode [] expected = new MatchCode[] {
275         ScanQueryMatcher.MatchCode.INCLUDE,
276         ScanQueryMatcher.MatchCode.INCLUDE,
277         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
278         ScanQueryMatcher.MatchCode.INCLUDE,
279         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
280         ScanQueryMatcher.MatchCode.DONE
281     };
282 
283     long now = EnvironmentEdgeManager.currentTimeMillis();
284     ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
285         0, 1, testTTL, false, 0, rowComparator), null,
286         now - testTTL);
287 
288     KeyValue [] kvs = new KeyValue[] {
289         new KeyValue(row1, fam2, col1, now-100, data),
290         new KeyValue(row1, fam2, col2, now-50, data),
291         new KeyValue(row1, fam2, col3, now-5000, data),
292         new KeyValue(row1, fam2, col4, now-500, data),
293         new KeyValue(row1, fam2, col5, now-10000, data),
294         new KeyValue(row2, fam1, col1, now-10, data)
295     };
296     KeyValue k = kvs[0];
297     qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
298 
299     List<ScanQueryMatcher.MatchCode> actual =
300         new ArrayList<ScanQueryMatcher.MatchCode>(kvs.length);
301     for (KeyValue kv : kvs) {
302       actual.add( qm.match(kv) );
303     }
304 
305     assertEquals(expected.length, actual.size());
306     for (int i=0; i<expected.length; i++) {
307       if(PRINT){
308         System.out.println("expected "+expected[i]+
309             ", actual " +actual.get(i));
310       }
311       assertEquals(expected[i], actual.get(i));
312     }
313   }
314 
315   public void testMatch_PartialRangeDropDeletes() throws Exception {
316     // Some ranges.
317     testDropDeletes(
318         row2, row3, new byte[][] { row1, row2, row2, row3 }, INCLUDE, SKIP, SKIP, INCLUDE);
319     testDropDeletes(row2, row3, new byte[][] { row1, row1, row2 }, INCLUDE, INCLUDE, SKIP);
320     testDropDeletes(row2, row3, new byte[][] { row2, row3, row3 }, SKIP, INCLUDE, INCLUDE);
321     testDropDeletes(row1, row3, new byte[][] { row1, row2, row3 }, SKIP, SKIP, INCLUDE);
322     // Open ranges.
323     testDropDeletes(HConstants.EMPTY_START_ROW, row3,
324         new byte[][] { row1, row2, row3 }, SKIP, SKIP, INCLUDE);
325     testDropDeletes(row2, HConstants.EMPTY_END_ROW,
326         new byte[][] { row1, row2, row3 }, INCLUDE, SKIP, SKIP);
327     testDropDeletes(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
328         new byte[][] { row1, row2, row3, row3 }, SKIP, SKIP, SKIP, SKIP);
329 
330     // No KVs in range.
331     testDropDeletes(row2, row3, new byte[][] { row1, row1, row3 }, INCLUDE, INCLUDE, INCLUDE);
332     testDropDeletes(row2, row3, new byte[][] { row3, row3 }, INCLUDE, INCLUDE);
333     testDropDeletes(row2, row3, new byte[][] { row1, row1 }, INCLUDE, INCLUDE);
334   }
335 
336   private void testDropDeletes(
337       byte[] from, byte[] to, byte[][] rows, MatchCode... expected) throws IOException {
338     long now = EnvironmentEdgeManager.currentTimeMillis();
339     // Set time to purge deletes to negative value to avoid it ever happening.
340     ScanInfo scanInfo = new ScanInfo(fam2, 0, 1, ttl, false, -1L, rowComparator);
341     NavigableSet<byte[]> cols = get.getFamilyMap().get(fam2);
342 
343     ScanQueryMatcher qm = new ScanQueryMatcher(scan, scanInfo, cols, Long.MAX_VALUE,
344         HConstants.OLDEST_TIMESTAMP, HConstants.OLDEST_TIMESTAMP, from, to);
345     List<ScanQueryMatcher.MatchCode> actual =
346         new ArrayList<ScanQueryMatcher.MatchCode>(rows.length);
347     byte[] prevRow = null;
348     for (byte[] row : rows) {
349       if (prevRow == null || !Bytes.equals(prevRow, row)) {
350         qm.setRow(row, 0, (short)row.length);
351         prevRow = row;
352       }
353       actual.add(qm.match(new KeyValue(row, fam2, null, now, Type.Delete)));
354     }
355 
356     assertEquals(expected.length, actual.size());
357     for (int i = 0; i < expected.length; i++) {
358       if (PRINT) System.out.println("expected " + expected[i] + ", actual " + actual.get(i));
359       assertEquals(expected[i], actual.get(i));
360     }
361   }
362 }
363