View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.List;
25  import java.util.NavigableSet;
26  
27  import static org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode.*;
28  
29  import org.apache.hadoop.hbase.HBaseTestCase;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.KeyValue;
32  import org.apache.hadoop.hbase.KeyValue.KVComparator;
33  import org.apache.hadoop.hbase.KeyValue.Type;
34  import org.apache.hadoop.hbase.SmallTests;
35  import org.apache.hadoop.hbase.client.Get;
36  import org.apache.hadoop.hbase.client.Scan;
37  import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
40  import org.junit.experimental.categories.Category;
41  
42  @Category(SmallTests.class)
43  public class TestQueryMatcher extends HBaseTestCase {
44    private static final boolean PRINT = false;
45  
46    private byte[] row1;
47    private byte[] row2;
48    private byte[] row3;
49    private byte[] fam1;
50    private byte[] fam2;
51    private byte[] col1;
52    private byte[] col2;
53    private byte[] col3;
54    private byte[] col4;
55    private byte[] col5;
56  
57    private byte[] data;
58  
59    private Get get;
60  
61    long ttl = Long.MAX_VALUE;
62    KVComparator rowComparator;
63    private Scan scan;
64  
65    public void setUp() throws Exception {
66      super.setUp();
67      row1 = Bytes.toBytes("row1");
68      row2 = Bytes.toBytes("row2");
69      row3 = Bytes.toBytes("row3");
70      fam1 = Bytes.toBytes("fam1");
71      fam2 = Bytes.toBytes("fam2");
72      col1 = Bytes.toBytes("col1");
73      col2 = Bytes.toBytes("col2");
74      col3 = Bytes.toBytes("col3");
75      col4 = Bytes.toBytes("col4");
76      col5 = Bytes.toBytes("col5");
77  
78      data = Bytes.toBytes("data");
79  
80      //Create Get
81      get = new Get(row1);
82      get.addFamily(fam1);
83      get.addColumn(fam2, col2);
84      get.addColumn(fam2, col4);
85      get.addColumn(fam2, col5);
86      this.scan = new Scan(get);
87  
88      rowComparator = KeyValue.COMPARATOR;
89  
90    }
91  
92    public void testMatch_ExplicitColumns()
93    throws IOException {
94      //Moving up from the Tracker by using Gets and List<KeyValue> instead
95      //of just byte []
96  
97      //Expected result
98      List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
99      expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
100     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
101     expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
102     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
103     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
104     expected.add(ScanQueryMatcher.MatchCode.DONE);
105 
106     // 2,4,5
107     
108     ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
109         0, 1, ttl, false, 0, rowComparator), get.getFamilyMap().get(fam2),
110         EnvironmentEdgeManager.currentTimeMillis() - ttl);
111 
112     List<KeyValue> memstore = new ArrayList<KeyValue>();
113     memstore.add(new KeyValue(row1, fam2, col1, 1, data));
114     memstore.add(new KeyValue(row1, fam2, col2, 1, data));
115     memstore.add(new KeyValue(row1, fam2, col3, 1, data));
116     memstore.add(new KeyValue(row1, fam2, col4, 1, data));
117     memstore.add(new KeyValue(row1, fam2, col5, 1, data));
118 
119     memstore.add(new KeyValue(row2, fam1, col1, data));
120 
121     List<ScanQueryMatcher.MatchCode> actual = new ArrayList<ScanQueryMatcher.MatchCode>();
122     KeyValue k = memstore.get(0);
123     qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
124 
125     for (KeyValue kv : memstore){
126       actual.add(qm.match(kv));
127     }
128 
129     assertEquals(expected.size(), actual.size());
130     for(int i=0; i< expected.size(); i++){
131       assertEquals(expected.get(i), actual.get(i));
132       if(PRINT){
133         System.out.println("expected "+expected.get(i)+
134             ", actual " +actual.get(i));
135       }
136     }
137   }
138 
139 
140   public void testMatch_Wildcard()
141   throws IOException {
142     //Moving up from the Tracker by using Gets and List<KeyValue> instead
143     //of just byte []
144 
145     //Expected result
146     List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
147     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
148     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
149     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
150     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
151     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
152     expected.add(ScanQueryMatcher.MatchCode.DONE);
153 
154     ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
155         0, 1, ttl, false, 0, rowComparator), null,
156         EnvironmentEdgeManager.currentTimeMillis() - ttl);
157 
158     List<KeyValue> memstore = new ArrayList<KeyValue>();
159     memstore.add(new KeyValue(row1, fam2, col1, 1, data));
160     memstore.add(new KeyValue(row1, fam2, col2, 1, data));
161     memstore.add(new KeyValue(row1, fam2, col3, 1, data));
162     memstore.add(new KeyValue(row1, fam2, col4, 1, data));
163     memstore.add(new KeyValue(row1, fam2, col5, 1, data));
164     memstore.add(new KeyValue(row2, fam1, col1, 1, data));
165 
166     List<ScanQueryMatcher.MatchCode> actual = new ArrayList<ScanQueryMatcher.MatchCode>();
167 
168     KeyValue k = memstore.get(0);
169     qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
170 
171     for(KeyValue kv : memstore) {
172       actual.add(qm.match(kv));
173     }
174 
175     assertEquals(expected.size(), actual.size());
176     for(int i=0; i< expected.size(); i++){
177       assertEquals(expected.get(i), actual.get(i));
178       if(PRINT){
179         System.out.println("expected "+expected.get(i)+
180             ", actual " +actual.get(i));
181       }
182     }
183   }
184 
185 
186   /**
187    * Verify that {@link ScanQueryMatcher} only skips expired KeyValue
188    * instances and does not exit early from the row (skipping
189    * later non-expired KeyValues).  This version mimics a Get with
190    * explicitly specified column qualifiers.
191    *
192    * @throws IOException
193    */
194   public void testMatch_ExpiredExplicit()
195   throws IOException {
196 
197     long testTTL = 1000;
198     MatchCode [] expected = new MatchCode[] {
199         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
200         ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL,
201         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
202         ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL,
203         ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW,
204         ScanQueryMatcher.MatchCode.DONE
205     };
206 
207     long now = EnvironmentEdgeManager.currentTimeMillis();
208     ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
209         0, 1, testTTL, false, 0, rowComparator), get.getFamilyMap().get(fam2),
210         now - testTTL);
211 
212     KeyValue [] kvs = new KeyValue[] {
213         new KeyValue(row1, fam2, col1, now-100, data),
214         new KeyValue(row1, fam2, col2, now-50, data),
215         new KeyValue(row1, fam2, col3, now-5000, data),
216         new KeyValue(row1, fam2, col4, now-500, data),
217         new KeyValue(row1, fam2, col5, now-10000, data),
218         new KeyValue(row2, fam1, col1, now-10, data)
219     };
220 
221     KeyValue k = kvs[0];
222     qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
223 
224     List<MatchCode> actual = new ArrayList<MatchCode>(kvs.length);
225     for (KeyValue kv : kvs) {
226       actual.add( qm.match(kv) );
227     }
228 
229     assertEquals(expected.length, actual.size());
230     for (int i=0; i<expected.length; i++) {
231       if(PRINT){
232         System.out.println("expected "+expected[i]+
233             ", actual " +actual.get(i));
234       }
235       assertEquals(expected[i], actual.get(i));
236     }
237   }
238 
239 
240   /**
241    * Verify that {@link ScanQueryMatcher} only skips expired KeyValue
242    * instances and does not exit early from the row (skipping
243    * later non-expired KeyValues).  This version mimics a Get with
244    * wildcard-inferred column qualifiers.
245    *
246    * @throws IOException
247    */
248   public void testMatch_ExpiredWildcard()
249   throws IOException {
250 
251     long testTTL = 1000;
252     MatchCode [] expected = new MatchCode[] {
253         ScanQueryMatcher.MatchCode.INCLUDE,
254         ScanQueryMatcher.MatchCode.INCLUDE,
255         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
256         ScanQueryMatcher.MatchCode.INCLUDE,
257         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
258         ScanQueryMatcher.MatchCode.DONE
259     };
260 
261     long now = EnvironmentEdgeManager.currentTimeMillis();
262     ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
263         0, 1, testTTL, false, 0, rowComparator), null,
264         now - testTTL);
265 
266     KeyValue [] kvs = new KeyValue[] {
267         new KeyValue(row1, fam2, col1, now-100, data),
268         new KeyValue(row1, fam2, col2, now-50, data),
269         new KeyValue(row1, fam2, col3, now-5000, data),
270         new KeyValue(row1, fam2, col4, now-500, data),
271         new KeyValue(row1, fam2, col5, now-10000, data),
272         new KeyValue(row2, fam1, col1, now-10, data)
273     };
274     KeyValue k = kvs[0];
275     qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
276 
277     List<ScanQueryMatcher.MatchCode> actual =
278         new ArrayList<ScanQueryMatcher.MatchCode>(kvs.length);
279     for (KeyValue kv : kvs) {
280       actual.add( qm.match(kv) );
281     }
282 
283     assertEquals(expected.length, actual.size());
284     for (int i=0; i<expected.length; i++) {
285       if(PRINT){
286         System.out.println("expected "+expected[i]+
287             ", actual " +actual.get(i));
288       }
289       assertEquals(expected[i], actual.get(i));
290     }
291   }
292 
293   public void testMatch_PartialRangeDropDeletes() throws Exception {
294     // Some ranges.
295     testDropDeletes(
296         row2, row3, new byte[][] { row1, row2, row2, row3 }, INCLUDE, SKIP, SKIP, INCLUDE);
297     testDropDeletes(row2, row3, new byte[][] { row1, row1, row2 }, INCLUDE, INCLUDE, SKIP);
298     testDropDeletes(row2, row3, new byte[][] { row2, row3, row3 }, SKIP, INCLUDE, INCLUDE);
299     testDropDeletes(row1, row3, new byte[][] { row1, row2, row3 }, SKIP, SKIP, INCLUDE);
300     // Open ranges.
301     testDropDeletes(HConstants.EMPTY_START_ROW, row3,
302         new byte[][] { row1, row2, row3 }, SKIP, SKIP, INCLUDE);
303     testDropDeletes(row2, HConstants.EMPTY_END_ROW,
304         new byte[][] { row1, row2, row3 }, INCLUDE, SKIP, SKIP);
305     testDropDeletes(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
306         new byte[][] { row1, row2, row3, row3 }, SKIP, SKIP, SKIP, SKIP);
307 
308     // No KVs in range.
309     testDropDeletes(row2, row3, new byte[][] { row1, row1, row3 }, INCLUDE, INCLUDE, INCLUDE);
310     testDropDeletes(row2, row3, new byte[][] { row3, row3 }, INCLUDE, INCLUDE);
311     testDropDeletes(row2, row3, new byte[][] { row1, row1 }, INCLUDE, INCLUDE);
312   }
313 
314   private void testDropDeletes(
315       byte[] from, byte[] to, byte[][] rows, MatchCode... expected) throws IOException {
316     long now = EnvironmentEdgeManager.currentTimeMillis();
317     // Set time to purge deletes to negative value to avoid it ever happening.
318     ScanInfo scanInfo = new ScanInfo(fam2, 0, 1, ttl, false, -1L, rowComparator);
319     NavigableSet<byte[]> cols = get.getFamilyMap().get(fam2);
320 
321     ScanQueryMatcher qm = new ScanQueryMatcher(scan, scanInfo, cols, Long.MAX_VALUE,
322         HConstants.OLDEST_TIMESTAMP, HConstants.OLDEST_TIMESTAMP, from, to);
323     List<ScanQueryMatcher.MatchCode> actual =
324         new ArrayList<ScanQueryMatcher.MatchCode>(rows.length);
325     byte[] prevRow = null;
326     for (byte[] row : rows) {
327       if (prevRow == null || !Bytes.equals(prevRow, row)) {
328         qm.setRow(row, 0, (short)row.length);
329         prevRow = row;
330       }
331       actual.add(qm.match(new KeyValue(row, fam2, null, now, Type.Delete)));
332     }
333 
334     assertEquals(expected.length, actual.size());
335     for (int i = 0; i < expected.length; i++) {
336       if (PRINT) System.out.println("expected " + expected[i] + ", actual " + actual.get(i));
337       assertEquals(expected[i], actual.get(i));
338     }
339   }
340 }
341