1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.List;
25 import java.util.NavigableSet;
26
27 import static org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode.*;
28
29 import org.apache.hadoop.hbase.HBaseTestCase;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.KeyValue;
32 import org.apache.hadoop.hbase.KeyValue.KVComparator;
33 import org.apache.hadoop.hbase.KeyValue.Type;
34 import org.apache.hadoop.hbase.SmallTests;
35 import org.apache.hadoop.hbase.client.Get;
36 import org.apache.hadoop.hbase.client.Scan;
37 import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
40 import org.junit.experimental.categories.Category;
41
42 @Category(SmallTests.class)
43 public class TestQueryMatcher extends HBaseTestCase {
44 private static final boolean PRINT = false;
45
46 private byte[] row1;
47 private byte[] row2;
48 private byte[] row3;
49 private byte[] fam1;
50 private byte[] fam2;
51 private byte[] col1;
52 private byte[] col2;
53 private byte[] col3;
54 private byte[] col4;
55 private byte[] col5;
56
57 private byte[] data;
58
59 private Get get;
60
61 long ttl = Long.MAX_VALUE;
62 KVComparator rowComparator;
63 private Scan scan;
64
65 public void setUp() throws Exception {
66 super.setUp();
67 row1 = Bytes.toBytes("row1");
68 row2 = Bytes.toBytes("row2");
69 row3 = Bytes.toBytes("row3");
70 fam1 = Bytes.toBytes("fam1");
71 fam2 = Bytes.toBytes("fam2");
72 col1 = Bytes.toBytes("col1");
73 col2 = Bytes.toBytes("col2");
74 col3 = Bytes.toBytes("col3");
75 col4 = Bytes.toBytes("col4");
76 col5 = Bytes.toBytes("col5");
77
78 data = Bytes.toBytes("data");
79
80
81 get = new Get(row1);
82 get.addFamily(fam1);
83 get.addColumn(fam2, col2);
84 get.addColumn(fam2, col4);
85 get.addColumn(fam2, col5);
86 this.scan = new Scan(get);
87
88 rowComparator = KeyValue.COMPARATOR;
89
90 }
91
92 private void _testMatch_ExplicitColumns(Scan scan, List<MatchCode> expected) throws IOException {
93
94 ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
95 0, 1, ttl, false, 0, rowComparator), get.getFamilyMap().get(fam2),
96 EnvironmentEdgeManager.currentTimeMillis() - ttl);
97
98 List<KeyValue> memstore = new ArrayList<KeyValue>();
99 memstore.add(new KeyValue(row1, fam2, col1, 1, data));
100 memstore.add(new KeyValue(row1, fam2, col2, 1, data));
101 memstore.add(new KeyValue(row1, fam2, col3, 1, data));
102 memstore.add(new KeyValue(row1, fam2, col4, 1, data));
103 memstore.add(new KeyValue(row1, fam2, col5, 1, data));
104
105 memstore.add(new KeyValue(row2, fam1, col1, data));
106
107 List<ScanQueryMatcher.MatchCode> actual = new ArrayList<ScanQueryMatcher.MatchCode>();
108 KeyValue k = memstore.get(0);
109 qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
110
111 for (KeyValue kv : memstore){
112 actual.add(qm.match(kv));
113 }
114
115 assertEquals(expected.size(), actual.size());
116 for(int i=0; i< expected.size(); i++){
117 assertEquals(expected.get(i), actual.get(i));
118 if(PRINT){
119 System.out.println("expected "+expected.get(i)+
120 ", actual " +actual.get(i));
121 }
122 }
123 }
124
125 public void testMatch_ExplicitColumns()
126 throws IOException {
127
128
129
130
131 List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
132 expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
133 expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
134 expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
135 expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
136 expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
137 expected.add(ScanQueryMatcher.MatchCode.DONE);
138
139 _testMatch_ExplicitColumns(scan, expected);
140 }
141
142 public void testMatch_ExplicitColumnsWithLookAhead()
143 throws IOException {
144
145
146
147
148 List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
149 expected.add(ScanQueryMatcher.MatchCode.SKIP);
150 expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
151 expected.add(ScanQueryMatcher.MatchCode.SKIP);
152 expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
153 expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
154 expected.add(ScanQueryMatcher.MatchCode.DONE);
155
156 Scan s = new Scan(scan);
157 s.setAttribute(Scan.HINT_LOOKAHEAD, Bytes.toBytes(2));
158 _testMatch_ExplicitColumns(s, expected);
159 }
160
161
162 public void testMatch_Wildcard()
163 throws IOException {
164
165
166
167
168 List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
169 expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
170 expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
171 expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
172 expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
173 expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
174 expected.add(ScanQueryMatcher.MatchCode.DONE);
175
176 ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
177 0, 1, ttl, false, 0, rowComparator), null,
178 EnvironmentEdgeManager.currentTimeMillis() - ttl);
179
180 List<KeyValue> memstore = new ArrayList<KeyValue>();
181 memstore.add(new KeyValue(row1, fam2, col1, 1, data));
182 memstore.add(new KeyValue(row1, fam2, col2, 1, data));
183 memstore.add(new KeyValue(row1, fam2, col3, 1, data));
184 memstore.add(new KeyValue(row1, fam2, col4, 1, data));
185 memstore.add(new KeyValue(row1, fam2, col5, 1, data));
186 memstore.add(new KeyValue(row2, fam1, col1, 1, data));
187
188 List<ScanQueryMatcher.MatchCode> actual = new ArrayList<ScanQueryMatcher.MatchCode>();
189
190 KeyValue k = memstore.get(0);
191 qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
192
193 for(KeyValue kv : memstore) {
194 actual.add(qm.match(kv));
195 }
196
197 assertEquals(expected.size(), actual.size());
198 for(int i=0; i< expected.size(); i++){
199 assertEquals(expected.get(i), actual.get(i));
200 if(PRINT){
201 System.out.println("expected "+expected.get(i)+
202 ", actual " +actual.get(i));
203 }
204 }
205 }
206
207
208
209
210
211
212
213
214
215
216 public void testMatch_ExpiredExplicit()
217 throws IOException {
218
219 long testTTL = 1000;
220 MatchCode [] expected = new MatchCode[] {
221 ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
222 ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL,
223 ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
224 ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL,
225 ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW,
226 ScanQueryMatcher.MatchCode.DONE
227 };
228
229 long now = EnvironmentEdgeManager.currentTimeMillis();
230 ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
231 0, 1, testTTL, false, 0, rowComparator), get.getFamilyMap().get(fam2),
232 now - testTTL);
233
234 KeyValue [] kvs = new KeyValue[] {
235 new KeyValue(row1, fam2, col1, now-100, data),
236 new KeyValue(row1, fam2, col2, now-50, data),
237 new KeyValue(row1, fam2, col3, now-5000, data),
238 new KeyValue(row1, fam2, col4, now-500, data),
239 new KeyValue(row1, fam2, col5, now-10000, data),
240 new KeyValue(row2, fam1, col1, now-10, data)
241 };
242
243 KeyValue k = kvs[0];
244 qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
245
246 List<MatchCode> actual = new ArrayList<MatchCode>(kvs.length);
247 for (KeyValue kv : kvs) {
248 actual.add( qm.match(kv) );
249 }
250
251 assertEquals(expected.length, actual.size());
252 for (int i=0; i<expected.length; i++) {
253 if(PRINT){
254 System.out.println("expected "+expected[i]+
255 ", actual " +actual.get(i));
256 }
257 assertEquals(expected[i], actual.get(i));
258 }
259 }
260
261
262
263
264
265
266
267
268
269
270 public void testMatch_ExpiredWildcard()
271 throws IOException {
272
273 long testTTL = 1000;
274 MatchCode [] expected = new MatchCode[] {
275 ScanQueryMatcher.MatchCode.INCLUDE,
276 ScanQueryMatcher.MatchCode.INCLUDE,
277 ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
278 ScanQueryMatcher.MatchCode.INCLUDE,
279 ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
280 ScanQueryMatcher.MatchCode.DONE
281 };
282
283 long now = EnvironmentEdgeManager.currentTimeMillis();
284 ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
285 0, 1, testTTL, false, 0, rowComparator), null,
286 now - testTTL);
287
288 KeyValue [] kvs = new KeyValue[] {
289 new KeyValue(row1, fam2, col1, now-100, data),
290 new KeyValue(row1, fam2, col2, now-50, data),
291 new KeyValue(row1, fam2, col3, now-5000, data),
292 new KeyValue(row1, fam2, col4, now-500, data),
293 new KeyValue(row1, fam2, col5, now-10000, data),
294 new KeyValue(row2, fam1, col1, now-10, data)
295 };
296 KeyValue k = kvs[0];
297 qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
298
299 List<ScanQueryMatcher.MatchCode> actual =
300 new ArrayList<ScanQueryMatcher.MatchCode>(kvs.length);
301 for (KeyValue kv : kvs) {
302 actual.add( qm.match(kv) );
303 }
304
305 assertEquals(expected.length, actual.size());
306 for (int i=0; i<expected.length; i++) {
307 if(PRINT){
308 System.out.println("expected "+expected[i]+
309 ", actual " +actual.get(i));
310 }
311 assertEquals(expected[i], actual.get(i));
312 }
313 }
314
315 public void testMatch_PartialRangeDropDeletes() throws Exception {
316 long now = EnvironmentEdgeManager.currentTimeMillis();
317 ScanInfo scanInfo = new ScanInfo(fam2, 0, 1, ttl, false, 0, rowComparator);
318 NavigableSet<byte[]> cols = get.getFamilyMap().get(fam2);
319
320
321 testDropDeletes(
322 row2, row3, new byte[][] { row1, row2, row2, row3 }, INCLUDE, SKIP, SKIP, INCLUDE);
323 testDropDeletes(row2, row3, new byte[][] { row1, row1, row2 }, INCLUDE, INCLUDE, SKIP);
324 testDropDeletes(row2, row3, new byte[][] { row2, row3, row3 }, SKIP, INCLUDE, INCLUDE);
325 testDropDeletes(row1, row3, new byte[][] { row1, row2, row3 }, SKIP, SKIP, INCLUDE);
326
327 testDropDeletes(HConstants.EMPTY_START_ROW, row3,
328 new byte[][] { row1, row2, row3 }, SKIP, SKIP, INCLUDE);
329 testDropDeletes(row2, HConstants.EMPTY_END_ROW,
330 new byte[][] { row1, row2, row3 }, INCLUDE, SKIP, SKIP);
331 testDropDeletes(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
332 new byte[][] { row1, row2, row3, row3 }, SKIP, SKIP, SKIP, SKIP);
333
334
335 testDropDeletes(row2, row3, new byte[][] { row1, row1, row3 }, INCLUDE, INCLUDE, INCLUDE);
336 testDropDeletes(row2, row3, new byte[][] { row3, row3 }, INCLUDE, INCLUDE);
337 testDropDeletes(row2, row3, new byte[][] { row1, row1 }, INCLUDE, INCLUDE);
338 }
339
340 private void testDropDeletes(
341 byte[] from, byte[] to, byte[][] rows, MatchCode... expected) throws IOException {
342 long now = EnvironmentEdgeManager.currentTimeMillis();
343
344 ScanInfo scanInfo = new ScanInfo(fam2, 0, 1, ttl, false, -1L, rowComparator);
345 NavigableSet<byte[]> cols = get.getFamilyMap().get(fam2);
346
347 ScanQueryMatcher qm = new ScanQueryMatcher(scan, scanInfo, cols, Long.MAX_VALUE,
348 HConstants.OLDEST_TIMESTAMP, HConstants.OLDEST_TIMESTAMP, from, to);
349 List<ScanQueryMatcher.MatchCode> actual =
350 new ArrayList<ScanQueryMatcher.MatchCode>(rows.length);
351 byte[] prevRow = null;
352 for (byte[] row : rows) {
353 if (prevRow == null || !Bytes.equals(prevRow, row)) {
354 qm.setRow(row, 0, (short)row.length);
355 prevRow = row;
356 }
357 actual.add(qm.match(new KeyValue(row, fam2, null, now, Type.Delete)));
358 }
359
360 assertEquals(expected.length, actual.size());
361 for (int i = 0; i < expected.length; i++) {
362 if (PRINT) System.out.println("expected " + expected[i] + ", actual " + actual.get(i));
363 assertEquals(expected[i], actual.get(i));
364 }
365 }
366 }
367