1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.regionserver;
22
23 import junit.framework.TestCase;
24 import org.apache.hadoop.hbase.KeyValue;
25 import org.apache.hadoop.hbase.KeyValueTestUtil;
26 import org.apache.hadoop.hbase.client.Scan;
27 import org.apache.hadoop.hbase.util.Bytes;
28 import org.mockito.Mockito;
29 import org.mockito.stubbing.OngoingStubbing;
30
31 import com.google.common.collect.Lists;
32
33 import java.io.IOException;
34 import java.util.ArrayList;
35 import java.util.Arrays;
36 import java.util.Collections;
37 import java.util.List;
38 import java.util.NavigableSet;
39 import java.util.TreeSet;
40 import static org.apache.hadoop.hbase.regionserver.KeyValueScanFixture.scanFixture;
41
42 public class TestStoreScanner extends TestCase {
43 private static final String CF_STR = "cf";
44 final byte [] CF = Bytes.toBytes(CF_STR);
45
46
47
48
49
50
51 NavigableSet<byte[]> getCols(String ...strCols) {
52 NavigableSet<byte[]> cols = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
53 for (String col : strCols) {
54 byte[] bytes = Bytes.toBytes(col);
55 cols.add(bytes);
56 }
57 return cols;
58 }
59
60 public void testScanTimeRange() throws IOException {
61 String r1 = "R1";
62
63 KeyValue [] kvs = new KeyValue[] {
64 KeyValueTestUtil.create(r1, CF_STR, "a", 1, KeyValue.Type.Put, "dont-care"),
65 KeyValueTestUtil.create(r1, CF_STR, "a", 2, KeyValue.Type.Put, "dont-care"),
66 KeyValueTestUtil.create(r1, CF_STR, "a", 3, KeyValue.Type.Put, "dont-care"),
67 KeyValueTestUtil.create(r1, CF_STR, "a", 4, KeyValue.Type.Put, "dont-care"),
68 KeyValueTestUtil.create(r1, CF_STR, "a", 5, KeyValue.Type.Put, "dont-care"),
69 };
70 List<KeyValueScanner> scanners = Arrays.<KeyValueScanner>asList(
71 new KeyValueScanner[] {
72 new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
73 });
74 Scan scanSpec = new Scan(Bytes.toBytes(r1));
75 scanSpec.setTimeRange(0, 6);
76 scanSpec.setMaxVersions();
77 StoreScanner scan =
78 new StoreScanner(scanSpec, CF, Long.MAX_VALUE,
79 KeyValue.COMPARATOR, getCols("a"), scanners);
80 List<KeyValue> results = new ArrayList<KeyValue>();
81 assertEquals(true, scan.next(results));
82 assertEquals(5, results.size());
83 assertEquals(kvs[kvs.length - 1], results.get(0));
84
85 scanSpec = new Scan(Bytes.toBytes(r1));
86 scanSpec.setTimeRange(1, 3);
87 scanSpec.setMaxVersions();
88 scan = new StoreScanner(scanSpec, CF, Long.MAX_VALUE,
89 KeyValue.COMPARATOR, getCols("a"), scanners);
90 results = new ArrayList<KeyValue>();
91 assertEquals(true, scan.next(results));
92 assertEquals(2, results.size());
93
94 scanSpec = new Scan(Bytes.toBytes(r1));
95 scanSpec.setTimeRange(5, 10);
96 scanSpec.setMaxVersions();
97 scan = new StoreScanner(scanSpec, CF, Long.MAX_VALUE,
98 KeyValue.COMPARATOR, getCols("a"), scanners);
99 results = new ArrayList<KeyValue>();
100 assertEquals(true, scan.next(results));
101 assertEquals(1, results.size());
102
103
104 scanSpec = new Scan(Bytes.toBytes(r1));
105 scanSpec.setTimeRange(0, 10);
106 scanSpec.setMaxVersions(3);
107 scan = new StoreScanner(scanSpec, CF, Long.MAX_VALUE,
108 KeyValue.COMPARATOR, getCols("a"), scanners);
109 results = new ArrayList<KeyValue>();
110 assertEquals(true, scan.next(results));
111 assertEquals(3, results.size());
112 }
113
114 public void testScanSameTimestamp() throws IOException {
115
116 KeyValue [] kvs = new KeyValue[] {
117 KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
118 KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
119 };
120 List<KeyValueScanner> scanners = Arrays.asList(
121 new KeyValueScanner[] {
122 new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
123 });
124
125 Scan scanSpec = new Scan(Bytes.toBytes("R1"));
126
127 StoreScanner scan =
128 new StoreScanner(scanSpec, CF, Long.MAX_VALUE,
129 KeyValue.COMPARATOR, getCols("a"),
130 scanners);
131
132 List<KeyValue> results = new ArrayList<KeyValue>();
133 assertEquals(true, scan.next(results));
134 assertEquals(1, results.size());
135 assertEquals(kvs[0], results.get(0));
136 }
137
138
139
140
141
142
143
144
145 public void testWontNextToNext() throws IOException {
146
147 KeyValue [] kvs = new KeyValue[] {
148 KeyValueTestUtil.create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
149 KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
150 KeyValueTestUtil.create("R2", "cf", "a", 1, KeyValue.Type.Put, "dont-care")
151 };
152 List<KeyValueScanner> scanners = scanFixture(kvs);
153
154 Scan scanSpec = new Scan(Bytes.toBytes("R1"));
155
156 StoreScanner scan =
157 new StoreScanner(scanSpec, CF, Long.MAX_VALUE,
158 KeyValue.COMPARATOR, getCols("a"),
159 scanners);
160
161 List<KeyValue> results = new ArrayList<KeyValue>();
162 scan.next(results);
163 assertEquals(1, results.size());
164 assertEquals(kvs[0], results.get(0));
165
166
167 results.clear();
168 scan.next(results);
169 assertEquals(1, results.size());
170 assertEquals(kvs[2], results.get(0));
171
172 results.clear();
173 scan.next(results);
174 assertEquals(0, results.size());
175
176 }
177
178
179 public void testDeleteVersionSameTimestamp() throws IOException {
180 KeyValue [] kvs = new KeyValue [] {
181 KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
182 KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"),
183 };
184 List<KeyValueScanner> scanners = scanFixture(kvs);
185 Scan scanSpec = new Scan(Bytes.toBytes("R1"));
186 StoreScanner scan =
187 new StoreScanner(scanSpec, CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
188 getCols("a"), scanners);
189
190 List<KeyValue> results = new ArrayList<KeyValue>();
191 assertFalse(scan.next(results));
192 assertEquals(0, results.size());
193 }
194
195
196
197
198
199 public void testDeletedRowThenGoodRow() throws IOException {
200 KeyValue [] kvs = new KeyValue [] {
201 KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
202 KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"),
203 KeyValueTestUtil.create("R2", "cf", "a", 20, KeyValue.Type.Put, "dont-care")
204 };
205 List<KeyValueScanner> scanners = scanFixture(kvs);
206 Scan scanSpec = new Scan(Bytes.toBytes("R1"));
207 StoreScanner scan =
208 new StoreScanner(scanSpec, CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
209 getCols("a"), scanners);
210
211 List<KeyValue> results = new ArrayList<KeyValue>();
212 assertEquals(true, scan.next(results));
213 assertEquals(0, results.size());
214
215 assertEquals(true, scan.next(results));
216 assertEquals(1, results.size());
217 assertEquals(kvs[2], results.get(0));
218
219 assertEquals(false, scan.next(results));
220 }
221
222 public void testDeleteVersionMaskingMultiplePuts() throws IOException {
223 long now = System.currentTimeMillis();
224 KeyValue [] kvs1 = new KeyValue[] {
225 KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
226 KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care")
227 };
228 KeyValue [] kvs2 = new KeyValue[] {
229 KeyValueTestUtil.create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"),
230 KeyValueTestUtil.create("R1", "cf", "a", now-100, KeyValue.Type.Put, "dont-care"),
231 KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care")
232 };
233 List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2);
234
235 StoreScanner scan =
236 new StoreScanner(new Scan(Bytes.toBytes("R1")), CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
237 getCols("a"), scanners);
238 List<KeyValue> results = new ArrayList<KeyValue>();
239
240
241
242 assertEquals(true, scan.next(results));
243 assertEquals(1, results.size());
244 assertEquals(kvs2[1], results.get(0));
245 }
246 public void testDeleteVersionsMixedAndMultipleVersionReturn() throws IOException {
247 long now = System.currentTimeMillis();
248 KeyValue [] kvs1 = new KeyValue[] {
249 KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
250 KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care")
251 };
252 KeyValue [] kvs2 = new KeyValue[] {
253 KeyValueTestUtil.create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"),
254 KeyValueTestUtil.create("R1", "cf", "a", now+500, KeyValue.Type.Put, "dont-care"),
255 KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
256 KeyValueTestUtil.create("R2", "cf", "z", now, KeyValue.Type.Put, "dont-care")
257 };
258 List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2);
259
260 Scan scanSpec = new Scan(Bytes.toBytes("R1")).setMaxVersions(2);
261 StoreScanner scan =
262 new StoreScanner(scanSpec, CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
263 getCols("a"), scanners);
264 List<KeyValue> results = new ArrayList<KeyValue>();
265 assertEquals(true, scan.next(results));
266 assertEquals(2, results.size());
267 assertEquals(kvs2[1], results.get(0));
268 assertEquals(kvs2[0], results.get(1));
269 }
270
271 public void testWildCardOneVersionScan() throws IOException {
272 KeyValue [] kvs = new KeyValue [] {
273 KeyValueTestUtil.create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
274 KeyValueTestUtil.create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"),
275 KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"),
276 };
277 List<KeyValueScanner> scanners = scanFixture(kvs);
278 StoreScanner scan =
279 new StoreScanner(new Scan(Bytes.toBytes("R1")), CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
280 null, scanners);
281 List<KeyValue> results = new ArrayList<KeyValue>();
282 assertEquals(true, scan.next(results));
283 assertEquals(2, results.size());
284 assertEquals(kvs[0], results.get(0));
285 assertEquals(kvs[1], results.get(1));
286 }
287
288 public void testWildCardScannerUnderDeletes() throws IOException {
289 KeyValue [] kvs = new KeyValue [] {
290 KeyValueTestUtil.create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
291
292 KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"),
293
294 KeyValueTestUtil.create("R1", "cf", "b", 2, KeyValue.Type.Put, "dont-care"),
295 KeyValueTestUtil.create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"),
296
297 KeyValueTestUtil.create("R1", "cf", "c", 10, KeyValue.Type.Delete, "dont-care"),
298 KeyValueTestUtil.create("R1", "cf", "c", 10, KeyValue.Type.Put, "dont-care"),
299 KeyValueTestUtil.create("R1", "cf", "c", 9, KeyValue.Type.Put, "dont-care"),
300
301 KeyValueTestUtil.create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"),
302 KeyValueTestUtil.create("R1", "cf", "d", 10, KeyValue.Type.DeleteColumn, "dont-care"),
303 KeyValueTestUtil.create("R1", "cf", "d", 9, KeyValue.Type.Put, "dont-care"),
304 KeyValueTestUtil.create("R1", "cf", "d", 8, KeyValue.Type.Put, "dont-care"),
305
306 };
307 List<KeyValueScanner> scanners = scanFixture(kvs);
308 StoreScanner scan =
309 new StoreScanner(new Scan().setMaxVersions(2), CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
310 null, scanners);
311 List<KeyValue> results = new ArrayList<KeyValue>();
312 assertEquals(true, scan.next(results));
313 assertEquals(5, results.size());
314 assertEquals(kvs[0], results.get(0));
315 assertEquals(kvs[2], results.get(1));
316 assertEquals(kvs[3], results.get(2));
317 assertEquals(kvs[6], results.get(3));
318 assertEquals(kvs[7], results.get(4));
319 }
320
321 public void testDeleteFamily() throws IOException {
322 KeyValue [] kvs = new KeyValue[] {
323 KeyValueTestUtil.create("R1", "cf", "a", 100, KeyValue.Type.DeleteFamily, "dont-care"),
324 KeyValueTestUtil.create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"),
325 KeyValueTestUtil.create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
326 KeyValueTestUtil.create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"),
327 KeyValueTestUtil.create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"),
328 KeyValueTestUtil.create("R1", "cf", "e", 11, KeyValue.Type.DeleteColumn, "dont-care"),
329 KeyValueTestUtil.create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"),
330 KeyValueTestUtil.create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"),
331 KeyValueTestUtil.create("R1", "cf", "g", 11, KeyValue.Type.Delete, "dont-care"),
332 KeyValueTestUtil.create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"),
333 KeyValueTestUtil.create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"),
334 KeyValueTestUtil.create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
335 };
336 List<KeyValueScanner> scanners = scanFixture(kvs);
337 StoreScanner scan =
338 new StoreScanner(new Scan().setMaxVersions(Integer.MAX_VALUE), CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
339 null, scanners);
340 List<KeyValue> results = new ArrayList<KeyValue>();
341 assertEquals(true, scan.next(results));
342 assertEquals(0, results.size());
343 assertEquals(true, scan.next(results));
344 assertEquals(1, results.size());
345 assertEquals(kvs[kvs.length-1], results.get(0));
346
347 assertEquals(false, scan.next(results));
348 }
349
350 public void testDeleteColumn() throws IOException {
351 KeyValue [] kvs = new KeyValue[] {
352 KeyValueTestUtil.create("R1", "cf", "a", 10, KeyValue.Type.DeleteColumn, "dont-care"),
353 KeyValueTestUtil.create("R1", "cf", "a", 9, KeyValue.Type.Delete, "dont-care"),
354 KeyValueTestUtil.create("R1", "cf", "a", 8, KeyValue.Type.Put, "dont-care"),
355 KeyValueTestUtil.create("R1", "cf", "b", 5, KeyValue.Type.Put, "dont-care")
356 };
357 List<KeyValueScanner> scanners = scanFixture(kvs);
358 StoreScanner scan =
359 new StoreScanner(new Scan(), CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
360 null, scanners);
361 List<KeyValue> results = new ArrayList<KeyValue>();
362 assertEquals(true, scan.next(results));
363 assertEquals(1, results.size());
364 assertEquals(kvs[3], results.get(0));
365 }
366
367 private static final KeyValue [] kvs = new KeyValue[] {
368 KeyValueTestUtil.create("R1", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
369 KeyValueTestUtil.create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"),
370 KeyValueTestUtil.create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
371 KeyValueTestUtil.create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"),
372 KeyValueTestUtil.create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"),
373 KeyValueTestUtil.create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"),
374 KeyValueTestUtil.create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"),
375 KeyValueTestUtil.create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"),
376 KeyValueTestUtil.create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"),
377 KeyValueTestUtil.create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
378 };
379
380 public void testSkipColumn() throws IOException {
381 List<KeyValueScanner> scanners = scanFixture(kvs);
382 StoreScanner scan =
383 new StoreScanner(new Scan(), CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
384 getCols("a", "d"), scanners);
385
386 List<KeyValue> results = new ArrayList<KeyValue>();
387 assertEquals(true, scan.next(results));
388 assertEquals(2, results.size());
389 assertEquals(kvs[0], results.get(0));
390 assertEquals(kvs[3], results.get(1));
391 results.clear();
392
393 assertEquals(true, scan.next(results));
394 assertEquals(1, results.size());
395 assertEquals(kvs[kvs.length-1], results.get(0));
396
397 results.clear();
398 assertEquals(false, scan.next(results));
399 }
400
401
402
403
404
405 public void testWildCardTtlScan() throws IOException {
406 long now = System.currentTimeMillis();
407 KeyValue [] kvs = new KeyValue[] {
408 KeyValueTestUtil.create("R1", "cf", "a", now-1000, KeyValue.Type.Put, "dont-care"),
409 KeyValueTestUtil.create("R1", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
410 KeyValueTestUtil.create("R1", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
411 KeyValueTestUtil.create("R1", "cf", "d", now-10000, KeyValue.Type.Put, "dont-care"),
412 KeyValueTestUtil.create("R2", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
413 KeyValueTestUtil.create("R2", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
414 KeyValueTestUtil.create("R2", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
415 KeyValueTestUtil.create("R2", "cf", "c", now-1000, KeyValue.Type.Put, "dont-care")
416 };
417 List<KeyValueScanner> scanners = scanFixture(kvs);
418 Scan scan = new Scan();
419 scan.setMaxVersions(1);
420 StoreScanner scanner =
421 new StoreScanner(scan, CF, 500, KeyValue.COMPARATOR,
422 null, scanners);
423
424 List<KeyValue> results = new ArrayList<KeyValue>();
425 assertEquals(true, scanner.next(results));
426 assertEquals(2, results.size());
427 assertEquals(kvs[1], results.get(0));
428 assertEquals(kvs[2], results.get(1));
429 results.clear();
430
431 assertEquals(true, scanner.next(results));
432 assertEquals(3, results.size());
433 assertEquals(kvs[4], results.get(0));
434 assertEquals(kvs[5], results.get(1));
435 assertEquals(kvs[6], results.get(2));
436 results.clear();
437
438 assertEquals(false, scanner.next(results));
439 }
440
441 public void testScannerReseekDoesntNPE() throws Exception {
442 List<KeyValueScanner> scanners = scanFixture(kvs);
443 StoreScanner scan =
444 new StoreScanner(new Scan(), CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
445 getCols("a", "d"), scanners);
446
447
448
449
450
451
452 scan.updateReaders();
453
454 scan.updateReaders();
455
456 scan.peek();
457 }
458
459
460
461
462
463 public void SKIP_testPeek() throws Exception {
464 KeyValue [] kvs = new KeyValue [] {
465 KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
466 KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"),
467 };
468 List<KeyValueScanner> scanners = scanFixture(kvs);
469 Scan scanSpec = new Scan(Bytes.toBytes("R1"));
470 StoreScanner scan =
471 new StoreScanner(scanSpec, CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
472 getCols("a"), scanners);
473 assertNull(scan.peek());
474 }
475 }