1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.List;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.hbase.HBaseTestCase;
28 import org.apache.hadoop.hbase.HBaseTestingUtility;
29 import org.apache.hadoop.hbase.HColumnDescriptor;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.HRegionInfo;
32 import org.apache.hadoop.hbase.HTableDescriptor;
33 import org.apache.hadoop.hbase.KeyValue;
34 import org.apache.hadoop.hbase.SmallTests;
35 import org.apache.hadoop.hbase.TableName;
36 import org.apache.hadoop.hbase.UnknownScannerException;
37 import org.apache.hadoop.hbase.client.Delete;
38 import org.apache.hadoop.hbase.client.Get;
39 import org.apache.hadoop.hbase.client.Put;
40 import org.apache.hadoop.hbase.client.Result;
41 import org.apache.hadoop.hbase.client.Scan;
42 import org.apache.hadoop.hbase.filter.Filter;
43 import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
44 import org.apache.hadoop.hbase.filter.PrefixFilter;
45 import org.apache.hadoop.hbase.filter.WhileMatchFilter;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.junit.experimental.categories.Category;
48
49
50
51
52 @Category(SmallTests.class)
53 public class TestScanner extends HBaseTestCase {
54 private final Log LOG = LogFactory.getLog(this.getClass());
55
56 private static final byte [] FIRST_ROW = HConstants.EMPTY_START_ROW;
57 private static final byte [][] COLS = { HConstants.CATALOG_FAMILY };
58 private static final byte [][] EXPLICIT_COLS = {
59 HConstants.REGIONINFO_QUALIFIER, HConstants.SERVER_QUALIFIER,
60
61
62 };
63
64 static final HTableDescriptor TESTTABLEDESC =
65 new HTableDescriptor(TableName.valueOf("testscanner"));
66 static {
67 TESTTABLEDESC.addFamily(
68 new HColumnDescriptor(HConstants.CATALOG_FAMILY)
69
70 .setMaxVersions(10)
71 .setBlockCacheEnabled(false)
72 .setBlocksize(8 * 1024)
73 );
74 }
75
76 public static final HRegionInfo REGION_INFO =
77 new HRegionInfo(TESTTABLEDESC.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
78 HConstants.EMPTY_BYTE_ARRAY);
79
80 private static final byte [] ROW_KEY = REGION_INFO.getRegionName();
81
82 private static final long START_CODE = Long.MAX_VALUE;
83
84 private HRegion r;
85 private HRegionIncommon region;
86
87 private byte[] firstRowBytes, secondRowBytes, thirdRowBytes;
88 final private byte[] col1, col2;
89
90 public TestScanner() {
91 super();
92
93 firstRowBytes = START_KEY_BYTES;
94 secondRowBytes = START_KEY_BYTES.clone();
95
96 secondRowBytes[START_KEY_BYTES.length - 1]++;
97 thirdRowBytes = START_KEY_BYTES.clone();
98 thirdRowBytes[START_KEY_BYTES.length - 1] += 2;
99 col1 = Bytes.toBytes("column1");
100 col2 = Bytes.toBytes("column2");
101 }
102
103
104
105
106
107 public void testStopRow() throws Exception {
108 byte [] startrow = Bytes.toBytes("bbb");
109 byte [] stoprow = Bytes.toBytes("ccc");
110 try {
111 this.r = createNewHRegion(TESTTABLEDESC, null, null);
112 addContent(this.r, HConstants.CATALOG_FAMILY);
113 List<KeyValue> results = new ArrayList<KeyValue>();
114
115 Scan scan = new Scan(Bytes.toBytes("abc"), Bytes.toBytes("abd"));
116 scan.addFamily(HConstants.CATALOG_FAMILY);
117
118 InternalScanner s = r.getScanner(scan);
119 int count = 0;
120 while (s.next(results)) {
121 count++;
122 }
123 s.close();
124 assertEquals(0, count);
125
126 scan = new Scan(startrow, stoprow);
127 scan.addFamily(HConstants.CATALOG_FAMILY);
128
129 s = r.getScanner(scan);
130 count = 0;
131 KeyValue kv = null;
132 results = new ArrayList<KeyValue>();
133 for (boolean first = true; s.next(results);) {
134 kv = results.get(0);
135 if (first) {
136 assertTrue(Bytes.BYTES_COMPARATOR.compare(startrow, kv.getRow()) == 0);
137 first = false;
138 }
139 count++;
140 }
141 assertTrue(Bytes.BYTES_COMPARATOR.compare(stoprow, kv.getRow()) > 0);
142
143 assertTrue(count > 10);
144 s.close();
145 } finally {
146 HRegion.closeHRegion(this.r);
147 }
148 }
149
150 void rowPrefixFilter(Scan scan) throws IOException {
151 List<KeyValue> results = new ArrayList<KeyValue>();
152 scan.addFamily(HConstants.CATALOG_FAMILY);
153 InternalScanner s = r.getScanner(scan);
154 boolean hasMore = true;
155 while (hasMore) {
156 hasMore = s.next(results);
157 for (KeyValue kv : results) {
158 assertEquals((byte)'a', kv.getRow()[0]);
159 assertEquals((byte)'b', kv.getRow()[1]);
160 }
161 results.clear();
162 }
163 s.close();
164 }
165
166 void rowInclusiveStopFilter(Scan scan, byte[] stopRow) throws IOException {
167 List<KeyValue> results = new ArrayList<KeyValue>();
168 scan.addFamily(HConstants.CATALOG_FAMILY);
169 InternalScanner s = r.getScanner(scan);
170 boolean hasMore = true;
171 while (hasMore) {
172 hasMore = s.next(results);
173 for (KeyValue kv : results) {
174 assertTrue(Bytes.compareTo(kv.getRow(), stopRow) <= 0);
175 }
176 results.clear();
177 }
178 s.close();
179 }
180
181 public void testFilters() throws IOException {
182 try {
183 this.r = createNewHRegion(TESTTABLEDESC, null, null);
184 addContent(this.r, HConstants.CATALOG_FAMILY);
185 byte [] prefix = Bytes.toBytes("ab");
186 Filter newFilter = new PrefixFilter(prefix);
187 Scan scan = new Scan();
188 scan.setFilter(newFilter);
189 rowPrefixFilter(scan);
190
191 byte[] stopRow = Bytes.toBytes("bbc");
192 newFilter = new WhileMatchFilter(new InclusiveStopFilter(stopRow));
193 scan = new Scan();
194 scan.setFilter(newFilter);
195 rowInclusiveStopFilter(scan, stopRow);
196
197 } finally {
198 HRegion.closeHRegion(this.r);
199 }
200 }
201
202
203
204
205
206
207 public void testRaceBetweenClientAndTimeout() throws Exception {
208 try {
209 this.r = createNewHRegion(TESTTABLEDESC, null, null);
210 addContent(this.r, HConstants.CATALOG_FAMILY);
211 Scan scan = new Scan();
212 InternalScanner s = r.getScanner(scan);
213 List<KeyValue> results = new ArrayList<KeyValue>();
214 try {
215 s.next(results);
216 s.close();
217 s.next(results);
218 fail("We don't want anything more, we should be failing");
219 } catch (UnknownScannerException ex) {
220
221 return;
222 }
223 } finally {
224 HRegion.closeHRegion(this.r);
225 }
226 }
227
228
229
230
231 public void testScanner() throws IOException {
232 try {
233 r = createNewHRegion(TESTTABLEDESC, null, null);
234 region = new HRegionIncommon(r);
235
236
237
238 Put put = new Put(ROW_KEY, System.currentTimeMillis());
239
240 put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
241 REGION_INFO.toByteArray());
242 region.put(put);
243
244
245
246
247 scan(false, null);
248 getRegionInfo();
249
250
251
252 r.close();
253 r = openClosedRegion(r);
254 region = new HRegionIncommon(r);
255
256
257
258 scan(false, null);
259 getRegionInfo();
260
261
262
263 String address = HConstants.LOCALHOST_IP + ":" + HBaseTestingUtility.randomFreePort();
264
265 put = new Put(ROW_KEY, System.currentTimeMillis());
266 put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
267 Bytes.toBytes(address));
268
269
270
271 region.put(put);
272
273
274
275
276 scan(true, address.toString());
277 getRegionInfo();
278
279
280
281 region.flushcache();
282
283
284
285 scan(true, address.toString());
286 getRegionInfo();
287
288
289
290 r.close();
291 r = openClosedRegion(r);
292 region = new HRegionIncommon(r);
293
294
295
296 scan(true, address.toString());
297 getRegionInfo();
298
299
300
301 address = "bar.foo.com:4321";
302
303 put = new Put(ROW_KEY, System.currentTimeMillis());
304
305 put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
306 Bytes.toBytes(address));
307 region.put(put);
308
309
310
311 scan(true, address.toString());
312 getRegionInfo();
313
314
315
316 region.flushcache();
317
318
319
320 scan(true, address.toString());
321 getRegionInfo();
322
323
324
325 r.close();
326 r = openClosedRegion(r);
327 region = new HRegionIncommon(r);
328
329
330
331 scan(true, address.toString());
332 getRegionInfo();
333
334 } finally {
335
336 HRegion.closeHRegion(r);
337 }
338 }
339
340
341 private void validateRegionInfo(byte [] regionBytes) throws IOException {
342 HRegionInfo info = HRegionInfo.parseFromOrNull(regionBytes);
343
344 assertEquals(REGION_INFO.getRegionId(), info.getRegionId());
345 assertEquals(0, info.getStartKey().length);
346 assertEquals(0, info.getEndKey().length);
347 assertEquals(0, Bytes.compareTo(info.getRegionName(), REGION_INFO.getRegionName()));
348
349 }
350
351
352 private void scan(boolean validateStartcode, String serverName)
353 throws IOException {
354 InternalScanner scanner = null;
355 Scan scan = null;
356 List<KeyValue> results = new ArrayList<KeyValue>();
357 byte [][][] scanColumns = {
358 COLS,
359 EXPLICIT_COLS
360 };
361
362 for(int i = 0; i < scanColumns.length; i++) {
363 try {
364 scan = new Scan(FIRST_ROW);
365 for (int ii = 0; ii < EXPLICIT_COLS.length; ii++) {
366 scan.addColumn(COLS[0], EXPLICIT_COLS[ii]);
367 }
368 scanner = r.getScanner(scan);
369 while (scanner.next(results)) {
370 assertTrue(hasColumn(results, HConstants.CATALOG_FAMILY,
371 HConstants.REGIONINFO_QUALIFIER));
372 byte [] val = getColumn(results, HConstants.CATALOG_FAMILY,
373 HConstants.REGIONINFO_QUALIFIER).getValue();
374 validateRegionInfo(val);
375 if(validateStartcode) {
376
377
378
379
380 assertNotNull(val);
381 assertFalse(val.length == 0);
382 long startCode = Bytes.toLong(val);
383 assertEquals(START_CODE, startCode);
384 }
385
386 if(serverName != null) {
387 assertTrue(hasColumn(results, HConstants.CATALOG_FAMILY,
388 HConstants.SERVER_QUALIFIER));
389 val = getColumn(results, HConstants.CATALOG_FAMILY,
390 HConstants.SERVER_QUALIFIER).getValue();
391 assertNotNull(val);
392 assertFalse(val.length == 0);
393 String server = Bytes.toString(val);
394 assertEquals(0, server.compareTo(serverName));
395 }
396 }
397 } finally {
398 InternalScanner s = scanner;
399 scanner = null;
400 if(s != null) {
401 s.close();
402 }
403 }
404 }
405 }
406
407 private boolean hasColumn(final List<KeyValue> kvs, final byte [] family,
408 final byte [] qualifier) {
409 for (KeyValue kv: kvs) {
410 if (kv.matchingFamily(family) && kv.matchingQualifier(qualifier)) {
411 return true;
412 }
413 }
414 return false;
415 }
416
417 private KeyValue getColumn(final List<KeyValue> kvs, final byte [] family,
418 final byte [] qualifier) {
419 for (KeyValue kv: kvs) {
420 if (kv.matchingFamily(family) && kv.matchingQualifier(qualifier)) {
421 return kv;
422 }
423 }
424 return null;
425 }
426
427
428
429 private void getRegionInfo() throws IOException {
430 Get get = new Get(ROW_KEY);
431 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
432 Result result = region.get(get);
433 byte [] bytes = result.value();
434 validateRegionInfo(bytes);
435 }
436
437
438
439
440
441
442
443 public void testScanAndSyncFlush() throws Exception {
444 this.r = createNewHRegion(TESTTABLEDESC, null, null);
445 HRegionIncommon hri = new HRegionIncommon(r);
446 try {
447 LOG.info("Added: " + addContent(hri, Bytes.toString(HConstants.CATALOG_FAMILY),
448 Bytes.toString(HConstants.REGIONINFO_QUALIFIER)));
449 int count = count(hri, -1, false);
450 assertEquals(count, count(hri, 100, false));
451 } catch (Exception e) {
452 LOG.error("Failed", e);
453 throw e;
454 } finally {
455 HRegion.closeHRegion(this.r);
456 }
457 }
458
459
460
461
462
463
464
465 public void testScanAndRealConcurrentFlush() throws Exception {
466 this.r = createNewHRegion(TESTTABLEDESC, null, null);
467 HRegionIncommon hri = new HRegionIncommon(r);
468 try {
469 LOG.info("Added: " + addContent(hri, Bytes.toString(HConstants.CATALOG_FAMILY),
470 Bytes.toString(HConstants.REGIONINFO_QUALIFIER)));
471 int count = count(hri, -1, false);
472 assertEquals(count, count(hri, 100, true));
473 } catch (Exception e) {
474 LOG.error("Failed", e);
475 throw e;
476 } finally {
477 HRegion.closeHRegion(this.r);
478 }
479 }
480
481
482
483
484
485
486
487 @SuppressWarnings("deprecation")
488 public void testScanAndConcurrentMajorCompact() throws Exception {
489 HTableDescriptor htd = createTableDescriptor(getName());
490 this.r = createNewHRegion(htd, null, null);
491 HRegionIncommon hri = new HRegionIncommon(r);
492
493 try {
494 addContent(hri, Bytes.toString(fam1), Bytes.toString(col1),
495 firstRowBytes, secondRowBytes);
496 addContent(hri, Bytes.toString(fam2), Bytes.toString(col1),
497 firstRowBytes, secondRowBytes);
498
499 Delete dc = new Delete(firstRowBytes);
500
501 dc.deleteColumns(fam1, col1);
502 r.delete(dc);
503 r.flushcache();
504
505 addContent(hri, Bytes.toString(fam1), Bytes.toString(col1),
506 secondRowBytes, thirdRowBytes);
507 addContent(hri, Bytes.toString(fam2), Bytes.toString(col1),
508 secondRowBytes, thirdRowBytes);
509 r.flushcache();
510
511 InternalScanner s = r.getScanner(new Scan());
512
513 r.compactStores(true);
514
515 List<KeyValue> results = new ArrayList<KeyValue>();
516 s.next(results);
517
518
519 assertTrue("result is not correct, keyValues : " + results,
520 results.size() == 1);
521 assertTrue(Bytes.BYTES_COMPARATOR.compare(firstRowBytes, results.get(0)
522 .getRow()) == 0);
523 assertTrue(Bytes.BYTES_COMPARATOR.compare(fam2, results.get(0)
524 .getFamily()) == 0);
525
526 results = new ArrayList<KeyValue>();
527 s.next(results);
528
529
530 assertTrue(results.size() == 2);
531 assertTrue(Bytes.BYTES_COMPARATOR.compare(secondRowBytes, results.get(0)
532 .getRow()) == 0);
533 assertTrue(Bytes.BYTES_COMPARATOR.compare(fam1, results.get(0)
534 .getFamily()) == 0);
535 assertTrue(Bytes.BYTES_COMPARATOR.compare(fam2, results.get(1)
536 .getFamily()) == 0);
537 } finally {
538 HRegion.closeHRegion(this.r);
539 }
540 }
541
542
543
544
545
546
547
548
549
550 private int count(final HRegionIncommon hri, final int flushIndex,
551 boolean concurrent)
552 throws IOException {
553 LOG.info("Taking out counting scan");
554 ScannerIncommon s = hri.getScanner(HConstants.CATALOG_FAMILY, EXPLICIT_COLS,
555 HConstants.EMPTY_START_ROW, HConstants.LATEST_TIMESTAMP);
556 List<KeyValue> values = new ArrayList<KeyValue>();
557 int count = 0;
558 boolean justFlushed = false;
559 while (s.next(values)) {
560 if (justFlushed) {
561 LOG.info("after next() just after next flush");
562 justFlushed=false;
563 }
564 count++;
565 if (flushIndex == count) {
566 LOG.info("Starting flush at flush index " + flushIndex);
567 Thread t = new Thread() {
568 public void run() {
569 try {
570 hri.flushcache();
571 LOG.info("Finishing flush");
572 } catch (IOException e) {
573 LOG.info("Failed flush cache");
574 }
575 }
576 };
577 if (concurrent) {
578 t.start();
579 } else {
580 t.run();
581 }
582 LOG.info("Continuing on after kicking off background flush");
583 justFlushed = true;
584 }
585 }
586 s.close();
587 LOG.info("Found " + count + " items");
588 return count;
589 }
590
591 }