1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.ByteArrayOutputStream;
23 import java.io.DataOutputStream;
24 import java.io.IOException;
25 import java.util.ArrayList;
26 import java.util.List;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.hbase.HBaseTestCase;
31 import org.apache.hadoop.hbase.HColumnDescriptor;
32 import org.apache.hadoop.hbase.HConstants;
33 import org.apache.hadoop.hbase.HRegionInfo;
34 import org.apache.hadoop.hbase.HTableDescriptor;
35 import org.apache.hadoop.hbase.KeyValue;
36 import org.apache.hadoop.hbase.SmallTests;
37 import org.apache.hadoop.hbase.UnknownScannerException;
38 import org.apache.hadoop.hbase.client.Delete;
39 import org.apache.hadoop.hbase.client.Get;
40 import org.apache.hadoop.hbase.client.Put;
41 import org.apache.hadoop.hbase.client.Result;
42 import org.apache.hadoop.hbase.client.Scan;
43 import org.apache.hadoop.hbase.filter.Filter;
44 import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
45 import org.apache.hadoop.hbase.filter.PrefixFilter;
46 import org.apache.hadoop.hbase.filter.WhileMatchFilter;
47 import org.apache.hadoop.hbase.io.hfile.Compression;
48 import org.apache.hadoop.hbase.util.Bytes;
49 import org.apache.hadoop.hbase.util.Writables;
50 import org.junit.experimental.categories.Category;
51
52
53
54
55 @Category(SmallTests.class)
56 public class TestScanner extends HBaseTestCase {
57 private final Log LOG = LogFactory.getLog(this.getClass());
58
59 private static final byte [] FIRST_ROW = HConstants.EMPTY_START_ROW;
60 private static final byte [][] COLS = { HConstants.CATALOG_FAMILY };
61 private static final byte [][] EXPLICIT_COLS = {
62 HConstants.REGIONINFO_QUALIFIER, HConstants.SERVER_QUALIFIER,
63
64
65 };
66
67 static final HTableDescriptor TESTTABLEDESC =
68 new HTableDescriptor("testscanner");
69 static {
70 TESTTABLEDESC.addFamily(
71 new HColumnDescriptor(HConstants.CATALOG_FAMILY)
72
73 .setMaxVersions(10)
74 .setBlockCacheEnabled(false)
75 .setBlocksize(8 * 1024)
76 );
77 }
78
79 public static final HRegionInfo REGION_INFO =
80 new HRegionInfo(TESTTABLEDESC.getName(), HConstants.EMPTY_BYTE_ARRAY,
81 HConstants.EMPTY_BYTE_ARRAY);
82
83 private static final byte [] ROW_KEY = REGION_INFO.getRegionName();
84
85 private static final long START_CODE = Long.MAX_VALUE;
86
87 private HRegion r;
88 private HRegionIncommon region;
89
90 private byte[] firstRowBytes, secondRowBytes, thirdRowBytes;
91 final private byte[] col1, col2;
92
93 public TestScanner() throws Exception {
94 super();
95
96 firstRowBytes = START_KEY.getBytes(HConstants.UTF8_ENCODING);
97 secondRowBytes = START_KEY.getBytes(HConstants.UTF8_ENCODING);
98
99 secondRowBytes[START_KEY_BYTES.length - 1]++;
100 thirdRowBytes = START_KEY.getBytes(HConstants.UTF8_ENCODING);
101 thirdRowBytes[START_KEY_BYTES.length - 1]++;
102 thirdRowBytes[START_KEY_BYTES.length - 1]++;
103 col1 = "column1".getBytes(HConstants.UTF8_ENCODING);
104 col2 = "column2".getBytes(HConstants.UTF8_ENCODING);
105 }
106
107
108
109
110
111 public void testStopRow() throws Exception {
112 byte [] startrow = Bytes.toBytes("bbb");
113 byte [] stoprow = Bytes.toBytes("ccc");
114 try {
115 this.r = createNewHRegion(TESTTABLEDESC, null, null);
116 addContent(this.r, HConstants.CATALOG_FAMILY);
117 List<KeyValue> results = new ArrayList<KeyValue>();
118
119 Scan scan = new Scan(Bytes.toBytes("abc"), Bytes.toBytes("abd"));
120 scan.addFamily(HConstants.CATALOG_FAMILY);
121
122 InternalScanner s = r.getScanner(scan);
123 int count = 0;
124 while (s.next(results)) {
125 count++;
126 }
127 s.close();
128 assertEquals(0, count);
129
130 scan = new Scan(startrow, stoprow);
131 scan.addFamily(HConstants.CATALOG_FAMILY);
132
133 s = r.getScanner(scan);
134 count = 0;
135 KeyValue kv = null;
136 results = new ArrayList<KeyValue>();
137 for (boolean first = true; s.next(results);) {
138 kv = results.get(0);
139 if (first) {
140 assertTrue(Bytes.BYTES_COMPARATOR.compare(startrow, kv.getRow()) == 0);
141 first = false;
142 }
143 count++;
144 }
145 assertTrue(Bytes.BYTES_COMPARATOR.compare(stoprow, kv.getRow()) > 0);
146
147 assertTrue(count > 10);
148 s.close();
149 } finally {
150 this.r.close();
151 this.r.getLog().closeAndDelete();
152 }
153 }
154
155 void rowPrefixFilter(Scan scan) throws IOException {
156 List<KeyValue> results = new ArrayList<KeyValue>();
157 scan.addFamily(HConstants.CATALOG_FAMILY);
158 InternalScanner s = r.getScanner(scan);
159 boolean hasMore = true;
160 while (hasMore) {
161 hasMore = s.next(results);
162 for (KeyValue kv : results) {
163 assertEquals((byte)'a', kv.getRow()[0]);
164 assertEquals((byte)'b', kv.getRow()[1]);
165 }
166 results.clear();
167 }
168 s.close();
169 }
170
171 void rowInclusiveStopFilter(Scan scan, byte[] stopRow) throws IOException {
172 List<KeyValue> results = new ArrayList<KeyValue>();
173 scan.addFamily(HConstants.CATALOG_FAMILY);
174 InternalScanner s = r.getScanner(scan);
175 boolean hasMore = true;
176 while (hasMore) {
177 hasMore = s.next(results);
178 for (KeyValue kv : results) {
179 assertTrue(Bytes.compareTo(kv.getRow(), stopRow) <= 0);
180 }
181 results.clear();
182 }
183 s.close();
184 }
185
186 public void testFilters() throws IOException {
187 try {
188 this.r = createNewHRegion(TESTTABLEDESC, null, null);
189 addContent(this.r, HConstants.CATALOG_FAMILY);
190 byte [] prefix = Bytes.toBytes("ab");
191 Filter newFilter = new PrefixFilter(prefix);
192 Scan scan = new Scan();
193 scan.setFilter(newFilter);
194 rowPrefixFilter(scan);
195
196 byte[] stopRow = Bytes.toBytes("bbc");
197 newFilter = new WhileMatchFilter(new InclusiveStopFilter(stopRow));
198 scan = new Scan();
199 scan.setFilter(newFilter);
200 rowInclusiveStopFilter(scan, stopRow);
201
202 } finally {
203 this.r.close();
204 this.r.getLog().closeAndDelete();
205 }
206 }
207
208
209
210
211
212
213 public void testRaceBetweenClientAndTimeout() throws Exception {
214 try {
215 this.r = createNewHRegion(TESTTABLEDESC, null, null);
216 addContent(this.r, HConstants.CATALOG_FAMILY);
217 Scan scan = new Scan();
218 InternalScanner s = r.getScanner(scan);
219 List<KeyValue> results = new ArrayList<KeyValue>();
220 try {
221 s.next(results);
222 s.close();
223 s.next(results);
224 fail("We don't want anything more, we should be failing");
225 } catch (UnknownScannerException ex) {
226
227 return;
228 }
229 } finally {
230 this.r.close();
231 this.r.getLog().closeAndDelete();
232 }
233 }
234
235
236
237
238 public void testScanner() throws IOException {
239 try {
240 r = createNewHRegion(TESTTABLEDESC, null, null);
241 region = new HRegionIncommon(r);
242
243
244
245 Put put = new Put(ROW_KEY, System.currentTimeMillis(), null);
246
247 ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
248 DataOutputStream s = new DataOutputStream(byteStream);
249 REGION_INFO.write(s);
250 put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
251 byteStream.toByteArray());
252 region.put(put);
253
254
255
256
257 scan(false, null);
258 getRegionInfo();
259
260
261
262 r.close();
263 r = openClosedRegion(r);
264 region = new HRegionIncommon(r);
265
266
267
268 scan(false, null);
269 getRegionInfo();
270
271
272
273 String address = "www.example.com:1234";
274
275 put = new Put(ROW_KEY, System.currentTimeMillis(), null);
276 put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
277 Bytes.toBytes(address));
278
279
280
281 region.put(put);
282
283
284
285
286 scan(true, address.toString());
287 getRegionInfo();
288
289
290
291 region.flushcache();
292
293
294
295 scan(true, address.toString());
296 getRegionInfo();
297
298
299
300 r.close();
301 r = openClosedRegion(r);
302 region = new HRegionIncommon(r);
303
304
305
306 scan(true, address.toString());
307 getRegionInfo();
308
309
310
311 address = "bar.foo.com:4321";
312
313 put = new Put(ROW_KEY, System.currentTimeMillis(), null);
314
315 put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
316 Bytes.toBytes(address));
317 region.put(put);
318
319
320
321 scan(true, address.toString());
322 getRegionInfo();
323
324
325
326 region.flushcache();
327
328
329
330 scan(true, address.toString());
331 getRegionInfo();
332
333
334
335 r.close();
336 r = openClosedRegion(r);
337 region = new HRegionIncommon(r);
338
339
340
341 scan(true, address.toString());
342 getRegionInfo();
343
344 } finally {
345
346 r.close();
347 r.getLog().closeAndDelete();
348 }
349 }
350
351
352 private void validateRegionInfo(byte [] regionBytes) throws IOException {
353 HRegionInfo info =
354 (HRegionInfo) Writables.getWritable(regionBytes, new HRegionInfo());
355
356 assertEquals(REGION_INFO.getRegionId(), info.getRegionId());
357 assertEquals(0, info.getStartKey().length);
358 assertEquals(0, info.getEndKey().length);
359 assertEquals(0, Bytes.compareTo(info.getRegionName(), REGION_INFO.getRegionName()));
360
361 }
362
363
364 private void scan(boolean validateStartcode, String serverName)
365 throws IOException {
366 InternalScanner scanner = null;
367 Scan scan = null;
368 List<KeyValue> results = new ArrayList<KeyValue>();
369 byte [][][] scanColumns = {
370 COLS,
371 EXPLICIT_COLS
372 };
373
374 for(int i = 0; i < scanColumns.length; i++) {
375 try {
376 scan = new Scan(FIRST_ROW);
377 for (int ii = 0; ii < EXPLICIT_COLS.length; ii++) {
378 scan.addColumn(COLS[0], EXPLICIT_COLS[ii]);
379 }
380 scanner = r.getScanner(scan);
381 while (scanner.next(results)) {
382 assertTrue(hasColumn(results, HConstants.CATALOG_FAMILY,
383 HConstants.REGIONINFO_QUALIFIER));
384 byte [] val = getColumn(results, HConstants.CATALOG_FAMILY,
385 HConstants.REGIONINFO_QUALIFIER).getValue();
386 validateRegionInfo(val);
387 if(validateStartcode) {
388
389
390
391
392 assertNotNull(val);
393 assertFalse(val.length == 0);
394 long startCode = Bytes.toLong(val);
395 assertEquals(START_CODE, startCode);
396 }
397
398 if(serverName != null) {
399 assertTrue(hasColumn(results, HConstants.CATALOG_FAMILY,
400 HConstants.SERVER_QUALIFIER));
401 val = getColumn(results, HConstants.CATALOG_FAMILY,
402 HConstants.SERVER_QUALIFIER).getValue();
403 assertNotNull(val);
404 assertFalse(val.length == 0);
405 String server = Bytes.toString(val);
406 assertEquals(0, server.compareTo(serverName));
407 }
408 }
409 } finally {
410 InternalScanner s = scanner;
411 scanner = null;
412 if(s != null) {
413 s.close();
414 }
415 }
416 }
417 }
418
419 private boolean hasColumn(final List<KeyValue> kvs, final byte [] family,
420 final byte [] qualifier) {
421 for (KeyValue kv: kvs) {
422 if (kv.matchingFamily(family) && kv.matchingQualifier(qualifier)) {
423 return true;
424 }
425 }
426 return false;
427 }
428
429 private KeyValue getColumn(final List<KeyValue> kvs, final byte [] family,
430 final byte [] qualifier) {
431 for (KeyValue kv: kvs) {
432 if (kv.matchingFamily(family) && kv.matchingQualifier(qualifier)) {
433 return kv;
434 }
435 }
436 return null;
437 }
438
439
440
441 private void getRegionInfo() throws IOException {
442 Get get = new Get(ROW_KEY);
443 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
444 Result result = region.get(get, null);
445 byte [] bytes = result.value();
446 validateRegionInfo(bytes);
447 }
448
449
450
451
452
453
454
455 public void testScanAndSyncFlush() throws Exception {
456 this.r = createNewHRegion(TESTTABLEDESC, null, null);
457 HRegionIncommon hri = new HRegionIncommon(r);
458 try {
459 LOG.info("Added: " + addContent(hri, Bytes.toString(HConstants.CATALOG_FAMILY),
460 Bytes.toString(HConstants.REGIONINFO_QUALIFIER)));
461 int count = count(hri, -1, false);
462 assertEquals(count, count(hri, 100, false));
463 } catch (Exception e) {
464 LOG.error("Failed", e);
465 throw e;
466 } finally {
467 this.r.close();
468 this.r.getLog().closeAndDelete();
469 }
470 }
471
472
473
474
475
476
477
478 public void testScanAndRealConcurrentFlush() throws Exception {
479 this.r = createNewHRegion(TESTTABLEDESC, null, null);
480 HRegionIncommon hri = new HRegionIncommon(r);
481 try {
482 LOG.info("Added: " + addContent(hri, Bytes.toString(HConstants.CATALOG_FAMILY),
483 Bytes.toString(HConstants.REGIONINFO_QUALIFIER)));
484 int count = count(hri, -1, false);
485 assertEquals(count, count(hri, 100, true));
486 } catch (Exception e) {
487 LOG.error("Failed", e);
488 throw e;
489 } finally {
490 this.r.close();
491 this.r.getLog().closeAndDelete();
492 }
493 }
494
495
496
497
498
499
500
501 @SuppressWarnings("deprecation")
502 public void testScanAndConcurrentMajorCompact() throws Exception {
503 HTableDescriptor htd = createTableDescriptor(getName());
504 this.r = createNewHRegion(htd, null, null);
505 HRegionIncommon hri = new HRegionIncommon(r);
506
507 try {
508 addContent(hri, Bytes.toString(fam1), Bytes.toString(col1),
509 firstRowBytes, secondRowBytes);
510 addContent(hri, Bytes.toString(fam2), Bytes.toString(col1),
511 firstRowBytes, secondRowBytes);
512
513 Delete dc = new Delete(firstRowBytes);
514
515 dc.deleteColumns(fam1, col1);
516 r.delete(dc, null, true);
517 r.flushcache();
518
519 addContent(hri, Bytes.toString(fam1), Bytes.toString(col1),
520 secondRowBytes, thirdRowBytes);
521 addContent(hri, Bytes.toString(fam2), Bytes.toString(col1),
522 secondRowBytes, thirdRowBytes);
523 r.flushcache();
524
525 InternalScanner s = r.getScanner(new Scan());
526
527 r.compactStores(true);
528
529 List<KeyValue> results = new ArrayList<KeyValue>();
530 s.next(results);
531
532
533 assertTrue("result is not correct, keyValues : " + results,
534 results.size() == 1);
535 assertTrue(Bytes.BYTES_COMPARATOR.compare(firstRowBytes, results.get(0)
536 .getRow()) == 0);
537 assertTrue(Bytes.BYTES_COMPARATOR.compare(fam2, results.get(0)
538 .getFamily()) == 0);
539
540 results = new ArrayList<KeyValue>();
541 s.next(results);
542
543
544 assertTrue(results.size() == 2);
545 assertTrue(Bytes.BYTES_COMPARATOR.compare(secondRowBytes, results.get(0)
546 .getRow()) == 0);
547 assertTrue(Bytes.BYTES_COMPARATOR.compare(fam1, results.get(0)
548 .getFamily()) == 0);
549 assertTrue(Bytes.BYTES_COMPARATOR.compare(fam2, results.get(1)
550 .getFamily()) == 0);
551 } finally {
552 this.r.close();
553 this.r.getLog().closeAndDelete();
554 }
555 }
556
557
558
559
560
561
562
563
564
565 private int count(final HRegionIncommon hri, final int flushIndex,
566 boolean concurrent)
567 throws IOException {
568 LOG.info("Taking out counting scan");
569 ScannerIncommon s = hri.getScanner(HConstants.CATALOG_FAMILY, EXPLICIT_COLS,
570 HConstants.EMPTY_START_ROW, HConstants.LATEST_TIMESTAMP);
571 List<KeyValue> values = new ArrayList<KeyValue>();
572 int count = 0;
573 boolean justFlushed = false;
574 while (s.next(values)) {
575 if (justFlushed) {
576 LOG.info("after next() just after next flush");
577 justFlushed=false;
578 }
579 count++;
580 if (flushIndex == count) {
581 LOG.info("Starting flush at flush index " + flushIndex);
582 Thread t = new Thread() {
583 public void run() {
584 try {
585 hri.flushcache();
586 LOG.info("Finishing flush");
587 } catch (IOException e) {
588 LOG.info("Failed flush cache");
589 }
590 }
591 };
592 if (concurrent) {
593 t.start();
594 } else {
595 t.run();
596 }
597 LOG.info("Continuing on after kicking off background flush");
598 justFlushed = true;
599 }
600 }
601 s.close();
602 LOG.info("Found " + count + " items");
603 return count;
604 }
605
606 @org.junit.Rule
607 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
608 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
609 }
610