1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase;
21
22 import java.io.File;
23 import java.io.IOException;
24 import java.io.UnsupportedEncodingException;
25 import java.util.Iterator;
26 import java.util.List;
27 import java.util.NavigableMap;
28
29 import junit.framework.AssertionFailedError;
30 import junit.framework.TestCase;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hbase.client.Delete;
38 import org.apache.hadoop.hbase.client.Get;
39 import org.apache.hadoop.hbase.client.HTable;
40 import org.apache.hadoop.hbase.client.Put;
41 import org.apache.hadoop.hbase.client.Result;
42 import org.apache.hadoop.hbase.client.ResultScanner;
43 import org.apache.hadoop.hbase.client.Scan;
44 import org.apache.hadoop.hbase.regionserver.HRegion;
45 import org.apache.hadoop.hbase.regionserver.InternalScanner;
46 import org.apache.hadoop.hbase.util.*;
47 import org.apache.hadoop.hdfs.MiniDFSCluster;
48
49
50
51
52
53
54 public abstract class HBaseTestCase extends TestCase {
55 private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
56
57
58
59
60 private static final String TEST_DIRECTORY_KEY = "test.build.data";
61
62
63
64
65
66
67 protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
68 protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
69 protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
70
71 protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
72
73 private boolean localfs = false;
74 protected static Path testDir = null;
75 protected FileSystem fs = null;
76 protected HRegion root = null;
77 protected HRegion meta = null;
78 protected static final char FIRST_CHAR = 'a';
79 protected static final char LAST_CHAR = 'z';
80 protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
81 protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
82 protected String START_KEY;
83 protected static final int MAXVERSIONS = 3;
84
85 static {
86 initialize();
87 }
88
89 public volatile Configuration conf;
90
91
92 public HBaseTestCase() {
93 super();
94 init();
95 }
96
97
98
99
100 public HBaseTestCase(String name) {
101 super(name);
102 init();
103 }
104
105 private void init() {
106 conf = HBaseConfiguration.create();
107 try {
108 START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_ENCODING);
109 } catch (UnsupportedEncodingException e) {
110 LOG.fatal("error during initialization", e);
111 fail();
112 }
113 }
114
115
116
117
118
119 @Override
120 protected void setUp() throws Exception {
121 super.setUp();
122 localfs =
123 (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
124
125 if (fs == null) {
126 this.fs = FileSystem.get(conf);
127 }
128 try {
129 if (localfs) {
130 this.testDir = getUnitTestdir(getName());
131 if (fs.exists(testDir)) {
132 fs.delete(testDir, true);
133 }
134 } else {
135 this.testDir =
136 this.fs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR)));
137 }
138 } catch (Exception e) {
139 LOG.fatal("error during setup", e);
140 throw e;
141 }
142 }
143
144 @Override
145 protected void tearDown() throws Exception {
146 try {
147 if (localfs) {
148 if (this.fs.exists(testDir)) {
149 this.fs.delete(testDir, true);
150 }
151 }
152 } catch (Exception e) {
153 LOG.fatal("error during tear down", e);
154 }
155 super.tearDown();
156 }
157
158
159
160
161
162
163 protected Path getUnitTestdir(String testName) {
164 return new Path(
165 System.getProperty(
166 HBaseTestingUtility.BASE_TEST_DIRECTORY_KEY,
167 HBaseTestingUtility.DEFAULT_BASE_TEST_DIRECTORY
168 ),
169 testName
170 );
171 }
172
173
174
175
176
177
178
179
180
181
182
183 public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
184 byte [] endKey)
185 throws IOException {
186 return createNewHRegion(desc, startKey, endKey, this.conf);
187 }
188
189 public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
190 byte [] endKey, Configuration conf)
191 throws IOException {
192 FileSystem filesystem = FileSystem.get(conf);
193 HRegionInfo hri = new HRegionInfo(desc.getName(), startKey, endKey);
194 return HRegion.createHRegion(hri, testDir, conf, desc);
195 }
196
197 protected HRegion openClosedRegion(final HRegion closedRegion)
198 throws IOException {
199 HRegion r = new HRegion(closedRegion.getTableDir(), closedRegion.getLog(),
200 closedRegion.getFilesystem(), closedRegion.getConf(),
201 closedRegion.getRegionInfo(), closedRegion.getTableDesc(), null);
202 r.initialize();
203 return r;
204 }
205
206
207
208
209
210
211
212 protected HTableDescriptor createTableDescriptor(final String name) {
213 return createTableDescriptor(name, MAXVERSIONS);
214 }
215
216
217
218
219
220
221
222
223 protected HTableDescriptor createTableDescriptor(final String name,
224 final int versions) {
225 return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
226 versions, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
227 }
228
229
230
231
232
233
234
235
236 protected HTableDescriptor createTableDescriptor(final String name,
237 final int minVersions, final int versions, final int ttl, boolean keepDeleted) {
238 HTableDescriptor htd = new HTableDescriptor(name);
239 for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
240 htd.addFamily(new HColumnDescriptor(cfName)
241 .setMinVersions(minVersions)
242 .setMaxVersions(versions)
243 .setKeepDeletedCells(keepDeleted)
244 .setBlockCacheEnabled(false)
245 .setTimeToLive(ttl)
246 );
247 }
248 return htd;
249 }
250
251
252
253
254
255
256
257
258
259
260
261 public static long addContent(final HRegion r, final byte [] columnFamily, final byte[] column)
262 throws IOException {
263 byte [] startKey = r.getRegionInfo().getStartKey();
264 byte [] endKey = r.getRegionInfo().getEndKey();
265 byte [] startKeyBytes = startKey;
266 if (startKeyBytes == null || startKeyBytes.length == 0) {
267 startKeyBytes = START_KEY_BYTES;
268 }
269 return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), Bytes.toString(column),
270 startKeyBytes, endKey, -1);
271 }
272
273
274
275
276
277
278
279
280
281
282 protected static long addContent(final HRegion r, final byte [] columnFamily)
283 throws IOException {
284 return addContent(r, columnFamily, null);
285 }
286
287
288
289
290
291
292
293
294
295
296 protected static long addContent(final Incommon updater,
297 final String columnFamily) throws IOException {
298 return addContent(updater, columnFamily, START_KEY_BYTES, null);
299 }
300
301 protected static long addContent(final Incommon updater, final String family,
302 final String column) throws IOException {
303 return addContent(updater, family, column, START_KEY_BYTES, null);
304 }
305
306
307
308
309
310
311
312
313
314
315
316
317 protected static long addContent(final Incommon updater, final String columnFamily,
318 final byte [] startKeyBytes, final byte [] endKey)
319 throws IOException {
320 return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
321 }
322
323 protected static long addContent(final Incommon updater, final String family,
324 final String column, final byte [] startKeyBytes,
325 final byte [] endKey) throws IOException {
326 return addContent(updater, family, column, startKeyBytes, endKey, -1);
327 }
328
329
330
331
332
333
334
335
336
337
338
339
340
341 protected static long addContent(final Incommon updater,
342 final String columnFamily,
343 final String column,
344 final byte [] startKeyBytes, final byte [] endKey, final long ts)
345 throws IOException {
346 long count = 0;
347
348
349
350
351 char secondCharStart = (char)startKeyBytes[1];
352 char thirdCharStart = (char)startKeyBytes[2];
353 EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
354 for (char d = secondCharStart; d <= LAST_CHAR; d++) {
355 for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
356 byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
357 if (endKey != null && endKey.length > 0
358 && Bytes.compareTo(endKey, t) <= 0) {
359 break EXIT;
360 }
361 try {
362 Put put;
363 if(ts != -1) {
364 put = new Put(t, ts, null);
365 } else {
366 put = new Put(t);
367 }
368 try {
369 StringBuilder sb = new StringBuilder();
370 if (column != null && column.contains(":")) {
371 sb.append(column);
372 } else {
373 if (columnFamily != null) {
374 sb.append(columnFamily);
375 if (!columnFamily.endsWith(":")) {
376 sb.append(":");
377 }
378 if (column != null) {
379 sb.append(column);
380 }
381 }
382 }
383 byte[][] split =
384 KeyValue.parseColumn(Bytes.toBytes(sb.toString()));
385 if(split.length == 1) {
386 put.add(split[0], new byte[0], t);
387 } else {
388 put.add(split[0], split[1], t);
389 }
390 updater.put(put);
391 count++;
392 } catch (RuntimeException ex) {
393 ex.printStackTrace();
394 throw ex;
395 } catch (IOException ex) {
396 ex.printStackTrace();
397 throw ex;
398 }
399 } catch (RuntimeException ex) {
400 ex.printStackTrace();
401 throw ex;
402 } catch (IOException ex) {
403 ex.printStackTrace();
404 throw ex;
405 }
406 }
407
408 thirdCharStart = FIRST_CHAR;
409 }
410 secondCharStart = FIRST_CHAR;
411 }
412 return count;
413 }
414
415
416
417
418 public static interface FlushCache {
419
420
421
422 public void flushcache() throws IOException;
423 }
424
425
426
427
428
429
430
431 public static interface Incommon {
432
433
434
435
436
437
438
439 public void delete(Delete delete, Integer lockid, boolean writeToWAL)
440 throws IOException;
441
442
443
444
445
446 public void put(Put put) throws IOException;
447
448 public Result get(Get get) throws IOException;
449
450
451
452
453
454
455
456
457
458 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
459 byte [] firstRow, long ts)
460 throws IOException;
461 }
462
463
464
465
466 public static class HRegionIncommon implements Incommon, FlushCache {
467 final HRegion region;
468
469
470
471
472 public HRegionIncommon(final HRegion HRegion) {
473 this.region = HRegion;
474 }
475
476 public void put(Put put) throws IOException {
477 region.put(put);
478 }
479
480 public void delete(Delete delete, Integer lockid, boolean writeToWAL)
481 throws IOException {
482 this.region.delete(delete, lockid, writeToWAL);
483 }
484
485 public Result get(Get get) throws IOException {
486 return region.get(get, null);
487 }
488
489 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
490 byte [] firstRow, long ts)
491 throws IOException {
492 Scan scan = new Scan(firstRow);
493 if(qualifiers == null || qualifiers.length == 0) {
494 scan.addFamily(family);
495 } else {
496 for(int i=0; i<qualifiers.length; i++){
497 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
498 }
499 }
500 scan.setTimeRange(0, ts);
501 return new
502 InternalScannerIncommon(region.getScanner(scan));
503 }
504
505 public Result get(Get get, Integer lockid) throws IOException{
506 return this.region.get(get, lockid);
507 }
508
509
510 public void flushcache() throws IOException {
511 this.region.flushcache();
512 }
513 }
514
515
516
517
518 public static class HTableIncommon implements Incommon {
519 final HTable table;
520
521
522
523
524 public HTableIncommon(final HTable table) {
525 super();
526 this.table = table;
527 }
528
529 public void put(Put put) throws IOException {
530 table.put(put);
531 }
532
533
534 public void delete(Delete delete, Integer lockid, boolean writeToWAL)
535 throws IOException {
536 this.table.delete(delete);
537 }
538
539 public Result get(Get get) throws IOException {
540 return table.get(get);
541 }
542
543 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
544 byte [] firstRow, long ts)
545 throws IOException {
546 Scan scan = new Scan(firstRow);
547 if(qualifiers == null || qualifiers.length == 0) {
548 scan.addFamily(family);
549 } else {
550 for(int i=0; i<qualifiers.length; i++){
551 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
552 }
553 }
554 scan.setTimeRange(0, ts);
555 return new
556 ClientScannerIncommon(table.getScanner(scan));
557 }
558 }
559
560 public interface ScannerIncommon
561 extends Iterable<Result> {
562 public boolean next(List<KeyValue> values)
563 throws IOException;
564
565 public void close() throws IOException;
566 }
567
568 public static class ClientScannerIncommon implements ScannerIncommon {
569 ResultScanner scanner;
570 public ClientScannerIncommon(ResultScanner scanner) {
571 this.scanner = scanner;
572 }
573
574 public boolean next(List<KeyValue> values)
575 throws IOException {
576 Result results = scanner.next();
577 if (results == null) {
578 return false;
579 }
580 values.clear();
581 values.addAll(results.list());
582 return true;
583 }
584
585 public void close() throws IOException {
586 scanner.close();
587 }
588
589 @SuppressWarnings("unchecked")
590 public Iterator iterator() {
591 return scanner.iterator();
592 }
593 }
594
595 public static class InternalScannerIncommon implements ScannerIncommon {
596 InternalScanner scanner;
597
598 public InternalScannerIncommon(InternalScanner scanner) {
599 this.scanner = scanner;
600 }
601
602 public boolean next(List<KeyValue> results)
603 throws IOException {
604 return scanner.next(results);
605 }
606
607 public void close() throws IOException {
608 scanner.close();
609 }
610
611 public Iterator<Result> iterator() {
612 throw new UnsupportedOperationException();
613 }
614 }
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636 protected void assertResultEquals(final HRegion region, final byte [] row,
637 final byte [] family, final byte [] qualifier, final long timestamp,
638 final byte [] value)
639 throws IOException {
640 Get get = new Get(row);
641 get.setTimeStamp(timestamp);
642 Result res = region.get(get, null);
643 NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
644 res.getMap();
645 byte [] res_value = map.get(family).get(qualifier).get(timestamp);
646
647 if (value == null) {
648 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
649 " at timestamp " + timestamp, null, res_value);
650 } else {
651 if (res_value == null) {
652 fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
653 " at timestamp " + timestamp + "\" was expected to be \"" +
654 Bytes.toStringBinary(value) + " but was null");
655 }
656 if (res_value != null) {
657 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
658 " at timestamp " +
659 timestamp, value, new String(res_value));
660 }
661 }
662 }
663
664
665
666
667
668
669
670
671
672 public static void initialize() {
673 if (System.getProperty(TEST_DIRECTORY_KEY) == null) {
674 System.setProperty(TEST_DIRECTORY_KEY, new File(
675 "build/hbase/test").getAbsolutePath());
676 }
677 }
678
679
680
681
682
683
684 public static void shutdownDfs(MiniDFSCluster cluster) {
685 if (cluster != null) {
686 LOG.info("Shutting down Mini DFS ");
687 try {
688 cluster.shutdown();
689 } catch (Exception e) {
690
691
692
693 }
694 try {
695 FileSystem fs = cluster.getFileSystem();
696 if (fs != null) {
697 LOG.info("Shutting down FileSystem");
698 fs.close();
699 }
700 FileSystem.closeAll();
701 } catch (IOException e) {
702 LOG.error("error closing file system", e);
703 }
704 }
705 }
706
707
708
709
710
711
712 protected void createRootAndMetaRegions() throws IOException {
713 root = HRegion.createHRegion(HRegionInfo.ROOT_REGIONINFO, testDir,
714 conf, HTableDescriptor.ROOT_TABLEDESC);
715 meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir,
716 conf, HTableDescriptor.META_TABLEDESC);
717 HRegion.addRegionToMETA(root, meta);
718 }
719
720 protected void closeRootAndMeta() throws IOException {
721 if (meta != null) {
722 meta.close();
723 meta.getLog().closeAndDelete();
724 }
725 if (root != null) {
726 root.close();
727 root.getLog().closeAndDelete();
728 }
729 }
730
731 public static void assertByteEquals(byte[] expected,
732 byte[] actual) {
733 if (Bytes.compareTo(expected, actual) != 0) {
734 throw new AssertionFailedError("expected:<" +
735 Bytes.toString(expected) + "> but was:<" +
736 Bytes.toString(actual) + ">");
737 }
738 }
739
740 public static void assertEquals(byte[] expected,
741 byte[] actual) {
742 if (Bytes.compareTo(expected, actual) != 0) {
743 throw new AssertionFailedError("expected:<" +
744 Bytes.toStringBinary(expected) + "> but was:<" +
745 Bytes.toStringBinary(actual) + ">");
746 }
747 }
748
749 }