1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase;
21
22 import java.io.File;
23 import java.io.IOException;
24 import java.io.UnsupportedEncodingException;
25 import java.util.Iterator;
26 import java.util.List;
27 import java.util.NavigableMap;
28
29 import junit.framework.AssertionFailedError;
30 import junit.framework.TestCase;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hbase.client.Delete;
38 import org.apache.hadoop.hbase.client.Get;
39 import org.apache.hadoop.hbase.client.HTable;
40 import org.apache.hadoop.hbase.client.Put;
41 import org.apache.hadoop.hbase.client.Result;
42 import org.apache.hadoop.hbase.client.ResultScanner;
43 import org.apache.hadoop.hbase.client.Scan;
44 import org.apache.hadoop.hbase.regionserver.HRegion;
45 import org.apache.hadoop.hbase.regionserver.InternalScanner;
46 import org.apache.hadoop.hbase.util.*;
47 import org.apache.hadoop.hdfs.MiniDFSCluster;
48
49
50
51
52
53
54 public abstract class HBaseTestCase extends TestCase {
55 private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
56
57
58
59
60 private static final String TEST_DIRECTORY_KEY = "test.build.data";
61
62
63
64
65
66
67 protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
68 protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
69 protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
70
71 protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
72
73 private boolean localfs = false;
74 protected static Path testDir = null;
75 protected FileSystem fs = null;
76 protected HRegion root = null;
77 protected HRegion meta = null;
78 protected static final char FIRST_CHAR = 'a';
79 protected static final char LAST_CHAR = 'z';
80 protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
81 protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
82 protected String START_KEY;
83 protected static final int MAXVERSIONS = 3;
84
85 static {
86 initialize();
87 }
88
89 public volatile Configuration conf;
90
91
92 public HBaseTestCase() {
93 super();
94 init();
95 }
96
97
98
99
100 public HBaseTestCase(String name) {
101 super(name);
102 init();
103 }
104
105 private void init() {
106 conf = HBaseConfiguration.create();
107 try {
108 START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_ENCODING);
109 } catch (UnsupportedEncodingException e) {
110 LOG.fatal("error during initialization", e);
111 fail();
112 }
113 }
114
115
116
117
118
119 @Override
120 protected void setUp() throws Exception {
121 super.setUp();
122 localfs =
123 (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
124
125 if (fs == null) {
126 this.fs = FileSystem.get(conf);
127 }
128 try {
129 if (localfs) {
130 this.testDir = getUnitTestdir(getName());
131 if (fs.exists(testDir)) {
132 fs.delete(testDir, true);
133 }
134 } else {
135 this.testDir =
136 this.fs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR)));
137 }
138 } catch (Exception e) {
139 LOG.fatal("error during setup", e);
140 throw e;
141 }
142 }
143
144 @Override
145 protected void tearDown() throws Exception {
146 try {
147 if (localfs) {
148 if (this.fs.exists(testDir)) {
149 this.fs.delete(testDir, true);
150 }
151 }
152 } catch (Exception e) {
153 LOG.fatal("error during tear down", e);
154 }
155 super.tearDown();
156 }
157
158
159
160
161
162
163 protected Path getUnitTestdir(String testName) {
164 return new Path(
165 System.getProperty(
166 HBaseTestingUtility.BASE_TEST_DIRECTORY_KEY,
167 HBaseTestingUtility.DEFAULT_BASE_TEST_DIRECTORY
168 ),
169 testName
170 );
171 }
172
173
174
175
176
177
178
179
180
181
182
183 public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
184 byte [] endKey)
185 throws IOException {
186 return createNewHRegion(desc, startKey, endKey, this.conf);
187 }
188
189 public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
190 byte [] endKey, Configuration conf)
191 throws IOException {
192 FileSystem filesystem = FileSystem.get(conf);
193 HRegionInfo hri = new HRegionInfo(desc.getName(), startKey, endKey);
194 return HRegion.createHRegion(hri, testDir, conf, desc);
195 }
196
197 protected HRegion openClosedRegion(final HRegion closedRegion)
198 throws IOException {
199 HRegion r = new HRegion(closedRegion);
200 r.initialize();
201 return r;
202 }
203
204
205
206
207
208
209
210 protected HTableDescriptor createTableDescriptor(final String name) {
211 return createTableDescriptor(name, MAXVERSIONS);
212 }
213
214
215
216
217
218
219
220
221 protected HTableDescriptor createTableDescriptor(final String name,
222 final int versions) {
223 return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
224 versions, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
225 }
226
227
228
229
230
231
232
233
234 protected HTableDescriptor createTableDescriptor(final String name,
235 final int minVersions, final int versions, final int ttl, boolean keepDeleted) {
236 HTableDescriptor htd = new HTableDescriptor(name);
237 for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
238 htd.addFamily(new HColumnDescriptor(cfName)
239 .setMinVersions(minVersions)
240 .setMaxVersions(versions)
241 .setKeepDeletedCells(keepDeleted)
242 .setBlockCacheEnabled(false)
243 .setTimeToLive(ttl)
244 );
245 }
246 return htd;
247 }
248
249
250
251
252
253
254
255
256
257
258
259 public static long addContent(final HRegion r, final byte [] columnFamily, final byte[] column)
260 throws IOException {
261 byte [] startKey = r.getRegionInfo().getStartKey();
262 byte [] endKey = r.getRegionInfo().getEndKey();
263 byte [] startKeyBytes = startKey;
264 if (startKeyBytes == null || startKeyBytes.length == 0) {
265 startKeyBytes = START_KEY_BYTES;
266 }
267 return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), Bytes.toString(column),
268 startKeyBytes, endKey, -1);
269 }
270
271
272
273
274
275
276
277
278
279
280 protected static long addContent(final HRegion r, final byte [] columnFamily)
281 throws IOException {
282 return addContent(r, columnFamily, null);
283 }
284
285
286
287
288
289
290
291
292
293
294 protected static long addContent(final Incommon updater,
295 final String columnFamily) throws IOException {
296 return addContent(updater, columnFamily, START_KEY_BYTES, null);
297 }
298
299 protected static long addContent(final Incommon updater, final String family,
300 final String column) throws IOException {
301 return addContent(updater, family, column, START_KEY_BYTES, null);
302 }
303
304
305
306
307
308
309
310
311
312
313
314
315 protected static long addContent(final Incommon updater, final String columnFamily,
316 final byte [] startKeyBytes, final byte [] endKey)
317 throws IOException {
318 return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
319 }
320
321 protected static long addContent(final Incommon updater, final String family,
322 final String column, final byte [] startKeyBytes,
323 final byte [] endKey) throws IOException {
324 return addContent(updater, family, column, startKeyBytes, endKey, -1);
325 }
326
327
328
329
330
331
332
333
334
335
336
337
338
339 protected static long addContent(final Incommon updater,
340 final String columnFamily,
341 final String column,
342 final byte [] startKeyBytes, final byte [] endKey, final long ts)
343 throws IOException {
344 long count = 0;
345
346
347
348
349 char secondCharStart = (char)startKeyBytes[1];
350 char thirdCharStart = (char)startKeyBytes[2];
351 EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
352 for (char d = secondCharStart; d <= LAST_CHAR; d++) {
353 for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
354 byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
355 if (endKey != null && endKey.length > 0
356 && Bytes.compareTo(endKey, t) <= 0) {
357 break EXIT;
358 }
359 try {
360 Put put;
361 if(ts != -1) {
362 put = new Put(t, ts, null);
363 } else {
364 put = new Put(t);
365 }
366 try {
367 StringBuilder sb = new StringBuilder();
368 if (column != null && column.contains(":")) {
369 sb.append(column);
370 } else {
371 if (columnFamily != null) {
372 sb.append(columnFamily);
373 if (!columnFamily.endsWith(":")) {
374 sb.append(":");
375 }
376 if (column != null) {
377 sb.append(column);
378 }
379 }
380 }
381 byte[][] split =
382 KeyValue.parseColumn(Bytes.toBytes(sb.toString()));
383 if(split.length == 1) {
384 put.add(split[0], new byte[0], t);
385 } else {
386 put.add(split[0], split[1], t);
387 }
388 updater.put(put);
389 count++;
390 } catch (RuntimeException ex) {
391 ex.printStackTrace();
392 throw ex;
393 } catch (IOException ex) {
394 ex.printStackTrace();
395 throw ex;
396 }
397 } catch (RuntimeException ex) {
398 ex.printStackTrace();
399 throw ex;
400 } catch (IOException ex) {
401 ex.printStackTrace();
402 throw ex;
403 }
404 }
405
406 thirdCharStart = FIRST_CHAR;
407 }
408 secondCharStart = FIRST_CHAR;
409 }
410 return count;
411 }
412
413
414
415
416 public static interface FlushCache {
417
418
419
420 public void flushcache() throws IOException;
421 }
422
423
424
425
426
427
428
429 public static interface Incommon {
430
431
432
433
434
435
436
437 public void delete(Delete delete, Integer lockid, boolean writeToWAL)
438 throws IOException;
439
440
441
442
443
444 public void put(Put put) throws IOException;
445
446 public Result get(Get get) throws IOException;
447
448
449
450
451
452
453
454
455
456 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
457 byte [] firstRow, long ts)
458 throws IOException;
459 }
460
461
462
463
464 public static class HRegionIncommon implements Incommon, FlushCache {
465 final HRegion region;
466
467
468
469
470 public HRegionIncommon(final HRegion HRegion) {
471 this.region = HRegion;
472 }
473
474 public void put(Put put) throws IOException {
475 region.put(put);
476 }
477
478 public void delete(Delete delete, Integer lockid, boolean writeToWAL)
479 throws IOException {
480 this.region.delete(delete, lockid, writeToWAL);
481 }
482
483 public Result get(Get get) throws IOException {
484 return region.get(get, null);
485 }
486
487 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
488 byte [] firstRow, long ts)
489 throws IOException {
490 Scan scan = new Scan(firstRow);
491 if(qualifiers == null || qualifiers.length == 0) {
492 scan.addFamily(family);
493 } else {
494 for(int i=0; i<qualifiers.length; i++){
495 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
496 }
497 }
498 scan.setTimeRange(0, ts);
499 return new
500 InternalScannerIncommon(region.getScanner(scan));
501 }
502
503 public Result get(Get get, Integer lockid) throws IOException{
504 return this.region.get(get, lockid);
505 }
506
507
508 public void flushcache() throws IOException {
509 this.region.flushcache();
510 }
511 }
512
513
514
515
516 public static class HTableIncommon implements Incommon {
517 final HTable table;
518
519
520
521
522 public HTableIncommon(final HTable table) {
523 super();
524 this.table = table;
525 }
526
527 public void put(Put put) throws IOException {
528 table.put(put);
529 }
530
531
532 public void delete(Delete delete, Integer lockid, boolean writeToWAL)
533 throws IOException {
534 this.table.delete(delete);
535 }
536
537 public Result get(Get get) throws IOException {
538 return table.get(get);
539 }
540
541 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
542 byte [] firstRow, long ts)
543 throws IOException {
544 Scan scan = new Scan(firstRow);
545 if(qualifiers == null || qualifiers.length == 0) {
546 scan.addFamily(family);
547 } else {
548 for(int i=0; i<qualifiers.length; i++){
549 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
550 }
551 }
552 scan.setTimeRange(0, ts);
553 return new
554 ClientScannerIncommon(table.getScanner(scan));
555 }
556 }
557
558 public interface ScannerIncommon
559 extends Iterable<Result> {
560 public boolean next(List<KeyValue> values)
561 throws IOException;
562
563 public void close() throws IOException;
564 }
565
566 public static class ClientScannerIncommon implements ScannerIncommon {
567 ResultScanner scanner;
568 public ClientScannerIncommon(ResultScanner scanner) {
569 this.scanner = scanner;
570 }
571
572 public boolean next(List<KeyValue> values)
573 throws IOException {
574 Result results = scanner.next();
575 if (results == null) {
576 return false;
577 }
578 values.clear();
579 values.addAll(results.list());
580 return true;
581 }
582
583 public void close() throws IOException {
584 scanner.close();
585 }
586
587 @SuppressWarnings("unchecked")
588 public Iterator iterator() {
589 return scanner.iterator();
590 }
591 }
592
593 public static class InternalScannerIncommon implements ScannerIncommon {
594 InternalScanner scanner;
595
596 public InternalScannerIncommon(InternalScanner scanner) {
597 this.scanner = scanner;
598 }
599
600 public boolean next(List<KeyValue> results)
601 throws IOException {
602 return scanner.next(results);
603 }
604
605 public void close() throws IOException {
606 scanner.close();
607 }
608
609 public Iterator<Result> iterator() {
610 throw new UnsupportedOperationException();
611 }
612 }
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634 protected void assertResultEquals(final HRegion region, final byte [] row,
635 final byte [] family, final byte [] qualifier, final long timestamp,
636 final byte [] value)
637 throws IOException {
638 Get get = new Get(row);
639 get.setTimeStamp(timestamp);
640 Result res = region.get(get, null);
641 NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
642 res.getMap();
643 byte [] res_value = map.get(family).get(qualifier).get(timestamp);
644
645 if (value == null) {
646 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
647 " at timestamp " + timestamp, null, res_value);
648 } else {
649 if (res_value == null) {
650 fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
651 " at timestamp " + timestamp + "\" was expected to be \"" +
652 Bytes.toStringBinary(value) + " but was null");
653 }
654 if (res_value != null) {
655 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
656 " at timestamp " +
657 timestamp, value, new String(res_value));
658 }
659 }
660 }
661
662
663
664
665
666
667
668
669
670 public static void initialize() {
671 if (System.getProperty(TEST_DIRECTORY_KEY) == null) {
672 System.setProperty(TEST_DIRECTORY_KEY, new File(
673 "build/hbase/test").getAbsolutePath());
674 }
675 }
676
677
678
679
680
681
682 public static void shutdownDfs(MiniDFSCluster cluster) {
683 if (cluster != null) {
684 LOG.info("Shutting down Mini DFS ");
685 try {
686 cluster.shutdown();
687 } catch (Exception e) {
688
689
690
691 }
692 try {
693 FileSystem fs = cluster.getFileSystem();
694 if (fs != null) {
695 LOG.info("Shutting down FileSystem");
696 fs.close();
697 }
698 FileSystem.closeAll();
699 } catch (IOException e) {
700 LOG.error("error closing file system", e);
701 }
702 }
703 }
704
705
706
707
708
709
710 protected void createRootAndMetaRegions() throws IOException {
711 root = HRegion.createHRegion(HRegionInfo.ROOT_REGIONINFO, testDir,
712 conf, HTableDescriptor.ROOT_TABLEDESC);
713 meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir,
714 conf, HTableDescriptor.META_TABLEDESC);
715 HRegion.addRegionToMETA(root, meta);
716 }
717
718 protected void closeRootAndMeta() throws IOException {
719 if (meta != null) {
720 meta.close();
721 meta.getLog().closeAndDelete();
722 }
723 if (root != null) {
724 root.close();
725 root.getLog().closeAndDelete();
726 }
727 }
728
729 public static void assertByteEquals(byte[] expected,
730 byte[] actual) {
731 if (Bytes.compareTo(expected, actual) != 0) {
732 throw new AssertionFailedError("expected:<" +
733 Bytes.toString(expected) + "> but was:<" +
734 Bytes.toString(actual) + ">");
735 }
736 }
737
738 public static void assertEquals(byte[] expected,
739 byte[] actual) {
740 if (Bytes.compareTo(expected, actual) != 0) {
741 throw new AssertionFailedError("expected:<" +
742 Bytes.toStringBinary(expected) + "> but was:<" +
743 Bytes.toStringBinary(actual) + ">");
744 }
745 }
746
747 }