1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase;
21
22 import java.io.File;
23 import java.io.IOException;
24 import java.io.UnsupportedEncodingException;
25 import java.util.Iterator;
26 import java.util.List;
27 import java.util.NavigableMap;
28
29 import junit.framework.AssertionFailedError;
30 import junit.framework.TestCase;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hbase.client.Delete;
38 import org.apache.hadoop.hbase.client.Get;
39 import org.apache.hadoop.hbase.client.HTable;
40 import org.apache.hadoop.hbase.client.Put;
41 import org.apache.hadoop.hbase.client.Result;
42 import org.apache.hadoop.hbase.client.ResultScanner;
43 import org.apache.hadoop.hbase.client.Scan;
44 import org.apache.hadoop.hbase.regionserver.HRegion;
45 import org.apache.hadoop.hbase.regionserver.InternalScanner;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hdfs.MiniDFSCluster;
48
49
50
51
52
53
54 public abstract class HBaseTestCase extends TestCase {
55 private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
56
57
58 public static final String TEST_DIRECTORY_KEY = "test.build.data";
59
60 protected final static byte [] fam1 = Bytes.toBytes("colfamily1");
61 protected final static byte [] fam2 = Bytes.toBytes("colfamily2");
62 protected final static byte [] fam3 = Bytes.toBytes("colfamily3");
63 protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
64
65 private boolean localfs = false;
66 protected Path testDir = null;
67 protected FileSystem fs = null;
68 protected HRegion root = null;
69 protected HRegion meta = null;
70 protected static final char FIRST_CHAR = 'a';
71 protected static final char LAST_CHAR = 'z';
72 protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
73 protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
74 protected String START_KEY;
75 protected static final int MAXVERSIONS = 3;
76
77 static {
78 initialize();
79 }
80
81 public volatile Configuration conf;
82
83
84 public HBaseTestCase() {
85 super();
86 init();
87 }
88
89
90
91
92 public HBaseTestCase(String name) {
93 super(name);
94 init();
95 }
96
97 private void init() {
98 conf = HBaseConfiguration.create();
99 try {
100 START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_ENCODING);
101 } catch (UnsupportedEncodingException e) {
102 LOG.fatal("error during initialization", e);
103 fail();
104 }
105 }
106
107
108
109
110
111 @Override
112 protected void setUp() throws Exception {
113 super.setUp();
114 localfs =
115 (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
116
117 if (fs == null) {
118 this.fs = FileSystem.get(conf);
119 }
120 try {
121 if (localfs) {
122 this.testDir = getUnitTestdir(getName());
123 if (fs.exists(testDir)) {
124 fs.delete(testDir, true);
125 }
126 } else {
127 this.testDir =
128 this.fs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR)));
129 }
130 } catch (Exception e) {
131 LOG.fatal("error during setup", e);
132 throw e;
133 }
134 }
135
136 @Override
137 protected void tearDown() throws Exception {
138 try {
139 if (localfs) {
140 if (this.fs.exists(testDir)) {
141 this.fs.delete(testDir, true);
142 }
143 }
144 } catch (Exception e) {
145 LOG.fatal("error during tear down", e);
146 }
147 super.tearDown();
148 }
149
150 protected Path getUnitTestdir(String testName) {
151 return new Path(
152 conf.get(TEST_DIRECTORY_KEY, "target/test/data"), testName);
153 }
154
155 protected HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
156 byte [] endKey)
157 throws IOException {
158 FileSystem filesystem = FileSystem.get(conf);
159 Path rootdir = filesystem.makeQualified(
160 new Path(conf.get(HConstants.HBASE_DIR)));
161 filesystem.mkdirs(rootdir);
162
163 return HRegion.createHRegion(new HRegionInfo(desc, startKey, endKey),
164 rootdir, conf);
165 }
166
167 protected HRegion openClosedRegion(final HRegion closedRegion)
168 throws IOException {
169 HRegion r = new HRegion(closedRegion.getTableDir(), closedRegion.getLog(),
170 closedRegion.getFilesystem(), closedRegion.getConf(),
171 closedRegion.getRegionInfo(), null);
172 r.initialize();
173 return r;
174 }
175
176
177
178
179
180
181
182 protected HTableDescriptor createTableDescriptor(final String name) {
183 return createTableDescriptor(name, MAXVERSIONS);
184 }
185
186
187
188
189
190
191
192
193 protected HTableDescriptor createTableDescriptor(final String name,
194 final int versions) {
195 HTableDescriptor htd = new HTableDescriptor(name);
196 htd.addFamily(new HColumnDescriptor(fam1, versions,
197 HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
198 Integer.MAX_VALUE, HConstants.FOREVER,
199 HColumnDescriptor.DEFAULT_BLOOMFILTER,
200 HConstants.REPLICATION_SCOPE_LOCAL));
201 htd.addFamily(new HColumnDescriptor(fam2, versions,
202 HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
203 Integer.MAX_VALUE, HConstants.FOREVER,
204 HColumnDescriptor.DEFAULT_BLOOMFILTER,
205 HConstants.REPLICATION_SCOPE_LOCAL));
206 htd.addFamily(new HColumnDescriptor(fam3, versions,
207 HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
208 Integer.MAX_VALUE, HConstants.FOREVER,
209 HColumnDescriptor.DEFAULT_BLOOMFILTER,
210 HConstants.REPLICATION_SCOPE_LOCAL));
211 return htd;
212 }
213
214
215
216
217
218
219
220
221
222
223 protected static long addContent(final HRegion r, final byte [] columnFamily)
224 throws IOException {
225 byte [] startKey = r.getRegionInfo().getStartKey();
226 byte [] endKey = r.getRegionInfo().getEndKey();
227 byte [] startKeyBytes = startKey;
228 if (startKeyBytes == null || startKeyBytes.length == 0) {
229 startKeyBytes = START_KEY_BYTES;
230 }
231 return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), null,
232 startKeyBytes, endKey, -1);
233 }
234
235
236
237
238
239
240
241
242
243
244 protected static long addContent(final Incommon updater,
245 final String columnFamily) throws IOException {
246 return addContent(updater, columnFamily, START_KEY_BYTES, null);
247 }
248
249 protected static long addContent(final Incommon updater, final String family,
250 final String column) throws IOException {
251 return addContent(updater, family, column, START_KEY_BYTES, null);
252 }
253
254
255
256
257
258
259
260
261
262
263
264
265 protected static long addContent(final Incommon updater, final String columnFamily,
266 final byte [] startKeyBytes, final byte [] endKey)
267 throws IOException {
268 return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
269 }
270
271 protected static long addContent(final Incommon updater, final String family,
272 final String column, final byte [] startKeyBytes,
273 final byte [] endKey) throws IOException {
274 return addContent(updater, family, column, startKeyBytes, endKey, -1);
275 }
276
277
278
279
280
281
282
283
284
285
286
287
288
289 protected static long addContent(final Incommon updater,
290 final String columnFamily,
291 final String column,
292 final byte [] startKeyBytes, final byte [] endKey, final long ts)
293 throws IOException {
294 long count = 0;
295
296
297
298
299 char secondCharStart = (char)startKeyBytes[1];
300 char thirdCharStart = (char)startKeyBytes[2];
301 EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
302 for (char d = secondCharStart; d <= LAST_CHAR; d++) {
303 for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
304 byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
305 if (endKey != null && endKey.length > 0
306 && Bytes.compareTo(endKey, t) <= 0) {
307 break EXIT;
308 }
309 try {
310 Put put;
311 if(ts != -1) {
312 put = new Put(t, ts, null);
313 } else {
314 put = new Put(t);
315 }
316 try {
317 StringBuilder sb = new StringBuilder();
318 if (column != null && column.contains(":")) {
319 sb.append(column);
320 } else {
321 if (columnFamily != null) {
322 sb.append(columnFamily);
323 if (!columnFamily.endsWith(":")) {
324 sb.append(":");
325 }
326 if (column != null) {
327 sb.append(column);
328 }
329 }
330 }
331 byte[][] split =
332 KeyValue.parseColumn(Bytes.toBytes(sb.toString()));
333 if(split.length == 1) {
334 put.add(split[0], new byte[0], t);
335 } else {
336 put.add(split[0], split[1], t);
337 }
338 updater.put(put);
339 count++;
340 } catch (RuntimeException ex) {
341 ex.printStackTrace();
342 throw ex;
343 } catch (IOException ex) {
344 ex.printStackTrace();
345 throw ex;
346 }
347 } catch (RuntimeException ex) {
348 ex.printStackTrace();
349 throw ex;
350 } catch (IOException ex) {
351 ex.printStackTrace();
352 throw ex;
353 }
354 }
355
356 thirdCharStart = FIRST_CHAR;
357 }
358 secondCharStart = FIRST_CHAR;
359 }
360 return count;
361 }
362
363
364
365
366 public static interface FlushCache {
367
368
369
370 public void flushcache() throws IOException;
371 }
372
373
374
375
376
377
378
379 public static interface Incommon {
380
381
382
383
384
385
386
387 public void delete(Delete delete, Integer lockid, boolean writeToWAL)
388 throws IOException;
389
390
391
392
393
394 public void put(Put put) throws IOException;
395
396 public Result get(Get get) throws IOException;
397
398
399
400
401
402
403
404
405
406 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
407 byte [] firstRow, long ts)
408 throws IOException;
409 }
410
411
412
413
414 public static class HRegionIncommon implements Incommon, FlushCache {
415 final HRegion region;
416
417
418
419
420 public HRegionIncommon(final HRegion HRegion) {
421 this.region = HRegion;
422 }
423
424 public void put(Put put) throws IOException {
425 region.put(put);
426 }
427
428 public void delete(Delete delete, Integer lockid, boolean writeToWAL)
429 throws IOException {
430 this.region.delete(delete, lockid, writeToWAL);
431 }
432
433 public Result get(Get get) throws IOException {
434 return region.get(get, null);
435 }
436
437 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
438 byte [] firstRow, long ts)
439 throws IOException {
440 Scan scan = new Scan(firstRow);
441 if(qualifiers == null || qualifiers.length == 0) {
442 scan.addFamily(family);
443 } else {
444 for(int i=0; i<qualifiers.length; i++){
445 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
446 }
447 }
448 scan.setTimeRange(0, ts);
449 return new
450 InternalScannerIncommon(region.getScanner(scan));
451 }
452
453 public Result get(Get get, Integer lockid) throws IOException{
454 return this.region.get(get, lockid);
455 }
456
457
458 public void flushcache() throws IOException {
459 this.region.flushcache();
460 }
461 }
462
463
464
465
466 public static class HTableIncommon implements Incommon {
467 final HTable table;
468
469
470
471
472 public HTableIncommon(final HTable table) {
473 super();
474 this.table = table;
475 }
476
477 public void put(Put put) throws IOException {
478 table.put(put);
479 }
480
481
482 public void delete(Delete delete, Integer lockid, boolean writeToWAL)
483 throws IOException {
484 this.table.delete(delete);
485 }
486
487 public Result get(Get get) throws IOException {
488 return table.get(get);
489 }
490
491 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
492 byte [] firstRow, long ts)
493 throws IOException {
494 Scan scan = new Scan(firstRow);
495 if(qualifiers == null || qualifiers.length == 0) {
496 scan.addFamily(family);
497 } else {
498 for(int i=0; i<qualifiers.length; i++){
499 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
500 }
501 }
502 scan.setTimeRange(0, ts);
503 return new
504 ClientScannerIncommon(table.getScanner(scan));
505 }
506 }
507
508 public interface ScannerIncommon
509 extends Iterable<Result> {
510 public boolean next(List<KeyValue> values)
511 throws IOException;
512
513 public void close() throws IOException;
514 }
515
516 public static class ClientScannerIncommon implements ScannerIncommon {
517 ResultScanner scanner;
518 public ClientScannerIncommon(ResultScanner scanner) {
519 this.scanner = scanner;
520 }
521
522 public boolean next(List<KeyValue> values)
523 throws IOException {
524 Result results = scanner.next();
525 if (results == null) {
526 return false;
527 }
528 values.clear();
529 values.addAll(results.list());
530 return true;
531 }
532
533 public void close() throws IOException {
534 scanner.close();
535 }
536
537 @SuppressWarnings("unchecked")
538 public Iterator iterator() {
539 return scanner.iterator();
540 }
541 }
542
543 public static class InternalScannerIncommon implements ScannerIncommon {
544 InternalScanner scanner;
545
546 public InternalScannerIncommon(InternalScanner scanner) {
547 this.scanner = scanner;
548 }
549
550 public boolean next(List<KeyValue> results)
551 throws IOException {
552 return scanner.next(results);
553 }
554
555 public void close() throws IOException {
556 scanner.close();
557 }
558
559 public Iterator<Result> iterator() {
560 throw new UnsupportedOperationException();
561 }
562 }
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584 protected void assertResultEquals(final HRegion region, final byte [] row,
585 final byte [] family, final byte [] qualifier, final long timestamp,
586 final byte [] value)
587 throws IOException {
588 Get get = new Get(row);
589 get.setTimeStamp(timestamp);
590 Result res = region.get(get, null);
591 NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
592 res.getMap();
593 byte [] res_value = map.get(family).get(qualifier).get(timestamp);
594
595 if (value == null) {
596 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
597 " at timestamp " + timestamp, null, res_value);
598 } else {
599 if (res_value == null) {
600 fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
601 " at timestamp " + timestamp + "\" was expected to be \"" +
602 Bytes.toStringBinary(value) + " but was null");
603 }
604 if (res_value != null) {
605 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
606 " at timestamp " +
607 timestamp, value, new String(res_value));
608 }
609 }
610 }
611
612
613
614
615
616
617
618
619
620 public static void initialize() {
621 if (System.getProperty(TEST_DIRECTORY_KEY) == null) {
622 System.setProperty(TEST_DIRECTORY_KEY, new File(
623 "build/hbase/test").getAbsolutePath());
624 }
625 }
626
627
628
629
630
631
632 public static void shutdownDfs(MiniDFSCluster cluster) {
633 if (cluster != null) {
634 LOG.info("Shutting down Mini DFS ");
635 try {
636 cluster.shutdown();
637 } catch (Exception e) {
638
639
640
641 }
642 try {
643 FileSystem fs = cluster.getFileSystem();
644 if (fs != null) {
645 LOG.info("Shutting down FileSystem");
646 fs.close();
647 }
648 FileSystem.closeAll();
649 } catch (IOException e) {
650 LOG.error("error closing file system", e);
651 }
652 }
653 }
654
655 protected void createRootAndMetaRegions() throws IOException {
656 root = HRegion.createHRegion(HRegionInfo.ROOT_REGIONINFO, testDir, conf);
657 meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir,
658 conf);
659 HRegion.addRegionToMETA(root, meta);
660 }
661
662 protected void closeRootAndMeta() throws IOException {
663 if (meta != null) {
664 meta.close();
665 meta.getLog().closeAndDelete();
666 }
667 if (root != null) {
668 root.close();
669 root.getLog().closeAndDelete();
670 }
671 }
672
673 public static void assertByteEquals(byte[] expected,
674 byte[] actual) {
675 if (Bytes.compareTo(expected, actual) != 0) {
676 throw new AssertionFailedError("expected:<" +
677 Bytes.toString(expected) + "> but was:<" +
678 Bytes.toString(actual) + ">");
679 }
680 }
681
682 public static void assertEquals(byte[] expected,
683 byte[] actual) {
684 if (Bytes.compareTo(expected, actual) != 0) {
685 throw new AssertionFailedError("expected:<" +
686 Bytes.toStringBinary(expected) + "> but was:<" +
687 Bytes.toStringBinary(actual) + ">");
688 }
689 }
690
691 }