1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import java.io.IOException;
22 import java.util.Iterator;
23 import java.util.List;
24 import java.util.NavigableMap;
25
26 import junit.framework.AssertionFailedError;
27 import junit.framework.TestCase;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.client.Delete;
35 import org.apache.hadoop.hbase.client.Get;
36 import org.apache.hadoop.hbase.client.HTable;
37 import org.apache.hadoop.hbase.client.Put;
38 import org.apache.hadoop.hbase.client.Result;
39 import org.apache.hadoop.hbase.client.ResultScanner;
40 import org.apache.hadoop.hbase.client.Scan;
41 import org.apache.hadoop.hbase.regionserver.HRegion;
42 import org.apache.hadoop.hbase.regionserver.InternalScanner;
43 import org.apache.hadoop.hbase.util.*;
44 import org.apache.hadoop.hdfs.MiniDFSCluster;
45
46
47
48
49
50
51 public abstract class HBaseTestCase extends TestCase {
52 private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
53
54 protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
55 protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
56 protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
57
58 protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
59
60 private boolean localfs = false;
61 protected static Path testDir = null;
62 protected FileSystem fs = null;
63 protected HRegion meta = null;
64 protected static final char FIRST_CHAR = 'a';
65 protected static final char LAST_CHAR = 'z';
66 protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
67 protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
68 protected String START_KEY;
69 protected static final int MAXVERSIONS = 3;
70
71 protected final HBaseTestingUtility testUtil = new HBaseTestingUtility();
72
73 public volatile Configuration conf;
74
75
76 public HBaseTestCase() {
77 super();
78 init();
79 }
80
81
82
83
84 public HBaseTestCase(String name) {
85 super(name);
86 init();
87 }
88
89 private void init() {
90 conf = HBaseConfiguration.create();
91 START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET);
92 }
93
94
95
96
97
98 @Override
99 protected void setUp() throws Exception {
100 super.setUp();
101 localfs =
102 (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
103
104 if (fs == null) {
105 this.fs = FileSystem.get(conf);
106 }
107 try {
108 if (localfs) {
109 this.testDir = getUnitTestdir(getName());
110 if (fs.exists(testDir)) {
111 fs.delete(testDir, true);
112 }
113 } else {
114 this.testDir = FSUtils.getRootDir(conf);
115 }
116 } catch (Exception e) {
117 LOG.fatal("error during setup", e);
118 throw e;
119 }
120 }
121
122 @Override
123 protected void tearDown() throws Exception {
124 try {
125 if (localfs) {
126 if (this.fs.exists(testDir)) {
127 this.fs.delete(testDir, true);
128 }
129 }
130 } catch (Exception e) {
131 LOG.fatal("error during tear down", e);
132 }
133 super.tearDown();
134 }
135
136
137
138
139
140
141 protected Path getUnitTestdir(String testName) {
142 return testUtil.getDataTestDir(testName);
143 }
144
145
146
147
148
149
150
151
152
153
154
155 public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
156 byte [] endKey)
157 throws IOException {
158 return createNewHRegion(desc, startKey, endKey, this.conf);
159 }
160
161 public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
162 byte [] endKey, Configuration conf)
163 throws IOException {
164 HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
165 return HRegion.createHRegion(hri, testDir, conf, desc);
166 }
167
168 protected HRegion openClosedRegion(final HRegion closedRegion)
169 throws IOException {
170 return HRegion.openHRegion(closedRegion, null);
171 }
172
173
174
175
176
177
178
179 protected HTableDescriptor createTableDescriptor(final String name) {
180 return createTableDescriptor(name, MAXVERSIONS);
181 }
182
183
184
185
186
187
188
189
190 protected HTableDescriptor createTableDescriptor(final String name,
191 final int versions) {
192 return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
193 versions, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
194 }
195
196
197
198
199
200
201
202
203 protected HTableDescriptor createTableDescriptor(final String name,
204 final int minVersions, final int versions, final int ttl, boolean keepDeleted) {
205 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
206 for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
207 htd.addFamily(new HColumnDescriptor(cfName)
208 .setMinVersions(minVersions)
209 .setMaxVersions(versions)
210 .setKeepDeletedCells(keepDeleted)
211 .setBlockCacheEnabled(false)
212 .setTimeToLive(ttl)
213 );
214 }
215 return htd;
216 }
217
218
219
220
221
222
223
224
225
226
227
228 public static long addContent(final HRegion r, final byte [] columnFamily, final byte[] column)
229 throws IOException {
230 byte [] startKey = r.getRegionInfo().getStartKey();
231 byte [] endKey = r.getRegionInfo().getEndKey();
232 byte [] startKeyBytes = startKey;
233 if (startKeyBytes == null || startKeyBytes.length == 0) {
234 startKeyBytes = START_KEY_BYTES;
235 }
236 return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), Bytes.toString(column),
237 startKeyBytes, endKey, -1);
238 }
239
240
241
242
243
244
245
246
247
248
249 protected static long addContent(final HRegion r, final byte [] columnFamily)
250 throws IOException {
251 return addContent(r, columnFamily, null);
252 }
253
254
255
256
257
258
259
260
261
262
263 protected static long addContent(final Incommon updater,
264 final String columnFamily) throws IOException {
265 return addContent(updater, columnFamily, START_KEY_BYTES, null);
266 }
267
268 protected static long addContent(final Incommon updater, final String family,
269 final String column) throws IOException {
270 return addContent(updater, family, column, START_KEY_BYTES, null);
271 }
272
273
274
275
276
277
278
279
280
281
282
283
284 protected static long addContent(final Incommon updater, final String columnFamily,
285 final byte [] startKeyBytes, final byte [] endKey)
286 throws IOException {
287 return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
288 }
289
290 protected static long addContent(final Incommon updater, final String family,
291 final String column, final byte [] startKeyBytes,
292 final byte [] endKey) throws IOException {
293 return addContent(updater, family, column, startKeyBytes, endKey, -1);
294 }
295
296
297
298
299
300
301
302
303
304
305
306
307
308 protected static long addContent(final Incommon updater,
309 final String columnFamily,
310 final String column,
311 final byte [] startKeyBytes, final byte [] endKey, final long ts)
312 throws IOException {
313 long count = 0;
314
315
316
317
318 char secondCharStart = (char)startKeyBytes[1];
319 char thirdCharStart = (char)startKeyBytes[2];
320 EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
321 for (char d = secondCharStart; d <= LAST_CHAR; d++) {
322 for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
323 byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
324 if (endKey != null && endKey.length > 0
325 && Bytes.compareTo(endKey, t) <= 0) {
326 break EXIT;
327 }
328 try {
329 Put put;
330 if(ts != -1) {
331 put = new Put(t, ts);
332 } else {
333 put = new Put(t);
334 }
335 try {
336 StringBuilder sb = new StringBuilder();
337 if (column != null && column.contains(":")) {
338 sb.append(column);
339 } else {
340 if (columnFamily != null) {
341 sb.append(columnFamily);
342 if (!columnFamily.endsWith(":")) {
343 sb.append(":");
344 }
345 if (column != null) {
346 sb.append(column);
347 }
348 }
349 }
350 byte[][] split =
351 KeyValue.parseColumn(Bytes.toBytes(sb.toString()));
352 if(split.length == 1) {
353 put.add(split[0], new byte[0], t);
354 } else {
355 put.add(split[0], split[1], t);
356 }
357 updater.put(put);
358 count++;
359 } catch (RuntimeException ex) {
360 ex.printStackTrace();
361 throw ex;
362 } catch (IOException ex) {
363 ex.printStackTrace();
364 throw ex;
365 }
366 } catch (RuntimeException ex) {
367 ex.printStackTrace();
368 throw ex;
369 } catch (IOException ex) {
370 ex.printStackTrace();
371 throw ex;
372 }
373 }
374
375 thirdCharStart = FIRST_CHAR;
376 }
377 secondCharStart = FIRST_CHAR;
378 }
379 return count;
380 }
381
382
383
384
385 public interface FlushCache {
386
387
388
389 void flushcache() throws IOException;
390 }
391
392
393
394
395
396
397
398 public interface Incommon {
399
400
401
402
403
404
405 void delete(Delete delete, boolean writeToWAL)
406 throws IOException;
407
408
409
410
411
412 void put(Put put) throws IOException;
413
414 Result get(Get get) throws IOException;
415
416
417
418
419
420
421
422
423
424 ScannerIncommon getScanner(
425 byte[] family, byte[][] qualifiers, byte[] firstRow, long ts
426 )
427 throws IOException;
428 }
429
430
431
432
433 public static class HRegionIncommon implements Incommon, FlushCache {
434 final HRegion region;
435
436
437
438
439 public HRegionIncommon(final HRegion HRegion) {
440 this.region = HRegion;
441 }
442
443 public void put(Put put) throws IOException {
444 region.put(put);
445 }
446
447 public void delete(Delete delete, boolean writeToWAL)
448 throws IOException {
449 this.region.delete(delete);
450 }
451
452 public Result get(Get get) throws IOException {
453 return region.get(get);
454 }
455
456 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
457 byte [] firstRow, long ts)
458 throws IOException {
459 Scan scan = new Scan(firstRow);
460 if(qualifiers == null || qualifiers.length == 0) {
461 scan.addFamily(family);
462 } else {
463 for(int i=0; i<qualifiers.length; i++){
464 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
465 }
466 }
467 scan.setTimeRange(0, ts);
468 return new
469 InternalScannerIncommon(region.getScanner(scan));
470 }
471
472 public void flushcache() throws IOException {
473 this.region.flushcache();
474 }
475 }
476
477
478
479
480 public static class HTableIncommon implements Incommon {
481 final HTable table;
482
483
484
485
486 public HTableIncommon(final HTable table) {
487 super();
488 this.table = table;
489 }
490
491 public void put(Put put) throws IOException {
492 table.put(put);
493 }
494
495
496 public void delete(Delete delete, boolean writeToWAL)
497 throws IOException {
498 this.table.delete(delete);
499 }
500
501 public Result get(Get get) throws IOException {
502 return table.get(get);
503 }
504
505 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
506 byte [] firstRow, long ts)
507 throws IOException {
508 Scan scan = new Scan(firstRow);
509 if(qualifiers == null || qualifiers.length == 0) {
510 scan.addFamily(family);
511 } else {
512 for(int i=0; i<qualifiers.length; i++){
513 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
514 }
515 }
516 scan.setTimeRange(0, ts);
517 return new
518 ClientScannerIncommon(table.getScanner(scan));
519 }
520 }
521
522 public interface ScannerIncommon
523 extends Iterable<Result> {
524 boolean next(List<KeyValue> values)
525 throws IOException;
526
527 void close() throws IOException;
528 }
529
530 public static class ClientScannerIncommon implements ScannerIncommon {
531 ResultScanner scanner;
532 public ClientScannerIncommon(ResultScanner scanner) {
533 this.scanner = scanner;
534 }
535
536 public boolean next(List<KeyValue> values)
537 throws IOException {
538 Result results = scanner.next();
539 if (results == null) {
540 return false;
541 }
542 values.clear();
543 values.addAll(results.list());
544 return true;
545 }
546
547 public void close() throws IOException {
548 scanner.close();
549 }
550
551 @SuppressWarnings("unchecked")
552 public Iterator iterator() {
553 return scanner.iterator();
554 }
555 }
556
557 public static class InternalScannerIncommon implements ScannerIncommon {
558 InternalScanner scanner;
559
560 public InternalScannerIncommon(InternalScanner scanner) {
561 this.scanner = scanner;
562 }
563
564 public boolean next(List<KeyValue> results)
565 throws IOException {
566 return scanner.next(results);
567 }
568
569 public void close() throws IOException {
570 scanner.close();
571 }
572
573 public Iterator<Result> iterator() {
574 throw new UnsupportedOperationException();
575 }
576 }
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598 protected void assertResultEquals(final HRegion region, final byte [] row,
599 final byte [] family, final byte [] qualifier, final long timestamp,
600 final byte [] value)
601 throws IOException {
602 Get get = new Get(row);
603 get.setTimeStamp(timestamp);
604 Result res = region.get(get);
605 NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
606 res.getMap();
607 byte [] res_value = map.get(family).get(qualifier).get(timestamp);
608
609 if (value == null) {
610 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
611 " at timestamp " + timestamp, null, res_value);
612 } else {
613 if (res_value == null) {
614 fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
615 " at timestamp " + timestamp + "\" was expected to be \"" +
616 Bytes.toStringBinary(value) + " but was null");
617 }
618 if (res_value != null) {
619 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
620 " at timestamp " +
621 timestamp, value, new String(res_value));
622 }
623 }
624 }
625
626
627
628
629
630
631 public static void shutdownDfs(MiniDFSCluster cluster) {
632 if (cluster != null) {
633 LOG.info("Shutting down Mini DFS ");
634 try {
635 cluster.shutdown();
636 } catch (Exception e) {
637
638
639
640 }
641 try {
642 FileSystem fs = cluster.getFileSystem();
643 if (fs != null) {
644 LOG.info("Shutting down FileSystem");
645 fs.close();
646 }
647 FileSystem.closeAll();
648 } catch (IOException e) {
649 LOG.error("error closing file system", e);
650 }
651 }
652 }
653
654
655
656
657
658
659 protected void createMetaRegion() throws IOException {
660 meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir,
661 conf, HTableDescriptor.META_TABLEDESC);
662 }
663
664 protected void closeRootAndMeta() throws IOException {
665 HRegion.closeHRegion(meta);
666 }
667
668 public static void assertByteEquals(byte[] expected,
669 byte[] actual) {
670 if (Bytes.compareTo(expected, actual) != 0) {
671 throw new AssertionFailedError("expected:<" +
672 Bytes.toString(expected) + "> but was:<" +
673 Bytes.toString(actual) + ">");
674 }
675 }
676
677 public static void assertEquals(byte[] expected,
678 byte[] actual) {
679 if (Bytes.compareTo(expected, actual) != 0) {
680 throw new AssertionFailedError("expected:<" +
681 Bytes.toStringBinary(expected) + "> but was:<" +
682 Bytes.toStringBinary(actual) + ">");
683 }
684 }
685
686 }