1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import java.io.IOException;
22 import java.util.Iterator;
23 import java.util.List;
24 import java.util.NavigableMap;
25
26 import junit.framework.AssertionFailedError;
27 import junit.framework.TestCase;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.client.Delete;
35 import org.apache.hadoop.hbase.client.Get;
36 import org.apache.hadoop.hbase.client.HTable;
37 import org.apache.hadoop.hbase.client.Put;
38 import org.apache.hadoop.hbase.client.Result;
39 import org.apache.hadoop.hbase.client.ResultScanner;
40 import org.apache.hadoop.hbase.client.Scan;
41 import org.apache.hadoop.hbase.regionserver.HRegion;
42 import org.apache.hadoop.hbase.regionserver.InternalScanner;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.util.FSUtils;
45 import org.apache.hadoop.hdfs.MiniDFSCluster;
46
47
48
49
50
51
52 public abstract class HBaseTestCase extends TestCase {
53 private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
54
55 protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
56 protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
57 protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
58
59 protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
60
61 private boolean localfs = false;
62 protected static Path testDir = null;
63 protected FileSystem fs = null;
64 protected HRegion meta = null;
65 protected static final char FIRST_CHAR = 'a';
66 protected static final char LAST_CHAR = 'z';
67 protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
68 protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
69 protected String START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET);
70 protected static final int MAXVERSIONS = 3;
71
72 protected final HBaseTestingUtility testUtil = new HBaseTestingUtility();
73
74 public volatile Configuration conf = HBaseConfiguration.create();
75
76
77 public HBaseTestCase() {
78 super();
79 }
80
81
82
83
84 public HBaseTestCase(String name) {
85 super(name);
86 }
87
88
89
90
91
92 @Override
93 protected void setUp() throws Exception {
94 super.setUp();
95 localfs =
96 (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
97
98 if (fs == null) {
99 this.fs = FileSystem.get(conf);
100 }
101 try {
102 if (localfs) {
103 this.testDir = getUnitTestdir(getName());
104 if (fs.exists(testDir)) {
105 fs.delete(testDir, true);
106 }
107 } else {
108 this.testDir = FSUtils.getRootDir(conf);
109 }
110 } catch (Exception e) {
111 LOG.fatal("error during setup", e);
112 throw e;
113 }
114 }
115
116 @Override
117 protected void tearDown() throws Exception {
118 try {
119 if (localfs) {
120 if (this.fs.exists(testDir)) {
121 this.fs.delete(testDir, true);
122 }
123 }
124 } catch (Exception e) {
125 LOG.fatal("error during tear down", e);
126 }
127 super.tearDown();
128 }
129
130
131
132
133
134
135 protected Path getUnitTestdir(String testName) {
136 return testUtil.getDataTestDir(testName);
137 }
138
139
140
141
142
143
144
145
146
147
148
149 public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
150 byte [] endKey)
151 throws IOException {
152 return createNewHRegion(desc, startKey, endKey, this.conf);
153 }
154
155 public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
156 byte [] endKey, Configuration conf)
157 throws IOException {
158 HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
159 return HRegion.createHRegion(hri, testDir, conf, desc);
160 }
161
162 protected HRegion openClosedRegion(final HRegion closedRegion)
163 throws IOException {
164 return HRegion.openHRegion(closedRegion, null);
165 }
166
167
168
169
170
171
172
173 protected HTableDescriptor createTableDescriptor(final String name) {
174 return createTableDescriptor(name, MAXVERSIONS);
175 }
176
177
178
179
180
181
182
183
184 protected HTableDescriptor createTableDescriptor(final String name,
185 final int versions) {
186 return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
187 versions, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
188 }
189
190
191
192
193
194
195
196
197 protected HTableDescriptor createTableDescriptor(final String name,
198 final int minVersions, final int versions, final int ttl, boolean keepDeleted) {
199 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
200 for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
201 htd.addFamily(new HColumnDescriptor(cfName)
202 .setMinVersions(minVersions)
203 .setMaxVersions(versions)
204 .setKeepDeletedCells(keepDeleted)
205 .setBlockCacheEnabled(false)
206 .setTimeToLive(ttl)
207 );
208 }
209 return htd;
210 }
211
212
213
214
215
216
217
218
219
220
221
222 public static long addContent(final HRegion r, final byte [] columnFamily, final byte[] column)
223 throws IOException {
224 byte [] startKey = r.getRegionInfo().getStartKey();
225 byte [] endKey = r.getRegionInfo().getEndKey();
226 byte [] startKeyBytes = startKey;
227 if (startKeyBytes == null || startKeyBytes.length == 0) {
228 startKeyBytes = START_KEY_BYTES;
229 }
230 return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), Bytes.toString(column),
231 startKeyBytes, endKey, -1);
232 }
233
234
235
236
237
238
239
240
241
242
243 public static long addContent(final HRegion r, final byte [] columnFamily)
244 throws IOException {
245 return addContent(r, columnFamily, null);
246 }
247
248
249
250
251
252
253
254
255
256
257 public static long addContent(final Incommon updater,
258 final String columnFamily) throws IOException {
259 return addContent(updater, columnFamily, START_KEY_BYTES, null);
260 }
261
262 public static long addContent(final Incommon updater, final String family,
263 final String column) throws IOException {
264 return addContent(updater, family, column, START_KEY_BYTES, null);
265 }
266
267
268
269
270
271
272
273
274
275
276
277
278 public static long addContent(final Incommon updater, final String columnFamily,
279 final byte [] startKeyBytes, final byte [] endKey)
280 throws IOException {
281 return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
282 }
283
284 public static long addContent(final Incommon updater, final String family,
285 final String column, final byte [] startKeyBytes,
286 final byte [] endKey) throws IOException {
287 return addContent(updater, family, column, startKeyBytes, endKey, -1);
288 }
289
290
291
292
293
294
295
296
297
298
299
300
301
302 public static long addContent(final Incommon updater,
303 final String columnFamily,
304 final String column,
305 final byte [] startKeyBytes, final byte [] endKey, final long ts)
306 throws IOException {
307 long count = 0;
308
309
310
311
312 char secondCharStart = (char)startKeyBytes[1];
313 char thirdCharStart = (char)startKeyBytes[2];
314 EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
315 for (char d = secondCharStart; d <= LAST_CHAR; d++) {
316 for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
317 byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
318 if (endKey != null && endKey.length > 0
319 && Bytes.compareTo(endKey, t) <= 0) {
320 break EXIT;
321 }
322 try {
323 Put put;
324 if(ts != -1) {
325 put = new Put(t, ts);
326 } else {
327 put = new Put(t);
328 }
329 try {
330 StringBuilder sb = new StringBuilder();
331 if (column != null && column.contains(":")) {
332 sb.append(column);
333 } else {
334 if (columnFamily != null) {
335 sb.append(columnFamily);
336 if (!columnFamily.endsWith(":")) {
337 sb.append(":");
338 }
339 if (column != null) {
340 sb.append(column);
341 }
342 }
343 }
344 byte[][] split =
345 KeyValue.parseColumn(Bytes.toBytes(sb.toString()));
346 if(split.length == 1) {
347 put.add(split[0], new byte[0], t);
348 } else {
349 put.add(split[0], split[1], t);
350 }
351 updater.put(put);
352 count++;
353 } catch (RuntimeException ex) {
354 ex.printStackTrace();
355 throw ex;
356 } catch (IOException ex) {
357 ex.printStackTrace();
358 throw ex;
359 }
360 } catch (RuntimeException ex) {
361 ex.printStackTrace();
362 throw ex;
363 } catch (IOException ex) {
364 ex.printStackTrace();
365 throw ex;
366 }
367 }
368
369 thirdCharStart = FIRST_CHAR;
370 }
371 secondCharStart = FIRST_CHAR;
372 }
373 return count;
374 }
375
376
377
378
379 public interface FlushCache {
380
381
382
383 void flushcache() throws IOException;
384 }
385
386
387
388
389
390
391
392 public interface Incommon {
393
394
395
396
397
398
399 void delete(Delete delete, boolean writeToWAL)
400 throws IOException;
401
402
403
404
405
406 void put(Put put) throws IOException;
407
408 Result get(Get get) throws IOException;
409
410
411
412
413
414
415
416
417
418 ScannerIncommon getScanner(
419 byte[] family, byte[][] qualifiers, byte[] firstRow, long ts
420 )
421 throws IOException;
422 }
423
424
425
426
427 public static class HRegionIncommon implements Incommon, FlushCache {
428 final HRegion region;
429
430
431
432
433 public HRegionIncommon(final HRegion HRegion) {
434 this.region = HRegion;
435 }
436
437 public void put(Put put) throws IOException {
438 region.put(put);
439 }
440
441 public void delete(Delete delete, boolean writeToWAL)
442 throws IOException {
443 this.region.delete(delete);
444 }
445
446 public Result get(Get get) throws IOException {
447 return region.get(get);
448 }
449
450 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
451 byte [] firstRow, long ts)
452 throws IOException {
453 Scan scan = new Scan(firstRow);
454 if(qualifiers == null || qualifiers.length == 0) {
455 scan.addFamily(family);
456 } else {
457 for(int i=0; i<qualifiers.length; i++){
458 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
459 }
460 }
461 scan.setTimeRange(0, ts);
462 return new
463 InternalScannerIncommon(region.getScanner(scan));
464 }
465
466 public void flushcache() throws IOException {
467 this.region.flushcache();
468 }
469 }
470
471
472
473
474 public static class HTableIncommon implements Incommon {
475 final HTable table;
476
477
478
479
480 public HTableIncommon(final HTable table) {
481 super();
482 this.table = table;
483 }
484
485 public void put(Put put) throws IOException {
486 table.put(put);
487 }
488
489
490 public void delete(Delete delete, boolean writeToWAL)
491 throws IOException {
492 this.table.delete(delete);
493 }
494
495 public Result get(Get get) throws IOException {
496 return table.get(get);
497 }
498
499 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
500 byte [] firstRow, long ts)
501 throws IOException {
502 Scan scan = new Scan(firstRow);
503 if(qualifiers == null || qualifiers.length == 0) {
504 scan.addFamily(family);
505 } else {
506 for(int i=0; i<qualifiers.length; i++){
507 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
508 }
509 }
510 scan.setTimeRange(0, ts);
511 return new
512 ClientScannerIncommon(table.getScanner(scan));
513 }
514 }
515
516 public interface ScannerIncommon
517 extends Iterable<Result> {
518 boolean next(List<Cell> values)
519 throws IOException;
520
521 void close() throws IOException;
522 }
523
524 public static class ClientScannerIncommon implements ScannerIncommon {
525 ResultScanner scanner;
526 public ClientScannerIncommon(ResultScanner scanner) {
527 this.scanner = scanner;
528 }
529
530 @Override
531 public boolean next(List<Cell> values)
532 throws IOException {
533 Result results = scanner.next();
534 if (results == null) {
535 return false;
536 }
537 values.clear();
538 values.addAll(results.listCells());
539 return true;
540 }
541
542 public void close() throws IOException {
543 scanner.close();
544 }
545
546 @SuppressWarnings("unchecked")
547 public Iterator<Result> iterator() {
548 return scanner.iterator();
549 }
550 }
551
552 public static class InternalScannerIncommon implements ScannerIncommon {
553 InternalScanner scanner;
554
555 public InternalScannerIncommon(InternalScanner scanner) {
556 this.scanner = scanner;
557 }
558
559 @Override
560 public boolean next(List<Cell> results)
561 throws IOException {
562 return scanner.next(results);
563 }
564
565 @Override
566 public void close() throws IOException {
567 scanner.close();
568 }
569
570 @Override
571 public Iterator<Result> iterator() {
572 throw new UnsupportedOperationException();
573 }
574 }
575
576 protected void assertResultEquals(final HRegion region, final byte [] row,
577 final byte [] family, final byte [] qualifier, final long timestamp,
578 final byte [] value)
579 throws IOException {
580 Get get = new Get(row);
581 get.setTimeStamp(timestamp);
582 Result res = region.get(get);
583 NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
584 res.getMap();
585 byte [] res_value = map.get(family).get(qualifier).get(timestamp);
586
587 if (value == null) {
588 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
589 " at timestamp " + timestamp, null, res_value);
590 } else {
591 if (res_value == null) {
592 fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
593 " at timestamp " + timestamp + "\" was expected to be \"" +
594 Bytes.toStringBinary(value) + " but was null");
595 }
596 if (res_value != null) {
597 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
598 " at timestamp " +
599 timestamp, value, new String(res_value));
600 }
601 }
602 }
603
604
605
606
607
608
609 public static void shutdownDfs(MiniDFSCluster cluster) {
610 if (cluster != null) {
611 LOG.info("Shutting down Mini DFS ");
612 try {
613 cluster.shutdown();
614 } catch (Exception e) {
615
616
617
618 }
619 try {
620 FileSystem fs = cluster.getFileSystem();
621 if (fs != null) {
622 LOG.info("Shutting down FileSystem");
623 fs.close();
624 }
625 FileSystem.closeAll();
626 } catch (IOException e) {
627 LOG.error("error closing file system", e);
628 }
629 }
630 }
631
632
633
634
635
636
637 protected void createMetaRegion() throws IOException {
638 meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir,
639 conf, HTableDescriptor.META_TABLEDESC);
640 }
641
642 protected void closeRootAndMeta() throws IOException {
643 HRegion.closeHRegion(meta);
644 }
645
646 public static void assertByteEquals(byte[] expected,
647 byte[] actual) {
648 if (Bytes.compareTo(expected, actual) != 0) {
649 throw new AssertionFailedError("expected:<" +
650 Bytes.toString(expected) + "> but was:<" +
651 Bytes.toString(actual) + ">");
652 }
653 }
654
655 public static void assertEquals(byte[] expected,
656 byte[] actual) {
657 if (Bytes.compareTo(expected, actual) != 0) {
658 throw new AssertionFailedError("expected:<" +
659 Bytes.toStringBinary(expected) + "> but was:<" +
660 Bytes.toStringBinary(actual) + ">");
661 }
662 }
663
664 }