1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import java.io.IOException;
22 import java.util.Iterator;
23 import java.util.List;
24 import java.util.NavigableMap;
25
26 import junit.framework.AssertionFailedError;
27 import junit.framework.TestCase;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.client.Delete;
35 import org.apache.hadoop.hbase.client.Durability;
36 import org.apache.hadoop.hbase.client.Get;
37 import org.apache.hadoop.hbase.client.HTable;
38 import org.apache.hadoop.hbase.client.Put;
39 import org.apache.hadoop.hbase.client.Result;
40 import org.apache.hadoop.hbase.client.ResultScanner;
41 import org.apache.hadoop.hbase.client.Scan;
42 import org.apache.hadoop.hbase.client.Table;
43 import org.apache.hadoop.hbase.regionserver.HRegion;
44 import org.apache.hadoop.hbase.regionserver.InternalScanner;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.hbase.util.FSTableDescriptors;
47 import org.apache.hadoop.hbase.util.FSUtils;
48 import org.apache.hadoop.hdfs.MiniDFSCluster;
49
50
51
52
53
54
55 public abstract class HBaseTestCase extends TestCase {
56 private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
57
58 protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
59 protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
60 protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
61
62 protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
63
64 private boolean localfs = false;
65 protected static Path testDir = null;
66 protected FileSystem fs = null;
67 protected HRegion meta = null;
68 protected static final char FIRST_CHAR = 'a';
69 protected static final char LAST_CHAR = 'z';
70 protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
71 protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
72 protected String START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET);
73 protected static final int MAXVERSIONS = 3;
74
75 protected final HBaseTestingUtility testUtil = new HBaseTestingUtility();
76
77 public volatile Configuration conf = HBaseConfiguration.create();
78 public final FSTableDescriptors fsTableDescriptors;
79 {
80 try {
81 fsTableDescriptors = new FSTableDescriptors(conf);
82 } catch (IOException e) {
83 throw new RuntimeException("Failed to init descriptors", e);
84 }
85 }
86
87
88 public HBaseTestCase() {
89 super();
90 }
91
92
93
94
95 public HBaseTestCase(String name) {
96 super(name);
97 }
98
99
100
101
102
103 @Override
104 protected void setUp() throws Exception {
105 super.setUp();
106 localfs =
107 (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
108
109 if (fs == null) {
110 this.fs = FileSystem.get(conf);
111 }
112 try {
113 if (localfs) {
114 this.testDir = getUnitTestdir(getName());
115 if (fs.exists(testDir)) {
116 fs.delete(testDir, true);
117 }
118 } else {
119 this.testDir = FSUtils.getRootDir(conf);
120 }
121 } catch (Exception e) {
122 LOG.fatal("error during setup", e);
123 throw e;
124 }
125 }
126
127 @Override
128 protected void tearDown() throws Exception {
129 try {
130 if (localfs) {
131 if (this.fs.exists(testDir)) {
132 this.fs.delete(testDir, true);
133 }
134 }
135 } catch (Exception e) {
136 LOG.fatal("error during tear down", e);
137 }
138 super.tearDown();
139 }
140
141
142
143
144
145
146 protected Path getUnitTestdir(String testName) {
147 return testUtil.getDataTestDir(testName);
148 }
149
150
151
152
153
154
155
156
157
158
159
160 public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
161 byte [] endKey)
162 throws IOException {
163 return createNewHRegion(desc, startKey, endKey, this.conf);
164 }
165
166 public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
167 byte [] endKey, Configuration conf)
168 throws IOException {
169 HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
170 return HRegion.createHRegion(hri, testDir, conf, desc);
171 }
172
173 protected HRegion openClosedRegion(final HRegion closedRegion)
174 throws IOException {
175 return HRegion.openHRegion(closedRegion, null);
176 }
177
178
179
180
181
182
183
184 protected HTableDescriptor createTableDescriptor(final String name) {
185 return createTableDescriptor(name, MAXVERSIONS);
186 }
187
188
189
190
191
192
193
194
195 protected HTableDescriptor createTableDescriptor(final String name,
196 final int versions) {
197 return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
198 versions, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
199 }
200
201
202
203
204
205
206
207
208 protected HTableDescriptor createTableDescriptor(final String name,
209 final int minVersions, final int versions, final int ttl, KeepDeletedCells keepDeleted) {
210 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
211 for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
212 htd.addFamily(new HColumnDescriptor(cfName)
213 .setMinVersions(minVersions)
214 .setMaxVersions(versions)
215 .setKeepDeletedCells(keepDeleted)
216 .setBlockCacheEnabled(false)
217 .setTimeToLive(ttl)
218 );
219 }
220 return htd;
221 }
222
223
224
225
226
227
228
229
230
231
232
233 public static long addContent(final HRegion r, final byte [] columnFamily, final byte[] column)
234 throws IOException {
235 byte [] startKey = r.getRegionInfo().getStartKey();
236 byte [] endKey = r.getRegionInfo().getEndKey();
237 byte [] startKeyBytes = startKey;
238 if (startKeyBytes == null || startKeyBytes.length == 0) {
239 startKeyBytes = START_KEY_BYTES;
240 }
241 return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), Bytes.toString(column),
242 startKeyBytes, endKey, -1);
243 }
244
245 public static long addContent(final HRegion r, final byte [] columnFamily)
246 throws IOException {
247 return addContent(r, columnFamily, null);
248 }
249
250
251
252
253
254
255
256
257
258
259
260 public static long addContent(final Incommon updater,
261 final String columnFamily) throws IOException {
262 return addContent(updater, columnFamily, START_KEY_BYTES, null);
263 }
264
265 public static long addContent(final Incommon updater, final String family,
266 final String column) throws IOException {
267 return addContent(updater, family, column, START_KEY_BYTES, null);
268 }
269
270
271
272
273
274
275
276
277
278
279
280
281
282 public static long addContent(final Incommon updater, final String columnFamily,
283 final byte [] startKeyBytes, final byte [] endKey)
284 throws IOException {
285 return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
286 }
287
288 public static long addContent(final Incommon updater, final String family, String column,
289 final byte [] startKeyBytes, final byte [] endKey) throws IOException {
290 return addContent(updater, family, column, startKeyBytes, endKey, -1);
291 }
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306 public static long addContent(final Incommon updater,
307 final String columnFamily,
308 final String column,
309 final byte [] startKeyBytes, final byte [] endKey, final long ts)
310 throws IOException {
311 long count = 0;
312
313
314
315
316 char secondCharStart = (char)startKeyBytes[1];
317 char thirdCharStart = (char)startKeyBytes[2];
318 EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
319 for (char d = secondCharStart; d <= LAST_CHAR; d++) {
320 for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
321 byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
322 if (endKey != null && endKey.length > 0
323 && Bytes.compareTo(endKey, t) <= 0) {
324 break EXIT;
325 }
326 try {
327 Put put;
328 if(ts != -1) {
329 put = new Put(t, ts);
330 } else {
331 put = new Put(t);
332 }
333 try {
334 StringBuilder sb = new StringBuilder();
335 if (column != null && column.contains(":")) {
336 sb.append(column);
337 } else {
338 if (columnFamily != null) {
339 sb.append(columnFamily);
340 if (!columnFamily.endsWith(":")) {
341 sb.append(":");
342 }
343 if (column != null) {
344 sb.append(column);
345 }
346 }
347 }
348 byte[][] split =
349 KeyValue.parseColumn(Bytes.toBytes(sb.toString()));
350 if(split.length == 1) {
351 put.add(split[0], new byte[0], t);
352 } else {
353 put.add(split[0], split[1], t);
354 }
355 put.setDurability(Durability.SKIP_WAL);
356 updater.put(put);
357 count++;
358 } catch (RuntimeException ex) {
359 ex.printStackTrace();
360 throw ex;
361 } catch (IOException ex) {
362 ex.printStackTrace();
363 throw ex;
364 }
365 } catch (RuntimeException ex) {
366 ex.printStackTrace();
367 throw ex;
368 } catch (IOException ex) {
369 ex.printStackTrace();
370 throw ex;
371 }
372 }
373
374 thirdCharStart = FIRST_CHAR;
375 }
376 secondCharStart = FIRST_CHAR;
377 }
378 return count;
379 }
380
381
382
383
384 public interface FlushCache {
385
386
387
388 void flushcache() throws IOException;
389 }
390
391
392
393
394
395
396
397 public interface Incommon {
398
399
400
401
402
403
404 void delete(Delete delete, boolean writeToWAL)
405 throws IOException;
406
407
408
409
410
411 void put(Put put) throws IOException;
412
413 Result get(Get get) throws IOException;
414
415
416
417
418
419
420
421
422
423 ScannerIncommon getScanner(
424 byte[] family, byte[][] qualifiers, byte[] firstRow, long ts
425 )
426 throws IOException;
427 }
428
429
430
431
432 public static class HRegionIncommon implements Incommon, FlushCache {
433 final HRegion region;
434
435
436
437
438 public HRegionIncommon(final HRegion HRegion) {
439 this.region = HRegion;
440 }
441
442 public void put(Put put) throws IOException {
443 region.put(put);
444 }
445
446 public void delete(Delete delete, boolean writeToWAL)
447 throws IOException {
448 this.region.delete(delete);
449 }
450
451 public Result get(Get get) throws IOException {
452 return region.get(get);
453 }
454
455 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
456 byte [] firstRow, long ts)
457 throws IOException {
458 Scan scan = new Scan(firstRow);
459 if(qualifiers == null || qualifiers.length == 0) {
460 scan.addFamily(family);
461 } else {
462 for(int i=0; i<qualifiers.length; i++){
463 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
464 }
465 }
466 scan.setTimeRange(0, ts);
467 return new
468 InternalScannerIncommon(region.getScanner(scan));
469 }
470
471 public void flushcache() throws IOException {
472 this.region.flushcache();
473 }
474 }
475
476
477
478
479 public static class HTableIncommon implements Incommon {
480 final Table table;
481
482
483
484
485 public HTableIncommon(final Table table) {
486 super();
487 this.table = table;
488 }
489
490 public void put(Put put) throws IOException {
491 table.put(put);
492 }
493
494
495 public void delete(Delete delete, boolean writeToWAL)
496 throws IOException {
497 this.table.delete(delete);
498 }
499
500 public Result get(Get get) throws IOException {
501 return table.get(get);
502 }
503
504 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
505 byte [] firstRow, long ts)
506 throws IOException {
507 Scan scan = new Scan(firstRow);
508 if(qualifiers == null || qualifiers.length == 0) {
509 scan.addFamily(family);
510 } else {
511 for(int i=0; i<qualifiers.length; i++){
512 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
513 }
514 }
515 scan.setTimeRange(0, ts);
516 return new
517 ClientScannerIncommon(table.getScanner(scan));
518 }
519 }
520
521 public interface ScannerIncommon
522 extends Iterable<Result> {
523 boolean next(List<Cell> values)
524 throws IOException;
525
526 void close() throws IOException;
527 }
528
529 public static class ClientScannerIncommon implements ScannerIncommon {
530 ResultScanner scanner;
531 public ClientScannerIncommon(ResultScanner scanner) {
532 this.scanner = scanner;
533 }
534
535 @Override
536 public boolean next(List<Cell> values)
537 throws IOException {
538 Result results = scanner.next();
539 if (results == null) {
540 return false;
541 }
542 values.clear();
543 values.addAll(results.listCells());
544 return true;
545 }
546
547 public void close() throws IOException {
548 scanner.close();
549 }
550
551 public Iterator<Result> iterator() {
552 return scanner.iterator();
553 }
554 }
555
556 public static class InternalScannerIncommon implements ScannerIncommon {
557 InternalScanner scanner;
558
559 public InternalScannerIncommon(InternalScanner scanner) {
560 this.scanner = scanner;
561 }
562
563 @Override
564 public boolean next(List<Cell> results)
565 throws IOException {
566 return scanner.next(results);
567 }
568
569 @Override
570 public void close() throws IOException {
571 scanner.close();
572 }
573
574 @Override
575 public Iterator<Result> iterator() {
576 throw new UnsupportedOperationException();
577 }
578 }
579
580 protected void assertResultEquals(final HRegion region, final byte [] row,
581 final byte [] family, final byte [] qualifier, final long timestamp,
582 final byte [] value)
583 throws IOException {
584 Get get = new Get(row);
585 get.setTimeStamp(timestamp);
586 Result res = region.get(get);
587 NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
588 res.getMap();
589 byte [] res_value = map.get(family).get(qualifier).get(timestamp);
590
591 if (value == null) {
592 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
593 " at timestamp " + timestamp, null, res_value);
594 } else {
595 if (res_value == null) {
596 fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
597 " at timestamp " + timestamp + "\" was expected to be \"" +
598 Bytes.toStringBinary(value) + " but was null");
599 }
600 if (res_value != null) {
601 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
602 " at timestamp " +
603 timestamp, value, new String(res_value));
604 }
605 }
606 }
607
608
609
610
611
612
613 public static void shutdownDfs(MiniDFSCluster cluster) {
614 if (cluster != null) {
615 LOG.info("Shutting down Mini DFS ");
616 try {
617 cluster.shutdown();
618 } catch (Exception e) {
619
620
621
622 }
623 try {
624 FileSystem fs = cluster.getFileSystem();
625 if (fs != null) {
626 LOG.info("Shutting down FileSystem");
627 fs.close();
628 }
629 FileSystem.closeAll();
630 } catch (IOException e) {
631 LOG.error("error closing file system", e);
632 }
633 }
634 }
635
636
637
638
639
640
641 protected void createMetaRegion() throws IOException {
642 FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(conf);
643 meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir, conf,
644 fsTableDescriptors.get(TableName.META_TABLE_NAME));
645 }
646
647 protected void closeRootAndMeta() throws IOException {
648 HRegion.closeHRegion(meta);
649 }
650
651 public static void assertByteEquals(byte[] expected,
652 byte[] actual) {
653 if (Bytes.compareTo(expected, actual) != 0) {
654 throw new AssertionFailedError("expected:<" +
655 Bytes.toString(expected) + "> but was:<" +
656 Bytes.toString(actual) + ">");
657 }
658 }
659
660 public static void assertEquals(byte[] expected,
661 byte[] actual) {
662 if (Bytes.compareTo(expected, actual) != 0) {
663 throw new AssertionFailedError("expected:<" +
664 Bytes.toStringBinary(expected) + "> but was:<" +
665 Bytes.toStringBinary(actual) + ">");
666 }
667 }
668
669 }