1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase;
21
22 import java.io.File;
23 import java.io.IOException;
24 import java.io.UnsupportedEncodingException;
25 import java.util.Iterator;
26 import java.util.List;
27 import java.util.NavigableMap;
28
29 import junit.framework.AssertionFailedError;
30 import junit.framework.TestCase;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.fs.FileSystem;
35 import org.apache.hadoop.fs.Path;
36 import org.apache.hadoop.hbase.client.Delete;
37 import org.apache.hadoop.hbase.client.Get;
38 import org.apache.hadoop.hbase.client.HTable;
39 import org.apache.hadoop.hbase.client.Put;
40 import org.apache.hadoop.hbase.client.Result;
41 import org.apache.hadoop.hbase.client.ResultScanner;
42 import org.apache.hadoop.hbase.client.Scan;
43 import org.apache.hadoop.hbase.regionserver.HRegion;
44 import org.apache.hadoop.hbase.regionserver.InternalScanner;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.hdfs.MiniDFSCluster;
47
48
49
50
51
52
53 public abstract class HBaseTestCase extends TestCase {
54 private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
55
56
57 public static final String TEST_DIRECTORY_KEY = "test.build.data";
58
59 protected final static byte [] fam1 = Bytes.toBytes("colfamily1");
60 protected final static byte [] fam2 = Bytes.toBytes("colfamily2");
61 protected final static byte [] fam3 = Bytes.toBytes("colfamily3");
62 protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
63
64 private boolean localfs = false;
65 protected Path testDir = null;
66 protected FileSystem fs = null;
67 protected HRegion root = null;
68 protected HRegion meta = null;
69 protected static final char FIRST_CHAR = 'a';
70 protected static final char LAST_CHAR = 'z';
71 protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
72 protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
73 protected String START_KEY;
74 protected static final int MAXVERSIONS = 3;
75
76 static {
77 initialize();
78 }
79
80 public volatile HBaseConfiguration conf;
81
82
83 public HBaseTestCase() {
84 super();
85 init();
86 }
87
88
89
90
91 public HBaseTestCase(String name) {
92 super(name);
93 init();
94 }
95
96 private void init() {
97 conf = new HBaseConfiguration();
98 try {
99 START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_ENCODING);
100 } catch (UnsupportedEncodingException e) {
101 LOG.fatal("error during initialization", e);
102 fail();
103 }
104 }
105
106
107
108
109
110 @Override
111 protected void setUp() throws Exception {
112 super.setUp();
113 localfs =
114 (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
115
116 if (fs == null) {
117 this.fs = FileSystem.get(conf);
118 }
119 try {
120 if (localfs) {
121 this.testDir = getUnitTestdir(getName());
122 if (fs.exists(testDir)) {
123 fs.delete(testDir, true);
124 }
125 } else {
126 this.testDir =
127 this.fs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR)));
128 }
129 } catch (Exception e) {
130 LOG.fatal("error during setup", e);
131 throw e;
132 }
133 }
134
135 @Override
136 protected void tearDown() throws Exception {
137 try {
138 if (localfs) {
139 if (this.fs.exists(testDir)) {
140 this.fs.delete(testDir, true);
141 }
142 }
143 } catch (Exception e) {
144 LOG.fatal("error during tear down", e);
145 }
146 super.tearDown();
147 }
148
149 protected Path getUnitTestdir(String testName) {
150 return new Path(
151 conf.get(TEST_DIRECTORY_KEY, "target/test/data"), testName);
152 }
153
154 protected HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
155 byte [] endKey)
156 throws IOException {
157 FileSystem filesystem = FileSystem.get(conf);
158 Path rootdir = filesystem.makeQualified(
159 new Path(conf.get(HConstants.HBASE_DIR)));
160 filesystem.mkdirs(rootdir);
161
162 return HRegion.createHRegion(new HRegionInfo(desc, startKey, endKey),
163 rootdir, conf);
164 }
165
166 protected HRegion openClosedRegion(final HRegion closedRegion)
167 throws IOException {
168 HRegion r = new HRegion(closedRegion.getTableDir(), closedRegion.getLog(),
169 closedRegion.getFilesystem(), closedRegion.getConf(),
170 closedRegion.getRegionInfo(), null);
171 r.initialize();
172 return r;
173 }
174
175
176
177
178
179
180
181 protected HTableDescriptor createTableDescriptor(final String name) {
182 return createTableDescriptor(name, MAXVERSIONS);
183 }
184
185
186
187
188
189
190
191
192 protected HTableDescriptor createTableDescriptor(final String name,
193 final int versions) {
194 HTableDescriptor htd = new HTableDescriptor(name);
195 htd.addFamily(new HColumnDescriptor(fam1, versions,
196 HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
197 Integer.MAX_VALUE, HConstants.FOREVER,
198 HColumnDescriptor.DEFAULT_BLOOMFILTER,
199 HConstants.REPLICATION_SCOPE_LOCAL));
200 htd.addFamily(new HColumnDescriptor(fam2, versions,
201 HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
202 Integer.MAX_VALUE, HConstants.FOREVER,
203 HColumnDescriptor.DEFAULT_BLOOMFILTER,
204 HConstants.REPLICATION_SCOPE_LOCAL));
205 htd.addFamily(new HColumnDescriptor(fam3, versions,
206 HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
207 Integer.MAX_VALUE, HConstants.FOREVER,
208 HColumnDescriptor.DEFAULT_BLOOMFILTER,
209 HConstants.REPLICATION_SCOPE_LOCAL));
210 return htd;
211 }
212
213
214
215
216
217
218
219
220
221
222 protected static long addContent(final HRegion r, final byte [] columnFamily)
223 throws IOException {
224 byte [] startKey = r.getRegionInfo().getStartKey();
225 byte [] endKey = r.getRegionInfo().getEndKey();
226 byte [] startKeyBytes = startKey;
227 if (startKeyBytes == null || startKeyBytes.length == 0) {
228 startKeyBytes = START_KEY_BYTES;
229 }
230 return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), null,
231 startKeyBytes, endKey, -1);
232 }
233
234
235
236
237
238
239
240
241
242
243 protected static long addContent(final Incommon updater,
244 final String columnFamily) throws IOException {
245 return addContent(updater, columnFamily, START_KEY_BYTES, null);
246 }
247
248 protected static long addContent(final Incommon updater, final String family,
249 final String column) throws IOException {
250 return addContent(updater, family, column, START_KEY_BYTES, null);
251 }
252
253
254
255
256
257
258
259
260
261
262
263
264 protected static long addContent(final Incommon updater, final String columnFamily,
265 final byte [] startKeyBytes, final byte [] endKey)
266 throws IOException {
267 return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
268 }
269
270 protected static long addContent(final Incommon updater, final String family,
271 final String column, final byte [] startKeyBytes,
272 final byte [] endKey) throws IOException {
273 return addContent(updater, family, column, startKeyBytes, endKey, -1);
274 }
275
276
277
278
279
280
281
282
283
284
285
286
287
288 protected static long addContent(final Incommon updater,
289 final String columnFamily,
290 final String column,
291 final byte [] startKeyBytes, final byte [] endKey, final long ts)
292 throws IOException {
293 long count = 0;
294
295
296
297
298 char secondCharStart = (char)startKeyBytes[1];
299 char thirdCharStart = (char)startKeyBytes[2];
300 EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
301 for (char d = secondCharStart; d <= LAST_CHAR; d++) {
302 for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
303 byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
304 if (endKey != null && endKey.length > 0
305 && Bytes.compareTo(endKey, t) <= 0) {
306 break EXIT;
307 }
308 try {
309 Put put;
310 if(ts != -1) {
311 put = new Put(t, ts, null);
312 } else {
313 put = new Put(t);
314 }
315 try {
316 StringBuilder sb = new StringBuilder();
317 if (column != null && column.contains(":")) {
318 sb.append(column);
319 } else {
320 if (columnFamily != null) {
321 sb.append(columnFamily);
322 if (!columnFamily.endsWith(":")) {
323 sb.append(":");
324 }
325 if (column != null) {
326 sb.append(column);
327 }
328 }
329 }
330 byte[][] split =
331 KeyValue.parseColumn(Bytes.toBytes(sb.toString()));
332 if(split.length == 1) {
333 put.add(split[0], new byte[0], t);
334 } else {
335 put.add(split[0], split[1], t);
336 }
337 updater.put(put);
338 count++;
339 } catch (RuntimeException ex) {
340 ex.printStackTrace();
341 throw ex;
342 } catch (IOException ex) {
343 ex.printStackTrace();
344 throw ex;
345 }
346 } catch (RuntimeException ex) {
347 ex.printStackTrace();
348 throw ex;
349 } catch (IOException ex) {
350 ex.printStackTrace();
351 throw ex;
352 }
353 }
354
355 thirdCharStart = FIRST_CHAR;
356 }
357 secondCharStart = FIRST_CHAR;
358 }
359 return count;
360 }
361
362
363
364
365 public static interface FlushCache {
366
367
368
369 public void flushcache() throws IOException;
370 }
371
372
373
374
375
376
377
378 public static interface Incommon {
379
380
381
382
383
384
385
386 public void delete(Delete delete, Integer lockid, boolean writeToWAL)
387 throws IOException;
388
389
390
391
392
393 public void put(Put put) throws IOException;
394
395 public Result get(Get get) throws IOException;
396
397
398
399
400
401
402
403
404
405 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
406 byte [] firstRow, long ts)
407 throws IOException;
408 }
409
410
411
412
413 public static class HRegionIncommon implements Incommon, FlushCache {
414 final HRegion region;
415
416
417
418
419 public HRegionIncommon(final HRegion HRegion) {
420 this.region = HRegion;
421 }
422
423 public void put(Put put) throws IOException {
424 region.put(put);
425 }
426
427 public void delete(Delete delete, Integer lockid, boolean writeToWAL)
428 throws IOException {
429 this.region.delete(delete, lockid, writeToWAL);
430 }
431
432 public Result get(Get get) throws IOException {
433 return region.get(get, null);
434 }
435
436 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
437 byte [] firstRow, long ts)
438 throws IOException {
439 Scan scan = new Scan(firstRow);
440 if(qualifiers == null || qualifiers.length == 0) {
441 scan.addFamily(family);
442 } else {
443 for(int i=0; i<qualifiers.length; i++){
444 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
445 }
446 }
447 scan.setTimeRange(0, ts);
448 return new
449 InternalScannerIncommon(region.getScanner(scan));
450 }
451
452 public Result get(Get get, Integer lockid) throws IOException{
453 return this.region.get(get, lockid);
454 }
455
456
457 public void flushcache() throws IOException {
458 this.region.flushcache();
459 }
460 }
461
462
463
464
465 public static class HTableIncommon implements Incommon {
466 final HTable table;
467
468
469
470
471 public HTableIncommon(final HTable table) {
472 super();
473 this.table = table;
474 }
475
476 public void put(Put put) throws IOException {
477 table.put(put);
478 }
479
480
481 public void delete(Delete delete, Integer lockid, boolean writeToWAL)
482 throws IOException {
483 this.table.delete(delete);
484 }
485
486 public Result get(Get get) throws IOException {
487 return table.get(get);
488 }
489
490 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
491 byte [] firstRow, long ts)
492 throws IOException {
493 Scan scan = new Scan(firstRow);
494 if(qualifiers == null || qualifiers.length == 0) {
495 scan.addFamily(family);
496 } else {
497 for(int i=0; i<qualifiers.length; i++){
498 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
499 }
500 }
501 scan.setTimeRange(0, ts);
502 return new
503 ClientScannerIncommon(table.getScanner(scan));
504 }
505 }
506
507 public interface ScannerIncommon
508 extends Iterable<Result> {
509 public boolean next(List<KeyValue> values)
510 throws IOException;
511
512 public void close() throws IOException;
513 }
514
515 public static class ClientScannerIncommon implements ScannerIncommon {
516 ResultScanner scanner;
517 public ClientScannerIncommon(ResultScanner scanner) {
518 this.scanner = scanner;
519 }
520
521 public boolean next(List<KeyValue> values)
522 throws IOException {
523 Result results = scanner.next();
524 if (results == null) {
525 return false;
526 }
527 values.clear();
528 values.addAll(results.list());
529 return true;
530 }
531
532 public void close() throws IOException {
533 scanner.close();
534 }
535
536 @SuppressWarnings("unchecked")
537 public Iterator iterator() {
538 return scanner.iterator();
539 }
540 }
541
542 public static class InternalScannerIncommon implements ScannerIncommon {
543 InternalScanner scanner;
544
545 public InternalScannerIncommon(InternalScanner scanner) {
546 this.scanner = scanner;
547 }
548
549 public boolean next(List<KeyValue> results)
550 throws IOException {
551 return scanner.next(results);
552 }
553
554 public void close() throws IOException {
555 scanner.close();
556 }
557
558 public Iterator<Result> iterator() {
559 throw new UnsupportedOperationException();
560 }
561 }
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583 protected void assertResultEquals(final HRegion region, final byte [] row,
584 final byte [] family, final byte [] qualifier, final long timestamp,
585 final byte [] value)
586 throws IOException {
587 Get get = new Get(row);
588 get.setTimeStamp(timestamp);
589 Result res = region.get(get, null);
590 NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
591 res.getMap();
592 byte [] res_value = map.get(family).get(qualifier).get(timestamp);
593
594 if (value == null) {
595 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
596 " at timestamp " + timestamp, null, res_value);
597 } else {
598 if (res_value == null) {
599 fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
600 " at timestamp " + timestamp + "\" was expected to be \"" +
601 Bytes.toStringBinary(value) + " but was null");
602 }
603 if (res_value != null) {
604 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
605 " at timestamp " +
606 timestamp, value, new String(res_value));
607 }
608 }
609 }
610
611
612
613
614
615
616
617
618
619 public static void initialize() {
620 if (System.getProperty(TEST_DIRECTORY_KEY) == null) {
621 System.setProperty(TEST_DIRECTORY_KEY, new File(
622 "build/hbase/test").getAbsolutePath());
623 }
624 }
625
626
627
628
629
630
631 public static void shutdownDfs(MiniDFSCluster cluster) {
632 if (cluster != null) {
633 LOG.info("Shutting down Mini DFS ");
634 try {
635 cluster.shutdown();
636 } catch (Exception e) {
637
638
639
640 }
641 try {
642 FileSystem fs = cluster.getFileSystem();
643 if (fs != null) {
644 LOG.info("Shutting down FileSystem");
645 fs.close();
646 }
647 FileSystem.closeAll();
648 } catch (IOException e) {
649 LOG.error("error closing file system", e);
650 }
651 }
652 }
653
654 protected void createRootAndMetaRegions() throws IOException {
655 root = HRegion.createHRegion(HRegionInfo.ROOT_REGIONINFO, testDir, conf);
656 meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir,
657 conf);
658 HRegion.addRegionToMETA(root, meta);
659 }
660
661 protected void closeRootAndMeta() throws IOException {
662 if (meta != null) {
663 meta.close();
664 meta.getLog().closeAndDelete();
665 }
666 if (root != null) {
667 root.close();
668 root.getLog().closeAndDelete();
669 }
670 }
671
672 public void assertByteEquals(byte[] expected,
673 byte[] actual) {
674 if (Bytes.compareTo(expected, actual) != 0) {
675 throw new AssertionFailedError("expected:<" +
676 Bytes.toString(expected) + "> but was:<" +
677 Bytes.toString(actual) + ">");
678 }
679 }
680
681 public static void assertEquals(byte[] expected,
682 byte[] actual) {
683 if (Bytes.compareTo(expected, actual) != 0) {
684 throw new AssertionFailedError("expected:<" +
685 Bytes.toStringBinary(expected) + "> but was:<" +
686 Bytes.toStringBinary(actual) + ">");
687 }
688 }
689
690 }