1   /**
2    * Copyright 2007 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase;
21  
22  import java.io.File;
23  import java.io.IOException;
24  import java.io.UnsupportedEncodingException;
25  import java.util.Iterator;
26  import java.util.List;
27  import java.util.NavigableMap;
28  
29  import junit.framework.AssertionFailedError;
30  import junit.framework.TestCase;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.fs.FileSystem;
36  import org.apache.hadoop.fs.Path;
37  import org.apache.hadoop.hbase.client.Delete;
38  import org.apache.hadoop.hbase.client.Get;
39  import org.apache.hadoop.hbase.client.HTable;
40  import org.apache.hadoop.hbase.client.Put;
41  import org.apache.hadoop.hbase.client.Result;
42  import org.apache.hadoop.hbase.client.ResultScanner;
43  import org.apache.hadoop.hbase.client.Scan;
44  import org.apache.hadoop.hbase.regionserver.HRegion;
45  import org.apache.hadoop.hbase.regionserver.InternalScanner;
46  import org.apache.hadoop.hbase.util.Bytes;
47  import org.apache.hadoop.hdfs.MiniDFSCluster;
48  
49  /**
50   * Abstract HBase test class.  Initializes a few things that can come in handly
51   * like an HBaseConfiguration and filesystem.
52   * @deprecated Write junit4 unit tests using {@link HBaseTestingUtility}
53   */
54  public abstract class HBaseTestCase extends TestCase {
55    private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
56  
57    /** configuration parameter name for test directory */
58    public static final String TEST_DIRECTORY_KEY = "test.build.data";
59  
60    protected final static byte [] fam1 = Bytes.toBytes("colfamily1");
61    protected final static byte [] fam2 = Bytes.toBytes("colfamily2");
62    protected final static byte [] fam3 = Bytes.toBytes("colfamily3");
63    protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
64  
65    private boolean localfs = false;
66    protected Path testDir = null;
67    protected FileSystem fs = null;
68    protected HRegion root = null;
69    protected HRegion meta = null;
70    protected static final char FIRST_CHAR = 'a';
71    protected static final char LAST_CHAR = 'z';
72    protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
73    protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
74    protected String START_KEY;
75    protected static final int MAXVERSIONS = 3;
76  
77    static {
78      initialize();
79    }
80  
81    public volatile Configuration conf;
82  
83    /** constructor */
84    public HBaseTestCase() {
85      super();
86      init();
87    }
88  
89    /**
90     * @param name
91     */
92    public HBaseTestCase(String name) {
93      super(name);
94      init();
95    }
96  
97    private void init() {
98      conf = HBaseConfiguration.create();
99      try {
100       START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_ENCODING);
101     } catch (UnsupportedEncodingException e) {
102       LOG.fatal("error during initialization", e);
103       fail();
104     }
105   }
106 
107   /**
108    * Note that this method must be called after the mini hdfs cluster has
109    * started or we end up with a local file system.
110    */
111   @Override
112   protected void setUp() throws Exception {
113     super.setUp();
114     localfs =
115       (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
116 
117     if (fs == null) {
118       this.fs = FileSystem.get(conf);
119     }
120     try {
121       if (localfs) {
122         this.testDir = getUnitTestdir(getName());
123         if (fs.exists(testDir)) {
124           fs.delete(testDir, true);
125         }
126       } else {
127         this.testDir =
128           this.fs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR)));
129       }
130     } catch (Exception e) {
131       LOG.fatal("error during setup", e);
132       throw e;
133     }
134   }
135 
136   @Override
137   protected void tearDown() throws Exception {
138     try {
139       if (localfs) {
140         if (this.fs.exists(testDir)) {
141           this.fs.delete(testDir, true);
142         }
143       }
144     } catch (Exception e) {
145       LOG.fatal("error during tear down", e);
146     }
147     super.tearDown();
148   }
149 
150   protected Path getUnitTestdir(String testName) {
151     return new Path(
152         conf.get(TEST_DIRECTORY_KEY, "target/test/data"), testName);
153   }
154 
155   protected HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
156       byte [] endKey)
157   throws IOException {
158     FileSystem filesystem = FileSystem.get(conf);
159     Path rootdir = filesystem.makeQualified(
160         new Path(conf.get(HConstants.HBASE_DIR)));
161     filesystem.mkdirs(rootdir);
162 
163     return HRegion.createHRegion(new HRegionInfo(desc, startKey, endKey),
164         rootdir, conf);
165   }
166 
167   protected HRegion openClosedRegion(final HRegion closedRegion)
168   throws IOException {
169     HRegion r = new HRegion(closedRegion.getTableDir(), closedRegion.getLog(),
170         closedRegion.getFilesystem(), closedRegion.getConf(),
171         closedRegion.getRegionInfo(), null);
172     r.initialize();
173     return r;
174   }
175 
176   /**
177    * Create a table of name <code>name</code> with {@link COLUMNS} for
178    * families.
179    * @param name Name to give table.
180    * @return Column descriptor.
181    */
182   protected HTableDescriptor createTableDescriptor(final String name) {
183     return createTableDescriptor(name, MAXVERSIONS);
184   }
185 
186   /**
187    * Create a table of name <code>name</code> with {@link COLUMNS} for
188    * families.
189    * @param name Name to give table.
190    * @param versions How many versions to allow per column.
191    * @return Column descriptor.
192    */
193   protected HTableDescriptor createTableDescriptor(final String name,
194       final int versions) {
195     HTableDescriptor htd = new HTableDescriptor(name);
196     htd.addFamily(new HColumnDescriptor(fam1, versions,
197       HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
198       Integer.MAX_VALUE, HConstants.FOREVER, 
199       HColumnDescriptor.DEFAULT_BLOOMFILTER,
200       HConstants.REPLICATION_SCOPE_LOCAL));
201     htd.addFamily(new HColumnDescriptor(fam2, versions,
202         HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
203         Integer.MAX_VALUE, HConstants.FOREVER,
204         HColumnDescriptor.DEFAULT_BLOOMFILTER,
205         HConstants.REPLICATION_SCOPE_LOCAL));
206     htd.addFamily(new HColumnDescriptor(fam3, versions,
207         HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
208         Integer.MAX_VALUE,  HConstants.FOREVER,
209         HColumnDescriptor.DEFAULT_BLOOMFILTER,
210         HConstants.REPLICATION_SCOPE_LOCAL));
211     return htd;
212   }
213 
214   /**
215    * Add content to region <code>r</code> on the passed column
216    * <code>column</code>.
217    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
218    * @param r
219    * @param columnFamily
220    * @throws IOException
221    * @return count of what we added.
222    */
223   protected static long addContent(final HRegion r, final byte [] columnFamily)
224   throws IOException {
225     byte [] startKey = r.getRegionInfo().getStartKey();
226     byte [] endKey = r.getRegionInfo().getEndKey();
227     byte [] startKeyBytes = startKey;
228     if (startKeyBytes == null || startKeyBytes.length == 0) {
229       startKeyBytes = START_KEY_BYTES;
230     }
231     return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), null,
232       startKeyBytes, endKey, -1);
233   }
234 
235   /**
236    * Add content to region <code>r</code> on the passed column
237    * <code>column</code>.
238    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
239    * @param updater  An instance of {@link Incommon}.
240    * @param columnFamily
241    * @throws IOException
242    * @return count of what we added.
243    */
244   protected static long addContent(final Incommon updater,
245                                    final String columnFamily) throws IOException {
246     return addContent(updater, columnFamily, START_KEY_BYTES, null);
247   }
248 
249   protected static long addContent(final Incommon updater, final String family,
250                                    final String column) throws IOException {
251     return addContent(updater, family, column, START_KEY_BYTES, null);
252   }
253 
254   /**
255    * Add content to region <code>r</code> on the passed column
256    * <code>column</code>.
257    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
258    * @param updater  An instance of {@link Incommon}.
259    * @param columnFamily
260    * @param startKeyBytes Where to start the rows inserted
261    * @param endKey Where to stop inserting rows.
262    * @return count of what we added.
263    * @throws IOException
264    */
265   protected static long addContent(final Incommon updater, final String columnFamily,
266       final byte [] startKeyBytes, final byte [] endKey)
267   throws IOException {
268     return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
269   }
270 
271   protected static long addContent(final Incommon updater, final String family,
272                                    final String column, final byte [] startKeyBytes,
273                                    final byte [] endKey) throws IOException {
274     return addContent(updater, family, column, startKeyBytes, endKey, -1);
275   }
276 
277   /**
278    * Add content to region <code>r</code> on the passed column
279    * <code>column</code>.
280    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
281    * @param updater  An instance of {@link Incommon}.
282    * @param column
283    * @param startKeyBytes Where to start the rows inserted
284    * @param endKey Where to stop inserting rows.
285    * @param ts Timestamp to write the content with.
286    * @return count of what we added.
287    * @throws IOException
288    */
289   protected static long addContent(final Incommon updater,
290                                    final String columnFamily,
291                                    final String column,
292       final byte [] startKeyBytes, final byte [] endKey, final long ts)
293   throws IOException {
294     long count = 0;
295     // Add rows of three characters.  The first character starts with the
296     // 'a' character and runs up to 'z'.  Per first character, we run the
297     // second character over same range.  And same for the third so rows
298     // (and values) look like this: 'aaa', 'aab', 'aac', etc.
299     char secondCharStart = (char)startKeyBytes[1];
300     char thirdCharStart = (char)startKeyBytes[2];
301     EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
302       for (char d = secondCharStart; d <= LAST_CHAR; d++) {
303         for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
304           byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
305           if (endKey != null && endKey.length > 0
306               && Bytes.compareTo(endKey, t) <= 0) {
307             break EXIT;
308           }
309           try {
310             Put put;
311             if(ts != -1) {
312               put = new Put(t, ts, null);
313             } else {
314               put = new Put(t);
315             }
316             try {
317               StringBuilder sb = new StringBuilder();
318               if (column != null && column.contains(":")) {
319                 sb.append(column);
320               } else {
321                 if (columnFamily != null) {
322                   sb.append(columnFamily);
323                   if (!columnFamily.endsWith(":")) {
324                     sb.append(":");
325                   }
326                   if (column != null) {
327                     sb.append(column);
328                   }
329                 }
330               }
331               byte[][] split =
332                 KeyValue.parseColumn(Bytes.toBytes(sb.toString()));
333               if(split.length == 1) {
334                 put.add(split[0], new byte[0], t);
335               } else {
336                 put.add(split[0], split[1], t);
337               }
338               updater.put(put);
339               count++;
340             } catch (RuntimeException ex) {
341               ex.printStackTrace();
342               throw ex;
343             } catch (IOException ex) {
344               ex.printStackTrace();
345               throw ex;
346             }
347           } catch (RuntimeException ex) {
348             ex.printStackTrace();
349             throw ex;
350           } catch (IOException ex) {
351             ex.printStackTrace();
352             throw ex;
353           }
354         }
355         // Set start character back to FIRST_CHAR after we've done first loop.
356         thirdCharStart = FIRST_CHAR;
357       }
358       secondCharStart = FIRST_CHAR;
359     }
360     return count;
361   }
362 
363   /**
364    * Implementors can flushcache.
365    */
366   public static interface FlushCache {
367     /**
368      * @throws IOException
369      */
370     public void flushcache() throws IOException;
371   }
372 
373   /**
374    * Interface used by tests so can do common operations against an HTable
375    * or an HRegion.
376    *
377    * TOOD: Come up w/ a better name for this interface.
378    */
379   public static interface Incommon {
380     /**
381      *
382      * @param delete
383      * @param lockid
384      * @param writeToWAL
385      * @throws IOException
386      */
387     public void delete(Delete delete,  Integer lockid, boolean writeToWAL)
388     throws IOException;
389 
390     /**
391      * @param put
392      * @throws IOException
393      */
394     public void put(Put put) throws IOException;
395 
396     public Result get(Get get) throws IOException;
397 
398     /**
399      * @param family
400      * @param qualifiers
401      * @param firstRow
402      * @param ts
403      * @return scanner for specified columns, first row and timestamp
404      * @throws IOException
405      */
406     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
407         byte [] firstRow, long ts)
408     throws IOException;
409   }
410 
411   /**
412    * A class that makes a {@link Incommon} out of a {@link HRegion}
413    */
414   public static class HRegionIncommon implements Incommon, FlushCache {
415     final HRegion region;
416 
417     /**
418      * @param HRegion
419      */
420     public HRegionIncommon(final HRegion HRegion) {
421       this.region = HRegion;
422     }
423 
424     public void put(Put put) throws IOException {
425       region.put(put);
426     }
427 
428     public void delete(Delete delete,  Integer lockid, boolean writeToWAL)
429     throws IOException {
430       this.region.delete(delete, lockid, writeToWAL);
431     }
432 
433     public Result get(Get get) throws IOException {
434       return region.get(get, null);
435     }
436 
437     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
438         byte [] firstRow, long ts)
439       throws IOException {
440         Scan scan = new Scan(firstRow);
441         if(qualifiers == null || qualifiers.length == 0) {
442           scan.addFamily(family);
443         } else {
444           for(int i=0; i<qualifiers.length; i++){
445             scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
446           }
447         }
448         scan.setTimeRange(0, ts);
449         return new
450           InternalScannerIncommon(region.getScanner(scan));
451       }
452 
453     public Result get(Get get, Integer lockid) throws IOException{
454       return this.region.get(get, lockid);
455     }
456 
457 
458     public void flushcache() throws IOException {
459       this.region.flushcache();
460     }
461   }
462 
463   /**
464    * A class that makes a {@link Incommon} out of a {@link HTable}
465    */
466   public static class HTableIncommon implements Incommon {
467     final HTable table;
468 
469     /**
470      * @param table
471      */
472     public HTableIncommon(final HTable table) {
473       super();
474       this.table = table;
475     }
476 
477     public void put(Put put) throws IOException {
478       table.put(put);
479     }
480 
481 
482     public void delete(Delete delete,  Integer lockid, boolean writeToWAL)
483     throws IOException {
484       this.table.delete(delete);
485     }
486 
487     public Result get(Get get) throws IOException {
488       return table.get(get);
489     }
490 
491     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
492         byte [] firstRow, long ts)
493       throws IOException {
494       Scan scan = new Scan(firstRow);
495       if(qualifiers == null || qualifiers.length == 0) {
496         scan.addFamily(family);
497       } else {
498         for(int i=0; i<qualifiers.length; i++){
499           scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
500         }
501       }
502       scan.setTimeRange(0, ts);
503       return new
504         ClientScannerIncommon(table.getScanner(scan));
505     }
506   }
507 
508   public interface ScannerIncommon
509   extends Iterable<Result> {
510     public boolean next(List<KeyValue> values)
511     throws IOException;
512 
513     public void close() throws IOException;
514   }
515 
516   public static class ClientScannerIncommon implements ScannerIncommon {
517     ResultScanner scanner;
518     public ClientScannerIncommon(ResultScanner scanner) {
519       this.scanner = scanner;
520     }
521 
522     public boolean next(List<KeyValue> values)
523     throws IOException {
524       Result results = scanner.next();
525       if (results == null) {
526         return false;
527       }
528       values.clear();
529       values.addAll(results.list());
530       return true;
531     }
532 
533     public void close() throws IOException {
534       scanner.close();
535     }
536 
537     @SuppressWarnings("unchecked")
538     public Iterator iterator() {
539       return scanner.iterator();
540     }
541   }
542 
543   public static class InternalScannerIncommon implements ScannerIncommon {
544     InternalScanner scanner;
545 
546     public InternalScannerIncommon(InternalScanner scanner) {
547       this.scanner = scanner;
548     }
549 
550     public boolean next(List<KeyValue> results)
551     throws IOException {
552       return scanner.next(results);
553     }
554 
555     public void close() throws IOException {
556       scanner.close();
557     }
558 
559     public Iterator<Result> iterator() {
560       throw new UnsupportedOperationException();
561     }
562   }
563 
564 //  protected void assertCellEquals(final HRegion region, final byte [] row,
565 //    final byte [] column, final long timestamp, final String value)
566 //  throws IOException {
567 //    Map<byte [], Cell> result = region.getFull(row, null, timestamp, 1, null);
568 //    Cell cell_value = result.get(column);
569 //    if (value == null) {
570 //      assertEquals(Bytes.toString(column) + " at timestamp " + timestamp, null,
571 //        cell_value);
572 //    } else {
573 //      if (cell_value == null) {
574 //        fail(Bytes.toString(column) + " at timestamp " + timestamp +
575 //          "\" was expected to be \"" + value + " but was null");
576 //      }
577 //      if (cell_value != null) {
578 //        assertEquals(Bytes.toString(column) + " at timestamp "
579 //            + timestamp, value, new String(cell_value.getValue()));
580 //      }
581 //    }
582 //  }
583 
584   protected void assertResultEquals(final HRegion region, final byte [] row,
585       final byte [] family, final byte [] qualifier, final long timestamp,
586       final byte [] value)
587     throws IOException {
588       Get get = new Get(row);
589       get.setTimeStamp(timestamp);
590       Result res = region.get(get, null);
591       NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
592         res.getMap();
593       byte [] res_value = map.get(family).get(qualifier).get(timestamp);
594 
595       if (value == null) {
596         assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
597             " at timestamp " + timestamp, null, res_value);
598       } else {
599         if (res_value == null) {
600           fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
601               " at timestamp " + timestamp + "\" was expected to be \"" +
602               Bytes.toStringBinary(value) + " but was null");
603         }
604         if (res_value != null) {
605           assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
606               " at timestamp " +
607               timestamp, value, new String(res_value));
608         }
609       }
610     }
611 
612   /**
613    * Initializes parameters used in the test environment:
614    *
615    * Sets the configuration parameter TEST_DIRECTORY_KEY if not already set.
616    * Sets the boolean debugging if "DEBUGGING" is set in the environment.
617    * If debugging is enabled, reconfigures logging so that the root log level is
618    * set to WARN and the logging level for the package is set to DEBUG.
619    */
620   public static void initialize() {
621     if (System.getProperty(TEST_DIRECTORY_KEY) == null) {
622       System.setProperty(TEST_DIRECTORY_KEY, new File(
623           "build/hbase/test").getAbsolutePath());
624     }
625   }
626 
627   /**
628    * Common method to close down a MiniDFSCluster and the associated file system
629    *
630    * @param cluster
631    */
632   public static void shutdownDfs(MiniDFSCluster cluster) {
633     if (cluster != null) {
634       LOG.info("Shutting down Mini DFS ");
635       try {
636         cluster.shutdown();
637       } catch (Exception e) {
638         /// Can get a java.lang.reflect.UndeclaredThrowableException thrown
639         // here because of an InterruptedException. Don't let exceptions in
640         // here be cause of test failure.
641       }
642       try {
643         FileSystem fs = cluster.getFileSystem();
644         if (fs != null) {
645           LOG.info("Shutting down FileSystem");
646           fs.close();
647         }
648         FileSystem.closeAll();
649       } catch (IOException e) {
650         LOG.error("error closing file system", e);
651       }
652     }
653   }
654 
655   protected void createRootAndMetaRegions() throws IOException {
656     root = HRegion.createHRegion(HRegionInfo.ROOT_REGIONINFO, testDir, conf);
657     meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir,
658         conf);
659     HRegion.addRegionToMETA(root, meta);
660   }
661 
662   protected void closeRootAndMeta() throws IOException {
663     if (meta != null) {
664       meta.close();
665       meta.getLog().closeAndDelete();
666     }
667     if (root != null) {
668       root.close();
669       root.getLog().closeAndDelete();
670     }
671   }
672 
673   public static void assertByteEquals(byte[] expected,
674                                byte[] actual) {
675     if (Bytes.compareTo(expected, actual) != 0) {
676       throw new AssertionFailedError("expected:<" +
677       Bytes.toString(expected) + "> but was:<" +
678       Bytes.toString(actual) + ">");
679     }
680   }
681 
682   public static void assertEquals(byte[] expected,
683                                byte[] actual) {
684     if (Bytes.compareTo(expected, actual) != 0) {
685       throw new AssertionFailedError("expected:<" +
686       Bytes.toStringBinary(expected) + "> but was:<" +
687       Bytes.toStringBinary(actual) + ">");
688     }
689   }
690 
691 }