1   /**
2    * Copyright 2007 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase;
21  
22  import java.io.File;
23  import java.io.IOException;
24  import java.io.UnsupportedEncodingException;
25  import java.util.Iterator;
26  import java.util.List;
27  import java.util.NavigableMap;
28  
29  import junit.framework.AssertionFailedError;
30  import junit.framework.TestCase;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.fs.FileSystem;
35  import org.apache.hadoop.fs.Path;
36  import org.apache.hadoop.hbase.client.Delete;
37  import org.apache.hadoop.hbase.client.Get;
38  import org.apache.hadoop.hbase.client.HTable;
39  import org.apache.hadoop.hbase.client.Put;
40  import org.apache.hadoop.hbase.client.Result;
41  import org.apache.hadoop.hbase.client.ResultScanner;
42  import org.apache.hadoop.hbase.client.Scan;
43  import org.apache.hadoop.hbase.regionserver.HRegion;
44  import org.apache.hadoop.hbase.regionserver.InternalScanner;
45  import org.apache.hadoop.hbase.util.Bytes;
46  import org.apache.hadoop.hdfs.MiniDFSCluster;
47  
48  /**
49   * Abstract HBase test class.  Initializes a few things that can come in handly
50   * like an HBaseConfiguration and filesystem.
51   * @deprecated Write junit4 unit tests using {@link HBaseTestingUtility}
52   */
53  public abstract class HBaseTestCase extends TestCase {
54    private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
55  
56    /** configuration parameter name for test directory */
57    public static final String TEST_DIRECTORY_KEY = "test.build.data";
58  
59    protected final static byte [] fam1 = Bytes.toBytes("colfamily1");
60    protected final static byte [] fam2 = Bytes.toBytes("colfamily2");
61    protected final static byte [] fam3 = Bytes.toBytes("colfamily3");
62    protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
63  
64    private boolean localfs = false;
65    protected Path testDir = null;
66    protected FileSystem fs = null;
67    protected HRegion root = null;
68    protected HRegion meta = null;
69    protected static final char FIRST_CHAR = 'a';
70    protected static final char LAST_CHAR = 'z';
71    protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
72    protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
73    protected String START_KEY;
74    protected static final int MAXVERSIONS = 3;
75  
76    static {
77      initialize();
78    }
79  
80    public volatile HBaseConfiguration conf;
81  
82    /** constructor */
83    public HBaseTestCase() {
84      super();
85      init();
86    }
87  
88    /**
89     * @param name
90     */
91    public HBaseTestCase(String name) {
92      super(name);
93      init();
94    }
95  
96    private void init() {
97      conf = new HBaseConfiguration();
98      try {
99        START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_ENCODING);
100     } catch (UnsupportedEncodingException e) {
101       LOG.fatal("error during initialization", e);
102       fail();
103     }
104   }
105 
106   /**
107    * Note that this method must be called after the mini hdfs cluster has
108    * started or we end up with a local file system.
109    */
110   @Override
111   protected void setUp() throws Exception {
112     super.setUp();
113     localfs =
114       (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
115 
116     if (fs == null) {
117       this.fs = FileSystem.get(conf);
118     }
119     try {
120       if (localfs) {
121         this.testDir = getUnitTestdir(getName());
122         if (fs.exists(testDir)) {
123           fs.delete(testDir, true);
124         }
125       } else {
126         this.testDir =
127           this.fs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR)));
128       }
129     } catch (Exception e) {
130       LOG.fatal("error during setup", e);
131       throw e;
132     }
133   }
134 
135   @Override
136   protected void tearDown() throws Exception {
137     try {
138       if (localfs) {
139         if (this.fs.exists(testDir)) {
140           this.fs.delete(testDir, true);
141         }
142       }
143     } catch (Exception e) {
144       LOG.fatal("error during tear down", e);
145     }
146     super.tearDown();
147   }
148 
149   protected Path getUnitTestdir(String testName) {
150     return new Path(
151         conf.get(TEST_DIRECTORY_KEY, "target/test/data"), testName);
152   }
153 
154   protected HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
155       byte [] endKey)
156   throws IOException {
157     FileSystem filesystem = FileSystem.get(conf);
158     Path rootdir = filesystem.makeQualified(
159         new Path(conf.get(HConstants.HBASE_DIR)));
160     filesystem.mkdirs(rootdir);
161 
162     return HRegion.createHRegion(new HRegionInfo(desc, startKey, endKey),
163         rootdir, conf);
164   }
165 
166   protected HRegion openClosedRegion(final HRegion closedRegion)
167   throws IOException {
168     HRegion r = new HRegion(closedRegion.getTableDir(), closedRegion.getLog(),
169         closedRegion.getFilesystem(), closedRegion.getConf(),
170         closedRegion.getRegionInfo(), null);
171     r.initialize();
172     return r;
173   }
174 
175   /**
176    * Create a table of name <code>name</code> with {@link COLUMNS} for
177    * families.
178    * @param name Name to give table.
179    * @return Column descriptor.
180    */
181   protected HTableDescriptor createTableDescriptor(final String name) {
182     return createTableDescriptor(name, MAXVERSIONS);
183   }
184 
185   /**
186    * Create a table of name <code>name</code> with {@link COLUMNS} for
187    * families.
188    * @param name Name to give table.
189    * @param versions How many versions to allow per column.
190    * @return Column descriptor.
191    */
192   protected HTableDescriptor createTableDescriptor(final String name,
193       final int versions) {
194     HTableDescriptor htd = new HTableDescriptor(name);
195     htd.addFamily(new HColumnDescriptor(fam1, versions,
196       HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
197       Integer.MAX_VALUE, HConstants.FOREVER, 
198       HColumnDescriptor.DEFAULT_BLOOMFILTER,
199       HConstants.REPLICATION_SCOPE_LOCAL));
200     htd.addFamily(new HColumnDescriptor(fam2, versions,
201         HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
202         Integer.MAX_VALUE, HConstants.FOREVER,
203         HColumnDescriptor.DEFAULT_BLOOMFILTER,
204         HConstants.REPLICATION_SCOPE_LOCAL));
205     htd.addFamily(new HColumnDescriptor(fam3, versions,
206         HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
207         Integer.MAX_VALUE,  HConstants.FOREVER,
208         HColumnDescriptor.DEFAULT_BLOOMFILTER,
209         HConstants.REPLICATION_SCOPE_LOCAL));
210     return htd;
211   }
212 
213   /**
214    * Add content to region <code>r</code> on the passed column
215    * <code>column</code>.
216    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
217    * @param r
218    * @param columnFamily
219    * @throws IOException
220    * @return count of what we added.
221    */
222   protected static long addContent(final HRegion r, final byte [] columnFamily)
223   throws IOException {
224     byte [] startKey = r.getRegionInfo().getStartKey();
225     byte [] endKey = r.getRegionInfo().getEndKey();
226     byte [] startKeyBytes = startKey;
227     if (startKeyBytes == null || startKeyBytes.length == 0) {
228       startKeyBytes = START_KEY_BYTES;
229     }
230     return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), null,
231       startKeyBytes, endKey, -1);
232   }
233 
234   /**
235    * Add content to region <code>r</code> on the passed column
236    * <code>column</code>.
237    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
238    * @param updater  An instance of {@link Incommon}.
239    * @param columnFamily
240    * @throws IOException
241    * @return count of what we added.
242    */
243   protected static long addContent(final Incommon updater,
244                                    final String columnFamily) throws IOException {
245     return addContent(updater, columnFamily, START_KEY_BYTES, null);
246   }
247 
248   protected static long addContent(final Incommon updater, final String family,
249                                    final String column) throws IOException {
250     return addContent(updater, family, column, START_KEY_BYTES, null);
251   }
252 
253   /**
254    * Add content to region <code>r</code> on the passed column
255    * <code>column</code>.
256    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
257    * @param updater  An instance of {@link Incommon}.
258    * @param columnFamily
259    * @param startKeyBytes Where to start the rows inserted
260    * @param endKey Where to stop inserting rows.
261    * @return count of what we added.
262    * @throws IOException
263    */
264   protected static long addContent(final Incommon updater, final String columnFamily,
265       final byte [] startKeyBytes, final byte [] endKey)
266   throws IOException {
267     return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
268   }
269 
270   protected static long addContent(final Incommon updater, final String family,
271                                    final String column, final byte [] startKeyBytes,
272                                    final byte [] endKey) throws IOException {
273     return addContent(updater, family, column, startKeyBytes, endKey, -1);
274   }
275 
276   /**
277    * Add content to region <code>r</code> on the passed column
278    * <code>column</code>.
279    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
280    * @param updater  An instance of {@link Incommon}.
281    * @param column
282    * @param startKeyBytes Where to start the rows inserted
283    * @param endKey Where to stop inserting rows.
284    * @param ts Timestamp to write the content with.
285    * @return count of what we added.
286    * @throws IOException
287    */
288   protected static long addContent(final Incommon updater,
289                                    final String columnFamily,
290                                    final String column,
291       final byte [] startKeyBytes, final byte [] endKey, final long ts)
292   throws IOException {
293     long count = 0;
294     // Add rows of three characters.  The first character starts with the
295     // 'a' character and runs up to 'z'.  Per first character, we run the
296     // second character over same range.  And same for the third so rows
297     // (and values) look like this: 'aaa', 'aab', 'aac', etc.
298     char secondCharStart = (char)startKeyBytes[1];
299     char thirdCharStart = (char)startKeyBytes[2];
300     EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
301       for (char d = secondCharStart; d <= LAST_CHAR; d++) {
302         for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
303           byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
304           if (endKey != null && endKey.length > 0
305               && Bytes.compareTo(endKey, t) <= 0) {
306             break EXIT;
307           }
308           try {
309             Put put;
310             if(ts != -1) {
311               put = new Put(t, ts, null);
312             } else {
313               put = new Put(t);
314             }
315             try {
316               StringBuilder sb = new StringBuilder();
317               if (column != null && column.contains(":")) {
318                 sb.append(column);
319               } else {
320                 if (columnFamily != null) {
321                   sb.append(columnFamily);
322                   if (!columnFamily.endsWith(":")) {
323                     sb.append(":");
324                   }
325                   if (column != null) {
326                     sb.append(column);
327                   }
328                 }
329               }
330               byte[][] split =
331                 KeyValue.parseColumn(Bytes.toBytes(sb.toString()));
332               if(split.length == 1) {
333                 put.add(split[0], new byte[0], t);
334               } else {
335                 put.add(split[0], split[1], t);
336               }
337               updater.put(put);
338               count++;
339             } catch (RuntimeException ex) {
340               ex.printStackTrace();
341               throw ex;
342             } catch (IOException ex) {
343               ex.printStackTrace();
344               throw ex;
345             }
346           } catch (RuntimeException ex) {
347             ex.printStackTrace();
348             throw ex;
349           } catch (IOException ex) {
350             ex.printStackTrace();
351             throw ex;
352           }
353         }
354         // Set start character back to FIRST_CHAR after we've done first loop.
355         thirdCharStart = FIRST_CHAR;
356       }
357       secondCharStart = FIRST_CHAR;
358     }
359     return count;
360   }
361 
362   /**
363    * Implementors can flushcache.
364    */
365   public static interface FlushCache {
366     /**
367      * @throws IOException
368      */
369     public void flushcache() throws IOException;
370   }
371 
372   /**
373    * Interface used by tests so can do common operations against an HTable
374    * or an HRegion.
375    *
376    * TOOD: Come up w/ a better name for this interface.
377    */
378   public static interface Incommon {
379     /**
380      *
381      * @param delete
382      * @param lockid
383      * @param writeToWAL
384      * @throws IOException
385      */
386     public void delete(Delete delete,  Integer lockid, boolean writeToWAL)
387     throws IOException;
388 
389     /**
390      * @param put
391      * @throws IOException
392      */
393     public void put(Put put) throws IOException;
394 
395     public Result get(Get get) throws IOException;
396 
397     /**
398      * @param family
399      * @param qualifiers
400      * @param firstRow
401      * @param ts
402      * @return scanner for specified columns, first row and timestamp
403      * @throws IOException
404      */
405     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
406         byte [] firstRow, long ts)
407     throws IOException;
408   }
409 
410   /**
411    * A class that makes a {@link Incommon} out of a {@link HRegion}
412    */
413   public static class HRegionIncommon implements Incommon, FlushCache {
414     final HRegion region;
415 
416     /**
417      * @param HRegion
418      */
419     public HRegionIncommon(final HRegion HRegion) {
420       this.region = HRegion;
421     }
422 
423     public void put(Put put) throws IOException {
424       region.put(put);
425     }
426 
427     public void delete(Delete delete,  Integer lockid, boolean writeToWAL)
428     throws IOException {
429       this.region.delete(delete, lockid, writeToWAL);
430     }
431 
432     public Result get(Get get) throws IOException {
433       return region.get(get, null);
434     }
435 
436     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
437         byte [] firstRow, long ts)
438       throws IOException {
439         Scan scan = new Scan(firstRow);
440         if(qualifiers == null || qualifiers.length == 0) {
441           scan.addFamily(family);
442         } else {
443           for(int i=0; i<qualifiers.length; i++){
444             scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
445           }
446         }
447         scan.setTimeRange(0, ts);
448         return new
449           InternalScannerIncommon(region.getScanner(scan));
450       }
451 
452     public Result get(Get get, Integer lockid) throws IOException{
453       return this.region.get(get, lockid);
454     }
455 
456 
457     public void flushcache() throws IOException {
458       this.region.flushcache();
459     }
460   }
461 
462   /**
463    * A class that makes a {@link Incommon} out of a {@link HTable}
464    */
465   public static class HTableIncommon implements Incommon {
466     final HTable table;
467 
468     /**
469      * @param table
470      */
471     public HTableIncommon(final HTable table) {
472       super();
473       this.table = table;
474     }
475 
476     public void put(Put put) throws IOException {
477       table.put(put);
478     }
479 
480 
481     public void delete(Delete delete,  Integer lockid, boolean writeToWAL)
482     throws IOException {
483       this.table.delete(delete);
484     }
485 
486     public Result get(Get get) throws IOException {
487       return table.get(get);
488     }
489 
490     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
491         byte [] firstRow, long ts)
492       throws IOException {
493       Scan scan = new Scan(firstRow);
494       if(qualifiers == null || qualifiers.length == 0) {
495         scan.addFamily(family);
496       } else {
497         for(int i=0; i<qualifiers.length; i++){
498           scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
499         }
500       }
501       scan.setTimeRange(0, ts);
502       return new
503         ClientScannerIncommon(table.getScanner(scan));
504     }
505   }
506 
507   public interface ScannerIncommon
508   extends Iterable<Result> {
509     public boolean next(List<KeyValue> values)
510     throws IOException;
511 
512     public void close() throws IOException;
513   }
514 
515   public static class ClientScannerIncommon implements ScannerIncommon {
516     ResultScanner scanner;
517     public ClientScannerIncommon(ResultScanner scanner) {
518       this.scanner = scanner;
519     }
520 
521     public boolean next(List<KeyValue> values)
522     throws IOException {
523       Result results = scanner.next();
524       if (results == null) {
525         return false;
526       }
527       values.clear();
528       values.addAll(results.list());
529       return true;
530     }
531 
532     public void close() throws IOException {
533       scanner.close();
534     }
535 
536     @SuppressWarnings("unchecked")
537     public Iterator iterator() {
538       return scanner.iterator();
539     }
540   }
541 
542   public static class InternalScannerIncommon implements ScannerIncommon {
543     InternalScanner scanner;
544 
545     public InternalScannerIncommon(InternalScanner scanner) {
546       this.scanner = scanner;
547     }
548 
549     public boolean next(List<KeyValue> results)
550     throws IOException {
551       return scanner.next(results);
552     }
553 
554     public void close() throws IOException {
555       scanner.close();
556     }
557 
558     public Iterator<Result> iterator() {
559       throw new UnsupportedOperationException();
560     }
561   }
562 
563 //  protected void assertCellEquals(final HRegion region, final byte [] row,
564 //    final byte [] column, final long timestamp, final String value)
565 //  throws IOException {
566 //    Map<byte [], Cell> result = region.getFull(row, null, timestamp, 1, null);
567 //    Cell cell_value = result.get(column);
568 //    if (value == null) {
569 //      assertEquals(Bytes.toString(column) + " at timestamp " + timestamp, null,
570 //        cell_value);
571 //    } else {
572 //      if (cell_value == null) {
573 //        fail(Bytes.toString(column) + " at timestamp " + timestamp +
574 //          "\" was expected to be \"" + value + " but was null");
575 //      }
576 //      if (cell_value != null) {
577 //        assertEquals(Bytes.toString(column) + " at timestamp "
578 //            + timestamp, value, new String(cell_value.getValue()));
579 //      }
580 //    }
581 //  }
582 
583   protected void assertResultEquals(final HRegion region, final byte [] row,
584       final byte [] family, final byte [] qualifier, final long timestamp,
585       final byte [] value)
586     throws IOException {
587       Get get = new Get(row);
588       get.setTimeStamp(timestamp);
589       Result res = region.get(get, null);
590       NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
591         res.getMap();
592       byte [] res_value = map.get(family).get(qualifier).get(timestamp);
593 
594       if (value == null) {
595         assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
596             " at timestamp " + timestamp, null, res_value);
597       } else {
598         if (res_value == null) {
599           fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
600               " at timestamp " + timestamp + "\" was expected to be \"" +
601               Bytes.toStringBinary(value) + " but was null");
602         }
603         if (res_value != null) {
604           assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
605               " at timestamp " +
606               timestamp, value, new String(res_value));
607         }
608       }
609     }
610 
611   /**
612    * Initializes parameters used in the test environment:
613    *
614    * Sets the configuration parameter TEST_DIRECTORY_KEY if not already set.
615    * Sets the boolean debugging if "DEBUGGING" is set in the environment.
616    * If debugging is enabled, reconfigures logging so that the root log level is
617    * set to WARN and the logging level for the package is set to DEBUG.
618    */
619   public static void initialize() {
620     if (System.getProperty(TEST_DIRECTORY_KEY) == null) {
621       System.setProperty(TEST_DIRECTORY_KEY, new File(
622           "build/hbase/test").getAbsolutePath());
623     }
624   }
625 
626   /**
627    * Common method to close down a MiniDFSCluster and the associated file system
628    *
629    * @param cluster
630    */
631   public static void shutdownDfs(MiniDFSCluster cluster) {
632     if (cluster != null) {
633       LOG.info("Shutting down Mini DFS ");
634       try {
635         cluster.shutdown();
636       } catch (Exception e) {
637         /// Can get a java.lang.reflect.UndeclaredThrowableException thrown
638         // here because of an InterruptedException. Don't let exceptions in
639         // here be cause of test failure.
640       }
641       try {
642         FileSystem fs = cluster.getFileSystem();
643         if (fs != null) {
644           LOG.info("Shutting down FileSystem");
645           fs.close();
646         }
647         FileSystem.closeAll();
648       } catch (IOException e) {
649         LOG.error("error closing file system", e);
650       }
651     }
652   }
653 
654   protected void createRootAndMetaRegions() throws IOException {
655     root = HRegion.createHRegion(HRegionInfo.ROOT_REGIONINFO, testDir, conf);
656     meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir,
657         conf);
658     HRegion.addRegionToMETA(root, meta);
659   }
660 
661   protected void closeRootAndMeta() throws IOException {
662     if (meta != null) {
663       meta.close();
664       meta.getLog().closeAndDelete();
665     }
666     if (root != null) {
667       root.close();
668       root.getLog().closeAndDelete();
669     }
670   }
671 
672   public void assertByteEquals(byte[] expected,
673                                byte[] actual) {
674     if (Bytes.compareTo(expected, actual) != 0) {
675       throw new AssertionFailedError("expected:<" +
676       Bytes.toString(expected) + "> but was:<" +
677       Bytes.toString(actual) + ">");
678     }
679   }
680 
681   public static void assertEquals(byte[] expected,
682                                byte[] actual) {
683     if (Bytes.compareTo(expected, actual) != 0) {
684       throw new AssertionFailedError("expected:<" +
685       Bytes.toStringBinary(expected) + "> but was:<" +
686       Bytes.toStringBinary(actual) + ">");
687     }
688   }
689 
690 }