View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import java.io.IOException;
22  import java.util.Iterator;
23  import java.util.List;
24  import java.util.NavigableMap;
25  
26  import junit.framework.AssertionFailedError;
27  import junit.framework.TestCase;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.client.Delete;
35  import org.apache.hadoop.hbase.client.Get;
36  import org.apache.hadoop.hbase.client.HTable;
37  import org.apache.hadoop.hbase.client.Put;
38  import org.apache.hadoop.hbase.client.Result;
39  import org.apache.hadoop.hbase.client.ResultScanner;
40  import org.apache.hadoop.hbase.client.Scan;
41  import org.apache.hadoop.hbase.regionserver.HRegion;
42  import org.apache.hadoop.hbase.regionserver.InternalScanner;
43  import org.apache.hadoop.hbase.util.*;
44  import org.apache.hadoop.hdfs.MiniDFSCluster;
45  
46  /**
47   * Abstract HBase test class.  Initializes a few things that can come in handly
48   * like an HBaseConfiguration and filesystem.
49   * @deprecated Write junit4 unit tests using {@link HBaseTestingUtility}
50   */
51  public abstract class HBaseTestCase extends TestCase {
52    private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
53  
54    protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
55    protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
56    protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
57  
58    protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
59  
60    private boolean localfs = false;
61    protected static Path testDir = null;
62    protected FileSystem fs = null;
63    protected HRegion meta = null;
64    protected static final char FIRST_CHAR = 'a';
65    protected static final char LAST_CHAR = 'z';
66    protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
67    protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
68    protected String START_KEY;
69    protected static final int MAXVERSIONS = 3;
70  
71    protected final HBaseTestingUtility testUtil = new HBaseTestingUtility();
72  
73    public volatile Configuration conf;
74  
75    /** constructor */
76    public HBaseTestCase() {
77      super();
78      init();
79    }
80  
81    /**
82     * @param name
83     */
84    public HBaseTestCase(String name) {
85      super(name);
86      init();
87    }
88  
89    private void init() {
90      conf = HBaseConfiguration.create();
91      START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET);
92    }
93  
94    /**
95     * Note that this method must be called after the mini hdfs cluster has
96     * started or we end up with a local file system.
97     */
98    @Override
99    protected void setUp() throws Exception {
100     super.setUp();
101     localfs =
102       (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
103 
104     if (fs == null) {
105       this.fs = FileSystem.get(conf);
106     }
107     try {
108       if (localfs) {
109         this.testDir = getUnitTestdir(getName());
110         if (fs.exists(testDir)) {
111           fs.delete(testDir, true);
112         }
113       } else {
114         this.testDir = FSUtils.getRootDir(conf);
115       }
116     } catch (Exception e) {
117       LOG.fatal("error during setup", e);
118       throw e;
119     }
120   }
121 
122   @Override
123   protected void tearDown() throws Exception {
124     try {
125       if (localfs) {
126         if (this.fs.exists(testDir)) {
127           this.fs.delete(testDir, true);
128         }
129       }
130     } catch (Exception e) {
131       LOG.fatal("error during tear down", e);
132     }
133     super.tearDown();
134   }
135 
136   /**
137    * @see HBaseTestingUtility#getBaseTestDir
138    * @param testName
139    * @return directory to use for this test
140    */
141     protected Path getUnitTestdir(String testName) {
142       return testUtil.getDataTestDir(testName);
143     }
144 
145   /**
146    * You must call close on the returned region and then close on the log file
147    * it created. Do {@link HRegion#close()} followed by {@link HRegion#getLog()}
148    * and on it call close.
149    * @param desc
150    * @param startKey
151    * @param endKey
152    * @return An {@link HRegion}
153    * @throws IOException
154    */
155   public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
156       byte [] endKey)
157   throws IOException {
158     return createNewHRegion(desc, startKey, endKey, this.conf);
159   }
160 
161   public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
162       byte [] endKey, Configuration conf)
163   throws IOException {
164     HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
165     return HRegion.createHRegion(hri, testDir, conf, desc);
166   }
167 
168   protected HRegion openClosedRegion(final HRegion closedRegion)
169   throws IOException {
170     return HRegion.openHRegion(closedRegion, null);
171   }
172 
173   /**
174    * Create a table of name <code>name</code> with {@link COLUMNS} for
175    * families.
176    * @param name Name to give table.
177    * @return Column descriptor.
178    */
179   protected HTableDescriptor createTableDescriptor(final String name) {
180     return createTableDescriptor(name, MAXVERSIONS);
181   }
182 
183   /**
184    * Create a table of name <code>name</code> with {@link COLUMNS} for
185    * families.
186    * @param name Name to give table.
187    * @param versions How many versions to allow per column.
188    * @return Column descriptor.
189    */
190   protected HTableDescriptor createTableDescriptor(final String name,
191       final int versions) {
192     return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
193         versions, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
194   }
195 
196   /**
197    * Create a table of name <code>name</code> with {@link COLUMNS} for
198    * families.
199    * @param name Name to give table.
200    * @param versions How many versions to allow per column.
201    * @return Column descriptor.
202    */
203   protected HTableDescriptor createTableDescriptor(final String name,
204       final int minVersions, final int versions, final int ttl, boolean keepDeleted) {
205     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
206     for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
207       htd.addFamily(new HColumnDescriptor(cfName)
208           .setMinVersions(minVersions)
209           .setMaxVersions(versions)
210           .setKeepDeletedCells(keepDeleted)
211           .setBlockCacheEnabled(false)
212           .setTimeToLive(ttl)
213       );
214     }
215     return htd;
216   }
217 
218   /**
219    * Add content to region <code>r</code> on the passed column
220    * <code>column</code>.
221    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
222    * @param r
223    * @param columnFamily
224    * @param column
225    * @throws IOException
226    * @return count of what we added.
227    */
228   public static long addContent(final HRegion r, final byte [] columnFamily, final byte[] column)
229   throws IOException {
230     byte [] startKey = r.getRegionInfo().getStartKey();
231     byte [] endKey = r.getRegionInfo().getEndKey();
232     byte [] startKeyBytes = startKey;
233     if (startKeyBytes == null || startKeyBytes.length == 0) {
234       startKeyBytes = START_KEY_BYTES;
235     }
236     return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), Bytes.toString(column),
237       startKeyBytes, endKey, -1);
238   }
239 
240   /**
241    * Add content to region <code>r</code> on the passed column
242    * <code>column</code>.
243    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
244    * @param r
245    * @param columnFamily
246    * @throws IOException
247    * @return count of what we added.
248    */
249   protected static long addContent(final HRegion r, final byte [] columnFamily)
250   throws IOException {
251     return addContent(r, columnFamily, null);
252   }
253 
254   /**
255    * Add content to region <code>r</code> on the passed column
256    * <code>column</code>.
257    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
258    * @param updater  An instance of {@link Incommon}.
259    * @param columnFamily
260    * @throws IOException
261    * @return count of what we added.
262    */
263   protected static long addContent(final Incommon updater,
264       final String columnFamily) throws IOException {
265     return addContent(updater, columnFamily, START_KEY_BYTES, null);
266   }
267 
268   protected static long addContent(final Incommon updater, final String family,
269       final String column) throws IOException {
270     return addContent(updater, family, column, START_KEY_BYTES, null);
271   }
272 
273   /**
274    * Add content to region <code>r</code> on the passed column
275    * <code>column</code>.
276    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
277    * @param updater  An instance of {@link Incommon}.
278    * @param columnFamily
279    * @param startKeyBytes Where to start the rows inserted
280    * @param endKey Where to stop inserting rows.
281    * @return count of what we added.
282    * @throws IOException
283    */
284   protected static long addContent(final Incommon updater, final String columnFamily,
285       final byte [] startKeyBytes, final byte [] endKey)
286   throws IOException {
287     return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
288   }
289 
290   protected static long addContent(final Incommon updater, final String family,
291                                    final String column, final byte [] startKeyBytes,
292                                    final byte [] endKey) throws IOException {
293     return addContent(updater, family, column, startKeyBytes, endKey, -1);
294   }
295 
296   /**
297    * Add content to region <code>r</code> on the passed column
298    * <code>column</code>.
299    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
300    * @param updater  An instance of {@link Incommon}.
301    * @param column
302    * @param startKeyBytes Where to start the rows inserted
303    * @param endKey Where to stop inserting rows.
304    * @param ts Timestamp to write the content with.
305    * @return count of what we added.
306    * @throws IOException
307    */
308   protected static long addContent(final Incommon updater,
309                                    final String columnFamily,
310                                    final String column,
311       final byte [] startKeyBytes, final byte [] endKey, final long ts)
312   throws IOException {
313     long count = 0;
314     // Add rows of three characters.  The first character starts with the
315     // 'a' character and runs up to 'z'.  Per first character, we run the
316     // second character over same range.  And same for the third so rows
317     // (and values) look like this: 'aaa', 'aab', 'aac', etc.
318     char secondCharStart = (char)startKeyBytes[1];
319     char thirdCharStart = (char)startKeyBytes[2];
320     EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
321       for (char d = secondCharStart; d <= LAST_CHAR; d++) {
322         for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
323           byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
324           if (endKey != null && endKey.length > 0
325               && Bytes.compareTo(endKey, t) <= 0) {
326             break EXIT;
327           }
328           try {
329             Put put;
330             if(ts != -1) {
331               put = new Put(t, ts);
332             } else {
333               put = new Put(t);
334             }
335             try {
336               StringBuilder sb = new StringBuilder();
337               if (column != null && column.contains(":")) {
338                 sb.append(column);
339               } else {
340                 if (columnFamily != null) {
341                   sb.append(columnFamily);
342                   if (!columnFamily.endsWith(":")) {
343                     sb.append(":");
344                   }
345                   if (column != null) {
346                     sb.append(column);
347                   }
348                 }
349               }
350               byte[][] split =
351                 KeyValue.parseColumn(Bytes.toBytes(sb.toString()));
352               if(split.length == 1) {
353                 put.add(split[0], new byte[0], t);
354               } else {
355                 put.add(split[0], split[1], t);
356               }
357               updater.put(put);
358               count++;
359             } catch (RuntimeException ex) {
360               ex.printStackTrace();
361               throw ex;
362             } catch (IOException ex) {
363               ex.printStackTrace();
364               throw ex;
365             }
366           } catch (RuntimeException ex) {
367             ex.printStackTrace();
368             throw ex;
369           } catch (IOException ex) {
370             ex.printStackTrace();
371             throw ex;
372           }
373         }
374         // Set start character back to FIRST_CHAR after we've done first loop.
375         thirdCharStart = FIRST_CHAR;
376       }
377       secondCharStart = FIRST_CHAR;
378     }
379     return count;
380   }
381 
382   /**
383    * Implementors can flushcache.
384    */
385   public interface FlushCache {
386     /**
387      * @throws IOException
388      */
389     void flushcache() throws IOException;
390   }
391 
392   /**
393    * Interface used by tests so can do common operations against an HTable
394    * or an HRegion.
395    *
396    * TOOD: Come up w/ a better name for this interface.
397    */
398   public interface Incommon {
399     /**
400      *
401      * @param delete
402      * @param writeToWAL
403      * @throws IOException
404      */
405     void delete(Delete delete, boolean writeToWAL)
406     throws IOException;
407 
408     /**
409      * @param put
410      * @throws IOException
411      */
412     void put(Put put) throws IOException;
413 
414     Result get(Get get) throws IOException;
415 
416     /**
417      * @param family
418      * @param qualifiers
419      * @param firstRow
420      * @param ts
421      * @return scanner for specified columns, first row and timestamp
422      * @throws IOException
423      */
424     ScannerIncommon getScanner(
425       byte[] family, byte[][] qualifiers, byte[] firstRow, long ts
426     )
427     throws IOException;
428   }
429 
430   /**
431    * A class that makes a {@link Incommon} out of a {@link HRegion}
432    */
433   public static class HRegionIncommon implements Incommon, FlushCache {
434     final HRegion region;
435 
436     /**
437      * @param HRegion
438      */
439     public HRegionIncommon(final HRegion HRegion) {
440       this.region = HRegion;
441     }
442 
443     public void put(Put put) throws IOException {
444       region.put(put);
445     }
446 
447     public void delete(Delete delete,  boolean writeToWAL)
448     throws IOException {
449       this.region.delete(delete);
450     }
451 
452     public Result get(Get get) throws IOException {
453       return region.get(get);
454     }
455 
456     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
457         byte [] firstRow, long ts)
458       throws IOException {
459         Scan scan = new Scan(firstRow);
460         if(qualifiers == null || qualifiers.length == 0) {
461           scan.addFamily(family);
462         } else {
463           for(int i=0; i<qualifiers.length; i++){
464             scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
465           }
466         }
467         scan.setTimeRange(0, ts);
468         return new
469           InternalScannerIncommon(region.getScanner(scan));
470       }
471 
472     public void flushcache() throws IOException {
473       this.region.flushcache();
474     }
475   }
476 
477   /**
478    * A class that makes a {@link Incommon} out of a {@link HTable}
479    */
480   public static class HTableIncommon implements Incommon {
481     final HTable table;
482 
483     /**
484      * @param table
485      */
486     public HTableIncommon(final HTable table) {
487       super();
488       this.table = table;
489     }
490 
491     public void put(Put put) throws IOException {
492       table.put(put);
493     }
494 
495 
496     public void delete(Delete delete, boolean writeToWAL)
497     throws IOException {
498       this.table.delete(delete);
499     }
500 
501     public Result get(Get get) throws IOException {
502       return table.get(get);
503     }
504 
505     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
506         byte [] firstRow, long ts)
507       throws IOException {
508       Scan scan = new Scan(firstRow);
509       if(qualifiers == null || qualifiers.length == 0) {
510         scan.addFamily(family);
511       } else {
512         for(int i=0; i<qualifiers.length; i++){
513           scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
514         }
515       }
516       scan.setTimeRange(0, ts);
517       return new
518         ClientScannerIncommon(table.getScanner(scan));
519     }
520   }
521 
522   public interface ScannerIncommon
523   extends Iterable<Result> {
524     boolean next(List<KeyValue> values)
525     throws IOException;
526 
527     void close() throws IOException;
528   }
529 
530   public static class ClientScannerIncommon implements ScannerIncommon {
531     ResultScanner scanner;
532     public ClientScannerIncommon(ResultScanner scanner) {
533       this.scanner = scanner;
534     }
535 
536     public boolean next(List<KeyValue> values)
537     throws IOException {
538       Result results = scanner.next();
539       if (results == null) {
540         return false;
541       }
542       values.clear();
543       values.addAll(results.list());
544       return true;
545     }
546 
547     public void close() throws IOException {
548       scanner.close();
549     }
550 
551     @SuppressWarnings("unchecked")
552     public Iterator iterator() {
553       return scanner.iterator();
554     }
555   }
556 
557   public static class InternalScannerIncommon implements ScannerIncommon {
558     InternalScanner scanner;
559 
560     public InternalScannerIncommon(InternalScanner scanner) {
561       this.scanner = scanner;
562     }
563 
564     public boolean next(List<KeyValue> results)
565     throws IOException {
566       return scanner.next(results);
567     }
568 
569     public void close() throws IOException {
570       scanner.close();
571     }
572 
573     public Iterator<Result> iterator() {
574       throw new UnsupportedOperationException();
575     }
576   }
577 
578 //  protected void assertCellEquals(final HRegion region, final byte [] row,
579 //    final byte [] column, final long timestamp, final String value)
580 //  throws IOException {
581 //    Map<byte [], Cell> result = region.getFull(row, null, timestamp, 1, null);
582 //    Cell cell_value = result.get(column);
583 //    if (value == null) {
584 //      assertEquals(Bytes.toString(column) + " at timestamp " + timestamp, null,
585 //        cell_value);
586 //    } else {
587 //      if (cell_value == null) {
588 //        fail(Bytes.toString(column) + " at timestamp " + timestamp +
589 //          "\" was expected to be \"" + value + " but was null");
590 //      }
591 //      if (cell_value != null) {
592 //        assertEquals(Bytes.toString(column) + " at timestamp "
593 //            + timestamp, value, new String(cell_value.getValue()));
594 //      }
595 //    }
596 //  }
597 
598   protected void assertResultEquals(final HRegion region, final byte [] row,
599       final byte [] family, final byte [] qualifier, final long timestamp,
600       final byte [] value)
601     throws IOException {
602       Get get = new Get(row);
603       get.setTimeStamp(timestamp);
604       Result res = region.get(get);
605       NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
606         res.getMap();
607       byte [] res_value = map.get(family).get(qualifier).get(timestamp);
608 
609       if (value == null) {
610         assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
611             " at timestamp " + timestamp, null, res_value);
612       } else {
613         if (res_value == null) {
614           fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
615               " at timestamp " + timestamp + "\" was expected to be \"" +
616               Bytes.toStringBinary(value) + " but was null");
617         }
618         if (res_value != null) {
619           assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
620               " at timestamp " +
621               timestamp, value, new String(res_value));
622         }
623       }
624     }
625 
626   /**
627    * Common method to close down a MiniDFSCluster and the associated file system
628    *
629    * @param cluster
630    */
631   public static void shutdownDfs(MiniDFSCluster cluster) {
632     if (cluster != null) {
633       LOG.info("Shutting down Mini DFS ");
634       try {
635         cluster.shutdown();
636       } catch (Exception e) {
637         /// Can get a java.lang.reflect.UndeclaredThrowableException thrown
638         // here because of an InterruptedException. Don't let exceptions in
639         // here be cause of test failure.
640       }
641       try {
642         FileSystem fs = cluster.getFileSystem();
643         if (fs != null) {
644           LOG.info("Shutting down FileSystem");
645           fs.close();
646         }
647         FileSystem.closeAll();
648       } catch (IOException e) {
649         LOG.error("error closing file system", e);
650       }
651     }
652   }
653 
654   /**
655    * You must call {@link #closeRootAndMeta()} when done after calling this
656    * method. It does cleanup.
657    * @throws IOException
658    */
659   protected void createMetaRegion() throws IOException {
660     meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir,
661         conf, HTableDescriptor.META_TABLEDESC);
662   }
663 
664   protected void closeRootAndMeta() throws IOException {
665     HRegion.closeHRegion(meta);
666   }
667 
668   public static void assertByteEquals(byte[] expected,
669                                byte[] actual) {
670     if (Bytes.compareTo(expected, actual) != 0) {
671       throw new AssertionFailedError("expected:<" +
672       Bytes.toString(expected) + "> but was:<" +
673       Bytes.toString(actual) + ">");
674     }
675   }
676 
677   public static void assertEquals(byte[] expected,
678                                byte[] actual) {
679     if (Bytes.compareTo(expected, actual) != 0) {
680       throw new AssertionFailedError("expected:<" +
681       Bytes.toStringBinary(expected) + "> but was:<" +
682       Bytes.toStringBinary(actual) + ">");
683     }
684   }
685 
686 }