View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import java.io.IOException;
22  import java.util.Iterator;
23  import java.util.List;
24  import java.util.NavigableMap;
25  
26  import junit.framework.AssertionFailedError;
27  import junit.framework.TestCase;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.client.Delete;
35  import org.apache.hadoop.hbase.client.Durability;
36  import org.apache.hadoop.hbase.client.Get;
37  import org.apache.hadoop.hbase.client.HTable;
38  import org.apache.hadoop.hbase.client.Put;
39  import org.apache.hadoop.hbase.client.Result;
40  import org.apache.hadoop.hbase.client.ResultScanner;
41  import org.apache.hadoop.hbase.client.Scan;
42  import org.apache.hadoop.hbase.client.Table;
43  import org.apache.hadoop.hbase.regionserver.HRegion;
44  import org.apache.hadoop.hbase.regionserver.InternalScanner;
45  import org.apache.hadoop.hbase.util.Bytes;
46  import org.apache.hadoop.hbase.util.FSTableDescriptors;
47  import org.apache.hadoop.hbase.util.FSUtils;
48  import org.apache.hadoop.hdfs.MiniDFSCluster;
49  
50  /**
51   * Abstract HBase test class.  Initializes a few things that can come in handly
52   * like an HBaseConfiguration and filesystem.
53   * @deprecated Write junit4 unit tests using {@link HBaseTestingUtility}
54   */
55  public abstract class HBaseTestCase extends TestCase {
56    private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
57  
58    protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
59    protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
60    protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
61  
62    protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
63  
64    private boolean localfs = false;
65    protected static Path testDir = null;
66    protected FileSystem fs = null;
67    protected HRegion meta = null;
68    protected static final char FIRST_CHAR = 'a';
69    protected static final char LAST_CHAR = 'z';
70    protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
71    protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
72    protected String START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET);
73    protected static final int MAXVERSIONS = 3;
74  
75    protected final HBaseTestingUtility testUtil = new HBaseTestingUtility();
76  
77    public volatile Configuration conf = HBaseConfiguration.create();
78    public final FSTableDescriptors fsTableDescriptors;
79    {
80      try {
81        fsTableDescriptors = new FSTableDescriptors(conf);
82      } catch (IOException e) {
83        throw new RuntimeException("Failed to init descriptors", e);
84      }
85    }
86  
87    /** constructor */
88    public HBaseTestCase() {
89      super();
90    }
91  
92    /**
93     * @param name
94     */
95    public HBaseTestCase(String name) {
96      super(name);
97    }
98  
99    /**
100    * Note that this method must be called after the mini hdfs cluster has
101    * started or we end up with a local file system.
102    */
103   @Override
104   protected void setUp() throws Exception {
105     super.setUp();
106     localfs =
107       (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
108 
109     if (fs == null) {
110       this.fs = FileSystem.get(conf);
111     }
112     try {
113       if (localfs) {
114         this.testDir = getUnitTestdir(getName());
115         if (fs.exists(testDir)) {
116           fs.delete(testDir, true);
117         }
118       } else {
119         this.testDir = FSUtils.getRootDir(conf);
120       }
121     } catch (Exception e) {
122       LOG.fatal("error during setup", e);
123       throw e;
124     }
125   }
126 
127   @Override
128   protected void tearDown() throws Exception {
129     try {
130       if (localfs) {
131         if (this.fs.exists(testDir)) {
132           this.fs.delete(testDir, true);
133         }
134       }
135     } catch (Exception e) {
136       LOG.fatal("error during tear down", e);
137     }
138     super.tearDown();
139   }
140 
141   /**
142    * @see HBaseTestingUtility#getBaseTestDir
143    * @param testName
144    * @return directory to use for this test
145    */
146     protected Path getUnitTestdir(String testName) {
147       return testUtil.getDataTestDir(testName);
148     }
149 
150   /**
151    * You must call close on the returned region and then close on the log file
152    * it created. Do {@link HRegion#close()} followed by {@link HRegion#getWAL()}
153    * and on it call close.
154    * @param desc
155    * @param startKey
156    * @param endKey
157    * @return An {@link HRegion}
158    * @throws IOException
159    */
160   public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
161       byte [] endKey)
162   throws IOException {
163     return createNewHRegion(desc, startKey, endKey, this.conf);
164   }
165 
166   public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
167       byte [] endKey, Configuration conf)
168   throws IOException {
169     HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
170     return HRegion.createHRegion(hri, testDir, conf, desc);
171   }
172 
173   protected HRegion openClosedRegion(final HRegion closedRegion)
174   throws IOException {
175     return HRegion.openHRegion(closedRegion, null);
176   }
177 
178   /**
179    * Create a table of name <code>name</code> with {@link COLUMNS} for
180    * families.
181    * @param name Name to give table.
182    * @return Column descriptor.
183    */
184   protected HTableDescriptor createTableDescriptor(final String name) {
185     return createTableDescriptor(name, MAXVERSIONS);
186   }
187 
188   /**
189    * Create a table of name <code>name</code> with {@link COLUMNS} for
190    * families.
191    * @param name Name to give table.
192    * @param versions How many versions to allow per column.
193    * @return Column descriptor.
194    */
195   protected HTableDescriptor createTableDescriptor(final String name,
196       final int versions) {
197     return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
198         versions, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
199   }
200 
201   /**
202    * Create a table of name <code>name</code> with {@link COLUMNS} for
203    * families.
204    * @param name Name to give table.
205    * @param versions How many versions to allow per column.
206    * @return Column descriptor.
207    */
208   protected HTableDescriptor createTableDescriptor(final String name,
209       final int minVersions, final int versions, final int ttl, KeepDeletedCells keepDeleted) {
210     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
211     for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
212       htd.addFamily(new HColumnDescriptor(cfName)
213           .setMinVersions(minVersions)
214           .setMaxVersions(versions)
215           .setKeepDeletedCells(keepDeleted)
216           .setBlockCacheEnabled(false)
217           .setTimeToLive(ttl)
218       );
219     }
220     return htd;
221   }
222 
223   /**
224    * Add content to region <code>r</code> on the passed column
225    * <code>column</code>.
226    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
227    * @param r
228    * @param columnFamily
229    * @param column
230    * @throws IOException
231    * @return count of what we added.
232    */
233   public static long addContent(final HRegion r, final byte [] columnFamily, final byte[] column)
234   throws IOException {
235     byte [] startKey = r.getRegionInfo().getStartKey();
236     byte [] endKey = r.getRegionInfo().getEndKey();
237     byte [] startKeyBytes = startKey;
238     if (startKeyBytes == null || startKeyBytes.length == 0) {
239       startKeyBytes = START_KEY_BYTES;
240     }
241     return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), Bytes.toString(column),
242       startKeyBytes, endKey, -1);
243   }
244 
245   public static long addContent(final HRegion r, final byte [] columnFamily)
246   throws IOException {
247     return addContent(r, columnFamily, null);
248   }
249 
250   /**
251    * Add content to region <code>r</code> on the passed column
252    * <code>column</code>.
253    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
254    * @param updater  An instance of {@link Incommon}.
255    * @param columnFamily
256    * @param writeToWAL
257    * @throws IOException
258    * @return count of what we added.
259    */
260   public static long addContent(final Incommon updater,
261       final String columnFamily) throws IOException {
262     return addContent(updater, columnFamily, START_KEY_BYTES, null);
263   }
264 
265   public static long addContent(final Incommon updater, final String family,
266       final String column) throws IOException {
267     return addContent(updater, family, column, START_KEY_BYTES, null);
268   }
269 
270   /**
271    * Add content to region <code>r</code> on the passed column
272    * <code>column</code>.
273    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
274    * @param updater  An instance of {@link Incommon}.
275    * @param columnFamily
276    * @param startKeyBytes Where to start the rows inserted
277    * @param endKey Where to stop inserting rows.
278    * @param writeToWAL
279    * @return count of what we added.
280    * @throws IOException
281    */
282   public static long addContent(final Incommon updater, final String columnFamily,
283       final byte [] startKeyBytes, final byte [] endKey)
284   throws IOException {
285     return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
286   }
287 
288   public static long addContent(final Incommon updater, final String family, String column,
289       final byte [] startKeyBytes, final byte [] endKey) throws IOException {
290     return addContent(updater, family, column, startKeyBytes, endKey, -1);
291   }
292 
293   /**
294    * Add content to region <code>r</code> on the passed column
295    * <code>column</code>.
296    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
297    * @param updater  An instance of {@link Incommon}.
298    * @param column
299    * @param startKeyBytes Where to start the rows inserted
300    * @param endKey Where to stop inserting rows.
301    * @param ts Timestamp to write the content with.
302    * @param writeToWAL
303    * @return count of what we added.
304    * @throws IOException
305    */
306   public static long addContent(final Incommon updater,
307                                    final String columnFamily, 
308                                    final String column,
309       final byte [] startKeyBytes, final byte [] endKey, final long ts)
310   throws IOException {
311     long count = 0;
312     // Add rows of three characters.  The first character starts with the
313     // 'a' character and runs up to 'z'.  Per first character, we run the
314     // second character over same range.  And same for the third so rows
315     // (and values) look like this: 'aaa', 'aab', 'aac', etc.
316     char secondCharStart = (char)startKeyBytes[1];
317     char thirdCharStart = (char)startKeyBytes[2];
318     EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
319       for (char d = secondCharStart; d <= LAST_CHAR; d++) {
320         for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
321           byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
322           if (endKey != null && endKey.length > 0
323               && Bytes.compareTo(endKey, t) <= 0) {
324             break EXIT;
325           }
326           try {
327             Put put;
328             if(ts != -1) {
329               put = new Put(t, ts);
330             } else {
331               put = new Put(t);
332             }
333             try {
334               StringBuilder sb = new StringBuilder();
335               if (column != null && column.contains(":")) {
336                 sb.append(column);
337               } else {
338                 if (columnFamily != null) {
339                   sb.append(columnFamily);
340                   if (!columnFamily.endsWith(":")) {
341                     sb.append(":");
342                   }
343                   if (column != null) {
344                     sb.append(column);
345                   }
346                 }
347               }
348               byte[][] split =
349                 KeyValue.parseColumn(Bytes.toBytes(sb.toString()));
350               if(split.length == 1) {
351                 put.add(split[0], new byte[0], t);
352               } else {
353                 put.add(split[0], split[1], t);
354               }
355               put.setDurability(Durability.SKIP_WAL);
356               updater.put(put);
357               count++;
358             } catch (RuntimeException ex) {
359               ex.printStackTrace();
360               throw ex;
361             } catch (IOException ex) {
362               ex.printStackTrace();
363               throw ex;
364             }
365           } catch (RuntimeException ex) {
366             ex.printStackTrace();
367             throw ex;
368           } catch (IOException ex) {
369             ex.printStackTrace();
370             throw ex;
371           }
372         }
373         // Set start character back to FIRST_CHAR after we've done first loop.
374         thirdCharStart = FIRST_CHAR;
375       }
376       secondCharStart = FIRST_CHAR;
377     }
378     return count;
379   }
380 
381   /**
382    * Implementors can flushcache.
383    */
384   public interface FlushCache {
385     /**
386      * @throws IOException
387      */
388     void flushcache() throws IOException;
389   }
390 
391   /**
392    * Interface used by tests so can do common operations against an HTable
393    * or an HRegion.
394    *
395    * TOOD: Come up w/ a better name for this interface.
396    */
397   public interface Incommon {
398     /**
399      *
400      * @param delete
401      * @param writeToWAL
402      * @throws IOException
403      */
404     void delete(Delete delete, boolean writeToWAL)
405     throws IOException;
406 
407     /**
408      * @param put
409      * @throws IOException
410      */
411     void put(Put put) throws IOException;
412 
413     Result get(Get get) throws IOException;
414 
415     /**
416      * @param family
417      * @param qualifiers
418      * @param firstRow
419      * @param ts
420      * @return scanner for specified columns, first row and timestamp
421      * @throws IOException
422      */
423     ScannerIncommon getScanner(
424       byte[] family, byte[][] qualifiers, byte[] firstRow, long ts
425     )
426     throws IOException;
427   }
428 
429   /**
430    * A class that makes a {@link Incommon} out of a {@link HRegion}
431    */
432   public static class HRegionIncommon implements Incommon, FlushCache {
433     final HRegion region;
434 
435     /**
436      * @param HRegion
437      */
438     public HRegionIncommon(final HRegion HRegion) {
439       this.region = HRegion;
440     }
441 
442     public void put(Put put) throws IOException {
443       region.put(put);
444     }
445 
446     public void delete(Delete delete,  boolean writeToWAL)
447     throws IOException {
448       this.region.delete(delete);
449     }
450 
451     public Result get(Get get) throws IOException {
452       return region.get(get);
453     }
454 
455     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
456         byte [] firstRow, long ts)
457       throws IOException {
458         Scan scan = new Scan(firstRow);
459         if(qualifiers == null || qualifiers.length == 0) {
460           scan.addFamily(family);
461         } else {
462           for(int i=0; i<qualifiers.length; i++){
463             scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
464           }
465         }
466         scan.setTimeRange(0, ts);
467         return new
468           InternalScannerIncommon(region.getScanner(scan));
469       }
470 
471     public void flushcache() throws IOException {
472       this.region.flushcache();
473     }
474   }
475 
476   /**
477    * A class that makes a {@link Incommon} out of a {@link HTable}
478    */
479   public static class HTableIncommon implements Incommon {
480     final Table table;
481 
482     /**
483      * @param table
484      */
485     public HTableIncommon(final Table table) {
486       super();
487       this.table = table;
488     }
489 
490     public void put(Put put) throws IOException {
491       table.put(put);
492     }
493 
494 
495     public void delete(Delete delete, boolean writeToWAL)
496     throws IOException {
497       this.table.delete(delete);
498     }
499 
500     public Result get(Get get) throws IOException {
501       return table.get(get);
502     }
503 
504     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
505         byte [] firstRow, long ts)
506       throws IOException {
507       Scan scan = new Scan(firstRow);
508       if(qualifiers == null || qualifiers.length == 0) {
509         scan.addFamily(family);
510       } else {
511         for(int i=0; i<qualifiers.length; i++){
512           scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
513         }
514       }
515       scan.setTimeRange(0, ts);
516       return new
517         ClientScannerIncommon(table.getScanner(scan));
518     }
519   }
520 
521   public interface ScannerIncommon
522   extends Iterable<Result> {
523     boolean next(List<Cell> values)
524     throws IOException;
525 
526     void close() throws IOException;
527   }
528 
529   public static class ClientScannerIncommon implements ScannerIncommon {
530     ResultScanner scanner;
531     public ClientScannerIncommon(ResultScanner scanner) {
532       this.scanner = scanner;
533     }
534 
535     @Override
536     public boolean next(List<Cell> values)
537     throws IOException {
538       Result results = scanner.next();
539       if (results == null) {
540         return false;
541       }
542       values.clear();
543       values.addAll(results.listCells());
544       return true;
545     }
546 
547     public void close() throws IOException {
548       scanner.close();
549     }
550 
551     public Iterator<Result> iterator() {
552       return scanner.iterator();
553     }
554   }
555 
556   public static class InternalScannerIncommon implements ScannerIncommon {
557     InternalScanner scanner;
558 
559     public InternalScannerIncommon(InternalScanner scanner) {
560       this.scanner = scanner;
561     }
562 
563     @Override
564     public boolean next(List<Cell> results)
565     throws IOException {
566       return scanner.next(results);
567     }
568 
569     @Override
570     public void close() throws IOException {
571       scanner.close();
572     }
573 
574     @Override
575     public Iterator<Result> iterator() {
576       throw new UnsupportedOperationException();
577     }
578   }
579 
580   protected void assertResultEquals(final HRegion region, final byte [] row,
581       final byte [] family, final byte [] qualifier, final long timestamp,
582       final byte [] value)
583     throws IOException {
584       Get get = new Get(row);
585       get.setTimeStamp(timestamp);
586       Result res = region.get(get);
587       NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
588         res.getMap();
589       byte [] res_value = map.get(family).get(qualifier).get(timestamp);
590 
591       if (value == null) {
592         assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
593             " at timestamp " + timestamp, null, res_value);
594       } else {
595         if (res_value == null) {
596           fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
597               " at timestamp " + timestamp + "\" was expected to be \"" +
598               Bytes.toStringBinary(value) + " but was null");
599         }
600         if (res_value != null) {
601           assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
602               " at timestamp " +
603               timestamp, value, new String(res_value));
604         }
605       }
606     }
607 
608   /**
609    * Common method to close down a MiniDFSCluster and the associated file system
610    *
611    * @param cluster
612    */
613   public static void shutdownDfs(MiniDFSCluster cluster) {
614     if (cluster != null) {
615       LOG.info("Shutting down Mini DFS ");
616       try {
617         cluster.shutdown();
618       } catch (Exception e) {
619         /// Can get a java.lang.reflect.UndeclaredThrowableException thrown
620         // here because of an InterruptedException. Don't let exceptions in
621         // here be cause of test failure.
622       }
623       try {
624         FileSystem fs = cluster.getFileSystem();
625         if (fs != null) {
626           LOG.info("Shutting down FileSystem");
627           fs.close();
628         }
629         FileSystem.closeAll();
630       } catch (IOException e) {
631         LOG.error("error closing file system", e);
632       }
633     }
634   }
635 
636   /**
637    * You must call {@link #closeRootAndMeta()} when done after calling this
638    * method. It does cleanup.
639    * @throws IOException
640    */
641   protected void createMetaRegion() throws IOException {
642     FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(conf);
643     meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir, conf,
644       fsTableDescriptors.get(TableName.META_TABLE_NAME));
645   }
646 
647   protected void closeRootAndMeta() throws IOException {
648     HRegion.closeHRegion(meta);
649   }
650 
651   public static void assertByteEquals(byte[] expected,
652                                byte[] actual) {
653     if (Bytes.compareTo(expected, actual) != 0) {
654       throw new AssertionFailedError("expected:<" +
655       Bytes.toString(expected) + "> but was:<" +
656       Bytes.toString(actual) + ">");
657     }
658   }
659 
660   public static void assertEquals(byte[] expected,
661                                byte[] actual) {
662     if (Bytes.compareTo(expected, actual) != 0) {
663       throw new AssertionFailedError("expected:<" +
664       Bytes.toStringBinary(expected) + "> but was:<" +
665       Bytes.toStringBinary(actual) + ">");
666     }
667   }
668 
669 }