View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import java.io.IOException;
22  import java.util.Iterator;
23  import java.util.List;
24  import java.util.NavigableMap;
25  
26  import junit.framework.AssertionFailedError;
27  import junit.framework.TestCase;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.client.Delete;
35  import org.apache.hadoop.hbase.client.Get;
36  import org.apache.hadoop.hbase.client.HTable;
37  import org.apache.hadoop.hbase.client.Put;
38  import org.apache.hadoop.hbase.client.Result;
39  import org.apache.hadoop.hbase.client.ResultScanner;
40  import org.apache.hadoop.hbase.client.Scan;
41  import org.apache.hadoop.hbase.regionserver.HRegion;
42  import org.apache.hadoop.hbase.regionserver.InternalScanner;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.util.FSUtils;
45  import org.apache.hadoop.hdfs.MiniDFSCluster;
46  
47  /**
48   * Abstract HBase test class.  Initializes a few things that can come in handly
49   * like an HBaseConfiguration and filesystem.
50   * @deprecated Write junit4 unit tests using {@link HBaseTestingUtility}
51   */
52  public abstract class HBaseTestCase extends TestCase {
53    private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
54  
55    protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
56    protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
57    protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
58  
59    protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
60  
61    private boolean localfs = false;
62    protected static Path testDir = null;
63    protected FileSystem fs = null;
64    protected HRegion meta = null;
65    protected static final char FIRST_CHAR = 'a';
66    protected static final char LAST_CHAR = 'z';
67    protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
68    protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
69    protected String START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET);
70    protected static final int MAXVERSIONS = 3;
71  
72    protected final HBaseTestingUtility testUtil = new HBaseTestingUtility();
73  
74    public volatile Configuration conf = HBaseConfiguration.create();
75  
76    /** constructor */
77    public HBaseTestCase() {
78      super();
79    }
80  
81    /**
82     * @param name
83     */
84    public HBaseTestCase(String name) {
85      super(name);
86    }
87  
88    /**
89     * Note that this method must be called after the mini hdfs cluster has
90     * started or we end up with a local file system.
91     */
92    @Override
93    protected void setUp() throws Exception {
94      super.setUp();
95      localfs =
96        (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
97  
98      if (fs == null) {
99        this.fs = FileSystem.get(conf);
100     }
101     try {
102       if (localfs) {
103         this.testDir = getUnitTestdir(getName());
104         if (fs.exists(testDir)) {
105           fs.delete(testDir, true);
106         }
107       } else {
108         this.testDir = FSUtils.getRootDir(conf);
109       }
110     } catch (Exception e) {
111       LOG.fatal("error during setup", e);
112       throw e;
113     }
114   }
115 
116   @Override
117   protected void tearDown() throws Exception {
118     try {
119       if (localfs) {
120         if (this.fs.exists(testDir)) {
121           this.fs.delete(testDir, true);
122         }
123       }
124     } catch (Exception e) {
125       LOG.fatal("error during tear down", e);
126     }
127     super.tearDown();
128   }
129 
130   /**
131    * @see HBaseTestingUtility#getBaseTestDir
132    * @param testName
133    * @return directory to use for this test
134    */
135     protected Path getUnitTestdir(String testName) {
136       return testUtil.getDataTestDir(testName);
137     }
138 
139   /**
140    * You must call close on the returned region and then close on the log file
141    * it created. Do {@link HRegion#close()} followed by {@link HRegion#getLog()}
142    * and on it call close.
143    * @param desc
144    * @param startKey
145    * @param endKey
146    * @return An {@link HRegion}
147    * @throws IOException
148    */
149   public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
150       byte [] endKey)
151   throws IOException {
152     return createNewHRegion(desc, startKey, endKey, this.conf);
153   }
154 
155   public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
156       byte [] endKey, Configuration conf)
157   throws IOException {
158     HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
159     return HRegion.createHRegion(hri, testDir, conf, desc);
160   }
161 
162   protected HRegion openClosedRegion(final HRegion closedRegion)
163   throws IOException {
164     return HRegion.openHRegion(closedRegion, null);
165   }
166 
167   /**
168    * Create a table of name <code>name</code> with {@link COLUMNS} for
169    * families.
170    * @param name Name to give table.
171    * @return Column descriptor.
172    */
173   protected HTableDescriptor createTableDescriptor(final String name) {
174     return createTableDescriptor(name, MAXVERSIONS);
175   }
176 
177   /**
178    * Create a table of name <code>name</code> with {@link COLUMNS} for
179    * families.
180    * @param name Name to give table.
181    * @param versions How many versions to allow per column.
182    * @return Column descriptor.
183    */
184   protected HTableDescriptor createTableDescriptor(final String name,
185       final int versions) {
186     return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
187         versions, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
188   }
189 
190   /**
191    * Create a table of name <code>name</code> with {@link COLUMNS} for
192    * families.
193    * @param name Name to give table.
194    * @param versions How many versions to allow per column.
195    * @return Column descriptor.
196    */
197   protected HTableDescriptor createTableDescriptor(final String name,
198       final int minVersions, final int versions, final int ttl, boolean keepDeleted) {
199     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
200     for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
201       htd.addFamily(new HColumnDescriptor(cfName)
202           .setMinVersions(minVersions)
203           .setMaxVersions(versions)
204           .setKeepDeletedCells(keepDeleted)
205           .setBlockCacheEnabled(false)
206           .setTimeToLive(ttl)
207       );
208     }
209     return htd;
210   }
211 
212   /**
213    * Add content to region <code>r</code> on the passed column
214    * <code>column</code>.
215    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
216    * @param r
217    * @param columnFamily
218    * @param column
219    * @throws IOException
220    * @return count of what we added.
221    */
222   public static long addContent(final HRegion r, final byte [] columnFamily, final byte[] column)
223   throws IOException {
224     byte [] startKey = r.getRegionInfo().getStartKey();
225     byte [] endKey = r.getRegionInfo().getEndKey();
226     byte [] startKeyBytes = startKey;
227     if (startKeyBytes == null || startKeyBytes.length == 0) {
228       startKeyBytes = START_KEY_BYTES;
229     }
230     return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), Bytes.toString(column),
231       startKeyBytes, endKey, -1);
232   }
233 
234   /**
235    * Add content to region <code>r</code> on the passed column
236    * <code>column</code>.
237    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
238    * @param r
239    * @param columnFamily
240    * @throws IOException
241    * @return count of what we added.
242    */
243   public static long addContent(final HRegion r, final byte [] columnFamily)
244   throws IOException {
245     return addContent(r, columnFamily, null);
246   }
247 
248   /**
249    * Add content to region <code>r</code> on the passed column
250    * <code>column</code>.
251    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
252    * @param updater  An instance of {@link Incommon}.
253    * @param columnFamily
254    * @throws IOException
255    * @return count of what we added.
256    */
257   public static long addContent(final Incommon updater,
258       final String columnFamily) throws IOException {
259     return addContent(updater, columnFamily, START_KEY_BYTES, null);
260   }
261 
262   public static long addContent(final Incommon updater, final String family,
263       final String column) throws IOException {
264     return addContent(updater, family, column, START_KEY_BYTES, null);
265   }
266 
267   /**
268    * Add content to region <code>r</code> on the passed column
269    * <code>column</code>.
270    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
271    * @param updater  An instance of {@link Incommon}.
272    * @param columnFamily
273    * @param startKeyBytes Where to start the rows inserted
274    * @param endKey Where to stop inserting rows.
275    * @return count of what we added.
276    * @throws IOException
277    */
278   public static long addContent(final Incommon updater, final String columnFamily,
279       final byte [] startKeyBytes, final byte [] endKey)
280   throws IOException {
281     return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
282   }
283 
284   public static long addContent(final Incommon updater, final String family,
285                                    final String column, final byte [] startKeyBytes,
286                                    final byte [] endKey) throws IOException {
287     return addContent(updater, family, column, startKeyBytes, endKey, -1);
288   }
289 
290   /**
291    * Add content to region <code>r</code> on the passed column
292    * <code>column</code>.
293    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
294    * @param updater  An instance of {@link Incommon}.
295    * @param column
296    * @param startKeyBytes Where to start the rows inserted
297    * @param endKey Where to stop inserting rows.
298    * @param ts Timestamp to write the content with.
299    * @return count of what we added.
300    * @throws IOException
301    */
302   public static long addContent(final Incommon updater,
303                                    final String columnFamily, 
304                                    final String column,
305       final byte [] startKeyBytes, final byte [] endKey, final long ts)
306   throws IOException {
307     long count = 0;
308     // Add rows of three characters.  The first character starts with the
309     // 'a' character and runs up to 'z'.  Per first character, we run the
310     // second character over same range.  And same for the third so rows
311     // (and values) look like this: 'aaa', 'aab', 'aac', etc.
312     char secondCharStart = (char)startKeyBytes[1];
313     char thirdCharStart = (char)startKeyBytes[2];
314     EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
315       for (char d = secondCharStart; d <= LAST_CHAR; d++) {
316         for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
317           byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
318           if (endKey != null && endKey.length > 0
319               && Bytes.compareTo(endKey, t) <= 0) {
320             break EXIT;
321           }
322           try {
323             Put put;
324             if(ts != -1) {
325               put = new Put(t, ts);
326             } else {
327               put = new Put(t);
328             }
329             try {
330               StringBuilder sb = new StringBuilder();
331               if (column != null && column.contains(":")) {
332                 sb.append(column);
333               } else {
334                 if (columnFamily != null) {
335                   sb.append(columnFamily);
336                   if (!columnFamily.endsWith(":")) {
337                     sb.append(":");
338                   }
339                   if (column != null) {
340                     sb.append(column);
341                   }
342                 }
343               }
344               byte[][] split =
345                 KeyValue.parseColumn(Bytes.toBytes(sb.toString()));
346               if(split.length == 1) {
347                 put.add(split[0], new byte[0], t);
348               } else {
349                 put.add(split[0], split[1], t);
350               }
351               updater.put(put);
352               count++;
353             } catch (RuntimeException ex) {
354               ex.printStackTrace();
355               throw ex;
356             } catch (IOException ex) {
357               ex.printStackTrace();
358               throw ex;
359             }
360           } catch (RuntimeException ex) {
361             ex.printStackTrace();
362             throw ex;
363           } catch (IOException ex) {
364             ex.printStackTrace();
365             throw ex;
366           }
367         }
368         // Set start character back to FIRST_CHAR after we've done first loop.
369         thirdCharStart = FIRST_CHAR;
370       }
371       secondCharStart = FIRST_CHAR;
372     }
373     return count;
374   }
375 
376   /**
377    * Implementors can flushcache.
378    */
379   public interface FlushCache {
380     /**
381      * @throws IOException
382      */
383     void flushcache() throws IOException;
384   }
385 
386   /**
387    * Interface used by tests so can do common operations against an HTable
388    * or an HRegion.
389    *
390    * TOOD: Come up w/ a better name for this interface.
391    */
392   public interface Incommon {
393     /**
394      *
395      * @param delete
396      * @param writeToWAL
397      * @throws IOException
398      */
399     void delete(Delete delete, boolean writeToWAL)
400     throws IOException;
401 
402     /**
403      * @param put
404      * @throws IOException
405      */
406     void put(Put put) throws IOException;
407 
408     Result get(Get get) throws IOException;
409 
410     /**
411      * @param family
412      * @param qualifiers
413      * @param firstRow
414      * @param ts
415      * @return scanner for specified columns, first row and timestamp
416      * @throws IOException
417      */
418     ScannerIncommon getScanner(
419       byte[] family, byte[][] qualifiers, byte[] firstRow, long ts
420     )
421     throws IOException;
422   }
423 
424   /**
425    * A class that makes a {@link Incommon} out of a {@link HRegion}
426    */
427   public static class HRegionIncommon implements Incommon, FlushCache {
428     final HRegion region;
429 
430     /**
431      * @param HRegion
432      */
433     public HRegionIncommon(final HRegion HRegion) {
434       this.region = HRegion;
435     }
436 
437     public void put(Put put) throws IOException {
438       region.put(put);
439     }
440 
441     public void delete(Delete delete,  boolean writeToWAL)
442     throws IOException {
443       this.region.delete(delete);
444     }
445 
446     public Result get(Get get) throws IOException {
447       return region.get(get);
448     }
449 
450     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
451         byte [] firstRow, long ts)
452       throws IOException {
453         Scan scan = new Scan(firstRow);
454         if(qualifiers == null || qualifiers.length == 0) {
455           scan.addFamily(family);
456         } else {
457           for(int i=0; i<qualifiers.length; i++){
458             scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
459           }
460         }
461         scan.setTimeRange(0, ts);
462         return new
463           InternalScannerIncommon(region.getScanner(scan));
464       }
465 
466     public void flushcache() throws IOException {
467       this.region.flushcache();
468     }
469   }
470 
471   /**
472    * A class that makes a {@link Incommon} out of a {@link HTable}
473    */
474   public static class HTableIncommon implements Incommon {
475     final HTable table;
476 
477     /**
478      * @param table
479      */
480     public HTableIncommon(final HTable table) {
481       super();
482       this.table = table;
483     }
484 
485     public void put(Put put) throws IOException {
486       table.put(put);
487     }
488 
489 
490     public void delete(Delete delete, boolean writeToWAL)
491     throws IOException {
492       this.table.delete(delete);
493     }
494 
495     public Result get(Get get) throws IOException {
496       return table.get(get);
497     }
498 
499     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
500         byte [] firstRow, long ts)
501       throws IOException {
502       Scan scan = new Scan(firstRow);
503       if(qualifiers == null || qualifiers.length == 0) {
504         scan.addFamily(family);
505       } else {
506         for(int i=0; i<qualifiers.length; i++){
507           scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
508         }
509       }
510       scan.setTimeRange(0, ts);
511       return new
512         ClientScannerIncommon(table.getScanner(scan));
513     }
514   }
515 
516   public interface ScannerIncommon
517   extends Iterable<Result> {
518     boolean next(List<Cell> values)
519     throws IOException;
520 
521     void close() throws IOException;
522   }
523 
524   public static class ClientScannerIncommon implements ScannerIncommon {
525     ResultScanner scanner;
526     public ClientScannerIncommon(ResultScanner scanner) {
527       this.scanner = scanner;
528     }
529 
530     @Override
531     public boolean next(List<Cell> values)
532     throws IOException {
533       Result results = scanner.next();
534       if (results == null) {
535         return false;
536       }
537       values.clear();
538       values.addAll(results.listCells());
539       return true;
540     }
541 
542     public void close() throws IOException {
543       scanner.close();
544     }
545 
546     @SuppressWarnings("unchecked")
547     public Iterator<Result> iterator() {
548       return scanner.iterator();
549     }
550   }
551 
552   public static class InternalScannerIncommon implements ScannerIncommon {
553     InternalScanner scanner;
554 
555     public InternalScannerIncommon(InternalScanner scanner) {
556       this.scanner = scanner;
557     }
558 
559     @Override
560     public boolean next(List<Cell> results)
561     throws IOException {
562       return scanner.next(results);
563     }
564 
565     @Override
566     public void close() throws IOException {
567       scanner.close();
568     }
569 
570     @Override
571     public Iterator<Result> iterator() {
572       throw new UnsupportedOperationException();
573     }
574   }
575 
576   protected void assertResultEquals(final HRegion region, final byte [] row,
577       final byte [] family, final byte [] qualifier, final long timestamp,
578       final byte [] value)
579     throws IOException {
580       Get get = new Get(row);
581       get.setTimeStamp(timestamp);
582       Result res = region.get(get);
583       NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
584         res.getMap();
585       byte [] res_value = map.get(family).get(qualifier).get(timestamp);
586 
587       if (value == null) {
588         assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
589             " at timestamp " + timestamp, null, res_value);
590       } else {
591         if (res_value == null) {
592           fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
593               " at timestamp " + timestamp + "\" was expected to be \"" +
594               Bytes.toStringBinary(value) + " but was null");
595         }
596         if (res_value != null) {
597           assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
598               " at timestamp " +
599               timestamp, value, new String(res_value));
600         }
601       }
602     }
603 
604   /**
605    * Common method to close down a MiniDFSCluster and the associated file system
606    *
607    * @param cluster
608    */
609   public static void shutdownDfs(MiniDFSCluster cluster) {
610     if (cluster != null) {
611       LOG.info("Shutting down Mini DFS ");
612       try {
613         cluster.shutdown();
614       } catch (Exception e) {
615         /// Can get a java.lang.reflect.UndeclaredThrowableException thrown
616         // here because of an InterruptedException. Don't let exceptions in
617         // here be cause of test failure.
618       }
619       try {
620         FileSystem fs = cluster.getFileSystem();
621         if (fs != null) {
622           LOG.info("Shutting down FileSystem");
623           fs.close();
624         }
625         FileSystem.closeAll();
626       } catch (IOException e) {
627         LOG.error("error closing file system", e);
628       }
629     }
630   }
631 
632   /**
633    * You must call {@link #closeRootAndMeta()} when done after calling this
634    * method. It does cleanup.
635    * @throws IOException
636    */
637   protected void createMetaRegion() throws IOException {
638     meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir,
639         conf, HTableDescriptor.META_TABLEDESC);
640   }
641 
642   protected void closeRootAndMeta() throws IOException {
643     HRegion.closeHRegion(meta);
644   }
645 
646   public static void assertByteEquals(byte[] expected,
647                                byte[] actual) {
648     if (Bytes.compareTo(expected, actual) != 0) {
649       throw new AssertionFailedError("expected:<" +
650       Bytes.toString(expected) + "> but was:<" +
651       Bytes.toString(actual) + ">");
652     }
653   }
654 
655   public static void assertEquals(byte[] expected,
656                                byte[] actual) {
657     if (Bytes.compareTo(expected, actual) != 0) {
658       throw new AssertionFailedError("expected:<" +
659       Bytes.toStringBinary(expected) + "> but was:<" +
660       Bytes.toStringBinary(actual) + ">");
661     }
662   }
663 
664 }