View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase;
19  
20  import static org.junit.Assert.assertTrue;
21  import static org.junit.Assert.fail;
22  
23  import java.io.File;
24  import java.io.IOException;
25  import java.io.OutputStream;
26  import java.lang.reflect.Field;
27  import java.lang.reflect.Method;
28  import java.lang.reflect.Modifier;
29  import java.net.InetAddress;
30  import java.net.InetSocketAddress;
31  import java.net.ServerSocket;
32  import java.net.Socket;
33  import java.net.UnknownHostException;
34  import java.security.MessageDigest;
35  import java.util.ArrayList;
36  import java.util.Arrays;
37  import java.util.Collection;
38  import java.util.Collections;
39  import java.util.HashSet;
40  import java.util.List;
41  import java.util.Map;
42  import java.util.NavigableSet;
43  import java.util.Random;
44  import java.util.Set;
45  import java.util.UUID;
46  import java.util.concurrent.TimeUnit;
47  
48  import org.apache.commons.logging.Log;
49  import org.apache.commons.logging.LogFactory;
50  import org.apache.commons.logging.impl.Jdk14Logger;
51  import org.apache.commons.logging.impl.Log4JLogger;
52  import org.apache.hadoop.hbase.classification.InterfaceAudience;
53  import org.apache.hadoop.hbase.classification.InterfaceStability;
54  import org.apache.hadoop.conf.Configuration;
55  import org.apache.hadoop.fs.FileSystem;
56  import org.apache.hadoop.fs.Path;
57  import org.apache.hadoop.hbase.Waiter.Predicate;
58  import org.apache.hadoop.hbase.catalog.MetaEditor;
59  import org.apache.hadoop.hbase.client.Delete;
60  import org.apache.hadoop.hbase.client.Durability;
61  import org.apache.hadoop.hbase.client.Get;
62  import org.apache.hadoop.hbase.client.HBaseAdmin;
63  import org.apache.hadoop.hbase.client.HConnection;
64  import org.apache.hadoop.hbase.client.HTable;
65  import org.apache.hadoop.hbase.client.Put;
66  import org.apache.hadoop.hbase.client.Result;
67  import org.apache.hadoop.hbase.client.ResultScanner;
68  import org.apache.hadoop.hbase.client.Scan;
69  import org.apache.hadoop.hbase.fs.HFileSystem;
70  import org.apache.hadoop.hbase.io.compress.Compression;
71  import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
72  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
73  import org.apache.hadoop.hbase.io.hfile.ChecksumUtil;
74  import org.apache.hadoop.hbase.io.hfile.HFile;
75  import org.apache.hadoop.hbase.ipc.RpcServerInterface;
76  import org.apache.hadoop.hbase.mapreduce.MapreduceTestingShim;
77  import org.apache.hadoop.hbase.master.HMaster;
78  import org.apache.hadoop.hbase.master.RegionStates;
79  import org.apache.hadoop.hbase.master.ServerManager;
80  import org.apache.hadoop.hbase.regionserver.BloomType;
81  import org.apache.hadoop.hbase.regionserver.HRegion;
82  import org.apache.hadoop.hbase.regionserver.HRegionServer;
83  import org.apache.hadoop.hbase.regionserver.HStore;
84  import org.apache.hadoop.hbase.regionserver.InternalScanner;
85  import org.apache.hadoop.hbase.regionserver.RegionServerServices;
86  import org.apache.hadoop.hbase.regionserver.wal.HLog;
87  import org.apache.hadoop.hbase.security.User;
88  import org.apache.hadoop.hbase.tool.Canary;
89  import org.apache.hadoop.hbase.util.Bytes;
90  import org.apache.hadoop.hbase.util.FSUtils;
91  import org.apache.hadoop.hbase.util.JVMClusterUtil;
92  import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
93  import org.apache.hadoop.hbase.util.RegionSplitter;
94  import org.apache.hadoop.hbase.util.RetryCounter;
95  import org.apache.hadoop.hbase.util.Threads;
96  import org.apache.hadoop.hbase.zookeeper.EmptyWatcher;
97  import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
98  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
99  import org.apache.hadoop.hbase.zookeeper.ZKConfig;
100 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
101 import org.apache.hadoop.hdfs.DFSClient;
102 import org.apache.hadoop.hdfs.DistributedFileSystem;
103 import org.apache.hadoop.hdfs.MiniDFSCluster;
104 import org.apache.hadoop.mapred.JobConf;
105 import org.apache.hadoop.mapred.MiniMRCluster;
106 import org.apache.hadoop.mapred.TaskLog;
107 import org.apache.hadoop.security.UserGroupInformation;
108 import org.apache.zookeeper.KeeperException;
109 import org.apache.zookeeper.KeeperException.NodeExistsException;
110 import org.apache.zookeeper.WatchedEvent;
111 import org.apache.zookeeper.ZooKeeper;
112 import org.apache.zookeeper.ZooKeeper.States;
113 
114 /**
115  * Facility for testing HBase. Replacement for
116  * old HBaseTestCase and HBaseClusterTestCase functionality.
117  * Create an instance and keep it around testing HBase.  This class is
118  * meant to be your one-stop shop for anything you might need testing.  Manages
119  * one cluster at a time only. Managed cluster can be an in-process
120  * {@link MiniHBaseCluster}, or a deployed cluster of type {@link DistributedHBaseCluster}.
121  * Not all methods work with the real cluster.
122  * Depends on log4j being on classpath and
123  * hbase-site.xml for logging and test-run configuration.  It does not set
124  * logging levels nor make changes to configuration parameters.
125  * <p>To preserve test data directories, pass the system property "hbase.testing.preserve.testdir"
126  * setting it to true.
127  */
128 @InterfaceAudience.Public
129 @InterfaceStability.Evolving
130 public class HBaseTestingUtility extends HBaseCommonTestingUtility {
131    private MiniZooKeeperCluster zkCluster = null;
132 
133   public static final String REGIONS_PER_SERVER_KEY = "hbase.test.regions-per-server";
134   /**
135    * The default number of regions per regionserver when creating a pre-split
136    * table.
137    */
138   public static final int DEFAULT_REGIONS_PER_SERVER = 5;
139 
140   /**
141    * Set if we were passed a zkCluster.  If so, we won't shutdown zk as
142    * part of general shutdown.
143    */
144   private boolean passedZkCluster = false;
145   private MiniDFSCluster dfsCluster = null;
146 
147   private HBaseCluster hbaseCluster = null;
148   private MiniMRCluster mrCluster = null;
149 
150   /** If there is a mini cluster running for this testing utility instance. */
151   private boolean miniClusterRunning;
152 
153   private String hadoopLogDir;
154 
155   /** Directory (a subdirectory of dataTestDir) used by the dfs cluster if any */
156   private File clusterTestDir = null;
157 
158   /** Directory on test filesystem where we put the data for this instance of
159     * HBaseTestingUtility*/
160   private Path dataTestDirOnTestFS = null;
161 
162   /**
163    * System property key to get test directory value.
164    * Name is as it is because mini dfs has hard-codings to put test data here.
165    * It should NOT be used directly in HBase, as it's a property used in
166    *  mini dfs.
167    *  @deprecated can be used only with mini dfs
168    */
169   @Deprecated
170   private static final String TEST_DIRECTORY_KEY = "test.build.data";
171 
172   /** Filesystem URI used for map-reduce mini-cluster setup */
173   private static String FS_URI;
174 
175   /** A set of ports that have been claimed using {@link #randomFreePort()}. */
176   private static final Set<Integer> takenRandomPorts = new HashSet<Integer>();
177 
178   /** Compression algorithms to use in parameterized JUnit 4 tests */
179   public static final List<Object[]> COMPRESSION_ALGORITHMS_PARAMETERIZED =
180     Arrays.asList(new Object[][] {
181       { Compression.Algorithm.NONE },
182       { Compression.Algorithm.GZ }
183     });
184 
185   /** This is for unit tests parameterized with a two booleans. */
186   public static final List<Object[]> BOOLEAN_PARAMETERIZED =
187       Arrays.asList(new Object[][] {
188           { new Boolean(false) },
189           { new Boolean(true) }
190       });
191 
192   /** This is for unit tests parameterized with a single boolean. */
193   public static final List<Object[]> MEMSTORETS_TAGS_PARAMETRIZED = memStoreTSAndTagsCombination()  ;
194   /** Compression algorithms to use in testing */
195   public static final Compression.Algorithm[] COMPRESSION_ALGORITHMS ={
196       Compression.Algorithm.NONE, Compression.Algorithm.GZ
197     };
198 
199   /**
200    * Create all combinations of Bloom filters and compression algorithms for
201    * testing.
202    */
203   private static List<Object[]> bloomAndCompressionCombinations() {
204     List<Object[]> configurations = new ArrayList<Object[]>();
205     for (Compression.Algorithm comprAlgo :
206          HBaseTestingUtility.COMPRESSION_ALGORITHMS) {
207       for (BloomType bloomType : BloomType.values()) {
208         configurations.add(new Object[] { comprAlgo, bloomType });
209       }
210     }
211     return Collections.unmodifiableList(configurations);
212   }
213 
214   /**
215    * Create combination of memstoreTS and tags
216    */
217   private static List<Object[]> memStoreTSAndTagsCombination() {
218     List<Object[]> configurations = new ArrayList<Object[]>();
219     configurations.add(new Object[] { false, false });
220     configurations.add(new Object[] { false, true });
221     configurations.add(new Object[] { true, false });
222     configurations.add(new Object[] { true, true });
223     return Collections.unmodifiableList(configurations);
224   }
225 
226   public static final Collection<Object[]> BLOOM_AND_COMPRESSION_COMBINATIONS =
227       bloomAndCompressionCombinations();
228 
229   public HBaseTestingUtility() {
230     this(HBaseConfiguration.create());
231   }
232 
233   public HBaseTestingUtility(Configuration conf) {
234     super(conf);
235 
236     // a hbase checksum verification failure will cause unit tests to fail
237     ChecksumUtil.generateExceptionForChecksumFailureForTest(true);
238   }
239 
240   /**
241    * Create an HBaseTestingUtility where all tmp files are written to the local test data dir.
242    * It is needed to properly base FSUtil.getRootDirs so that they drop temp files in the proper
243    * test dir.  Use this when you aren't using an Mini HDFS cluster.
244    * @return HBaseTestingUtility that use local fs for temp files.
245    */
246   public static HBaseTestingUtility createLocalHTU() {
247     Configuration c = HBaseConfiguration.create();
248     return createLocalHTU(c);
249   }
250 
251   /**
252    * Create an HBaseTestingUtility where all tmp files are written to the local test data dir.
253    * It is needed to properly base FSUtil.getRootDirs so that they drop temp files in the proper
254    * test dir.  Use this when you aren't using an Mini HDFS cluster.
255    * @param c Configuration (will be modified)
256    * @return HBaseTestingUtility that use local fs for temp files.
257    */
258   public static HBaseTestingUtility createLocalHTU(Configuration c) {
259     HBaseTestingUtility htu = new HBaseTestingUtility(c);
260     String dataTestDir = htu.getDataTestDir().toString();
261     htu.getConfiguration().set(HConstants.HBASE_DIR, dataTestDir);
262     LOG.debug("Setting " + HConstants.HBASE_DIR + " to " + dataTestDir);
263     return htu;
264   }
265 
266   /**
267    * Controls how many attempts we will make in the face of failures in HDFS.
268    * @deprecated to be removed with Hadoop 1.x support
269    */
270   @Deprecated
271   public void setHDFSClientRetry(final int retries) {
272     this.conf.setInt("hdfs.client.retries.number", retries);
273     if (0 == retries) {
274       makeDFSClientNonRetrying();
275     }
276   }
277 
278   /**
279    * Returns this classes's instance of {@link Configuration}.  Be careful how
280    * you use the returned Configuration since {@link HConnection} instances
281    * can be shared.  The Map of HConnections is keyed by the Configuration.  If
282    * say, a Connection was being used against a cluster that had been shutdown,
283    * see {@link #shutdownMiniCluster()}, then the Connection will no longer
284    * be wholesome.  Rather than use the return direct, its usually best to
285    * make a copy and use that.  Do
286    * <code>Configuration c = new Configuration(INSTANCE.getConfiguration());</code>
287    * @return Instance of Configuration.
288    */
289   @Override
290   public Configuration getConfiguration() {
291     return super.getConfiguration();
292   }
293 
294   public void setHBaseCluster(HBaseCluster hbaseCluster) {
295     this.hbaseCluster = hbaseCluster;
296   }
297 
298   /**
299    * Home our data in a dir under {@link #DEFAULT_BASE_TEST_DIRECTORY}.
300    * Give it a random name so can have many concurrent tests running if
301    * we need to.  It needs to amend the {@link #TEST_DIRECTORY_KEY}
302    * System property, as it's what minidfscluster bases
303    * it data dir on.  Moding a System property is not the way to do concurrent
304    * instances -- another instance could grab the temporary
305    * value unintentionally -- but not anything can do about it at moment;
306    * single instance only is how the minidfscluster works.
307    *
308    * We also create the underlying directory for
309    *  hadoop.log.dir, mapred.local.dir and hadoop.tmp.dir, and set the values
310    *  in the conf, and as a system property for hadoop.tmp.dir
311    *
312    * @return The calculated data test build directory, if newly-created.
313    */
314   @Override
315   protected Path setupDataTestDir() {
316     Path testPath = super.setupDataTestDir();
317     if (null == testPath) {
318       return null;
319     }
320 
321     createSubDirAndSystemProperty(
322       "hadoop.log.dir",
323       testPath, "hadoop-log-dir");
324 
325     // This is defaulted in core-default.xml to /tmp/hadoop-${user.name}, but
326     //  we want our own value to ensure uniqueness on the same machine
327     createSubDirAndSystemProperty(
328       "hadoop.tmp.dir",
329       testPath, "hadoop-tmp-dir");
330 
331     // Read and modified in org.apache.hadoop.mapred.MiniMRCluster
332     createSubDir(
333       "mapred.local.dir",
334       testPath, "mapred-local-dir");
335 
336     return testPath;
337   }
338 
339   private void createSubDirAndSystemProperty(
340     String propertyName, Path parent, String subDirName){
341 
342     String sysValue = System.getProperty(propertyName);
343 
344     if (sysValue != null) {
345       // There is already a value set. So we do nothing but hope
346       //  that there will be no conflicts
347       LOG.info("System.getProperty(\""+propertyName+"\") already set to: "+
348         sysValue + " so I do NOT create it in " + parent);
349       String confValue = conf.get(propertyName);
350       if (confValue != null && !confValue.endsWith(sysValue)){
351        LOG.warn(
352          propertyName + " property value differs in configuration and system: "+
353          "Configuration="+confValue+" while System="+sysValue+
354          " Erasing configuration value by system value."
355        );
356       }
357       conf.set(propertyName, sysValue);
358     } else {
359       // Ok, it's not set, so we create it as a subdirectory
360       createSubDir(propertyName, parent, subDirName);
361       System.setProperty(propertyName, conf.get(propertyName));
362     }
363   }
364 
365   /**
366    * @return Where to write test data on the test filesystem; Returns working directory
367    * for the test filesystem by default
368    * @see #setupDataTestDirOnTestFS()
369    * @see #getTestFileSystem()
370    */
371   private Path getBaseTestDirOnTestFS() throws IOException {
372     FileSystem fs = getTestFileSystem();
373     return new Path(fs.getWorkingDirectory(), "test-data");
374   }
375 
376   /**
377    * @return Where the DFS cluster will write data on the local subsystem.
378    * Creates it if it does not exist already.  A subdir of {@link #getBaseTestDir()}
379    * @see #getTestFileSystem()
380    */
381   Path getClusterTestDir() {
382     if (clusterTestDir == null){
383       setupClusterTestDir();
384     }
385     return new Path(clusterTestDir.getAbsolutePath());
386   }
387 
388   /**
389    * Creates a directory for the DFS cluster, under the test data
390    */
391   private void setupClusterTestDir() {
392     if (clusterTestDir != null) {
393       return;
394     }
395 
396     // Using randomUUID ensures that multiple clusters can be launched by
397     //  a same test, if it stops & starts them
398     Path testDir = getDataTestDir("dfscluster_" + UUID.randomUUID().toString());
399     clusterTestDir = new File(testDir.toString()).getAbsoluteFile();
400     // Have it cleaned up on exit
401     boolean b = deleteOnExit();
402     if (b) clusterTestDir.deleteOnExit();
403     conf.set(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
404     LOG.info("Created new mini-cluster data directory: " + clusterTestDir + ", deleteOnExit=" + b);
405   }
406 
407   /**
408    * Returns a Path in the test filesystem, obtained from {@link #getTestFileSystem()}
409    * to write temporary test data. Call this method after setting up the mini dfs cluster
410    * if the test relies on it.
411    * @return a unique path in the test filesystem
412    */
413   public Path getDataTestDirOnTestFS() throws IOException {
414     if (dataTestDirOnTestFS == null) {
415       setupDataTestDirOnTestFS();
416     }
417 
418     return dataTestDirOnTestFS;
419   }
420 
421   /**
422    * Returns a Path in the test filesystem, obtained from {@link #getTestFileSystem()}
423    * to write temporary test data. Call this method after setting up the mini dfs cluster
424    * if the test relies on it.
425    * @return a unique path in the test filesystem
426    * @param subdirName name of the subdir to create under the base test dir
427    */
428   public Path getDataTestDirOnTestFS(final String subdirName) throws IOException {
429     return new Path(getDataTestDirOnTestFS(), subdirName);
430   }
431 
432   /**
433    * Sets up a path in test filesystem to be used by tests
434    */
435   private void setupDataTestDirOnTestFS() throws IOException {
436     if (dataTestDirOnTestFS != null) {
437       LOG.warn("Data test on test fs dir already setup in "
438           + dataTestDirOnTestFS.toString());
439       return;
440     }
441 
442     //The file system can be either local, mini dfs, or if the configuration
443     //is supplied externally, it can be an external cluster FS. If it is a local
444     //file system, the tests should use getBaseTestDir, otherwise, we can use
445     //the working directory, and create a unique sub dir there
446     FileSystem fs = getTestFileSystem();
447     if (fs.getUri().getScheme().equals(FileSystem.getLocal(conf).getUri().getScheme())) {
448       File dataTestDir = new File(getDataTestDir().toString());
449       if (deleteOnExit()) dataTestDir.deleteOnExit();
450       dataTestDirOnTestFS = new Path(dataTestDir.getAbsolutePath());
451     } else {
452       Path base = getBaseTestDirOnTestFS();
453       String randomStr = UUID.randomUUID().toString();
454       dataTestDirOnTestFS = new Path(base, randomStr);
455       if (deleteOnExit()) fs.deleteOnExit(dataTestDirOnTestFS);
456     }
457   }
458 
459   /**
460    * Cleans the test data directory on the test filesystem.
461    * @return True if we removed the test dirs
462    * @throws IOException
463    */
464   public boolean cleanupDataTestDirOnTestFS() throws IOException {
465     boolean ret = getTestFileSystem().delete(dataTestDirOnTestFS, true);
466     if (ret)
467       dataTestDirOnTestFS = null;
468     return ret;
469   }
470 
471   /**
472    * Cleans a subdirectory under the test data directory on the test filesystem.
473    * @return True if we removed child
474    * @throws IOException
475    */
476   public boolean cleanupDataTestDirOnTestFS(String subdirName) throws IOException {
477     Path cpath = getDataTestDirOnTestFS(subdirName);
478     return getTestFileSystem().delete(cpath, true);
479   }
480 
481   /**
482    * Start a minidfscluster.
483    * @param servers How many DNs to start.
484    * @throws Exception
485    * @see {@link #shutdownMiniDFSCluster()}
486    * @return The mini dfs cluster created.
487    */
488   public MiniDFSCluster startMiniDFSCluster(int servers) throws Exception {
489     return startMiniDFSCluster(servers, null);
490   }
491 
492   /**
493    * Start a minidfscluster.
494    * This is useful if you want to run datanode on distinct hosts for things
495    * like HDFS block location verification.
496    * If you start MiniDFSCluster without host names, all instances of the
497    * datanodes will have the same host name.
498    * @param hosts hostnames DNs to run on.
499    * @throws Exception
500    * @see {@link #shutdownMiniDFSCluster()}
501    * @return The mini dfs cluster created.
502    */
503   public MiniDFSCluster startMiniDFSCluster(final String hosts[])
504   throws Exception {
505     if ( hosts != null && hosts.length != 0) {
506       return startMiniDFSCluster(hosts.length, hosts);
507     } else {
508       return startMiniDFSCluster(1, null);
509     }
510   }
511 
512   /**
513    * Start a minidfscluster.
514    * Can only create one.
515    * @param servers How many DNs to start.
516    * @param hosts hostnames DNs to run on.
517    * @throws Exception
518    * @see {@link #shutdownMiniDFSCluster()}
519    * @return The mini dfs cluster created.
520    */
521   public MiniDFSCluster startMiniDFSCluster(int servers, final String hosts[])
522   throws Exception {
523     createDirsAndSetProperties();
524     try {
525       Method m = Class.forName("org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream")
526           .getMethod("setShouldSkipFsyncForTesting", new Class<?> []{ boolean.class });
527       m.invoke(null, new Object[] {true});
528     } catch (ClassNotFoundException e) {
529       LOG.info("EditLogFileOutputStream not found");
530     }
531 
532     // Error level to skip some warnings specific to the minicluster. See HBASE-4709
533     org.apache.log4j.Logger.getLogger(org.apache.hadoop.metrics2.util.MBeans.class).
534         setLevel(org.apache.log4j.Level.ERROR);
535     org.apache.log4j.Logger.getLogger(org.apache.hadoop.metrics2.impl.MetricsSystemImpl.class).
536         setLevel(org.apache.log4j.Level.ERROR);
537 
538 
539     this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true,
540       true, null, null, hosts, null);
541 
542     // Set this just-started cluster as our filesystem.
543     FileSystem fs = this.dfsCluster.getFileSystem();
544     FSUtils.setFsDefault(this.conf, new Path(fs.getUri()));
545 
546     // Wait for the cluster to be totally up
547     this.dfsCluster.waitClusterUp();
548 
549     //reset the test directory for test file system
550     dataTestDirOnTestFS = null;
551 
552     return this.dfsCluster;
553   }
554 
555 
556   public MiniDFSCluster startMiniDFSCluster(int servers, final  String racks[], String hosts[])
557       throws Exception {
558     createDirsAndSetProperties();
559     this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true,
560         true, null, racks, hosts, null);
561 
562     // Set this just-started cluster as our filesystem.
563     FileSystem fs = this.dfsCluster.getFileSystem();
564     FSUtils.setFsDefault(this.conf, new Path(fs.getUri()));
565 
566     // Wait for the cluster to be totally up
567     this.dfsCluster.waitClusterUp();
568 
569     //reset the test directory for test file system
570     dataTestDirOnTestFS = null;
571 
572     return this.dfsCluster;
573   }
574 
575   public MiniDFSCluster startMiniDFSClusterForTestHLog(int namenodePort) throws IOException {
576     createDirsAndSetProperties();
577     dfsCluster = new MiniDFSCluster(namenodePort, conf, 5, false, true, true, null,
578         null, null, null);
579     return dfsCluster;
580   }
581 
582   /** This is used before starting HDFS and map-reduce mini-clusters */
583   private void createDirsAndSetProperties() throws IOException {
584     setupClusterTestDir();
585     System.setProperty(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
586     createDirAndSetProperty("cache_data", "test.cache.data");
587     createDirAndSetProperty("hadoop_tmp", "hadoop.tmp.dir");
588     hadoopLogDir = createDirAndSetProperty("hadoop_logs", "hadoop.log.dir");
589     createDirAndSetProperty("mapred_local", "mapred.local.dir");
590     createDirAndSetProperty("mapred_temp", "mapred.temp.dir");
591     enableShortCircuit();
592 
593     Path root = getDataTestDirOnTestFS("hadoop");
594     conf.set(MapreduceTestingShim.getMROutputDirProp(),
595       new Path(root, "mapred-output-dir").toString());
596     conf.set("mapred.system.dir", new Path(root, "mapred-system-dir").toString());
597     conf.set("mapreduce.jobtracker.staging.root.dir",
598       new Path(root, "mapreduce-jobtracker-staging-root-dir").toString());
599     conf.set("mapred.working.dir", new Path(root, "mapred-working-dir").toString());
600   }
601 
602 
603   /**
604    *  Get the HBase setting for dfs.client.read.shortcircuit from the conf or a system property.
605    *  This allows to specify this parameter on the command line.
606    *   If not set, default is true.
607    */
608   public boolean isReadShortCircuitOn(){
609     final String propName = "hbase.tests.use.shortcircuit.reads";
610     String readOnProp = System.getProperty(propName);
611     if (readOnProp != null){
612       return  Boolean.parseBoolean(readOnProp);
613     } else {
614       return conf.getBoolean(propName, false);
615     }
616   }
617 
618   /** Enable the short circuit read, unless configured differently.
619    * Set both HBase and HDFS settings, including skipping the hdfs checksum checks.
620    */
621   private void enableShortCircuit() {
622     if (isReadShortCircuitOn()) {
623       String curUser = System.getProperty("user.name");
624       LOG.info("read short circuit is ON for user " + curUser);
625       // read short circuit, for hdfs
626       conf.set("dfs.block.local-path-access.user", curUser);
627       // read short circuit, for hbase
628       conf.setBoolean("dfs.client.read.shortcircuit", true);
629       // Skip checking checksum, for the hdfs client and the datanode
630       conf.setBoolean("dfs.client.read.shortcircuit.skip.checksum", true);
631     } else {
632       LOG.info("read short circuit is OFF");
633     }
634   }
635 
636   private String createDirAndSetProperty(final String relPath, String property) {
637     String path = getDataTestDir(relPath).toString();
638     System.setProperty(property, path);
639     conf.set(property, path);
640     new File(path).mkdirs();
641     LOG.info("Setting " + property + " to " + path + " in system properties and HBase conf");
642     return path;
643   }
644 
645   /**
646    * Shuts down instance created by call to {@link #startMiniDFSCluster(int)}
647    * or does nothing.
648    * @throws IOException
649    */
650   public void shutdownMiniDFSCluster() throws IOException {
651     if (this.dfsCluster != null) {
652       // The below throws an exception per dn, AsynchronousCloseException.
653       this.dfsCluster.shutdown();
654       dfsCluster = null;
655       dataTestDirOnTestFS = null;
656       FSUtils.setFsDefault(this.conf, new Path("file:///"));
657     }
658   }
659 
660   /**
661    * Call this if you only want a zk cluster.
662    * @see #startMiniZKCluster() if you want zk + dfs + hbase mini cluster.
663    * @throws Exception
664    * @see #shutdownMiniZKCluster()
665    * @return zk cluster started.
666    */
667   public MiniZooKeeperCluster startMiniZKCluster() throws Exception {
668     return startMiniZKCluster(1);
669   }
670 
671   /**
672    * Call this if you only want a zk cluster.
673    * @param zooKeeperServerNum
674    * @see #startMiniZKCluster() if you want zk + dfs + hbase mini cluster.
675    * @throws Exception
676    * @see #shutdownMiniZKCluster()
677    * @return zk cluster started.
678    */
679   public MiniZooKeeperCluster startMiniZKCluster(int zooKeeperServerNum)
680       throws Exception {
681     setupClusterTestDir();
682     return startMiniZKCluster(clusterTestDir, zooKeeperServerNum);
683   }
684 
685   private MiniZooKeeperCluster startMiniZKCluster(final File dir)
686     throws Exception {
687     return startMiniZKCluster(dir,1);
688   }
689 
690   /**
691    * Start a mini ZK cluster. If the property "test.hbase.zookeeper.property.clientPort" is set
692    *  the port mentionned is used as the default port for ZooKeeper.
693    */
694   private MiniZooKeeperCluster startMiniZKCluster(final File dir,
695       int zooKeeperServerNum)
696   throws Exception {
697     if (this.zkCluster != null) {
698       throw new IOException("Cluster already running at " + dir);
699     }
700     this.passedZkCluster = false;
701     this.zkCluster = new MiniZooKeeperCluster(this.getConfiguration());
702     final int defPort = this.conf.getInt("test.hbase.zookeeper.property.clientPort", 0);
703     if (defPort > 0){
704       // If there is a port in the config file, we use it.
705       this.zkCluster.setDefaultClientPort(defPort);
706     }
707     int clientPort =   this.zkCluster.startup(dir,zooKeeperServerNum);
708     this.conf.set(HConstants.ZOOKEEPER_CLIENT_PORT,
709       Integer.toString(clientPort));
710     return this.zkCluster;
711   }
712 
713   /**
714    * Shuts down zk cluster created by call to {@link #startMiniZKCluster(File)}
715    * or does nothing.
716    * @throws IOException
717    * @see #startMiniZKCluster()
718    */
719   public void shutdownMiniZKCluster() throws IOException {
720     if (this.zkCluster != null) {
721       this.zkCluster.shutdown();
722       this.zkCluster = null;
723     }
724   }
725 
726   /**
727    * Start up a minicluster of hbase, dfs, and zookeeper.
728    * @throws Exception
729    * @return Mini hbase cluster instance created.
730    * @see {@link #shutdownMiniDFSCluster()}
731    */
732   public MiniHBaseCluster startMiniCluster() throws Exception {
733     return startMiniCluster(1, 1);
734   }
735 
736   /**
737    * Start up a minicluster of hbase, optionally dfs, and zookeeper.
738    * Modifies Configuration.  Homes the cluster data directory under a random
739    * subdirectory in a directory under System property test.build.data.
740    * Directory is cleaned up on exit.
741    * @param numSlaves Number of slaves to start up.  We'll start this many
742    * datanodes and regionservers.  If numSlaves is > 1, then make sure
743    * hbase.regionserver.info.port is -1 (i.e. no ui per regionserver) otherwise
744    * bind errors.
745    * @throws Exception
746    * @see {@link #shutdownMiniCluster()}
747    * @return Mini hbase cluster instance created.
748    */
749   public MiniHBaseCluster startMiniCluster(final int numSlaves)
750   throws Exception {
751     return startMiniCluster(1, numSlaves);
752   }
753 
754 
755   /**
756    * start minicluster
757    * @throws Exception
758    * @see {@link #shutdownMiniCluster()}
759    * @return Mini hbase cluster instance created.
760    */
761   public MiniHBaseCluster startMiniCluster(final int numMasters,
762     final int numSlaves)
763   throws Exception {
764     return startMiniCluster(numMasters, numSlaves, null);
765   }
766 
767   /**
768    * Start up a minicluster of hbase, optionally dfs, and zookeeper.
769    * Modifies Configuration.  Homes the cluster data directory under a random
770    * subdirectory in a directory under System property test.build.data.
771    * Directory is cleaned up on exit.
772    * @param numMasters Number of masters to start up.  We'll start this many
773    * hbase masters.  If numMasters > 1, you can find the active/primary master
774    * with {@link MiniHBaseCluster#getMaster()}.
775    * @param numSlaves Number of slaves to start up.  We'll start this many
776    * regionservers. If dataNodeHosts == null, this also indicates the number of
777    * datanodes to start. If dataNodeHosts != null, the number of datanodes is
778    * based on dataNodeHosts.length.
779    * If numSlaves is > 1, then make sure
780    * hbase.regionserver.info.port is -1 (i.e. no ui per regionserver) otherwise
781    * bind errors.
782    * @param dataNodeHosts hostnames DNs to run on.
783    * This is useful if you want to run datanode on distinct hosts for things
784    * like HDFS block location verification.
785    * If you start MiniDFSCluster without host names,
786    * all instances of the datanodes will have the same host name.
787    * @throws Exception
788    * @see {@link #shutdownMiniCluster()}
789    * @return Mini hbase cluster instance created.
790    */
791   public MiniHBaseCluster startMiniCluster(final int numMasters,
792       final int numSlaves, final String[] dataNodeHosts) throws Exception {
793     return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts, null, null);
794   }
795 
796   /**
797    * Same as {@link #startMiniCluster(int, int)}, but with custom number of datanodes.
798    * @param numDataNodes Number of data nodes.
799    */
800   public MiniHBaseCluster startMiniCluster(final int numMasters,
801       final int numSlaves, final int numDataNodes) throws Exception {
802     return startMiniCluster(numMasters, numSlaves, numDataNodes, null, null, null);
803   }
804 
805   /**
806    * Start up a minicluster of hbase, optionally dfs, and zookeeper.
807    * Modifies Configuration.  Homes the cluster data directory under a random
808    * subdirectory in a directory under System property test.build.data.
809    * Directory is cleaned up on exit.
810    * @param numMasters Number of masters to start up.  We'll start this many
811    * hbase masters.  If numMasters > 1, you can find the active/primary master
812    * with {@link MiniHBaseCluster#getMaster()}.
813    * @param numSlaves Number of slaves to start up.  We'll start this many
814    * regionservers. If dataNodeHosts == null, this also indicates the number of
815    * datanodes to start. If dataNodeHosts != null, the number of datanodes is
816    * based on dataNodeHosts.length.
817    * If numSlaves is > 1, then make sure
818    * hbase.regionserver.info.port is -1 (i.e. no ui per regionserver) otherwise
819    * bind errors.
820    * @param dataNodeHosts hostnames DNs to run on.
821    * This is useful if you want to run datanode on distinct hosts for things
822    * like HDFS block location verification.
823    * If you start MiniDFSCluster without host names,
824    * all instances of the datanodes will have the same host name.
825    * @param masterClass The class to use as HMaster, or null for default
826    * @param regionserverClass The class to use as HRegionServer, or null for
827    * default
828    * @throws Exception
829    * @see {@link #shutdownMiniCluster()}
830    * @return Mini hbase cluster instance created.
831    */
832   public MiniHBaseCluster startMiniCluster(final int numMasters,
833       final int numSlaves, final String[] dataNodeHosts, Class<? extends HMaster> masterClass,
834       Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)
835           throws Exception {
836     return startMiniCluster(
837         numMasters, numSlaves, numSlaves, dataNodeHosts, masterClass, regionserverClass);
838   }
839 
840   /**
841    * Same as {@link #startMiniCluster(int, int, String[], Class, Class)}, but with custom
842    * number of datanodes.
843    * @param numDataNodes Number of data nodes.
844    */
845   public MiniHBaseCluster startMiniCluster(final int numMasters,
846     final int numSlaves, int numDataNodes, final String[] dataNodeHosts,
847     Class<? extends HMaster> masterClass,
848     Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)
849   throws Exception {
850     if (dataNodeHosts != null && dataNodeHosts.length != 0) {
851       numDataNodes = dataNodeHosts.length;
852     }
853 
854     LOG.info("Starting up minicluster with " + numMasters + " master(s) and " +
855         numSlaves + " regionserver(s) and " + numDataNodes + " datanode(s)");
856 
857     // If we already put up a cluster, fail.
858     if (miniClusterRunning) {
859       throw new IllegalStateException("A mini-cluster is already running");
860     }
861     miniClusterRunning = true;
862 
863     setupClusterTestDir();
864     System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestDir.getPath());
865 
866     // Bring up mini dfs cluster. This spews a bunch of warnings about missing
867     // scheme. Complaints are 'Scheme is undefined for build/test/data/dfs/name1'.
868     startMiniDFSCluster(numDataNodes, dataNodeHosts);
869 
870     // Start up a zk cluster.
871     if (this.zkCluster == null) {
872       startMiniZKCluster(clusterTestDir);
873     }
874 
875     // Start the MiniHBaseCluster
876     return startMiniHBaseCluster(numMasters, numSlaves, masterClass, regionserverClass);
877   }
878 
879   public MiniHBaseCluster startMiniHBaseCluster(final int numMasters, final int numSlaves)
880       throws IOException, InterruptedException{
881     return startMiniHBaseCluster(numMasters, numSlaves, null, null);
882   }
883 
884   /**
885    * Starts up mini hbase cluster.  Usually used after call to
886    * {@link #startMiniCluster(int, int)} when doing stepped startup of clusters.
887    * Usually you won't want this.  You'll usually want {@link #startMiniCluster()}.
888    * @param numMasters
889    * @param numSlaves
890    * @return Reference to the hbase mini hbase cluster.
891    * @throws IOException
892    * @throws InterruptedException
893    * @see {@link #startMiniCluster()}
894    */
895   public MiniHBaseCluster startMiniHBaseCluster(final int numMasters,
896         final int numSlaves, Class<? extends HMaster> masterClass,
897         Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)
898   throws IOException, InterruptedException {
899     // Now do the mini hbase cluster.  Set the hbase.rootdir in config.
900     createRootDir();
901 
902     // These settings will make the server waits until this exact number of
903     // regions servers are connected.
904     if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1) == -1) {
905       conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, numSlaves);
906     }
907     if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1) == -1) {
908       conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, numSlaves);
909     }
910 
911     Configuration c = new Configuration(this.conf);
912     this.hbaseCluster =
913         new MiniHBaseCluster(c, numMasters, numSlaves, masterClass, regionserverClass);
914     // Don't leave here till we've done a successful scan of the hbase:meta
915     HTable t = new HTable(c, TableName.META_TABLE_NAME);
916     ResultScanner s = t.getScanner(new Scan());
917     while (s.next() != null) {
918       continue;
919     }
920     s.close();
921     t.close();
922 
923     getHBaseAdmin(); // create immediately the hbaseAdmin
924     LOG.info("Minicluster is up");
925     return (MiniHBaseCluster)this.hbaseCluster;
926   }
927 
928   /**
929    * Starts the hbase cluster up again after shutting it down previously in a
930    * test.  Use this if you want to keep dfs/zk up and just stop/start hbase.
931    * @param servers number of region servers
932    * @throws IOException
933    */
934   public void restartHBaseCluster(int servers) throws IOException, InterruptedException {
935     this.hbaseCluster = new MiniHBaseCluster(this.conf, servers);
936     // Don't leave here till we've done a successful scan of the hbase:meta
937     HTable t = new HTable(new Configuration(this.conf), TableName.META_TABLE_NAME);
938     ResultScanner s = t.getScanner(new Scan());
939     while (s.next() != null) {
940       // do nothing
941     }
942     LOG.info("HBase has been restarted");
943     s.close();
944     t.close();
945   }
946 
947   /**
948    * @return Current mini hbase cluster. Only has something in it after a call
949    * to {@link #startMiniCluster()}.
950    * @see #startMiniCluster()
951    */
952   public MiniHBaseCluster getMiniHBaseCluster() {
953     if (this.hbaseCluster == null || this.hbaseCluster instanceof MiniHBaseCluster) {
954       return (MiniHBaseCluster)this.hbaseCluster;
955     }
956     throw new RuntimeException(hbaseCluster + " not an instance of " +
957                                MiniHBaseCluster.class.getName());
958   }
959 
960   /**
961    * Stops mini hbase, zk, and hdfs clusters.
962    * @throws IOException
963    * @see {@link #startMiniCluster(int)}
964    */
965   public void shutdownMiniCluster() throws Exception {
966     LOG.info("Shutting down minicluster");
967     shutdownMiniHBaseCluster();
968     if (!this.passedZkCluster){
969       shutdownMiniZKCluster();
970     }
971     shutdownMiniDFSCluster();
972 
973     cleanupTestDir();
974     miniClusterRunning = false;
975     LOG.info("Minicluster is down");
976   }
977 
978   /**
979    * @return True if we removed the test dirs
980    * @throws IOException
981    */
982   @Override
983   public boolean cleanupTestDir() throws IOException {
984     boolean ret = super.cleanupTestDir();
985     if (deleteDir(this.clusterTestDir)) {
986       this.clusterTestDir = null;
987       return ret & true;
988     }
989     return false;
990   }
991 
992   /**
993    * Shutdown HBase mini cluster.  Does not shutdown zk or dfs if running.
994    * @throws IOException
995    */
996   public void shutdownMiniHBaseCluster() throws IOException {
997     if (hbaseAdmin != null) {
998       hbaseAdmin.close0();
999       hbaseAdmin = null;
1000     }
1001 
1002     // unset the configuration for MIN and MAX RS to start
1003     conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1);
1004     conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1);
1005     if (this.hbaseCluster != null) {
1006       this.hbaseCluster.shutdown();
1007       // Wait till hbase is down before going on to shutdown zk.
1008       this.hbaseCluster.waitUntilShutDown();
1009       this.hbaseCluster = null;
1010     }
1011 
1012     if (zooKeeperWatcher != null) {
1013       zooKeeperWatcher.close();
1014       zooKeeperWatcher = null;
1015     }
1016   }
1017 
1018   /**
1019    * Returns the path to the default root dir the minicluster uses.
1020    * Note: this does not cause the root dir to be created.
1021    * @return Fully qualified path for the default hbase root dir
1022    * @throws IOException
1023    */
1024   public Path getDefaultRootDirPath() throws IOException {
1025 	FileSystem fs = FileSystem.get(this.conf);
1026 	return new Path(fs.makeQualified(fs.getHomeDirectory()),"hbase");
1027   }
1028 
1029   /**
1030    * Creates an hbase rootdir in user home directory.  Also creates hbase
1031    * version file.  Normally you won't make use of this method.  Root hbasedir
1032    * is created for you as part of mini cluster startup.  You'd only use this
1033    * method if you were doing manual operation.
1034    * @return Fully qualified path to hbase root dir
1035    * @throws IOException
1036    */
1037   public Path createRootDir() throws IOException {
1038     FileSystem fs = FileSystem.get(this.conf);
1039     Path hbaseRootdir = getDefaultRootDirPath();
1040     FSUtils.setRootDir(this.conf, hbaseRootdir);
1041     fs.mkdirs(hbaseRootdir);
1042     FSUtils.setVersion(fs, hbaseRootdir);
1043     return hbaseRootdir;
1044   }
1045 
1046   /**
1047    * Flushes all caches in the mini hbase cluster
1048    * @throws IOException
1049    */
1050   public void flush() throws IOException {
1051     getMiniHBaseCluster().flushcache();
1052   }
1053 
1054   /**
1055    * Flushes all caches in the mini hbase cluster
1056    * @throws IOException
1057    */
1058   public void flush(TableName tableName) throws IOException {
1059     getMiniHBaseCluster().flushcache(tableName);
1060   }
1061 
1062   /**
1063    * Compact all regions in the mini hbase cluster
1064    * @throws IOException
1065    */
1066   public void compact(boolean major) throws IOException {
1067     getMiniHBaseCluster().compact(major);
1068   }
1069 
1070   /**
1071    * Compact all of a table's reagion in the mini hbase cluster
1072    * @throws IOException
1073    */
1074   public void compact(TableName tableName, boolean major) throws IOException {
1075     getMiniHBaseCluster().compact(tableName, major);
1076   }
1077 
1078   /**
1079    * Create a table.
1080    * @param tableName
1081    * @param family
1082    * @return An HTable instance for the created table.
1083    * @throws IOException
1084    */
1085   public HTable createTable(String tableName, String family)
1086   throws IOException{
1087     return createTable(TableName.valueOf(tableName), new String[]{family});
1088   }
1089 
1090   /**
1091    * Create a table.
1092    * @param tableName
1093    * @param family
1094    * @return An HTable instance for the created table.
1095    * @throws IOException
1096    */
1097   public HTable createTable(byte[] tableName, byte[] family)
1098   throws IOException{
1099     return createTable(TableName.valueOf(tableName), new byte[][]{family});
1100   }
1101 
1102   /**
1103    * Create a table.
1104    * @param tableName
1105    * @param families
1106    * @return An HTable instance for the created table.
1107    * @throws IOException
1108    */
1109   public HTable createTable(TableName tableName, String[] families)
1110   throws IOException {
1111     List<byte[]> fams = new ArrayList<byte[]>(families.length);
1112     for (String family : families) {
1113       fams.add(Bytes.toBytes(family));
1114     }
1115     return createTable(tableName, fams.toArray(new byte[0][]));
1116   }
1117 
1118   /**
1119    * Create a table.
1120    * @param tableName
1121    * @param family
1122    * @return An HTable instance for the created table.
1123    * @throws IOException
1124    */
1125   public HTable createTable(TableName tableName, byte[] family)
1126   throws IOException{
1127     return createTable(tableName, new byte[][]{family});
1128   }
1129 
1130 
1131   /**
1132    * Create a table.
1133    * @param tableName
1134    * @param families
1135    * @return An HTable instance for the created table.
1136    * @throws IOException
1137    */
1138   public HTable createTable(byte[] tableName, byte[][] families)
1139   throws IOException {
1140     return createTable(tableName, families,
1141         new Configuration(getConfiguration()));
1142   }
1143 
1144   /**
1145    * Create a table.
1146    * @param tableName
1147    * @param families
1148    * @return An HTable instance for the created table.
1149    * @throws IOException
1150    */
1151   public HTable createTable(TableName tableName, byte[][] families)
1152   throws IOException {
1153     return createTable(tableName, families,
1154         new Configuration(getConfiguration()));
1155   }
1156 
1157   public HTable createTable(byte[] tableName, byte[][] families,
1158       int numVersions, byte[] startKey, byte[] endKey, int numRegions) throws IOException {
1159     return createTable(TableName.valueOf(tableName), families, numVersions,
1160         startKey, endKey, numRegions);
1161   }
1162 
1163   public HTable createTable(String tableName, byte[][] families,
1164       int numVersions, byte[] startKey, byte[] endKey, int numRegions) throws IOException {
1165     return createTable(TableName.valueOf(tableName), families, numVersions,
1166         startKey, endKey, numRegions);
1167   }
1168 
1169   public HTable createTable(TableName tableName, byte[][] families,
1170       int numVersions, byte[] startKey, byte[] endKey, int numRegions)
1171   throws IOException{
1172     HTableDescriptor desc = new HTableDescriptor(tableName);
1173     for (byte[] family : families) {
1174       HColumnDescriptor hcd = new HColumnDescriptor(family)
1175           .setMaxVersions(numVersions);
1176       desc.addFamily(hcd);
1177     }
1178     getHBaseAdmin().createTable(desc, startKey, endKey, numRegions);
1179     // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned
1180     waitUntilAllRegionsAssigned(tableName);
1181     return new HTable(getConfiguration(), tableName);
1182   }
1183 
1184   /**
1185    * Create a table.
1186    * @param htd
1187    * @param families
1188    * @param c Configuration to use
1189    * @return An HTable instance for the created table.
1190    * @throws IOException
1191    */
1192   public HTable createTable(HTableDescriptor htd, byte[][] families, Configuration c)
1193   throws IOException {
1194     for(byte[] family : families) {
1195       HColumnDescriptor hcd = new HColumnDescriptor(family);
1196       // Disable blooms (they are on by default as of 0.95) but we disable them here because
1197       // tests have hard coded counts of what to expect in block cache, etc., and blooms being
1198       // on is interfering.
1199       hcd.setBloomFilterType(BloomType.NONE);
1200       htd.addFamily(hcd);
1201     }
1202     getHBaseAdmin().createTable(htd);
1203     // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned
1204     waitUntilAllRegionsAssigned(htd.getTableName());
1205     return new HTable(c, htd.getTableName());
1206   }
1207 
1208   /**
1209    * Create a table.
1210    * @param tableName
1211    * @param families
1212    * @param c Configuration to use
1213    * @return An HTable instance for the created table.
1214    * @throws IOException
1215    */
1216   public HTable createTable(TableName tableName, byte[][] families,
1217       final Configuration c)
1218   throws IOException {
1219     return createTable(new HTableDescriptor(tableName), families, c);
1220   }
1221 
1222   /**
1223    * Create a table.
1224    * @param tableName
1225    * @param families
1226    * @param c Configuration to use
1227    * @return An HTable instance for the created table.
1228    * @throws IOException
1229    */
1230   public HTable createTable(byte[] tableName, byte[][] families,
1231       final Configuration c)
1232   throws IOException {
1233     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
1234     for(byte[] family : families) {
1235       HColumnDescriptor hcd = new HColumnDescriptor(family);
1236       // Disable blooms (they are on by default as of 0.95) but we disable them here because
1237       // tests have hard coded counts of what to expect in block cache, etc., and blooms being
1238       // on is interfering.
1239       hcd.setBloomFilterType(BloomType.NONE);
1240       desc.addFamily(hcd);
1241     }
1242     getHBaseAdmin().createTable(desc);
1243     return new HTable(c, tableName);
1244   }
1245 
1246   /**
1247    * Create a table.
1248    * @param tableName
1249    * @param families
1250    * @param c Configuration to use
1251    * @param numVersions
1252    * @return An HTable instance for the created table.
1253    * @throws IOException
1254    */
1255   public HTable createTable(TableName tableName, byte[][] families,
1256       final Configuration c, int numVersions)
1257   throws IOException {
1258     HTableDescriptor desc = new HTableDescriptor(tableName);
1259     for(byte[] family : families) {
1260       HColumnDescriptor hcd = new HColumnDescriptor(family)
1261           .setMaxVersions(numVersions);
1262       desc.addFamily(hcd);
1263     }
1264     getHBaseAdmin().createTable(desc);
1265     // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned
1266     waitUntilAllRegionsAssigned(tableName);
1267     return new HTable(c, tableName);
1268   }
1269 
1270   /**
1271    * Create a table.
1272    * @param tableName
1273    * @param families
1274    * @param c Configuration to use
1275    * @param numVersions
1276    * @return An HTable instance for the created table.
1277    * @throws IOException
1278    */
1279   public HTable createTable(byte[] tableName, byte[][] families,
1280       final Configuration c, int numVersions)
1281   throws IOException {
1282     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
1283     for(byte[] family : families) {
1284       HColumnDescriptor hcd = new HColumnDescriptor(family)
1285           .setMaxVersions(numVersions);
1286       desc.addFamily(hcd);
1287     }
1288     getHBaseAdmin().createTable(desc);
1289     return new HTable(c, tableName);
1290   }
1291 
1292   /**
1293    * Create a table.
1294    * @param tableName
1295    * @param family
1296    * @param numVersions
1297    * @return An HTable instance for the created table.
1298    * @throws IOException
1299    */
1300   public HTable createTable(byte[] tableName, byte[] family, int numVersions)
1301   throws IOException {
1302     return createTable(tableName, new byte[][]{family}, numVersions);
1303   }
1304 
1305   /**
1306    * Create a table.
1307    * @param tableName
1308    * @param family
1309    * @param numVersions
1310    * @return An HTable instance for the created table.
1311    * @throws IOException
1312    */
1313   public HTable createTable(TableName tableName, byte[] family, int numVersions)
1314   throws IOException {
1315     return createTable(tableName, new byte[][]{family}, numVersions);
1316   }
1317 
1318   /**
1319    * Create a table.
1320    * @param tableName
1321    * @param families
1322    * @param numVersions
1323    * @return An HTable instance for the created table.
1324    * @throws IOException
1325    */
1326   public HTable createTable(byte[] tableName, byte[][] families,
1327       int numVersions)
1328   throws IOException {
1329     return createTable(TableName.valueOf(tableName), families, numVersions);
1330   }
1331 
1332   /**
1333    * Create a table.
1334    * @param tableName
1335    * @param families
1336    * @param numVersions
1337    * @return An HTable instance for the created table.
1338    * @throws IOException
1339    */
1340   public HTable createTable(TableName tableName, byte[][] families,
1341       int numVersions)
1342   throws IOException {
1343     HTableDescriptor desc = new HTableDescriptor(tableName);
1344     for (byte[] family : families) {
1345       HColumnDescriptor hcd = new HColumnDescriptor(family).setMaxVersions(numVersions);
1346       desc.addFamily(hcd);
1347     }
1348     getHBaseAdmin().createTable(desc);
1349     // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned
1350     waitUntilAllRegionsAssigned(tableName);
1351     return new HTable(new Configuration(getConfiguration()), tableName);
1352   }
1353 
1354   /**
1355    * Create a table.
1356    * @param tableName
1357    * @param families
1358    * @param numVersions
1359    * @return An HTable instance for the created table.
1360    * @throws IOException
1361    */
1362   public HTable createTable(byte[] tableName, byte[][] families,
1363     int numVersions, int blockSize) throws IOException {
1364     return createTable(TableName.valueOf(tableName),
1365         families, numVersions, blockSize);
1366   }
1367 
1368   /**
1369    * Create a table.
1370    * @param tableName
1371    * @param families
1372    * @param numVersions
1373    * @return An HTable instance for the created table.
1374    * @throws IOException
1375    */
1376   public HTable createTable(TableName tableName, byte[][] families,
1377     int numVersions, int blockSize) throws IOException {
1378     HTableDescriptor desc = new HTableDescriptor(tableName);
1379     for (byte[] family : families) {
1380       HColumnDescriptor hcd = new HColumnDescriptor(family)
1381           .setMaxVersions(numVersions)
1382           .setBlocksize(blockSize);
1383       desc.addFamily(hcd);
1384     }
1385     getHBaseAdmin().createTable(desc);
1386     // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned
1387     waitUntilAllRegionsAssigned(tableName);
1388     return new HTable(new Configuration(getConfiguration()), tableName);
1389   }
1390 
1391   /**
1392    * Create a table.
1393    * @param tableName
1394    * @param families
1395    * @param numVersions
1396    * @return An HTable instance for the created table.
1397    * @throws IOException
1398    */
1399   public HTable createTable(byte[] tableName, byte[][] families,
1400       int[] numVersions)
1401   throws IOException {
1402     return createTable(TableName.valueOf(tableName), families, numVersions);
1403   }
1404 
1405   /**
1406    * Create a table.
1407    * @param tableName
1408    * @param families
1409    * @param numVersions
1410    * @return An HTable instance for the created table.
1411    * @throws IOException
1412    */
1413   public HTable createTable(TableName tableName, byte[][] families,
1414       int[] numVersions)
1415   throws IOException {
1416     HTableDescriptor desc = new HTableDescriptor(tableName);
1417     int i = 0;
1418     for (byte[] family : families) {
1419       HColumnDescriptor hcd = new HColumnDescriptor(family)
1420           .setMaxVersions(numVersions[i]);
1421       desc.addFamily(hcd);
1422       i++;
1423     }
1424     getHBaseAdmin().createTable(desc);
1425     // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned
1426     waitUntilAllRegionsAssigned(tableName);
1427     return new HTable(new Configuration(getConfiguration()), tableName);
1428   }
1429 
1430   /**
1431    * Create a table.
1432    * @param tableName
1433    * @param family
1434    * @param splitRows
1435    * @return An HTable instance for the created table.
1436    * @throws IOException
1437    */
1438   public HTable createTable(byte[] tableName, byte[] family, byte[][] splitRows)
1439     throws IOException{
1440     return createTable(TableName.valueOf(tableName), family, splitRows);
1441   }
1442 
1443   /**
1444    * Create a table.
1445    * @param tableName
1446    * @param family
1447    * @param splitRows
1448    * @return An HTable instance for the created table.
1449    * @throws IOException
1450    */
1451   public HTable createTable(TableName tableName, byte[] family, byte[][] splitRows)
1452       throws IOException {
1453     HTableDescriptor desc = new HTableDescriptor(tableName);
1454     HColumnDescriptor hcd = new HColumnDescriptor(family);
1455     desc.addFamily(hcd);
1456     getHBaseAdmin().createTable(desc, splitRows);
1457     // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned
1458     waitUntilAllRegionsAssigned(tableName);
1459     return new HTable(getConfiguration(), tableName);
1460   }
1461 
1462   /**
1463    * Create a table.
1464    * @param tableName
1465    * @param families
1466    * @param splitRows
1467    * @return An HTable instance for the created table.
1468    * @throws IOException
1469    */
1470   public HTable createTable(byte[] tableName, byte[][] families, byte[][] splitRows)
1471       throws IOException {
1472     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
1473     for(byte[] family:families) {
1474       HColumnDescriptor hcd = new HColumnDescriptor(family);
1475       desc.addFamily(hcd);
1476     }
1477     getHBaseAdmin().createTable(desc, splitRows);
1478     // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned
1479     waitUntilAllRegionsAssigned(TableName.valueOf(tableName));
1480     return new HTable(getConfiguration(), tableName);
1481   }
1482 
1483   /**
1484    * Drop an existing table
1485    * @param tableName existing table
1486    */
1487   public void deleteTable(String tableName) throws IOException {
1488     deleteTable(TableName.valueOf(tableName));
1489   }
1490 
1491   /**
1492    * Drop an existing table
1493    * @param tableName existing table
1494    */
1495   public void deleteTable(byte[] tableName) throws IOException {
1496     deleteTable(TableName.valueOf(tableName));
1497   }
1498 
1499   /**
1500    * Drop an existing table
1501    * @param tableName existing table
1502    */
1503   public void deleteTable(TableName tableName) throws IOException {
1504     try {
1505       getHBaseAdmin().disableTable(tableName);
1506     } catch (TableNotEnabledException e) {
1507       LOG.debug("Table: " + tableName + " already disabled, so just deleting it.");
1508     }
1509     getHBaseAdmin().deleteTable(tableName);
1510   }
1511 
1512   // ==========================================================================
1513   // Canned table and table descriptor creation
1514   // TODO replace HBaseTestCase
1515 
1516   public final static byte [] fam1 = Bytes.toBytes("colfamily11");
1517   public final static byte [] fam2 = Bytes.toBytes("colfamily21");
1518   public final static byte [] fam3 = Bytes.toBytes("colfamily31");
1519   public static final byte[][] COLUMNS = {fam1, fam2, fam3};
1520   private static final int MAXVERSIONS = 3;
1521 
1522   public static final char FIRST_CHAR = 'a';
1523   public static final char LAST_CHAR = 'z';
1524   public static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
1525   public static final String START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET);
1526 
1527   /**
1528    * Create a table of name <code>name</code> with {@link COLUMNS} for
1529    * families.
1530    * @param name Name to give table.
1531    * @param versions How many versions to allow per column.
1532    * @return Column descriptor.
1533    */
1534   public HTableDescriptor createTableDescriptor(final String name,
1535       final int minVersions, final int versions, final int ttl, boolean keepDeleted) {
1536     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
1537     for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
1538       htd.addFamily(new HColumnDescriptor(cfName)
1539           .setMinVersions(minVersions)
1540           .setMaxVersions(versions)
1541           .setKeepDeletedCells(keepDeleted)
1542           .setBlockCacheEnabled(false)
1543           .setTimeToLive(ttl)
1544       );
1545     }
1546     return htd;
1547   }
1548 
1549   /**
1550    * Create a table of name <code>name</code> with {@link COLUMNS} for
1551    * families.
1552    * @param name Name to give table.
1553    * @return Column descriptor.
1554    */
1555   public HTableDescriptor createTableDescriptor(final String name) {
1556     return createTableDescriptor(name,  HColumnDescriptor.DEFAULT_MIN_VERSIONS,
1557         MAXVERSIONS, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
1558   }
1559 
1560   /**
1561    * Create an HRegion that writes to the local tmp dirs
1562    * @param desc
1563    * @param startKey
1564    * @param endKey
1565    * @return
1566    * @throws IOException
1567    */
1568   public HRegion createLocalHRegion(HTableDescriptor desc, byte [] startKey,
1569       byte [] endKey)
1570   throws IOException {
1571     HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
1572     return createLocalHRegion(hri, desc);
1573   }
1574 
1575   /**
1576    * Create an HRegion that writes to the local tmp dirs
1577    * @param info
1578    * @param desc
1579    * @return
1580    * @throws IOException
1581    */
1582   public HRegion createLocalHRegion(HRegionInfo info, HTableDescriptor desc) throws IOException {
1583     return HRegion.createHRegion(info, getDataTestDir(), getConfiguration(), desc);
1584   }
1585 
1586   /**
1587    * Create an HRegion that writes to the local tmp dirs with specified hlog
1588    * @param info regioninfo
1589    * @param desc table descriptor
1590    * @param hlog hlog for this region.
1591    * @return created hregion
1592    * @throws IOException
1593    */
1594   public HRegion createLocalHRegion(HRegionInfo info, HTableDescriptor desc, HLog hlog) throws IOException {
1595     return HRegion.createHRegion(info, getDataTestDir(), getConfiguration(), desc, hlog);
1596   }
1597 
1598 
1599   /**
1600    * @param tableName
1601    * @param startKey
1602    * @param stopKey
1603    * @param callingMethod
1604    * @param conf
1605    * @param isReadOnly
1606    * @param families
1607    * @throws IOException
1608    * @return A region on which you must call
1609    *         {@link HRegion#closeHRegion(HRegion)} when done.
1610    */
1611   public HRegion createLocalHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
1612       String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
1613       HLog hlog, byte[]... families) throws IOException {
1614     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
1615     htd.setReadOnly(isReadOnly);
1616     for (byte[] family : families) {
1617       HColumnDescriptor hcd = new HColumnDescriptor(family);
1618       // Set default to be three versions.
1619       hcd.setMaxVersions(Integer.MAX_VALUE);
1620       htd.addFamily(hcd);
1621     }
1622     htd.setDurability(durability);
1623     HRegionInfo info = new HRegionInfo(htd.getTableName(), startKey, stopKey, false);
1624     return createLocalHRegion(info, htd, hlog);
1625   }
1626   //
1627   // ==========================================================================
1628 
1629   /**
1630    * Provide an existing table name to truncate
1631    * @param tableName existing table
1632    * @return HTable to that new table
1633    * @throws IOException
1634    */
1635   public HTable truncateTable(byte[] tableName) throws IOException {
1636     return truncateTable(TableName.valueOf(tableName));
1637   }
1638 
1639   /**
1640    * Provide an existing table name to truncate
1641    * @param tableName existing table
1642    * @return HTable to that new table
1643    * @throws IOException
1644    */
1645   public HTable truncateTable(TableName tableName) throws IOException {
1646     HTable table = new HTable(getConfiguration(), tableName);
1647     Scan scan = new Scan();
1648     ResultScanner resScan = table.getScanner(scan);
1649     for(Result res : resScan) {
1650       Delete del = new Delete(res.getRow());
1651       table.delete(del);
1652     }
1653     resScan = table.getScanner(scan);
1654     resScan.close();
1655     return table;
1656   }
1657 
1658   /**
1659    * Load table with rows from 'aaa' to 'zzz'.
1660    * @param t Table
1661    * @param f Family
1662    * @return Count of rows loaded.
1663    * @throws IOException
1664    */
1665   public int loadTable(final HTable t, final byte[] f) throws IOException {
1666     return loadTable(t, new byte[][] {f});
1667   }
1668 
1669   /**
1670    * Load table with rows from 'aaa' to 'zzz'.
1671    * @param t Table
1672    * @param f Family
1673    * @return Count of rows loaded.
1674    * @throws IOException
1675    */
1676   public int loadTable(final HTable t, final byte[] f, boolean writeToWAL) throws IOException {
1677     return loadTable(t, new byte[][] {f}, null, writeToWAL);
1678   }
1679 
1680   /**
1681    * Load table of multiple column families with rows from 'aaa' to 'zzz'.
1682    * @param t Table
1683    * @param f Array of Families to load
1684    * @return Count of rows loaded.
1685    * @throws IOException
1686    */
1687   public int loadTable(final HTable t, final byte[][] f) throws IOException {
1688     return loadTable(t, f, null);
1689   }
1690 
1691   /**
1692    * Load table of multiple column families with rows from 'aaa' to 'zzz'.
1693    * @param t Table
1694    * @param f Array of Families to load
1695    * @param value the values of the cells. If null is passed, the row key is used as value
1696    * @return Count of rows loaded.
1697    * @throws IOException
1698    */
1699   public int loadTable(final HTable t, final byte[][] f, byte[] value) throws IOException {
1700     return loadTable(t, f, value, true);
1701   }
1702 
1703   /**
1704    * Load table of multiple column families with rows from 'aaa' to 'zzz'.
1705    * @param t Table
1706    * @param f Array of Families to load
1707    * @param value the values of the cells. If null is passed, the row key is used as value
1708    * @return Count of rows loaded.
1709    * @throws IOException
1710    */
1711   public int loadTable(final HTable t, final byte[][] f, byte[] value, boolean writeToWAL) throws IOException {
1712     t.setAutoFlush(false);
1713     int rowCount = 0;
1714     for (byte[] row : HBaseTestingUtility.ROWS) {
1715       Put put = new Put(row);
1716       put.setDurability(writeToWAL ? Durability.USE_DEFAULT : Durability.SKIP_WAL);
1717       for (int i = 0; i < f.length; i++) {
1718         put.add(f[i], null, value != null ? value : row);
1719       }
1720       t.put(put);
1721       rowCount++;
1722     }
1723     t.flushCommits();
1724     return rowCount;
1725   }
1726 
1727   /** A tracker for tracking and validating table rows
1728    * generated with {@link HBaseTestingUtility#loadTable(HTable, byte[])}
1729    */
1730   public static class SeenRowTracker {
1731     int dim = 'z' - 'a' + 1;
1732     int[][][] seenRows = new int[dim][dim][dim]; //count of how many times the row is seen
1733     byte[] startRow;
1734     byte[] stopRow;
1735 
1736     public SeenRowTracker(byte[] startRow, byte[] stopRow) {
1737       this.startRow = startRow;
1738       this.stopRow = stopRow;
1739     }
1740 
1741     void reset() {
1742       for (byte[] row : ROWS) {
1743         seenRows[i(row[0])][i(row[1])][i(row[2])] = 0;
1744       }
1745     }
1746 
1747     int i(byte b) {
1748       return b - 'a';
1749     }
1750 
1751     public void addRow(byte[] row) {
1752       seenRows[i(row[0])][i(row[1])][i(row[2])]++;
1753     }
1754 
1755     /** Validate that all the rows between startRow and stopRow are seen exactly once, and
1756      * all other rows none
1757      */
1758     public void validate() {
1759       for (byte b1 = 'a'; b1 <= 'z'; b1++) {
1760         for (byte b2 = 'a'; b2 <= 'z'; b2++) {
1761           for (byte b3 = 'a'; b3 <= 'z'; b3++) {
1762             int count = seenRows[i(b1)][i(b2)][i(b3)];
1763             int expectedCount = 0;
1764             if (Bytes.compareTo(new byte[] {b1,b2,b3}, startRow) >= 0
1765                 && Bytes.compareTo(new byte[] {b1,b2,b3}, stopRow) < 0) {
1766               expectedCount = 1;
1767             }
1768             if (count != expectedCount) {
1769               String row = new String(new byte[] {b1,b2,b3});
1770               throw new RuntimeException("Row:" + row + " has a seen count of " + count + " instead of " + expectedCount);
1771             }
1772           }
1773         }
1774       }
1775     }
1776   }
1777 
1778   public int loadRegion(final HRegion r, final byte[] f) throws IOException {
1779     return loadRegion(r, f, false);
1780   }
1781 
1782   /**
1783    * Load region with rows from 'aaa' to 'zzz'.
1784    * @param r Region
1785    * @param f Family
1786    * @param flush flush the cache if true
1787    * @return Count of rows loaded.
1788    * @throws IOException
1789    */
1790   public int loadRegion(final HRegion r, final byte[] f, final boolean flush)
1791   throws IOException {
1792     byte[] k = new byte[3];
1793     int rowCount = 0;
1794     for (byte b1 = 'a'; b1 <= 'z'; b1++) {
1795       for (byte b2 = 'a'; b2 <= 'z'; b2++) {
1796         for (byte b3 = 'a'; b3 <= 'z'; b3++) {
1797           k[0] = b1;
1798           k[1] = b2;
1799           k[2] = b3;
1800           Put put = new Put(k);
1801           put.setDurability(Durability.SKIP_WAL);
1802           put.add(f, null, k);
1803           if (r.getLog() == null) put.setDurability(Durability.SKIP_WAL);
1804 
1805           int preRowCount = rowCount;
1806           int pause = 10;
1807           int maxPause = 1000;
1808           while (rowCount == preRowCount) {
1809             try {
1810               r.put(put);
1811               rowCount++;
1812             } catch (RegionTooBusyException e) {
1813               pause = (pause * 2 >= maxPause) ? maxPause : pause * 2;
1814               Threads.sleep(pause);
1815             }
1816           }
1817         }
1818       }
1819       if (flush) {
1820         r.flushcache();
1821       }
1822     }
1823     return rowCount;
1824   }
1825 
1826   public void loadNumericRows(final HTable t, final byte[] f, int startRow, int endRow) throws IOException {
1827     for (int i = startRow; i < endRow; i++) {
1828       byte[] data = Bytes.toBytes(String.valueOf(i));
1829       Put put = new Put(data);
1830       put.add(f, null, data);
1831       t.put(put);
1832     }
1833   }
1834 
1835   /**
1836    * Return the number of rows in the given table.
1837    */
1838   public int countRows(final HTable table) throws IOException {
1839     Scan scan = new Scan();
1840     ResultScanner results = table.getScanner(scan);
1841     int count = 0;
1842     for (@SuppressWarnings("unused") Result res : results) {
1843       count++;
1844     }
1845     results.close();
1846     return count;
1847   }
1848 
1849   public int countRows(final HTable table, final byte[]... families) throws IOException {
1850     Scan scan = new Scan();
1851     for (byte[] family: families) {
1852       scan.addFamily(family);
1853     }
1854     ResultScanner results = table.getScanner(scan);
1855     int count = 0;
1856     for (@SuppressWarnings("unused") Result res : results) {
1857       count++;
1858     }
1859     results.close();
1860     return count;
1861   }
1862 
1863   /**
1864    * Return an md5 digest of the entire contents of a table.
1865    */
1866   public String checksumRows(final HTable table) throws Exception {
1867     Scan scan = new Scan();
1868     ResultScanner results = table.getScanner(scan);
1869     MessageDigest digest = MessageDigest.getInstance("MD5");
1870     for (Result res : results) {
1871       digest.update(res.getRow());
1872     }
1873     results.close();
1874     return digest.toString();
1875   }
1876 
1877   /**
1878    * Creates many regions names "aaa" to "zzz".
1879    *
1880    * @param table  The table to use for the data.
1881    * @param columnFamily  The family to insert the data into.
1882    * @return count of regions created.
1883    * @throws IOException When creating the regions fails.
1884    */
1885   public int createMultiRegions(HTable table, byte[] columnFamily)
1886   throws IOException {
1887     return createMultiRegions(getConfiguration(), table, columnFamily);
1888   }
1889 
1890   /** All the row values for the data loaded by {@link #loadTable(HTable, byte[])} */
1891   public static final byte[][] ROWS = new byte[(int) Math.pow('z' - 'a' + 1, 3)][3]; // ~52KB
1892   static {
1893     int i = 0;
1894     for (byte b1 = 'a'; b1 <= 'z'; b1++) {
1895       for (byte b2 = 'a'; b2 <= 'z'; b2++) {
1896         for (byte b3 = 'a'; b3 <= 'z'; b3++) {
1897           ROWS[i][0] = b1;
1898           ROWS[i][1] = b2;
1899           ROWS[i][2] = b3;
1900           i++;
1901         }
1902       }
1903     }
1904   }
1905 
1906   public static final byte[][] KEYS = {
1907     HConstants.EMPTY_BYTE_ARRAY, Bytes.toBytes("bbb"),
1908     Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"),
1909     Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
1910     Bytes.toBytes("iii"), Bytes.toBytes("jjj"), Bytes.toBytes("kkk"),
1911     Bytes.toBytes("lll"), Bytes.toBytes("mmm"), Bytes.toBytes("nnn"),
1912     Bytes.toBytes("ooo"), Bytes.toBytes("ppp"), Bytes.toBytes("qqq"),
1913     Bytes.toBytes("rrr"), Bytes.toBytes("sss"), Bytes.toBytes("ttt"),
1914     Bytes.toBytes("uuu"), Bytes.toBytes("vvv"), Bytes.toBytes("www"),
1915     Bytes.toBytes("xxx"), Bytes.toBytes("yyy")
1916   };
1917 
1918   public static final byte[][] KEYS_FOR_HBA_CREATE_TABLE = {
1919       Bytes.toBytes("bbb"),
1920       Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"),
1921       Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
1922       Bytes.toBytes("iii"), Bytes.toBytes("jjj"), Bytes.toBytes("kkk"),
1923       Bytes.toBytes("lll"), Bytes.toBytes("mmm"), Bytes.toBytes("nnn"),
1924       Bytes.toBytes("ooo"), Bytes.toBytes("ppp"), Bytes.toBytes("qqq"),
1925       Bytes.toBytes("rrr"), Bytes.toBytes("sss"), Bytes.toBytes("ttt"),
1926       Bytes.toBytes("uuu"), Bytes.toBytes("vvv"), Bytes.toBytes("www"),
1927       Bytes.toBytes("xxx"), Bytes.toBytes("yyy"), Bytes.toBytes("zzz")
1928   };
1929 
1930   /**
1931    * Creates many regions names "aaa" to "zzz".
1932    * @param c Configuration to use.
1933    * @param table  The table to use for the data.
1934    * @param columnFamily  The family to insert the data into.
1935    * @return count of regions created.
1936    * @throws IOException When creating the regions fails.
1937    */
1938   public int createMultiRegions(final Configuration c, final HTable table,
1939       final byte[] columnFamily)
1940   throws IOException {
1941     return createMultiRegions(c, table, columnFamily, KEYS);
1942   }
1943 
1944   void makeDFSClientNonRetrying() {
1945     if (null == this.dfsCluster) {
1946       LOG.debug("dfsCluster has not started, can't make client non-retrying.");
1947       return;
1948     }
1949     try {
1950       final FileSystem filesystem = this.dfsCluster.getFileSystem();
1951       if (!(filesystem instanceof DistributedFileSystem)) {
1952         LOG.debug("dfsCluster is not backed by a DistributedFileSystem, can't make client non-retrying.");
1953         return;
1954       }
1955       // rely on FileSystem.CACHE to alter how we talk via DFSClient
1956       final DistributedFileSystem fs = (DistributedFileSystem)filesystem;
1957       // retrieve the backing DFSClient instance
1958       final Field dfsField = fs.getClass().getDeclaredField("dfs");
1959       dfsField.setAccessible(true);
1960       final Class<?> dfsClazz = dfsField.getType();
1961       final DFSClient dfs = DFSClient.class.cast(dfsField.get(fs));
1962 
1963       // expose the method for creating direct RPC connections.
1964       final Method createRPCNamenode = dfsClazz.getDeclaredMethod("createRPCNamenode", InetSocketAddress.class, Configuration.class, UserGroupInformation.class);
1965       createRPCNamenode.setAccessible(true);
1966 
1967       // grab the DFSClient instance's backing connection information
1968       final Field nnField = dfsClazz.getDeclaredField("nnAddress");
1969       nnField.setAccessible(true);
1970       final InetSocketAddress nnAddress = InetSocketAddress.class.cast(nnField.get(dfs));
1971       final Field confField = dfsClazz.getDeclaredField("conf");
1972       confField.setAccessible(true);
1973       final Configuration conf = Configuration.class.cast(confField.get(dfs));
1974       final Field ugiField = dfsClazz.getDeclaredField("ugi");
1975       ugiField.setAccessible(true);
1976       final UserGroupInformation ugi = UserGroupInformation.class.cast(ugiField.get(dfs));
1977 
1978       // replace the proxy for the namenode rpc with a direct instance
1979       final Field namenodeField = dfsClazz.getDeclaredField("namenode");
1980       namenodeField.setAccessible(true);
1981       namenodeField.set(dfs, createRPCNamenode.invoke(null, nnAddress, conf, ugi));
1982       LOG.debug("Set DSFClient namenode to bare RPC");
1983     } catch (Exception exception) {
1984       LOG.info("Could not alter DFSClient to be non-retrying.", exception);
1985     }
1986   }
1987 
1988   /**
1989    * Creates the specified number of regions in the specified table.
1990    * @param c
1991    * @param table
1992    * @param family
1993    * @param numRegions
1994    * @return
1995    * @throws IOException
1996    */
1997   public int createMultiRegions(final Configuration c, final HTable table,
1998       final byte [] family, int numRegions)
1999   throws IOException {
2000     if (numRegions < 3) throw new IOException("Must create at least 3 regions");
2001     byte [] startKey = Bytes.toBytes("aaaaa");
2002     byte [] endKey = Bytes.toBytes("zzzzz");
2003     byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
2004     byte [][] regionStartKeys = new byte[splitKeys.length+1][];
2005     System.arraycopy(splitKeys, 0, regionStartKeys, 1, splitKeys.length);
2006     regionStartKeys[0] = HConstants.EMPTY_BYTE_ARRAY;
2007     return createMultiRegions(c, table, family, regionStartKeys);
2008   }
2009 
2010   @SuppressWarnings("deprecation")
2011   public int createMultiRegions(final Configuration c, final HTable table,
2012       final byte[] columnFamily, byte [][] startKeys)
2013   throws IOException {
2014     Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR);
2015     HTable meta = new HTable(c, TableName.META_TABLE_NAME);
2016     HTableDescriptor htd = table.getTableDescriptor();
2017     if(!htd.hasFamily(columnFamily)) {
2018       HColumnDescriptor hcd = new HColumnDescriptor(columnFamily);
2019       htd.addFamily(hcd);
2020     }
2021     // remove empty region - this is tricky as the mini cluster during the test
2022     // setup already has the "<tablename>,,123456789" row with an empty start
2023     // and end key. Adding the custom regions below adds those blindly,
2024     // including the new start region from empty to "bbb". lg
2025     List<byte[]> rows = getMetaTableRows(htd.getTableName());
2026     String regionToDeleteInFS = table
2027         .getRegionsInRange(Bytes.toBytes(""), Bytes.toBytes("")).get(0)
2028         .getRegionInfo().getEncodedName();
2029     List<HRegionInfo> newRegions = new ArrayList<HRegionInfo>(startKeys.length);
2030     // add custom ones
2031     int count = 0;
2032     for (int i = 0; i < startKeys.length; i++) {
2033       int j = (i + 1) % startKeys.length;
2034       HRegionInfo hri = new HRegionInfo(table.getName(),
2035         startKeys[i], startKeys[j]);
2036       MetaEditor.addRegionToMeta(meta, hri);
2037       newRegions.add(hri);
2038       count++;
2039     }
2040     // see comment above, remove "old" (or previous) single region
2041     for (byte[] row : rows) {
2042       LOG.info("createMultiRegions: deleting meta row -> " +
2043         Bytes.toStringBinary(row));
2044       meta.delete(new Delete(row));
2045     }
2046     // remove the "old" region from FS
2047     Path tableDir = new Path(getDefaultRootDirPath().toString()
2048         + System.getProperty("file.separator") + htd.getTableName()
2049         + System.getProperty("file.separator") + regionToDeleteInFS);
2050     FileSystem.get(c).delete(tableDir);
2051     // flush cache of regions
2052     HConnection conn = table.getConnection();
2053     conn.clearRegionCache();
2054     // assign all the new regions IF table is enabled.
2055     HBaseAdmin admin = getHBaseAdmin();
2056     if (admin.isTableEnabled(table.getTableName())) {
2057       for(HRegionInfo hri : newRegions) {
2058         admin.assign(hri.getRegionName());
2059       }
2060     }
2061 
2062     meta.close();
2063 
2064     return count;
2065   }
2066 
2067   /**
2068    * Create rows in hbase:meta for regions of the specified table with the specified
2069    * start keys.  The first startKey should be a 0 length byte array if you
2070    * want to form a proper range of regions.
2071    * @param conf
2072    * @param htd
2073    * @param startKeys
2074    * @return list of region info for regions added to meta
2075    * @throws IOException
2076    */
2077   public List<HRegionInfo> createMultiRegionsInMeta(final Configuration conf,
2078       final HTableDescriptor htd, byte [][] startKeys)
2079   throws IOException {
2080     HTable meta = new HTable(conf, TableName.META_TABLE_NAME);
2081     Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR);
2082     List<HRegionInfo> newRegions = new ArrayList<HRegionInfo>(startKeys.length);
2083     // add custom ones
2084     for (int i = 0; i < startKeys.length; i++) {
2085       int j = (i + 1) % startKeys.length;
2086       HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKeys[i],
2087           startKeys[j]);
2088       MetaEditor.addRegionToMeta(meta, hri);
2089       newRegions.add(hri);
2090     }
2091 
2092     meta.close();
2093     return newRegions;
2094   }
2095 
2096   /**
2097    * Returns all rows from the hbase:meta table.
2098    *
2099    * @throws IOException When reading the rows fails.
2100    */
2101   public List<byte[]> getMetaTableRows() throws IOException {
2102     // TODO: Redo using MetaReader class
2103     HTable t = new HTable(new Configuration(this.conf), TableName.META_TABLE_NAME);
2104     List<byte[]> rows = new ArrayList<byte[]>();
2105     ResultScanner s = t.getScanner(new Scan());
2106     for (Result result : s) {
2107       LOG.info("getMetaTableRows: row -> " +
2108         Bytes.toStringBinary(result.getRow()));
2109       rows.add(result.getRow());
2110     }
2111     s.close();
2112     t.close();
2113     return rows;
2114   }
2115 
2116   /**
2117    * Returns all rows from the hbase:meta table for a given user table
2118    *
2119    * @throws IOException When reading the rows fails.
2120    */
2121   public List<byte[]> getMetaTableRows(TableName tableName) throws IOException {
2122     // TODO: Redo using MetaReader.
2123     HTable t = new HTable(new Configuration(this.conf), TableName.META_TABLE_NAME);
2124     List<byte[]> rows = new ArrayList<byte[]>();
2125     ResultScanner s = t.getScanner(new Scan());
2126     for (Result result : s) {
2127       HRegionInfo info = HRegionInfo.getHRegionInfo(result);
2128       if (info == null) {
2129         LOG.error("No region info for row " + Bytes.toString(result.getRow()));
2130         // TODO figure out what to do for this new hosed case.
2131         continue;
2132       }
2133 
2134       if (info.getTable().equals(tableName)) {
2135         LOG.info("getMetaTableRows: row -> " +
2136             Bytes.toStringBinary(result.getRow()) + info);
2137         rows.add(result.getRow());
2138       }
2139     }
2140     s.close();
2141     t.close();
2142     return rows;
2143   }
2144 
2145   /**
2146    * Tool to get the reference to the region server object that holds the
2147    * region of the specified user table.
2148    * It first searches for the meta rows that contain the region of the
2149    * specified table, then gets the index of that RS, and finally retrieves
2150    * the RS's reference.
2151    * @param tableName user table to lookup in hbase:meta
2152    * @return region server that holds it, null if the row doesn't exist
2153    * @throws IOException
2154    * @throws InterruptedException
2155    */
2156   public HRegionServer getRSForFirstRegionInTable(byte[] tableName)
2157       throws IOException, InterruptedException {
2158     return getRSForFirstRegionInTable(TableName.valueOf(tableName));
2159   }
2160   /**
2161    * Tool to get the reference to the region server object that holds the
2162    * region of the specified user table.
2163    * It first searches for the meta rows that contain the region of the
2164    * specified table, then gets the index of that RS, and finally retrieves
2165    * the RS's reference.
2166    * @param tableName user table to lookup in hbase:meta
2167    * @return region server that holds it, null if the row doesn't exist
2168    * @throws IOException
2169    */
2170   public HRegionServer getRSForFirstRegionInTable(TableName tableName)
2171       throws IOException, InterruptedException {
2172     List<byte[]> metaRows = getMetaTableRows(tableName);
2173     if (metaRows == null || metaRows.isEmpty()) {
2174       return null;
2175     }
2176     LOG.debug("Found " + metaRows.size() + " rows for table " +
2177       tableName);
2178     byte [] firstrow = metaRows.get(0);
2179     LOG.debug("FirstRow=" + Bytes.toString(firstrow));
2180     long pause = getConfiguration().getLong(HConstants.HBASE_CLIENT_PAUSE,
2181       HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
2182     int numRetries = getConfiguration().getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
2183       HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
2184     RetryCounter retrier = new RetryCounter(numRetries+1, (int)pause, TimeUnit.MICROSECONDS);
2185     while(retrier.shouldRetry()) {
2186       int index = getMiniHBaseCluster().getServerWith(firstrow);
2187       if (index != -1) {
2188         return getMiniHBaseCluster().getRegionServerThreads().get(index).getRegionServer();
2189       }
2190       // Came back -1.  Region may not be online yet.  Sleep a while.
2191       retrier.sleepUntilNextRetry();
2192     }
2193     return null;
2194   }
2195 
2196   /**
2197    * Starts a <code>MiniMRCluster</code> with a default number of
2198    * <code>TaskTracker</code>'s.
2199    *
2200    * @throws IOException When starting the cluster fails.
2201    */
2202   public MiniMRCluster startMiniMapReduceCluster() throws IOException {
2203     startMiniMapReduceCluster(2);
2204     return mrCluster;
2205   }
2206 
2207   /**
2208    * Tasktracker has a bug where changing the hadoop.log.dir system property
2209    * will not change its internal static LOG_DIR variable.
2210    */
2211   private void forceChangeTaskLogDir() {
2212     Field logDirField;
2213     try {
2214       logDirField = TaskLog.class.getDeclaredField("LOG_DIR");
2215       logDirField.setAccessible(true);
2216 
2217       Field modifiersField = Field.class.getDeclaredField("modifiers");
2218       modifiersField.setAccessible(true);
2219       modifiersField.setInt(logDirField, logDirField.getModifiers() & ~Modifier.FINAL);
2220 
2221       logDirField.set(null, new File(hadoopLogDir, "userlogs"));
2222     } catch (SecurityException e) {
2223       throw new RuntimeException(e);
2224     } catch (NoSuchFieldException e) {
2225       // TODO Auto-generated catch block
2226       throw new RuntimeException(e);
2227     } catch (IllegalArgumentException e) {
2228       throw new RuntimeException(e);
2229     } catch (IllegalAccessException e) {
2230       throw new RuntimeException(e);
2231     }
2232   }
2233 
2234   /**
2235    * Starts a <code>MiniMRCluster</code>. Call {@link #setFileSystemURI(String)} to use a different
2236    * filesystem.
2237    * @param servers  The number of <code>TaskTracker</code>'s to start.
2238    * @throws IOException When starting the cluster fails.
2239    */
2240   private void startMiniMapReduceCluster(final int servers) throws IOException {
2241     if (mrCluster != null) {
2242       throw new IllegalStateException("MiniMRCluster is already running");
2243     }
2244     LOG.info("Starting mini mapreduce cluster...");
2245     setupClusterTestDir();
2246     createDirsAndSetProperties();
2247 
2248     forceChangeTaskLogDir();
2249 
2250     //// hadoop2 specific settings
2251     // Tests were failing because this process used 6GB of virtual memory and was getting killed.
2252     // we up the VM usable so that processes don't get killed.
2253     conf.setFloat("yarn.nodemanager.vmem-pmem-ratio", 8.0f);
2254 
2255     // Tests were failing due to MAPREDUCE-4880 / MAPREDUCE-4607 against hadoop 2.0.2-alpha and
2256     // this avoids the problem by disabling speculative task execution in tests.
2257     conf.setBoolean("mapreduce.map.speculative", false);
2258     conf.setBoolean("mapreduce.reduce.speculative", false);
2259     ////
2260 
2261     // Allow the user to override FS URI for this map-reduce cluster to use.
2262     mrCluster = new MiniMRCluster(servers,
2263       FS_URI != null ? FS_URI : FileSystem.get(conf).getUri().toString(), 1,
2264       null, null, new JobConf(this.conf));
2265     JobConf jobConf = MapreduceTestingShim.getJobConf(mrCluster);
2266     if (jobConf == null) {
2267       jobConf = mrCluster.createJobConf();
2268     }
2269 
2270     jobConf.set("mapred.local.dir",
2271       conf.get("mapred.local.dir")); //Hadoop MiniMR overwrites this while it should not
2272     LOG.info("Mini mapreduce cluster started");
2273 
2274     // In hadoop2, YARN/MR2 starts a mini cluster with its own conf instance and updates settings.
2275     // Our HBase MR jobs need several of these settings in order to properly run.  So we copy the
2276     // necessary config properties here.  YARN-129 required adding a few properties.
2277     conf.set("mapred.job.tracker", jobConf.get("mapred.job.tracker"));
2278     // this for mrv2 support; mr1 ignores this
2279     conf.set("mapreduce.framework.name", "yarn");
2280     conf.setBoolean("yarn.is.minicluster", true);
2281     String rmAddress = jobConf.get("yarn.resourcemanager.address");
2282     if (rmAddress != null) {
2283       conf.set("yarn.resourcemanager.address", rmAddress);
2284     }
2285     String historyAddress = jobConf.get("mapreduce.jobhistory.address");
2286     if (historyAddress != null) {
2287       conf.set("mapreduce.jobhistory.address", historyAddress);
2288     }
2289     String schedulerAddress =
2290       jobConf.get("yarn.resourcemanager.scheduler.address");
2291     if (schedulerAddress != null) {
2292       conf.set("yarn.resourcemanager.scheduler.address", schedulerAddress);
2293     }
2294   }
2295 
2296   /**
2297    * Stops the previously started <code>MiniMRCluster</code>.
2298    */
2299   public void shutdownMiniMapReduceCluster() {
2300     LOG.info("Stopping mini mapreduce cluster...");
2301     if (mrCluster != null) {
2302       mrCluster.shutdown();
2303       mrCluster = null;
2304     }
2305     // Restore configuration to point to local jobtracker
2306     conf.set("mapred.job.tracker", "local");
2307     LOG.info("Mini mapreduce cluster stopped");
2308   }
2309 
2310   /**
2311    * Create a stubbed out RegionServerService, mainly for getting FS.
2312    */
2313   public RegionServerServices createMockRegionServerService() throws IOException {
2314     return createMockRegionServerService((ServerName)null);
2315   }
2316 
2317   /**
2318    * Create a stubbed out RegionServerService, mainly for getting FS.
2319    * This version is used by TestTokenAuthentication
2320    */
2321   public RegionServerServices createMockRegionServerService(RpcServerInterface rpc) throws IOException {
2322     final MockRegionServerServices rss = new MockRegionServerServices(getZooKeeperWatcher());
2323     rss.setFileSystem(getTestFileSystem());
2324     rss.setRpcServer(rpc);
2325     return rss;
2326   }
2327 
2328   /**
2329    * Create a stubbed out RegionServerService, mainly for getting FS.
2330    * This version is used by TestOpenRegionHandler
2331    */
2332   public RegionServerServices createMockRegionServerService(ServerName name) throws IOException {
2333     final MockRegionServerServices rss = new MockRegionServerServices(getZooKeeperWatcher(), name);
2334     rss.setFileSystem(getTestFileSystem());
2335     return rss;
2336   }
2337 
2338   /**
2339    * Switches the logger for the given class to DEBUG level.
2340    *
2341    * @param clazz  The class for which to switch to debug logging.
2342    */
2343   public void enableDebug(Class<?> clazz) {
2344     Log l = LogFactory.getLog(clazz);
2345     if (l instanceof Log4JLogger) {
2346       ((Log4JLogger) l).getLogger().setLevel(org.apache.log4j.Level.DEBUG);
2347     } else if (l instanceof Jdk14Logger) {
2348       ((Jdk14Logger) l).getLogger().setLevel(java.util.logging.Level.ALL);
2349     }
2350   }
2351 
2352   /**
2353    * Expire the Master's session
2354    * @throws Exception
2355    */
2356   public void expireMasterSession() throws Exception {
2357     HMaster master = getMiniHBaseCluster().getMaster();
2358     expireSession(master.getZooKeeper(), false);
2359   }
2360 
2361   /**
2362    * Expire a region server's session
2363    * @param index which RS
2364    * @throws Exception
2365    */
2366   public void expireRegionServerSession(int index) throws Exception {
2367     HRegionServer rs = getMiniHBaseCluster().getRegionServer(index);
2368     expireSession(rs.getZooKeeper(), false);
2369     decrementMinRegionServerCount();
2370   }
2371 
2372   private void decrementMinRegionServerCount() {
2373     // decrement the count for this.conf, for newly spwaned master
2374     // this.hbaseCluster shares this configuration too
2375     decrementMinRegionServerCount(getConfiguration());
2376 
2377     // each master thread keeps a copy of configuration
2378     for (MasterThread master : getHBaseCluster().getMasterThreads()) {
2379       decrementMinRegionServerCount(master.getMaster().getConfiguration());
2380     }
2381   }
2382 
2383   private void decrementMinRegionServerCount(Configuration conf) {
2384     int currentCount = conf.getInt(
2385         ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1);
2386     if (currentCount != -1) {
2387       conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
2388           Math.max(currentCount - 1, 1));
2389     }
2390   }
2391 
2392   public void expireSession(ZooKeeperWatcher nodeZK) throws Exception {
2393    expireSession(nodeZK, false);
2394   }
2395 
2396   @Deprecated
2397   public void expireSession(ZooKeeperWatcher nodeZK, Server server)
2398     throws Exception {
2399     expireSession(nodeZK, false);
2400   }
2401 
2402   /**
2403    * Expire a ZooKeeper session as recommended in ZooKeeper documentation
2404    * http://wiki.apache.org/hadoop/ZooKeeper/FAQ#A4
2405    * There are issues when doing this:
2406    * [1] http://www.mail-archive.com/dev@zookeeper.apache.org/msg01942.html
2407    * [2] https://issues.apache.org/jira/browse/ZOOKEEPER-1105
2408    *
2409    * @param nodeZK - the ZK watcher to expire
2410    * @param checkStatus - true to check if we can create an HTable with the
2411    *                    current configuration.
2412    */
2413   public void expireSession(ZooKeeperWatcher nodeZK, boolean checkStatus)
2414     throws Exception {
2415     Configuration c = new Configuration(this.conf);
2416     String quorumServers = ZKConfig.getZKQuorumServersString(c);
2417     ZooKeeper zk = nodeZK.getRecoverableZooKeeper().getZooKeeper();
2418     byte[] password = zk.getSessionPasswd();
2419     long sessionID = zk.getSessionId();
2420 
2421     // Expiry seems to be asynchronous (see comment from P. Hunt in [1]),
2422     //  so we create a first watcher to be sure that the
2423     //  event was sent. We expect that if our watcher receives the event
2424     //  other watchers on the same machine will get is as well.
2425     // When we ask to close the connection, ZK does not close it before
2426     //  we receive all the events, so don't have to capture the event, just
2427     //  closing the connection should be enough.
2428     ZooKeeper monitor = new ZooKeeper(quorumServers,
2429       1000, new org.apache.zookeeper.Watcher(){
2430       @Override
2431       public void process(WatchedEvent watchedEvent) {
2432         LOG.info("Monitor ZKW received event="+watchedEvent);
2433       }
2434     } , sessionID, password);
2435 
2436     // Making it expire
2437     ZooKeeper newZK = new ZooKeeper(quorumServers,
2438         1000, EmptyWatcher.instance, sessionID, password);
2439 
2440     //ensure that we have connection to the server before closing down, otherwise
2441     //the close session event will be eaten out before we start CONNECTING state
2442     long start = System.currentTimeMillis();
2443     while (newZK.getState() != States.CONNECTED
2444          && System.currentTimeMillis() - start < 1000) {
2445        Thread.sleep(1);
2446     }
2447     newZK.close();
2448     LOG.info("ZK Closed Session 0x" + Long.toHexString(sessionID));
2449 
2450     // Now closing & waiting to be sure that the clients get it.
2451     monitor.close();
2452 
2453     if (checkStatus) {
2454       new HTable(new Configuration(conf), TableName.META_TABLE_NAME).close();
2455     }
2456   }
2457 
2458   /**
2459    * Get the Mini HBase cluster.
2460    *
2461    * @return hbase cluster
2462    * @see #getHBaseClusterInterface()
2463    */
2464   public MiniHBaseCluster getHBaseCluster() {
2465     return getMiniHBaseCluster();
2466   }
2467 
2468   /**
2469    * Returns the HBaseCluster instance.
2470    * <p>Returned object can be any of the subclasses of HBaseCluster, and the
2471    * tests referring this should not assume that the cluster is a mini cluster or a
2472    * distributed one. If the test only works on a mini cluster, then specific
2473    * method {@link #getMiniHBaseCluster()} can be used instead w/o the
2474    * need to type-cast.
2475    */
2476   public HBaseCluster getHBaseClusterInterface() {
2477     //implementation note: we should rename this method as #getHBaseCluster(),
2478     //but this would require refactoring 90+ calls.
2479     return hbaseCluster;
2480   }
2481 
2482   /**
2483    * Returns a HBaseAdmin instance.
2484    * This instance is shared between HBaseTestingUtility instance users.
2485    * Closing it has no effect, it will be closed automatically when the
2486    * cluster shutdowns
2487    *
2488    * @return The HBaseAdmin instance.
2489    * @throws IOException
2490    */
2491   public synchronized HBaseAdmin getHBaseAdmin()
2492   throws IOException {
2493     if (hbaseAdmin == null){
2494       hbaseAdmin = new HBaseAdminForTests(getConfiguration());
2495     }
2496     return hbaseAdmin;
2497   }
2498 
2499   private HBaseAdminForTests hbaseAdmin = null;
2500   private static class HBaseAdminForTests extends HBaseAdmin {
2501     public HBaseAdminForTests(Configuration c) throws MasterNotRunningException,
2502         ZooKeeperConnectionException, IOException {
2503       super(c);
2504     }
2505 
2506     @Override
2507     public synchronized void close() throws IOException {
2508       LOG.warn("close() called on HBaseAdmin instance returned from HBaseTestingUtility.getHBaseAdmin()");
2509     }
2510 
2511     private synchronized void close0() throws IOException {
2512       super.close();
2513     }
2514   }
2515 
2516   /**
2517    * Returns a ZooKeeperWatcher instance.
2518    * This instance is shared between HBaseTestingUtility instance users.
2519    * Don't close it, it will be closed automatically when the
2520    * cluster shutdowns
2521    *
2522    * @return The ZooKeeperWatcher instance.
2523    * @throws IOException
2524    */
2525   public synchronized ZooKeeperWatcher getZooKeeperWatcher()
2526     throws IOException {
2527     if (zooKeeperWatcher == null) {
2528       zooKeeperWatcher = new ZooKeeperWatcher(conf, "testing utility",
2529         new Abortable() {
2530         @Override public void abort(String why, Throwable e) {
2531           throw new RuntimeException("Unexpected abort in HBaseTestingUtility:"+why, e);
2532         }
2533         @Override public boolean isAborted() {return false;}
2534       });
2535     }
2536     return zooKeeperWatcher;
2537   }
2538   private ZooKeeperWatcher zooKeeperWatcher;
2539 
2540 
2541 
2542   /**
2543    * Closes the named region.
2544    *
2545    * @param regionName  The region to close.
2546    * @throws IOException
2547    */
2548   public void closeRegion(String regionName) throws IOException {
2549     closeRegion(Bytes.toBytes(regionName));
2550   }
2551 
2552   /**
2553    * Closes the named region.
2554    *
2555    * @param regionName  The region to close.
2556    * @throws IOException
2557    */
2558   public void closeRegion(byte[] regionName) throws IOException {
2559     getHBaseAdmin().closeRegion(regionName, null);
2560   }
2561 
2562   /**
2563    * Closes the region containing the given row.
2564    *
2565    * @param row  The row to find the containing region.
2566    * @param table  The table to find the region.
2567    * @throws IOException
2568    */
2569   public void closeRegionByRow(String row, HTable table) throws IOException {
2570     closeRegionByRow(Bytes.toBytes(row), table);
2571   }
2572 
2573   /**
2574    * Closes the region containing the given row.
2575    *
2576    * @param row  The row to find the containing region.
2577    * @param table  The table to find the region.
2578    * @throws IOException
2579    */
2580   public void closeRegionByRow(byte[] row, HTable table) throws IOException {
2581     HRegionLocation hrl = table.getRegionLocation(row);
2582     closeRegion(hrl.getRegionInfo().getRegionName());
2583   }
2584 
2585   /*
2586    * Retrieves a splittable region randomly from tableName
2587    *
2588    * @param tableName name of table
2589    * @param maxAttempts maximum number of attempts, unlimited for value of -1
2590    * @return the HRegion chosen, null if none was found within limit of maxAttempts
2591    */
2592   public HRegion getSplittableRegion(TableName tableName, int maxAttempts) {
2593     List<HRegion> regions = getHBaseCluster().getRegions(tableName);
2594     int regCount = regions.size();
2595     Set<Integer> attempted = new HashSet<Integer>();
2596     int idx;
2597     int attempts = 0;
2598     do {
2599       regions = getHBaseCluster().getRegions(tableName);
2600       if (regCount != regions.size()) {
2601         // if there was region movement, clear attempted Set
2602         attempted.clear();
2603       }
2604       regCount = regions.size();
2605       // There are chances that before we get the region for the table from an RS the region may
2606       // be going for CLOSE.  This may be because online schema change is enabled
2607       if (regCount > 0) {
2608         idx = random.nextInt(regCount);
2609         // if we have just tried this region, there is no need to try again
2610         if (attempted.contains(idx))
2611           continue;
2612         try {
2613           regions.get(idx).checkSplit();
2614           return regions.get(idx);
2615         } catch (Exception ex) {
2616           LOG.warn("Caught exception", ex);
2617           attempted.add(idx);
2618         }
2619       }
2620       attempts++;
2621     } while (maxAttempts == -1 || attempts < maxAttempts);
2622     return null;
2623   }
2624 
2625   public MiniZooKeeperCluster getZkCluster() {
2626     return zkCluster;
2627   }
2628 
2629   public void setZkCluster(MiniZooKeeperCluster zkCluster) {
2630     this.passedZkCluster = true;
2631     this.zkCluster = zkCluster;
2632     conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkCluster.getClientPort());
2633   }
2634 
2635   public MiniDFSCluster getDFSCluster() {
2636     return dfsCluster;
2637   }
2638 
2639   public void setDFSCluster(MiniDFSCluster cluster) throws IOException {
2640     if (dfsCluster != null && dfsCluster.isClusterUp()) {
2641       throw new IOException("DFSCluster is already running! Shut it down first.");
2642     }
2643     this.dfsCluster = cluster;
2644   }
2645 
2646   public FileSystem getTestFileSystem() throws IOException {
2647     return HFileSystem.get(conf);
2648   }
2649 
2650   /**
2651    * Wait until all regions in a table have been assigned.  Waits default timeout before giving up
2652    * (30 seconds).
2653    * @param table Table to wait on.
2654    * @throws InterruptedException
2655    * @throws IOException
2656    */
2657   public void waitTableAvailable(byte[] table)
2658       throws InterruptedException, IOException {
2659     waitTableAvailable(getHBaseAdmin(), table, 30000);
2660   }
2661 
2662   public void waitTableAvailable(HBaseAdmin admin, byte[] table)
2663       throws InterruptedException, IOException {
2664     waitTableAvailable(admin, table, 30000);
2665   }
2666 
2667   /**
2668    * Wait until all regions in a table have been assigned
2669    * @param table Table to wait on.
2670    * @param timeoutMillis Timeout.
2671    * @throws InterruptedException
2672    * @throws IOException
2673    */
2674   public void waitTableAvailable(byte[] table, long timeoutMillis)
2675   throws InterruptedException, IOException {
2676     waitTableAvailable(getHBaseAdmin(), table, timeoutMillis);
2677   }
2678 
2679   public void waitTableAvailable(HBaseAdmin admin, byte[] table, long timeoutMillis)
2680   throws InterruptedException, IOException {
2681     long startWait = System.currentTimeMillis();
2682     while (!admin.isTableAvailable(table)) {
2683       assertTrue("Timed out waiting for table to become available " +
2684         Bytes.toStringBinary(table),
2685         System.currentTimeMillis() - startWait < timeoutMillis);
2686       Thread.sleep(200);
2687     }
2688   }
2689 
2690   /**
2691    * Waits for a table to be 'enabled'.  Enabled means that table is set as 'enabled' and the
2692    * regions have been all assigned.  Will timeout after default period (30 seconds)
2693    * @see #waitTableAvailable(byte[])
2694    * @param table Table to wait on.
2695    * @param table
2696    * @throws InterruptedException
2697    * @throws IOException
2698    */
2699   public void waitTableEnabled(byte[] table)
2700       throws InterruptedException, IOException {
2701     waitTableEnabled(getHBaseAdmin(), table, 30000);
2702   }
2703 
2704   public void waitTableEnabled(HBaseAdmin admin, byte[] table)
2705       throws InterruptedException, IOException {
2706     waitTableEnabled(admin, table, 30000);
2707   }
2708 
2709   /**
2710    * Waits for a table to be 'enabled'.  Enabled means that table is set as 'enabled' and the
2711    * regions have been all assigned.
2712    * @see #waitTableAvailable(byte[])
2713    * @param table Table to wait on.
2714    * @param timeoutMillis Time to wait on it being marked enabled.
2715    * @throws InterruptedException
2716    * @throws IOException
2717    */
2718   public void waitTableEnabled(byte[] table, long timeoutMillis)
2719   throws InterruptedException, IOException {
2720     waitTableEnabled(getHBaseAdmin(), table, timeoutMillis);
2721   }
2722 
2723   public void waitTableEnabled(HBaseAdmin admin, byte[] table, long timeoutMillis)
2724   throws InterruptedException, IOException {
2725     long startWait = System.currentTimeMillis();
2726     waitTableAvailable(admin, table, timeoutMillis);
2727     while (!admin.isTableEnabled(table)) {
2728       assertTrue("Timed out waiting for table to become available and enabled " +
2729          Bytes.toStringBinary(table),
2730          System.currentTimeMillis() - startWait < timeoutMillis);
2731       Thread.sleep(200);
2732     }
2733     // Finally make sure all regions are fully open and online out on the cluster. Regions may be
2734     // in the hbase:meta table and almost open on all regionservers but there setting the region
2735     // online in the regionserver is the very last thing done and can take a little while to happen.
2736     // Below we do a get.  The get will retry if a NotServeringRegionException or a
2737     // RegionOpeningException.  It is crass but when done all will be online.
2738     try {
2739       Canary.sniff(admin, TableName.valueOf(table));
2740     } catch (Exception e) {
2741       throw new IOException(e);
2742     }
2743   }
2744 
2745   /**
2746    * Make sure that at least the specified number of region servers
2747    * are running
2748    * @param num minimum number of region servers that should be running
2749    * @return true if we started some servers
2750    * @throws IOException
2751    */
2752   public boolean ensureSomeRegionServersAvailable(final int num)
2753       throws IOException {
2754     boolean startedServer = false;
2755     MiniHBaseCluster hbaseCluster = getMiniHBaseCluster();
2756     for (int i=hbaseCluster.getLiveRegionServerThreads().size(); i<num; ++i) {
2757       LOG.info("Started new server=" + hbaseCluster.startRegionServer());
2758       startedServer = true;
2759     }
2760 
2761     return startedServer;
2762   }
2763 
2764 
2765   /**
2766    * Make sure that at least the specified number of region servers
2767    * are running. We don't count the ones that are currently stopping or are
2768    * stopped.
2769    * @param num minimum number of region servers that should be running
2770    * @return true if we started some servers
2771    * @throws IOException
2772    */
2773   public boolean ensureSomeNonStoppedRegionServersAvailable(final int num)
2774     throws IOException {
2775     boolean startedServer = ensureSomeRegionServersAvailable(num);
2776 
2777     int nonStoppedServers = 0;
2778     for (JVMClusterUtil.RegionServerThread rst :
2779       getMiniHBaseCluster().getRegionServerThreads()) {
2780 
2781       HRegionServer hrs = rst.getRegionServer();
2782       if (hrs.isStopping() || hrs.isStopped()) {
2783         LOG.info("A region server is stopped or stopping:"+hrs);
2784       } else {
2785         nonStoppedServers++;
2786       }
2787     }
2788     for (int i=nonStoppedServers; i<num; ++i) {
2789       LOG.info("Started new server=" + getMiniHBaseCluster().startRegionServer());
2790       startedServer = true;
2791     }
2792     return startedServer;
2793   }
2794 
2795 
2796   /**
2797    * This method clones the passed <code>c</code> configuration setting a new
2798    * user into the clone.  Use it getting new instances of FileSystem.  Only
2799    * works for DistributedFileSystem.
2800    * @param c Initial configuration
2801    * @param differentiatingSuffix Suffix to differentiate this user from others.
2802    * @return A new configuration instance with a different user set into it.
2803    * @throws IOException
2804    */
2805   public static User getDifferentUser(final Configuration c,
2806     final String differentiatingSuffix)
2807   throws IOException {
2808     FileSystem currentfs = FileSystem.get(c);
2809     if (!(currentfs instanceof DistributedFileSystem)) {
2810       return User.getCurrent();
2811     }
2812     // Else distributed filesystem.  Make a new instance per daemon.  Below
2813     // code is taken from the AppendTestUtil over in hdfs.
2814     String username = User.getCurrent().getName() +
2815       differentiatingSuffix;
2816     User user = User.createUserForTesting(c, username,
2817         new String[]{"supergroup"});
2818     return user;
2819   }
2820 
2821   /**
2822    * Set maxRecoveryErrorCount in DFSClient.  In 0.20 pre-append its hard-coded to 5 and
2823    * makes tests linger.  Here is the exception you'll see:
2824    * <pre>
2825    * 2010-06-15 11:52:28,511 WARN  [DataStreamer for file /hbase/.logs/hlog.1276627923013 block blk_928005470262850423_1021] hdfs.DFSClient$DFSOutputStream(2657): Error Recovery for block blk_928005470262850423_1021 failed  because recovery from primary datanode 127.0.0.1:53683 failed 4 times.  Pipeline was 127.0.0.1:53687, 127.0.0.1:53683. Will retry...
2826    * </pre>
2827    * @param stream A DFSClient.DFSOutputStream.
2828    * @param max
2829    * @throws NoSuchFieldException
2830    * @throws SecurityException
2831    * @throws IllegalAccessException
2832    * @throws IllegalArgumentException
2833    */
2834   public static void setMaxRecoveryErrorCount(final OutputStream stream,
2835       final int max) {
2836     try {
2837       Class<?> [] clazzes = DFSClient.class.getDeclaredClasses();
2838       for (Class<?> clazz: clazzes) {
2839         String className = clazz.getSimpleName();
2840         if (className.equals("DFSOutputStream")) {
2841           if (clazz.isInstance(stream)) {
2842             Field maxRecoveryErrorCountField =
2843               stream.getClass().getDeclaredField("maxRecoveryErrorCount");
2844             maxRecoveryErrorCountField.setAccessible(true);
2845             maxRecoveryErrorCountField.setInt(stream, max);
2846             break;
2847           }
2848         }
2849       }
2850     } catch (Exception e) {
2851       LOG.info("Could not set max recovery field", e);
2852     }
2853   }
2854 
2855   /**
2856    * Wait until all regions for a table in hbase:meta have a non-empty
2857    * info:server, up to 60 seconds. This means all regions have been deployed,
2858    * master has been informed and updated hbase:meta with the regions deployed
2859    * server.
2860    * @param tableName the table name
2861    * @throws IOException
2862    */
2863   public void waitUntilAllRegionsAssigned(final TableName tableName) throws IOException {
2864     waitUntilAllRegionsAssigned(tableName, 60000);
2865   }
2866 
2867   /**
2868    * Wait until all regions for a table in hbase:meta have a non-empty
2869    * info:server, or until timeout.  This means all regions have been deployed,
2870    * master has been informed and updated hbase:meta with the regions deployed
2871    * server.
2872    * @param tableName the table name
2873    * @param timeout timeout, in milliseconds
2874    * @throws IOException
2875    */
2876   public void waitUntilAllRegionsAssigned(final TableName tableName, final long timeout)
2877       throws IOException {
2878     final HTable meta = new HTable(getConfiguration(), TableName.META_TABLE_NAME);
2879     try {
2880       waitFor(timeout, 200, true, new Predicate<IOException>() {
2881         @Override
2882         public boolean evaluate() throws IOException {
2883           boolean allRegionsAssigned = true;
2884           Scan scan = new Scan();
2885           scan.addFamily(HConstants.CATALOG_FAMILY);
2886           ResultScanner s = meta.getScanner(scan);
2887           try {
2888             Result r;
2889             while ((r = s.next()) != null) {
2890               byte [] b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
2891               HRegionInfo info = HRegionInfo.parseFromOrNull(b);
2892               if (info != null && info.getTable().equals(tableName)) {
2893                 b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
2894                 allRegionsAssigned &= (b != null);
2895               }
2896             }
2897           } finally {
2898             s.close();
2899           }
2900           return allRegionsAssigned;
2901         }
2902       });
2903     } finally {
2904       meta.close();
2905     }
2906   }
2907 
2908   /**
2909    * Do a small get/scan against one store. This is required because store
2910    * has no actual methods of querying itself, and relies on StoreScanner.
2911    */
2912   public static List<Cell> getFromStoreFile(HStore store,
2913                                                 Get get) throws IOException {
2914     Scan scan = new Scan(get);
2915     InternalScanner scanner = (InternalScanner) store.getScanner(scan,
2916         scan.getFamilyMap().get(store.getFamily().getName()),
2917         // originally MultiVersionConsistencyControl.resetThreadReadPoint() was called to set
2918         // readpoint 0.
2919         0);
2920 
2921     List<Cell> result = new ArrayList<Cell>();
2922     scanner.next(result);
2923     if (!result.isEmpty()) {
2924       // verify that we are on the row we want:
2925       Cell kv = result.get(0);
2926       if (!CellUtil.matchingRow(kv, get.getRow())) {
2927         result.clear();
2928       }
2929     }
2930     scanner.close();
2931     return result;
2932   }
2933 
2934   /**
2935    * Create region split keys between startkey and endKey
2936    *
2937    * @param startKey
2938    * @param endKey
2939    * @param numRegions the number of regions to be created. it has to be greater than 3.
2940    * @return
2941    */
2942   public byte[][] getRegionSplitStartKeys(byte[] startKey, byte[] endKey, int numRegions){
2943     assertTrue(numRegions>3);
2944     byte [][] tmpSplitKeys = Bytes.split(startKey, endKey, numRegions - 3);
2945     byte [][] result = new byte[tmpSplitKeys.length+1][];
2946     System.arraycopy(tmpSplitKeys, 0, result, 1, tmpSplitKeys.length);
2947     result[0] = HConstants.EMPTY_BYTE_ARRAY;
2948     return result;
2949   }
2950 
2951   /**
2952    * Do a small get/scan against one store. This is required because store
2953    * has no actual methods of querying itself, and relies on StoreScanner.
2954    */
2955   public static List<Cell> getFromStoreFile(HStore store,
2956                                                 byte [] row,
2957                                                 NavigableSet<byte[]> columns
2958                                                 ) throws IOException {
2959     Get get = new Get(row);
2960     Map<byte[], NavigableSet<byte[]>> s = get.getFamilyMap();
2961     s.put(store.getFamily().getName(), columns);
2962 
2963     return getFromStoreFile(store,get);
2964   }
2965 
2966   /**
2967    * Gets a ZooKeeperWatcher.
2968    * @param TEST_UTIL
2969    */
2970   public static ZooKeeperWatcher getZooKeeperWatcher(
2971       HBaseTestingUtility TEST_UTIL) throws ZooKeeperConnectionException,
2972       IOException {
2973     ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
2974         "unittest", new Abortable() {
2975           boolean aborted = false;
2976 
2977           @Override
2978           public void abort(String why, Throwable e) {
2979             aborted = true;
2980             throw new RuntimeException("Fatal ZK error, why=" + why, e);
2981           }
2982 
2983           @Override
2984           public boolean isAborted() {
2985             return aborted;
2986           }
2987         });
2988     return zkw;
2989   }
2990 
2991   /**
2992    * Creates a znode with OPENED state.
2993    * @param TEST_UTIL
2994    * @param region
2995    * @param serverName
2996    * @return
2997    * @throws IOException
2998    * @throws org.apache.hadoop.hbase.ZooKeeperConnectionException
2999    * @throws KeeperException
3000    * @throws NodeExistsException
3001    */
3002   public static ZooKeeperWatcher createAndForceNodeToOpenedState(
3003       HBaseTestingUtility TEST_UTIL, HRegion region,
3004       ServerName serverName) throws ZooKeeperConnectionException,
3005       IOException, KeeperException, NodeExistsException {
3006     ZooKeeperWatcher zkw = getZooKeeperWatcher(TEST_UTIL);
3007     ZKAssign.createNodeOffline(zkw, region.getRegionInfo(), serverName);
3008     int version = ZKAssign.transitionNodeOpening(zkw, region
3009         .getRegionInfo(), serverName);
3010     ZKAssign.transitionNodeOpened(zkw, region.getRegionInfo(), serverName,
3011         version);
3012     return zkw;
3013   }
3014 
3015   public static void assertKVListsEqual(String additionalMsg,
3016       final List<? extends Cell> expected,
3017       final List<? extends Cell> actual) {
3018     final int eLen = expected.size();
3019     final int aLen = actual.size();
3020     final int minLen = Math.min(eLen, aLen);
3021 
3022     int i;
3023     for (i = 0; i < minLen
3024         && KeyValue.COMPARATOR.compare(expected.get(i), actual.get(i)) == 0;
3025         ++i) {}
3026 
3027     if (additionalMsg == null) {
3028       additionalMsg = "";
3029     }
3030     if (!additionalMsg.isEmpty()) {
3031       additionalMsg = ". " + additionalMsg;
3032     }
3033 
3034     if (eLen != aLen || i != minLen) {
3035       throw new AssertionError(
3036           "Expected and actual KV arrays differ at position " + i + ": " +
3037           safeGetAsStr(expected, i) + " (length " + eLen +") vs. " +
3038           safeGetAsStr(actual, i) + " (length " + aLen + ")" + additionalMsg);
3039     }
3040   }
3041 
3042   private static <T> String safeGetAsStr(List<T> lst, int i) {
3043     if (0 <= i && i < lst.size()) {
3044       return lst.get(i).toString();
3045     } else {
3046       return "<out_of_range>";
3047     }
3048   }
3049 
3050   public String getClusterKey() {
3051     return conf.get(HConstants.ZOOKEEPER_QUORUM) + ":"
3052         + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT) + ":"
3053         + conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
3054             HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
3055   }
3056 
3057   /** Creates a random table with the given parameters */
3058   public HTable createRandomTable(String tableName,
3059       final Collection<String> families,
3060       final int maxVersions,
3061       final int numColsPerRow,
3062       final int numFlushes,
3063       final int numRegions,
3064       final int numRowsPerFlush)
3065       throws IOException, InterruptedException {
3066 
3067     LOG.info("\n\nCreating random table " + tableName + " with " + numRegions +
3068         " regions, " + numFlushes + " storefiles per region, " +
3069         numRowsPerFlush + " rows per flush, maxVersions=" +  maxVersions +
3070         "\n");
3071 
3072     final Random rand = new Random(tableName.hashCode() * 17L + 12938197137L);
3073     final int numCF = families.size();
3074     final byte[][] cfBytes = new byte[numCF][];
3075     {
3076       int cfIndex = 0;
3077       for (String cf : families) {
3078         cfBytes[cfIndex++] = Bytes.toBytes(cf);
3079       }
3080     }
3081 
3082     final int actualStartKey = 0;
3083     final int actualEndKey = Integer.MAX_VALUE;
3084     final int keysPerRegion = (actualEndKey - actualStartKey) / numRegions;
3085     final int splitStartKey = actualStartKey + keysPerRegion;
3086     final int splitEndKey = actualEndKey - keysPerRegion;
3087     final String keyFormat = "%08x";
3088     final HTable table = createTable(tableName, cfBytes,
3089         maxVersions,
3090         Bytes.toBytes(String.format(keyFormat, splitStartKey)),
3091         Bytes.toBytes(String.format(keyFormat, splitEndKey)),
3092         numRegions);
3093 
3094     if (hbaseCluster != null) {
3095       getMiniHBaseCluster().flushcache(TableName.META_TABLE_NAME);
3096     }
3097 
3098     for (int iFlush = 0; iFlush < numFlushes; ++iFlush) {
3099       for (int iRow = 0; iRow < numRowsPerFlush; ++iRow) {
3100         final byte[] row = Bytes.toBytes(String.format(keyFormat,
3101             actualStartKey + rand.nextInt(actualEndKey - actualStartKey)));
3102 
3103         Put put = new Put(row);
3104         Delete del = new Delete(row);
3105         for (int iCol = 0; iCol < numColsPerRow; ++iCol) {
3106           final byte[] cf = cfBytes[rand.nextInt(numCF)];
3107           final long ts = rand.nextInt();
3108           final byte[] qual = Bytes.toBytes("col" + iCol);
3109           if (rand.nextBoolean()) {
3110             final byte[] value = Bytes.toBytes("value_for_row_" + iRow +
3111                 "_cf_" + Bytes.toStringBinary(cf) + "_col_" + iCol + "_ts_" +
3112                 ts + "_random_" + rand.nextLong());
3113             put.add(cf, qual, ts, value);
3114           } else if (rand.nextDouble() < 0.8) {
3115             del.deleteColumn(cf, qual, ts);
3116           } else {
3117             del.deleteColumns(cf, qual, ts);
3118           }
3119         }
3120 
3121         if (!put.isEmpty()) {
3122           table.put(put);
3123         }
3124 
3125         if (!del.isEmpty()) {
3126           table.delete(del);
3127         }
3128       }
3129       LOG.info("Initiating flush #" + iFlush + " for table " + tableName);
3130       table.flushCommits();
3131       if (hbaseCluster != null) {
3132         getMiniHBaseCluster().flushcache(table.getName());
3133       }
3134     }
3135 
3136     return table;
3137   }
3138 
3139   private static final int MIN_RANDOM_PORT = 0xc000;
3140   private static final int MAX_RANDOM_PORT = 0xfffe;
3141   private static Random random = new Random();
3142 
3143   /**
3144    * Returns a random port. These ports cannot be registered with IANA and are
3145    * intended for dynamic allocation (see http://bit.ly/dynports).
3146    */
3147   public static int randomPort() {
3148     return MIN_RANDOM_PORT
3149         + random.nextInt(MAX_RANDOM_PORT - MIN_RANDOM_PORT);
3150   }
3151 
3152   /**
3153    * Returns a random free port and marks that port as taken. Not thread-safe. Expected to be
3154    * called from single-threaded test setup code/
3155    */
3156   public static int randomFreePort() {
3157     int port = 0;
3158     do {
3159       port = randomPort();
3160       if (takenRandomPorts.contains(port)) {
3161         continue;
3162       }
3163       takenRandomPorts.add(port);
3164 
3165       try {
3166         ServerSocket sock = new ServerSocket(port);
3167         sock.close();
3168       } catch (IOException ex) {
3169         port = 0;
3170       }
3171     } while (port == 0);
3172     return port;
3173   }
3174 
3175 
3176   public static String randomMultiCastAddress() {
3177     return "226.1.1." + random.nextInt(254);
3178   }
3179 
3180 
3181 
3182   public static void waitForHostPort(String host, int port)
3183       throws IOException {
3184     final int maxTimeMs = 10000;
3185     final int maxNumAttempts = maxTimeMs / HConstants.SOCKET_RETRY_WAIT_MS;
3186     IOException savedException = null;
3187     LOG.info("Waiting for server at " + host + ":" + port);
3188     for (int attempt = 0; attempt < maxNumAttempts; ++attempt) {
3189       try {
3190         Socket sock = new Socket(InetAddress.getByName(host), port);
3191         sock.close();
3192         savedException = null;
3193         LOG.info("Server at " + host + ":" + port + " is available");
3194         break;
3195       } catch (UnknownHostException e) {
3196         throw new IOException("Failed to look up " + host, e);
3197       } catch (IOException e) {
3198         savedException = e;
3199       }
3200       Threads.sleepWithoutInterrupt(HConstants.SOCKET_RETRY_WAIT_MS);
3201     }
3202 
3203     if (savedException != null) {
3204       throw savedException;
3205     }
3206   }
3207 
3208   /**
3209    * Creates a pre-split table for load testing. If the table already exists,
3210    * logs a warning and continues.
3211    * @return the number of regions the table was split into
3212    */
3213   public static int createPreSplitLoadTestTable(Configuration conf,
3214       TableName tableName, byte[] columnFamily, Algorithm compression,
3215       DataBlockEncoding dataBlockEncoding) throws IOException {
3216     HTableDescriptor desc = new HTableDescriptor(tableName);
3217     HColumnDescriptor hcd = new HColumnDescriptor(columnFamily);
3218     hcd.setDataBlockEncoding(dataBlockEncoding);
3219     hcd.setCompressionType(compression);
3220     return createPreSplitLoadTestTable(conf, desc, hcd);
3221   }
3222 
3223   /**
3224    * Creates a pre-split table for load testing. If the table already exists,
3225    * logs a warning and continues.
3226    * @return the number of regions the table was split into
3227    */
3228   public static int createPreSplitLoadTestTable(Configuration conf,
3229       HTableDescriptor desc, HColumnDescriptor hcd) throws IOException {
3230     if (!desc.hasFamily(hcd.getName())) {
3231       desc.addFamily(hcd);
3232     }
3233 
3234     int totalNumberOfRegions = 0;
3235     HBaseAdmin admin = new HBaseAdmin(conf);
3236     try {
3237       // create a table a pre-splits regions.
3238       // The number of splits is set as:
3239       //    region servers * regions per region server).
3240       int numberOfServers = admin.getClusterStatus().getServers().size();
3241       if (numberOfServers == 0) {
3242         throw new IllegalStateException("No live regionservers");
3243       }
3244 
3245       int regionsPerServer = conf.getInt(REGIONS_PER_SERVER_KEY, DEFAULT_REGIONS_PER_SERVER);
3246       totalNumberOfRegions = numberOfServers * regionsPerServer;
3247       LOG.info("Number of live regionservers: " + numberOfServers + ", " +
3248           "pre-splitting table into " + totalNumberOfRegions + " regions " +
3249           "(default regions per server: " + regionsPerServer + ")");
3250 
3251       byte[][] splits = new RegionSplitter.HexStringSplit().split(
3252           totalNumberOfRegions);
3253 
3254       admin.createTable(desc, splits);
3255     } catch (MasterNotRunningException e) {
3256       LOG.error("Master not running", e);
3257       throw new IOException(e);
3258     } catch (TableExistsException e) {
3259       LOG.warn("Table " + desc.getTableName() +
3260           " already exists, continuing");
3261     } finally {
3262       admin.close();
3263     }
3264     return totalNumberOfRegions;
3265   }
3266 
3267   public static int getMetaRSPort(Configuration conf) throws IOException {
3268     HTable table = new HTable(conf, TableName.META_TABLE_NAME);
3269     HRegionLocation hloc = table.getRegionLocation(Bytes.toBytes(""));
3270     table.close();
3271     return hloc.getPort();
3272   }
3273 
3274   /**
3275    *  Due to async racing issue, a region may not be in
3276    *  the online region list of a region server yet, after
3277    *  the assignment znode is deleted and the new assignment
3278    *  is recorded in master.
3279    */
3280   public void assertRegionOnServer(
3281       final HRegionInfo hri, final ServerName server,
3282       final long timeout) throws IOException, InterruptedException {
3283     long timeoutTime = System.currentTimeMillis() + timeout;
3284     while (true) {
3285       List<HRegionInfo> regions = getHBaseAdmin().getOnlineRegions(server);
3286       if (regions.contains(hri)) return;
3287       long now = System.currentTimeMillis();
3288       if (now > timeoutTime) break;
3289       Thread.sleep(10);
3290     }
3291     fail("Could not find region " + hri.getRegionNameAsString()
3292       + " on server " + server);
3293   }
3294 
3295   /**
3296    * Check to make sure the region is open on the specified
3297    * region server, but not on any other one.
3298    */
3299   public void assertRegionOnlyOnServer(
3300       final HRegionInfo hri, final ServerName server,
3301       final long timeout) throws IOException, InterruptedException {
3302     long timeoutTime = System.currentTimeMillis() + timeout;
3303     while (true) {
3304       List<HRegionInfo> regions = getHBaseAdmin().getOnlineRegions(server);
3305       if (regions.contains(hri)) {
3306         List<JVMClusterUtil.RegionServerThread> rsThreads =
3307           getHBaseCluster().getLiveRegionServerThreads();
3308         for (JVMClusterUtil.RegionServerThread rsThread: rsThreads) {
3309           HRegionServer rs = rsThread.getRegionServer();
3310           if (server.equals(rs.getServerName())) {
3311             continue;
3312           }
3313           Collection<HRegion> hrs = rs.getOnlineRegionsLocalContext();
3314           for (HRegion r: hrs) {
3315             assertTrue("Region should not be double assigned",
3316               r.getRegionId() != hri.getRegionId());
3317           }
3318         }
3319         return; // good, we are happy
3320       }
3321       long now = System.currentTimeMillis();
3322       if (now > timeoutTime) break;
3323       Thread.sleep(10);
3324     }
3325     fail("Could not find region " + hri.getRegionNameAsString()
3326       + " on server " + server);
3327   }
3328 
3329   public HRegion createTestRegion(String tableName, HColumnDescriptor hcd)
3330       throws IOException {
3331     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3332     htd.addFamily(hcd);
3333     HRegionInfo info =
3334         new HRegionInfo(TableName.valueOf(tableName), null, null, false);
3335     HRegion region =
3336         HRegion.createHRegion(info, getDataTestDir(), getConfiguration(), htd);
3337     return region;
3338   }
3339 
3340   public void setFileSystemURI(String fsURI) {
3341     FS_URI = fsURI;
3342   }
3343 
3344   /**
3345    * Wrapper method for {@link Waiter#waitFor(Configuration, long, Predicate)}.
3346    */
3347   public <E extends Exception> long waitFor(long timeout, Predicate<E> predicate)
3348       throws E {
3349     return Waiter.waitFor(this.conf, timeout, predicate);
3350   }
3351 
3352   /**
3353    * Wrapper method for {@link Waiter#waitFor(Configuration, long, long, Predicate)}.
3354    */
3355   public <E extends Exception> long waitFor(long timeout, long interval, Predicate<E> predicate)
3356       throws E {
3357     return Waiter.waitFor(this.conf, timeout, interval, predicate);
3358   }
3359 
3360   /**
3361    * Wrapper method for {@link Waiter#waitFor(Configuration, long, long, boolean, Predicate)}.
3362    */
3363   public <E extends Exception> long waitFor(long timeout, long interval,
3364       boolean failIfTimeout, Predicate<E> predicate) throws E {
3365     return Waiter.waitFor(this.conf, timeout, interval, failIfTimeout, predicate);
3366   }
3367 
3368   /**
3369    * Returns a {@link Predicate} for checking that there are no regions in transition in master
3370    */
3371   public Waiter.Predicate<Exception> predicateNoRegionsInTransition() {
3372     return new Waiter.Predicate<Exception>() {
3373       @Override
3374       public boolean evaluate() throws Exception {
3375         final RegionStates regionStates = getMiniHBaseCluster().getMaster()
3376             .getAssignmentManager().getRegionStates();
3377         return !regionStates.isRegionsInTransition();
3378       }
3379     };
3380   }
3381 
3382   /**
3383    * Returns a {@link Predicate} for checking that table is enabled
3384    */
3385   public Waiter.Predicate<Exception> predicateTableEnabled(final TableName tableName) {
3386     return new Waiter.Predicate<Exception>() {
3387      @Override
3388      public boolean evaluate() throws Exception {
3389        return getHBaseAdmin().isTableEnabled(tableName);
3390       }
3391     };
3392   }
3393 
3394   /**
3395    * Create a set of column descriptors with the combination of compression,
3396    * encoding, bloom codecs available.
3397    * @return the list of column descriptors
3398    */
3399   public static List<HColumnDescriptor> generateColumnDescriptors() {
3400     return generateColumnDescriptors("");
3401   }
3402 
3403   /**
3404    * Create a set of column descriptors with the combination of compression,
3405    * encoding, bloom codecs available.
3406    * @param prefix family names prefix
3407    * @return the list of column descriptors
3408    */
3409   public static List<HColumnDescriptor> generateColumnDescriptors(final String prefix) {
3410     List<HColumnDescriptor> htds = new ArrayList<HColumnDescriptor>();
3411     long familyId = 0;
3412     for (Compression.Algorithm compressionType: getSupportedCompressionAlgorithms()) {
3413       for (DataBlockEncoding encodingType: DataBlockEncoding.values()) {
3414         for (BloomType bloomType: BloomType.values()) {
3415           String name = String.format("%s-cf-!@#&-%d!@#", prefix, familyId);
3416           HColumnDescriptor htd = new HColumnDescriptor(name);
3417           htd.setCompressionType(compressionType);
3418           htd.setDataBlockEncoding(encodingType);
3419           htd.setBloomFilterType(bloomType);
3420           htds.add(htd);
3421           familyId++;
3422         }
3423       }
3424     }
3425     return htds;
3426   }
3427 
3428   /**
3429    * Get supported compression algorithms.
3430    * @return supported compression algorithms.
3431    */
3432   public static Compression.Algorithm[] getSupportedCompressionAlgorithms() {
3433     String[] allAlgos = HFile.getSupportedCompressionAlgorithms();
3434     List<Compression.Algorithm> supportedAlgos = new ArrayList<Compression.Algorithm>();
3435     for (String algoName : allAlgos) {
3436       try {
3437         Compression.Algorithm algo = Compression.getCompressionAlgorithmByName(algoName);
3438         algo.getCompressor();
3439         supportedAlgos.add(algo);
3440       } catch (Throwable t) {
3441         // this algo is not available
3442       }
3443     }
3444     return supportedAlgos.toArray(new Algorithm[supportedAlgos.size()]);
3445   }
3446 }