View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations
15   * under the License.
16   */
17  package org.apache.hadoop.hbase.util;
18  
19  import java.io.IOException;
20  import java.io.InterruptedIOException;
21  import java.lang.reflect.Constructor;
22  import java.net.InetAddress;
23  import java.security.SecureRandom;
24  import java.util.ArrayList;
25  import java.util.Arrays;
26  import java.util.List;
27  import java.util.Properties;
28  import java.util.Random;
29  import java.util.concurrent.atomic.AtomicReference;
30  
31  import javax.crypto.spec.SecretKeySpec;
32  
33  import org.apache.commons.cli.CommandLine;
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.hbase.HBaseConfiguration;
38  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
39  import org.apache.hadoop.hbase.HBaseTestingUtility;
40  import org.apache.hadoop.hbase.HColumnDescriptor;
41  import org.apache.hadoop.hbase.HConstants;
42  import org.apache.hadoop.hbase.HTableDescriptor;
43  import org.apache.hadoop.hbase.TableName;
44  import org.apache.hadoop.hbase.classification.InterfaceAudience;
45  import org.apache.hadoop.hbase.client.Admin;
46  import org.apache.hadoop.hbase.client.Durability;
47  import org.apache.hadoop.hbase.client.HBaseAdmin;
48  import org.apache.hadoop.hbase.io.compress.Compression;
49  import org.apache.hadoop.hbase.io.crypto.Cipher;
50  import org.apache.hadoop.hbase.io.crypto.Encryption;
51  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
52  import org.apache.hadoop.hbase.regionserver.BloomType;
53  import org.apache.hadoop.hbase.security.EncryptionUtil;
54  import org.apache.hadoop.hbase.security.User;
55  import org.apache.hadoop.hbase.security.access.AccessControlClient;
56  import org.apache.hadoop.hbase.security.access.Permission;
57  import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator;
58  import org.apache.hadoop.hbase.util.test.LoadTestDataGeneratorWithACL;
59  import org.apache.hadoop.security.SecurityUtil;
60  import org.apache.hadoop.security.UserGroupInformation;
61  import org.apache.hadoop.util.ToolRunner;
62  
63  /**
64   * A command-line utility that reads, writes, and verifies data. Unlike
65   * {@link PerformanceEvaluation}, this tool validates the data written,
66   * and supports simultaneously writing and reading the same set of keys.
67   */
68  @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
69  public class LoadTestTool extends AbstractHBaseTool {
70  
71    private static final Log LOG = LogFactory.getLog(LoadTestTool.class);
72    private static final String COLON = ":";
73  
74    /** Table name for the test */
75    private TableName tableName;
76  
77    /** Table name to use of not overridden on the command line */
78    protected static final String DEFAULT_TABLE_NAME = "cluster_test";
79  
80    /** Column family used by the test */
81    public static byte[] COLUMN_FAMILY = Bytes.toBytes("test_cf");
82  
83    /** Column families used by the test */
84    protected static final byte[][] COLUMN_FAMILIES = { COLUMN_FAMILY };
85  
86    /** The default data size if not specified */
87    protected static final int DEFAULT_DATA_SIZE = 64;
88  
89    /** The number of reader/writer threads if not specified */
90    protected static final int DEFAULT_NUM_THREADS = 20;
91  
92    /** Usage string for the load option */
93    protected static final String OPT_USAGE_LOAD =
94        "<avg_cols_per_key>:<avg_data_size>" +
95        "[:<#threads=" + DEFAULT_NUM_THREADS + ">]";
96  
97    /** Usage string for the read option */
98    protected static final String OPT_USAGE_READ =
99        "<verify_percent>[:<#threads=" + DEFAULT_NUM_THREADS + ">]";
100 
101   /** Usage string for the update option */
102   protected static final String OPT_USAGE_UPDATE =
103       "<update_percent>[:<#threads=" + DEFAULT_NUM_THREADS
104       + ">][:<#whether to ignore nonce collisions=0>]";
105 
106   protected static final String OPT_USAGE_BLOOM = "Bloom filter type, one of " +
107       Arrays.toString(BloomType.values());
108 
109   protected static final String OPT_USAGE_COMPRESSION = "Compression type, " +
110       "one of " + Arrays.toString(Compression.Algorithm.values());
111 
112   public static final String OPT_DATA_BLOCK_ENCODING_USAGE =
113     "Encoding algorithm (e.g. prefix "
114         + "compression) to use for data blocks in the test column family, "
115         + "one of " + Arrays.toString(DataBlockEncoding.values()) + ".";
116 
117   public static final String OPT_BLOOM = "bloom";
118   public static final String OPT_COMPRESSION = "compression";
119   public static final String OPT_DEFERRED_LOG_FLUSH = "deferredlogflush";
120   public static final String OPT_DEFERRED_LOG_FLUSH_USAGE = "Enable deferred log flush.";
121 
122   public static final String OPT_DATA_BLOCK_ENCODING =
123       HColumnDescriptor.DATA_BLOCK_ENCODING.toLowerCase();
124 
125   public static final String OPT_INMEMORY = "in_memory";
126   public static final String OPT_USAGE_IN_MEMORY = "Tries to keep the HFiles of the CF " +
127       "inmemory as far as possible.  Not guaranteed that reads are always served from inmemory";
128 
129   public static final String OPT_GENERATOR = "generator";
130   public static final String OPT_GENERATOR_USAGE = "The class which generates load for the tool."
131       + " Any args for this class can be passed as colon separated after class name";
132 
133   public static final String OPT_READER = "reader";
134   public static final String OPT_READER_USAGE = "The class for executing the read requests";
135 
136   protected static final String OPT_KEY_WINDOW = "key_window";
137   protected static final String OPT_WRITE = "write";
138   protected static final String OPT_MAX_READ_ERRORS = "max_read_errors";
139   protected static final String OPT_MULTIPUT = "multiput";
140   public static final String OPT_MULTIGET = "multiget_batchsize";
141   protected static final String OPT_NUM_KEYS = "num_keys";
142   protected static final String OPT_READ = "read";
143   protected static final String OPT_START_KEY = "start_key";
144   public static final String OPT_TABLE_NAME = "tn";
145   protected static final String OPT_ZK_QUORUM = "zk";
146   protected static final String OPT_ZK_PARENT_NODE = "zk_root";
147   protected static final String OPT_SKIP_INIT = "skip_init";
148   protected static final String OPT_INIT_ONLY = "init_only";
149   protected static final String NUM_TABLES = "num_tables";
150   protected static final String OPT_REGIONS_PER_SERVER = "regions_per_server";
151   protected static final String OPT_BATCHUPDATE = "batchupdate";
152   protected static final String OPT_UPDATE = "update";
153 
154   public static final String OPT_ENCRYPTION = "encryption";
155   protected static final String OPT_ENCRYPTION_USAGE =
156     "Enables transparent encryption on the test table, one of " +
157     Arrays.toString(Encryption.getSupportedCiphers());
158 
159   public static final String OPT_NUM_REGIONS_PER_SERVER = "num_regions_per_server";
160   protected static final String OPT_NUM_REGIONS_PER_SERVER_USAGE
161     = "Desired number of regions per region server. Defaults to 5.";
162   public static int DEFAULT_NUM_REGIONS_PER_SERVER = 5;
163 
164   public static final String OPT_REGION_REPLICATION = "region_replication";
165   protected static final String OPT_REGION_REPLICATION_USAGE =
166       "Desired number of replicas per region";
167 
168   public static final String OPT_REGION_REPLICA_ID = "region_replica_id";
169   protected static final String OPT_REGION_REPLICA_ID_USAGE =
170       "Region replica id to do the reads from";
171 
172   protected static final long DEFAULT_START_KEY = 0;
173 
174   /** This will be removed as we factor out the dependency on command line */
175   protected CommandLine cmd;
176 
177   protected MultiThreadedWriter writerThreads = null;
178   protected MultiThreadedReader readerThreads = null;
179   protected MultiThreadedUpdater updaterThreads = null;
180 
181   protected long startKey, endKey;
182 
183   protected boolean isWrite, isRead, isUpdate;
184   protected boolean deferredLogFlush;
185 
186   // Column family options
187   protected DataBlockEncoding dataBlockEncodingAlgo;
188   protected Compression.Algorithm compressAlgo;
189   protected BloomType bloomType;
190   private boolean inMemoryCF;
191 
192   private User userOwner;
193   // Writer options
194   protected int numWriterThreads = DEFAULT_NUM_THREADS;
195   protected int minColsPerKey, maxColsPerKey;
196   protected int minColDataSize = DEFAULT_DATA_SIZE, maxColDataSize = DEFAULT_DATA_SIZE;
197   protected boolean isMultiPut;
198 
199   // Updater options
200   protected int numUpdaterThreads = DEFAULT_NUM_THREADS;
201   protected int updatePercent;
202   protected boolean ignoreConflicts = false;
203   protected boolean isBatchUpdate;
204 
205   // Reader options
206   private int numReaderThreads = DEFAULT_NUM_THREADS;
207   private int keyWindow = MultiThreadedReader.DEFAULT_KEY_WINDOW;
208   private int multiGetBatchSize = MultiThreadedReader.DEFAULT_BATCH_SIZE;
209   private int maxReadErrors = MultiThreadedReader.DEFAULT_MAX_ERRORS;
210   private int verifyPercent;
211 
212   private int numTables = 1;
213 
214   private String superUser;
215 
216   private String userNames;
217   //This file is used to read authentication information in secure clusters.
218   private String authnFileName;
219 
220   private int numRegionsPerServer = DEFAULT_NUM_REGIONS_PER_SERVER;
221   private int regionReplication = -1; // not set
222   private int regionReplicaId = -1; // not set
223 
224   // TODO: refactor LoadTestToolImpl somewhere to make the usage from tests less bad,
225   //       console tool itself should only be used from console.
226   protected boolean isSkipInit = false;
227   protected boolean isInitOnly = false;
228 
229   protected Cipher cipher = null;
230 
231   protected String[] splitColonSeparated(String option,
232       int minNumCols, int maxNumCols) {
233     String optVal = cmd.getOptionValue(option);
234     String[] cols = optVal.split(COLON);
235     if (cols.length < minNumCols || cols.length > maxNumCols) {
236       throw new IllegalArgumentException("Expected at least "
237           + minNumCols + " columns but no more than " + maxNumCols +
238           " in the colon-separated value '" + optVal + "' of the " +
239           "-" + option + " option");
240     }
241     return cols;
242   }
243 
244   protected int getNumThreads(String numThreadsStr) {
245     return parseInt(numThreadsStr, 1, Short.MAX_VALUE);
246   }
247 
248   /**
249    * Apply column family options such as Bloom filters, compression, and data
250    * block encoding.
251    */
252   protected void applyColumnFamilyOptions(TableName tableName,
253       byte[][] columnFamilies) throws IOException {
254     Admin admin = new HBaseAdmin(conf);
255     HTableDescriptor tableDesc = admin.getTableDescriptor(tableName);
256     LOG.info("Disabling table " + tableName);
257     admin.disableTable(tableName);
258     for (byte[] cf : columnFamilies) {
259       HColumnDescriptor columnDesc = tableDesc.getFamily(cf);
260       boolean isNewCf = columnDesc == null;
261       if (isNewCf) {
262         columnDesc = new HColumnDescriptor(cf);
263       }
264       if (bloomType != null) {
265         columnDesc.setBloomFilterType(bloomType);
266       }
267       if (compressAlgo != null) {
268         columnDesc.setCompressionType(compressAlgo);
269       }
270       if (dataBlockEncodingAlgo != null) {
271         columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo);
272       }
273       if (inMemoryCF) {
274         columnDesc.setInMemory(inMemoryCF);
275       }
276       if (cipher != null) {
277         byte[] keyBytes = new byte[cipher.getKeyLength()];
278         new SecureRandom().nextBytes(keyBytes);
279         columnDesc.setEncryptionType(cipher.getName());
280         columnDesc.setEncryptionKey(EncryptionUtil.wrapKey(conf,
281           User.getCurrent().getShortName(),
282           new SecretKeySpec(keyBytes, cipher.getName())));
283       }
284       if (isNewCf) {
285         admin.addColumn(tableName, columnDesc);
286       } else {
287         admin.modifyColumn(tableName, columnDesc);
288       }
289     }
290     LOG.info("Enabling table " + tableName);
291     admin.enableTable(tableName);
292     admin.close();
293   }
294 
295   @Override
296   protected void addOptions() {
297     addOptWithArg(OPT_ZK_QUORUM, "ZK quorum as comma-separated host names " +
298         "without port numbers");
299     addOptWithArg(OPT_ZK_PARENT_NODE, "name of parent znode in zookeeper");
300     addOptWithArg(OPT_TABLE_NAME, "The name of the table to read or write");
301     addOptWithArg(OPT_WRITE, OPT_USAGE_LOAD);
302     addOptWithArg(OPT_READ, OPT_USAGE_READ);
303     addOptWithArg(OPT_UPDATE, OPT_USAGE_UPDATE);
304     addOptNoArg(OPT_INIT_ONLY, "Initialize the test table only, don't do any loading");
305     addOptWithArg(OPT_BLOOM, OPT_USAGE_BLOOM);
306     addOptWithArg(OPT_COMPRESSION, OPT_USAGE_COMPRESSION);
307     addOptWithArg(OPT_DATA_BLOCK_ENCODING, OPT_DATA_BLOCK_ENCODING_USAGE);
308     addOptWithArg(OPT_MAX_READ_ERRORS, "The maximum number of read errors " +
309         "to tolerate before terminating all reader threads. The default is " +
310         MultiThreadedReader.DEFAULT_MAX_ERRORS + ".");
311     addOptWithArg(OPT_MULTIGET, "Whether to use multi-gets as opposed to " +
312         "separate gets for every column in a row");
313     addOptWithArg(OPT_KEY_WINDOW, "The 'key window' to maintain between " +
314         "reads and writes for concurrent write/read workload. The default " +
315         "is " + MultiThreadedReader.DEFAULT_KEY_WINDOW + ".");
316 
317     addOptNoArg(OPT_MULTIPUT, "Whether to use multi-puts as opposed to " +
318         "separate puts for every column in a row");
319     addOptNoArg(OPT_BATCHUPDATE, "Whether to use batch as opposed to " +
320         "separate updates for every column in a row");
321     addOptNoArg(OPT_INMEMORY, OPT_USAGE_IN_MEMORY);
322     addOptWithArg(OPT_GENERATOR, OPT_GENERATOR_USAGE);
323     addOptWithArg(OPT_READER, OPT_READER_USAGE);
324 
325     addOptWithArg(OPT_NUM_KEYS, "The number of keys to read/write");
326     addOptWithArg(OPT_START_KEY, "The first key to read/write " +
327         "(a 0-based index). The default value is " +
328         DEFAULT_START_KEY + ".");
329     addOptNoArg(OPT_SKIP_INIT, "Skip the initialization; assume test table "
330         + "already exists");
331 
332     addOptWithArg(NUM_TABLES,
333       "A positive integer number. When a number n is speicfied, load test "
334           + "tool  will load n table parallely. -tn parameter value becomes "
335           + "table name prefix. Each table name is in format <tn>_1...<tn>_n");
336 
337     addOptWithArg(OPT_REGIONS_PER_SERVER,
338       "A positive integer number. When a number n is specified, load test "
339           + "tool will create the test table with n regions per server");
340 
341     addOptWithArg(OPT_ENCRYPTION, OPT_ENCRYPTION_USAGE);
342     addOptNoArg(OPT_DEFERRED_LOG_FLUSH, OPT_DEFERRED_LOG_FLUSH_USAGE);
343     addOptWithArg(OPT_NUM_REGIONS_PER_SERVER, OPT_NUM_REGIONS_PER_SERVER_USAGE);
344     addOptWithArg(OPT_REGION_REPLICATION, OPT_REGION_REPLICATION_USAGE);
345     addOptWithArg(OPT_REGION_REPLICA_ID, OPT_REGION_REPLICA_ID_USAGE);
346   }
347 
348   @Override
349   protected void processOptions(CommandLine cmd) {
350     this.cmd = cmd;
351 
352     tableName = TableName.valueOf(cmd.getOptionValue(OPT_TABLE_NAME,
353         DEFAULT_TABLE_NAME));
354 
355     isWrite = cmd.hasOption(OPT_WRITE);
356     isRead = cmd.hasOption(OPT_READ);
357     isUpdate = cmd.hasOption(OPT_UPDATE);
358     isInitOnly = cmd.hasOption(OPT_INIT_ONLY);
359     deferredLogFlush = cmd.hasOption(OPT_DEFERRED_LOG_FLUSH);
360 
361     if (!isWrite && !isRead && !isUpdate && !isInitOnly) {
362       throw new IllegalArgumentException("Either -" + OPT_WRITE + " or " +
363         "-" + OPT_UPDATE + " or -" + OPT_READ + " has to be specified");
364     }
365 
366     if (isInitOnly && (isRead || isWrite || isUpdate)) {
367       throw new IllegalArgumentException(OPT_INIT_ONLY + " cannot be specified with"
368           + " either -" + OPT_WRITE + " or -" + OPT_UPDATE + " or -" + OPT_READ);
369     }
370 
371     if (!isInitOnly) {
372       if (!cmd.hasOption(OPT_NUM_KEYS)) {
373         throw new IllegalArgumentException(OPT_NUM_KEYS + " must be specified in "
374             + "read or write mode");
375       }
376       startKey = parseLong(cmd.getOptionValue(OPT_START_KEY,
377           String.valueOf(DEFAULT_START_KEY)), 0, Long.MAX_VALUE);
378       long numKeys = parseLong(cmd.getOptionValue(OPT_NUM_KEYS), 1,
379           Long.MAX_VALUE - startKey);
380       endKey = startKey + numKeys;
381       isSkipInit = cmd.hasOption(OPT_SKIP_INIT);
382       System.out.println("Key range: [" + startKey + ".." + (endKey - 1) + "]");
383     }
384 
385     parseColumnFamilyOptions(cmd);
386 
387     if (isWrite) {
388       String[] writeOpts = splitColonSeparated(OPT_WRITE, 2, 3);
389 
390       int colIndex = 0;
391       minColsPerKey = 1;
392       maxColsPerKey = 2 * Integer.parseInt(writeOpts[colIndex++]);
393       int avgColDataSize =
394           parseInt(writeOpts[colIndex++], 1, Integer.MAX_VALUE);
395       minColDataSize = avgColDataSize / 2;
396       maxColDataSize = avgColDataSize * 3 / 2;
397 
398       if (colIndex < writeOpts.length) {
399         numWriterThreads = getNumThreads(writeOpts[colIndex++]);
400       }
401 
402       isMultiPut = cmd.hasOption(OPT_MULTIPUT);
403 
404       System.out.println("Multi-puts: " + isMultiPut);
405       System.out.println("Columns per key: " + minColsPerKey + ".."
406           + maxColsPerKey);
407       System.out.println("Data size per column: " + minColDataSize + ".."
408           + maxColDataSize);
409     }
410 
411     if (isUpdate) {
412       String[] mutateOpts = splitColonSeparated(OPT_UPDATE, 1, 3);
413       int colIndex = 0;
414       updatePercent = parseInt(mutateOpts[colIndex++], 0, 100);
415       if (colIndex < mutateOpts.length) {
416         numUpdaterThreads = getNumThreads(mutateOpts[colIndex++]);
417       }
418       if (colIndex < mutateOpts.length) {
419         ignoreConflicts = parseInt(mutateOpts[colIndex++], 0, 1) == 1;
420       }
421 
422       isBatchUpdate = cmd.hasOption(OPT_BATCHUPDATE);
423 
424       System.out.println("Batch updates: " + isBatchUpdate);
425       System.out.println("Percent of keys to update: " + updatePercent);
426       System.out.println("Updater threads: " + numUpdaterThreads);
427       System.out.println("Ignore nonce conflicts: " + ignoreConflicts);
428     }
429 
430     if (isRead) {
431       String[] readOpts = splitColonSeparated(OPT_READ, 1, 2);
432       int colIndex = 0;
433       verifyPercent = parseInt(readOpts[colIndex++], 0, 100);
434       if (colIndex < readOpts.length) {
435         numReaderThreads = getNumThreads(readOpts[colIndex++]);
436       }
437 
438       if (cmd.hasOption(OPT_MAX_READ_ERRORS)) {
439         maxReadErrors = parseInt(cmd.getOptionValue(OPT_MAX_READ_ERRORS),
440             0, Integer.MAX_VALUE);
441       }
442 
443       if (cmd.hasOption(OPT_KEY_WINDOW)) {
444         keyWindow = parseInt(cmd.getOptionValue(OPT_KEY_WINDOW),
445             0, Integer.MAX_VALUE);
446       }
447 
448       if (cmd.hasOption(OPT_MULTIGET)) {
449         multiGetBatchSize = parseInt(cmd.getOptionValue(OPT_MULTIGET),
450             0, Integer.MAX_VALUE);
451       }
452 
453       System.out.println("Multi-gets (value of 1 means no multigets): " + multiGetBatchSize);
454       System.out.println("Percent of keys to verify: " + verifyPercent);
455       System.out.println("Reader threads: " + numReaderThreads);
456     }
457 
458     numTables = 1;
459     if (cmd.hasOption(NUM_TABLES)) {
460       numTables = parseInt(cmd.getOptionValue(NUM_TABLES), 1, Short.MAX_VALUE);
461     }
462 
463     numRegionsPerServer = DEFAULT_NUM_REGIONS_PER_SERVER;
464     if (cmd.hasOption(OPT_NUM_REGIONS_PER_SERVER)) {
465       numRegionsPerServer = Integer.parseInt(cmd.getOptionValue(OPT_NUM_REGIONS_PER_SERVER));
466     }
467 
468     regionReplication = 1;
469     if (cmd.hasOption(OPT_REGION_REPLICATION)) {
470       regionReplication = Integer.parseInt(cmd.getOptionValue(OPT_REGION_REPLICATION));
471     }
472 
473     regionReplicaId = -1;
474     if (cmd.hasOption(OPT_REGION_REPLICA_ID)) {
475       regionReplicaId = Integer.parseInt(cmd.getOptionValue(OPT_REGION_REPLICA_ID));
476     }
477   }
478 
479   private void parseColumnFamilyOptions(CommandLine cmd) {
480     String dataBlockEncodingStr = cmd.getOptionValue(OPT_DATA_BLOCK_ENCODING);
481     dataBlockEncodingAlgo = dataBlockEncodingStr == null ? null :
482         DataBlockEncoding.valueOf(dataBlockEncodingStr);
483 
484     String compressStr = cmd.getOptionValue(OPT_COMPRESSION);
485     compressAlgo = compressStr == null ? Compression.Algorithm.NONE :
486         Compression.Algorithm.valueOf(compressStr);
487 
488     String bloomStr = cmd.getOptionValue(OPT_BLOOM);
489     bloomType = bloomStr == null ? BloomType.ROW :
490         BloomType.valueOf(bloomStr);
491 
492     inMemoryCF = cmd.hasOption(OPT_INMEMORY);
493     if (cmd.hasOption(OPT_ENCRYPTION)) {
494       cipher = Encryption.getCipher(conf, cmd.getOptionValue(OPT_ENCRYPTION));
495     }
496 
497   }
498 
499   public void initTestTable() throws IOException {
500     Durability durability = Durability.USE_DEFAULT;
501     if (deferredLogFlush) {
502       durability = Durability.ASYNC_WAL;
503     }
504 
505     HBaseTestingUtility.createPreSplitLoadTestTable(conf, tableName,
506         COLUMN_FAMILY, compressAlgo, dataBlockEncodingAlgo, numRegionsPerServer,
507         regionReplication, durability);
508     applyColumnFamilyOptions(tableName, COLUMN_FAMILIES);
509   }
510 
511   @Override
512   protected int doWork() throws IOException {
513     if (numTables > 1) {
514       return parallelLoadTables();
515     } else {
516       return loadTable();
517     }
518   }
519 
520   protected int loadTable() throws IOException {
521     if (cmd.hasOption(OPT_ZK_QUORUM)) {
522       conf.set(HConstants.ZOOKEEPER_QUORUM, cmd.getOptionValue(OPT_ZK_QUORUM));
523     }
524     if (cmd.hasOption(OPT_ZK_PARENT_NODE)) {
525       conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, cmd.getOptionValue(OPT_ZK_PARENT_NODE));
526     }
527 
528     if (isInitOnly) {
529       LOG.info("Initializing only; no reads or writes");
530       initTestTable();
531       return 0;
532     }
533 
534     if (!isSkipInit) {
535       initTestTable();
536     }
537     LoadTestDataGenerator dataGen = null;
538     if (cmd.hasOption(OPT_GENERATOR)) {
539       String[] clazzAndArgs = cmd.getOptionValue(OPT_GENERATOR).split(COLON);
540       dataGen = getLoadGeneratorInstance(clazzAndArgs[0]);
541       String[] args;
542       if (dataGen instanceof LoadTestDataGeneratorWithACL) {
543         LOG.info("Using LoadTestDataGeneratorWithACL");
544         if (User.isHBaseSecurityEnabled(conf)) {
545           LOG.info("Security is enabled");
546           authnFileName = clazzAndArgs[1];
547           superUser = clazzAndArgs[2];
548           userNames = clazzAndArgs[3];
549           args = Arrays.copyOfRange(clazzAndArgs, 2, clazzAndArgs.length);
550           Properties authConfig = new Properties();
551           authConfig.load(this.getClass().getClassLoader().getResourceAsStream(authnFileName));
552           try {
553             addAuthInfoToConf(authConfig, conf, superUser, userNames);
554           } catch (IOException exp) {
555             LOG.error(exp);
556             return EXIT_FAILURE;
557           }
558           userOwner = User.create(loginAndReturnUGI(conf, superUser));
559         } else {
560           superUser = clazzAndArgs[1];
561           userNames = clazzAndArgs[2];
562           args = Arrays.copyOfRange(clazzAndArgs, 1, clazzAndArgs.length);
563           userOwner = User.createUserForTesting(conf, superUser, new String[0]);
564         }
565       } else {
566         args = clazzAndArgs.length == 1 ? new String[0] : Arrays.copyOfRange(clazzAndArgs, 1,
567             clazzAndArgs.length);
568       }
569       dataGen.initialize(args);
570     } else {
571       // Default DataGenerator is MultiThreadedAction.DefaultDataGenerator
572       dataGen = new MultiThreadedAction.DefaultDataGenerator(minColDataSize, maxColDataSize,
573           minColsPerKey, maxColsPerKey, COLUMN_FAMILY);
574     }
575 
576     if (userOwner != null) {
577       LOG.info("Granting permissions for user " + userOwner.getShortName());
578       Permission.Action[] actions = {
579         Permission.Action.ADMIN, Permission.Action.CREATE,
580         Permission.Action.READ, Permission.Action.WRITE };
581       try {
582         AccessControlClient.grant(conf, tableName, userOwner.getShortName(), null, null, actions);
583       } catch (Throwable e) {
584         LOG.fatal("Error in granting permission for the user " + userOwner.getShortName(), e);
585         return EXIT_FAILURE;
586       }
587     }
588 
589     if (userNames != null) {
590       // This will be comma separated list of expressions.
591       String users[] = userNames.split(",");
592       User user = null;
593       for (String userStr : users) {
594         if (User.isHBaseSecurityEnabled(conf)) {
595           user = User.create(loginAndReturnUGI(conf, userStr));
596         } else {
597           user = User.createUserForTesting(conf, userStr, new String[0]);
598         }
599       }
600     }
601 
602     if (isWrite) {
603       if (userOwner != null) {
604         writerThreads = new MultiThreadedWriterWithACL(dataGen, conf, tableName, userOwner);
605       } else {
606         writerThreads = new MultiThreadedWriter(dataGen, conf, tableName);
607       }
608       writerThreads.setMultiPut(isMultiPut);
609     }
610 
611     if (isUpdate) {
612       if (userOwner != null) {
613         updaterThreads = new MultiThreadedUpdaterWithACL(dataGen, conf, tableName, updatePercent,
614             userOwner, userNames);
615       } else {
616         updaterThreads = new MultiThreadedUpdater(dataGen, conf, tableName, updatePercent);
617       }
618       updaterThreads.setBatchUpdate(isBatchUpdate);
619       updaterThreads.setIgnoreNonceConflicts(ignoreConflicts);
620     }
621 
622     if (isRead) {
623       if (userOwner != null) {
624         readerThreads = new MultiThreadedReaderWithACL(dataGen, conf, tableName, verifyPercent,
625             userNames);
626       } else {
627         String readerClass = null;
628         if (cmd.hasOption(OPT_READER)) {
629           readerClass = cmd.getOptionValue(OPT_READER);
630         } else {
631           readerClass = MultiThreadedReader.class.getCanonicalName();
632         }
633         readerThreads = getMultiThreadedReaderInstance(readerClass, dataGen);
634       }
635       readerThreads.setMaxErrors(maxReadErrors);
636       readerThreads.setKeyWindow(keyWindow);
637       readerThreads.setMultiGetBatchSize(multiGetBatchSize);
638       readerThreads.setRegionReplicaId(regionReplicaId);
639     }
640 
641     if (isUpdate && isWrite) {
642       LOG.info("Concurrent write/update workload: making updaters aware of the " +
643         "write point");
644       updaterThreads.linkToWriter(writerThreads);
645     }
646 
647     if (isRead && (isUpdate || isWrite)) {
648       LOG.info("Concurrent write/read workload: making readers aware of the " +
649         "write point");
650       readerThreads.linkToWriter(isUpdate ? updaterThreads : writerThreads);
651     }
652 
653     if (isWrite) {
654       System.out.println("Starting to write data...");
655       writerThreads.start(startKey, endKey, numWriterThreads);
656     }
657 
658     if (isUpdate) {
659       LOG.info("Starting to mutate data...");
660       System.out.println("Starting to mutate data...");
661       // TODO : currently append and increment operations not tested with tags
662       // Will update this aftet it is done
663       updaterThreads.start(startKey, endKey, numUpdaterThreads);
664     }
665 
666     if (isRead) {
667       System.out.println("Starting to read data...");
668       readerThreads.start(startKey, endKey, numReaderThreads);
669     }
670 
671     if (isWrite) {
672       writerThreads.waitForFinish();
673     }
674 
675     if (isUpdate) {
676       updaterThreads.waitForFinish();
677     }
678 
679     if (isRead) {
680       readerThreads.waitForFinish();
681     }
682 
683     boolean success = true;
684     if (isWrite) {
685       success = success && writerThreads.getNumWriteFailures() == 0;
686     }
687     if (isUpdate) {
688       success = success && updaterThreads.getNumWriteFailures() == 0;
689     }
690     if (isRead) {
691       success = success && readerThreads.getNumReadErrors() == 0
692           && readerThreads.getNumReadFailures() == 0;
693     }
694     return success ? EXIT_SUCCESS : EXIT_FAILURE;
695   }
696 
697   private LoadTestDataGenerator getLoadGeneratorInstance(String clazzName) throws IOException {
698     try {
699       Class<?> clazz = Class.forName(clazzName);
700       Constructor<?> constructor = clazz.getConstructor(int.class, int.class, int.class, int.class,
701           byte[][].class);
702       return (LoadTestDataGenerator) constructor.newInstance(minColDataSize, maxColDataSize,
703           minColsPerKey, maxColsPerKey, COLUMN_FAMILIES);
704     } catch (Exception e) {
705       throw new IOException(e);
706     }
707   }
708 
709   private MultiThreadedReader getMultiThreadedReaderInstance(String clazzName
710       , LoadTestDataGenerator dataGen) throws IOException {
711     try {
712       Class<?> clazz = Class.forName(clazzName);
713       Constructor<?> constructor = clazz.getConstructor(
714         LoadTestDataGenerator.class, Configuration.class, TableName.class, double.class);
715       return (MultiThreadedReader) constructor.newInstance(dataGen, conf, tableName, verifyPercent);
716     } catch (Exception e) {
717       throw new IOException(e);
718     }
719   }
720 
721   public static byte[] generateData(final Random r, int length) {
722     byte [] b = new byte [length];
723     int i = 0;
724 
725     for(i = 0; i < (length-8); i += 8) {
726       b[i] = (byte) (65 + r.nextInt(26));
727       b[i+1] = b[i];
728       b[i+2] = b[i];
729       b[i+3] = b[i];
730       b[i+4] = b[i];
731       b[i+5] = b[i];
732       b[i+6] = b[i];
733       b[i+7] = b[i];
734     }
735 
736     byte a = (byte) (65 + r.nextInt(26));
737     for(; i < length; i++) {
738       b[i] = a;
739     }
740     return b;
741   }
742   public static void main(String[] args) {
743     new LoadTestTool().doStaticMain(args);
744   }
745 
746   /**
747    * When NUM_TABLES is specified, the function starts multiple worker threads
748    * which individually start a LoadTestTool instance to load a table. Each
749    * table name is in format <tn>_<index>. For example, "-tn test -num_tables 2"
750    * , table names will be "test_1", "test_2"
751    *
752    * @throws IOException
753    */
754   private int parallelLoadTables()
755       throws IOException {
756     // create new command args
757     String tableName = cmd.getOptionValue(OPT_TABLE_NAME, DEFAULT_TABLE_NAME);
758     String[] newArgs = null;
759     if (!cmd.hasOption(LoadTestTool.OPT_TABLE_NAME)) {
760       newArgs = new String[cmdLineArgs.length + 2];
761       newArgs[0] = "-" + LoadTestTool.OPT_TABLE_NAME;
762       newArgs[1] = LoadTestTool.DEFAULT_TABLE_NAME;
763       System.arraycopy(cmdLineArgs, 0, newArgs, 2, cmdLineArgs.length);
764     } else {
765       newArgs = cmdLineArgs;
766     }
767 
768     int tableNameValueIndex = -1;
769     for (int j = 0; j < newArgs.length; j++) {
770       if (newArgs[j].endsWith(OPT_TABLE_NAME)) {
771         tableNameValueIndex = j + 1;
772       } else if (newArgs[j].endsWith(NUM_TABLES)) {
773         // change NUM_TABLES to 1 so that each worker loads one table
774         newArgs[j + 1] = "1";
775       }
776     }
777 
778     // starting to load multiple tables
779     List<WorkerThread> workers = new ArrayList<WorkerThread>();
780     for (int i = 0; i < numTables; i++) {
781       String[] workerArgs = newArgs.clone();
782       workerArgs[tableNameValueIndex] = tableName + "_" + (i+1);
783       WorkerThread worker = new WorkerThread(i, workerArgs);
784       workers.add(worker);
785       LOG.info(worker + " starting");
786       worker.start();
787     }
788 
789     // wait for all workers finish
790     LOG.info("Waiting for worker threads to finish");
791     for (WorkerThread t : workers) {
792       try {
793         t.join();
794       } catch (InterruptedException ie) {
795         IOException iie = new InterruptedIOException();
796         iie.initCause(ie);
797         throw iie;
798       }
799       checkForErrors();
800     }
801 
802     return EXIT_SUCCESS;
803   }
804 
805   // If an exception is thrown by one of worker threads, it will be
806   // stored here.
807   protected AtomicReference<Throwable> thrown = new AtomicReference<Throwable>();
808 
809   private void workerThreadError(Throwable t) {
810     thrown.compareAndSet(null, t);
811   }
812 
813   /**
814    * Check for errors in the writer threads. If any is found, rethrow it.
815    */
816   private void checkForErrors() throws IOException {
817     Throwable thrown = this.thrown.get();
818     if (thrown == null) return;
819     if (thrown instanceof IOException) {
820       throw (IOException) thrown;
821     } else {
822       throw new RuntimeException(thrown);
823     }
824   }
825 
826   class WorkerThread extends Thread {
827     private String[] workerArgs;
828 
829     WorkerThread(int i, String[] args) {
830       super("WorkerThread-" + i);
831       workerArgs = args;
832     }
833 
834     @Override
835     public void run() {
836       try {
837         int ret = ToolRunner.run(HBaseConfiguration.create(), new LoadTestTool(), workerArgs);
838         if (ret != 0) {
839           throw new RuntimeException("LoadTestTool exit with non-zero return code.");
840         }
841       } catch (Exception ex) {
842         LOG.error("Error in worker thread", ex);
843         workerThreadError(ex);
844       }
845     }
846   }
847 
848   private void addAuthInfoToConf(Properties authConfig, Configuration conf, String owner,
849       String userList) throws IOException {
850     List<String> users = Arrays.asList(userList.split(","));
851     users.add(owner);
852     for (String user : users) {
853       String keyTabFileConfKey = "hbase." + user + ".keytab.file";
854       String principalConfKey = "hbase." + user + ".kerberos.principal";
855       if (!authConfig.containsKey(keyTabFileConfKey) || !authConfig.containsKey(principalConfKey)) {
856         throw new IOException("Authentication configs missing for user : " + user);
857       }
858     }
859     for (String key : authConfig.stringPropertyNames()) {
860       conf.set(key, authConfig.getProperty(key));
861     }
862     LOG.debug("Added authentication properties to config successfully.");
863   }
864 
865   public static UserGroupInformation loginAndReturnUGI(Configuration conf, String username)
866       throws IOException {
867     String hostname = InetAddress.getLocalHost().getHostName();
868     String keyTabFileConfKey = "hbase." + username + ".keytab.file";
869     String keyTabFileLocation = conf.get(keyTabFileConfKey);
870     String principalConfKey = "hbase." + username + ".kerberos.principal";
871     String principal = SecurityUtil.getServerPrincipal(conf.get(principalConfKey), hostname);
872     if (keyTabFileLocation == null || principal == null) {
873       LOG.warn("Principal or key tab file null for : " + principalConfKey + ", "
874           + keyTabFileConfKey);
875     }
876     UserGroupInformation ugi =
877         UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keyTabFileLocation);
878     return ugi;
879   }
880 }