View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations
15   * under the License.
16   */
17  package org.apache.hadoop.hbase.util;
18  
19  import java.io.IOException;
20  import java.io.InterruptedIOException;
21  import java.lang.reflect.Constructor;
22  import java.security.SecureRandom;
23  import java.util.ArrayList;
24  import java.util.Arrays;
25  import java.util.List;
26  import java.util.Random;
27  import java.util.concurrent.atomic.AtomicReference;
28  
29  import javax.crypto.spec.SecretKeySpec;
30  
31  import org.apache.commons.cli.CommandLine;
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.hbase.HBaseConfiguration;
35  import org.apache.hadoop.hbase.HBaseTestingUtility;
36  import org.apache.hadoop.hbase.HColumnDescriptor;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.HTableDescriptor;
39  import org.apache.hadoop.hbase.PerformanceEvaluation;
40  import org.apache.hadoop.hbase.TableName;
41  import org.apache.hadoop.hbase.client.HBaseAdmin;
42  import org.apache.hadoop.hbase.client.HTable;
43  import org.apache.hadoop.hbase.io.compress.Compression;
44  import org.apache.hadoop.hbase.io.crypto.Cipher;
45  import org.apache.hadoop.hbase.io.crypto.Encryption;
46  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
47  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos;
48  import org.apache.hadoop.hbase.regionserver.BloomType;
49  import org.apache.hadoop.hbase.security.EncryptionUtil;
50  import org.apache.hadoop.hbase.security.User;
51  import org.apache.hadoop.hbase.security.access.AccessControlClient;
52  import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator;
53  import org.apache.hadoop.hbase.util.test.LoadTestDataGeneratorWithACL;
54  import org.apache.hadoop.util.ToolRunner;
55  
56  /**
57   * A command-line utility that reads, writes, and verifies data. Unlike
58   * {@link PerformanceEvaluation}, this tool validates the data written,
59   * and supports simultaneously writing and reading the same set of keys.
60   */
61  public class LoadTestTool extends AbstractHBaseTool {
62  
63    private static final Log LOG = LogFactory.getLog(LoadTestTool.class);
64    private static final String COLON = ":";
65  
66    /** Table name for the test */
67    private TableName tableName;
68  
69    /** Table name to use of not overridden on the command line */
70    protected static final String DEFAULT_TABLE_NAME = "cluster_test";
71  
72    /** Column family used by the test */
73    public static byte[] COLUMN_FAMILY = Bytes.toBytes("test_cf");
74  
75    /** Column families used by the test */
76    protected static final byte[][] COLUMN_FAMILIES = { COLUMN_FAMILY };
77  
78    /** The default data size if not specified */
79    protected static final int DEFAULT_DATA_SIZE = 64;
80  
81    /** The number of reader/writer threads if not specified */
82    protected static final int DEFAULT_NUM_THREADS = 20;
83  
84    /** Usage string for the load option */
85    protected static final String OPT_USAGE_LOAD =
86        "<avg_cols_per_key>:<avg_data_size>" +
87        "[:<#threads=" + DEFAULT_NUM_THREADS + ">]";
88  
89    /** Usage string for the read option */
90    protected static final String OPT_USAGE_READ =
91        "<verify_percent>[:<#threads=" + DEFAULT_NUM_THREADS + ">]";
92  
93    /** Usage string for the update option */
94    protected static final String OPT_USAGE_UPDATE =
95        "<update_percent>[:<#threads=" + DEFAULT_NUM_THREADS
96        + ">][:<#whether to ignore nonce collisions=0>]";
97  
98    protected static final String OPT_USAGE_BLOOM = "Bloom filter type, one of " +
99        Arrays.toString(BloomType.values());
100 
101   protected static final String OPT_USAGE_COMPRESSION = "Compression type, " +
102       "one of " + Arrays.toString(Compression.Algorithm.values());
103 
104   public static final String OPT_DATA_BLOCK_ENCODING_USAGE =
105     "Encoding algorithm (e.g. prefix "
106         + "compression) to use for data blocks in the test column family, "
107         + "one of " + Arrays.toString(DataBlockEncoding.values()) + ".";
108 
109   private static final String OPT_BLOOM = "bloom";
110   private static final String OPT_COMPRESSION = "compression";
111   public static final String OPT_DATA_BLOCK_ENCODING =
112       HColumnDescriptor.DATA_BLOCK_ENCODING.toLowerCase();
113 
114   public static final String OPT_INMEMORY = "in_memory";
115   public static final String OPT_USAGE_IN_MEMORY = "Tries to keep the HFiles of the CF " +
116   		"inmemory as far as possible.  Not guaranteed that reads are always served from inmemory";
117 
118   public static final String OPT_GENERATOR = "generator";
119   public static final String OPT_GENERATOR_USAGE = "The class which generates load for the tool."
120       + " Any args for this class can be passed as colon separated after class name";
121 
122   protected static final String OPT_KEY_WINDOW = "key_window";
123   protected static final String OPT_WRITE = "write";
124   protected static final String OPT_MAX_READ_ERRORS = "max_read_errors";
125   protected static final String OPT_MULTIPUT = "multiput";
126   protected static final String OPT_NUM_KEYS = "num_keys";
127   protected static final String OPT_READ = "read";
128   protected static final String OPT_START_KEY = "start_key";
129   protected static final String OPT_TABLE_NAME = "tn";
130   protected static final String OPT_ZK_QUORUM = "zk";
131   protected static final String OPT_ZK_PARENT_NODE = "zk_root";
132   protected static final String OPT_SKIP_INIT = "skip_init";
133   protected static final String OPT_INIT_ONLY = "init_only";
134   private static final String NUM_TABLES = "num_tables";
135   protected static final String OPT_BATCHUPDATE = "batchupdate";
136   protected static final String OPT_UPDATE = "update";
137 
138   protected static final String OPT_ENCRYPTION = "encryption";
139   protected static final String OPT_ENCRYPTION_USAGE =
140     "Enables transparent encryption on the test table, one of " +
141     Arrays.toString(Encryption.getSupportedCiphers());
142 
143   protected static final long DEFAULT_START_KEY = 0;
144 
145   /** This will be removed as we factor out the dependency on command line */
146   protected CommandLine cmd;
147 
148   protected MultiThreadedWriter writerThreads = null;
149   protected MultiThreadedReader readerThreads = null;
150   protected MultiThreadedUpdater updaterThreads = null;
151 
152   protected long startKey, endKey;
153 
154   protected boolean isWrite, isRead, isUpdate;
155 
156   // Column family options
157   protected DataBlockEncoding dataBlockEncodingAlgo;
158   protected Compression.Algorithm compressAlgo;
159   protected BloomType bloomType;
160   private boolean inMemoryCF;
161 
162   private User userOwner;
163   // Writer options
164   protected int numWriterThreads = DEFAULT_NUM_THREADS;
165   protected int minColsPerKey, maxColsPerKey;
166   protected int minColDataSize = DEFAULT_DATA_SIZE, maxColDataSize = DEFAULT_DATA_SIZE;
167   protected boolean isMultiPut;
168 
169   // Updater options
170   protected int numUpdaterThreads = DEFAULT_NUM_THREADS;
171   protected int updatePercent;
172   protected boolean ignoreConflicts = false;
173   protected boolean isBatchUpdate;
174 
175   // Reader options
176   private int numReaderThreads = DEFAULT_NUM_THREADS;
177   private int keyWindow = MultiThreadedReader.DEFAULT_KEY_WINDOW;
178   private int maxReadErrors = MultiThreadedReader.DEFAULT_MAX_ERRORS;
179   private int verifyPercent;
180 
181   private int numTables = 1;
182 
183   private String superUser;
184 
185   private String userNames = "user1, user2, user3, user4";
186 
187   // TODO: refactor LoadTestToolImpl somewhere to make the usage from tests less bad,
188   //       console tool itself should only be used from console.
189   protected boolean isSkipInit = false;
190   protected boolean isInitOnly = false;
191 
192   protected Cipher cipher = null;
193 
194   protected String[] splitColonSeparated(String option,
195       int minNumCols, int maxNumCols) {
196     String optVal = cmd.getOptionValue(option);
197     String[] cols = optVal.split(COLON);
198     if (cols.length < minNumCols || cols.length > maxNumCols) {
199       throw new IllegalArgumentException("Expected at least "
200           + minNumCols + " columns but no more than " + maxNumCols +
201           " in the colon-separated value '" + optVal + "' of the " +
202           "-" + option + " option");
203     }
204     return cols;
205   }
206 
207   protected int getNumThreads(String numThreadsStr) {
208     return parseInt(numThreadsStr, 1, Short.MAX_VALUE);
209   }
210 
211   /**
212    * Apply column family options such as Bloom filters, compression, and data
213    * block encoding.
214    */
215   protected void applyColumnFamilyOptions(TableName tableName,
216       byte[][] columnFamilies) throws IOException {
217     HBaseAdmin admin = new HBaseAdmin(conf);
218     HTableDescriptor tableDesc = admin.getTableDescriptor(tableName);
219     LOG.info("Disabling table " + tableName);
220     admin.disableTable(tableName);
221     for (byte[] cf : columnFamilies) {
222       HColumnDescriptor columnDesc = tableDesc.getFamily(cf);
223       boolean isNewCf = columnDesc == null;
224       if (isNewCf) {
225         columnDesc = new HColumnDescriptor(cf);
226       }
227       if (bloomType != null) {
228         columnDesc.setBloomFilterType(bloomType);
229       }
230       if (compressAlgo != null) {
231         columnDesc.setCompressionType(compressAlgo);
232       }
233       if (dataBlockEncodingAlgo != null) {
234         columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo);
235       }
236       if (inMemoryCF) {
237         columnDesc.setInMemory(inMemoryCF);
238       }
239       if (cipher != null) {
240         byte[] keyBytes = new byte[cipher.getKeyLength()];
241         new SecureRandom().nextBytes(keyBytes);
242         columnDesc.setEncryptionType(cipher.getName());
243         columnDesc.setEncryptionKey(EncryptionUtil.wrapKey(conf,
244           User.getCurrent().getShortName(),
245           new SecretKeySpec(keyBytes, cipher.getName())));
246       }
247       if (isNewCf) {
248         admin.addColumn(tableName, columnDesc);
249       } else {
250         admin.modifyColumn(tableName, columnDesc);
251       }
252     }
253     LOG.info("Enabling table " + tableName);
254     admin.enableTable(tableName);
255   }
256 
257   @Override
258   protected void addOptions() {
259     addOptWithArg(OPT_ZK_QUORUM, "ZK quorum as comma-separated host names " +
260         "without port numbers");
261     addOptWithArg(OPT_ZK_PARENT_NODE, "name of parent znode in zookeeper");
262     addOptWithArg(OPT_TABLE_NAME, "The name of the table to read or write");
263     addOptWithArg(OPT_WRITE, OPT_USAGE_LOAD);
264     addOptWithArg(OPT_READ, OPT_USAGE_READ);
265     addOptWithArg(OPT_UPDATE, OPT_USAGE_UPDATE);
266     addOptNoArg(OPT_INIT_ONLY, "Initialize the test table only, don't do any loading");
267     addOptWithArg(OPT_BLOOM, OPT_USAGE_BLOOM);
268     addOptWithArg(OPT_COMPRESSION, OPT_USAGE_COMPRESSION);
269     addOptWithArg(OPT_DATA_BLOCK_ENCODING, OPT_DATA_BLOCK_ENCODING_USAGE);
270     addOptWithArg(OPT_MAX_READ_ERRORS, "The maximum number of read errors " +
271         "to tolerate before terminating all reader threads. The default is " +
272         MultiThreadedReader.DEFAULT_MAX_ERRORS + ".");
273     addOptWithArg(OPT_KEY_WINDOW, "The 'key window' to maintain between " +
274         "reads and writes for concurrent write/read workload. The default " +
275         "is " + MultiThreadedReader.DEFAULT_KEY_WINDOW + ".");
276 
277     addOptNoArg(OPT_MULTIPUT, "Whether to use multi-puts as opposed to " +
278         "separate puts for every column in a row");
279     addOptNoArg(OPT_BATCHUPDATE, "Whether to use batch as opposed to " +
280         "separate updates for every column in a row");
281     addOptNoArg(OPT_INMEMORY, OPT_USAGE_IN_MEMORY);
282     addOptWithArg(OPT_GENERATOR, OPT_GENERATOR_USAGE);
283 
284     addOptWithArg(OPT_NUM_KEYS, "The number of keys to read/write");
285     addOptWithArg(OPT_START_KEY, "The first key to read/write " +
286         "(a 0-based index). The default value is " +
287         DEFAULT_START_KEY + ".");
288     addOptNoArg(OPT_SKIP_INIT, "Skip the initialization; assume test table "
289         + "already exists");
290 
291     addOptWithArg(NUM_TABLES,
292       "A positive integer number. When a number n is speicfied, load test "
293           + "tool  will load n table parallely. -tn parameter value becomes "
294           + "table name prefix. Each table name is in format <tn>_1...<tn>_n");
295 
296     addOptWithArg(OPT_ENCRYPTION, OPT_ENCRYPTION_USAGE);
297   }
298 
299   @Override
300   protected void processOptions(CommandLine cmd) {
301     this.cmd = cmd;
302 
303     tableName = TableName.valueOf(cmd.getOptionValue(OPT_TABLE_NAME,
304         DEFAULT_TABLE_NAME));
305 
306     isWrite = cmd.hasOption(OPT_WRITE);
307     isRead = cmd.hasOption(OPT_READ);
308     isUpdate = cmd.hasOption(OPT_UPDATE);
309     isInitOnly = cmd.hasOption(OPT_INIT_ONLY);
310 
311     if (!isWrite && !isRead && !isUpdate && !isInitOnly) {
312       throw new IllegalArgumentException("Either -" + OPT_WRITE + " or " +
313         "-" + OPT_UPDATE + "-" + OPT_READ + " has to be specified");
314     }
315 
316     if (isInitOnly && (isRead || isWrite || isUpdate)) {
317       throw new IllegalArgumentException(OPT_INIT_ONLY + " cannot be specified with"
318           + " either -" + OPT_WRITE + " or -" + OPT_UPDATE + " or -" + OPT_READ);
319     }
320 
321     if (!isInitOnly) {
322       if (!cmd.hasOption(OPT_NUM_KEYS)) {
323         throw new IllegalArgumentException(OPT_NUM_KEYS + " must be specified in "
324             + "read or write mode");
325       }
326       startKey = parseLong(cmd.getOptionValue(OPT_START_KEY,
327           String.valueOf(DEFAULT_START_KEY)), 0, Long.MAX_VALUE);
328       long numKeys = parseLong(cmd.getOptionValue(OPT_NUM_KEYS), 1,
329           Long.MAX_VALUE - startKey);
330       endKey = startKey + numKeys;
331       isSkipInit = cmd.hasOption(OPT_SKIP_INIT);
332       System.out.println("Key range: [" + startKey + ".." + (endKey - 1) + "]");
333     }
334 
335     parseColumnFamilyOptions(cmd);
336 
337     if (isWrite) {
338       String[] writeOpts = splitColonSeparated(OPT_WRITE, 2, 3);
339 
340       int colIndex = 0;
341       minColsPerKey = 1;
342       maxColsPerKey = 2 * Integer.parseInt(writeOpts[colIndex++]);
343       int avgColDataSize =
344           parseInt(writeOpts[colIndex++], 1, Integer.MAX_VALUE);
345       minColDataSize = avgColDataSize / 2;
346       maxColDataSize = avgColDataSize * 3 / 2;
347 
348       if (colIndex < writeOpts.length) {
349         numWriterThreads = getNumThreads(writeOpts[colIndex++]);
350       }
351 
352       isMultiPut = cmd.hasOption(OPT_MULTIPUT);
353 
354       System.out.println("Multi-puts: " + isMultiPut);
355       System.out.println("Columns per key: " + minColsPerKey + ".."
356           + maxColsPerKey);
357       System.out.println("Data size per column: " + minColDataSize + ".."
358           + maxColDataSize);
359     }
360 
361     if (isUpdate) {
362       String[] mutateOpts = splitColonSeparated(OPT_UPDATE, 1, 3);
363       int colIndex = 0;
364       updatePercent = parseInt(mutateOpts[colIndex++], 0, 100);
365       if (colIndex < mutateOpts.length) {
366         numUpdaterThreads = getNumThreads(mutateOpts[colIndex++]);
367       }
368       if (colIndex < mutateOpts.length) {
369         ignoreConflicts = parseInt(mutateOpts[colIndex++], 0, 1) == 1;
370       }
371 
372       isBatchUpdate = cmd.hasOption(OPT_BATCHUPDATE);
373 
374       System.out.println("Batch updates: " + isBatchUpdate);
375       System.out.println("Percent of keys to update: " + updatePercent);
376       System.out.println("Updater threads: " + numUpdaterThreads);
377       System.out.println("Ignore nonce conflicts: " + ignoreConflicts);
378     }
379 
380     if (isRead) {
381       String[] readOpts = splitColonSeparated(OPT_READ, 1, 2);
382       int colIndex = 0;
383       verifyPercent = parseInt(readOpts[colIndex++], 0, 100);
384       if (colIndex < readOpts.length) {
385         numReaderThreads = getNumThreads(readOpts[colIndex++]);
386       }
387 
388       if (cmd.hasOption(OPT_MAX_READ_ERRORS)) {
389         maxReadErrors = parseInt(cmd.getOptionValue(OPT_MAX_READ_ERRORS),
390             0, Integer.MAX_VALUE);
391       }
392 
393       if (cmd.hasOption(OPT_KEY_WINDOW)) {
394         keyWindow = parseInt(cmd.getOptionValue(OPT_KEY_WINDOW),
395             0, Integer.MAX_VALUE);
396       }
397 
398       System.out.println("Percent of keys to verify: " + verifyPercent);
399       System.out.println("Reader threads: " + numReaderThreads);
400     }
401 
402     numTables = 1;
403     if(cmd.hasOption(NUM_TABLES)) {
404       numTables = parseInt(cmd.getOptionValue(NUM_TABLES), 1, Short.MAX_VALUE);
405     }
406   }
407 
408   private void parseColumnFamilyOptions(CommandLine cmd) {
409     String dataBlockEncodingStr = cmd.getOptionValue(OPT_DATA_BLOCK_ENCODING);
410     dataBlockEncodingAlgo = dataBlockEncodingStr == null ? null :
411         DataBlockEncoding.valueOf(dataBlockEncodingStr);
412 
413     String compressStr = cmd.getOptionValue(OPT_COMPRESSION);
414     compressAlgo = compressStr == null ? Compression.Algorithm.NONE :
415         Compression.Algorithm.valueOf(compressStr);
416 
417     String bloomStr = cmd.getOptionValue(OPT_BLOOM);
418     bloomType = bloomStr == null ? null :
419         BloomType.valueOf(bloomStr);
420 
421     inMemoryCF = cmd.hasOption(OPT_INMEMORY);
422     if (cmd.hasOption(OPT_ENCRYPTION)) {
423       cipher = Encryption.getCipher(conf, cmd.getOptionValue(OPT_ENCRYPTION));
424     }
425   }
426 
427   public void initTestTable() throws IOException {
428     HBaseTestingUtility.createPreSplitLoadTestTable(conf, tableName,
429         COLUMN_FAMILY, compressAlgo, dataBlockEncodingAlgo);
430     applyColumnFamilyOptions(tableName, COLUMN_FAMILIES);
431   }
432 
433   @Override
434   protected int doWork() throws IOException {
435     if (numTables > 1) {
436       return parallelLoadTables();
437     } else {
438       return loadTable();
439     }
440   }
441 
442   protected int loadTable() throws IOException {
443     if (cmd.hasOption(OPT_ZK_QUORUM)) {
444       conf.set(HConstants.ZOOKEEPER_QUORUM, cmd.getOptionValue(OPT_ZK_QUORUM));
445     }
446     if (cmd.hasOption(OPT_ZK_PARENT_NODE)) {
447       conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, cmd.getOptionValue(OPT_ZK_PARENT_NODE));
448     }
449 
450     if (isInitOnly) {
451       LOG.info("Initializing only; no reads or writes");
452       initTestTable();
453       return 0;
454     }
455 
456     if (!isSkipInit) {
457       initTestTable();
458     }
459     LoadTestDataGenerator dataGen = null;
460     if (cmd.hasOption(OPT_GENERATOR)) {
461       String[] clazzAndArgs = cmd.getOptionValue(OPT_GENERATOR).split(COLON);
462       dataGen = getLoadGeneratorInstance(clazzAndArgs[0]);
463       String args[];
464       if (dataGen instanceof LoadTestDataGeneratorWithACL) {
465         LOG.info("ACL is on");
466         superUser = clazzAndArgs[1];
467         userNames = clazzAndArgs[2];
468         args = Arrays.copyOfRange(clazzAndArgs, 1,
469             clazzAndArgs.length);
470         userOwner = User.createUserForTesting(conf, superUser, new String[0]);
471       } else {
472         args = clazzAndArgs.length == 1 ? new String[0] : Arrays.copyOfRange(clazzAndArgs, 1,
473             clazzAndArgs.length);
474       }
475       dataGen.initialize(args);
476     } else {
477       // Default DataGenerator is MultiThreadedAction.DefaultDataGenerator
478       dataGen = new MultiThreadedAction.DefaultDataGenerator(minColDataSize, maxColDataSize,
479           minColsPerKey, maxColsPerKey, COLUMN_FAMILY);
480     }
481 
482     if(userOwner != null) {
483       conf.set("hadoop.security.authorization", "false");
484       conf.set("hadoop.security.authentication", "simple");
485       LOG.info("Granting permission for the user " + userOwner.getShortName());
486       HTable table = new HTable(conf, tableName);
487       AccessControlProtos.Permission.Action[] actions = {
488           AccessControlProtos.Permission.Action.ADMIN,
489           AccessControlProtos.Permission.Action.CREATE, AccessControlProtos.Permission.Action.READ,
490           AccessControlProtos.Permission.Action.WRITE };
491 
492       try {
493         AccessControlClient.grant(conf, table.getName(), userOwner.getShortName(), COLUMN_FAMILY,
494             null, actions);
495       } catch (Throwable e) {
496         LOG.fatal("Error in granting permission for the user " + userOwner.getShortName(), e);
497         return EXIT_FAILURE;
498       }
499     }
500 
501     if (isWrite) {
502       if (userOwner != null) {
503         writerThreads = new MultiThreadedWriterWithACL(dataGen, conf, tableName, userOwner);
504       } else {
505         writerThreads = new MultiThreadedWriter(dataGen, conf, tableName);
506       }
507       writerThreads.setMultiPut(isMultiPut);
508     }
509 
510     if (isUpdate) {
511       if (userOwner != null) {
512         updaterThreads = new MultiThreadedUpdaterWithACL(dataGen, conf, tableName, updatePercent,
513             userOwner, userNames);
514       } else {
515         updaterThreads = new MultiThreadedUpdater(dataGen, conf, tableName, updatePercent);
516       }
517       updaterThreads.setBatchUpdate(isBatchUpdate);
518       updaterThreads.setIgnoreNonceConflicts(ignoreConflicts);
519     }
520 
521     if (isRead) {
522       if (userOwner != null) {
523         readerThreads = new MultiThreadedReaderWithACL(dataGen, conf, tableName, verifyPercent,
524             userNames);
525       } else {
526         readerThreads = new MultiThreadedReader(dataGen, conf, tableName, verifyPercent);
527       }
528       readerThreads.setMaxErrors(maxReadErrors);
529       readerThreads.setKeyWindow(keyWindow);
530     }
531 
532     if (isUpdate && isWrite) {
533       LOG.info("Concurrent write/update workload: making updaters aware of the " +
534         "write point");
535       updaterThreads.linkToWriter(writerThreads);
536     }
537 
538     if (isRead && (isUpdate || isWrite)) {
539       LOG.info("Concurrent write/read workload: making readers aware of the " +
540         "write point");
541       readerThreads.linkToWriter(isUpdate ? updaterThreads : writerThreads);
542     }
543 
544     if (isWrite) {
545       System.out.println("Starting to write data...");
546       writerThreads.start(startKey, endKey, numWriterThreads);
547     }
548 
549     if (isUpdate) {
550       LOG.info("Starting to mutate data...");
551       System.out.println("Starting to mutate data...");
552       // TODO : currently append and increment operations not tested with tags
553       // Will update this aftet it is done
554       updaterThreads.start(startKey, endKey, numUpdaterThreads);
555     }
556 
557     if (isRead) {
558       System.out.println("Starting to read data...");
559       readerThreads.start(startKey, endKey, numReaderThreads);
560     }
561 
562     if (isWrite) {
563       writerThreads.waitForFinish();
564     }
565 
566     if (isUpdate) {
567       updaterThreads.waitForFinish();
568     }
569 
570     if (isRead) {
571       readerThreads.waitForFinish();
572     }
573 
574     boolean success = true;
575     if (isWrite) {
576       success = success && writerThreads.getNumWriteFailures() == 0;
577     }
578     if (isUpdate) {
579       success = success && updaterThreads.getNumWriteFailures() == 0;
580     }
581     if (isRead) {
582       success = success && readerThreads.getNumReadErrors() == 0
583           && readerThreads.getNumReadFailures() == 0;
584     }
585     return success ? EXIT_SUCCESS : EXIT_FAILURE;
586   }
587 
588   private LoadTestDataGenerator getLoadGeneratorInstance(String clazzName) throws IOException {
589     try {
590       Class<?> clazz = Class.forName(clazzName);
591       Constructor<?> constructor = clazz.getConstructor(int.class, int.class, int.class, int.class,
592           byte[][].class);
593       return (LoadTestDataGenerator) constructor.newInstance(minColDataSize, maxColDataSize,
594           minColsPerKey, maxColsPerKey, COLUMN_FAMILIES);
595     } catch (Exception e) {
596       throw new IOException(e);
597     }
598   }
599 
600   public static byte[] generateData(final Random r, int length) {
601     byte [] b = new byte [length];
602     int i = 0;
603 
604     for(i = 0; i < (length-8); i += 8) {
605       b[i] = (byte) (65 + r.nextInt(26));
606       b[i+1] = b[i];
607       b[i+2] = b[i];
608       b[i+3] = b[i];
609       b[i+4] = b[i];
610       b[i+5] = b[i];
611       b[i+6] = b[i];
612       b[i+7] = b[i];
613     }
614 
615     byte a = (byte) (65 + r.nextInt(26));
616     for(; i < length; i++) {
617       b[i] = a;
618     }
619     return b;
620   }
621   public static void main(String[] args) {
622     new LoadTestTool().doStaticMain(args);
623   }
624 
625   /**
626    * When NUM_TABLES is specified, the function starts multiple worker threads
627    * which individually start a LoadTestTool instance to load a table. Each
628    * table name is in format <tn>_<index>. For example, "-tn test -num_tables 2"
629    * , table names will be "test_1", "test_2"
630    *
631    * @throws IOException
632    */
633   private int parallelLoadTables()
634       throws IOException {
635     // create new command args
636     String tableName = cmd.getOptionValue(OPT_TABLE_NAME, DEFAULT_TABLE_NAME);
637     String[] newArgs = null;
638     if (!cmd.hasOption(LoadTestTool.OPT_TABLE_NAME)) {
639       newArgs = new String[cmdLineArgs.length + 2];
640       newArgs[0] = "-" + LoadTestTool.OPT_TABLE_NAME;
641       newArgs[1] = LoadTestTool.DEFAULT_TABLE_NAME;
642       for (int i = 0; i < cmdLineArgs.length; i++) {
643         newArgs[i + 2] = cmdLineArgs[i];
644       }
645     } else {
646       newArgs = cmdLineArgs;
647     }
648 
649     int tableNameValueIndex = -1;
650     for (int j = 0; j < newArgs.length; j++) {
651       if (newArgs[j].endsWith(OPT_TABLE_NAME)) {
652         tableNameValueIndex = j + 1;
653       } else if (newArgs[j].endsWith(NUM_TABLES)) {
654         // change NUM_TABLES to 1 so that each worker loads one table
655         newArgs[j + 1] = "1";
656       }
657     }
658 
659     // starting to load multiple tables
660     List<WorkerThread> workers = new ArrayList<WorkerThread>();
661     for (int i = 0; i < numTables; i++) {
662       String[] workerArgs = newArgs.clone();
663       workerArgs[tableNameValueIndex] = tableName + "_" + (i+1);
664       WorkerThread worker = new WorkerThread(i, workerArgs);
665       workers.add(worker);
666       LOG.info(worker + " starting");
667       worker.start();
668     }
669 
670     // wait for all workers finish
671     LOG.info("Waiting for worker threads to finish");
672     for (WorkerThread t : workers) {
673       try {
674         t.join();
675       } catch (InterruptedException ie) {
676         IOException iie = new InterruptedIOException();
677         iie.initCause(ie);
678         throw iie;
679       }
680       checkForErrors();
681     }
682 
683     return EXIT_SUCCESS;
684   }
685 
686   // If an exception is thrown by one of worker threads, it will be
687   // stored here.
688   protected AtomicReference<Throwable> thrown = new AtomicReference<Throwable>();
689 
690   private void workerThreadError(Throwable t) {
691     thrown.compareAndSet(null, t);
692   }
693 
694   /**
695    * Check for errors in the writer threads. If any is found, rethrow it.
696    */
697   private void checkForErrors() throws IOException {
698     Throwable thrown = this.thrown.get();
699     if (thrown == null) return;
700     if (thrown instanceof IOException) {
701       throw (IOException) thrown;
702     } else {
703       throw new RuntimeException(thrown);
704     }
705   }
706 
707   class WorkerThread extends Thread {
708     private String[] workerArgs;
709 
710     WorkerThread(int i, String[] args) {
711       super("WorkerThread-" + i);
712       workerArgs = args;
713     }
714 
715     @Override
716     public void run() {
717       try {
718         int ret = ToolRunner.run(HBaseConfiguration.create(), new LoadTestTool(), workerArgs);
719         if (ret != 0) {
720           throw new RuntimeException("LoadTestTool exit with non-zero return code.");
721         }
722       } catch (Exception ex) {
723         LOG.error("Error in worker thread", ex);
724         workerThreadError(ex);
725       }
726     }
727   }
728 }