1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.util;
18
19 import java.io.IOException;
20 import java.io.InterruptedIOException;
21 import java.lang.reflect.Constructor;
22 import java.security.SecureRandom;
23 import java.util.ArrayList;
24 import java.util.Arrays;
25 import java.util.List;
26 import java.util.Random;
27 import java.util.concurrent.atomic.AtomicReference;
28
29 import javax.crypto.spec.SecretKeySpec;
30
31 import org.apache.commons.cli.CommandLine;
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.hbase.HBaseConfiguration;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HColumnDescriptor;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HTableDescriptor;
39 import org.apache.hadoop.hbase.PerformanceEvaluation;
40 import org.apache.hadoop.hbase.TableName;
41 import org.apache.hadoop.hbase.client.HBaseAdmin;
42 import org.apache.hadoop.hbase.client.HTable;
43 import org.apache.hadoop.hbase.io.compress.Compression;
44 import org.apache.hadoop.hbase.io.crypto.Cipher;
45 import org.apache.hadoop.hbase.io.crypto.Encryption;
46 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
47 import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos;
48 import org.apache.hadoop.hbase.regionserver.BloomType;
49 import org.apache.hadoop.hbase.security.EncryptionUtil;
50 import org.apache.hadoop.hbase.security.User;
51 import org.apache.hadoop.hbase.security.access.AccessControlClient;
52 import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator;
53 import org.apache.hadoop.hbase.util.test.LoadTestDataGeneratorWithACL;
54 import org.apache.hadoop.util.ToolRunner;
55
56
57
58
59
60
61 public class LoadTestTool extends AbstractHBaseTool {
62
63 private static final Log LOG = LogFactory.getLog(LoadTestTool.class);
64 private static final String COLON = ":";
65
66
67 private TableName tableName;
68
69
70 protected static final String DEFAULT_TABLE_NAME = "cluster_test";
71
72
73 public static byte[] COLUMN_FAMILY = Bytes.toBytes("test_cf");
74
75
76 protected static final byte[][] COLUMN_FAMILIES = { COLUMN_FAMILY };
77
78
79 protected static final int DEFAULT_DATA_SIZE = 64;
80
81
82 protected static final int DEFAULT_NUM_THREADS = 20;
83
84
85 protected static final String OPT_USAGE_LOAD =
86 "<avg_cols_per_key>:<avg_data_size>" +
87 "[:<#threads=" + DEFAULT_NUM_THREADS + ">]";
88
89
90 protected static final String OPT_USAGE_READ =
91 "<verify_percent>[:<#threads=" + DEFAULT_NUM_THREADS + ">]";
92
93
94 protected static final String OPT_USAGE_UPDATE =
95 "<update_percent>[:<#threads=" + DEFAULT_NUM_THREADS
96 + ">][:<#whether to ignore nonce collisions=0>]";
97
98 protected static final String OPT_USAGE_BLOOM = "Bloom filter type, one of " +
99 Arrays.toString(BloomType.values());
100
101 protected static final String OPT_USAGE_COMPRESSION = "Compression type, " +
102 "one of " + Arrays.toString(Compression.Algorithm.values());
103
104 public static final String OPT_DATA_BLOCK_ENCODING_USAGE =
105 "Encoding algorithm (e.g. prefix "
106 + "compression) to use for data blocks in the test column family, "
107 + "one of " + Arrays.toString(DataBlockEncoding.values()) + ".";
108
109 private static final String OPT_BLOOM = "bloom";
110 private static final String OPT_COMPRESSION = "compression";
111 public static final String OPT_DATA_BLOCK_ENCODING =
112 HColumnDescriptor.DATA_BLOCK_ENCODING.toLowerCase();
113
114 public static final String OPT_INMEMORY = "in_memory";
115 public static final String OPT_USAGE_IN_MEMORY = "Tries to keep the HFiles of the CF " +
116 "inmemory as far as possible. Not guaranteed that reads are always served from inmemory";
117
118 public static final String OPT_GENERATOR = "generator";
119 public static final String OPT_GENERATOR_USAGE = "The class which generates load for the tool."
120 + " Any args for this class can be passed as colon separated after class name";
121
122 protected static final String OPT_KEY_WINDOW = "key_window";
123 protected static final String OPT_WRITE = "write";
124 protected static final String OPT_MAX_READ_ERRORS = "max_read_errors";
125 protected static final String OPT_MULTIPUT = "multiput";
126 protected static final String OPT_NUM_KEYS = "num_keys";
127 protected static final String OPT_READ = "read";
128 protected static final String OPT_START_KEY = "start_key";
129 protected static final String OPT_TABLE_NAME = "tn";
130 protected static final String OPT_ZK_QUORUM = "zk";
131 protected static final String OPT_ZK_PARENT_NODE = "zk_root";
132 protected static final String OPT_SKIP_INIT = "skip_init";
133 protected static final String OPT_INIT_ONLY = "init_only";
134 private static final String NUM_TABLES = "num_tables";
135 protected static final String OPT_BATCHUPDATE = "batchupdate";
136 protected static final String OPT_UPDATE = "update";
137
138 protected static final String OPT_ENCRYPTION = "encryption";
139 protected static final String OPT_ENCRYPTION_USAGE =
140 "Enables transparent encryption on the test table, one of " +
141 Arrays.toString(Encryption.getSupportedCiphers());
142
143 protected static final long DEFAULT_START_KEY = 0;
144
145
146 protected CommandLine cmd;
147
148 protected MultiThreadedWriter writerThreads = null;
149 protected MultiThreadedReader readerThreads = null;
150 protected MultiThreadedUpdater updaterThreads = null;
151
152 protected long startKey, endKey;
153
154 protected boolean isWrite, isRead, isUpdate;
155
156
157 protected DataBlockEncoding dataBlockEncodingAlgo;
158 protected Compression.Algorithm compressAlgo;
159 protected BloomType bloomType;
160 private boolean inMemoryCF;
161
162 private User userOwner;
163
164 protected int numWriterThreads = DEFAULT_NUM_THREADS;
165 protected int minColsPerKey, maxColsPerKey;
166 protected int minColDataSize = DEFAULT_DATA_SIZE, maxColDataSize = DEFAULT_DATA_SIZE;
167 protected boolean isMultiPut;
168
169
170 protected int numUpdaterThreads = DEFAULT_NUM_THREADS;
171 protected int updatePercent;
172 protected boolean ignoreConflicts = false;
173 protected boolean isBatchUpdate;
174
175
176 private int numReaderThreads = DEFAULT_NUM_THREADS;
177 private int keyWindow = MultiThreadedReader.DEFAULT_KEY_WINDOW;
178 private int maxReadErrors = MultiThreadedReader.DEFAULT_MAX_ERRORS;
179 private int verifyPercent;
180
181 private int numTables = 1;
182
183 private String superUser;
184
185 private String userNames = "user1, user2, user3, user4";
186
187
188
189 protected boolean isSkipInit = false;
190 protected boolean isInitOnly = false;
191
192 protected Cipher cipher = null;
193
194 protected String[] splitColonSeparated(String option,
195 int minNumCols, int maxNumCols) {
196 String optVal = cmd.getOptionValue(option);
197 String[] cols = optVal.split(COLON);
198 if (cols.length < minNumCols || cols.length > maxNumCols) {
199 throw new IllegalArgumentException("Expected at least "
200 + minNumCols + " columns but no more than " + maxNumCols +
201 " in the colon-separated value '" + optVal + "' of the " +
202 "-" + option + " option");
203 }
204 return cols;
205 }
206
207 protected int getNumThreads(String numThreadsStr) {
208 return parseInt(numThreadsStr, 1, Short.MAX_VALUE);
209 }
210
211
212
213
214
215 protected void applyColumnFamilyOptions(TableName tableName,
216 byte[][] columnFamilies) throws IOException {
217 HBaseAdmin admin = new HBaseAdmin(conf);
218 HTableDescriptor tableDesc = admin.getTableDescriptor(tableName);
219 LOG.info("Disabling table " + tableName);
220 admin.disableTable(tableName);
221 for (byte[] cf : columnFamilies) {
222 HColumnDescriptor columnDesc = tableDesc.getFamily(cf);
223 boolean isNewCf = columnDesc == null;
224 if (isNewCf) {
225 columnDesc = new HColumnDescriptor(cf);
226 }
227 if (bloomType != null) {
228 columnDesc.setBloomFilterType(bloomType);
229 }
230 if (compressAlgo != null) {
231 columnDesc.setCompressionType(compressAlgo);
232 }
233 if (dataBlockEncodingAlgo != null) {
234 columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo);
235 }
236 if (inMemoryCF) {
237 columnDesc.setInMemory(inMemoryCF);
238 }
239 if (cipher != null) {
240 byte[] keyBytes = new byte[cipher.getKeyLength()];
241 new SecureRandom().nextBytes(keyBytes);
242 columnDesc.setEncryptionType(cipher.getName());
243 columnDesc.setEncryptionKey(EncryptionUtil.wrapKey(conf,
244 User.getCurrent().getShortName(),
245 new SecretKeySpec(keyBytes, cipher.getName())));
246 }
247 if (isNewCf) {
248 admin.addColumn(tableName, columnDesc);
249 } else {
250 admin.modifyColumn(tableName, columnDesc);
251 }
252 }
253 LOG.info("Enabling table " + tableName);
254 admin.enableTable(tableName);
255 }
256
257 @Override
258 protected void addOptions() {
259 addOptWithArg(OPT_ZK_QUORUM, "ZK quorum as comma-separated host names " +
260 "without port numbers");
261 addOptWithArg(OPT_ZK_PARENT_NODE, "name of parent znode in zookeeper");
262 addOptWithArg(OPT_TABLE_NAME, "The name of the table to read or write");
263 addOptWithArg(OPT_WRITE, OPT_USAGE_LOAD);
264 addOptWithArg(OPT_READ, OPT_USAGE_READ);
265 addOptWithArg(OPT_UPDATE, OPT_USAGE_UPDATE);
266 addOptNoArg(OPT_INIT_ONLY, "Initialize the test table only, don't do any loading");
267 addOptWithArg(OPT_BLOOM, OPT_USAGE_BLOOM);
268 addOptWithArg(OPT_COMPRESSION, OPT_USAGE_COMPRESSION);
269 addOptWithArg(OPT_DATA_BLOCK_ENCODING, OPT_DATA_BLOCK_ENCODING_USAGE);
270 addOptWithArg(OPT_MAX_READ_ERRORS, "The maximum number of read errors " +
271 "to tolerate before terminating all reader threads. The default is " +
272 MultiThreadedReader.DEFAULT_MAX_ERRORS + ".");
273 addOptWithArg(OPT_KEY_WINDOW, "The 'key window' to maintain between " +
274 "reads and writes for concurrent write/read workload. The default " +
275 "is " + MultiThreadedReader.DEFAULT_KEY_WINDOW + ".");
276
277 addOptNoArg(OPT_MULTIPUT, "Whether to use multi-puts as opposed to " +
278 "separate puts for every column in a row");
279 addOptNoArg(OPT_BATCHUPDATE, "Whether to use batch as opposed to " +
280 "separate updates for every column in a row");
281 addOptNoArg(OPT_INMEMORY, OPT_USAGE_IN_MEMORY);
282 addOptWithArg(OPT_GENERATOR, OPT_GENERATOR_USAGE);
283
284 addOptWithArg(OPT_NUM_KEYS, "The number of keys to read/write");
285 addOptWithArg(OPT_START_KEY, "The first key to read/write " +
286 "(a 0-based index). The default value is " +
287 DEFAULT_START_KEY + ".");
288 addOptNoArg(OPT_SKIP_INIT, "Skip the initialization; assume test table "
289 + "already exists");
290
291 addOptWithArg(NUM_TABLES,
292 "A positive integer number. When a number n is speicfied, load test "
293 + "tool will load n table parallely. -tn parameter value becomes "
294 + "table name prefix. Each table name is in format <tn>_1...<tn>_n");
295
296 addOptWithArg(OPT_ENCRYPTION, OPT_ENCRYPTION_USAGE);
297 }
298
299 @Override
300 protected void processOptions(CommandLine cmd) {
301 this.cmd = cmd;
302
303 tableName = TableName.valueOf(cmd.getOptionValue(OPT_TABLE_NAME,
304 DEFAULT_TABLE_NAME));
305
306 isWrite = cmd.hasOption(OPT_WRITE);
307 isRead = cmd.hasOption(OPT_READ);
308 isUpdate = cmd.hasOption(OPT_UPDATE);
309 isInitOnly = cmd.hasOption(OPT_INIT_ONLY);
310
311 if (!isWrite && !isRead && !isUpdate && !isInitOnly) {
312 throw new IllegalArgumentException("Either -" + OPT_WRITE + " or " +
313 "-" + OPT_UPDATE + "-" + OPT_READ + " has to be specified");
314 }
315
316 if (isInitOnly && (isRead || isWrite || isUpdate)) {
317 throw new IllegalArgumentException(OPT_INIT_ONLY + " cannot be specified with"
318 + " either -" + OPT_WRITE + " or -" + OPT_UPDATE + " or -" + OPT_READ);
319 }
320
321 if (!isInitOnly) {
322 if (!cmd.hasOption(OPT_NUM_KEYS)) {
323 throw new IllegalArgumentException(OPT_NUM_KEYS + " must be specified in "
324 + "read or write mode");
325 }
326 startKey = parseLong(cmd.getOptionValue(OPT_START_KEY,
327 String.valueOf(DEFAULT_START_KEY)), 0, Long.MAX_VALUE);
328 long numKeys = parseLong(cmd.getOptionValue(OPT_NUM_KEYS), 1,
329 Long.MAX_VALUE - startKey);
330 endKey = startKey + numKeys;
331 isSkipInit = cmd.hasOption(OPT_SKIP_INIT);
332 System.out.println("Key range: [" + startKey + ".." + (endKey - 1) + "]");
333 }
334
335 parseColumnFamilyOptions(cmd);
336
337 if (isWrite) {
338 String[] writeOpts = splitColonSeparated(OPT_WRITE, 2, 3);
339
340 int colIndex = 0;
341 minColsPerKey = 1;
342 maxColsPerKey = 2 * Integer.parseInt(writeOpts[colIndex++]);
343 int avgColDataSize =
344 parseInt(writeOpts[colIndex++], 1, Integer.MAX_VALUE);
345 minColDataSize = avgColDataSize / 2;
346 maxColDataSize = avgColDataSize * 3 / 2;
347
348 if (colIndex < writeOpts.length) {
349 numWriterThreads = getNumThreads(writeOpts[colIndex++]);
350 }
351
352 isMultiPut = cmd.hasOption(OPT_MULTIPUT);
353
354 System.out.println("Multi-puts: " + isMultiPut);
355 System.out.println("Columns per key: " + minColsPerKey + ".."
356 + maxColsPerKey);
357 System.out.println("Data size per column: " + minColDataSize + ".."
358 + maxColDataSize);
359 }
360
361 if (isUpdate) {
362 String[] mutateOpts = splitColonSeparated(OPT_UPDATE, 1, 3);
363 int colIndex = 0;
364 updatePercent = parseInt(mutateOpts[colIndex++], 0, 100);
365 if (colIndex < mutateOpts.length) {
366 numUpdaterThreads = getNumThreads(mutateOpts[colIndex++]);
367 }
368 if (colIndex < mutateOpts.length) {
369 ignoreConflicts = parseInt(mutateOpts[colIndex++], 0, 1) == 1;
370 }
371
372 isBatchUpdate = cmd.hasOption(OPT_BATCHUPDATE);
373
374 System.out.println("Batch updates: " + isBatchUpdate);
375 System.out.println("Percent of keys to update: " + updatePercent);
376 System.out.println("Updater threads: " + numUpdaterThreads);
377 System.out.println("Ignore nonce conflicts: " + ignoreConflicts);
378 }
379
380 if (isRead) {
381 String[] readOpts = splitColonSeparated(OPT_READ, 1, 2);
382 int colIndex = 0;
383 verifyPercent = parseInt(readOpts[colIndex++], 0, 100);
384 if (colIndex < readOpts.length) {
385 numReaderThreads = getNumThreads(readOpts[colIndex++]);
386 }
387
388 if (cmd.hasOption(OPT_MAX_READ_ERRORS)) {
389 maxReadErrors = parseInt(cmd.getOptionValue(OPT_MAX_READ_ERRORS),
390 0, Integer.MAX_VALUE);
391 }
392
393 if (cmd.hasOption(OPT_KEY_WINDOW)) {
394 keyWindow = parseInt(cmd.getOptionValue(OPT_KEY_WINDOW),
395 0, Integer.MAX_VALUE);
396 }
397
398 System.out.println("Percent of keys to verify: " + verifyPercent);
399 System.out.println("Reader threads: " + numReaderThreads);
400 }
401
402 numTables = 1;
403 if(cmd.hasOption(NUM_TABLES)) {
404 numTables = parseInt(cmd.getOptionValue(NUM_TABLES), 1, Short.MAX_VALUE);
405 }
406 }
407
408 private void parseColumnFamilyOptions(CommandLine cmd) {
409 String dataBlockEncodingStr = cmd.getOptionValue(OPT_DATA_BLOCK_ENCODING);
410 dataBlockEncodingAlgo = dataBlockEncodingStr == null ? null :
411 DataBlockEncoding.valueOf(dataBlockEncodingStr);
412
413 String compressStr = cmd.getOptionValue(OPT_COMPRESSION);
414 compressAlgo = compressStr == null ? Compression.Algorithm.NONE :
415 Compression.Algorithm.valueOf(compressStr);
416
417 String bloomStr = cmd.getOptionValue(OPT_BLOOM);
418 bloomType = bloomStr == null ? null :
419 BloomType.valueOf(bloomStr);
420
421 inMemoryCF = cmd.hasOption(OPT_INMEMORY);
422 if (cmd.hasOption(OPT_ENCRYPTION)) {
423 cipher = Encryption.getCipher(conf, cmd.getOptionValue(OPT_ENCRYPTION));
424 }
425 }
426
427 public void initTestTable() throws IOException {
428 HBaseTestingUtility.createPreSplitLoadTestTable(conf, tableName,
429 COLUMN_FAMILY, compressAlgo, dataBlockEncodingAlgo);
430 applyColumnFamilyOptions(tableName, COLUMN_FAMILIES);
431 }
432
433 @Override
434 protected int doWork() throws IOException {
435 if (numTables > 1) {
436 return parallelLoadTables();
437 } else {
438 return loadTable();
439 }
440 }
441
442 protected int loadTable() throws IOException {
443 if (cmd.hasOption(OPT_ZK_QUORUM)) {
444 conf.set(HConstants.ZOOKEEPER_QUORUM, cmd.getOptionValue(OPT_ZK_QUORUM));
445 }
446 if (cmd.hasOption(OPT_ZK_PARENT_NODE)) {
447 conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, cmd.getOptionValue(OPT_ZK_PARENT_NODE));
448 }
449
450 if (isInitOnly) {
451 LOG.info("Initializing only; no reads or writes");
452 initTestTable();
453 return 0;
454 }
455
456 if (!isSkipInit) {
457 initTestTable();
458 }
459 LoadTestDataGenerator dataGen = null;
460 if (cmd.hasOption(OPT_GENERATOR)) {
461 String[] clazzAndArgs = cmd.getOptionValue(OPT_GENERATOR).split(COLON);
462 dataGen = getLoadGeneratorInstance(clazzAndArgs[0]);
463 String args[];
464 if (dataGen instanceof LoadTestDataGeneratorWithACL) {
465 LOG.info("ACL is on");
466 superUser = clazzAndArgs[1];
467 userNames = clazzAndArgs[2];
468 args = Arrays.copyOfRange(clazzAndArgs, 1,
469 clazzAndArgs.length);
470 userOwner = User.createUserForTesting(conf, superUser, new String[0]);
471 } else {
472 args = clazzAndArgs.length == 1 ? new String[0] : Arrays.copyOfRange(clazzAndArgs, 1,
473 clazzAndArgs.length);
474 }
475 dataGen.initialize(args);
476 } else {
477
478 dataGen = new MultiThreadedAction.DefaultDataGenerator(minColDataSize, maxColDataSize,
479 minColsPerKey, maxColsPerKey, COLUMN_FAMILY);
480 }
481
482 if(userOwner != null) {
483 conf.set("hadoop.security.authorization", "false");
484 conf.set("hadoop.security.authentication", "simple");
485 LOG.info("Granting permission for the user " + userOwner.getShortName());
486 HTable table = new HTable(conf, tableName);
487 AccessControlProtos.Permission.Action[] actions = {
488 AccessControlProtos.Permission.Action.ADMIN,
489 AccessControlProtos.Permission.Action.CREATE, AccessControlProtos.Permission.Action.READ,
490 AccessControlProtos.Permission.Action.WRITE };
491
492 try {
493 AccessControlClient.grant(conf, table.getName(), userOwner.getShortName(), COLUMN_FAMILY,
494 null, actions);
495 } catch (Throwable e) {
496 LOG.fatal("Error in granting permission for the user " + userOwner.getShortName(), e);
497 return EXIT_FAILURE;
498 }
499 }
500
501 if (isWrite) {
502 if (userOwner != null) {
503 writerThreads = new MultiThreadedWriterWithACL(dataGen, conf, tableName, userOwner);
504 } else {
505 writerThreads = new MultiThreadedWriter(dataGen, conf, tableName);
506 }
507 writerThreads.setMultiPut(isMultiPut);
508 }
509
510 if (isUpdate) {
511 if (userOwner != null) {
512 updaterThreads = new MultiThreadedUpdaterWithACL(dataGen, conf, tableName, updatePercent,
513 userOwner, userNames);
514 } else {
515 updaterThreads = new MultiThreadedUpdater(dataGen, conf, tableName, updatePercent);
516 }
517 updaterThreads.setBatchUpdate(isBatchUpdate);
518 updaterThreads.setIgnoreNonceConflicts(ignoreConflicts);
519 }
520
521 if (isRead) {
522 if (userOwner != null) {
523 readerThreads = new MultiThreadedReaderWithACL(dataGen, conf, tableName, verifyPercent,
524 userNames);
525 } else {
526 readerThreads = new MultiThreadedReader(dataGen, conf, tableName, verifyPercent);
527 }
528 readerThreads.setMaxErrors(maxReadErrors);
529 readerThreads.setKeyWindow(keyWindow);
530 }
531
532 if (isUpdate && isWrite) {
533 LOG.info("Concurrent write/update workload: making updaters aware of the " +
534 "write point");
535 updaterThreads.linkToWriter(writerThreads);
536 }
537
538 if (isRead && (isUpdate || isWrite)) {
539 LOG.info("Concurrent write/read workload: making readers aware of the " +
540 "write point");
541 readerThreads.linkToWriter(isUpdate ? updaterThreads : writerThreads);
542 }
543
544 if (isWrite) {
545 System.out.println("Starting to write data...");
546 writerThreads.start(startKey, endKey, numWriterThreads);
547 }
548
549 if (isUpdate) {
550 LOG.info("Starting to mutate data...");
551 System.out.println("Starting to mutate data...");
552
553
554 updaterThreads.start(startKey, endKey, numUpdaterThreads);
555 }
556
557 if (isRead) {
558 System.out.println("Starting to read data...");
559 readerThreads.start(startKey, endKey, numReaderThreads);
560 }
561
562 if (isWrite) {
563 writerThreads.waitForFinish();
564 }
565
566 if (isUpdate) {
567 updaterThreads.waitForFinish();
568 }
569
570 if (isRead) {
571 readerThreads.waitForFinish();
572 }
573
574 boolean success = true;
575 if (isWrite) {
576 success = success && writerThreads.getNumWriteFailures() == 0;
577 }
578 if (isUpdate) {
579 success = success && updaterThreads.getNumWriteFailures() == 0;
580 }
581 if (isRead) {
582 success = success && readerThreads.getNumReadErrors() == 0
583 && readerThreads.getNumReadFailures() == 0;
584 }
585 return success ? EXIT_SUCCESS : EXIT_FAILURE;
586 }
587
588 private LoadTestDataGenerator getLoadGeneratorInstance(String clazzName) throws IOException {
589 try {
590 Class<?> clazz = Class.forName(clazzName);
591 Constructor<?> constructor = clazz.getConstructor(int.class, int.class, int.class, int.class,
592 byte[][].class);
593 return (LoadTestDataGenerator) constructor.newInstance(minColDataSize, maxColDataSize,
594 minColsPerKey, maxColsPerKey, COLUMN_FAMILIES);
595 } catch (Exception e) {
596 throw new IOException(e);
597 }
598 }
599
600 public static byte[] generateData(final Random r, int length) {
601 byte [] b = new byte [length];
602 int i = 0;
603
604 for(i = 0; i < (length-8); i += 8) {
605 b[i] = (byte) (65 + r.nextInt(26));
606 b[i+1] = b[i];
607 b[i+2] = b[i];
608 b[i+3] = b[i];
609 b[i+4] = b[i];
610 b[i+5] = b[i];
611 b[i+6] = b[i];
612 b[i+7] = b[i];
613 }
614
615 byte a = (byte) (65 + r.nextInt(26));
616 for(; i < length; i++) {
617 b[i] = a;
618 }
619 return b;
620 }
621 public static void main(String[] args) {
622 new LoadTestTool().doStaticMain(args);
623 }
624
625
626
627
628
629
630
631
632
633 private int parallelLoadTables()
634 throws IOException {
635
636 String tableName = cmd.getOptionValue(OPT_TABLE_NAME, DEFAULT_TABLE_NAME);
637 String[] newArgs = null;
638 if (!cmd.hasOption(LoadTestTool.OPT_TABLE_NAME)) {
639 newArgs = new String[cmdLineArgs.length + 2];
640 newArgs[0] = "-" + LoadTestTool.OPT_TABLE_NAME;
641 newArgs[1] = LoadTestTool.DEFAULT_TABLE_NAME;
642 for (int i = 0; i < cmdLineArgs.length; i++) {
643 newArgs[i + 2] = cmdLineArgs[i];
644 }
645 } else {
646 newArgs = cmdLineArgs;
647 }
648
649 int tableNameValueIndex = -1;
650 for (int j = 0; j < newArgs.length; j++) {
651 if (newArgs[j].endsWith(OPT_TABLE_NAME)) {
652 tableNameValueIndex = j + 1;
653 } else if (newArgs[j].endsWith(NUM_TABLES)) {
654
655 newArgs[j + 1] = "1";
656 }
657 }
658
659
660 List<WorkerThread> workers = new ArrayList<WorkerThread>();
661 for (int i = 0; i < numTables; i++) {
662 String[] workerArgs = newArgs.clone();
663 workerArgs[tableNameValueIndex] = tableName + "_" + (i+1);
664 WorkerThread worker = new WorkerThread(i, workerArgs);
665 workers.add(worker);
666 LOG.info(worker + " starting");
667 worker.start();
668 }
669
670
671 LOG.info("Waiting for worker threads to finish");
672 for (WorkerThread t : workers) {
673 try {
674 t.join();
675 } catch (InterruptedException ie) {
676 IOException iie = new InterruptedIOException();
677 iie.initCause(ie);
678 throw iie;
679 }
680 checkForErrors();
681 }
682
683 return EXIT_SUCCESS;
684 }
685
686
687
688 protected AtomicReference<Throwable> thrown = new AtomicReference<Throwable>();
689
690 private void workerThreadError(Throwable t) {
691 thrown.compareAndSet(null, t);
692 }
693
694
695
696
697 private void checkForErrors() throws IOException {
698 Throwable thrown = this.thrown.get();
699 if (thrown == null) return;
700 if (thrown instanceof IOException) {
701 throw (IOException) thrown;
702 } else {
703 throw new RuntimeException(thrown);
704 }
705 }
706
707 class WorkerThread extends Thread {
708 private String[] workerArgs;
709
710 WorkerThread(int i, String[] args) {
711 super("WorkerThread-" + i);
712 workerArgs = args;
713 }
714
715 @Override
716 public void run() {
717 try {
718 int ret = ToolRunner.run(HBaseConfiguration.create(), new LoadTestTool(), workerArgs);
719 if (ret != 0) {
720 throw new RuntimeException("LoadTestTool exit with non-zero return code.");
721 }
722 } catch (Exception ex) {
723 LOG.error("Error in worker thread", ex);
724 workerThreadError(ex);
725 }
726 }
727 }
728 }