1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import static org.codehaus.jackson.map.SerializationConfig.Feature.SORT_PROPERTIES_ALPHABETICALLY;
22
23 import java.io.IOException;
24 import java.io.PrintStream;
25 import java.lang.reflect.Constructor;
26 import java.math.BigDecimal;
27 import java.math.MathContext;
28 import java.text.DecimalFormat;
29 import java.text.SimpleDateFormat;
30 import java.util.ArrayList;
31 import java.util.Arrays;
32 import java.util.Date;
33 import java.util.Map;
34 import java.util.Random;
35 import java.util.TreeMap;
36 import java.util.concurrent.Callable;
37 import java.util.concurrent.ExecutionException;
38 import java.util.concurrent.ExecutorService;
39 import java.util.concurrent.Executors;
40 import java.util.concurrent.Future;
41
42 import com.google.common.util.concurrent.ThreadFactoryBuilder;
43 import org.apache.commons.logging.Log;
44 import org.apache.commons.logging.LogFactory;
45 import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
46 import org.apache.hadoop.conf.Configuration;
47 import org.apache.hadoop.conf.Configured;
48 import org.apache.hadoop.fs.FileSystem;
49 import org.apache.hadoop.fs.Path;
50 import org.apache.hadoop.hbase.client.Durability;
51 import org.apache.hadoop.hbase.client.Get;
52 import org.apache.hadoop.hbase.client.HBaseAdmin;
53 import org.apache.hadoop.hbase.client.HConnection;
54 import org.apache.hadoop.hbase.client.HConnectionManager;
55 import org.apache.hadoop.hbase.client.HTableInterface;
56 import org.apache.hadoop.hbase.client.Put;
57 import org.apache.hadoop.hbase.client.Result;
58 import org.apache.hadoop.hbase.client.ResultScanner;
59 import org.apache.hadoop.hbase.client.Scan;
60 import org.apache.hadoop.hbase.filter.BinaryComparator;
61 import org.apache.hadoop.hbase.filter.CompareFilter;
62 import org.apache.hadoop.hbase.filter.Filter;
63 import org.apache.hadoop.hbase.filter.FilterAllFilter;
64 import org.apache.hadoop.hbase.filter.FilterList;
65 import org.apache.hadoop.hbase.filter.PageFilter;
66 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
67 import org.apache.hadoop.hbase.filter.WhileMatchFilter;
68 import org.apache.hadoop.hbase.io.compress.Compression;
69 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
70 import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
71 import org.apache.hadoop.hbase.util.Bytes;
72 import org.apache.hadoop.hbase.util.Hash;
73 import org.apache.hadoop.hbase.util.MurmurHash;
74 import org.apache.hadoop.hbase.util.Pair;
75 import org.apache.hadoop.io.LongWritable;
76 import org.apache.hadoop.io.Text;
77 import org.apache.hadoop.mapreduce.Job;
78 import org.apache.hadoop.mapreduce.Mapper;
79 import org.apache.hadoop.mapreduce.lib.input.NLineInputFormat;
80 import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
81 import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer;
82 import org.apache.hadoop.util.Tool;
83 import org.apache.hadoop.util.ToolRunner;
84 import org.codehaus.jackson.map.ObjectMapper;
85
86 import com.google.common.util.concurrent.ThreadFactoryBuilder;
87 import com.yammer.metrics.core.Histogram;
88 import com.yammer.metrics.core.MetricsRegistry;
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106 public class PerformanceEvaluation extends Configured implements Tool {
107 protected static final Log LOG = LogFactory.getLog(PerformanceEvaluation.class.getName());
108
109 public static final String TABLE_NAME = "TestTable";
110 public static final byte[] FAMILY_NAME = Bytes.toBytes("info");
111 public static final byte[] QUALIFIER_NAME = Bytes.toBytes("data");
112 public static final int VALUE_LENGTH = 1000;
113 public static final int ROW_LENGTH = 26;
114
115 private static final int ONE_GB = 1024 * 1024 * 1000;
116 private static final int ROWS_PER_GB = ONE_GB / VALUE_LENGTH;
117
118 private static final int TAG_LENGTH = 256;
119 private static final DecimalFormat FMT = new DecimalFormat("0.##");
120 private static final MathContext CXT = MathContext.DECIMAL64;
121 private static final BigDecimal MS_PER_SEC = BigDecimal.valueOf(1000);
122 private static final BigDecimal BYTES_PER_MB = BigDecimal.valueOf(1024 * 1024);
123 private static final TestOptions DEFAULT_OPTS = new TestOptions();
124
125 protected Map<String, CmdDescriptor> commands = new TreeMap<String, CmdDescriptor>();
126
127 private static final Path PERF_EVAL_DIR = new Path("performance_evaluation");
128
129
130
131
132
133 protected static enum Counter {
134
135 ELAPSED_TIME,
136
137 ROWS
138 }
139
140
141
142
143
144 public PerformanceEvaluation(final Configuration conf) {
145 super(conf);
146
147 addCommandDescriptor(RandomReadTest.class, "randomRead",
148 "Run random read test");
149 addCommandDescriptor(RandomSeekScanTest.class, "randomSeekScan",
150 "Run random seek and scan 100 test");
151 addCommandDescriptor(RandomScanWithRange10Test.class, "scanRange10",
152 "Run random seek scan with both start and stop row (max 10 rows)");
153 addCommandDescriptor(RandomScanWithRange100Test.class, "scanRange100",
154 "Run random seek scan with both start and stop row (max 100 rows)");
155 addCommandDescriptor(RandomScanWithRange1000Test.class, "scanRange1000",
156 "Run random seek scan with both start and stop row (max 1000 rows)");
157 addCommandDescriptor(RandomScanWithRange10000Test.class, "scanRange10000",
158 "Run random seek scan with both start and stop row (max 10000 rows)");
159 addCommandDescriptor(RandomWriteTest.class, "randomWrite",
160 "Run random write test");
161 addCommandDescriptor(SequentialReadTest.class, "sequentialRead",
162 "Run sequential read test");
163 addCommandDescriptor(SequentialWriteTest.class, "sequentialWrite",
164 "Run sequential write test");
165 addCommandDescriptor(ScanTest.class, "scan",
166 "Run scan test (read every row)");
167 addCommandDescriptor(FilteredScanTest.class, "filterScan",
168 "Run scan test using a filter to find a specific row based on it's value (make sure to use --rows=20)");
169 }
170
171 protected void addCommandDescriptor(Class<? extends Test> cmdClass,
172 String name, String description) {
173 CmdDescriptor cmdDescriptor =
174 new CmdDescriptor(cmdClass, name, description);
175 commands.put(name, cmdDescriptor);
176 }
177
178
179
180
181 interface Status {
182
183
184
185
186
187 void setStatus(final String msg) throws IOException;
188 }
189
190
191
192
193 public static class EvaluationMapTask
194 extends Mapper<LongWritable, Text, LongWritable, LongWritable> {
195
196
197 public final static String CMD_KEY = "EvaluationMapTask.command";
198
199 public static final String PE_KEY = "EvaluationMapTask.performanceEvalImpl";
200
201 private Class<? extends Test> cmd;
202 private PerformanceEvaluation pe;
203
204 @Override
205 protected void setup(Context context) throws IOException, InterruptedException {
206 this.cmd = forName(context.getConfiguration().get(CMD_KEY), Test.class);
207
208
209
210 Class<? extends PerformanceEvaluation> peClass =
211 forName(context.getConfiguration().get(PE_KEY), PerformanceEvaluation.class);
212 try {
213 this.pe = peClass.getConstructor(Configuration.class)
214 .newInstance(context.getConfiguration());
215 } catch (Exception e) {
216 throw new IllegalStateException("Could not instantiate PE instance", e);
217 }
218 }
219
220 private <Type> Class<? extends Type> forName(String className, Class<Type> type) {
221 try {
222 return Class.forName(className).asSubclass(type);
223 } catch (ClassNotFoundException e) {
224 throw new IllegalStateException("Could not find class for name: " + className, e);
225 }
226 }
227
228 protected void map(LongWritable key, Text value, final Context context)
229 throws IOException, InterruptedException {
230
231 Status status = new Status() {
232 public void setStatus(String msg) {
233 context.setStatus(msg);
234 }
235 };
236
237 ObjectMapper mapper = new ObjectMapper();
238 TestOptions opts = mapper.readValue(value.toString(), TestOptions.class);
239 Configuration conf = HBaseConfiguration.create(context.getConfiguration());
240
241
242 long elapsedTime = this.pe.runOneClient(this.cmd, conf, opts, status);
243
244
245 context.getCounter(Counter.ELAPSED_TIME).increment(elapsedTime);
246 context.getCounter(Counter.ROWS).increment(opts.perClientRunRows);
247 context.write(new LongWritable(opts.startRow), new LongWritable(elapsedTime));
248 context.progress();
249 }
250 }
251
252
253
254
255
256
257
258 private static boolean checkTable(HBaseAdmin admin, TestOptions opts) throws IOException {
259 HTableDescriptor tableDescriptor = getTableDescriptor(opts);
260 if (opts.presplitRegions > 0) {
261
262 if (admin.tableExists(tableDescriptor.getTableName())) {
263 admin.disableTable(tableDescriptor.getTableName());
264 admin.deleteTable(tableDescriptor.getTableName());
265 }
266
267 byte[][] splits = getSplits(opts);
268 for (int i=0; i < splits.length; i++) {
269 LOG.debug(" split " + i + ": " + Bytes.toStringBinary(splits[i]));
270 }
271 admin.createTable(tableDescriptor, splits);
272 LOG.info ("Table created with " + opts.presplitRegions + " splits");
273 }
274 else {
275 boolean tableExists = admin.tableExists(tableDescriptor.getTableName());
276 if (!tableExists) {
277 admin.createTable(tableDescriptor);
278 LOG.info("Table " + tableDescriptor + " created");
279 }
280 }
281 return admin.tableExists(tableDescriptor.getTableName());
282 }
283
284
285
286
287 protected static HTableDescriptor getTableDescriptor(TestOptions opts) {
288 HTableDescriptor desc = new HTableDescriptor(opts.tableName);
289 HColumnDescriptor family = new HColumnDescriptor(FAMILY_NAME);
290 family.setDataBlockEncoding(opts.blockEncoding);
291 family.setCompressionType(opts.compression);
292 if (opts.inMemoryCF) {
293 family.setInMemory(true);
294 }
295 desc.addFamily(family);
296 return desc;
297 }
298
299
300
301
302 protected static byte[][] getSplits(TestOptions opts) {
303 if (opts.presplitRegions == 0)
304 return new byte [0][];
305
306 int numSplitPoints = opts.presplitRegions - 1;
307 byte[][] splits = new byte[numSplitPoints][];
308 int jump = opts.totalRows / opts.presplitRegions;
309 for (int i = 0; i < numSplitPoints; i++) {
310 int rowkey = jump * (1 + i);
311 splits[i] = format(rowkey);
312 }
313 return splits;
314 }
315
316
317
318
319
320
321 private void doLocalClients(final Class<? extends Test> cmd, final TestOptions opts)
322 throws IOException, InterruptedException {
323 Future<Long>[] threads = new Future[opts.numClientThreads];
324 long[] timings = new long[opts.numClientThreads];
325 ExecutorService pool = Executors.newFixedThreadPool(opts.numClientThreads,
326 new ThreadFactoryBuilder().setNameFormat("TestClient-%s").build());
327 for (int i = 0; i < threads.length; i++) {
328 final int index = i;
329 threads[i] = pool.submit(new Callable<Long>() {
330 @Override
331 public Long call() throws Exception {
332 TestOptions threadOpts = new TestOptions(opts);
333 threadOpts.startRow = index * threadOpts.perClientRunRows;
334 long elapsedTime = runOneClient(cmd, getConf(), threadOpts, new Status() {
335 public void setStatus(final String msg) throws IOException {
336 LOG.info("client-" + Thread.currentThread().getName() + " " + msg);
337 }
338 });
339 LOG.info("Finished " + Thread.currentThread().getName() + " in " + elapsedTime +
340 "ms over " + threadOpts.perClientRunRows + " rows");
341 return elapsedTime;
342 }
343 });
344 }
345 pool.shutdown();
346 for (int i = 0; i < threads.length; i++) {
347 try {
348 timings[i] = threads[i].get();
349 } catch (ExecutionException e) {
350 throw new IOException(e.getCause());
351 }
352 }
353 final String test = cmd.getSimpleName();
354 LOG.info("[" + test + "] Summary of timings (ms): "
355 + Arrays.toString(timings));
356 Arrays.sort(timings);
357 long total = 0;
358 for (int i = 0; i < timings.length; i++) {
359 total += timings[i];
360 }
361 LOG.info("[" + test + "]"
362 + "\tMin: " + timings[0] + "ms"
363 + "\tMax: " + timings[timings.length - 1] + "ms"
364 + "\tAvg: " + (total / timings.length) + "ms");
365 }
366
367
368
369
370
371
372
373
374 private void doMapReduce(final Class<? extends Test> cmd, TestOptions opts) throws IOException,
375 InterruptedException, ClassNotFoundException {
376 Configuration conf = getConf();
377 Path inputDir = writeInputFile(conf, opts);
378 conf.set(EvaluationMapTask.CMD_KEY, cmd.getName());
379 conf.set(EvaluationMapTask.PE_KEY, getClass().getName());
380 Job job = new Job(conf);
381 job.setJarByClass(PerformanceEvaluation.class);
382 job.setJobName("HBase Performance Evaluation");
383
384 job.setInputFormatClass(NLineInputFormat.class);
385 NLineInputFormat.setInputPaths(job, inputDir);
386
387 NLineInputFormat.setNumLinesPerSplit(job, 1);
388
389 job.setOutputKeyClass(LongWritable.class);
390 job.setOutputValueClass(LongWritable.class);
391
392 job.setMapperClass(EvaluationMapTask.class);
393 job.setReducerClass(LongSumReducer.class);
394
395 job.setNumReduceTasks(1);
396
397 job.setOutputFormatClass(TextOutputFormat.class);
398 TextOutputFormat.setOutputPath(job, new Path(inputDir.getParent(), "outputs"));
399
400 TableMapReduceUtil.addDependencyJars(job);
401 TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
402 DescriptiveStatistics.class,
403 ObjectMapper.class);
404
405 TableMapReduceUtil.initCredentials(job);
406
407 job.waitForCompletion(true);
408 }
409
410
411
412
413
414
415
416 private Path writeInputFile(final Configuration c, final TestOptions opts) throws IOException {
417 SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
418 Path jobdir = new Path(PERF_EVAL_DIR, formatter.format(new Date()));
419 Path inputDir = new Path(jobdir, "inputs");
420
421 FileSystem fs = FileSystem.get(c);
422 fs.mkdirs(inputDir);
423
424 Path inputFile = new Path(inputDir, "input.txt");
425 PrintStream out = new PrintStream(fs.create(inputFile));
426
427 Map<Integer, String> m = new TreeMap<Integer, String>();
428 Hash h = MurmurHash.getInstance();
429 int perClientRows = (opts.totalRows / opts.numClientThreads);
430 ObjectMapper mapper = new ObjectMapper();
431 mapper.configure(SORT_PROPERTIES_ALPHABETICALLY, true);
432 try {
433 for (int i = 0; i < 10; i++) {
434 for (int j = 0; j < opts.numClientThreads; j++) {
435 TestOptions next = new TestOptions(opts);
436 next.startRow = (j * perClientRows) + (i * (perClientRows/10));
437 next.perClientRunRows = perClientRows / 10;
438 String s = mapper.writeValueAsString(next);
439 int hash = h.hash(Bytes.toBytes(s));
440 m.put(hash, s);
441 }
442 }
443 for (Map.Entry<Integer, String> e: m.entrySet()) {
444 out.println(e.getValue());
445 }
446 } finally {
447 out.close();
448 }
449 return inputDir;
450 }
451
452
453
454
455 static class CmdDescriptor {
456 private Class<? extends Test> cmdClass;
457 private String name;
458 private String description;
459
460 CmdDescriptor(Class<? extends Test> cmdClass, String name, String description) {
461 this.cmdClass = cmdClass;
462 this.name = name;
463 this.description = description;
464 }
465
466 public Class<? extends Test> getCmdClass() {
467 return cmdClass;
468 }
469
470 public String getName() {
471 return name;
472 }
473
474 public String getDescription() {
475 return description;
476 }
477 }
478
479
480
481
482
483 static class TestOptions {
484
485 public TestOptions() {}
486
487 public TestOptions(TestOptions that) {
488 this.nomapred = that.nomapred;
489 this.startRow = that.startRow;
490 this.perClientRunRows = that.perClientRunRows;
491 this.numClientThreads = that.numClientThreads;
492 this.totalRows = that.totalRows;
493 this.sampleRate = that.sampleRate;
494 this.tableName = that.tableName;
495 this.flushCommits = that.flushCommits;
496 this.writeToWAL = that.writeToWAL;
497 this.useTags = that.useTags;
498 this.noOfTags = that.noOfTags;
499 this.reportLatency = that.reportLatency;
500 this.multiGet = that.multiGet;
501 this.inMemoryCF = that.inMemoryCF;
502 this.presplitRegions = that.presplitRegions;
503 this.compression = that.compression;
504 this.blockEncoding = that.blockEncoding;
505 this.filterAll = that.filterAll;
506 }
507
508 public boolean nomapred = false;
509 public boolean filterAll = false;
510 public int startRow = 0;
511 public int perClientRunRows = ROWS_PER_GB;
512 public int numClientThreads = 1;
513 public int totalRows = ROWS_PER_GB;
514 public float sampleRate = 1.0f;
515 public String tableName = TABLE_NAME;
516 public boolean flushCommits = true;
517 public boolean writeToWAL = true;
518 public boolean useTags = false;
519 public int noOfTags = 1;
520 public boolean reportLatency = false;
521 public int multiGet = 0;
522 boolean inMemoryCF = false;
523 int presplitRegions = 0;
524 public Compression.Algorithm compression = Compression.Algorithm.NONE;
525 public DataBlockEncoding blockEncoding = DataBlockEncoding.NONE;
526 }
527
528
529
530
531
532 static abstract class Test {
533
534
535 private static final Random randomSeed = new Random(System.currentTimeMillis());
536 private static long nextRandomSeed() {
537 return randomSeed.nextLong();
538 }
539 protected final Random rand = new Random(nextRandomSeed());
540 protected final Configuration conf;
541 protected final TestOptions opts;
542
543 private final Status status;
544 protected HConnection connection;
545 protected HTableInterface table;
546
547
548
549
550
551 Test(final Configuration conf, final TestOptions options, final Status status) {
552 this.conf = conf;
553 this.opts = options;
554 this.status = status;
555 }
556
557 private String generateStatus(final int sr, final int i, final int lr) {
558 return sr + "/" + i + "/" + lr;
559 }
560
561 protected int getReportingPeriod() {
562 int period = opts.perClientRunRows / 10;
563 return period == 0 ? opts.perClientRunRows : period;
564 }
565
566 void testSetup() throws IOException {
567 this.connection = HConnectionManager.createConnection(conf);
568 this.table = connection.getTable(opts.tableName);
569 this.table.setAutoFlush(false, true);
570 }
571
572 void testTakedown() throws IOException {
573 if (opts.flushCommits) {
574 this.table.flushCommits();
575 }
576 table.close();
577 connection.close();
578 }
579
580
581
582
583
584
585 long test() throws IOException {
586 testSetup();
587 LOG.info("Timed test starting in thread " + Thread.currentThread().getName());
588 final long startTime = System.nanoTime();
589 try {
590 testTimed();
591 } finally {
592 testTakedown();
593 }
594 return (System.nanoTime() - startTime) / 1000000;
595 }
596
597
598
599
600 void testTimed() throws IOException {
601 int lastRow = opts.startRow + opts.perClientRunRows;
602
603 for (int i = opts.startRow; i < lastRow; i++) {
604 testRow(i);
605 if (status != null && i > 0 && (i % getReportingPeriod()) == 0) {
606 status.setStatus(generateStatus(opts.startRow, i, lastRow));
607 }
608 }
609 }
610
611
612
613
614
615 abstract void testRow(final int i) throws IOException;
616 }
617
618
619 @SuppressWarnings("unused")
620 static class RandomSeekScanTest extends Test {
621 RandomSeekScanTest(Configuration conf, TestOptions options, Status status) {
622 super(conf, options, status);
623 }
624
625 @Override
626 void testRow(final int i) throws IOException {
627 Scan scan = new Scan(getRandomRow(this.rand, opts.totalRows));
628 FilterList list = new FilterList();
629 scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
630 if (opts.filterAll) {
631 list.addFilter(new FilterAllFilter());
632 }
633 list.addFilter(new WhileMatchFilter(new PageFilter(120)));
634 scan.setFilter(list);
635 ResultScanner s = this.table.getScanner(scan);
636 for (Result rr; (rr = s.next()) != null;) ;
637 s.close();
638 }
639
640 @Override
641 protected int getReportingPeriod() {
642 int period = opts.perClientRunRows / 100;
643 return period == 0 ? opts.perClientRunRows : period;
644 }
645
646 }
647
648 @SuppressWarnings("unused")
649 static abstract class RandomScanWithRangeTest extends Test {
650 RandomScanWithRangeTest(Configuration conf, TestOptions options, Status status) {
651 super(conf, options, status);
652 }
653
654 @Override
655 void testRow(final int i) throws IOException {
656 Pair<byte[], byte[]> startAndStopRow = getStartAndStopRow();
657 Scan scan = new Scan(startAndStopRow.getFirst(), startAndStopRow.getSecond());
658 if (opts.filterAll) {
659 scan.setFilter(new FilterAllFilter());
660 }
661 scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
662 ResultScanner s = this.table.getScanner(scan);
663 int count = 0;
664 for (Result rr; (rr = s.next()) != null;) {
665 count++;
666 }
667
668 if (i % 100 == 0) {
669 LOG.info(String.format("Scan for key range %s - %s returned %s rows",
670 Bytes.toString(startAndStopRow.getFirst()),
671 Bytes.toString(startAndStopRow.getSecond()), count));
672 }
673
674 s.close();
675 }
676
677 protected abstract Pair<byte[],byte[]> getStartAndStopRow();
678
679 protected Pair<byte[], byte[]> generateStartAndStopRows(int maxRange) {
680 int start = this.rand.nextInt(Integer.MAX_VALUE) % opts.totalRows;
681 int stop = start + maxRange;
682 return new Pair<byte[],byte[]>(format(start), format(stop));
683 }
684
685 @Override
686 protected int getReportingPeriod() {
687 int period = opts.perClientRunRows / 100;
688 return period == 0? opts.perClientRunRows: period;
689 }
690 }
691
692 static class RandomScanWithRange10Test extends RandomScanWithRangeTest {
693 RandomScanWithRange10Test(Configuration conf, TestOptions options, Status status) {
694 super(conf, options, status);
695 }
696
697 @Override
698 protected Pair<byte[], byte[]> getStartAndStopRow() {
699 return generateStartAndStopRows(10);
700 }
701 }
702
703 static class RandomScanWithRange100Test extends RandomScanWithRangeTest {
704 RandomScanWithRange100Test(Configuration conf, TestOptions options, Status status) {
705 super(conf, options, status);
706 }
707
708 @Override
709 protected Pair<byte[], byte[]> getStartAndStopRow() {
710 return generateStartAndStopRows(100);
711 }
712 }
713
714 static class RandomScanWithRange1000Test extends RandomScanWithRangeTest {
715 RandomScanWithRange1000Test(Configuration conf, TestOptions options, Status status) {
716 super(conf, options, status);
717 }
718
719 @Override
720 protected Pair<byte[], byte[]> getStartAndStopRow() {
721 return generateStartAndStopRows(1000);
722 }
723 }
724
725 static class RandomScanWithRange10000Test extends RandomScanWithRangeTest {
726 RandomScanWithRange10000Test(Configuration conf, TestOptions options, Status status) {
727 super(conf, options, status);
728 }
729
730 @Override
731 protected Pair<byte[], byte[]> getStartAndStopRow() {
732 return generateStartAndStopRows(10000);
733 }
734 }
735
736 static class RandomReadTest extends Test {
737 private final int everyN;
738 private final double[] times;
739 private ArrayList<Get> gets;
740 int idx = 0;
741
742 RandomReadTest(Configuration conf, TestOptions options, Status status) {
743 super(conf, options, status);
744 everyN = (int) (opts.totalRows / (opts.totalRows * opts.sampleRate));
745 LOG.info("Sampling 1 every " + everyN + " out of " + opts.perClientRunRows + " total rows.");
746 if (opts.multiGet > 0) {
747 LOG.info("MultiGet enabled. Sending GETs in batches of " + opts.multiGet + ".");
748 this.gets = new ArrayList<Get>(opts.multiGet);
749 }
750 if (opts.reportLatency) {
751 this.times = new double[(int) Math.ceil(opts.perClientRunRows * opts.sampleRate / Math.max(1, opts.multiGet))];
752 } else {
753 this.times = null;
754 }
755 }
756
757 @Override
758 void testRow(final int i) throws IOException {
759 if (i % everyN == 0) {
760 Get get = new Get(getRandomRow(this.rand, opts.totalRows));
761 get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
762 if (opts.filterAll) {
763 get.setFilter(new FilterAllFilter());
764 }
765 if (opts.multiGet > 0) {
766 this.gets.add(get);
767 if (this.gets.size() == opts.multiGet) {
768 long start = System.nanoTime();
769 this.table.get(this.gets);
770 if (opts.reportLatency) {
771 times[idx++] = (System.nanoTime() - start) / 1e6;
772 }
773 this.gets.clear();
774 }
775 } else {
776 long start = System.nanoTime();
777 this.table.get(get);
778 if (opts.reportLatency) {
779 times[idx++] = (System.nanoTime() - start) / 1e6;
780 }
781 }
782 }
783 }
784
785 @Override
786 protected int getReportingPeriod() {
787 int period = opts.perClientRunRows / 100;
788 return period == 0 ? opts.perClientRunRows : period;
789 }
790
791 @Override
792 protected void testTakedown() throws IOException {
793 if (this.gets != null && this.gets.size() > 0) {
794 this.table.get(gets);
795 this.gets.clear();
796 }
797 super.testTakedown();
798 if (opts.reportLatency) {
799 Arrays.sort(times);
800 DescriptiveStatistics ds = new DescriptiveStatistics();
801 for (double t : times) {
802 ds.addValue(t);
803 }
804 LOG.info("randomRead latency log (ms), on " + times.length + " measures");
805 LOG.info("99.9999% = " + ds.getPercentile(99.9999d));
806 LOG.info(" 99.999% = " + ds.getPercentile(99.999d));
807 LOG.info(" 99.99% = " + ds.getPercentile(99.99d));
808 LOG.info(" 99.9% = " + ds.getPercentile(99.9d));
809 LOG.info(" 99% = " + ds.getPercentile(99d));
810 LOG.info(" 95% = " + ds.getPercentile(95d));
811 LOG.info(" 90% = " + ds.getPercentile(90d));
812 LOG.info(" 80% = " + ds.getPercentile(80d));
813 LOG.info("Standard Deviation = " + ds.getStandardDeviation());
814 LOG.info("Mean = " + ds.getMean());
815 }
816 }
817 }
818
819 static class RandomWriteTest extends Test {
820 RandomWriteTest(Configuration conf, TestOptions options, Status status) {
821 super(conf, options, status);
822 }
823
824 @Override
825 void testRow(final int i) throws IOException {
826 byte[] row = getRandomRow(this.rand, opts.totalRows);
827 Put put = new Put(row);
828 byte[] value = generateData(this.rand, VALUE_LENGTH);
829 if (opts.useTags) {
830 byte[] tag = generateData(this.rand, TAG_LENGTH);
831 Tag[] tags = new Tag[opts.noOfTags];
832 for (int n = 0; n < opts.noOfTags; n++) {
833 Tag t = new Tag((byte) n, tag);
834 tags[n] = t;
835 }
836 KeyValue kv = new KeyValue(row, FAMILY_NAME, QUALIFIER_NAME, HConstants.LATEST_TIMESTAMP,
837 value, tags);
838 put.add(kv);
839 } else {
840 put.add(FAMILY_NAME, QUALIFIER_NAME, value);
841 }
842 put.setDurability(opts.writeToWAL ? Durability.SYNC_WAL : Durability.SKIP_WAL);
843 table.put(put);
844 }
845 }
846
847
848 static class ScanTest extends Test {
849 private ResultScanner testScanner;
850
851 ScanTest(Configuration conf, TestOptions options, Status status) {
852 super(conf, options, status);
853 }
854
855 @Override
856 void testTakedown() throws IOException {
857 if (this.testScanner != null) {
858 this.testScanner.close();
859 }
860 super.testTakedown();
861 }
862
863
864 @Override
865 void testRow(final int i) throws IOException {
866 if (this.testScanner == null) {
867 Scan scan = new Scan(format(opts.startRow));
868 scan.setCaching(30);
869 scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
870 if (opts.filterAll) {
871 scan.setFilter(new FilterAllFilter());
872 }
873 this.testScanner = table.getScanner(scan);
874 }
875 testScanner.next();
876 }
877
878 }
879
880 static class SequentialReadTest extends Test {
881 SequentialReadTest(Configuration conf, TestOptions options, Status status) {
882 super(conf, options, status);
883 }
884
885 @Override
886 void testRow(final int i) throws IOException {
887 Get get = new Get(format(i));
888 get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
889 if (opts.filterAll) {
890 get.setFilter(new FilterAllFilter());
891 }
892 table.get(get);
893 }
894 }
895
896 static class SequentialWriteTest extends Test {
897 SequentialWriteTest(Configuration conf, TestOptions options, Status status) {
898 super(conf, options, status);
899 }
900
901 @Override
902 void testRow(final int i) throws IOException {
903 byte[] row = format(i);
904 Put put = new Put(row);
905 byte[] value = generateData(this.rand, VALUE_LENGTH);
906 if (opts.useTags) {
907 byte[] tag = generateData(this.rand, TAG_LENGTH);
908 Tag[] tags = new Tag[opts.noOfTags];
909 for (int n = 0; n < opts.noOfTags; n++) {
910 Tag t = new Tag((byte) n, tag);
911 tags[n] = t;
912 }
913 KeyValue kv = new KeyValue(row, FAMILY_NAME, QUALIFIER_NAME, HConstants.LATEST_TIMESTAMP,
914 value, tags);
915 put.add(kv);
916 } else {
917 put.add(FAMILY_NAME, QUALIFIER_NAME, value);
918 }
919 put.setDurability(opts.writeToWAL ? Durability.SYNC_WAL : Durability.SKIP_WAL);
920 table.put(put);
921 }
922 }
923
924 static class FilteredScanTest extends Test {
925 protected static final Log LOG = LogFactory.getLog(FilteredScanTest.class.getName());
926
927 FilteredScanTest(Configuration conf, TestOptions options, Status status) {
928 super(conf, options, status);
929 }
930
931 @Override
932 void testRow(int i) throws IOException {
933 byte[] value = generateData(this.rand, VALUE_LENGTH);
934 Scan scan = constructScan(value);
935 ResultScanner scanner = null;
936 try {
937 scanner = this.table.getScanner(scan);
938 while (scanner.next() != null) {
939 }
940 } finally {
941 if (scanner != null) scanner.close();
942 }
943 }
944
945 protected Scan constructScan(byte[] valuePrefix) throws IOException {
946 FilterList list = new FilterList();
947 Filter filter = new SingleColumnValueFilter(
948 FAMILY_NAME, QUALIFIER_NAME, CompareFilter.CompareOp.EQUAL,
949 new BinaryComparator(valuePrefix)
950 );
951 list.addFilter(filter);
952 if(opts.filterAll) {
953 list.addFilter(new FilterAllFilter());
954 }
955 Scan scan = new Scan();
956 scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
957 scan.setFilter(list);
958 return scan;
959 }
960 }
961
962
963
964
965
966
967
968 private static String calculateMbps(int rows, long timeMs) {
969
970
971 BigDecimal rowSize =
972 BigDecimal.valueOf(ROW_LENGTH + VALUE_LENGTH + FAMILY_NAME.length + QUALIFIER_NAME.length);
973 BigDecimal mbps = BigDecimal.valueOf(rows).multiply(rowSize, CXT)
974 .divide(BigDecimal.valueOf(timeMs), CXT).multiply(MS_PER_SEC, CXT)
975 .divide(BYTES_PER_MB, CXT);
976 return FMT.format(mbps) + " MB/s";
977 }
978
979
980
981
982
983
984
985 public static byte [] format(final int number) {
986 byte [] b = new byte[ROW_LENGTH];
987 int d = Math.abs(number);
988 for (int i = b.length - 1; i >= 0; i--) {
989 b[i] = (byte)((d % 10) + '0');
990 d /= 10;
991 }
992 return b;
993 }
994
995
996
997
998
999
1000
1001 public static byte[] generateData(final Random r, int length) {
1002 byte [] b = new byte [length];
1003 int i = 0;
1004
1005 for(i = 0; i < (length-8); i += 8) {
1006 b[i] = (byte) (65 + r.nextInt(26));
1007 b[i+1] = b[i];
1008 b[i+2] = b[i];
1009 b[i+3] = b[i];
1010 b[i+4] = b[i];
1011 b[i+5] = b[i];
1012 b[i+6] = b[i];
1013 b[i+7] = b[i];
1014 }
1015
1016 byte a = (byte) (65 + r.nextInt(26));
1017 for(; i < length; i++) {
1018 b[i] = a;
1019 }
1020 return b;
1021 }
1022
1023 static byte [] getRandomRow(final Random random, final int totalRows) {
1024 return format(random.nextInt(Integer.MAX_VALUE) % totalRows);
1025 }
1026
1027 static long runOneClient(final Class<? extends Test> cmd, Configuration conf, TestOptions opts,
1028 final Status status)
1029 throws IOException {
1030 status.setStatus("Start " + cmd + " at offset " + opts.startRow + " for " +
1031 opts.perClientRunRows + " rows");
1032 long totalElapsedTime = 0;
1033
1034 final Test t;
1035 try {
1036 Constructor<? extends Test> constructor =
1037 cmd.getDeclaredConstructor(Configuration.class, TestOptions.class, Status.class);
1038 t = constructor.newInstance(conf, opts, status);
1039 } catch (NoSuchMethodException e) {
1040 throw new IllegalArgumentException("Invalid command class: " +
1041 cmd.getName() + ". It does not provide a constructor as described by " +
1042 "the javadoc comment. Available constructors are: " +
1043 Arrays.toString(cmd.getConstructors()));
1044 } catch (Exception e) {
1045 throw new IllegalStateException("Failed to construct command class", e);
1046 }
1047 totalElapsedTime = t.test();
1048
1049 status.setStatus("Finished " + cmd + " in " + totalElapsedTime +
1050 "ms at offset " + opts.startRow + " for " + opts.perClientRunRows + " rows" +
1051 " (" + calculateMbps((int)(opts.perClientRunRows * opts.sampleRate), totalElapsedTime) + ")");
1052 return totalElapsedTime;
1053 }
1054
1055 private void runTest(final Class<? extends Test> cmd, TestOptions opts) throws IOException,
1056 InterruptedException, ClassNotFoundException {
1057 HBaseAdmin admin = null;
1058 try {
1059 admin = new HBaseAdmin(getConf());
1060 checkTable(admin, opts);
1061 } finally {
1062 if (admin != null) admin.close();
1063 }
1064 if (opts.nomapred) {
1065 doLocalClients(cmd, opts);
1066 } else {
1067 doMapReduce(cmd, opts);
1068 }
1069 }
1070
1071 protected void printUsage() {
1072 printUsage(null);
1073 }
1074
1075 protected void printUsage(final String message) {
1076 if (message != null && message.length() > 0) {
1077 System.err.println(message);
1078 }
1079 System.err.println("Usage: java " + this.getClass().getName() + " \\");
1080 System.err.println(" [--nomapred] [--rows=ROWS] [--table=NAME] \\");
1081 System.err.println(" [--compress=TYPE] [--blockEncoding=TYPE] " +
1082 "[-D<property=value>]* <command> <nclients>");
1083 System.err.println();
1084 System.err.println("Options:");
1085 System.err.println(" nomapred Run multiple clients using threads " +
1086 "(rather than use mapreduce)");
1087 System.err.println(" rows Rows each client runs. Default: One million");
1088 System.err.println(" sampleRate Execute test on a sample of total " +
1089 "rows. Only supported by randomRead. Default: 1.0");
1090 System.err.println(" table Alternate table name. Default: 'TestTable'");
1091 System.err.println(" compress Compression type to use (GZ, LZO, ...). Default: 'NONE'");
1092 System.err.println(" flushCommits Used to determine if the test should flush the table. " +
1093 "Default: false");
1094 System.err.println(" writeToWAL Set writeToWAL on puts. Default: True");
1095 System.err.println(" presplit Create presplit table. Recommended for accurate perf " +
1096 "analysis (see guide). Default: disabled");
1097 System.err.println(" inmemory Tries to keep the HFiles of the CF " +
1098 "inmemory as far as possible. Not guaranteed that reads are always served " +
1099 "from memory. Default: false");
1100 System.err.println(" usetags Writes tags along with KVs. Use with HFile V3. " +
1101 "Default: false");
1102 System.err.println(" numoftags Specify the no of tags that would be needed. " +
1103 "This works only if usetags is true.");
1104 System.err.println(" filterAll Helps to filter out all the rows on the server side"
1105 + " there by not returning any thing back to the client. Helps to check the server side"
1106 + " performance. Uses FilterAllFilter internally. ");
1107 System.err.println(" latency Set to report operation latencies. " +
1108 "Currently only supported by randomRead test. Default: False");
1109 System.err.println();
1110 System.err.println(" Note: -D properties will be applied to the conf used. ");
1111 System.err.println(" For example: ");
1112 System.err.println(" -Dmapred.output.compress=true");
1113 System.err.println(" -Dmapreduce.task.timeout=60000");
1114 System.err.println();
1115 System.err.println("Command:");
1116 for (CmdDescriptor command : commands.values()) {
1117 System.err.println(String.format(" %-15s %s", command.getName(), command.getDescription()));
1118 }
1119 System.err.println();
1120 System.err.println("Args:");
1121 System.err.println(" nclients Integer. Required. Total number of " +
1122 "clients (and HRegionServers)");
1123 System.err.println(" running: 1 <= value <= 500");
1124 System.err.println("Examples:");
1125 System.err.println(" To run a single evaluation client:");
1126 System.err.println(" $ bin/hbase " + this.getClass().getName()
1127 + " sequentialWrite 1");
1128 }
1129
1130 private static int getNumClients(final int start, final String[] args) {
1131 if(start + 1 > args.length) {
1132 throw new IllegalArgumentException("must supply the number of clients");
1133 }
1134 int N = Integer.parseInt(args[start]);
1135 if (N < 1) {
1136 throw new IllegalArgumentException("Number of clients must be > 1");
1137 }
1138 return N;
1139 }
1140
1141 public int run(String[] args) throws Exception {
1142
1143
1144 int errCode = -1;
1145 if (args.length < 1) {
1146 printUsage();
1147 return errCode;
1148 }
1149
1150 try {
1151
1152
1153
1154
1155
1156
1157 TestOptions opts = new TestOptions();
1158
1159 for (int i = 0; i < args.length; i++) {
1160 String cmd = args[i];
1161 if (cmd.equals("-h") || cmd.startsWith("--h")) {
1162 printUsage();
1163 errCode = 0;
1164 break;
1165 }
1166
1167 final String nmr = "--nomapred";
1168 if (cmd.startsWith(nmr)) {
1169 opts.nomapred = true;
1170 continue;
1171 }
1172
1173 final String rows = "--rows=";
1174 if (cmd.startsWith(rows)) {
1175 opts.perClientRunRows = Integer.parseInt(cmd.substring(rows.length()));
1176 continue;
1177 }
1178
1179 final String sampleRate = "--sampleRate=";
1180 if (cmd.startsWith(sampleRate)) {
1181 opts.sampleRate = Float.parseFloat(cmd.substring(sampleRate.length()));
1182 continue;
1183 }
1184
1185 final String table = "--table=";
1186 if (cmd.startsWith(table)) {
1187 opts.tableName = cmd.substring(table.length());
1188 continue;
1189 }
1190
1191 final String compress = "--compress=";
1192 if (cmd.startsWith(compress)) {
1193 opts.compression = Compression.Algorithm.valueOf(cmd.substring(compress.length()));
1194 continue;
1195 }
1196
1197 final String blockEncoding = "--blockEncoding=";
1198 if (cmd.startsWith(blockEncoding)) {
1199 opts.blockEncoding = DataBlockEncoding.valueOf(cmd.substring(blockEncoding.length()));
1200 continue;
1201 }
1202
1203 final String flushCommits = "--flushCommits=";
1204 if (cmd.startsWith(flushCommits)) {
1205 opts.flushCommits = Boolean.parseBoolean(cmd.substring(flushCommits.length()));
1206 continue;
1207 }
1208
1209 final String writeToWAL = "--writeToWAL=";
1210 if (cmd.startsWith(writeToWAL)) {
1211 opts.writeToWAL = Boolean.parseBoolean(cmd.substring(writeToWAL.length()));
1212 continue;
1213 }
1214
1215 final String presplit = "--presplit=";
1216 if (cmd.startsWith(presplit)) {
1217 opts.presplitRegions = Integer.parseInt(cmd.substring(presplit.length()));
1218 continue;
1219 }
1220
1221 final String inMemory = "--inmemory=";
1222 if (cmd.startsWith(inMemory)) {
1223 opts.inMemoryCF = Boolean.parseBoolean(cmd.substring(inMemory.length()));
1224 continue;
1225 }
1226
1227 final String latency = "--latency";
1228 if (cmd.startsWith(latency)) {
1229 opts.reportLatency = true;
1230 continue;
1231 }
1232
1233 final String multiGet = "--multiGet=";
1234 if (cmd.startsWith(multiGet)) {
1235 opts.multiGet = Integer.parseInt(cmd.substring(multiGet.length()));
1236 continue;
1237 }
1238
1239 final String useTags = "--usetags=";
1240 if (cmd.startsWith(useTags)) {
1241 opts.useTags = Boolean.parseBoolean(cmd.substring(useTags.length()));
1242 continue;
1243 }
1244
1245 final String noOfTags = "--nooftags=";
1246 if (cmd.startsWith(noOfTags)) {
1247 opts.noOfTags = Integer.parseInt(cmd.substring(noOfTags.length()));
1248 continue;
1249 }
1250
1251 final String filterOutAll = "--filterAll";
1252 if (cmd.startsWith(filterOutAll)) {
1253 opts.filterAll = true;
1254 continue;
1255 }
1256
1257 Class<? extends Test> cmdClass = determineCommandClass(cmd);
1258 if (cmdClass != null) {
1259 opts.numClientThreads = getNumClients(i + 1, args);
1260
1261 opts.totalRows = opts.perClientRunRows * opts.numClientThreads;
1262 runTest(cmdClass, opts);
1263 errCode = 0;
1264 break;
1265 }
1266
1267 printUsage();
1268 break;
1269 }
1270 } catch (Exception e) {
1271 e.printStackTrace();
1272 }
1273
1274 return errCode;
1275 }
1276
1277 private Class<? extends Test> determineCommandClass(String cmd) {
1278 CmdDescriptor descriptor = commands.get(cmd);
1279 return descriptor != null ? descriptor.getCmdClass() : null;
1280 }
1281
1282 public static void main(final String[] args) throws Exception {
1283 int res = ToolRunner.run(new PerformanceEvaluation(HBaseConfiguration.create()), args);
1284 System.exit(res);
1285 }
1286 }