View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import static org.codehaus.jackson.map.SerializationConfig.Feature.SORT_PROPERTIES_ALPHABETICALLY;
22  
23  import java.io.IOException;
24  import java.io.PrintStream;
25  import java.lang.reflect.Constructor;
26  import java.math.BigDecimal;
27  import java.math.MathContext;
28  import java.text.DecimalFormat;
29  import java.text.SimpleDateFormat;
30  import java.util.ArrayList;
31  import java.util.Arrays;
32  import java.util.Date;
33  import java.util.LinkedList;
34  import java.util.Map;
35  import java.util.Queue;
36  import java.util.Random;
37  import java.util.TreeMap;
38  import java.util.concurrent.Callable;
39  import java.util.concurrent.ExecutionException;
40  import java.util.concurrent.ExecutorService;
41  import java.util.concurrent.Executors;
42  import java.util.concurrent.Future;
43  
44  import com.google.common.base.Objects;
45  import com.google.common.util.concurrent.ThreadFactoryBuilder;
46  
47  import org.apache.commons.logging.Log;
48  import org.apache.commons.logging.LogFactory;
49  import org.apache.hadoop.conf.Configuration;
50  import org.apache.hadoop.conf.Configured;
51  import org.apache.hadoop.fs.FileSystem;
52  import org.apache.hadoop.fs.Path;
53  import org.apache.hadoop.hbase.classification.InterfaceAudience;
54  import org.apache.hadoop.hbase.client.Admin;
55  import org.apache.hadoop.hbase.client.BufferedMutator;
56  import org.apache.hadoop.hbase.client.Connection;
57  import org.apache.hadoop.hbase.client.ConnectionFactory;
58  import org.apache.hadoop.hbase.client.Consistency;
59  import org.apache.hadoop.hbase.client.Durability;
60  import org.apache.hadoop.hbase.client.Get;
61  import org.apache.hadoop.hbase.client.Put;
62  import org.apache.hadoop.hbase.client.Result;
63  import org.apache.hadoop.hbase.client.ResultScanner;
64  import org.apache.hadoop.hbase.client.Scan;
65  import org.apache.hadoop.hbase.client.Table;
66  import org.apache.hadoop.hbase.filter.BinaryComparator;
67  import org.apache.hadoop.hbase.filter.CompareFilter;
68  import org.apache.hadoop.hbase.filter.Filter;
69  import org.apache.hadoop.hbase.filter.FilterAllFilter;
70  import org.apache.hadoop.hbase.filter.FilterList;
71  import org.apache.hadoop.hbase.filter.PageFilter;
72  import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
73  import org.apache.hadoop.hbase.filter.WhileMatchFilter;
74  import org.apache.hadoop.hbase.io.compress.Compression;
75  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
76  import org.apache.hadoop.hbase.io.hfile.RandomDistribution;
77  import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
78  import org.apache.hadoop.hbase.regionserver.BloomType;
79  import org.apache.hadoop.hbase.trace.HBaseHTraceConfiguration;
80  import org.apache.hadoop.hbase.trace.SpanReceiverHost;
81  import org.apache.hadoop.hbase.util.*;
82  import org.apache.hadoop.io.LongWritable;
83  import org.apache.hadoop.io.Text;
84  import org.apache.hadoop.mapreduce.Job;
85  import org.apache.hadoop.mapreduce.Mapper;
86  import org.apache.hadoop.mapreduce.lib.input.NLineInputFormat;
87  import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
88  import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer;
89  import org.apache.hadoop.util.Tool;
90  import org.apache.hadoop.util.ToolRunner;
91  import org.codehaus.jackson.map.ObjectMapper;
92  
93  import com.yammer.metrics.core.Histogram;
94  import com.yammer.metrics.stats.UniformSample;
95  import com.yammer.metrics.stats.Snapshot;
96  
97  import org.apache.htrace.Sampler;
98  import org.apache.htrace.Trace;
99  import org.apache.htrace.TraceScope;
100 import org.apache.htrace.impl.ProbabilitySampler;
101 
102 /**
103  * Script used evaluating HBase performance and scalability.  Runs a HBase
104  * client that steps through one of a set of hardcoded tests or 'experiments'
105  * (e.g. a random reads test, a random writes test, etc.). Pass on the
106  * command-line which test to run and how many clients are participating in
107  * this experiment. Run {@code PerformanceEvaluation --help} to obtain usage.
108  *
109  * <p>This class sets up and runs the evaluation programs described in
110  * Section 7, <i>Performance Evaluation</i>, of the <a
111  * href="http://labs.google.com/papers/bigtable.html">Bigtable</a>
112  * paper, pages 8-10.
113  *
114  * <p>By default, runs as a mapreduce job where each mapper runs a single test
115  * client. Can also run as a non-mapreduce, multithreaded application by
116  * specifying {@code --nomapred}. Each client does about 1GB of data, unless
117  * specified otherwise.
118  */
119 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
120 public class PerformanceEvaluation extends Configured implements Tool {
121   protected static final Log LOG = LogFactory.getLog(PerformanceEvaluation.class.getName());
122   private static final ObjectMapper MAPPER = new ObjectMapper();
123   static {
124     MAPPER.configure(SORT_PROPERTIES_ALPHABETICALLY, true);
125   }
126 
127   public static final String TABLE_NAME = "TestTable";
128   public static final byte[] FAMILY_NAME = Bytes.toBytes("info");
129   public static final byte[] QUALIFIER_NAME = Bytes.toBytes("data");
130   public static final int DEFAULT_VALUE_LENGTH = 1000;
131   public static final int ROW_LENGTH = 26;
132 
133   private static final int ONE_GB = 1024 * 1024 * 1000;
134   private static final int DEFAULT_ROWS_PER_GB = ONE_GB / DEFAULT_VALUE_LENGTH;
135   // TODO : should we make this configurable
136   private static final int TAG_LENGTH = 256;
137   private static final DecimalFormat FMT = new DecimalFormat("0.##");
138   private static final MathContext CXT = MathContext.DECIMAL64;
139   private static final BigDecimal MS_PER_SEC = BigDecimal.valueOf(1000);
140   private static final BigDecimal BYTES_PER_MB = BigDecimal.valueOf(1024 * 1024);
141   private static final TestOptions DEFAULT_OPTS = new TestOptions();
142 
143   private static Map<String, CmdDescriptor> COMMANDS = new TreeMap<String, CmdDescriptor>();
144   private static final Path PERF_EVAL_DIR = new Path("performance_evaluation");
145 
146   static {
147     addCommandDescriptor(RandomReadTest.class, "randomRead",
148       "Run random read test");
149     addCommandDescriptor(RandomSeekScanTest.class, "randomSeekScan",
150       "Run random seek and scan 100 test");
151     addCommandDescriptor(RandomScanWithRange10Test.class, "scanRange10",
152       "Run random seek scan with both start and stop row (max 10 rows)");
153     addCommandDescriptor(RandomScanWithRange100Test.class, "scanRange100",
154       "Run random seek scan with both start and stop row (max 100 rows)");
155     addCommandDescriptor(RandomScanWithRange1000Test.class, "scanRange1000",
156       "Run random seek scan with both start and stop row (max 1000 rows)");
157     addCommandDescriptor(RandomScanWithRange10000Test.class, "scanRange10000",
158       "Run random seek scan with both start and stop row (max 10000 rows)");
159     addCommandDescriptor(RandomWriteTest.class, "randomWrite",
160       "Run random write test");
161     addCommandDescriptor(SequentialReadTest.class, "sequentialRead",
162       "Run sequential read test");
163     addCommandDescriptor(SequentialWriteTest.class, "sequentialWrite",
164       "Run sequential write test");
165     addCommandDescriptor(ScanTest.class, "scan",
166       "Run scan test (read every row)");
167     addCommandDescriptor(FilteredScanTest.class, "filterScan",
168       "Run scan test using a filter to find a specific row based on it's value " +
169         "(make sure to use --rows=20)");
170   }
171 
172   /**
173    * Enum for map metrics.  Keep it out here rather than inside in the Map
174    * inner-class so we can find associated properties.
175    */
176   protected static enum Counter {
177     /** elapsed time */
178     ELAPSED_TIME,
179     /** number of rows */
180     ROWS
181   }
182 
183   protected static class RunResult implements Comparable<RunResult> {
184     public RunResult(long duration, Histogram hist) {
185       this.duration = duration;
186       this.hist = hist;
187     }
188 
189     public final long duration;
190     public final Histogram hist;
191 
192     @Override
193     public String toString() {
194       return Long.toString(duration);
195     }
196 
197     @Override public int compareTo(RunResult o) {
198       return Long.compare(this.duration, o.duration);
199     }
200   }
201 
202   /**
203    * Constructor
204    * @param conf Configuration object
205    */
206   public PerformanceEvaluation(final Configuration conf) {
207     super(conf);
208   }
209 
210   protected static void addCommandDescriptor(Class<? extends Test> cmdClass,
211       String name, String description) {
212     CmdDescriptor cmdDescriptor = new CmdDescriptor(cmdClass, name, description);
213     COMMANDS.put(name, cmdDescriptor);
214   }
215 
216   /**
217    * Implementations can have their status set.
218    */
219   interface Status {
220     /**
221      * Sets status
222      * @param msg status message
223      * @throws IOException
224      */
225     void setStatus(final String msg) throws IOException;
226   }
227 
228   /**
229    * MapReduce job that runs a performance evaluation client in each map task.
230    */
231   public static class EvaluationMapTask
232       extends Mapper<LongWritable, Text, LongWritable, LongWritable> {
233 
234     /** configuration parameter name that contains the command */
235     public final static String CMD_KEY = "EvaluationMapTask.command";
236     /** configuration parameter name that contains the PE impl */
237     public static final String PE_KEY = "EvaluationMapTask.performanceEvalImpl";
238 
239     private Class<? extends Test> cmd;
240 
241     @Override
242     protected void setup(Context context) throws IOException, InterruptedException {
243       this.cmd = forName(context.getConfiguration().get(CMD_KEY), Test.class);
244 
245       // this is required so that extensions of PE are instantiated within the
246       // map reduce task...
247       Class<? extends PerformanceEvaluation> peClass =
248           forName(context.getConfiguration().get(PE_KEY), PerformanceEvaluation.class);
249       try {
250         peClass.getConstructor(Configuration.class).newInstance(context.getConfiguration());
251       } catch (Exception e) {
252         throw new IllegalStateException("Could not instantiate PE instance", e);
253       }
254     }
255 
256     private <Type> Class<? extends Type> forName(String className, Class<Type> type) {
257       try {
258         return Class.forName(className).asSubclass(type);
259       } catch (ClassNotFoundException e) {
260         throw new IllegalStateException("Could not find class for name: " + className, e);
261       }
262     }
263 
264     @Override
265     protected void map(LongWritable key, Text value, final Context context)
266            throws IOException, InterruptedException {
267 
268       Status status = new Status() {
269         @Override
270         public void setStatus(String msg) {
271            context.setStatus(msg);
272         }
273       };
274 
275       ObjectMapper mapper = new ObjectMapper();
276       TestOptions opts = mapper.readValue(value.toString(), TestOptions.class);
277       Configuration conf = HBaseConfiguration.create(context.getConfiguration());
278       final Connection con = ConnectionFactory.createConnection(conf);
279 
280       // Evaluation task
281       RunResult result = PerformanceEvaluation.runOneClient(this.cmd, conf, con, opts, status);
282       // Collect how much time the thing took. Report as map output and
283       // to the ELAPSED_TIME counter.
284       context.getCounter(Counter.ELAPSED_TIME).increment(result.duration);
285       context.getCounter(Counter.ROWS).increment(opts.perClientRunRows);
286       context.write(new LongWritable(opts.startRow), new LongWritable(result.duration));
287       context.progress();
288     }
289   }
290 
291   /*
292    * If table does not already exist, create. Also create a table when
293    * {@code opts.presplitRegions} is specified or when the existing table's
294    * region replica count doesn't match {@code opts.replicas}.
295    */
296   static boolean checkTable(Admin admin, TestOptions opts) throws IOException {
297     TableName tableName = TableName.valueOf(opts.tableName);
298     boolean needsDelete = false, exists = admin.tableExists(tableName);
299     boolean isReadCmd = opts.cmdName.toLowerCase().contains("read")
300       || opts.cmdName.toLowerCase().contains("scan");
301     if (!exists && isReadCmd) {
302       throw new IllegalStateException(
303         "Must specify an existing table for read commands. Run a write command first.");
304     }
305     HTableDescriptor desc =
306       exists ? admin.getTableDescriptor(TableName.valueOf(opts.tableName)) : null;
307     byte[][] splits = getSplits(opts);
308 
309     // recreate the table when user has requested presplit or when existing
310     // {RegionSplitPolicy,replica count} does not match requested.
311     if ((exists && opts.presplitRegions != DEFAULT_OPTS.presplitRegions)
312       || (!isReadCmd && desc != null && desc.getRegionSplitPolicyClassName() != opts.splitPolicy)
313       || (!isReadCmd && desc != null && desc.getRegionReplication() != opts.replicas)) {
314       needsDelete = true;
315       // wait, why did it delete my table?!?
316       LOG.debug(Objects.toStringHelper("needsDelete")
317         .add("needsDelete", needsDelete)
318         .add("isReadCmd", isReadCmd)
319         .add("exists", exists)
320         .add("desc", desc)
321         .add("presplit", opts.presplitRegions)
322         .add("splitPolicy", opts.splitPolicy)
323         .add("replicas", opts.replicas));
324     }
325 
326     // remove an existing table
327     if (needsDelete) {
328       if (admin.isTableEnabled(tableName)) {
329         admin.disableTable(tableName);
330       }
331       admin.deleteTable(tableName);
332     }
333 
334     // table creation is necessary
335     if (!exists || needsDelete) {
336       desc = getTableDescriptor(opts);
337       if (splits != null) {
338         if (LOG.isDebugEnabled()) {
339           for (int i = 0; i < splits.length; i++) {
340             LOG.debug(" split " + i + ": " + Bytes.toStringBinary(splits[i]));
341           }
342         }
343       }
344       admin.createTable(desc, splits);
345       LOG.info("Table " + desc + " created");
346     }
347     return admin.tableExists(tableName);
348   }
349 
350   /**
351    * Create an HTableDescriptor from provided TestOptions.
352    */
353   protected static HTableDescriptor getTableDescriptor(TestOptions opts) {
354     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(opts.tableName));
355     HColumnDescriptor family = new HColumnDescriptor(FAMILY_NAME);
356     family.setDataBlockEncoding(opts.blockEncoding);
357     family.setCompressionType(opts.compression);
358     family.setBloomFilterType(opts.bloomType);
359     if (opts.inMemoryCF) {
360       family.setInMemory(true);
361     }
362     desc.addFamily(family);
363     if (opts.replicas != DEFAULT_OPTS.replicas) {
364       desc.setRegionReplication(opts.replicas);
365     }
366     if (opts.splitPolicy != DEFAULT_OPTS.splitPolicy) {
367       desc.setRegionSplitPolicyClassName(opts.splitPolicy);
368     }
369     return desc;
370   }
371 
372   /**
373    * generates splits based on total number of rows and specified split regions
374    */
375   protected static byte[][] getSplits(TestOptions opts) {
376     if (opts.presplitRegions == DEFAULT_OPTS.presplitRegions)
377       return null;
378 
379     int numSplitPoints = opts.presplitRegions - 1;
380     byte[][] splits = new byte[numSplitPoints][];
381     int jump = opts.totalRows / opts.presplitRegions;
382     for (int i = 0; i < numSplitPoints; i++) {
383       int rowkey = jump * (1 + i);
384       splits[i] = format(rowkey);
385     }
386     return splits;
387   }
388 
389   /*
390    * Run all clients in this vm each to its own thread.
391    */
392   static RunResult[] doLocalClients(final TestOptions opts, final Configuration conf)
393       throws IOException, InterruptedException {
394     final Class<? extends Test> cmd = determineCommandClass(opts.cmdName);
395     assert cmd != null;
396     @SuppressWarnings("unchecked")
397     Future<RunResult>[] threads = new Future[opts.numClientThreads];
398     RunResult[] results = new RunResult[opts.numClientThreads];
399     ExecutorService pool = Executors.newFixedThreadPool(opts.numClientThreads,
400       new ThreadFactoryBuilder().setNameFormat("TestClient-%s").build());
401     final Connection con = ConnectionFactory.createConnection(conf);
402     for (int i = 0; i < threads.length; i++) {
403       final int index = i;
404       threads[i] = pool.submit(new Callable<RunResult>() {
405         @Override
406         public RunResult call() throws Exception {
407           TestOptions threadOpts = new TestOptions(opts);
408           if (threadOpts.startRow == 0) threadOpts.startRow = index * threadOpts.perClientRunRows;
409           RunResult run = runOneClient(cmd, conf, con, threadOpts, new Status() {
410             @Override
411             public void setStatus(final String msg) throws IOException {
412               LOG.info(msg);
413             }
414           });
415           LOG.info("Finished " + Thread.currentThread().getName() + " in " + run.duration +
416             "ms over " + threadOpts.perClientRunRows + " rows");
417           return run;
418         }
419       });
420     }
421     pool.shutdown();
422 
423     for (int i = 0; i < threads.length; i++) {
424       try {
425         results[i] = threads[i].get();
426       } catch (ExecutionException e) {
427         throw new IOException(e.getCause());
428       }
429     }
430     final String test = cmd.getSimpleName();
431     LOG.info("[" + test + "] Summary of timings (ms): "
432              + Arrays.toString(results));
433     Arrays.sort(results);
434     long total = 0;
435     for (RunResult result : results) {
436       total += result.duration;
437     }
438     LOG.info("[" + test + "]"
439       + "\tMin: " + results[0] + "ms"
440       + "\tMax: " + results[results.length - 1] + "ms"
441       + "\tAvg: " + (total / results.length) + "ms");
442 
443     con.close();
444 
445     return results;
446   }
447 
448   /*
449    * Run a mapreduce job.  Run as many maps as asked-for clients.
450    * Before we start up the job, write out an input file with instruction
451    * per client regards which row they are to start on.
452    * @param cmd Command to run.
453    * @throws IOException
454    */
455   static Job doMapReduce(TestOptions opts, final Configuration conf)
456       throws IOException, InterruptedException, ClassNotFoundException {
457     final Class<? extends Test> cmd = determineCommandClass(opts.cmdName);
458     assert cmd != null;
459     Path inputDir = writeInputFile(conf, opts);
460     conf.set(EvaluationMapTask.CMD_KEY, cmd.getName());
461     conf.set(EvaluationMapTask.PE_KEY, PerformanceEvaluation.class.getName());
462     Job job = Job.getInstance(conf);
463     job.setJarByClass(PerformanceEvaluation.class);
464     job.setJobName("HBase Performance Evaluation - " + opts.cmdName);
465 
466     job.setInputFormatClass(NLineInputFormat.class);
467     NLineInputFormat.setInputPaths(job, inputDir);
468     // this is default, but be explicit about it just in case.
469     NLineInputFormat.setNumLinesPerSplit(job, 1);
470 
471     job.setOutputKeyClass(LongWritable.class);
472     job.setOutputValueClass(LongWritable.class);
473 
474     job.setMapperClass(EvaluationMapTask.class);
475     job.setReducerClass(LongSumReducer.class);
476 
477     job.setNumReduceTasks(1);
478 
479     job.setOutputFormatClass(TextOutputFormat.class);
480     TextOutputFormat.setOutputPath(job, new Path(inputDir.getParent(), "outputs"));
481 
482     TableMapReduceUtil.addDependencyJars(job);
483     TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
484       Histogram.class,     // yammer metrics
485       ObjectMapper.class); // jackson-mapper-asl
486 
487     TableMapReduceUtil.initCredentials(job);
488 
489     job.waitForCompletion(true);
490     return job;
491   }
492 
493   /*
494    * Write input file of offsets-per-client for the mapreduce job.
495    * @param c Configuration
496    * @return Directory that contains file written.
497    * @throws IOException
498    */
499   private static Path writeInputFile(final Configuration c, final TestOptions opts) throws IOException {
500     SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
501     Path jobdir = new Path(PERF_EVAL_DIR, formatter.format(new Date()));
502     Path inputDir = new Path(jobdir, "inputs");
503 
504     FileSystem fs = FileSystem.get(c);
505     fs.mkdirs(inputDir);
506 
507     Path inputFile = new Path(inputDir, "input.txt");
508     PrintStream out = new PrintStream(fs.create(inputFile));
509     // Make input random.
510     Map<Integer, String> m = new TreeMap<Integer, String>();
511     Hash h = MurmurHash.getInstance();
512     int perClientRows = (opts.totalRows / opts.numClientThreads);
513     try {
514       for (int i = 0; i < 10; i++) {
515         for (int j = 0; j < opts.numClientThreads; j++) {
516           TestOptions next = new TestOptions(opts);
517           next.startRow = (j * perClientRows) + (i * (perClientRows/10));
518           next.perClientRunRows = perClientRows / 10;
519           String s = MAPPER.writeValueAsString(next);
520           LOG.info("maptask input=" + s);
521           int hash = h.hash(Bytes.toBytes(s));
522           m.put(hash, s);
523         }
524       }
525       for (Map.Entry<Integer, String> e: m.entrySet()) {
526         out.println(e.getValue());
527       }
528     } finally {
529       out.close();
530     }
531     return inputDir;
532   }
533 
534   /**
535    * Describes a command.
536    */
537   static class CmdDescriptor {
538     private Class<? extends Test> cmdClass;
539     private String name;
540     private String description;
541 
542     CmdDescriptor(Class<? extends Test> cmdClass, String name, String description) {
543       this.cmdClass = cmdClass;
544       this.name = name;
545       this.description = description;
546     }
547 
548     public Class<? extends Test> getCmdClass() {
549       return cmdClass;
550     }
551 
552     public String getName() {
553       return name;
554     }
555 
556     public String getDescription() {
557       return description;
558     }
559   }
560 
561   /**
562    * Wraps up options passed to {@link org.apache.hadoop.hbase.PerformanceEvaluation}.
563    * This makes tracking all these arguments a little easier.
564    * NOTE: ADDING AN OPTION, you need to add a data member, a getter/setter (to make JSON
565    * serialization of this TestOptions class behave), and you need to add to the clone constructor
566    * below copying your new option from the 'that' to the 'this'.  Look for 'clone' below.
567    */
568   static class TestOptions {
569     String cmdName = null;
570     boolean nomapred = false;
571     boolean filterAll = false;
572     int startRow = 0;
573     float size = 1.0f;
574     int perClientRunRows = DEFAULT_ROWS_PER_GB;
575     int numClientThreads = 1;
576     int totalRows = DEFAULT_ROWS_PER_GB;
577     float sampleRate = 1.0f;
578     double traceRate = 0.0;
579     String tableName = TABLE_NAME;
580     boolean flushCommits = true;
581     boolean writeToWAL = true;
582     boolean autoFlush = false;
583     boolean oneCon = false;
584     boolean useTags = false;
585     int noOfTags = 1;
586     boolean reportLatency = false;
587     int multiGet = 0;
588     int randomSleep = 0;
589     boolean inMemoryCF = false;
590     int presplitRegions = 0;
591     int replicas = HTableDescriptor.DEFAULT_REGION_REPLICATION;
592     String splitPolicy = null;
593     Compression.Algorithm compression = Compression.Algorithm.NONE;
594     BloomType bloomType = BloomType.ROW;
595     DataBlockEncoding blockEncoding = DataBlockEncoding.NONE;
596     boolean valueRandom = false;
597     boolean valueZipf = false;
598     int valueSize = DEFAULT_VALUE_LENGTH;
599     int period = (this.perClientRunRows / 10) == 0? perClientRunRows: perClientRunRows / 10;
600 
601     public TestOptions() {}
602 
603     /**
604      * Clone constructor.
605      * @param that Object to copy from.
606      */
607     public TestOptions(TestOptions that) {
608       this.cmdName = that.cmdName;
609       this.nomapred = that.nomapred;
610       this.startRow = that.startRow;
611       this.size = that.size;
612       this.perClientRunRows = that.perClientRunRows;
613       this.numClientThreads = that.numClientThreads;
614       this.totalRows = that.totalRows;
615       this.sampleRate = that.sampleRate;
616       this.traceRate = that.traceRate;
617       this.tableName = that.tableName;
618       this.flushCommits = that.flushCommits;
619       this.writeToWAL = that.writeToWAL;
620       this.autoFlush = that.autoFlush;
621       this.oneCon = that.oneCon;
622       this.useTags = that.useTags;
623       this.noOfTags = that.noOfTags;
624       this.reportLatency = that.reportLatency;
625       this.multiGet = that.multiGet;
626       this.inMemoryCF = that.inMemoryCF;
627       this.presplitRegions = that.presplitRegions;
628       this.replicas = that.replicas;
629       this.splitPolicy = that.splitPolicy;
630       this.compression = that.compression;
631       this.blockEncoding = that.blockEncoding;
632       this.filterAll = that.filterAll;
633       this.bloomType = that.bloomType;
634       this.valueRandom = that.valueRandom;
635       this.valueZipf = that.valueZipf;
636       this.valueSize = that.valueSize;
637       this.period = that.period;
638       this.randomSleep = that.randomSleep;
639     }
640 
641     public boolean isValueZipf() {
642       return valueZipf;
643     }
644 
645     public void setValueZipf(boolean valueZipf) {
646       this.valueZipf = valueZipf;
647     }
648 
649     public String getCmdName() {
650       return cmdName;
651     }
652 
653     public void setCmdName(String cmdName) {
654       this.cmdName = cmdName;
655     }
656 
657     public int getRandomSleep() {
658       return randomSleep;
659     }
660 
661     public void setRandomSleep(int randomSleep) {
662       this.randomSleep = randomSleep;
663     }
664 
665     public int getReplicas() {
666       return replicas;
667     }
668 
669     public void setReplicas(int replicas) {
670       this.replicas = replicas;
671     }
672 
673     public String getSplitPolicy() {
674       return splitPolicy;
675     }
676 
677     public void setSplitPolicy(String splitPolicy) {
678       this.splitPolicy = splitPolicy;
679     }
680 
681     public void setNomapred(boolean nomapred) {
682       this.nomapred = nomapred;
683     }
684 
685     public void setFilterAll(boolean filterAll) {
686       this.filterAll = filterAll;
687     }
688 
689     public void setStartRow(int startRow) {
690       this.startRow = startRow;
691     }
692 
693     public void setSize(float size) {
694       this.size = size;
695     }
696 
697     public void setPerClientRunRows(int perClientRunRows) {
698       this.perClientRunRows = perClientRunRows;
699     }
700 
701     public void setNumClientThreads(int numClientThreads) {
702       this.numClientThreads = numClientThreads;
703     }
704 
705     public void setTotalRows(int totalRows) {
706       this.totalRows = totalRows;
707     }
708 
709     public void setSampleRate(float sampleRate) {
710       this.sampleRate = sampleRate;
711     }
712 
713     public void setTraceRate(double traceRate) {
714       this.traceRate = traceRate;
715     }
716 
717     public void setTableName(String tableName) {
718       this.tableName = tableName;
719     }
720 
721     public void setFlushCommits(boolean flushCommits) {
722       this.flushCommits = flushCommits;
723     }
724 
725     public void setWriteToWAL(boolean writeToWAL) {
726       this.writeToWAL = writeToWAL;
727     }
728 
729     public void setAutoFlush(boolean autoFlush) {
730       this.autoFlush = autoFlush;
731     }
732 
733     public void setOneCon(boolean oneCon) {
734       this.oneCon = oneCon;
735     }
736 
737     public void setUseTags(boolean useTags) {
738       this.useTags = useTags;
739     }
740 
741     public void setNoOfTags(int noOfTags) {
742       this.noOfTags = noOfTags;
743     }
744 
745     public void setReportLatency(boolean reportLatency) {
746       this.reportLatency = reportLatency;
747     }
748 
749     public void setMultiGet(int multiGet) {
750       this.multiGet = multiGet;
751     }
752 
753     public void setInMemoryCF(boolean inMemoryCF) {
754       this.inMemoryCF = inMemoryCF;
755     }
756 
757     public void setPresplitRegions(int presplitRegions) {
758       this.presplitRegions = presplitRegions;
759     }
760 
761     public void setCompression(Compression.Algorithm compression) {
762       this.compression = compression;
763     }
764 
765     public void setBloomType(BloomType bloomType) {
766       this.bloomType = bloomType;
767     }
768 
769     public void setBlockEncoding(DataBlockEncoding blockEncoding) {
770       this.blockEncoding = blockEncoding;
771     }
772 
773     public void setValueRandom(boolean valueRandom) {
774       this.valueRandom = valueRandom;
775     }
776 
777     public void setValueSize(int valueSize) {
778       this.valueSize = valueSize;
779     }
780 
781     public void setPeriod(int period) {
782       this.period = period;
783     }
784 
785     public boolean isNomapred() {
786       return nomapred;
787     }
788 
789     public boolean isFilterAll() {
790       return filterAll;
791     }
792 
793     public int getStartRow() {
794       return startRow;
795     }
796 
797     public float getSize() {
798       return size;
799     }
800 
801     public int getPerClientRunRows() {
802       return perClientRunRows;
803     }
804 
805     public int getNumClientThreads() {
806       return numClientThreads;
807     }
808 
809     public int getTotalRows() {
810       return totalRows;
811     }
812 
813     public float getSampleRate() {
814       return sampleRate;
815     }
816 
817     public double getTraceRate() {
818       return traceRate;
819     }
820 
821     public String getTableName() {
822       return tableName;
823     }
824 
825     public boolean isFlushCommits() {
826       return flushCommits;
827     }
828 
829     public boolean isWriteToWAL() {
830       return writeToWAL;
831     }
832 
833     public boolean isAutoFlush() {
834       return autoFlush;
835     }
836 
837     public boolean isUseTags() {
838       return useTags;
839     }
840 
841     public int getNoOfTags() {
842       return noOfTags;
843     }
844 
845     public boolean isReportLatency() {
846       return reportLatency;
847     }
848 
849     public int getMultiGet() {
850       return multiGet;
851     }
852 
853     public boolean isInMemoryCF() {
854       return inMemoryCF;
855     }
856 
857     public int getPresplitRegions() {
858       return presplitRegions;
859     }
860 
861     public Compression.Algorithm getCompression() {
862       return compression;
863     }
864 
865     public DataBlockEncoding getBlockEncoding() {
866       return blockEncoding;
867     }
868 
869     public boolean isValueRandom() {
870       return valueRandom;
871     }
872 
873     public int getValueSize() {
874       return valueSize;
875     }
876 
877     public int getPeriod() {
878       return period;
879     }
880 
881     public BloomType getBloomType() {
882       return bloomType;
883     }
884 
885     public boolean isOneCon() {
886       return oneCon;
887     }
888   }
889 
890   /*
891    * A test.
892    * Subclass to particularize what happens per row.
893    */
894   static abstract class Test {
895     // Below is make it so when Tests are all running in the one
896     // jvm, that they each have a differently seeded Random.
897     private static final Random randomSeed = new Random(System.currentTimeMillis());
898 
899     private static long nextRandomSeed() {
900       return randomSeed.nextLong();
901     }
902     private final int everyN;
903 
904     protected final Random rand = new Random(nextRandomSeed());
905     protected final Configuration conf;
906     protected final TestOptions opts;
907 
908     private final Status status;
909     private final Sampler<?> traceSampler;
910     private final SpanReceiverHost receiverHost;
911     protected Connection connection;
912 
913     private String testName;
914     private Histogram latency;
915     private Histogram valueSize;
916     private RandomDistribution.Zipf zipf;
917 
918     /**
919      * Note that all subclasses of this class must provide a public constructor
920      * that has the exact same list of arguments.
921      */
922     Test(final Connection con, final TestOptions options, final Status status) {
923       this.connection = con;
924       this.conf = con == null ? HBaseConfiguration.create() : this.connection.getConfiguration();
925       this.opts = options;
926       this.status = status;
927       this.testName = this.getClass().getSimpleName();
928       receiverHost = SpanReceiverHost.getInstance(conf);
929       if (options.traceRate >= 1.0) {
930         this.traceSampler = Sampler.ALWAYS;
931       } else if (options.traceRate > 0.0) {
932         conf.setDouble("hbase.sampler.fraction", options.traceRate);
933         this.traceSampler = new ProbabilitySampler(new HBaseHTraceConfiguration(conf));
934       } else {
935         this.traceSampler = Sampler.NEVER;
936       }
937       everyN = (int) (opts.totalRows / (opts.totalRows * opts.sampleRate));
938       if (options.isValueZipf()) {
939         this.zipf = new RandomDistribution.Zipf(this.rand, 1, options.getValueSize(), 1.1);
940       }
941       LOG.info("Sampling 1 every " + everyN + " out of " + opts.perClientRunRows + " total rows.");
942     }
943 
944     int getValueLength(final Random r) {
945       if (this.opts.isValueRandom()) return Math.abs(r.nextInt() % opts.valueSize);
946       else if (this.opts.isValueZipf()) return Math.abs(this.zipf.nextInt());
947       else return opts.valueSize;
948     }
949 
950     void updateValueSize(final Result [] rs) throws IOException {
951       if (rs == null || !isRandomValueSize()) return;
952       for (Result r: rs) updateValueSize(r);
953     }
954 
955     void updateValueSize(final Result r) throws IOException {
956       if (r == null || !isRandomValueSize()) return;
957       int size = 0;
958       for (CellScanner scanner = r.cellScanner(); scanner.advance();) {
959         size += scanner.current().getValueLength();
960       }
961       updateValueSize(size);
962     }
963 
964     void updateValueSize(final int valueSize) {
965       if (!isRandomValueSize()) return;
966       this.valueSize.update(valueSize);
967     }
968 
969     String generateStatus(final int sr, final int i, final int lr) {
970       return sr + "/" + i + "/" + lr + ", latency " + getShortLatencyReport() +
971         (!isRandomValueSize()? "": ", value size " + getShortValueSizeReport());
972     }
973 
974     boolean isRandomValueSize() {
975       return opts.valueRandom;
976     }
977 
978     protected int getReportingPeriod() {
979       return opts.period;
980     }
981 
982     /**
983      * Populated by testTakedown. Only implemented by RandomReadTest at the moment.
984      */
985     public Histogram getLatency() {
986       return latency;
987     }
988 
989     void testSetup() throws IOException {
990       if (!opts.oneCon) {
991         this.connection = ConnectionFactory.createConnection(conf);
992       }
993       onStartup();
994       latency = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
995       valueSize = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
996     }
997 
998     abstract void onStartup() throws IOException;
999 
1000     void testTakedown() throws IOException {
1001       reportLatency();
1002       reportValueSize();
1003       onTakedown();
1004       if (!opts.oneCon) {
1005         connection.close();
1006       }
1007       receiverHost.closeReceivers();
1008     }
1009 
1010     abstract void onTakedown() throws IOException;
1011 
1012     /*
1013      * Run test
1014      * @return Elapsed time.
1015      * @throws IOException
1016      */
1017     long test() throws IOException, InterruptedException {
1018       testSetup();
1019       LOG.info("Timed test starting in thread " + Thread.currentThread().getName());
1020       final long startTime = System.nanoTime();
1021       try {
1022         testTimed();
1023       } finally {
1024         testTakedown();
1025       }
1026       return (System.nanoTime() - startTime) / 1000000;
1027     }
1028 
1029     /**
1030      * Provides an extension point for tests that don't want a per row invocation.
1031      */
1032     void testTimed() throws IOException, InterruptedException {
1033       int lastRow = opts.startRow + opts.perClientRunRows;
1034       // Report on completion of 1/10th of total.
1035       for (int i = opts.startRow; i < lastRow; i++) {
1036         if (i % everyN != 0) continue;
1037         long startTime = System.nanoTime();
1038         TraceScope scope = Trace.startSpan("test row", traceSampler);
1039         try {
1040           testRow(i);
1041         } finally {
1042           scope.close();
1043         }
1044         latency.update((System.nanoTime() - startTime) / 1000);
1045         if (status != null && i > 0 && (i % getReportingPeriod()) == 0) {
1046           status.setStatus(generateStatus(opts.startRow, i, lastRow));
1047         }
1048       }
1049     }
1050     /**
1051      * report percentiles of latency
1052      * @throws IOException
1053      */
1054     private void reportLatency() throws IOException {
1055       status.setStatus(testName + " latency log (microseconds), on " +
1056           latency.count() + " measures");
1057       reportHistogram(this.latency);
1058     }
1059 
1060     private void reportValueSize() throws IOException {
1061       status.setStatus(testName + " valueSize after " +
1062           valueSize.count() + " measures");
1063       reportHistogram(this.valueSize);
1064     }
1065 
1066     private void reportHistogram(final Histogram h) throws IOException {
1067       Snapshot sn = h.getSnapshot();
1068       status.setStatus(testName + " Min      = " + h.min());
1069       status.setStatus(testName + " Avg      = " + h.mean());
1070       status.setStatus(testName + " StdDev   = " + h.stdDev());
1071       status.setStatus(testName + " 50th     = " + sn.getMedian());
1072       status.setStatus(testName + " 75th     = " + sn.get75thPercentile());
1073       status.setStatus(testName + " 95th     = " + sn.get95thPercentile());
1074       status.setStatus(testName + " 99th     = " + sn.get99thPercentile());
1075       status.setStatus(testName + " 99.9th   = " + sn.get999thPercentile());
1076       status.setStatus(testName + " 99.99th  = " + sn.getValue(0.9999));
1077       status.setStatus(testName + " 99.999th = " + sn.getValue(0.99999));
1078       status.setStatus(testName + " Max      = " + h.max());
1079     }
1080 
1081     /**
1082      * @return Subset of the histograms' calculation.
1083      */
1084     public String getShortLatencyReport() {
1085       return YammerHistogramUtils.getShortHistogramReport(this.latency);
1086     }
1087 
1088     /**
1089      * @return Subset of the histograms' calculation.
1090      */
1091     public String getShortValueSizeReport() {
1092       return YammerHistogramUtils.getShortHistogramReport(this.valueSize);
1093     }
1094 
1095     /*
1096     * Test for individual row.
1097     * @param i Row index.
1098     */
1099     abstract void testRow(final int i) throws IOException, InterruptedException;
1100   }
1101 
1102   static abstract class TableTest extends Test {
1103     protected Table table;
1104 
1105     TableTest(Connection con, TestOptions options, Status status) {
1106       super(con, options, status);
1107     }
1108 
1109     @Override
1110     void onStartup() throws IOException {
1111       this.table = connection.getTable(TableName.valueOf(opts.tableName));
1112     }
1113     
1114     @Override
1115     void onTakedown() throws IOException {
1116       table.close();
1117     }
1118   }
1119 
1120   static abstract class BufferedMutatorTest extends Test {
1121     protected BufferedMutator mutator;
1122 
1123     BufferedMutatorTest(Connection con, TestOptions options, Status status) {
1124       super(con, options, status);
1125     }
1126 
1127     @Override
1128     void onStartup() throws IOException {
1129       this.mutator = connection.getBufferedMutator(TableName.valueOf(opts.tableName));
1130     }
1131     
1132     @Override
1133     void onTakedown() throws IOException {
1134       mutator.close();
1135     }
1136   }
1137 
1138   static class RandomSeekScanTest extends TableTest {
1139     RandomSeekScanTest(Connection con, TestOptions options, Status status) {
1140       super(con, options, status);
1141     }
1142 
1143     @Override
1144     void testRow(final int i) throws IOException {
1145       Scan scan = new Scan(getRandomRow(this.rand, opts.totalRows));
1146       FilterList list = new FilterList();
1147       scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
1148       if (opts.filterAll) {
1149         list.addFilter(new FilterAllFilter());
1150       }
1151       list.addFilter(new WhileMatchFilter(new PageFilter(120)));
1152       scan.setFilter(list);
1153       ResultScanner s = this.table.getScanner(scan);
1154       for (Result rr; (rr = s.next()) != null;) {
1155         updateValueSize(rr);
1156       }
1157       s.close();
1158     }
1159 
1160     @Override
1161     protected int getReportingPeriod() {
1162       int period = opts.perClientRunRows / 100;
1163       return period == 0 ? opts.perClientRunRows : period;
1164     }
1165 
1166   }
1167 
1168   static abstract class RandomScanWithRangeTest extends TableTest {
1169     RandomScanWithRangeTest(Connection con, TestOptions options, Status status) {
1170       super(con, options, status);
1171     }
1172 
1173     @Override
1174     void testRow(final int i) throws IOException {
1175       Pair<byte[], byte[]> startAndStopRow = getStartAndStopRow();
1176       Scan scan = new Scan(startAndStopRow.getFirst(), startAndStopRow.getSecond());
1177       if (opts.filterAll) {
1178         scan.setFilter(new FilterAllFilter());
1179       }
1180       scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
1181       Result r = null;
1182       int count = 0;
1183       ResultScanner s = this.table.getScanner(scan);
1184       for (; (r = s.next()) != null;) {
1185         updateValueSize(r);
1186         count++;
1187       }
1188       if (i % 100 == 0) {
1189         LOG.info(String.format("Scan for key range %s - %s returned %s rows",
1190             Bytes.toString(startAndStopRow.getFirst()),
1191             Bytes.toString(startAndStopRow.getSecond()), count));
1192       }
1193 
1194       s.close();
1195     }
1196 
1197     protected abstract Pair<byte[],byte[]> getStartAndStopRow();
1198 
1199     protected Pair<byte[], byte[]> generateStartAndStopRows(int maxRange) {
1200       int start = this.rand.nextInt(Integer.MAX_VALUE) % opts.totalRows;
1201       int stop = start + maxRange;
1202       return new Pair<byte[],byte[]>(format(start), format(stop));
1203     }
1204 
1205     @Override
1206     protected int getReportingPeriod() {
1207       int period = opts.perClientRunRows / 100;
1208       return period == 0? opts.perClientRunRows: period;
1209     }
1210   }
1211 
1212   static class RandomScanWithRange10Test extends RandomScanWithRangeTest {
1213     RandomScanWithRange10Test(Connection con, TestOptions options, Status status) {
1214       super(con, options, status);
1215     }
1216 
1217     @Override
1218     protected Pair<byte[], byte[]> getStartAndStopRow() {
1219       return generateStartAndStopRows(10);
1220     }
1221   }
1222 
1223   static class RandomScanWithRange100Test extends RandomScanWithRangeTest {
1224     RandomScanWithRange100Test(Connection con, TestOptions options, Status status) {
1225       super(con, options, status);
1226     }
1227 
1228     @Override
1229     protected Pair<byte[], byte[]> getStartAndStopRow() {
1230       return generateStartAndStopRows(100);
1231     }
1232   }
1233 
1234   static class RandomScanWithRange1000Test extends RandomScanWithRangeTest {
1235     RandomScanWithRange1000Test(Connection con, TestOptions options, Status status) {
1236       super(con, options, status);
1237     }
1238 
1239     @Override
1240     protected Pair<byte[], byte[]> getStartAndStopRow() {
1241       return generateStartAndStopRows(1000);
1242     }
1243   }
1244 
1245   static class RandomScanWithRange10000Test extends RandomScanWithRangeTest {
1246     RandomScanWithRange10000Test(Connection con, TestOptions options, Status status) {
1247       super(con, options, status);
1248     }
1249 
1250     @Override
1251     protected Pair<byte[], byte[]> getStartAndStopRow() {
1252       return generateStartAndStopRows(10000);
1253     }
1254   }
1255 
1256   static class RandomReadTest extends TableTest {
1257     private final Consistency consistency;
1258     private ArrayList<Get> gets;
1259     private Random rd = new Random();
1260 
1261     RandomReadTest(Connection con, TestOptions options, Status status) {
1262       super(con, options, status);
1263       consistency = options.replicas == DEFAULT_OPTS.replicas ? null : Consistency.TIMELINE;
1264       if (opts.multiGet > 0) {
1265         LOG.info("MultiGet enabled. Sending GETs in batches of " + opts.multiGet + ".");
1266         this.gets = new ArrayList<Get>(opts.multiGet);
1267       }
1268     }
1269 
1270     @Override
1271     void testRow(final int i) throws IOException, InterruptedException {
1272       if (opts.randomSleep > 0) {
1273         Thread.sleep(rd.nextInt(opts.randomSleep));
1274       }
1275       Get get = new Get(getRandomRow(this.rand, opts.totalRows));
1276       get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
1277       if (opts.filterAll) {
1278         get.setFilter(new FilterAllFilter());
1279       }
1280       get.setConsistency(consistency);
1281       if (LOG.isTraceEnabled()) LOG.trace(get.toString());
1282       if (opts.multiGet > 0) {
1283         this.gets.add(get);
1284         if (this.gets.size() == opts.multiGet) {
1285           Result [] rs = this.table.get(this.gets);
1286           updateValueSize(rs);
1287           this.gets.clear();
1288         }
1289       } else {
1290         updateValueSize(this.table.get(get));
1291       }
1292     }
1293 
1294     @Override
1295     protected int getReportingPeriod() {
1296       int period = opts.perClientRunRows / 10;
1297       return period == 0 ? opts.perClientRunRows : period;
1298     }
1299 
1300     @Override
1301     protected void testTakedown() throws IOException {
1302       if (this.gets != null && this.gets.size() > 0) {
1303         this.table.get(gets);
1304         this.gets.clear();
1305       }
1306       super.testTakedown();
1307     }
1308   }
1309 
1310   static class RandomWriteTest extends BufferedMutatorTest {
1311     RandomWriteTest(Connection con, TestOptions options, Status status) {
1312       super(con, options, status);
1313     }
1314 
1315     @Override
1316     void testRow(final int i) throws IOException {
1317       byte[] row = getRandomRow(this.rand, opts.totalRows);
1318       Put put = new Put(row);
1319       byte[] value = generateData(this.rand, getValueLength(this.rand));
1320       if (opts.useTags) {
1321         byte[] tag = generateData(this.rand, TAG_LENGTH);
1322         Tag[] tags = new Tag[opts.noOfTags];
1323         for (int n = 0; n < opts.noOfTags; n++) {
1324           Tag t = new Tag((byte) n, tag);
1325           tags[n] = t;
1326         }
1327         KeyValue kv = new KeyValue(row, FAMILY_NAME, QUALIFIER_NAME, HConstants.LATEST_TIMESTAMP,
1328             value, tags);
1329         put.add(kv);
1330         updateValueSize(kv.getValueLength());
1331       } else {
1332         put.add(FAMILY_NAME, QUALIFIER_NAME, value);
1333         updateValueSize(value.length);
1334       }
1335       put.setDurability(opts.writeToWAL ? Durability.SYNC_WAL : Durability.SKIP_WAL);
1336       mutator.mutate(put);
1337     }
1338   }
1339 
1340   static class ScanTest extends TableTest {
1341     private ResultScanner testScanner;
1342 
1343     ScanTest(Connection con, TestOptions options, Status status) {
1344       super(con, options, status);
1345     }
1346 
1347     @Override
1348     void testTakedown() throws IOException {
1349       if (this.testScanner != null) {
1350         this.testScanner.close();
1351       }
1352       super.testTakedown();
1353     }
1354 
1355 
1356     @Override
1357     void testRow(final int i) throws IOException {
1358       if (this.testScanner == null) {
1359         Scan scan = new Scan(format(opts.startRow));
1360         scan.setCaching(30);
1361         scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
1362         if (opts.filterAll) {
1363           scan.setFilter(new FilterAllFilter());
1364         }
1365        this.testScanner = table.getScanner(scan);
1366       }
1367       Result r = testScanner.next();
1368       updateValueSize(r);
1369     }
1370 
1371   }
1372 
1373   static class SequentialReadTest extends TableTest {
1374     SequentialReadTest(Connection con, TestOptions options, Status status) {
1375       super(con, options, status);
1376     }
1377 
1378     @Override
1379     void testRow(final int i) throws IOException {
1380       Get get = new Get(format(i));
1381       get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
1382       if (opts.filterAll) {
1383         get.setFilter(new FilterAllFilter());
1384       }
1385       updateValueSize(table.get(get));
1386     }
1387   }
1388 
1389   static class SequentialWriteTest extends BufferedMutatorTest {
1390     SequentialWriteTest(Connection con, TestOptions options, Status status) {
1391       super(con, options, status);
1392     }
1393 
1394     @Override
1395     void testRow(final int i) throws IOException {
1396       byte[] row = format(i);
1397       Put put = new Put(row);
1398       byte[] value = generateData(this.rand, getValueLength(this.rand));
1399       if (opts.useTags) {
1400         byte[] tag = generateData(this.rand, TAG_LENGTH);
1401         Tag[] tags = new Tag[opts.noOfTags];
1402         for (int n = 0; n < opts.noOfTags; n++) {
1403           Tag t = new Tag((byte) n, tag);
1404           tags[n] = t;
1405         }
1406         KeyValue kv = new KeyValue(row, FAMILY_NAME, QUALIFIER_NAME, HConstants.LATEST_TIMESTAMP,
1407             value, tags);
1408         put.add(kv);
1409         updateValueSize(kv.getValueLength());
1410       } else {
1411         put.add(FAMILY_NAME, QUALIFIER_NAME, value);
1412         updateValueSize(value.length);
1413       }
1414       put.setDurability(opts.writeToWAL ? Durability.SYNC_WAL : Durability.SKIP_WAL);
1415       mutator.mutate(put);
1416     }
1417   }
1418 
1419   static class FilteredScanTest extends TableTest {
1420     protected static final Log LOG = LogFactory.getLog(FilteredScanTest.class.getName());
1421 
1422     FilteredScanTest(Connection con, TestOptions options, Status status) {
1423       super(con, options, status);
1424     }
1425 
1426     @Override
1427     void testRow(int i) throws IOException {
1428       byte[] value = generateData(this.rand, getValueLength(this.rand));
1429       Scan scan = constructScan(value);
1430       ResultScanner scanner = null;
1431       try {
1432         scanner = this.table.getScanner(scan);
1433         for (Result r = null; (r = scanner.next()) != null;) {
1434           updateValueSize(r);
1435         }
1436       } finally {
1437         if (scanner != null) scanner.close();
1438       }
1439     }
1440 
1441     protected Scan constructScan(byte[] valuePrefix) throws IOException {
1442       FilterList list = new FilterList();
1443       Filter filter = new SingleColumnValueFilter(
1444           FAMILY_NAME, QUALIFIER_NAME, CompareFilter.CompareOp.EQUAL,
1445           new BinaryComparator(valuePrefix)
1446       );
1447       list.addFilter(filter);
1448       if(opts.filterAll) {
1449         list.addFilter(new FilterAllFilter());
1450       }
1451       Scan scan = new Scan();
1452       scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
1453       scan.setFilter(list);
1454       return scan;
1455     }
1456   }
1457 
1458   /**
1459    * Compute a throughput rate in MB/s.
1460    * @param rows Number of records consumed.
1461    * @param timeMs Time taken in milliseconds.
1462    * @return String value with label, ie '123.76 MB/s'
1463    */
1464   private static String calculateMbps(int rows, long timeMs, final int valueSize) {
1465     // MB/s = ((totalRows * ROW_SIZE_BYTES) / totalTimeMS)
1466     //        * 1000 MS_PER_SEC / (1024 * 1024) BYTES_PER_MB
1467     BigDecimal rowSize =
1468       BigDecimal.valueOf(ROW_LENGTH + valueSize + FAMILY_NAME.length + QUALIFIER_NAME.length);
1469     BigDecimal mbps = BigDecimal.valueOf(rows).multiply(rowSize, CXT)
1470       .divide(BigDecimal.valueOf(timeMs), CXT).multiply(MS_PER_SEC, CXT)
1471       .divide(BYTES_PER_MB, CXT);
1472     return FMT.format(mbps) + " MB/s";
1473   }
1474 
1475   /*
1476    * Format passed integer.
1477    * @param number
1478    * @return Returns zero-prefixed ROW_LENGTH-byte wide decimal version of passed
1479    * number (Does absolute in case number is negative).
1480    */
1481   public static byte [] format(final int number) {
1482     byte [] b = new byte[ROW_LENGTH];
1483     int d = Math.abs(number);
1484     for (int i = b.length - 1; i >= 0; i--) {
1485       b[i] = (byte)((d % 10) + '0');
1486       d /= 10;
1487     }
1488     return b;
1489   }
1490 
1491   /*
1492    * This method takes some time and is done inline uploading data.  For
1493    * example, doing the mapfile test, generation of the key and value
1494    * consumes about 30% of CPU time.
1495    * @return Generated random value to insert into a table cell.
1496    */
1497   public static byte[] generateData(final Random r, int length) {
1498     byte [] b = new byte [length];
1499     int i;
1500 
1501     for(i = 0; i < (length-8); i += 8) {
1502       b[i] = (byte) (65 + r.nextInt(26));
1503       b[i+1] = b[i];
1504       b[i+2] = b[i];
1505       b[i+3] = b[i];
1506       b[i+4] = b[i];
1507       b[i+5] = b[i];
1508       b[i+6] = b[i];
1509       b[i+7] = b[i];
1510     }
1511 
1512     byte a = (byte) (65 + r.nextInt(26));
1513     for(; i < length; i++) {
1514       b[i] = a;
1515     }
1516     return b;
1517   }
1518 
1519   /**
1520    * @deprecated Use {@link #generateData(java.util.Random, int)} instead.
1521    * @return Generated random value to insert into a table cell.
1522    */
1523   @Deprecated
1524   public static byte[] generateValue(final Random r) {
1525     return generateData(r, DEFAULT_VALUE_LENGTH);
1526   }
1527 
1528   static byte [] getRandomRow(final Random random, final int totalRows) {
1529     return format(random.nextInt(Integer.MAX_VALUE) % totalRows);
1530   }
1531 
1532   static RunResult runOneClient(final Class<? extends Test> cmd, Configuration conf, Connection con,
1533                            TestOptions opts, final Status status)
1534       throws IOException, InterruptedException {
1535     status.setStatus("Start " + cmd + " at offset " + opts.startRow + " for " +
1536       opts.perClientRunRows + " rows");
1537     long totalElapsedTime;
1538 
1539     final Test t;
1540     try {
1541       Constructor<? extends Test> constructor =
1542         cmd.getDeclaredConstructor(Connection.class, TestOptions.class, Status.class);
1543       t = constructor.newInstance(con, opts, status);
1544     } catch (NoSuchMethodException e) {
1545       throw new IllegalArgumentException("Invalid command class: " +
1546           cmd.getName() + ".  It does not provide a constructor as described by " +
1547           "the javadoc comment.  Available constructors are: " +
1548           Arrays.toString(cmd.getConstructors()));
1549     } catch (Exception e) {
1550       throw new IllegalStateException("Failed to construct command class", e);
1551     }
1552     totalElapsedTime = t.test();
1553 
1554     status.setStatus("Finished " + cmd + " in " + totalElapsedTime +
1555       "ms at offset " + opts.startRow + " for " + opts.perClientRunRows + " rows" +
1556       " (" + calculateMbps((int)(opts.perClientRunRows * opts.sampleRate), totalElapsedTime,
1557           getAverageValueLength(opts)) + ")");
1558 
1559     return new RunResult(totalElapsedTime, t.getLatency());
1560   }
1561 
1562   private static int getAverageValueLength(final TestOptions opts) {
1563     return opts.valueRandom? opts.valueSize/2: opts.valueSize;
1564   }
1565 
1566   private void runTest(final Class<? extends Test> cmd, TestOptions opts) throws IOException,
1567       InterruptedException, ClassNotFoundException {
1568     // Log the configuration we're going to run with. Uses JSON mapper because lazy. It'll do
1569     // the TestOptions introspection for us and dump the output in a readable format.
1570     LOG.info(cmd.getSimpleName() + " test run options=" + MAPPER.writeValueAsString(opts));
1571     try(Connection conn = ConnectionFactory.createConnection(getConf());
1572         Admin admin = conn.getAdmin()) {
1573       checkTable(admin, opts);
1574     }
1575     if (opts.nomapred) {
1576       doLocalClients(opts, getConf());
1577     } else {
1578       doMapReduce(opts, getConf());
1579     }
1580   }
1581 
1582   protected void printUsage() {
1583     printUsage(null);
1584   }
1585 
1586   protected void printUsage(final String message) {
1587     if (message != null && message.length() > 0) {
1588       System.err.println(message);
1589     }
1590     System.err.println("Usage: java " + this.getClass().getName() + " \\");
1591     System.err.println("  <OPTIONS> [-D<property=value>]* <command> <nclients>");
1592     System.err.println();
1593     System.err.println("Options:");
1594     System.err.println(" nomapred        Run multiple clients using threads " +
1595       "(rather than use mapreduce)");
1596     System.err.println(" rows            Rows each client runs. Default: One million");
1597     System.err.println(" size            Total size in GiB. Mutually exclusive with --rows. " +
1598       "Default: 1.0.");
1599     System.err.println(" sampleRate      Execute test on a sample of total " +
1600       "rows. Only supported by randomRead. Default: 1.0");
1601     System.err.println(" traceRate       Enable HTrace spans. Initiate tracing every N rows. " +
1602       "Default: 0");
1603     System.err.println(" table           Alternate table name. Default: 'TestTable'");
1604     System.err.println(" multiGet        If >0, when doing RandomRead, perform multiple gets " +
1605       "instead of single gets. Default: 0");
1606     System.err.println(" compress        Compression type to use (GZ, LZO, ...). Default: 'NONE'");
1607     System.err.println(" flushCommits    Used to determine if the test should flush the table. " +
1608       "Default: false");
1609     System.err.println(" writeToWAL      Set writeToWAL on puts. Default: True");
1610     System.err.println(" autoFlush       Set autoFlush on htable. Default: False");
1611     System.err.println(" oneCon          all the threads share the same connection. Default: False");
1612     System.err.println(" presplit        Create presplit table. Recommended for accurate perf " +
1613       "analysis (see guide).  Default: disabled");
1614     System.err.println(" inmemory        Tries to keep the HFiles of the CF " +
1615       "inmemory as far as possible. Not guaranteed that reads are always served " +
1616       "from memory.  Default: false");
1617     System.err.println(" usetags         Writes tags along with KVs. Use with HFile V3. " +
1618       "Default: false");
1619     System.err.println(" numoftags       Specify the no of tags that would be needed. " +
1620        "This works only if usetags is true.");
1621     System.err.println(" filterAll       Helps to filter out all the rows on the server side"
1622         + " there by not returning any thing back to the client.  Helps to check the server side"
1623         + " performance.  Uses FilterAllFilter internally. ");
1624     System.err.println(" latency         Set to report operation latencies. Default: False");
1625     System.err.println(" bloomFilter      Bloom filter type, one of " + Arrays.toString(BloomType.values()));
1626     System.err.println(" valueSize       Pass value size to use: Default: 1024");
1627     System.err.println(" valueRandom     Set if we should vary value size between 0 and " +
1628         "'valueSize'; set on read for stats on size: Default: Not set.");
1629     System.err.println(" valueZipf       Set if we should vary value size between 0 and " +
1630         "'valueSize' in zipf form: Default: Not set.");
1631     System.err.println(" period          Report every 'period' rows: " +
1632       "Default: opts.perClientRunRows / 10");
1633     System.err.println(" multiGet        Batch gets together into groups of N. Only supported " +
1634       "by randomRead. Default: disabled");
1635     System.err.println(" replicas        Enable region replica testing. Defaults: 1.");
1636     System.err.println(" splitPolicy     Specify a custom RegionSplitPolicy for the table.");
1637     System.err.println(" randomSleep     Do a random sleep before each get between 0 and entered value. Defaults: 0");
1638     System.err.println();
1639     System.err.println(" Note: -D properties will be applied to the conf used. ");
1640     System.err.println("  For example: ");
1641     System.err.println("   -Dmapreduce.output.fileoutputformat.compress=true");
1642     System.err.println("   -Dmapreduce.task.timeout=60000");
1643     System.err.println();
1644     System.err.println("Command:");
1645     for (CmdDescriptor command : COMMANDS.values()) {
1646       System.err.println(String.format(" %-15s %s", command.getName(), command.getDescription()));
1647     }
1648     System.err.println();
1649     System.err.println("Args:");
1650     System.err.println(" nclients        Integer. Required. Total number of " +
1651       "clients (and HRegionServers)");
1652     System.err.println("                 running: 1 <= value <= 500");
1653     System.err.println("Examples:");
1654     System.err.println(" To run a single evaluation client:");
1655     System.err.println(" $ bin/hbase " + this.getClass().getName()
1656         + " sequentialWrite 1");
1657   }
1658 
1659   /**
1660    * Parse options passed in via an arguments array. Assumes that array has been split
1661    * on white-space and placed into a {@code Queue}. Any unknown arguments will remain
1662    * in the queue at the conclusion of this method call. It's up to the caller to deal
1663    * with these unrecognized arguments.
1664    */
1665   static TestOptions parseOpts(Queue<String> args) {
1666     TestOptions opts = new TestOptions();
1667 
1668     String cmd = null;
1669     while ((cmd = args.poll()) != null) {
1670       if (cmd.equals("-h") || cmd.startsWith("--h")) {
1671         // place item back onto queue so that caller knows parsing was incomplete
1672         args.add(cmd);
1673         break;
1674       }
1675 
1676       final String nmr = "--nomapred";
1677       if (cmd.startsWith(nmr)) {
1678         opts.nomapred = true;
1679         continue;
1680       }
1681 
1682       final String rows = "--rows=";
1683       if (cmd.startsWith(rows)) {
1684         opts.perClientRunRows = Integer.parseInt(cmd.substring(rows.length()));
1685         continue;
1686       }
1687 
1688       final String sampleRate = "--sampleRate=";
1689       if (cmd.startsWith(sampleRate)) {
1690         opts.sampleRate = Float.parseFloat(cmd.substring(sampleRate.length()));
1691         continue;
1692       }
1693 
1694       final String table = "--table=";
1695       if (cmd.startsWith(table)) {
1696         opts.tableName = cmd.substring(table.length());
1697         continue;
1698       }
1699 
1700       final String startRow = "--startRow=";
1701       if (cmd.startsWith(startRow)) {
1702         opts.startRow = Integer.parseInt(cmd.substring(startRow.length()));
1703         continue;
1704       }
1705 
1706       final String compress = "--compress=";
1707       if (cmd.startsWith(compress)) {
1708         opts.compression = Compression.Algorithm.valueOf(cmd.substring(compress.length()));
1709         continue;
1710       }
1711 
1712       final String traceRate = "--traceRate=";
1713       if (cmd.startsWith(traceRate)) {
1714         opts.traceRate = Double.parseDouble(cmd.substring(traceRate.length()));
1715         continue;
1716       }
1717 
1718       final String blockEncoding = "--blockEncoding=";
1719       if (cmd.startsWith(blockEncoding)) {
1720         opts.blockEncoding = DataBlockEncoding.valueOf(cmd.substring(blockEncoding.length()));
1721         continue;
1722       }
1723 
1724       final String flushCommits = "--flushCommits=";
1725       if (cmd.startsWith(flushCommits)) {
1726         opts.flushCommits = Boolean.parseBoolean(cmd.substring(flushCommits.length()));
1727         continue;
1728       }
1729 
1730       final String writeToWAL = "--writeToWAL=";
1731       if (cmd.startsWith(writeToWAL)) {
1732         opts.writeToWAL = Boolean.parseBoolean(cmd.substring(writeToWAL.length()));
1733         continue;
1734       }
1735 
1736       final String presplit = "--presplit=";
1737       if (cmd.startsWith(presplit)) {
1738         opts.presplitRegions = Integer.parseInt(cmd.substring(presplit.length()));
1739         continue;
1740       }
1741 
1742       final String inMemory = "--inmemory=";
1743       if (cmd.startsWith(inMemory)) {
1744         opts.inMemoryCF = Boolean.parseBoolean(cmd.substring(inMemory.length()));
1745         continue;
1746       }
1747 
1748       final String autoFlush = "--autoFlush=";
1749       if (cmd.startsWith(autoFlush)) {
1750         opts.autoFlush = Boolean.parseBoolean(cmd.substring(autoFlush.length()));
1751         continue;
1752       }
1753 
1754       final String onceCon = "--oneCon=";
1755       if (cmd.startsWith(onceCon)) {
1756         opts.oneCon = Boolean.parseBoolean(cmd.substring(onceCon.length()));
1757         continue;
1758       }
1759 
1760       final String latency = "--latency";
1761       if (cmd.startsWith(latency)) {
1762         opts.reportLatency = true;
1763         continue;
1764       }
1765 
1766       final String multiGet = "--multiGet=";
1767       if (cmd.startsWith(multiGet)) {
1768         opts.multiGet = Integer.parseInt(cmd.substring(multiGet.length()));
1769         continue;
1770       }
1771 
1772       final String useTags = "--usetags=";
1773       if (cmd.startsWith(useTags)) {
1774         opts.useTags = Boolean.parseBoolean(cmd.substring(useTags.length()));
1775         continue;
1776       }
1777 
1778       final String noOfTags = "--numoftags=";
1779       if (cmd.startsWith(noOfTags)) {
1780         opts.noOfTags = Integer.parseInt(cmd.substring(noOfTags.length()));
1781         continue;
1782       }
1783 
1784       final String replicas = "--replicas=";
1785       if (cmd.startsWith(replicas)) {
1786         opts.replicas = Integer.parseInt(cmd.substring(replicas.length()));
1787         continue;
1788       }
1789 
1790       final String filterOutAll = "--filterAll";
1791       if (cmd.startsWith(filterOutAll)) {
1792         opts.filterAll = true;
1793         continue;
1794       }
1795 
1796       final String size = "--size=";
1797       if (cmd.startsWith(size)) {
1798         opts.size = Float.parseFloat(cmd.substring(size.length()));
1799         continue;
1800       }
1801 
1802       final String splitPolicy = "--splitPolicy=";
1803       if (cmd.startsWith(splitPolicy)) {
1804         opts.splitPolicy = cmd.substring(splitPolicy.length());
1805         continue;
1806       }
1807 
1808       final String randomSleep = "--randomSleep=";
1809       if (cmd.startsWith(randomSleep)) {
1810         opts.randomSleep = Integer.parseInt(cmd.substring(randomSleep.length()));
1811         continue;
1812       }
1813 
1814       final String bloomFilter = "--bloomFilter=";
1815       if (cmd.startsWith(bloomFilter)) {
1816         opts.bloomType = BloomType.valueOf(cmd.substring(bloomFilter.length()));
1817         continue;
1818       }
1819 
1820       final String valueSize = "--valueSize=";
1821       if (cmd.startsWith(valueSize)) {
1822         opts.valueSize = Integer.parseInt(cmd.substring(valueSize.length()));
1823         continue;
1824       }
1825 
1826       final String valueRandom = "--valueRandom";
1827       if (cmd.startsWith(valueRandom)) {
1828         opts.valueRandom = true;
1829         if (opts.valueZipf) {
1830           throw new IllegalStateException("Either valueZipf or valueRandom but not both");
1831         }
1832         continue;
1833       }
1834 
1835       final String valueZipf = "--valueZipf";
1836       if (cmd.startsWith(valueZipf)) {
1837         opts.valueZipf = true;
1838         if (opts.valueRandom) {
1839           throw new IllegalStateException("Either valueZipf or valueRandom but not both");
1840         }
1841         continue;
1842       }
1843 
1844       final String period = "--period=";
1845       if (cmd.startsWith(period)) {
1846         opts.period = Integer.parseInt(cmd.substring(period.length()));
1847         continue;
1848       }
1849 
1850       if (isCommandClass(cmd)) {
1851         opts.cmdName = cmd;
1852         opts.numClientThreads = Integer.parseInt(args.remove());
1853         int rowsPerGB = ONE_GB / (opts.valueRandom? opts.valueSize/2: opts.valueSize);
1854         if (opts.size != DEFAULT_OPTS.size &&
1855             opts.perClientRunRows != DEFAULT_OPTS.perClientRunRows) {
1856           throw new IllegalArgumentException(rows + " and " + size + " are mutually exclusive arguments.");
1857         }
1858         if (opts.size != DEFAULT_OPTS.size) {
1859           // total size in GB specified
1860           opts.totalRows = (int) opts.size * rowsPerGB;
1861           opts.perClientRunRows = opts.totalRows / opts.numClientThreads;
1862         } else if (opts.perClientRunRows != DEFAULT_OPTS.perClientRunRows) {
1863           // number of rows specified
1864           opts.totalRows = opts.perClientRunRows * opts.numClientThreads;
1865           opts.size = opts.totalRows / rowsPerGB;
1866         }
1867         break;
1868       }
1869 
1870       // Not matching any option or command.
1871       System.err.println("Error: Wrong option or command: " + cmd);
1872       args.add(cmd);
1873       break;
1874     }
1875     return opts;
1876   }
1877 
1878   @Override
1879   public int run(String[] args) throws Exception {
1880     // Process command-line args. TODO: Better cmd-line processing
1881     // (but hopefully something not as painful as cli options).
1882     int errCode = -1;
1883     if (args.length < 1) {
1884       printUsage();
1885       return errCode;
1886     }
1887 
1888     try {
1889       LinkedList<String> argv = new LinkedList<String>();
1890       argv.addAll(Arrays.asList(args));
1891       TestOptions opts = parseOpts(argv);
1892 
1893       // args remaining, print help and exit
1894       if (!argv.isEmpty()) {
1895         errCode = 0;
1896         printUsage();
1897         return errCode;
1898       }
1899 
1900       // must run at least 1 client
1901       if (opts.numClientThreads <= 0) {
1902         throw new IllegalArgumentException("Number of clients must be > 0");
1903       }
1904 
1905       Class<? extends Test> cmdClass = determineCommandClass(opts.cmdName);
1906       if (cmdClass != null) {
1907         runTest(cmdClass, opts);
1908         errCode = 0;
1909       }
1910 
1911     } catch (Exception e) {
1912       e.printStackTrace();
1913     }
1914 
1915     return errCode;
1916   }
1917 
1918   private static boolean isCommandClass(String cmd) {
1919     return COMMANDS.containsKey(cmd);
1920   }
1921 
1922   private static Class<? extends Test> determineCommandClass(String cmd) {
1923     CmdDescriptor descriptor = COMMANDS.get(cmd);
1924     return descriptor != null ? descriptor.getCmdClass() : null;
1925   }
1926 
1927   public static void main(final String[] args) throws Exception {
1928     int res = ToolRunner.run(new PerformanceEvaluation(HBaseConfiguration.create()), args);
1929     System.exit(res);
1930   }
1931 }