1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.test;
20
21 import java.io.DataInput;
22 import java.io.DataOutput;
23 import java.io.IOException;
24 import java.security.SecureRandom;
25 import java.util.ArrayList;
26 import java.util.Arrays;
27 import java.util.Iterator;
28 import java.util.List;
29 import java.util.Random;
30 import java.util.Set;
31 import java.util.UUID;
32 import java.util.concurrent.atomic.AtomicInteger;
33
34 import org.apache.commons.cli.CommandLine;
35 import org.apache.commons.cli.GnuParser;
36 import org.apache.commons.cli.HelpFormatter;
37 import org.apache.commons.cli.Options;
38 import org.apache.commons.cli.ParseException;
39 import org.apache.commons.logging.Log;
40 import org.apache.commons.logging.LogFactory;
41 import org.apache.hadoop.conf.Configuration;
42 import org.apache.hadoop.conf.Configured;
43 import org.apache.hadoop.fs.FileSystem;
44 import org.apache.hadoop.fs.Path;
45 import org.apache.hadoop.hbase.HBaseConfiguration;
46 import org.apache.hadoop.hbase.HBaseTestingUtility;
47 import org.apache.hadoop.hbase.HColumnDescriptor;
48 import org.apache.hadoop.hbase.HRegionLocation;
49 import org.apache.hadoop.hbase.HTableDescriptor;
50 import org.apache.hadoop.hbase.IntegrationTestBase;
51 import org.apache.hadoop.hbase.IntegrationTestingUtility;
52 import org.apache.hadoop.hbase.testclassification.IntegrationTests;
53 import org.apache.hadoop.hbase.fs.HFileSystem;
54 import org.apache.hadoop.hbase.MasterNotRunningException;
55 import org.apache.hadoop.hbase.TableName;
56 import org.apache.hadoop.hbase.client.Get;
57 import org.apache.hadoop.hbase.client.HBaseAdmin;
58 import org.apache.hadoop.hbase.client.HConnection;
59 import org.apache.hadoop.hbase.client.HConnectionManager;
60 import org.apache.hadoop.hbase.client.HTable;
61 import org.apache.hadoop.hbase.client.Put;
62 import org.apache.hadoop.hbase.client.Result;
63 import org.apache.hadoop.hbase.client.ResultScanner;
64 import org.apache.hadoop.hbase.client.Scan;
65 import org.apache.hadoop.hbase.client.ScannerCallable;
66 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
67 import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
68 import org.apache.hadoop.hbase.mapreduce.TableMapper;
69 import org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl;
70 import org.apache.hadoop.hbase.util.AbstractHBaseTool;
71 import org.apache.hadoop.hbase.util.Bytes;
72 import org.apache.hadoop.hbase.util.RegionSplitter;
73 import org.apache.hadoop.io.BytesWritable;
74 import org.apache.hadoop.io.NullWritable;
75 import org.apache.hadoop.io.Text;
76 import org.apache.hadoop.io.Writable;
77 import org.apache.hadoop.mapreduce.Counter;
78 import org.apache.hadoop.mapreduce.CounterGroup;
79 import org.apache.hadoop.mapreduce.Counters;
80 import org.apache.hadoop.mapreduce.InputFormat;
81 import org.apache.hadoop.mapreduce.InputSplit;
82 import org.apache.hadoop.mapreduce.Job;
83 import org.apache.hadoop.mapreduce.JobContext;
84 import org.apache.hadoop.mapreduce.Mapper;
85 import org.apache.hadoop.mapreduce.RecordReader;
86 import org.apache.hadoop.mapreduce.Reducer;
87 import org.apache.hadoop.mapreduce.TaskAttemptContext;
88 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
89 import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
90 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
91 import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
92 import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
93 import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
94 import org.apache.hadoop.util.Tool;
95 import org.apache.hadoop.util.ToolRunner;
96 import org.junit.Test;
97 import org.junit.experimental.categories.Category;
98
99 import com.google.common.collect.Sets;
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164 @Category(IntegrationTests.class)
165 public class IntegrationTestBigLinkedList extends IntegrationTestBase {
166 protected static final byte[] NO_KEY = new byte[1];
167
168 protected static String TABLE_NAME_KEY = "IntegrationTestBigLinkedList.table";
169
170 protected static String DEFAULT_TABLE_NAME = "IntegrationTestBigLinkedList";
171
172 protected static byte[] FAMILY_NAME = Bytes.toBytes("meta");
173
174
175 protected static final byte[] COLUMN_PREV = Bytes.toBytes("prev");
176
177
178 protected static final byte[] COLUMN_CLIENT = Bytes.toBytes("client");
179
180
181 protected static final byte[] COLUMN_COUNT = Bytes.toBytes("count");
182
183
184 private static final String GENERATOR_NUM_ROWS_PER_MAP_KEY
185 = "IntegrationTestBigLinkedList.generator.num_rows";
186
187 private static final String GENERATOR_NUM_MAPPERS_KEY
188 = "IntegrationTestBigLinkedList.generator.map.tasks";
189
190 private static final String GENERATOR_WIDTH_KEY
191 = "IntegrationTestBigLinkedList.generator.width";
192
193 private static final String GENERATOR_WRAP_KEY
194 = "IntegrationTestBigLinkedList.generator.wrap";
195
196 protected int NUM_SLAVES_BASE = 3;
197
198 private static final int MISSING_ROWS_TO_LOG = 50;
199
200 private static final int WIDTH_DEFAULT = 1000000;
201 private static final int WRAP_DEFAULT = 25;
202 private static final int ROWKEY_LENGTH = 16;
203
204 protected String toRun;
205 protected String[] otherArgs;
206
207 static class CINode {
208 byte[] key;
209 byte[] prev;
210 String client;
211 long count;
212 }
213
214
215
216
217 static class Generator extends Configured implements Tool {
218
219 private static final Log LOG = LogFactory.getLog(Generator.class);
220
221 static class GeneratorInputFormat extends InputFormat<BytesWritable,NullWritable> {
222 static class GeneratorInputSplit extends InputSplit implements Writable {
223 @Override
224 public long getLength() throws IOException, InterruptedException {
225 return 1;
226 }
227 @Override
228 public String[] getLocations() throws IOException, InterruptedException {
229 return new String[0];
230 }
231 @Override
232 public void readFields(DataInput arg0) throws IOException {
233 }
234 @Override
235 public void write(DataOutput arg0) throws IOException {
236 }
237 }
238
239 static class GeneratorRecordReader extends RecordReader<BytesWritable,NullWritable> {
240 private long count;
241 private long numNodes;
242 private Random rand;
243
244 @Override
245 public void close() throws IOException {
246 }
247
248 @Override
249 public BytesWritable getCurrentKey() throws IOException, InterruptedException {
250 byte[] bytes = new byte[ROWKEY_LENGTH];
251 rand.nextBytes(bytes);
252 return new BytesWritable(bytes);
253 }
254
255 @Override
256 public NullWritable getCurrentValue() throws IOException, InterruptedException {
257 return NullWritable.get();
258 }
259
260 @Override
261 public float getProgress() throws IOException, InterruptedException {
262 return (float)(count / (double)numNodes);
263 }
264
265 @Override
266 public void initialize(InputSplit arg0, TaskAttemptContext context)
267 throws IOException, InterruptedException {
268 numNodes = context.getConfiguration().getLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, 25000000);
269
270 rand = new SecureRandom();
271 }
272
273 @Override
274 public boolean nextKeyValue() throws IOException, InterruptedException {
275 return count++ < numNodes;
276 }
277
278 }
279
280 @Override
281 public RecordReader<BytesWritable,NullWritable> createRecordReader(
282 InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
283 GeneratorRecordReader rr = new GeneratorRecordReader();
284 rr.initialize(split, context);
285 return rr;
286 }
287
288 @Override
289 public List<InputSplit> getSplits(JobContext job) throws IOException, InterruptedException {
290 int numMappers = job.getConfiguration().getInt(GENERATOR_NUM_MAPPERS_KEY, 1);
291
292 ArrayList<InputSplit> splits = new ArrayList<InputSplit>(numMappers);
293
294 for (int i = 0; i < numMappers; i++) {
295 splits.add(new GeneratorInputSplit());
296 }
297
298 return splits;
299 }
300 }
301
302
303 static class OneFilePerMapperSFIF<K, V> extends SequenceFileInputFormat<K, V> {
304 @Override
305 protected boolean isSplitable(JobContext context, Path filename) {
306 return false;
307 }
308 }
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334 static class GeneratorMapper
335 extends Mapper<BytesWritable, NullWritable, NullWritable, NullWritable> {
336
337 byte[][] first = null;
338 byte[][] prev = null;
339 byte[][] current = null;
340 byte[] id;
341 long count = 0;
342 int i;
343 HTable table;
344 long numNodes;
345 long wrap;
346 int width;
347
348 @Override
349 protected void setup(Context context) throws IOException, InterruptedException {
350 id = Bytes.toBytes("Job: "+context.getJobID() + " Task: " + context.getTaskAttemptID());
351 Configuration conf = context.getConfiguration();
352 instantiateHTable(conf);
353 this.width = context.getConfiguration().getInt(GENERATOR_WIDTH_KEY, WIDTH_DEFAULT);
354 current = new byte[this.width][];
355 int wrapMultiplier = context.getConfiguration().getInt(GENERATOR_WRAP_KEY, WRAP_DEFAULT);
356 this.wrap = (long)wrapMultiplier * width;
357 this.numNodes = context.getConfiguration().getLong(
358 GENERATOR_NUM_ROWS_PER_MAP_KEY, (long)WIDTH_DEFAULT * WRAP_DEFAULT);
359 if (this.numNodes < this.wrap) {
360 this.wrap = this.numNodes;
361 }
362 }
363
364 protected void instantiateHTable(Configuration conf) throws IOException {
365 table = new HTable(conf, getTableName(conf));
366 table.setAutoFlush(false, true);
367 table.setWriteBufferSize(4 * 1024 * 1024);
368 }
369
370 @Override
371 protected void cleanup(Context context) throws IOException ,InterruptedException {
372 table.close();
373 }
374
375 @Override
376 protected void map(BytesWritable key, NullWritable value, Context output) throws IOException {
377 current[i] = new byte[key.getLength()];
378 System.arraycopy(key.getBytes(), 0, current[i], 0, key.getLength());
379 if (++i == current.length) {
380 persist(output, count, prev, current, id);
381 i = 0;
382
383 if (first == null)
384 first = current;
385 prev = current;
386 current = new byte[this.width][];
387
388 count += current.length;
389 output.setStatus("Count " + count);
390
391 if (count % wrap == 0) {
392
393
394 circularLeftShift(first);
395
396 persist(output, -1, prev, first, null);
397
398 first = null;
399 prev = null;
400 }
401 }
402 }
403
404 private static <T> void circularLeftShift(T[] first) {
405 T ez = first[0];
406 System.arraycopy(first, 1, first, 0, first.length - 1);
407 first[first.length - 1] = ez;
408 }
409
410 protected void persist(Context output, long count, byte[][] prev, byte[][] current, byte[] id)
411 throws IOException {
412 for (int i = 0; i < current.length; i++) {
413 Put put = new Put(current[i]);
414 put.add(FAMILY_NAME, COLUMN_PREV, prev == null ? NO_KEY : prev[i]);
415
416 if (count >= 0) {
417 put.add(FAMILY_NAME, COLUMN_COUNT, Bytes.toBytes(count + i));
418 }
419 if (id != null) {
420 put.add(FAMILY_NAME, COLUMN_CLIENT, id);
421 }
422 table.put(put);
423
424 if (i % 1000 == 0) {
425
426 output.progress();
427 }
428 }
429
430 table.flushCommits();
431 }
432 }
433
434 @Override
435 public int run(String[] args) throws Exception {
436 if (args.length < 3) {
437 System.out.println("Usage : " + Generator.class.getSimpleName() +
438 " <num mappers> <num nodes per map> <tmp output dir> [<width> <wrap multiplier>]");
439 System.out.println(" where <num nodes per map> should be a multiple of " +
440 " width*wrap multiplier, 25M by default");
441 return 0;
442 }
443
444 int numMappers = Integer.parseInt(args[0]);
445 long numNodes = Long.parseLong(args[1]);
446 Path tmpOutput = new Path(args[2]);
447 Integer width = (args.length < 4) ? null : Integer.parseInt(args[3]);
448 Integer wrapMuplitplier = (args.length < 5) ? null : Integer.parseInt(args[4]);
449 return run(numMappers, numNodes, tmpOutput, width, wrapMuplitplier);
450 }
451
452 protected void createSchema() throws IOException {
453 Configuration conf = getConf();
454 HBaseAdmin admin = new HBaseAdmin(conf);
455 TableName tableName = getTableName(conf);
456 try {
457 if (!admin.tableExists(tableName)) {
458 HTableDescriptor htd = new HTableDescriptor(getTableName(getConf()));
459 htd.addFamily(new HColumnDescriptor(FAMILY_NAME));
460 int numberOfServers = admin.getClusterStatus().getServers().size();
461 if (numberOfServers == 0) {
462 throw new IllegalStateException("No live regionservers");
463 }
464 int regionsPerServer = conf.getInt(HBaseTestingUtility.REGIONS_PER_SERVER_KEY,
465 HBaseTestingUtility.DEFAULT_REGIONS_PER_SERVER);
466 int totalNumberOfRegions = numberOfServers * regionsPerServer;
467 LOG.info("Number of live regionservers: " + numberOfServers + ", " +
468 "pre-splitting table into " + totalNumberOfRegions + " regions " +
469 "(default regions per server: " + regionsPerServer + ")");
470
471 byte[][] splits = new RegionSplitter.UniformSplit().split(
472 totalNumberOfRegions);
473
474 admin.createTable(htd, splits);
475 }
476 } catch (MasterNotRunningException e) {
477 LOG.error("Master not running", e);
478 throw new IOException(e);
479 } finally {
480 admin.close();
481 }
482 }
483
484 public int runRandomInputGenerator(int numMappers, long numNodes, Path tmpOutput,
485 Integer width, Integer wrapMuplitplier) throws Exception {
486 LOG.info("Running RandomInputGenerator with numMappers=" + numMappers
487 + ", numNodes=" + numNodes);
488 Job job = new Job(getConf());
489
490 job.setJobName("Random Input Generator");
491 job.setNumReduceTasks(0);
492 job.setJarByClass(getClass());
493
494 job.setInputFormatClass(GeneratorInputFormat.class);
495 job.setOutputKeyClass(BytesWritable.class);
496 job.setOutputValueClass(NullWritable.class);
497
498 setJobConf(job, numMappers, numNodes, width, wrapMuplitplier);
499
500 job.setMapperClass(Mapper.class);
501
502 FileOutputFormat.setOutputPath(job, tmpOutput);
503 job.setOutputFormatClass(SequenceFileOutputFormat.class);
504
505 boolean success = jobCompletion(job);
506
507 return success ? 0 : 1;
508 }
509
510 public int runGenerator(int numMappers, long numNodes, Path tmpOutput,
511 Integer width, Integer wrapMuplitplier) throws Exception {
512 LOG.info("Running Generator with numMappers=" + numMappers +", numNodes=" + numNodes);
513 createSchema();
514 Job job = new Job(getConf());
515
516 job.setJobName("Link Generator");
517 job.setNumReduceTasks(0);
518 job.setJarByClass(getClass());
519
520 FileInputFormat.setInputPaths(job, tmpOutput);
521 job.setInputFormatClass(OneFilePerMapperSFIF.class);
522 job.setOutputKeyClass(NullWritable.class);
523 job.setOutputValueClass(NullWritable.class);
524
525 setJobConf(job, numMappers, numNodes, width, wrapMuplitplier);
526
527 setMapperForGenerator(job);
528
529 job.setOutputFormatClass(NullOutputFormat.class);
530
531 job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false);
532 TableMapReduceUtil.addDependencyJars(job);
533 TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
534 TableMapReduceUtil.initCredentials(job);
535
536 boolean success = jobCompletion(job);
537
538 return success ? 0 : 1;
539 }
540
541 protected boolean jobCompletion(Job job) throws IOException, InterruptedException,
542 ClassNotFoundException {
543 boolean success = job.waitForCompletion(true);
544 return success;
545 }
546
547 protected void setMapperForGenerator(Job job) {
548 job.setMapperClass(GeneratorMapper.class);
549 }
550
551 public int run(int numMappers, long numNodes, Path tmpOutput,
552 Integer width, Integer wrapMuplitplier) throws Exception {
553 int ret = runRandomInputGenerator(numMappers, numNodes, tmpOutput, width, wrapMuplitplier);
554 if (ret > 0) {
555 return ret;
556 }
557 return runGenerator(numMappers, numNodes, tmpOutput, width, wrapMuplitplier);
558 }
559 }
560
561
562
563
564
565 static class Verify extends Configured implements Tool {
566
567 private static final Log LOG = LogFactory.getLog(Verify.class);
568 protected static final BytesWritable DEF = new BytesWritable(NO_KEY);
569
570 protected Job job;
571
572 public static class VerifyMapper extends TableMapper<BytesWritable, BytesWritable> {
573 private BytesWritable row = new BytesWritable();
574 private BytesWritable ref = new BytesWritable();
575
576 @Override
577 protected void map(ImmutableBytesWritable key, Result value, Context context)
578 throws IOException ,InterruptedException {
579 byte[] rowKey = key.get();
580 row.set(rowKey, 0, rowKey.length);
581 context.write(row, DEF);
582 byte[] prev = value.getValue(FAMILY_NAME, COLUMN_PREV);
583 if (prev != null && prev.length > 0) {
584 ref.set(prev, 0, prev.length);
585 context.write(ref, row);
586 } else {
587 LOG.warn(String.format("Prev is not set for: %s", Bytes.toStringBinary(rowKey)));
588 }
589 }
590 }
591
592 public static enum Counts {
593 UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT, EXTRAREFERENCES
594 }
595
596 public static class VerifyReducer extends Reducer<BytesWritable,BytesWritable,Text,Text> {
597 private ArrayList<byte[]> refs = new ArrayList<byte[]>();
598
599 private AtomicInteger rows = new AtomicInteger(0);
600
601 @Override
602 public void reduce(BytesWritable key, Iterable<BytesWritable> values, Context context)
603 throws IOException, InterruptedException {
604
605 int defCount = 0;
606
607 refs.clear();
608 for (BytesWritable type : values) {
609 if (type.getLength() == DEF.getLength()) {
610 defCount++;
611 } else {
612 byte[] bytes = new byte[type.getLength()];
613 System.arraycopy(type.getBytes(), 0, bytes, 0, type.getLength());
614 refs.add(bytes);
615 }
616 }
617
618
619
620 StringBuilder refsSb = null;
621 String keyString = null;
622 if (defCount == 0 || refs.size() != 1) {
623 refsSb = new StringBuilder();
624 String comma = "";
625 for (byte[] ref : refs) {
626 refsSb.append(comma);
627 comma = ",";
628 refsSb.append(Bytes.toStringBinary(ref));
629 }
630 keyString = Bytes.toStringBinary(key.getBytes(), 0, key.getLength());
631
632 LOG.error("Linked List error: Key = " + keyString + " References = " + refsSb.toString());
633 }
634
635 if (defCount == 0 && refs.size() > 0) {
636
637
638 context.write(new Text(keyString), new Text(refsSb.toString()));
639 context.getCounter(Counts.UNDEFINED).increment(1);
640 if (rows.addAndGet(1) < MISSING_ROWS_TO_LOG) {
641 context.getCounter("undef", keyString).increment(1);
642 }
643 } else if (defCount > 0 && refs.size() == 0) {
644
645 context.write(new Text(keyString), new Text("none"));
646 context.getCounter(Counts.UNREFERENCED).increment(1);
647 if (rows.addAndGet(1) < MISSING_ROWS_TO_LOG) {
648 context.getCounter("unref", keyString).increment(1);
649 }
650 } else {
651 if (refs.size() > 1) {
652 if (refsSb != null) {
653 context.write(new Text(keyString), new Text(refsSb.toString()));
654 }
655 context.getCounter(Counts.EXTRAREFERENCES).increment(refs.size() - 1);
656 }
657
658 context.getCounter(Counts.REFERENCED).increment(1);
659 }
660
661 }
662 }
663
664 @Override
665 public int run(String[] args) throws Exception {
666
667 if (args.length != 2) {
668 System.out.println("Usage : " + Verify.class.getSimpleName() + " <output dir> <num reducers>");
669 return 0;
670 }
671
672 String outputDir = args[0];
673 int numReducers = Integer.parseInt(args[1]);
674
675 return run(outputDir, numReducers);
676 }
677
678 public int run(String outputDir, int numReducers) throws Exception {
679 return run(new Path(outputDir), numReducers);
680 }
681
682 public int run(Path outputDir, int numReducers) throws Exception {
683 LOG.info("Running Verify with outputDir=" + outputDir +", numReducers=" + numReducers);
684
685 job = new Job(getConf());
686
687 job.setJobName("Link Verifier");
688 job.setNumReduceTasks(numReducers);
689 job.setJarByClass(getClass());
690
691 setJobScannerConf(job);
692
693 Scan scan = new Scan();
694 scan.addColumn(FAMILY_NAME, COLUMN_PREV);
695 scan.setCaching(10000);
696 scan.setCacheBlocks(false);
697
698 TableMapReduceUtil.initTableMapperJob(getTableName(getConf()).getName(), scan,
699 VerifyMapper.class, BytesWritable.class, BytesWritable.class, job);
700 TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
701
702 job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false);
703
704 job.setReducerClass(VerifyReducer.class);
705 job.setOutputFormatClass(TextOutputFormat.class);
706 TextOutputFormat.setOutputPath(job, outputDir);
707
708 boolean success = job.waitForCompletion(true);
709
710 return success ? 0 : 1;
711 }
712
713 @SuppressWarnings("deprecation")
714 public boolean verify(long expectedReferenced) throws Exception {
715 if (job == null) {
716 throw new IllegalStateException("You should call run() first");
717 }
718
719 Counters counters = job.getCounters();
720
721 Counter referenced = counters.findCounter(Counts.REFERENCED);
722 Counter unreferenced = counters.findCounter(Counts.UNREFERENCED);
723 Counter undefined = counters.findCounter(Counts.UNDEFINED);
724 Counter multiref = counters.findCounter(Counts.EXTRAREFERENCES);
725
726 boolean success = true;
727
728 if (expectedReferenced != referenced.getValue()) {
729 LOG.error("Expected referenced count does not match with actual referenced count. " +
730 "expected referenced=" + expectedReferenced + " ,actual=" + referenced.getValue());
731 success = false;
732 }
733
734 if (unreferenced.getValue() > 0) {
735 boolean couldBeMultiRef = (multiref.getValue() == unreferenced.getValue());
736 LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue()
737 + (couldBeMultiRef ? "; could be due to duplicate random numbers" : ""));
738 success = false;
739 }
740
741 if (undefined.getValue() > 0) {
742 LOG.error("Found an undefined node. Undefined count=" + undefined.getValue());
743 success = false;
744 }
745
746 if (!success) {
747 handleFailure(counters);
748 }
749 return success;
750 }
751
752 protected void handleFailure(Counters counters) throws IOException {
753 Configuration conf = job.getConfiguration();
754 HConnection conn = HConnectionManager.getConnection(conf);
755 TableName tableName = getTableName(conf);
756 CounterGroup g = counters.getGroup("undef");
757 Iterator<Counter> it = g.iterator();
758 while (it.hasNext()) {
759 String keyString = it.next().getName();
760 byte[] key = Bytes.toBytes(keyString);
761 HRegionLocation loc = conn.relocateRegion(tableName, key);
762 LOG.error("undefined row " + keyString + ", " + loc);
763 }
764 g = counters.getGroup("unref");
765 it = g.iterator();
766 while (it.hasNext()) {
767 String keyString = it.next().getName();
768 byte[] key = Bytes.toBytes(keyString);
769 HRegionLocation loc = conn.relocateRegion(tableName, key);
770 LOG.error("unreferred row " + keyString + ", " + loc);
771 }
772 }
773 }
774
775
776
777
778
779 static class Loop extends Configured implements Tool {
780
781 private static final Log LOG = LogFactory.getLog(Loop.class);
782
783 IntegrationTestBigLinkedList it;
784
785 protected void runGenerator(int numMappers, long numNodes,
786 String outputDir, Integer width, Integer wrapMuplitplier) throws Exception {
787 Path outputPath = new Path(outputDir);
788 UUID uuid = UUID.randomUUID();
789 Path generatorOutput = new Path(outputPath, uuid.toString());
790
791 Generator generator = new Generator();
792 generator.setConf(getConf());
793 int retCode = generator.run(numMappers, numNodes, generatorOutput, width, wrapMuplitplier);
794 if (retCode > 0) {
795 throw new RuntimeException("Generator failed with return code: " + retCode);
796 }
797 }
798
799 protected void runVerify(String outputDir,
800 int numReducers, long expectedNumNodes) throws Exception {
801 Path outputPath = new Path(outputDir);
802 UUID uuid = UUID.randomUUID();
803 Path iterationOutput = new Path(outputPath, uuid.toString());
804
805 Verify verify = new Verify();
806 verify.setConf(getConf());
807 int retCode = verify.run(iterationOutput, numReducers);
808 if (retCode > 0) {
809 throw new RuntimeException("Verify.run failed with return code: " + retCode);
810 }
811
812 if (!verify.verify(expectedNumNodes)) {
813 throw new RuntimeException("Verify.verify failed");
814 }
815
816 LOG.info("Verify finished with succees. Total nodes=" + expectedNumNodes);
817 }
818
819 @Override
820 public int run(String[] args) throws Exception {
821 if (args.length < 5) {
822 System.err.println("Usage: Loop <num iterations> <num mappers> <num nodes per mapper> <output dir> <num reducers> [<width> <wrap multiplier>]");
823 return 1;
824 }
825 LOG.info("Running Loop with args:" + Arrays.deepToString(args));
826
827 int numIterations = Integer.parseInt(args[0]);
828 int numMappers = Integer.parseInt(args[1]);
829 long numNodes = Long.parseLong(args[2]);
830 String outputDir = args[3];
831 int numReducers = Integer.parseInt(args[4]);
832 Integer width = (args.length < 6) ? null : Integer.parseInt(args[5]);
833 Integer wrapMuplitplier = (args.length < 7) ? null : Integer.parseInt(args[6]);
834
835 long expectedNumNodes = 0;
836
837 if (numIterations < 0) {
838 numIterations = Integer.MAX_VALUE;
839 }
840
841 for (int i = 0; i < numIterations; i++) {
842 LOG.info("Starting iteration = " + i);
843 runGenerator(numMappers, numNodes, outputDir, width, wrapMuplitplier);
844 expectedNumNodes += numMappers * numNodes;
845
846 runVerify(outputDir, numReducers, expectedNumNodes);
847 }
848
849 return 0;
850 }
851 }
852
853
854
855
856 private static class Print extends Configured implements Tool {
857 @Override
858 public int run(String[] args) throws Exception {
859 Options options = new Options();
860 options.addOption("s", "start", true, "start key");
861 options.addOption("e", "end", true, "end key");
862 options.addOption("l", "limit", true, "number to print");
863
864 GnuParser parser = new GnuParser();
865 CommandLine cmd = null;
866 try {
867 cmd = parser.parse(options, args);
868 if (cmd.getArgs().length != 0) {
869 throw new ParseException("Command takes no arguments");
870 }
871 } catch (ParseException e) {
872 System.err.println("Failed to parse command line " + e.getMessage());
873 System.err.println();
874 HelpFormatter formatter = new HelpFormatter();
875 formatter.printHelp(getClass().getSimpleName(), options);
876 System.exit(-1);
877 }
878
879 HTable table = new HTable(getConf(), getTableName(getConf()));
880
881 Scan scan = new Scan();
882 scan.setBatch(10000);
883
884 if (cmd.hasOption("s"))
885 scan.setStartRow(Bytes.toBytesBinary(cmd.getOptionValue("s")));
886
887 if (cmd.hasOption("e"))
888 scan.setStopRow(Bytes.toBytesBinary(cmd.getOptionValue("e")));
889
890 int limit = 0;
891 if (cmd.hasOption("l"))
892 limit = Integer.parseInt(cmd.getOptionValue("l"));
893 else
894 limit = 100;
895
896 ResultScanner scanner = table.getScanner(scan);
897
898 CINode node = new CINode();
899 Result result = scanner.next();
900 int count = 0;
901 while (result != null && count++ < limit) {
902 node = getCINode(result, node);
903 System.out.printf("%s:%s:%012d:%s\n", Bytes.toStringBinary(node.key),
904 Bytes.toStringBinary(node.prev), node.count, node.client);
905 result = scanner.next();
906 }
907 scanner.close();
908 table.close();
909
910 return 0;
911 }
912 }
913
914
915
916
917 private static class Delete extends Configured implements Tool {
918 @Override
919 public int run(String[] args) throws Exception {
920 if (args.length != 1) {
921 System.out.println("Usage : " + Delete.class.getSimpleName() + " <node to delete>");
922 return 0;
923 }
924 byte[] val = Bytes.toBytesBinary(args[0]);
925
926 org.apache.hadoop.hbase.client.Delete delete
927 = new org.apache.hadoop.hbase.client.Delete(val);
928
929 HTable table = new HTable(getConf(), getTableName(getConf()));
930
931 table.delete(delete);
932 table.flushCommits();
933 table.close();
934
935 System.out.println("Delete successful");
936 return 0;
937 }
938 }
939
940
941
942
943 private static class Walker extends Configured implements Tool {
944 @Override
945 public int run(String[] args) throws IOException {
946 Options options = new Options();
947 options.addOption("n", "num", true, "number of queries");
948 options.addOption("s", "start", true, "key to start at, binary string");
949 options.addOption("l", "logevery", true, "log every N queries");
950
951 GnuParser parser = new GnuParser();
952 CommandLine cmd = null;
953 try {
954 cmd = parser.parse(options, args);
955 if (cmd.getArgs().length != 0) {
956 throw new ParseException("Command takes no arguments");
957 }
958 } catch (ParseException e) {
959 System.err.println("Failed to parse command line " + e.getMessage());
960 System.err.println();
961 HelpFormatter formatter = new HelpFormatter();
962 formatter.printHelp(getClass().getSimpleName(), options);
963 System.exit(-1);
964 }
965
966 long maxQueries = Long.MAX_VALUE;
967 if (cmd.hasOption('n')) {
968 maxQueries = Long.parseLong(cmd.getOptionValue("n"));
969 }
970 Random rand = new SecureRandom();
971 boolean isSpecificStart = cmd.hasOption('s');
972 byte[] startKey = isSpecificStart ? Bytes.toBytesBinary(cmd.getOptionValue('s')) : null;
973 int logEvery = cmd.hasOption('l') ? Integer.parseInt(cmd.getOptionValue('l')) : 1;
974
975 HTable table = new HTable(getConf(), getTableName(getConf()));
976 long numQueries = 0;
977
978
979
980 while (numQueries < maxQueries && (numQueries == 0 || !isSpecificStart)) {
981 if (!isSpecificStart) {
982 startKey = new byte[ROWKEY_LENGTH];
983 rand.nextBytes(startKey);
984 }
985 CINode node = findStartNode(table, startKey);
986 if (node == null && isSpecificStart) {
987 System.err.printf("Start node not found: %s \n", Bytes.toStringBinary(startKey));
988 }
989 numQueries++;
990 while (node != null && node.prev.length != NO_KEY.length && numQueries < maxQueries) {
991 byte[] prev = node.prev;
992 long t1 = System.currentTimeMillis();
993 node = getNode(prev, table, node);
994 long t2 = System.currentTimeMillis();
995 if (numQueries % logEvery == 0) {
996 System.out.printf("CQ %d: %d %s \n", numQueries, t2 - t1, Bytes.toStringBinary(prev));
997 }
998 numQueries++;
999 if (node == null) {
1000 System.err.printf("UNDEFINED NODE %s \n", Bytes.toStringBinary(prev));
1001 } else if (node.prev.length == NO_KEY.length) {
1002 System.err.printf("TERMINATING NODE %s \n", Bytes.toStringBinary(node.key));
1003 }
1004 }
1005 }
1006
1007 table.close();
1008 return 0;
1009 }
1010
1011 private static CINode findStartNode(HTable table, byte[] startKey) throws IOException {
1012 Scan scan = new Scan();
1013 scan.setStartRow(startKey);
1014 scan.setBatch(1);
1015 scan.addColumn(FAMILY_NAME, COLUMN_PREV);
1016
1017 long t1 = System.currentTimeMillis();
1018 ResultScanner scanner = table.getScanner(scan);
1019 Result result = scanner.next();
1020 long t2 = System.currentTimeMillis();
1021 scanner.close();
1022
1023 if ( result != null) {
1024 CINode node = getCINode(result, new CINode());
1025 System.out.printf("FSR %d %s\n", t2 - t1, Bytes.toStringBinary(node.key));
1026 return node;
1027 }
1028
1029 System.out.println("FSR " + (t2 - t1));
1030
1031 return null;
1032 }
1033
1034 private CINode getNode(byte[] row, HTable table, CINode node) throws IOException {
1035 Get get = new Get(row);
1036 get.addColumn(FAMILY_NAME, COLUMN_PREV);
1037 Result result = table.get(get);
1038 return getCINode(result, node);
1039 }
1040 }
1041
1042 private static class Clean extends Configured implements Tool {
1043
1044 @Override public int run(String[] args) throws Exception {
1045 if (args.length < 1) {
1046 System.err.println("Usage: Clean <output dir>");
1047 return -1;
1048 }
1049
1050 Path p = new Path(args[0]);
1051 Configuration conf = getConf();
1052 TableName tableName = getTableName(conf);
1053
1054 FileSystem fs = HFileSystem.get(conf);
1055 HBaseAdmin admin = new HBaseAdmin(conf);
1056 try {
1057 if (admin.tableExists(tableName)) {
1058 admin.disableTable(tableName);
1059 admin.deleteTable(tableName);
1060 }
1061 } finally {
1062 admin.close();
1063 }
1064
1065 if (fs.exists(p)) {
1066 fs.delete(p, true);
1067 }
1068
1069 return 0;
1070 }
1071 }
1072
1073 static TableName getTableName(Configuration conf) {
1074 return TableName.valueOf(conf.get(TABLE_NAME_KEY, DEFAULT_TABLE_NAME));
1075 }
1076
1077 private static CINode getCINode(Result result, CINode node) {
1078 node.key = Bytes.copy(result.getRow());
1079 if (result.containsColumn(FAMILY_NAME, COLUMN_PREV)) {
1080 node.prev = Bytes.copy(result.getValue(FAMILY_NAME, COLUMN_PREV));
1081 } else {
1082 node.prev = NO_KEY;
1083 }
1084 if (result.containsColumn(FAMILY_NAME, COLUMN_COUNT)) {
1085 node.count = Bytes.toLong(result.getValue(FAMILY_NAME, COLUMN_COUNT));
1086 } else {
1087 node.count = -1;
1088 }
1089 if (result.containsColumn(FAMILY_NAME, COLUMN_CLIENT)) {
1090 node.client = Bytes.toString(result.getValue(FAMILY_NAME, COLUMN_CLIENT));
1091 } else {
1092 node.client = "";
1093 }
1094 return node;
1095 }
1096
1097 protected IntegrationTestingUtility util;
1098
1099 @Override
1100 public void setUpCluster() throws Exception {
1101 util = getTestingUtil(getConf());
1102 boolean isDistributed = util.isDistributedCluster();
1103 util.initializeCluster(isDistributed ? 1 : this.NUM_SLAVES_BASE);
1104 if (!isDistributed) {
1105 util.startMiniMapReduceCluster();
1106 }
1107 this.setConf(util.getConfiguration());
1108 }
1109
1110 @Override
1111 public void cleanUpCluster() throws Exception {
1112 super.cleanUpCluster();
1113 if (util.isDistributedCluster()) {
1114 util.shutdownMiniMapReduceCluster();
1115 }
1116 }
1117
1118 @Test
1119 public void testContinuousIngest() throws IOException, Exception {
1120
1121 int ret = ToolRunner.run(getTestingUtil(getConf()).getConfiguration(), new Loop(),
1122 new String[] {"1", "1", "2000000",
1123 util.getDataTestDirOnTestFS("IntegrationTestBigLinkedList").toString(), "1"});
1124 org.junit.Assert.assertEquals(0, ret);
1125 }
1126
1127 private void usage() {
1128 System.err.println("Usage: " + this.getClass().getSimpleName() + " COMMAND [COMMAND options]");
1129 printCommands();
1130 }
1131
1132 private void printCommands() {
1133 System.err.println("Commands:");
1134 System.err.println(" Generator Map only job that generates data.");
1135 System.err.println(" Verify A map reduce job that looks for holes. Look at the counts ");
1136 System.err.println(" after running. See REFERENCED and UNREFERENCED are ok. Any ");
1137 System.err.println(" UNDEFINED counts are bad. Do not run with the Generator.");
1138 System.err.println(" Walker " +
1139 "Standalong program that starts following a linked list & emits timing info.");
1140 System.err.println(" Print Standalone program that prints nodes in the linked list.");
1141 System.err.println(" Delete Standalone program that deletes a·single node.");
1142 System.err.println(" Loop Program to Loop through Generator and Verify steps");
1143 System.err.println(" Clean Program to clean all left over detritus.");
1144 System.err.flush();
1145 }
1146
1147 @Override
1148 protected void processOptions(CommandLine cmd) {
1149 super.processOptions(cmd);
1150 String[] args = cmd.getArgs();
1151
1152 if (args.length < 1) {
1153 printUsage(this.getClass().getSimpleName() +
1154 " <general options> COMMAND [<COMMAND options>]", "General options:", "");
1155 printCommands();
1156 throw new RuntimeException("Incorrect Number of args.");
1157 }
1158 toRun = args[0];
1159 otherArgs = Arrays.copyOfRange(args, 1, args.length);
1160 }
1161
1162 @Override
1163 public int runTestFromCommandLine() throws Exception {
1164
1165 Tool tool = null;
1166 if (toRun.equals("Generator")) {
1167 tool = new Generator();
1168 } else if (toRun.equalsIgnoreCase("Verify")) {
1169 tool = new Verify();
1170 } else if (toRun.equalsIgnoreCase("Loop")) {
1171 Loop loop = new Loop();
1172 loop.it = this;
1173 tool = loop;
1174 } else if (toRun.equalsIgnoreCase("Walker")) {
1175 tool = new Walker();
1176 } else if (toRun.equalsIgnoreCase("Print")) {
1177 tool = new Print();
1178 } else if (toRun.equalsIgnoreCase("Delete")) {
1179 tool = new Delete();
1180 } else if (toRun.equalsIgnoreCase("Clean")) {
1181 tool = new Clean();
1182 } else {
1183 usage();
1184 throw new RuntimeException("Unknown arg");
1185 }
1186
1187 return ToolRunner.run(getConf(), tool, otherArgs);
1188 }
1189
1190 @Override
1191 public String getTablename() {
1192 Configuration c = getConf();
1193 return c.get(TABLE_NAME_KEY, DEFAULT_TABLE_NAME);
1194 }
1195
1196 @Override
1197 protected Set<String> getColumnFamilies() {
1198 return Sets.newHashSet(Bytes.toString(FAMILY_NAME));
1199 }
1200
1201 private static void setJobConf(Job job, int numMappers, long numNodes,
1202 Integer width, Integer wrapMultiplier) {
1203 job.getConfiguration().setInt(GENERATOR_NUM_MAPPERS_KEY, numMappers);
1204 job.getConfiguration().setLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, numNodes);
1205 if (width != null) {
1206 job.getConfiguration().setInt(GENERATOR_WIDTH_KEY, width);
1207 }
1208 if (wrapMultiplier != null) {
1209 job.getConfiguration().setInt(GENERATOR_WRAP_KEY, wrapMultiplier);
1210 }
1211 }
1212
1213 public static void setJobScannerConf(Job job) {
1214
1215 job.getConfiguration().setBoolean(ScannerCallable.LOG_SCANNER_ACTIVITY, true);
1216 job.getConfiguration().setInt(TableRecordReaderImpl.LOG_PER_ROW_COUNT, 100000);
1217 }
1218
1219 public static void main(String[] args) throws Exception {
1220 Configuration conf = HBaseConfiguration.create();
1221 IntegrationTestingUtility.setUseDistributedCluster(conf);
1222 int ret = ToolRunner.run(conf, new IntegrationTestBigLinkedList(), args);
1223 System.exit(ret);
1224 }
1225 }