1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mapreduce;
20
21 import java.io.IOException;
22 import java.util.Set;
23 import java.util.TreeSet;
24
25 import org.apache.hadoop.classification.InterfaceAudience;
26 import org.apache.hadoop.classification.InterfaceStability;
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.hbase.HBaseConfiguration;
29 import org.apache.hadoop.hbase.client.Result;
30 import org.apache.hadoop.hbase.client.Scan;
31 import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
32 import org.apache.hadoop.hbase.filter.FirstKeyValueMatchingQualifiersFilter;
33 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
34 import org.apache.hadoop.hbase.util.Bytes;
35 import org.apache.hadoop.mapreduce.Job;
36 import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
37 import org.apache.hadoop.util.GenericOptionsParser;
38
39
40
41
42
43 @InterfaceAudience.Public
44 @InterfaceStability.Stable
45 public class RowCounter {
46
47
48 static final String NAME = "rowcounter";
49
50
51
52
53 static class RowCounterMapper
54 extends TableMapper<ImmutableBytesWritable, Result> {
55
56
57 public static enum Counters {ROWS}
58
59
60
61
62
63
64
65
66
67
68
69 @Override
70 public void map(ImmutableBytesWritable row, Result values,
71 Context context)
72 throws IOException {
73
74 context.getCounter(Counters.ROWS).increment(1);
75 }
76 }
77
78
79
80
81
82
83
84
85
86 public static Job createSubmittableJob(Configuration conf, String[] args)
87 throws IOException {
88 String tableName = args[0];
89 String startKey = null;
90 String endKey = null;
91 StringBuilder sb = new StringBuilder();
92
93 final String rangeSwitch = "--range=";
94
95
96 for (int i = 1; i < args.length; i++) {
97 if (args[i].startsWith(rangeSwitch)) {
98 String[] startEnd = args[i].substring(rangeSwitch.length()).split(",", 2);
99 if (startEnd.length != 2 || startEnd[1].contains(",")) {
100 printUsage("Please specify range in such format as \"--range=a,b\" " +
101 "or, with only one boundary, \"--range=,b\" or \"--range=a,\"");
102 return null;
103 }
104 startKey = startEnd[0];
105 endKey = startEnd[1];
106 }
107 else {
108
109 sb.append(args[i]);
110 sb.append(" ");
111 }
112 }
113
114 Job job = new Job(conf, NAME + "_" + tableName);
115 job.setJarByClass(RowCounter.class);
116 Scan scan = new Scan();
117 scan.setCacheBlocks(false);
118 Set<byte []> qualifiers = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
119 if (startKey != null && !startKey.equals("")) {
120 scan.setStartRow(Bytes.toBytes(startKey));
121 }
122 if (endKey != null && !endKey.equals("")) {
123 scan.setStopRow(Bytes.toBytes(endKey));
124 }
125 scan.setFilter(new FirstKeyOnlyFilter());
126 if (sb.length() > 0) {
127 for (String columnName : sb.toString().trim().split(" ")) {
128 String [] fields = columnName.split(":");
129 if(fields.length == 1) {
130 scan.addFamily(Bytes.toBytes(fields[0]));
131 } else {
132 byte[] qualifier = Bytes.toBytes(fields[1]);
133 qualifiers.add(qualifier);
134 scan.addColumn(Bytes.toBytes(fields[0]), qualifier);
135 }
136 }
137 }
138
139
140
141 if (qualifiers.size() == 0) {
142 scan.setFilter(new FirstKeyOnlyFilter());
143 } else {
144 scan.setFilter(new FirstKeyValueMatchingQualifiersFilter(qualifiers));
145 }
146 job.setOutputFormatClass(NullOutputFormat.class);
147 TableMapReduceUtil.initTableMapperJob(tableName, scan,
148 RowCounterMapper.class, ImmutableBytesWritable.class, Result.class, job);
149 job.setNumReduceTasks(0);
150 return job;
151 }
152
153
154
155
156 private static void printUsage(String errorMessage) {
157 System.err.println("ERROR: " + errorMessage);
158 printUsage();
159 }
160
161
162
163
164 private static void printUsage() {
165 System.err.println("Usage: RowCounter [options] <tablename> " +
166 "[--range=[startKey],[endKey]] [<column1> <column2>...]");
167 System.err.println("For performance consider the following options:\n"
168 + "-Dhbase.client.scanner.caching=100\n"
169 + "-Dmapred.map.tasks.speculative.execution=false");
170 }
171
172
173
174
175
176
177
178 public static void main(String[] args) throws Exception {
179 Configuration conf = HBaseConfiguration.create();
180 String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
181 if (otherArgs.length < 1) {
182 printUsage("Wrong number of parameters: " + args.length);
183 System.exit(-1);
184 }
185 Job job = createSubmittableJob(conf, otherArgs);
186 if (job == null) {
187 System.exit(-1);
188 }
189 System.exit(job.waitForCompletion(true) ? 0 : 1);
190 }
191 }