1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.mapred;
21
22 import java.io.IOException;
23
24 import org.apache.hadoop.hbase.HBaseConfiguration;
25 import org.apache.hadoop.hbase.client.HTable;
26 import org.apache.hadoop.hbase.client.Put;
27 import org.apache.hadoop.hbase.client.UserProvider;
28 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
29 import org.apache.hadoop.hbase.security.User;
30 import org.apache.hadoop.io.Writable;
31 import org.apache.hadoop.io.WritableComparable;
32 import org.apache.hadoop.mapred.FileInputFormat;
33 import org.apache.hadoop.mapred.JobConf;
34 import org.apache.hadoop.mapred.InputFormat;
35 import org.apache.hadoop.mapred.OutputFormat;
36 import org.apache.hadoop.mapred.TextInputFormat;
37 import org.apache.hadoop.mapred.TextOutputFormat;
38
39
40
41
42 @Deprecated
43 @SuppressWarnings("unchecked")
44 public class TableMapReduceUtil {
45
46
47
48
49
50
51
52
53
54
55
56
57 public static void initTableMapJob(String table, String columns,
58 Class<? extends TableMap> mapper,
59 Class<? extends WritableComparable> outputKeyClass,
60 Class<? extends Writable> outputValueClass, JobConf job) {
61 initTableMapJob(table, columns, mapper, outputKeyClass, outputValueClass, job, true);
62 }
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77 public static void initTableMapJob(String table, String columns,
78 Class<? extends TableMap> mapper,
79 Class<? extends WritableComparable> outputKeyClass,
80 Class<? extends Writable> outputValueClass, JobConf job, boolean addDependencyJars) {
81
82 job.setInputFormat(TableInputFormat.class);
83 job.setMapOutputValueClass(outputValueClass);
84 job.setMapOutputKeyClass(outputKeyClass);
85 job.setMapperClass(mapper);
86 FileInputFormat.addInputPaths(job, table);
87 job.set(TableInputFormat.COLUMN_LIST, columns);
88 if (addDependencyJars) {
89 try {
90 addDependencyJars(job);
91 } catch (IOException e) {
92 e.printStackTrace();
93 }
94 }
95 try {
96 initCredentials(job);
97 } catch (IOException ioe) {
98
99 ioe.printStackTrace();
100 }
101 }
102
103
104
105
106
107
108
109
110
111
112 public static void initTableReduceJob(String table,
113 Class<? extends TableReduce> reducer, JobConf job)
114 throws IOException {
115 initTableReduceJob(table, reducer, job, null);
116 }
117
118
119
120
121
122
123
124
125
126
127
128
129 public static void initTableReduceJob(String table,
130 Class<? extends TableReduce> reducer, JobConf job, Class partitioner)
131 throws IOException {
132 initTableReduceJob(table, reducer, job, partitioner, true);
133 }
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148 public static void initTableReduceJob(String table,
149 Class<? extends TableReduce> reducer, JobConf job, Class partitioner,
150 boolean addDependencyJars) throws IOException {
151 job.setOutputFormat(TableOutputFormat.class);
152 job.setReducerClass(reducer);
153 job.set(TableOutputFormat.OUTPUT_TABLE, table);
154 job.setOutputKeyClass(ImmutableBytesWritable.class);
155 job.setOutputValueClass(Put.class);
156 if (partitioner == HRegionPartitioner.class) {
157 job.setPartitionerClass(HRegionPartitioner.class);
158 HTable outputTable = new HTable(HBaseConfiguration.create(job), table);
159 int regions = outputTable.getRegionsInfo().size();
160 if (job.getNumReduceTasks() > regions) {
161 job.setNumReduceTasks(outputTable.getRegionsInfo().size());
162 }
163 } else if (partitioner != null) {
164 job.setPartitionerClass(partitioner);
165 }
166 if (addDependencyJars) {
167 addDependencyJars(job);
168 }
169 initCredentials(job);
170 }
171
172 public static void initCredentials(JobConf job) throws IOException {
173 UserProvider provider = UserProvider.instantiate(job);
174 if (provider.isHBaseSecurityEnabled()) {
175 try {
176 provider.getCurrent().obtainAuthTokenForJob(job);
177 } catch (InterruptedException ie) {
178 ie.printStackTrace();
179 Thread.interrupted();
180 }
181 }
182 }
183
184
185
186
187
188
189
190
191
192 public static void limitNumReduceTasks(String table, JobConf job)
193 throws IOException {
194 HTable outputTable = new HTable(HBaseConfiguration.create(job), table);
195 int regions = outputTable.getRegionsInfo().size();
196 if (job.getNumReduceTasks() > regions)
197 job.setNumReduceTasks(regions);
198 }
199
200
201
202
203
204
205
206
207
208 public static void limitNumMapTasks(String table, JobConf job)
209 throws IOException {
210 HTable outputTable = new HTable(HBaseConfiguration.create(job), table);
211 int regions = outputTable.getRegionsInfo().size();
212 if (job.getNumMapTasks() > regions)
213 job.setNumMapTasks(regions);
214 }
215
216
217
218
219
220
221
222
223
224 public static void setNumReduceTasks(String table, JobConf job)
225 throws IOException {
226 HTable outputTable = new HTable(HBaseConfiguration.create(job), table);
227 int regions = outputTable.getRegionsInfo().size();
228 job.setNumReduceTasks(regions);
229 }
230
231
232
233
234
235
236
237
238
239 public static void setNumMapTasks(String table, JobConf job)
240 throws IOException {
241 HTable outputTable = new HTable(HBaseConfiguration.create(job), table);
242 int regions = outputTable.getRegionsInfo().size();
243 job.setNumMapTasks(regions);
244 }
245
246
247
248
249
250
251
252
253
254
255 public static void setScannerCaching(JobConf job, int batchSize) {
256 job.setInt("hbase.client.scanner.caching", batchSize);
257 }
258
259
260
261
262 public static void addDependencyJars(JobConf job) throws IOException {
263 org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(
264 job,
265 org.apache.zookeeper.ZooKeeper.class,
266 com.google.common.base.Function.class,
267 com.google.protobuf.Message.class,
268 job.getMapOutputKeyClass(),
269 job.getMapOutputValueClass(),
270 job.getOutputKeyClass(),
271 job.getOutputValueClass(),
272 job.getPartitionerClass(),
273 job.getClass("mapred.input.format.class", TextInputFormat.class, InputFormat.class),
274 job.getClass("mapred.output.format.class", TextOutputFormat.class, OutputFormat.class),
275 job.getCombinerClass());
276 }
277 }