View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.mapreduce;
20  
21  import java.io.File;
22  import java.io.IOException;
23  import java.lang.reflect.InvocationTargetException;
24  import java.lang.reflect.Method;
25  import java.net.URL;
26  import java.net.URLDecoder;
27  import java.util.ArrayList;
28  import java.util.Enumeration;
29  import java.util.HashMap;
30  import java.util.HashSet;
31  import java.util.List;
32  import java.util.Map;
33  import java.util.Set;
34  import java.util.zip.ZipEntry;
35  import java.util.zip.ZipFile;
36  
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.hadoop.classification.InterfaceAudience;
40  import org.apache.hadoop.classification.InterfaceStability;
41  import org.apache.hadoop.conf.Configuration;
42  import org.apache.hadoop.fs.FileSystem;
43  import org.apache.hadoop.fs.Path;
44  import org.apache.hadoop.hbase.HBaseConfiguration;
45  import org.apache.hadoop.hbase.catalog.MetaReader;
46  import org.apache.hadoop.hbase.client.Put;
47  import org.apache.hadoop.hbase.client.Scan;
48  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
49  import org.apache.hadoop.hbase.mapreduce.hadoopbackport.JarFinder;
50  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
51  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
52  import org.apache.hadoop.hbase.security.User;
53  import org.apache.hadoop.hbase.security.UserProvider;
54  import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier;
55  import org.apache.hadoop.hbase.security.token.AuthenticationTokenSelector;
56  import org.apache.hadoop.hbase.util.Base64;
57  import org.apache.hadoop.hbase.util.Bytes;
58  import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
59  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
60  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
61  import org.apache.hadoop.io.Text;
62  import org.apache.hadoop.io.Writable;
63  import org.apache.hadoop.io.WritableComparable;
64  import org.apache.hadoop.mapreduce.InputFormat;
65  import org.apache.hadoop.mapreduce.Job;
66  import org.apache.hadoop.security.token.Token;
67  import org.apache.hadoop.util.StringUtils;
68  import org.apache.zookeeper.KeeperException;
69  
70  import com.google.protobuf.InvalidProtocolBufferException;
71  
72  /**
73   * Utility for {@link TableMapper} and {@link TableReducer}
74   */
75  @SuppressWarnings({ "rawtypes", "unchecked" })
76  @InterfaceAudience.Public
77  @InterfaceStability.Stable
78  public class TableMapReduceUtil {
79    static Log LOG = LogFactory.getLog(TableMapReduceUtil.class);
80  
81    /**
82     * Use this before submitting a TableMap job. It will appropriately set up
83     * the job.
84     *
85     * @param table  The table name to read from.
86     * @param scan  The scan instance with the columns, time range etc.
87     * @param mapper  The mapper class to use.
88     * @param outputKeyClass  The class of the output key.
89     * @param outputValueClass  The class of the output value.
90     * @param job  The current job to adjust.  Make sure the passed job is
91     * carrying all necessary HBase configuration.
92     * @throws IOException When setting up the details fails.
93     */
94    public static void initTableMapperJob(String table, Scan scan,
95        Class<? extends TableMapper> mapper,
96        Class<?> outputKeyClass,
97        Class<?> outputValueClass, Job job)
98    throws IOException {
99      initTableMapperJob(table, scan, mapper, outputKeyClass, outputValueClass,
100         job, true);
101   }
102 
103   /**
104    * Use this before submitting a TableMap job. It will appropriately set up
105    * the job.
106    *
107    * @param table Binary representation of the table name to read from.
108    * @param scan  The scan instance with the columns, time range etc.
109    * @param mapper  The mapper class to use.
110    * @param outputKeyClass  The class of the output key.
111    * @param outputValueClass  The class of the output value.
112    * @param job  The current job to adjust.  Make sure the passed job is
113    * carrying all necessary HBase configuration.
114    * @throws IOException When setting up the details fails.
115    */
116    public static void initTableMapperJob(byte[] table, Scan scan,
117       Class<? extends TableMapper> mapper,
118       Class<?> outputKeyClass,
119       Class<?> outputValueClass, Job job)
120   throws IOException {
121       initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass, outputValueClass,
122               job, true);
123   }
124 
125    /**
126     * Use this before submitting a TableMap job. It will appropriately set up
127     * the job.
128     *
129     * @param table  The table name to read from.
130     * @param scan  The scan instance with the columns, time range etc.
131     * @param mapper  The mapper class to use.
132     * @param outputKeyClass  The class of the output key.
133     * @param outputValueClass  The class of the output value.
134     * @param job  The current job to adjust.  Make sure the passed job is
135     * carrying all necessary HBase configuration.
136     * @param addDependencyJars upload HBase jars and jars for any of the configured
137     *           job classes via the distributed cache (tmpjars).
138     * @throws IOException When setting up the details fails.
139     */
140    public static void initTableMapperJob(String table, Scan scan,
141        Class<? extends TableMapper> mapper,
142        Class<?> outputKeyClass,
143        Class<?> outputValueClass, Job job,
144        boolean addDependencyJars, Class<? extends InputFormat> inputFormatClass)
145    throws IOException {
146      initTableMapperJob(table, scan, mapper, outputKeyClass, outputValueClass, job,
147          addDependencyJars, true, inputFormatClass);
148    }
149 
150 
151   /**
152    * Use this before submitting a TableMap job. It will appropriately set up
153    * the job.
154    *
155    * @param table  The table name to read from.
156    * @param scan  The scan instance with the columns, time range etc.
157    * @param mapper  The mapper class to use.
158    * @param outputKeyClass  The class of the output key.
159    * @param outputValueClass  The class of the output value.
160    * @param job  The current job to adjust.  Make sure the passed job is
161    * carrying all necessary HBase configuration.
162    * @param addDependencyJars upload HBase jars and jars for any of the configured
163    *           job classes via the distributed cache (tmpjars).
164    * @param initCredentials whether to initialize hbase auth credentials for the job
165    * @param inputFormatClass the input format
166    * @throws IOException When setting up the details fails.
167    */
168   public static void initTableMapperJob(String table, Scan scan,
169       Class<? extends TableMapper> mapper,
170       Class<?> outputKeyClass,
171       Class<?> outputValueClass, Job job,
172       boolean addDependencyJars, boolean initCredentials,
173       Class<? extends InputFormat> inputFormatClass)
174   throws IOException {
175     job.setInputFormatClass(inputFormatClass);
176     if (outputValueClass != null) job.setMapOutputValueClass(outputValueClass);
177     if (outputKeyClass != null) job.setMapOutputKeyClass(outputKeyClass);
178     job.setMapperClass(mapper);
179     if (Put.class.equals(outputValueClass)) {
180       job.setCombinerClass(PutCombiner.class);
181     }
182     Configuration conf = job.getConfiguration();
183     HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
184     conf.set(TableInputFormat.INPUT_TABLE, table);
185     conf.set(TableInputFormat.SCAN, convertScanToString(scan));
186     conf.setStrings("io.serializations", conf.get("io.serializations"),
187         MutationSerialization.class.getName(), ResultSerialization.class.getName(),
188         KeyValueSerialization.class.getName());
189     if (addDependencyJars) {
190       addDependencyJars(job);
191     }
192     if (initCredentials) {
193       initCredentials(job);
194     }
195   }
196 
197   /**
198    * Use this before submitting a TableMap job. It will appropriately set up
199    * the job.
200    *
201    * @param table Binary representation of the table name to read from.
202    * @param scan  The scan instance with the columns, time range etc.
203    * @param mapper  The mapper class to use.
204    * @param outputKeyClass  The class of the output key.
205    * @param outputValueClass  The class of the output value.
206    * @param job  The current job to adjust.  Make sure the passed job is
207    * carrying all necessary HBase configuration.
208    * @param addDependencyJars upload HBase jars and jars for any of the configured
209    *           job classes via the distributed cache (tmpjars).
210    * @param inputFormatClass The class of the input format
211    * @throws IOException When setting up the details fails.
212    */
213   public static void initTableMapperJob(byte[] table, Scan scan,
214       Class<? extends TableMapper> mapper,
215       Class<?> outputKeyClass,
216       Class<?> outputValueClass, Job job,
217       boolean addDependencyJars, Class<? extends InputFormat> inputFormatClass)
218   throws IOException {
219       initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass,
220               outputValueClass, job, addDependencyJars, inputFormatClass);
221   }
222 
223   /**
224    * Use this before submitting a TableMap job. It will appropriately set up
225    * the job.
226    *
227    * @param table Binary representation of the table name to read from.
228    * @param scan  The scan instance with the columns, time range etc.
229    * @param mapper  The mapper class to use.
230    * @param outputKeyClass  The class of the output key.
231    * @param outputValueClass  The class of the output value.
232    * @param job  The current job to adjust.  Make sure the passed job is
233    * carrying all necessary HBase configuration.
234    * @param addDependencyJars upload HBase jars and jars for any of the configured
235    *           job classes via the distributed cache (tmpjars).
236    * @throws IOException When setting up the details fails.
237    */
238   public static void initTableMapperJob(byte[] table, Scan scan,
239       Class<? extends TableMapper> mapper,
240       Class<?> outputKeyClass,
241       Class<?> outputValueClass, Job job,
242       boolean addDependencyJars)
243   throws IOException {
244       initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass,
245               outputValueClass, job, addDependencyJars, TableInputFormat.class);
246   }
247 
248   /**
249    * Use this before submitting a TableMap job. It will appropriately set up
250    * the job.
251    *
252    * @param table The table name to read from.
253    * @param scan  The scan instance with the columns, time range etc.
254    * @param mapper  The mapper class to use.
255    * @param outputKeyClass  The class of the output key.
256    * @param outputValueClass  The class of the output value.
257    * @param job  The current job to adjust.  Make sure the passed job is
258    * carrying all necessary HBase configuration.
259    * @param addDependencyJars upload HBase jars and jars for any of the configured
260    *           job classes via the distributed cache (tmpjars).
261    * @throws IOException When setting up the details fails.
262    */
263   public static void initTableMapperJob(String table, Scan scan,
264       Class<? extends TableMapper> mapper,
265       Class<?> outputKeyClass,
266       Class<?> outputValueClass, Job job,
267       boolean addDependencyJars)
268   throws IOException {
269       initTableMapperJob(table, scan, mapper, outputKeyClass,
270               outputValueClass, job, addDependencyJars, TableInputFormat.class);
271   }
272 
273   /**
274    * Sets up the job for reading from a table snapshot. It bypasses hbase servers
275    * and read directly from snapshot files.
276    *
277    * @param snapshotName The name of the snapshot (of a table) to read from.
278    * @param scan  The scan instance with the columns, time range etc.
279    * @param mapper  The mapper class to use.
280    * @param outputKeyClass  The class of the output key.
281    * @param outputValueClass  The class of the output value.
282    * @param job  The current job to adjust.  Make sure the passed job is
283    * carrying all necessary HBase configuration.
284    * @param addDependencyJars upload HBase jars and jars for any of the configured
285    *           job classes via the distributed cache (tmpjars).
286    *
287    * @param tmpRestoreDir a temporary directory to copy the snapshot files into. Current user should
288    * have write permissions to this directory, and this should not be a subdirectory of rootdir.
289    * After the job is finished, restore directory can be deleted.
290    * @throws IOException When setting up the details fails.
291    * @see TableSnapshotInputFormat
292    */
293   public static void initTableSnapshotMapperJob(String snapshotName, Scan scan,
294       Class<? extends TableMapper> mapper,
295       Class<?> outputKeyClass,
296       Class<?> outputValueClass, Job job,
297       boolean addDependencyJars, Path tmpRestoreDir)
298   throws IOException {
299     TableSnapshotInputFormat.setInput(job, snapshotName, tmpRestoreDir);
300     initTableMapperJob(snapshotName, scan, mapper, outputKeyClass,
301         outputValueClass, job, addDependencyJars, false, TableSnapshotInputFormat.class);
302   }
303 
304   /**
305    * Use this before submitting a Multi TableMap job. It will appropriately set
306    * up the job.
307    *
308    * @param scans The list of {@link Scan} objects to read from.
309    * @param mapper The mapper class to use.
310    * @param outputKeyClass The class of the output key.
311    * @param outputValueClass The class of the output value.
312    * @param job The current job to adjust. Make sure the passed job is carrying
313    *          all necessary HBase configuration.
314    * @throws IOException When setting up the details fails.
315    */
316   public static void initTableMapperJob(List<Scan> scans,
317       Class<? extends TableMapper> mapper,
318       Class<? extends WritableComparable> outputKeyClass,
319       Class<? extends Writable> outputValueClass, Job job) throws IOException {
320     initTableMapperJob(scans, mapper, outputKeyClass, outputValueClass, job,
321         true);
322   }
323 
324   /**
325    * Use this before submitting a Multi TableMap job. It will appropriately set
326    * up the job.
327    *
328    * @param scans The list of {@link Scan} objects to read from.
329    * @param mapper The mapper class to use.
330    * @param outputKeyClass The class of the output key.
331    * @param outputValueClass The class of the output value.
332    * @param job The current job to adjust. Make sure the passed job is carrying
333    *          all necessary HBase configuration.
334    * @param addDependencyJars upload HBase jars and jars for any of the
335    *          configured job classes via the distributed cache (tmpjars).
336    * @throws IOException When setting up the details fails.
337    */
338   public static void initTableMapperJob(List<Scan> scans,
339       Class<? extends TableMapper> mapper,
340       Class<? extends WritableComparable> outputKeyClass,
341       Class<? extends Writable> outputValueClass, Job job,
342       boolean addDependencyJars) throws IOException {
343     job.setInputFormatClass(MultiTableInputFormat.class);
344     if (outputValueClass != null) {
345       job.setMapOutputValueClass(outputValueClass);
346     }
347     if (outputKeyClass != null) {
348       job.setMapOutputKeyClass(outputKeyClass);
349     }
350     job.setMapperClass(mapper);
351     HBaseConfiguration.addHbaseResources(job.getConfiguration());
352     List<String> scanStrings = new ArrayList<String>();
353 
354     for (Scan scan : scans) {
355       scanStrings.add(convertScanToString(scan));
356     }
357     job.getConfiguration().setStrings(MultiTableInputFormat.SCANS,
358       scanStrings.toArray(new String[scanStrings.size()]));
359 
360     if (addDependencyJars) {
361       addDependencyJars(job);
362     }
363   }
364 
365   public static void initCredentials(Job job) throws IOException {
366     UserProvider userProvider = UserProvider.instantiate(job.getConfiguration());
367     if (userProvider.isHadoopSecurityEnabled()) {
368       // propagate delegation related props from launcher job to MR job
369       if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
370         job.getConfiguration().set("mapreduce.job.credentials.binary",
371                                    System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
372       }
373     }
374 
375     if (userProvider.isHBaseSecurityEnabled()) {
376       try {
377         // init credentials for remote cluster
378         String quorumAddress = job.getConfiguration().get(TableOutputFormat.QUORUM_ADDRESS);
379         User user = userProvider.getCurrent();
380         if (quorumAddress != null) {
381           Configuration peerConf = HBaseConfiguration.create(job.getConfiguration());
382           ZKUtil.applyClusterKeyToConf(peerConf, quorumAddress);
383           obtainAuthTokenForJob(job, peerConf, user);
384         }
385 
386         obtainAuthTokenForJob(job, job.getConfiguration(), user);
387       } catch (InterruptedException ie) {
388         LOG.info("Interrupted obtaining user authentication token");
389         Thread.interrupted();
390       }
391     }
392   }
393 
394   /**
395    * Obtain an authentication token, for the specified cluster, on behalf of the current user
396    * and add it to the credentials for the given map reduce job.
397    *
398    * The quorumAddress is the key to the ZK ensemble, which contains:
399    * hbase.zookeeper.quorum, hbase.zookeeper.client.port and zookeeper.znode.parent
400    *
401    * @param job The job that requires the permission.
402    * @param quorumAddress string that contains the 3 required configuratins
403    * @throws IOException When the authentication token cannot be obtained.
404    */
405   public static void initCredentialsForCluster(Job job, String quorumAddress)
406       throws IOException {
407     UserProvider userProvider = UserProvider.instantiate(job.getConfiguration());
408     if (userProvider.isHBaseSecurityEnabled()) {
409       try {
410         Configuration peerConf = HBaseConfiguration.create(job.getConfiguration());
411         ZKUtil.applyClusterKeyToConf(peerConf, quorumAddress);
412         obtainAuthTokenForJob(job, peerConf, userProvider.getCurrent());
413       } catch (InterruptedException e) {
414         LOG.info("Interrupted obtaining user authentication token");
415         Thread.interrupted();
416       }
417     }
418   }
419 
420   private static void obtainAuthTokenForJob(Job job, Configuration conf, User user)
421       throws IOException, InterruptedException {
422     Token<AuthenticationTokenIdentifier> authToken = getAuthToken(conf, user);
423     if (authToken == null) {
424       user.obtainAuthTokenForJob(conf, job);
425     } else {
426       job.getCredentials().addToken(authToken.getService(), authToken);
427     }
428   }
429 
430   /**
431    * Get the authentication token of the user for the cluster specified in the configuration
432    * @return null if the user does not have the token, otherwise the auth token for the cluster.
433    */
434   private static Token<AuthenticationTokenIdentifier> getAuthToken(Configuration conf, User user)
435       throws IOException, InterruptedException {
436     ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "mr-init-credentials", null);
437     try {
438       String clusterId = ZKClusterId.readClusterIdZNode(zkw);
439       return new AuthenticationTokenSelector().selectToken(new Text(clusterId), user.getUGI().getTokens());
440     } catch (KeeperException e) {
441       throw new IOException(e);
442     } finally {
443       zkw.close();
444     }
445   }
446 
447   /**
448    * Writes the given scan into a Base64 encoded string.
449    *
450    * @param scan  The scan to write out.
451    * @return The scan saved in a Base64 encoded string.
452    * @throws IOException When writing the scan fails.
453    */
454   static String convertScanToString(Scan scan) throws IOException {
455     ClientProtos.Scan proto = ProtobufUtil.toScan(scan);
456     return Base64.encodeBytes(proto.toByteArray());
457   }
458 
459   /**
460    * Converts the given Base64 string back into a Scan instance.
461    *
462    * @param base64  The scan details.
463    * @return The newly created Scan instance.
464    * @throws IOException When reading the scan instance fails.
465    */
466   static Scan convertStringToScan(String base64) throws IOException {
467     byte [] decoded = Base64.decode(base64);
468     ClientProtos.Scan scan;
469     try {
470       scan = ClientProtos.Scan.parseFrom(decoded);
471     } catch (InvalidProtocolBufferException ipbe) {
472       throw new IOException(ipbe);
473     }
474 
475     return ProtobufUtil.toScan(scan);
476   }
477 
478   /**
479    * Use this before submitting a TableReduce job. It will
480    * appropriately set up the JobConf.
481    *
482    * @param table  The output table.
483    * @param reducer  The reducer class to use.
484    * @param job  The current job to adjust.
485    * @throws IOException When determining the region count fails.
486    */
487   public static void initTableReducerJob(String table,
488     Class<? extends TableReducer> reducer, Job job)
489   throws IOException {
490     initTableReducerJob(table, reducer, job, null);
491   }
492 
493   /**
494    * Use this before submitting a TableReduce job. It will
495    * appropriately set up the JobConf.
496    *
497    * @param table  The output table.
498    * @param reducer  The reducer class to use.
499    * @param job  The current job to adjust.
500    * @param partitioner  Partitioner to use. Pass <code>null</code> to use
501    * default partitioner.
502    * @throws IOException When determining the region count fails.
503    */
504   public static void initTableReducerJob(String table,
505     Class<? extends TableReducer> reducer, Job job,
506     Class partitioner) throws IOException {
507     initTableReducerJob(table, reducer, job, partitioner, null, null, null);
508   }
509 
510   /**
511    * Use this before submitting a TableReduce job. It will
512    * appropriately set up the JobConf.
513    *
514    * @param table  The output table.
515    * @param reducer  The reducer class to use.
516    * @param job  The current job to adjust.  Make sure the passed job is
517    * carrying all necessary HBase configuration.
518    * @param partitioner  Partitioner to use. Pass <code>null</code> to use
519    * default partitioner.
520    * @param quorumAddress Distant cluster to write to; default is null for
521    * output to the cluster that is designated in <code>hbase-site.xml</code>.
522    * Set this String to the zookeeper ensemble of an alternate remote cluster
523    * when you would have the reduce write a cluster that is other than the
524    * default; e.g. copying tables between clusters, the source would be
525    * designated by <code>hbase-site.xml</code> and this param would have the
526    * ensemble address of the remote cluster.  The format to pass is particular.
527    * Pass <code> &lt;hbase.zookeeper.quorum>:&lt;hbase.zookeeper.client.port>:&lt;zookeeper.znode.parent>
528    * </code> such as <code>server,server2,server3:2181:/hbase</code>.
529    * @param serverClass redefined hbase.regionserver.class
530    * @param serverImpl redefined hbase.regionserver.impl
531    * @throws IOException When determining the region count fails.
532    */
533   public static void initTableReducerJob(String table,
534     Class<? extends TableReducer> reducer, Job job,
535     Class partitioner, String quorumAddress, String serverClass,
536     String serverImpl) throws IOException {
537     initTableReducerJob(table, reducer, job, partitioner, quorumAddress,
538         serverClass, serverImpl, true);
539   }
540 
541   /**
542    * Use this before submitting a TableReduce job. It will
543    * appropriately set up the JobConf.
544    *
545    * @param table  The output table.
546    * @param reducer  The reducer class to use.
547    * @param job  The current job to adjust.  Make sure the passed job is
548    * carrying all necessary HBase configuration.
549    * @param partitioner  Partitioner to use. Pass <code>null</code> to use
550    * default partitioner.
551    * @param quorumAddress Distant cluster to write to; default is null for
552    * output to the cluster that is designated in <code>hbase-site.xml</code>.
553    * Set this String to the zookeeper ensemble of an alternate remote cluster
554    * when you would have the reduce write a cluster that is other than the
555    * default; e.g. copying tables between clusters, the source would be
556    * designated by <code>hbase-site.xml</code> and this param would have the
557    * ensemble address of the remote cluster.  The format to pass is particular.
558    * Pass <code> &lt;hbase.zookeeper.quorum>:&lt;hbase.zookeeper.client.port>:&lt;zookeeper.znode.parent>
559    * </code> such as <code>server,server2,server3:2181:/hbase</code>.
560    * @param serverClass redefined hbase.regionserver.class
561    * @param serverImpl redefined hbase.regionserver.impl
562    * @param addDependencyJars upload HBase jars and jars for any of the configured
563    *           job classes via the distributed cache (tmpjars).
564    * @throws IOException When determining the region count fails.
565    */
566   public static void initTableReducerJob(String table,
567     Class<? extends TableReducer> reducer, Job job,
568     Class partitioner, String quorumAddress, String serverClass,
569     String serverImpl, boolean addDependencyJars) throws IOException {
570 
571     Configuration conf = job.getConfiguration();
572     HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
573     job.setOutputFormatClass(TableOutputFormat.class);
574     if (reducer != null) job.setReducerClass(reducer);
575     conf.set(TableOutputFormat.OUTPUT_TABLE, table);
576     conf.setStrings("io.serializations", conf.get("io.serializations"),
577         MutationSerialization.class.getName(), ResultSerialization.class.getName());
578     // If passed a quorum/ensemble address, pass it on to TableOutputFormat.
579     if (quorumAddress != null) {
580       // Calling this will validate the format
581       ZKUtil.transformClusterKey(quorumAddress);
582       conf.set(TableOutputFormat.QUORUM_ADDRESS,quorumAddress);
583     }
584     if (serverClass != null && serverImpl != null) {
585       conf.set(TableOutputFormat.REGION_SERVER_CLASS, serverClass);
586       conf.set(TableOutputFormat.REGION_SERVER_IMPL, serverImpl);
587     }
588     job.setOutputKeyClass(ImmutableBytesWritable.class);
589     job.setOutputValueClass(Writable.class);
590     if (partitioner == HRegionPartitioner.class) {
591       job.setPartitionerClass(HRegionPartitioner.class);
592       int regions = MetaReader.getRegionCount(conf, table);
593       if (job.getNumReduceTasks() > regions) {
594         job.setNumReduceTasks(regions);
595       }
596     } else if (partitioner != null) {
597       job.setPartitionerClass(partitioner);
598     }
599 
600     if (addDependencyJars) {
601       addDependencyJars(job);
602     }
603 
604     initCredentials(job);
605   }
606 
607   /**
608    * Ensures that the given number of reduce tasks for the given job
609    * configuration does not exceed the number of regions for the given table.
610    *
611    * @param table  The table to get the region count for.
612    * @param job  The current job to adjust.
613    * @throws IOException When retrieving the table details fails.
614    */
615   public static void limitNumReduceTasks(String table, Job job)
616   throws IOException {
617     int regions = MetaReader.getRegionCount(job.getConfiguration(), table);
618     if (job.getNumReduceTasks() > regions)
619       job.setNumReduceTasks(regions);
620   }
621 
622   /**
623    * Sets the number of reduce tasks for the given job configuration to the
624    * number of regions the given table has.
625    *
626    * @param table  The table to get the region count for.
627    * @param job  The current job to adjust.
628    * @throws IOException When retrieving the table details fails.
629    */
630   public static void setNumReduceTasks(String table, Job job)
631   throws IOException {
632     job.setNumReduceTasks(MetaReader.getRegionCount(job.getConfiguration(), table));
633   }
634 
635   /**
636    * Sets the number of rows to return and cache with each scanner iteration.
637    * Higher caching values will enable faster mapreduce jobs at the expense of
638    * requiring more heap to contain the cached rows.
639    *
640    * @param job The current job to adjust.
641    * @param batchSize The number of rows to return in batch with each scanner
642    * iteration.
643    */
644   public static void setScannerCaching(Job job, int batchSize) {
645     job.getConfiguration().setInt("hbase.client.scanner.caching", batchSize);
646   }
647 
648   /**
649    * Add HBase and its dependencies (only) to the job configuration.
650    * <p>
651    * This is intended as a low-level API, facilitating code reuse between this
652    * class and its mapred counterpart. It also of use to extenral tools that
653    * need to build a MapReduce job that interacts with HBase but want
654    * fine-grained control over the jars shipped to the cluster.
655    * </p>
656    * @param conf The Configuration object to extend with dependencies.
657    * @see org.apache.hadoop.hbase.mapred.TableMapReduceUtil
658    * @see <a href="https://issues.apache.org/jira/browse/PIG-3285">PIG-3285</a>
659    */
660   public static void addHBaseDependencyJars(Configuration conf) throws IOException {
661     addDependencyJars(conf,
662       // explicitly pull a class from each module
663       org.apache.hadoop.hbase.HConstants.class,                      // hbase-common
664       org.apache.hadoop.hbase.protobuf.generated.ClientProtos.class, // hbase-protocol
665       org.apache.hadoop.hbase.client.Put.class,                      // hbase-client
666       org.apache.hadoop.hbase.CompatibilityFactory.class,            // hbase-hadoop-compat
667       org.apache.hadoop.hbase.mapreduce.TableMapper.class,           // hbase-server
668       // pull necessary dependencies
669       org.apache.zookeeper.ZooKeeper.class,
670       org.jboss.netty.channel.ChannelFactory.class,
671       com.google.protobuf.Message.class,
672       com.google.common.collect.Lists.class,
673       org.cloudera.htrace.Trace.class);
674   }
675 
676   /**
677    * Returns a classpath string built from the content of the "tmpjars" value in {@code conf}.
678    * Also exposed to shell scripts via `bin/hbase mapredcp`.
679    */
680   public static String buildDependencyClasspath(Configuration conf) {
681     if (conf == null) {
682       throw new IllegalArgumentException("Must provide a configuration object.");
683     }
684     Set<String> paths = new HashSet<String>(conf.getStringCollection("tmpjars"));
685     if (paths.size() == 0) {
686       throw new IllegalArgumentException("Configuration contains no tmpjars.");
687     }
688     StringBuilder sb = new StringBuilder();
689     for (String s : paths) {
690       // entries can take the form 'file:/path/to/file.jar'.
691       int idx = s.indexOf(":");
692       if (idx != -1) s = s.substring(idx + 1);
693       if (sb.length() > 0) sb.append(File.pathSeparator);
694       sb.append(s);
695     }
696     return sb.toString();
697   }
698 
699   /**
700    * Add the HBase dependency jars as well as jars for any of the configured
701    * job classes to the job configuration, so that JobClient will ship them
702    * to the cluster and add them to the DistributedCache.
703    */
704   public static void addDependencyJars(Job job) throws IOException {
705     addHBaseDependencyJars(job.getConfiguration());
706     try {
707       addDependencyJars(job.getConfiguration(),
708           // when making changes here, consider also mapred.TableMapReduceUtil
709           // pull job classes
710           job.getMapOutputKeyClass(),
711           job.getMapOutputValueClass(),
712           job.getInputFormatClass(),
713           job.getOutputKeyClass(),
714           job.getOutputValueClass(),
715           job.getOutputFormatClass(),
716           job.getPartitionerClass(),
717           job.getCombinerClass());
718     } catch (ClassNotFoundException e) {
719       throw new IOException(e);
720     }
721   }
722 
723   /**
724    * Add the jars containing the given classes to the job's configuration
725    * such that JobClient will ship them to the cluster and add them to
726    * the DistributedCache.
727    */
728   public static void addDependencyJars(Configuration conf,
729       Class<?>... classes) throws IOException {
730 
731     FileSystem localFs = FileSystem.getLocal(conf);
732     Set<String> jars = new HashSet<String>();
733     // Add jars that are already in the tmpjars variable
734     jars.addAll(conf.getStringCollection("tmpjars"));
735 
736     // add jars as we find them to a map of contents jar name so that we can avoid
737     // creating new jars for classes that have already been packaged.
738     Map<String, String> packagedClasses = new HashMap<String, String>();
739 
740     // Add jars containing the specified classes
741     for (Class<?> clazz : classes) {
742       if (clazz == null) continue;
743 
744       Path path = findOrCreateJar(clazz, localFs, packagedClasses);
745       if (path == null) {
746         LOG.warn("Could not find jar for class " + clazz +
747                  " in order to ship it to the cluster.");
748         continue;
749       }
750       if (!localFs.exists(path)) {
751         LOG.warn("Could not validate jar file " + path + " for class "
752                  + clazz);
753         continue;
754       }
755       jars.add(path.toString());
756     }
757     if (jars.isEmpty()) return;
758 
759     conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[jars.size()])));
760   }
761 
762   /**
763    * If org.apache.hadoop.util.JarFinder is available (0.23+ hadoop), finds
764    * the Jar for a class or creates it if it doesn't exist. If the class is in
765    * a directory in the classpath, it creates a Jar on the fly with the
766    * contents of the directory and returns the path to that Jar. If a Jar is
767    * created, it is created in the system temporary directory. Otherwise,
768    * returns an existing jar that contains a class of the same name. Maintains
769    * a mapping from jar contents to the tmp jar created.
770    * @param my_class the class to find.
771    * @param fs the FileSystem with which to qualify the returned path.
772    * @param packagedClasses a map of class name to path.
773    * @return a jar file that contains the class.
774    * @throws IOException
775    */
776   private static Path findOrCreateJar(Class<?> my_class, FileSystem fs,
777       Map<String, String> packagedClasses)
778   throws IOException {
779     // attempt to locate an existing jar for the class.
780     String jar = findContainingJar(my_class, packagedClasses);
781     if (null == jar || jar.isEmpty()) {
782       jar = getJar(my_class);
783       updateMap(jar, packagedClasses);
784     }
785 
786     if (null == jar || jar.isEmpty()) {
787       return null;
788     }
789 
790     LOG.debug(String.format("For class %s, using jar %s", my_class.getName(), jar));
791     return new Path(jar).makeQualified(fs);
792   }
793 
794   /**
795    * Add entries to <code>packagedClasses</code> corresponding to class files
796    * contained in <code>jar</code>.
797    * @param jar The jar who's content to list.
798    * @param packagedClasses map[class -> jar]
799    */
800   private static void updateMap(String jar, Map<String, String> packagedClasses) throws IOException {
801     if (null == jar || jar.isEmpty()) {
802       return;
803     }
804     ZipFile zip = null;
805     try {
806       zip = new ZipFile(jar);
807       for (Enumeration<? extends ZipEntry> iter = zip.entries(); iter.hasMoreElements();) {
808         ZipEntry entry = iter.nextElement();
809         if (entry.getName().endsWith("class")) {
810           packagedClasses.put(entry.getName(), jar);
811         }
812       }
813     } finally {
814       if (null != zip) zip.close();
815     }
816   }
817 
818   /**
819    * Find a jar that contains a class of the same name, if any. It will return
820    * a jar file, even if that is not the first thing on the class path that
821    * has a class with the same name. Looks first on the classpath and then in
822    * the <code>packagedClasses</code> map.
823    * @param my_class the class to find.
824    * @return a jar file that contains the class, or null.
825    * @throws IOException
826    */
827   private static String findContainingJar(Class<?> my_class, Map<String, String> packagedClasses)
828       throws IOException {
829     ClassLoader loader = my_class.getClassLoader();
830     String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
831 
832     // first search the classpath
833     for (Enumeration<URL> itr = loader.getResources(class_file); itr.hasMoreElements();) {
834       URL url = itr.nextElement();
835       if ("jar".equals(url.getProtocol())) {
836         String toReturn = url.getPath();
837         if (toReturn.startsWith("file:")) {
838           toReturn = toReturn.substring("file:".length());
839         }
840         // URLDecoder is a misnamed class, since it actually decodes
841         // x-www-form-urlencoded MIME type rather than actual
842         // URL encoding (which the file path has). Therefore it would
843         // decode +s to ' 's which is incorrect (spaces are actually
844         // either unencoded or encoded as "%20"). Replace +s first, so
845         // that they are kept sacred during the decoding process.
846         toReturn = toReturn.replaceAll("\\+", "%2B");
847         toReturn = URLDecoder.decode(toReturn, "UTF-8");
848         return toReturn.replaceAll("!.*$", "");
849       }
850     }
851 
852     // now look in any jars we've packaged using JarFinder. Returns null when
853     // no jar is found.
854     return packagedClasses.get(class_file);
855   }
856 
857   /**
858    * Invoke 'getJar' on a JarFinder implementation. Useful for some job
859    * configuration contexts (HBASE-8140) and also for testing on MRv2. First
860    * check if we have HADOOP-9426. Lacking that, fall back to the backport.
861    * @param my_class the class to find.
862    * @return a jar file that contains the class, or null.
863    */
864   private static String getJar(Class<?> my_class) {
865     String ret = null;
866     String hadoopJarFinder = "org.apache.hadoop.util.JarFinder";
867     Class<?> jarFinder = null;
868     try {
869       LOG.debug("Looking for " + hadoopJarFinder + ".");
870       jarFinder = Class.forName(hadoopJarFinder);
871       LOG.debug(hadoopJarFinder + " found.");
872       Method getJar = jarFinder.getMethod("getJar", Class.class);
873       ret = (String) getJar.invoke(null, my_class);
874     } catch (ClassNotFoundException e) {
875       LOG.debug("Using backported JarFinder.");
876       ret = JarFinder.getJar(my_class);
877     } catch (InvocationTargetException e) {
878       // function was properly called, but threw it's own exception. Unwrap it
879       // and pass it on.
880       throw new RuntimeException(e.getCause());
881     } catch (Exception e) {
882       // toss all other exceptions, related to reflection failure
883       throw new RuntimeException("getJar invocation failed.", e);
884     }
885 
886     return ret;
887   }
888 }