View Javadoc

1   /**
2    * Copyright 2008 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.mapreduce;
21  
22  import java.io.ByteArrayInputStream;
23  import java.io.ByteArrayOutputStream;
24  import java.io.DataInputStream;
25  import java.io.DataOutputStream;
26  import java.io.File;
27  import java.io.IOException;
28  import java.lang.reflect.InvocationTargetException;
29  import java.lang.reflect.Method;
30  import java.net.URL;
31  import java.net.URLDecoder;
32  import java.util.ArrayList;
33  import java.util.Enumeration;
34  import java.util.HashMap;
35  import java.util.HashSet;
36  import java.util.List;
37  import java.util.Map;
38  import java.util.Set;
39  import java.util.zip.ZipEntry;
40  import java.util.zip.ZipFile;
41  
42  import org.apache.commons.logging.Log;
43  import org.apache.commons.logging.LogFactory;
44  import org.apache.hadoop.conf.Configuration;
45  import org.apache.hadoop.fs.FileSystem;
46  import org.apache.hadoop.fs.Path;
47  import org.apache.hadoop.hbase.HBaseConfiguration;
48  import org.apache.hadoop.hbase.HConstants;
49  import org.apache.hadoop.hbase.client.HTable;
50  import org.apache.hadoop.hbase.client.Scan;
51  import org.apache.hadoop.hbase.client.UserProvider;
52  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
53  import org.apache.hadoop.hbase.mapreduce.hadoopbackport.JarFinder;
54  import org.apache.hadoop.hbase.security.User;
55  import org.apache.hadoop.hbase.util.Base64;
56  import org.apache.hadoop.hbase.util.Bytes;
57  import org.apache.hadoop.hbase.zookeeper.ClusterId;
58  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
59  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
60  import org.apache.hadoop.io.Writable;
61  import org.apache.hadoop.io.WritableComparable;
62  import org.apache.hadoop.mapreduce.InputFormat;
63  import org.apache.hadoop.mapreduce.Job;
64  import org.apache.hadoop.util.StringUtils;
65  import org.apache.hadoop.security.token.Token;
66  import org.apache.zookeeper.KeeperException;
67  
68  /**
69   * Utility for {@link TableMapper} and {@link TableReducer}
70   */
71  @SuppressWarnings("unchecked")
72  public class TableMapReduceUtil {
73    static Log LOG = LogFactory.getLog(TableMapReduceUtil.class);
74    
75    /**
76     * Use this before submitting a TableMap job. It will appropriately set up
77     * the job.
78     *
79     * @param table  The table name to read from.
80     * @param scan  The scan instance with the columns, time range etc.
81     * @param mapper  The mapper class to use.
82     * @param outputKeyClass  The class of the output key.
83     * @param outputValueClass  The class of the output value.
84     * @param job  The current job to adjust.  Make sure the passed job is
85     * carrying all necessary HBase configuration.
86     * @throws IOException When setting up the details fails.
87     */
88    public static void initTableMapperJob(String table, Scan scan,
89        Class<? extends TableMapper> mapper,
90        Class<? extends WritableComparable> outputKeyClass,
91        Class<? extends Writable> outputValueClass, Job job)
92    throws IOException {
93      initTableMapperJob(table, scan, mapper, outputKeyClass, outputValueClass,
94          job, true);
95    }
96  
97  
98    /**
99     * Use this before submitting a TableMap job. It will appropriately set up
100    * the job.
101    *
102    * @param table Binary representation of the table name to read from.
103    * @param scan  The scan instance with the columns, time range etc.
104    * @param mapper  The mapper class to use.
105    * @param outputKeyClass  The class of the output key.
106    * @param outputValueClass  The class of the output value.
107    * @param job  The current job to adjust.  Make sure the passed job is
108    * carrying all necessary HBase configuration.
109    * @throws IOException When setting up the details fails.
110    */
111    public static void initTableMapperJob(byte[] table, Scan scan,
112       Class<? extends TableMapper> mapper,
113       Class<? extends WritableComparable> outputKeyClass,
114       Class<? extends Writable> outputValueClass, Job job)
115   throws IOException {
116       initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass, outputValueClass,
117               job, true);
118   }
119 
120   /**
121    * Use this before submitting a TableMap job. It will appropriately set up
122    * the job.
123    *
124    * @param table  The table name to read from.
125    * @param scan  The scan instance with the columns, time range etc.
126    * @param mapper  The mapper class to use.
127    * @param outputKeyClass  The class of the output key.
128    * @param outputValueClass  The class of the output value.
129    * @param job  The current job to adjust.  Make sure the passed job is
130    * carrying all necessary HBase configuration.
131    * @param addDependencyJars upload HBase jars and jars for any of the configured
132    *           job classes via the distributed cache (tmpjars).
133    * @throws IOException When setting up the details fails.
134    */
135   public static void initTableMapperJob(String table, Scan scan,
136       Class<? extends TableMapper> mapper,
137       Class<? extends WritableComparable> outputKeyClass,
138       Class<? extends Writable> outputValueClass, Job job,
139       boolean addDependencyJars, Class<? extends InputFormat> inputFormatClass)
140   throws IOException {
141     job.setInputFormatClass(inputFormatClass);
142     if (outputValueClass != null) job.setMapOutputValueClass(outputValueClass);
143     if (outputKeyClass != null) job.setMapOutputKeyClass(outputKeyClass);
144     job.setMapperClass(mapper);
145     Configuration conf = job.getConfiguration();
146     HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
147     conf.set(TableInputFormat.INPUT_TABLE, table);
148     conf.set(TableInputFormat.SCAN, convertScanToString(scan));
149     if (addDependencyJars) {
150       addDependencyJars(job);
151     }
152     initCredentials(job);
153   }
154   
155   /**
156    * Use this before submitting a TableMap job. It will appropriately set up
157    * the job.
158    *
159    * @param table Binary representation of the table name to read from.
160    * @param scan  The scan instance with the columns, time range etc.
161    * @param mapper  The mapper class to use.
162    * @param outputKeyClass  The class of the output key.
163    * @param outputValueClass  The class of the output value.
164    * @param job  The current job to adjust.  Make sure the passed job is
165    * carrying all necessary HBase configuration.
166    * @param addDependencyJars upload HBase jars and jars for any of the configured
167    *           job classes via the distributed cache (tmpjars).
168    * @param inputFormatClass The class of the input format
169    * @throws IOException When setting up the details fails.
170    */
171   public static void initTableMapperJob(byte[] table, Scan scan,
172       Class<? extends TableMapper> mapper,
173       Class<? extends WritableComparable> outputKeyClass,
174       Class<? extends Writable> outputValueClass, Job job,
175       boolean addDependencyJars, Class<? extends InputFormat> inputFormatClass)
176   throws IOException {
177       initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass,
178               outputValueClass, job, addDependencyJars, inputFormatClass);
179   }
180   
181   /**
182    * Use this before submitting a TableMap job. It will appropriately set up
183    * the job.
184    *
185    * @param table Binary representation of the table name to read from.
186    * @param scan  The scan instance with the columns, time range etc.
187    * @param mapper  The mapper class to use.
188    * @param outputKeyClass  The class of the output key.
189    * @param outputValueClass  The class of the output value.
190    * @param job  The current job to adjust.  Make sure the passed job is
191    * carrying all necessary HBase configuration.
192    * @param addDependencyJars upload HBase jars and jars for any of the configured
193    *           job classes via the distributed cache (tmpjars).
194    * @throws IOException When setting up the details fails.
195    */
196   public static void initTableMapperJob(byte[] table, Scan scan,
197       Class<? extends TableMapper> mapper,
198       Class<? extends WritableComparable> outputKeyClass,
199       Class<? extends Writable> outputValueClass, Job job,
200       boolean addDependencyJars)
201   throws IOException {
202       initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass,
203               outputValueClass, job, addDependencyJars, TableInputFormat.class);
204   }
205   
206   /**
207    * Use this before submitting a TableMap job. It will appropriately set up
208    * the job.
209    *
210    * @param table The table name to read from.
211    * @param scan  The scan instance with the columns, time range etc.
212    * @param mapper  The mapper class to use.
213    * @param outputKeyClass  The class of the output key.
214    * @param outputValueClass  The class of the output value.
215    * @param job  The current job to adjust.  Make sure the passed job is
216    * carrying all necessary HBase configuration.
217    * @param addDependencyJars upload HBase jars and jars for any of the configured
218    *           job classes via the distributed cache (tmpjars).
219    * @throws IOException When setting up the details fails.
220    */
221   public static void initTableMapperJob(String table, Scan scan,
222       Class<? extends TableMapper> mapper,
223       Class<? extends WritableComparable> outputKeyClass,
224       Class<? extends Writable> outputValueClass, Job job,
225       boolean addDependencyJars)
226   throws IOException {
227       initTableMapperJob(table, scan, mapper, outputKeyClass,
228               outputValueClass, job, addDependencyJars, TableInputFormat.class);
229   }
230   
231   /**
232    * Use this before submitting a Multi TableMap job. It will appropriately set
233    * up the job.
234    *
235    * @param scans The list of {@link Scan} objects to read from.
236    * @param mapper The mapper class to use.
237    * @param outputKeyClass The class of the output key.
238    * @param outputValueClass The class of the output value.
239    * @param job The current job to adjust. Make sure the passed job is carrying
240    *          all necessary HBase configuration.
241    * @throws IOException When setting up the details fails.
242    */
243   public static void initTableMapperJob(List<Scan> scans,
244       Class<? extends TableMapper> mapper,
245       Class<? extends WritableComparable> outputKeyClass,
246       Class<? extends Writable> outputValueClass, Job job) throws IOException {
247     initTableMapperJob(scans, mapper, outputKeyClass, outputValueClass, job,
248         true);
249   }
250 
251   /**
252    * Use this before submitting a Multi TableMap job. It will appropriately set
253    * up the job.
254    *
255    * @param scans The list of {@link Scan} objects to read from.
256    * @param mapper The mapper class to use.
257    * @param outputKeyClass The class of the output key.
258    * @param outputValueClass The class of the output value.
259    * @param job The current job to adjust. Make sure the passed job is carrying
260    *          all necessary HBase configuration.
261    * @param addDependencyJars upload HBase jars and jars for any of the
262    *          configured job classes via the distributed cache (tmpjars).
263    * @throws IOException When setting up the details fails.
264    */
265   public static void initTableMapperJob(List<Scan> scans,
266       Class<? extends TableMapper> mapper,
267       Class<? extends WritableComparable> outputKeyClass,
268       Class<? extends Writable> outputValueClass, Job job,
269       boolean addDependencyJars) throws IOException {
270     job.setInputFormatClass(MultiTableInputFormat.class);
271     if (outputValueClass != null) {
272       job.setMapOutputValueClass(outputValueClass);
273     }
274     if (outputKeyClass != null) {
275       job.setMapOutputKeyClass(outputKeyClass);
276     }
277     job.setMapperClass(mapper);
278     HBaseConfiguration.addHbaseResources(job.getConfiguration());
279     List<String> scanStrings = new ArrayList<String>();
280 
281     for (Scan scan : scans) {
282       scanStrings.add(convertScanToString(scan));
283     }
284     job.getConfiguration().setStrings(MultiTableInputFormat.SCANS,
285       scanStrings.toArray(new String[scanStrings.size()]));
286 
287     if (addDependencyJars) {
288       addDependencyJars(job);
289     }
290   }
291 
292   public static void initCredentials(Job job) throws IOException {
293     UserProvider provider = UserProvider.instantiate(job.getConfiguration());
294 
295     if (provider.isHadoopSecurityEnabled()) {
296       // propagate delegation related props from launcher job to MR job
297       if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
298         job.getConfiguration().set("mapreduce.job.credentials.binary",
299                                    System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
300       }
301     }
302 
303     if (provider.isHBaseSecurityEnabled()) {
304       try {
305         // init credentials for remote cluster
306         String quorumAddress = job.getConfiguration().get(
307             TableOutputFormat.QUORUM_ADDRESS);
308         User user = provider.getCurrent();
309         if (quorumAddress != null) {
310           String[] parts = ZKUtil.transformClusterKey(quorumAddress);
311           Configuration peerConf = HBaseConfiguration.create(job
312               .getConfiguration());
313           peerConf.set(HConstants.ZOOKEEPER_QUORUM, parts[0]);
314           peerConf.set("hbase.zookeeper.client.port", parts[1]);
315           peerConf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parts[2]);
316           obtainAuthTokenForJob(job, peerConf, user);
317         }
318 
319         obtainAuthTokenForJob(job, job.getConfiguration(), user);
320       } catch (InterruptedException ie) {
321         LOG.info("Interrupted obtaining user authentication token");
322         Thread.interrupted();
323       }
324     }
325   }
326   
327   /**
328    * Obtain an authentication token, for the specified cluster, on behalf of the current user
329    * and add it to the credentials for the given map reduce job.
330    *
331    * The quorumAddress is the key to the ZK ensemble, which contains:
332    * hbase.zookeeper.quorum, hbase.zookeeper.client.port and zookeeper.znode.parent
333    *
334    * @param job The job that requires the permission.
335    * @param quorumAddress string that contains the 3 required configuratins
336    * @throws IOException When the authentication token cannot be obtained.
337    */
338   public static void initCredentialsForCluster(Job job, String quorumAddress)
339       throws IOException {
340     UserProvider userProvider = UserProvider.instantiate(job.getConfiguration());
341     if (userProvider.isHBaseSecurityEnabled()) {
342       try {
343         Configuration peerConf = HBaseConfiguration.create(job.getConfiguration());
344         ZKUtil.applyClusterKeyToConf(peerConf, quorumAddress);
345         obtainAuthTokenForJob(job, peerConf, userProvider.getCurrent());
346       } catch (InterruptedException e) {
347         LOG.info("Interrupted obtaining user authentication token");
348         Thread.interrupted();
349       }
350     }
351   }
352 
353   private static void obtainAuthTokenForJob(Job job, Configuration conf, User user)
354       throws IOException, InterruptedException {
355     Token<?> authToken = getAuthToken(conf, user);
356     if (authToken == null) {
357       user.obtainAuthTokenForJob(conf, job);
358     } else {
359       job.getCredentials().addToken(authToken.getService(), authToken);
360     }
361   }
362 
363   /**
364    * Get the authentication token of the user for the cluster specified in the configuration
365    * @return null if the user does not have the token, otherwise the auth token for the cluster.
366    */
367   private static Token<?> getAuthToken(Configuration conf, User user)
368       throws IOException, InterruptedException {
369     ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "mr-init-credentials", null);
370     try {
371       String clusterId = ClusterId.readClusterIdZNode(zkw);
372       return user.getToken("HBASE_AUTH_TOKEN", clusterId);
373     } catch (KeeperException e) {
374       throw new IOException(e);
375     } finally {
376       zkw.close();
377     }
378   }
379 
380   /**
381    * Writes the given scan into a Base64 encoded string.
382    *
383    * @param scan  The scan to write out.
384    * @return The scan saved in a Base64 encoded string.
385    * @throws IOException When writing the scan fails.
386    */
387   static String convertScanToString(Scan scan) throws IOException {
388     ByteArrayOutputStream out = new ByteArrayOutputStream();
389     DataOutputStream dos = new DataOutputStream(out);
390     scan.write(dos);
391     return Base64.encodeBytes(out.toByteArray());
392   }
393 
394   /**
395    * Converts the given Base64 string back into a Scan instance.
396    *
397    * @param base64  The scan details.
398    * @return The newly created Scan instance.
399    * @throws IOException When reading the scan instance fails.
400    */
401   static Scan convertStringToScan(String base64) throws IOException {
402     ByteArrayInputStream bis = new ByteArrayInputStream(Base64.decode(base64));
403     DataInputStream dis = new DataInputStream(bis);
404     Scan scan = new Scan();
405     scan.readFields(dis);
406     return scan;
407   }
408 
409   /**
410    * Use this before submitting a TableReduce job. It will
411    * appropriately set up the JobConf.
412    *
413    * @param table  The output table.
414    * @param reducer  The reducer class to use.
415    * @param job  The current job to adjust.
416    * @throws IOException When determining the region count fails.
417    */
418   public static void initTableReducerJob(String table,
419     Class<? extends TableReducer> reducer, Job job)
420   throws IOException {
421     initTableReducerJob(table, reducer, job, null);
422   }
423 
424   /**
425    * Use this before submitting a TableReduce job. It will
426    * appropriately set up the JobConf.
427    *
428    * @param table  The output table.
429    * @param reducer  The reducer class to use.
430    * @param job  The current job to adjust.
431    * @param partitioner  Partitioner to use. Pass <code>null</code> to use
432    * default partitioner.
433    * @throws IOException When determining the region count fails.
434    */
435   public static void initTableReducerJob(String table,
436     Class<? extends TableReducer> reducer, Job job,
437     Class partitioner) throws IOException {
438     initTableReducerJob(table, reducer, job, partitioner, null, null, null);
439   }
440 
441   /**
442    * Use this before submitting a TableReduce job. It will
443    * appropriately set up the JobConf.
444    *
445    * @param table  The output table.
446    * @param reducer  The reducer class to use.
447    * @param job  The current job to adjust.  Make sure the passed job is
448    * carrying all necessary HBase configuration.
449    * @param partitioner  Partitioner to use. Pass <code>null</code> to use
450    * default partitioner.
451    * @param quorumAddress Distant cluster to write to; default is null for
452    * output to the cluster that is designated in <code>hbase-site.xml</code>.
453    * Set this String to the zookeeper ensemble of an alternate remote cluster
454    * when you would have the reduce write a cluster that is other than the
455    * default; e.g. copying tables between clusters, the source would be
456    * designated by <code>hbase-site.xml</code> and this param would have the
457    * ensemble address of the remote cluster.  The format to pass is particular.
458    * Pass <code> &lt;hbase.zookeeper.quorum>:&lt;hbase.zookeeper.client.port>:&lt;zookeeper.znode.parent>
459    * </code> such as <code>server,server2,server3:2181:/hbase</code>.
460    * @param serverClass redefined hbase.regionserver.class
461    * @param serverImpl redefined hbase.regionserver.impl
462    * @throws IOException When determining the region count fails.
463    */
464   public static void initTableReducerJob(String table,
465     Class<? extends TableReducer> reducer, Job job,
466     Class partitioner, String quorumAddress, String serverClass,
467     String serverImpl) throws IOException {
468     initTableReducerJob(table, reducer, job, partitioner, quorumAddress,
469         serverClass, serverImpl, true);
470   }
471 
472   /**
473    * Use this before submitting a TableReduce job. It will
474    * appropriately set up the JobConf.
475    *
476    * @param table  The output table.
477    * @param reducer  The reducer class to use.
478    * @param job  The current job to adjust.  Make sure the passed job is
479    * carrying all necessary HBase configuration.
480    * @param partitioner  Partitioner to use. Pass <code>null</code> to use
481    * default partitioner.
482    * @param quorumAddress Distant cluster to write to; default is null for
483    * output to the cluster that is designated in <code>hbase-site.xml</code>.
484    * Set this String to the zookeeper ensemble of an alternate remote cluster
485    * when you would have the reduce write a cluster that is other than the
486    * default; e.g. copying tables between clusters, the source would be
487    * designated by <code>hbase-site.xml</code> and this param would have the
488    * ensemble address of the remote cluster.  The format to pass is particular.
489    * Pass <code> &lt;hbase.zookeeper.quorum>:&lt;hbase.zookeeper.client.port>:&lt;zookeeper.znode.parent>
490    * </code> such as <code>server,server2,server3:2181:/hbase</code>.
491    * @param serverClass redefined hbase.regionserver.class
492    * @param serverImpl redefined hbase.regionserver.impl
493    * @param addDependencyJars upload HBase jars and jars for any of the configured
494    *           job classes via the distributed cache (tmpjars).
495    * @throws IOException When determining the region count fails.
496    */
497   public static void initTableReducerJob(String table,
498     Class<? extends TableReducer> reducer, Job job,
499     Class partitioner, String quorumAddress, String serverClass,
500     String serverImpl, boolean addDependencyJars) throws IOException {
501 
502     Configuration conf = job.getConfiguration();    
503     HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
504     job.setOutputFormatClass(TableOutputFormat.class);
505     if (reducer != null) job.setReducerClass(reducer);
506     conf.set(TableOutputFormat.OUTPUT_TABLE, table);
507     // If passed a quorum/ensemble address, pass it on to TableOutputFormat.
508     if (quorumAddress != null) {
509       // Calling this will validate the format
510       ZKUtil.transformClusterKey(quorumAddress);
511       conf.set(TableOutputFormat.QUORUM_ADDRESS,quorumAddress);
512     }
513     if (serverClass != null && serverImpl != null) {
514       conf.set(TableOutputFormat.REGION_SERVER_CLASS, serverClass);
515       conf.set(TableOutputFormat.REGION_SERVER_IMPL, serverImpl);
516     }
517     job.setOutputKeyClass(ImmutableBytesWritable.class);
518     job.setOutputValueClass(Writable.class);
519     if (partitioner == HRegionPartitioner.class) {
520       job.setPartitionerClass(HRegionPartitioner.class);
521       HTable outputTable = new HTable(conf, table);
522       int regions = outputTable.getRegionsInfo().size();
523       if (job.getNumReduceTasks() > regions) {
524         job.setNumReduceTasks(outputTable.getRegionsInfo().size());
525       }
526     } else if (partitioner != null) {
527       job.setPartitionerClass(partitioner);
528     }
529 
530     if (addDependencyJars) {
531       addDependencyJars(job);
532     }
533 
534     initCredentials(job);
535   }
536 
537   /**
538    * Ensures that the given number of reduce tasks for the given job
539    * configuration does not exceed the number of regions for the given table.
540    *
541    * @param table  The table to get the region count for.
542    * @param job  The current job to adjust.
543    * @throws IOException When retrieving the table details fails.
544    */
545   public static void limitNumReduceTasks(String table, Job job)
546   throws IOException {
547     HTable outputTable = new HTable(job.getConfiguration(), table);
548     int regions = outputTable.getRegionsInfo().size();
549     if (job.getNumReduceTasks() > regions)
550       job.setNumReduceTasks(regions);
551   }
552 
553   /**
554    * Sets the number of reduce tasks for the given job configuration to the
555    * number of regions the given table has.
556    *
557    * @param table  The table to get the region count for.
558    * @param job  The current job to adjust.
559    * @throws IOException When retrieving the table details fails.
560    */
561   public static void setNumReduceTasks(String table, Job job)
562   throws IOException {
563     HTable outputTable = new HTable(job.getConfiguration(), table);
564     int regions = outputTable.getRegionsInfo().size();
565     job.setNumReduceTasks(regions);
566   }
567 
568   /**
569    * Sets the number of rows to return and cache with each scanner iteration.
570    * Higher caching values will enable faster mapreduce jobs at the expense of
571    * requiring more heap to contain the cached rows.
572    *
573    * @param job The current job to adjust.
574    * @param batchSize The number of rows to return in batch with each scanner
575    * iteration.
576    */
577   public static void setScannerCaching(Job job, int batchSize) {
578     job.getConfiguration().setInt("hbase.client.scanner.caching", batchSize);
579   }
580 
581   /**
582    * Add HBase and its dependencies (only) to the job configuration.
583    * <p>
584    * This is intended as a low-level API, facilitating code reuse between this
585    * class and its mapred counterpart. It also of use to extenral tools that
586    * need to build a MapReduce job that interacts with HBase but want
587    * fine-grained control over the jars shipped to the cluster.
588    * </p>
589    * @param conf The Configuration object to extend with dependencies.
590    * @see org.apache.hadoop.hbase.mapred.TableMapReduceUtil
591    * @see <a href="https://issues.apache.org/jira/browse/PIG-3285">PIG-3285</a>
592    */
593   public static void addHBaseDependencyJars(Configuration conf) throws IOException {
594     addDependencyJars(conf,
595       org.apache.zookeeper.ZooKeeper.class,
596       com.google.protobuf.Message.class,
597       com.google.common.base.Function.class,
598       com.google.common.collect.ImmutableSet.class,
599       org.apache.hadoop.hbase.util.Bytes.class); //one class from hbase.jar
600   }
601 
602    /**
603     * Returns a classpath string built from the content of the "tmpjars" value in {@code conf}.
604     * Also exposed to shell scripts via `bin/hbase mapredcp`.
605     */
606   public static String buildDependencyClasspath(Configuration conf) {
607     if (conf == null) {
608       throw new IllegalArgumentException("Must provide a configuration object.");
609     }
610     Set<String> paths = new HashSet<String>(conf.getStringCollection("tmpjars"));
611     if (paths.size() == 0) {
612       throw new IllegalArgumentException("Configuration contains no tmpjars.");
613     }
614     StringBuilder sb = new StringBuilder();
615     for (String s : paths) {
616       // entries can take the form 'file:/path/to/file.jar'.
617       int idx = s.indexOf(":");
618       if (idx != -1) s = s.substring(idx + 1);
619       if (sb.length() > 0) sb.append(File.pathSeparator);
620       sb.append(s);
621     }
622     return sb.toString();
623   }
624 
625   /**
626    * Add the HBase dependency jars as well as jars for any of the configured
627    * job classes to the job configuration, so that JobClient will ship them
628    * to the cluster and add them to the DistributedCache.
629    */
630   public static void addDependencyJars(Job job) throws IOException {
631     addHBaseDependencyJars(job.getConfiguration());
632     try {
633       addDependencyJars(job.getConfiguration(),
634         // when making changes here, consider also mapred.TableMapReduceUtil
635         job.getMapOutputKeyClass(),
636         job.getMapOutputValueClass(),
637         job.getInputFormatClass(),
638         job.getOutputKeyClass(),
639         job.getOutputValueClass(),
640         job.getOutputFormatClass(),
641         job.getPartitionerClass(),
642         job.getCombinerClass());
643     } catch (ClassNotFoundException e) {
644       throw new IOException(e);
645     }    
646   }
647   
648   /**
649    * Add the jars containing the given classes to the job's configuration
650    * such that JobClient will ship them to the cluster and add them to
651    * the DistributedCache.
652    */
653   public static void addDependencyJars(Configuration conf,
654       Class<?>... classes) throws IOException {
655 
656     FileSystem localFs = FileSystem.getLocal(conf);
657     Set<String> jars = new HashSet<String>();
658     // Add jars that are already in the tmpjars variable
659     jars.addAll(conf.getStringCollection("tmpjars"));
660 
661     // add jars as we find them to a map of contents jar name so that we can avoid
662     // creating new jars for classes that have already been packaged.
663     Map<String, String> packagedClasses = new HashMap<String, String>();
664 
665     // Add jars containing the specified classes
666     for (Class<?> clazz : classes) {
667       if (clazz == null) continue;
668 
669       Path path = findOrCreateJar(clazz, localFs, packagedClasses);
670       if (path == null) {
671         LOG.warn("Could not find jar for class " + clazz +
672                  " in order to ship it to the cluster.");
673         continue;
674       }
675       if (!localFs.exists(path)) {
676         LOG.warn("Could not validate jar file " + path + " for class "
677                  + clazz);
678         continue;
679       }
680       jars.add(path.toString());
681     }
682     if (jars.isEmpty()) return;
683 
684     conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[0])));
685   }
686 
687   /**
688    * If org.apache.hadoop.util.JarFinder is available (0.23+ hadoop), finds
689    * the Jar for a class or creates it if it doesn't exist. If the class is in
690    * a directory in the classpath, it creates a Jar on the fly with the
691    * contents of the directory and returns the path to that Jar. If a Jar is
692    * created, it is created in the system temporary directory. Otherwise,
693    * returns an existing jar that contains a class of the same name. Maintains
694    * a mapping from jar contents to the tmp jar created.
695    * @param my_class the class to find.
696    * @param fs the FileSystem with which to qualify the returned path.
697    * @param packagedClasses a map of class name to path.
698    * @return a jar file that contains the class.
699    * @throws IOException
700    */
701   private static Path findOrCreateJar(Class<?> my_class, FileSystem fs,
702       Map<String, String> packagedClasses)
703   throws IOException {
704     // attempt to locate an existing jar for the class.
705     String jar = findContainingJar(my_class, packagedClasses);
706     if (null == jar || jar.isEmpty()) {
707       jar = getJar(my_class);
708       updateMap(jar, packagedClasses);
709     }
710 
711     if (null == jar || jar.isEmpty()) {
712       return null;
713     }
714 
715     LOG.debug(String.format("For class %s, using jar %s", my_class.getName(), jar));
716     return new Path(jar).makeQualified(fs);
717   }
718 
719   /**
720    * Add entries to <code>packagedClasses</code> corresponding to class files
721    * contained in <code>jar</code>.
722    * @param jar The jar who's content to list.
723    * @param packagedClasses map[class -> jar]
724    */
725   private static void updateMap(String jar, Map<String, String> packagedClasses) throws IOException {
726     if (null == jar || jar.isEmpty()) {
727       return;
728     }
729     ZipFile zip = null;
730     try {
731       zip = new ZipFile(jar);
732       for (Enumeration<? extends ZipEntry> iter = zip.entries(); iter.hasMoreElements();) {
733         ZipEntry entry = iter.nextElement();
734         if (entry.getName().endsWith("class")) {
735           packagedClasses.put(entry.getName(), jar);
736         }
737       }
738     } finally {
739       if (null != zip) zip.close();
740     }
741   }
742 
743   /**
744    * Find a jar that contains a class of the same name, if any. It will return
745    * a jar file, even if that is not the first thing on the class path that
746    * has a class with the same name. Looks first on the classpath and then in
747    * the <code>packagedClasses</code> map.
748    * @param my_class the class to find.
749    * @return a jar file that contains the class, or null.
750    * @throws IOException
751    */
752   private static String findContainingJar(Class<?> my_class, Map<String, String> packagedClasses)
753       throws IOException {
754     ClassLoader loader = my_class.getClassLoader();
755     String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
756 
757     // first search the classpath
758     for (Enumeration<URL> itr = loader.getResources(class_file); itr.hasMoreElements();) {
759       URL url = itr.nextElement();
760       if ("jar".equals(url.getProtocol())) {
761         String toReturn = url.getPath();
762         if (toReturn.startsWith("file:")) {
763           toReturn = toReturn.substring("file:".length());
764         }
765         // URLDecoder is a misnamed class, since it actually decodes
766         // x-www-form-urlencoded MIME type rather than actual
767         // URL encoding (which the file path has). Therefore it would
768         // decode +s to ' 's which is incorrect (spaces are actually
769         // either unencoded or encoded as "%20"). Replace +s first, so
770         // that they are kept sacred during the decoding process.
771         toReturn = toReturn.replaceAll("\\+", "%2B");
772         toReturn = URLDecoder.decode(toReturn, "UTF-8");
773         return toReturn.replaceAll("!.*$", "");
774       }
775     }
776 
777     // now look in any jars we've packaged using JarFinder. Returns null when
778     // no jar is found.
779     return packagedClasses.get(class_file);
780   }
781 
782   /**
783    * Invoke 'getJar' on a JarFinder implementation. Useful for some job
784    * configuration contexts (HBASE-8140) and also for testing on MRv2. First
785    * check if we have HADOOP-9426. Lacking that, fall back to the backport.
786    * @param my_class the class to find.
787    * @return a jar file that contains the class, or null.
788    */
789   private static String getJar(Class<?> my_class) {
790     String ret = null;
791     String hadoopJarFinder = "org.apache.hadoop.util.JarFinder";
792     Class<?> jarFinder = null;
793     try {
794       LOG.debug("Looking for " + hadoopJarFinder + ".");
795       jarFinder = Class.forName(hadoopJarFinder);
796       LOG.debug(hadoopJarFinder + " found.");
797       Method getJar = jarFinder.getMethod("getJar", Class.class);
798       ret = (String) getJar.invoke(null, my_class);
799     } catch (ClassNotFoundException e) {
800       LOG.debug("Using backported JarFinder.");
801       ret = JarFinder.getJar(my_class);
802     } catch (InvocationTargetException e) {
803       // function was properly called, but threw it's own exception. Unwrap it
804       // and pass it on.
805       throw new RuntimeException(e.getCause());
806     } catch (Exception e) {
807       // toss all other exceptions, related to reflection failure
808       throw new RuntimeException("getJar invocation failed.", e);
809     }
810 
811     return ret;
812   }
813 }