View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.mapreduce;
20  
21  import java.io.File;
22  import java.io.IOException;
23  import java.lang.reflect.InvocationTargetException;
24  import java.lang.reflect.Method;
25  import java.net.URL;
26  import java.net.URLDecoder;
27  import java.util.ArrayList;
28  import java.util.Collection;
29  import java.util.Enumeration;
30  import java.util.HashMap;
31  import java.util.HashSet;
32  import java.util.List;
33  import java.util.Map;
34  import java.util.Set;
35  import java.util.zip.ZipEntry;
36  import java.util.zip.ZipFile;
37  
38  import com.google.protobuf.InvalidProtocolBufferException;
39  import com.yammer.metrics.core.MetricsRegistry;
40  
41  import org.apache.commons.logging.Log;
42  import org.apache.commons.logging.LogFactory;
43  import org.apache.hadoop.hbase.classification.InterfaceAudience;
44  import org.apache.hadoop.hbase.classification.InterfaceStability;
45  import org.apache.hadoop.conf.Configuration;
46  import org.apache.hadoop.fs.FileSystem;
47  import org.apache.hadoop.fs.Path;
48  import org.apache.hadoop.hbase.HBaseConfiguration;
49  import org.apache.hadoop.hbase.HConstants;
50  import org.apache.hadoop.hbase.catalog.MetaReader;
51  import org.apache.hadoop.hbase.client.HConnection;
52  import org.apache.hadoop.hbase.client.HConnectionManager;
53  import org.apache.hadoop.hbase.client.Put;
54  import org.apache.hadoop.hbase.client.Scan;
55  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
56  import org.apache.hadoop.hbase.mapreduce.hadoopbackport.JarFinder;
57  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
58  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
59  import org.apache.hadoop.hbase.security.User;
60  import org.apache.hadoop.hbase.security.UserProvider;
61  import org.apache.hadoop.hbase.security.token.TokenUtil;
62  import org.apache.hadoop.hbase.util.Base64;
63  import org.apache.hadoop.hbase.util.Bytes;
64  import org.apache.hadoop.hbase.zookeeper.ZKConfig;
65  import org.apache.hadoop.io.Writable;
66  import org.apache.hadoop.io.WritableComparable;
67  import org.apache.hadoop.mapreduce.InputFormat;
68  import org.apache.hadoop.mapreduce.Job;
69  import org.apache.hadoop.util.StringUtils;
70  
71  /**
72   * 
73   * Utility for {@link TableMapper} and {@link TableReducer}
74   */
75  @SuppressWarnings({ "rawtypes", "unchecked" })
76  @InterfaceAudience.Public
77  @InterfaceStability.Stable
78  public class TableMapReduceUtil {
79    static Log LOG = LogFactory.getLog(TableMapReduceUtil.class);
80  
81    /**
82     * Use this before submitting a TableMap job. It will appropriately set up
83     * the job.
84     *
85     * @param table  The table name to read from.
86     * @param scan  The scan instance with the columns, time range etc.
87     * @param mapper  The mapper class to use.
88     * @param outputKeyClass  The class of the output key.
89     * @param outputValueClass  The class of the output value.
90     * @param job  The current job to adjust.  Make sure the passed job is
91     * carrying all necessary HBase configuration.
92     * @throws IOException When setting up the details fails.
93     */
94    public static void initTableMapperJob(String table, Scan scan,
95        Class<? extends TableMapper> mapper,
96        Class<?> outputKeyClass,
97        Class<?> outputValueClass, Job job)
98    throws IOException {
99      initTableMapperJob(table, scan, mapper, outputKeyClass, outputValueClass,
100         job, true);
101   }
102 
103   /**
104    * Use this before submitting a TableMap job. It will appropriately set up
105    * the job.
106    *
107    * @param table Binary representation of the table name to read from.
108    * @param scan  The scan instance with the columns, time range etc.
109    * @param mapper  The mapper class to use.
110    * @param outputKeyClass  The class of the output key.
111    * @param outputValueClass  The class of the output value.
112    * @param job  The current job to adjust.  Make sure the passed job is
113    * carrying all necessary HBase configuration.
114    * @throws IOException When setting up the details fails.
115    */
116    public static void initTableMapperJob(byte[] table, Scan scan,
117       Class<? extends TableMapper> mapper,
118       Class<?> outputKeyClass,
119       Class<?> outputValueClass, Job job)
120   throws IOException {
121       initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass, outputValueClass,
122               job, true);
123   }
124 
125    /**
126     * Use this before submitting a TableMap job. It will appropriately set up
127     * the job.
128     *
129     * @param table  The table name to read from.
130     * @param scan  The scan instance with the columns, time range etc.
131     * @param mapper  The mapper class to use.
132     * @param outputKeyClass  The class of the output key.
133     * @param outputValueClass  The class of the output value.
134     * @param job  The current job to adjust.  Make sure the passed job is
135     * carrying all necessary HBase configuration.
136     * @param addDependencyJars upload HBase jars and jars for any of the configured
137     *           job classes via the distributed cache (tmpjars).
138     * @throws IOException When setting up the details fails.
139     */
140    public static void initTableMapperJob(String table, Scan scan,
141        Class<? extends TableMapper> mapper,
142        Class<?> outputKeyClass,
143        Class<?> outputValueClass, Job job,
144        boolean addDependencyJars, Class<? extends InputFormat> inputFormatClass)
145    throws IOException {
146      initTableMapperJob(table, scan, mapper, outputKeyClass, outputValueClass, job,
147          addDependencyJars, true, inputFormatClass);
148    }
149 
150 
151   /**
152    * Use this before submitting a TableMap job. It will appropriately set up
153    * the job.
154    *
155    * @param table  The table name to read from.
156    * @param scan  The scan instance with the columns, time range etc.
157    * @param mapper  The mapper class to use.
158    * @param outputKeyClass  The class of the output key.
159    * @param outputValueClass  The class of the output value.
160    * @param job  The current job to adjust.  Make sure the passed job is
161    * carrying all necessary HBase configuration.
162    * @param addDependencyJars upload HBase jars and jars for any of the configured
163    *           job classes via the distributed cache (tmpjars).
164    * @param initCredentials whether to initialize hbase auth credentials for the job
165    * @param inputFormatClass the input format
166    * @throws IOException When setting up the details fails.
167    */
168   public static void initTableMapperJob(String table, Scan scan,
169       Class<? extends TableMapper> mapper,
170       Class<?> outputKeyClass,
171       Class<?> outputValueClass, Job job,
172       boolean addDependencyJars, boolean initCredentials,
173       Class<? extends InputFormat> inputFormatClass)
174   throws IOException {
175     job.setInputFormatClass(inputFormatClass);
176     if (outputValueClass != null) job.setMapOutputValueClass(outputValueClass);
177     if (outputKeyClass != null) job.setMapOutputKeyClass(outputKeyClass);
178     job.setMapperClass(mapper);
179     if (Put.class.equals(outputValueClass)) {
180       job.setCombinerClass(PutCombiner.class);
181     }
182     Configuration conf = job.getConfiguration();
183     HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
184     conf.set(TableInputFormat.INPUT_TABLE, table);
185     conf.set(TableInputFormat.SCAN, convertScanToString(scan));
186     conf.setStrings("io.serializations", conf.get("io.serializations"),
187         MutationSerialization.class.getName(), ResultSerialization.class.getName(),
188         KeyValueSerialization.class.getName());
189     if (addDependencyJars) {
190       addDependencyJars(job);
191     }
192     if (initCredentials) {
193       initCredentials(job);
194     }
195   }
196 
197   /**
198    * Use this before submitting a TableMap job. It will appropriately set up
199    * the job.
200    *
201    * @param table Binary representation of the table name to read from.
202    * @param scan  The scan instance with the columns, time range etc.
203    * @param mapper  The mapper class to use.
204    * @param outputKeyClass  The class of the output key.
205    * @param outputValueClass  The class of the output value.
206    * @param job  The current job to adjust.  Make sure the passed job is
207    * carrying all necessary HBase configuration.
208    * @param addDependencyJars upload HBase jars and jars for any of the configured
209    *           job classes via the distributed cache (tmpjars).
210    * @param inputFormatClass The class of the input format
211    * @throws IOException When setting up the details fails.
212    */
213   public static void initTableMapperJob(byte[] table, Scan scan,
214       Class<? extends TableMapper> mapper,
215       Class<?> outputKeyClass,
216       Class<?> outputValueClass, Job job,
217       boolean addDependencyJars, Class<? extends InputFormat> inputFormatClass)
218   throws IOException {
219       initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass,
220               outputValueClass, job, addDependencyJars, inputFormatClass);
221   }
222 
223   /**
224    * Use this before submitting a TableMap job. It will appropriately set up
225    * the job.
226    *
227    * @param table Binary representation of the table name to read from.
228    * @param scan  The scan instance with the columns, time range etc.
229    * @param mapper  The mapper class to use.
230    * @param outputKeyClass  The class of the output key.
231    * @param outputValueClass  The class of the output value.
232    * @param job  The current job to adjust.  Make sure the passed job is
233    * carrying all necessary HBase configuration.
234    * @param addDependencyJars upload HBase jars and jars for any of the configured
235    *           job classes via the distributed cache (tmpjars).
236    * @throws IOException When setting up the details fails.
237    */
238   public static void initTableMapperJob(byte[] table, Scan scan,
239       Class<? extends TableMapper> mapper,
240       Class<?> outputKeyClass,
241       Class<?> outputValueClass, Job job,
242       boolean addDependencyJars)
243   throws IOException {
244       initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass,
245               outputValueClass, job, addDependencyJars, TableInputFormat.class);
246   }
247 
248   /**
249    * Use this before submitting a TableMap job. It will appropriately set up
250    * the job.
251    *
252    * @param table The table name to read from.
253    * @param scan  The scan instance with the columns, time range etc.
254    * @param mapper  The mapper class to use.
255    * @param outputKeyClass  The class of the output key.
256    * @param outputValueClass  The class of the output value.
257    * @param job  The current job to adjust.  Make sure the passed job is
258    * carrying all necessary HBase configuration.
259    * @param addDependencyJars upload HBase jars and jars for any of the configured
260    *           job classes via the distributed cache (tmpjars).
261    * @throws IOException When setting up the details fails.
262    */
263   public static void initTableMapperJob(String table, Scan scan,
264       Class<? extends TableMapper> mapper,
265       Class<?> outputKeyClass,
266       Class<?> outputValueClass, Job job,
267       boolean addDependencyJars)
268   throws IOException {
269       initTableMapperJob(table, scan, mapper, outputKeyClass,
270               outputValueClass, job, addDependencyJars, TableInputFormat.class);
271   }
272 
273   /**
274    * Enable a basic on-heap cache for these jobs. Any BlockCache implementation based on
275    * direct memory will likely cause the map tasks to OOM when opening the region. This
276    * is done here instead of in TableSnapshotRegionRecordReader in case an advanced user
277    * wants to override this behavior in their job.
278    */
279   public static void resetCacheConfig(Configuration conf) {
280     conf.setFloat(
281       HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
282     conf.setFloat("hbase.offheapcache.percentage", 0f);
283     conf.setFloat("hbase.bucketcache.size", 0f);
284     conf.unset("hbase.bucketcache.ioengine");
285   }
286 
287   /**
288    * Sets up the job for reading from one or more table snapshots, with one or more scans
289    * per snapshot.
290    * It bypasses hbase servers and read directly from snapshot files.
291    *
292    * @param snapshotScans     map of snapshot name to scans on that snapshot.
293    * @param mapper            The mapper class to use.
294    * @param outputKeyClass    The class of the output key.
295    * @param outputValueClass  The class of the output value.
296    * @param job               The current job to adjust.  Make sure the passed job is
297    *                          carrying all necessary HBase configuration.
298    * @param addDependencyJars upload HBase jars and jars for any of the configured
299    *                          job classes via the distributed cache (tmpjars).
300    */
301   public static void initMultiTableSnapshotMapperJob(Map<String, Collection<Scan>> snapshotScans,
302       Class<? extends TableMapper> mapper, Class<?> outputKeyClass, Class<?> outputValueClass,
303       Job job, boolean addDependencyJars, Path tmpRestoreDir) throws IOException {
304     MultiTableSnapshotInputFormat.setInput(job.getConfiguration(), snapshotScans, tmpRestoreDir);
305 
306     job.setInputFormatClass(MultiTableSnapshotInputFormat.class);
307     if (outputValueClass != null) {
308       job.setMapOutputValueClass(outputValueClass);
309     }
310     if (outputKeyClass != null) {
311       job.setMapOutputKeyClass(outputKeyClass);
312     }
313     job.setMapperClass(mapper);
314     Configuration conf = job.getConfiguration();
315     HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
316 
317     if (addDependencyJars) {
318       addDependencyJars(job);
319       addDependencyJars(job.getConfiguration(), MetricsRegistry.class);
320     }
321 
322     resetCacheConfig(job.getConfiguration());
323   }
324 
325   /**
326    * Sets up the job for reading from a table snapshot. It bypasses hbase servers
327    * and read directly from snapshot files.
328    *
329    * @param snapshotName The name of the snapshot (of a table) to read from.
330    * @param scan  The scan instance with the columns, time range etc.
331    * @param mapper  The mapper class to use.
332    * @param outputKeyClass  The class of the output key.
333    * @param outputValueClass  The class of the output value.
334    * @param job  The current job to adjust.  Make sure the passed job is
335    * carrying all necessary HBase configuration.
336    * @param addDependencyJars upload HBase jars and jars for any of the configured
337    *           job classes via the distributed cache (tmpjars).
338    *
339    * @param tmpRestoreDir a temporary directory to copy the snapshot files into. Current user should
340    * have write permissions to this directory, and this should not be a subdirectory of rootdir.
341    * After the job is finished, restore directory can be deleted.
342    * @throws IOException When setting up the details fails.
343    * @see TableSnapshotInputFormat
344    */
345   public static void initTableSnapshotMapperJob(String snapshotName, Scan scan,
346       Class<? extends TableMapper> mapper,
347       Class<?> outputKeyClass,
348       Class<?> outputValueClass, Job job,
349       boolean addDependencyJars, Path tmpRestoreDir)
350   throws IOException {
351     TableSnapshotInputFormat.setInput(job, snapshotName, tmpRestoreDir);
352     initTableMapperJob(snapshotName, scan, mapper, outputKeyClass,
353         outputValueClass, job, addDependencyJars, false, TableSnapshotInputFormat.class);
354     resetCacheConfig(job.getConfiguration());
355   }
356 
357   /**
358    * Use this before submitting a Multi TableMap job. It will appropriately set
359    * up the job.
360    *
361    * @param scans The list of {@link Scan} objects to read from.
362    * @param mapper The mapper class to use.
363    * @param outputKeyClass The class of the output key.
364    * @param outputValueClass The class of the output value.
365    * @param job The current job to adjust. Make sure the passed job is carrying
366    *          all necessary HBase configuration.
367    * @throws IOException When setting up the details fails.
368    */
369   public static void initTableMapperJob(List<Scan> scans,
370       Class<? extends TableMapper> mapper,
371       Class<? extends WritableComparable> outputKeyClass,
372       Class<? extends Writable> outputValueClass, Job job) throws IOException {
373     initTableMapperJob(scans, mapper, outputKeyClass, outputValueClass, job,
374         true);
375   }
376 
377   /**
378    * Use this before submitting a Multi TableMap job. It will appropriately set
379    * up the job.
380    *
381    * @param scans The list of {@link Scan} objects to read from.
382    * @param mapper The mapper class to use.
383    * @param outputKeyClass The class of the output key.
384    * @param outputValueClass The class of the output value.
385    * @param job The current job to adjust. Make sure the passed job is carrying
386    *          all necessary HBase configuration.
387    * @param addDependencyJars upload HBase jars and jars for any of the
388    *          configured job classes via the distributed cache (tmpjars).
389    * @throws IOException When setting up the details fails.
390    */
391   public static void initTableMapperJob(List<Scan> scans,
392       Class<? extends TableMapper> mapper,
393       Class<? extends WritableComparable> outputKeyClass,
394       Class<? extends Writable> outputValueClass, Job job,
395       boolean addDependencyJars) throws IOException {
396     initTableMapperJob(scans, mapper, outputKeyClass, outputValueClass, job,
397       addDependencyJars, true);
398   }
399 
400   /**
401    * Use this before submitting a Multi TableMap job. It will appropriately set
402    * up the job.
403    *
404    * @param scans The list of {@link Scan} objects to read from.
405    * @param mapper The mapper class to use.
406    * @param outputKeyClass The class of the output key.
407    * @param outputValueClass The class of the output value.
408    * @param job The current job to adjust. Make sure the passed job is carrying
409    *          all necessary HBase configuration.
410    * @param addDependencyJars upload HBase jars and jars for any of the
411    *          configured job classes via the distributed cache (tmpjars).
412    * @param initCredentials whether to initialize hbase auth credentials for the job
413    * @throws IOException When setting up the details fails.
414    */
415   public static void initTableMapperJob(List<Scan> scans,
416       Class<? extends TableMapper> mapper,
417       Class<? extends WritableComparable> outputKeyClass,
418       Class<? extends Writable> outputValueClass, Job job,
419       boolean addDependencyJars,
420       boolean initCredentials) throws IOException {
421     job.setInputFormatClass(MultiTableInputFormat.class);
422     if (outputValueClass != null) {
423       job.setMapOutputValueClass(outputValueClass);
424     }
425     if (outputKeyClass != null) {
426       job.setMapOutputKeyClass(outputKeyClass);
427     }
428     job.setMapperClass(mapper);
429     Configuration conf = job.getConfiguration();
430     HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
431     List<String> scanStrings = new ArrayList<String>();
432 
433     for (Scan scan : scans) {
434       scanStrings.add(convertScanToString(scan));
435     }
436     job.getConfiguration().setStrings(MultiTableInputFormat.SCANS,
437       scanStrings.toArray(new String[scanStrings.size()]));
438 
439     if (addDependencyJars) {
440       addDependencyJars(job);
441     }
442 
443     if (initCredentials) {
444       initCredentials(job);
445     }
446   }
447 
448   public static void initCredentials(Job job) throws IOException {
449     UserProvider userProvider = UserProvider.instantiate(job.getConfiguration());
450     if (userProvider.isHadoopSecurityEnabled()) {
451       // propagate delegation related props from launcher job to MR job
452       if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
453         job.getConfiguration().set("mapreduce.job.credentials.binary",
454                                    System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
455       }
456     }
457 
458     if (userProvider.isHBaseSecurityEnabled()) {
459       try {
460         // init credentials for remote cluster
461         String quorumAddress = job.getConfiguration().get(TableOutputFormat.QUORUM_ADDRESS);
462         User user = userProvider.getCurrent();
463         if (quorumAddress != null) {
464           Configuration peerConf = HBaseConfiguration.createClusterConf(job.getConfiguration(),
465               quorumAddress, TableOutputFormat.OUTPUT_CONF_PREFIX);
466           HConnection peerConn = HConnectionManager.createConnection(peerConf);
467           try {
468             TokenUtil.addTokenForJob(peerConn, user, job);
469           } finally {
470             peerConn.close();
471           }
472         }
473 
474         HConnection conn = HConnectionManager.createConnection(job.getConfiguration());
475         try {
476           TokenUtil.addTokenForJob(conn, user, job);
477         } finally {
478           conn.close();
479         }
480       } catch (InterruptedException ie) {
481         LOG.info("Interrupted obtaining user authentication token");
482         Thread.currentThread().interrupt();
483       }
484     }
485   }
486 
487   /**
488    * Obtain an authentication token, for the specified cluster, on behalf of the current user
489    * and add it to the credentials for the given map reduce job.
490    *
491    * The quorumAddress is the key to the ZK ensemble, which contains:
492    * hbase.zookeeper.quorum, hbase.zookeeper.client.port and zookeeper.znode.parent
493    *
494    * @param job The job that requires the permission.
495    * @param quorumAddress string that contains the 3 required configuratins
496    * @throws IOException When the authentication token cannot be obtained.
497    * @deprecated Since 1.2.0, use {@link #initCredentialsForCluster(Job, Configuration)} instead.
498    */
499   @Deprecated
500   public static void initCredentialsForCluster(Job job, String quorumAddress)
501       throws IOException {
502     Configuration peerConf = HBaseConfiguration.createClusterConf(job.getConfiguration(),
503         quorumAddress);
504     initCredentialsForCluster(job, peerConf);
505   }
506 
507   /**
508    * Obtain an authentication token, for the specified cluster, on behalf of the current user
509    * and add it to the credentials for the given map reduce job.
510    *
511    * @param job The job that requires the permission.
512    * @param conf The configuration to use in connecting to the peer cluster
513    * @throws IOException When the authentication token cannot be obtained.
514    */
515   public static void initCredentialsForCluster(Job job, Configuration conf)
516       throws IOException {
517     UserProvider userProvider = UserProvider.instantiate(job.getConfiguration());
518     if (userProvider.isHBaseSecurityEnabled()) {
519       try {
520         HConnection peerConn = HConnectionManager.createConnection(conf);
521         try {
522           TokenUtil.addTokenForJob(peerConn, userProvider.getCurrent(), job);
523         } finally {
524           peerConn.close();
525         }
526       } catch (InterruptedException e) {
527         LOG.info("Interrupted obtaining user authentication token");
528         Thread.interrupted();
529       }
530     }
531   }
532 
533   /**
534    * Writes the given scan into a Base64 encoded string.
535    *
536    * @param scan  The scan to write out.
537    * @return The scan saved in a Base64 encoded string.
538    * @throws IOException When writing the scan fails.
539    */
540   static String convertScanToString(Scan scan) throws IOException {
541     ClientProtos.Scan proto = ProtobufUtil.toScan(scan);
542     return Base64.encodeBytes(proto.toByteArray());
543   }
544 
545   /**
546    * Converts the given Base64 string back into a Scan instance.
547    *
548    * @param base64  The scan details.
549    * @return The newly created Scan instance.
550    * @throws IOException When reading the scan instance fails.
551    */
552   static Scan convertStringToScan(String base64) throws IOException {
553     byte [] decoded = Base64.decode(base64);
554     ClientProtos.Scan scan;
555     try {
556       scan = ClientProtos.Scan.parseFrom(decoded);
557     } catch (InvalidProtocolBufferException ipbe) {
558       throw new IOException(ipbe);
559     }
560 
561     return ProtobufUtil.toScan(scan);
562   }
563 
564   /**
565    * Use this before submitting a TableReduce job. It will
566    * appropriately set up the JobConf.
567    *
568    * @param table  The output table.
569    * @param reducer  The reducer class to use.
570    * @param job  The current job to adjust.
571    * @throws IOException When determining the region count fails.
572    */
573   public static void initTableReducerJob(String table,
574     Class<? extends TableReducer> reducer, Job job)
575   throws IOException {
576     initTableReducerJob(table, reducer, job, null);
577   }
578 
579   /**
580    * Use this before submitting a TableReduce job. It will
581    * appropriately set up the JobConf.
582    *
583    * @param table  The output table.
584    * @param reducer  The reducer class to use.
585    * @param job  The current job to adjust.
586    * @param partitioner  Partitioner to use. Pass <code>null</code> to use
587    * default partitioner.
588    * @throws IOException When determining the region count fails.
589    */
590   public static void initTableReducerJob(String table,
591     Class<? extends TableReducer> reducer, Job job,
592     Class partitioner) throws IOException {
593     initTableReducerJob(table, reducer, job, partitioner, null, null, null);
594   }
595 
596   /**
597    * Use this before submitting a TableReduce job. It will
598    * appropriately set up the JobConf.
599    *
600    * @param table  The output table.
601    * @param reducer  The reducer class to use.
602    * @param job  The current job to adjust.  Make sure the passed job is
603    * carrying all necessary HBase configuration.
604    * @param partitioner  Partitioner to use. Pass <code>null</code> to use
605    * default partitioner.
606    * @param quorumAddress Distant cluster to write to; default is null for
607    * output to the cluster that is designated in <code>hbase-site.xml</code>.
608    * Set this String to the zookeeper ensemble of an alternate remote cluster
609    * when you would have the reduce write a cluster that is other than the
610    * default; e.g. copying tables between clusters, the source would be
611    * designated by <code>hbase-site.xml</code> and this param would have the
612    * ensemble address of the remote cluster.  The format to pass is particular.
613    * Pass <code> &lt;hbase.zookeeper.quorum>:&lt;hbase.zookeeper.client.port>:&lt;zookeeper.znode.parent>
614    * </code> such as <code>server,server2,server3:2181:/hbase</code>.
615    * @param serverClass redefined hbase.regionserver.class
616    * @param serverImpl redefined hbase.regionserver.impl
617    * @throws IOException When determining the region count fails.
618    */
619   public static void initTableReducerJob(String table,
620     Class<? extends TableReducer> reducer, Job job,
621     Class partitioner, String quorumAddress, String serverClass,
622     String serverImpl) throws IOException {
623     initTableReducerJob(table, reducer, job, partitioner, quorumAddress,
624         serverClass, serverImpl, true);
625   }
626 
627   /**
628    * Use this before submitting a TableReduce job. It will
629    * appropriately set up the JobConf.
630    *
631    * @param table  The output table.
632    * @param reducer  The reducer class to use.
633    * @param job  The current job to adjust.  Make sure the passed job is
634    * carrying all necessary HBase configuration.
635    * @param partitioner  Partitioner to use. Pass <code>null</code> to use
636    * default partitioner.
637    * @param quorumAddress Distant cluster to write to; default is null for
638    * output to the cluster that is designated in <code>hbase-site.xml</code>.
639    * Set this String to the zookeeper ensemble of an alternate remote cluster
640    * when you would have the reduce write a cluster that is other than the
641    * default; e.g. copying tables between clusters, the source would be
642    * designated by <code>hbase-site.xml</code> and this param would have the
643    * ensemble address of the remote cluster.  The format to pass is particular.
644    * Pass <code> &lt;hbase.zookeeper.quorum>:&lt;hbase.zookeeper.client.port>:&lt;zookeeper.znode.parent>
645    * </code> such as <code>server,server2,server3:2181:/hbase</code>.
646    * @param serverClass redefined hbase.regionserver.class
647    * @param serverImpl redefined hbase.regionserver.impl
648    * @param addDependencyJars upload HBase jars and jars for any of the configured
649    *           job classes via the distributed cache (tmpjars).
650    * @throws IOException When determining the region count fails.
651    */
652   public static void initTableReducerJob(String table,
653     Class<? extends TableReducer> reducer, Job job,
654     Class partitioner, String quorumAddress, String serverClass,
655     String serverImpl, boolean addDependencyJars) throws IOException {
656 
657     Configuration conf = job.getConfiguration();
658     HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
659     job.setOutputFormatClass(TableOutputFormat.class);
660     if (reducer != null) job.setReducerClass(reducer);
661     conf.set(TableOutputFormat.OUTPUT_TABLE, table);
662     conf.setStrings("io.serializations", conf.get("io.serializations"),
663         MutationSerialization.class.getName(), ResultSerialization.class.getName());
664     // If passed a quorum/ensemble address, pass it on to TableOutputFormat.
665     if (quorumAddress != null) {
666       // Calling this will validate the format
667       ZKConfig.validateClusterKey(quorumAddress);
668       conf.set(TableOutputFormat.QUORUM_ADDRESS,quorumAddress);
669     }
670     if (serverClass != null && serverImpl != null) {
671       conf.set(TableOutputFormat.REGION_SERVER_CLASS, serverClass);
672       conf.set(TableOutputFormat.REGION_SERVER_IMPL, serverImpl);
673     }
674     job.setOutputKeyClass(ImmutableBytesWritable.class);
675     job.setOutputValueClass(Writable.class);
676     if (partitioner == HRegionPartitioner.class) {
677       job.setPartitionerClass(HRegionPartitioner.class);
678       int regions = MetaReader.getRegionCount(conf, table);
679       if (job.getNumReduceTasks() > regions) {
680         job.setNumReduceTasks(regions);
681       }
682     } else if (partitioner != null) {
683       job.setPartitionerClass(partitioner);
684     }
685 
686     if (addDependencyJars) {
687       addDependencyJars(job);
688     }
689 
690     initCredentials(job);
691   }
692 
693   /**
694    * Ensures that the given number of reduce tasks for the given job
695    * configuration does not exceed the number of regions for the given table.
696    *
697    * @param table  The table to get the region count for.
698    * @param job  The current job to adjust.
699    * @throws IOException When retrieving the table details fails.
700    */
701   public static void limitNumReduceTasks(String table, Job job)
702   throws IOException {
703     int regions = MetaReader.getRegionCount(job.getConfiguration(), table);
704     if (job.getNumReduceTasks() > regions)
705       job.setNumReduceTasks(regions);
706   }
707 
708   /**
709    * Sets the number of reduce tasks for the given job configuration to the
710    * number of regions the given table has.
711    *
712    * @param table  The table to get the region count for.
713    * @param job  The current job to adjust.
714    * @throws IOException When retrieving the table details fails.
715    */
716   public static void setNumReduceTasks(String table, Job job)
717   throws IOException {
718     job.setNumReduceTasks(MetaReader.getRegionCount(job.getConfiguration(), table));
719   }
720 
721   /**
722    * Sets the number of rows to return and cache with each scanner iteration.
723    * Higher caching values will enable faster mapreduce jobs at the expense of
724    * requiring more heap to contain the cached rows.
725    *
726    * @param job The current job to adjust.
727    * @param batchSize The number of rows to return in batch with each scanner
728    * iteration.
729    */
730   public static void setScannerCaching(Job job, int batchSize) {
731     job.getConfiguration().setInt("hbase.client.scanner.caching", batchSize);
732   }
733 
734   /**
735    * Add HBase and its dependencies (only) to the job configuration.
736    * <p>
737    * This is intended as a low-level API, facilitating code reuse between this
738    * class and its mapred counterpart. It also of use to external tools that
739    * need to build a MapReduce job that interacts with HBase but want
740    * fine-grained control over the jars shipped to the cluster.
741    * </p>
742    * @param conf The Configuration object to extend with dependencies.
743    * @see org.apache.hadoop.hbase.mapred.TableMapReduceUtil
744    * @see <a href="https://issues.apache.org/jira/browse/PIG-3285">PIG-3285</a>
745    */
746   public static void addHBaseDependencyJars(Configuration conf) throws IOException {
747 
748     // PrefixTreeCodec is part of the hbase-prefix-tree module. If not included in MR jobs jar
749     // dependencies, MR jobs that write encoded hfiles will fail.
750     // We used reflection here so to prevent a circular module dependency.
751     // TODO - if we extract the MR into a module, make it depend on hbase-prefix-tree.
752     Class prefixTreeCodecClass = null;
753     try {
754       prefixTreeCodecClass =
755           Class.forName("org.apache.hadoop.hbase.code.prefixtree.PrefixTreeCodec");
756     } catch (ClassNotFoundException e) {
757       // this will show up in unit tests but should not show in real deployments
758       LOG.warn("The hbase-prefix-tree module jar containing PrefixTreeCodec is not present." +
759           "  Continuing without it.");
760     }
761 
762     addDependencyJars(conf,
763       // explicitly pull a class from each module
764       org.apache.hadoop.hbase.HConstants.class,                      // hbase-common
765       org.apache.hadoop.hbase.protobuf.generated.ClientProtos.class, // hbase-protocol
766       org.apache.hadoop.hbase.client.Put.class,                      // hbase-client
767       org.apache.hadoop.hbase.CompatibilityFactory.class,            // hbase-hadoop-compat
768       org.apache.hadoop.hbase.mapreduce.TableMapper.class,           // hbase-server
769       prefixTreeCodecClass, //  hbase-prefix-tree (if null will be skipped)
770       // pull necessary dependencies
771       org.apache.zookeeper.ZooKeeper.class,
772       org.jboss.netty.channel.ChannelFactory.class,
773       com.google.protobuf.Message.class,
774       com.google.common.collect.Lists.class,
775       org.cloudera.htrace.Trace.class,
776       org.cliffc.high_scale_lib.Counter.class,
777       com.yammer.metrics.core.MetricsRegistry.class); // needed for mapred over snapshots
778   }
779 
780   /**
781    * Returns a classpath string built from the content of the "tmpjars" value in {@code conf}.
782    * Also exposed to shell scripts via `bin/hbase mapredcp`.
783    */
784   public static String buildDependencyClasspath(Configuration conf) {
785     if (conf == null) {
786       throw new IllegalArgumentException("Must provide a configuration object.");
787     }
788     Set<String> paths = new HashSet<String>(conf.getStringCollection("tmpjars"));
789     if (paths.size() == 0) {
790       throw new IllegalArgumentException("Configuration contains no tmpjars.");
791     }
792     StringBuilder sb = new StringBuilder();
793     for (String s : paths) {
794       // entries can take the form 'file:/path/to/file.jar'.
795       int idx = s.indexOf(":");
796       if (idx != -1) s = s.substring(idx + 1);
797       if (sb.length() > 0) sb.append(File.pathSeparator);
798       sb.append(s);
799     }
800     return sb.toString();
801   }
802 
803   /**
804    * Add the HBase dependency jars as well as jars for any of the configured
805    * job classes to the job configuration, so that JobClient will ship them
806    * to the cluster and add them to the DistributedCache.
807    */
808   public static void addDependencyJars(Job job) throws IOException {
809     addHBaseDependencyJars(job.getConfiguration());
810     try {
811       addDependencyJars(job.getConfiguration(),
812           // when making changes here, consider also mapred.TableMapReduceUtil
813           // pull job classes
814           job.getMapOutputKeyClass(),
815           job.getMapOutputValueClass(),
816           job.getInputFormatClass(),
817           job.getOutputKeyClass(),
818           job.getOutputValueClass(),
819           job.getOutputFormatClass(),
820           job.getPartitionerClass(),
821           job.getCombinerClass());
822     } catch (ClassNotFoundException e) {
823       throw new IOException(e);
824     }
825   }
826 
827   /**
828    * Add the jars containing the given classes to the job's configuration
829    * such that JobClient will ship them to the cluster and add them to
830    * the DistributedCache.
831    */
832   public static void addDependencyJars(Configuration conf,
833       Class<?>... classes) throws IOException {
834 
835     FileSystem localFs = FileSystem.getLocal(conf);
836     Set<String> jars = new HashSet<String>();
837     // Add jars that are already in the tmpjars variable
838     jars.addAll(conf.getStringCollection("tmpjars"));
839 
840     // add jars as we find them to a map of contents jar name so that we can avoid
841     // creating new jars for classes that have already been packaged.
842     Map<String, String> packagedClasses = new HashMap<String, String>();
843 
844     // Add jars containing the specified classes
845     for (Class<?> clazz : classes) {
846       if (clazz == null) continue;
847 
848       Path path = findOrCreateJar(clazz, localFs, packagedClasses);
849       if (path == null) {
850         LOG.warn("Could not find jar for class " + clazz +
851                  " in order to ship it to the cluster.");
852         continue;
853       }
854       if (!localFs.exists(path)) {
855         LOG.warn("Could not validate jar file " + path + " for class "
856                  + clazz);
857         continue;
858       }
859       jars.add(path.toString());
860     }
861     if (jars.isEmpty()) return;
862 
863     conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[jars.size()])));
864   }
865 
866   /**
867    * If org.apache.hadoop.util.JarFinder is available (0.23+ hadoop), finds
868    * the Jar for a class or creates it if it doesn't exist. If the class is in
869    * a directory in the classpath, it creates a Jar on the fly with the
870    * contents of the directory and returns the path to that Jar. If a Jar is
871    * created, it is created in the system temporary directory. Otherwise,
872    * returns an existing jar that contains a class of the same name. Maintains
873    * a mapping from jar contents to the tmp jar created.
874    * @param my_class the class to find.
875    * @param fs the FileSystem with which to qualify the returned path.
876    * @param packagedClasses a map of class name to path.
877    * @return a jar file that contains the class.
878    * @throws IOException
879    */
880   private static Path findOrCreateJar(Class<?> my_class, FileSystem fs,
881       Map<String, String> packagedClasses)
882   throws IOException {
883     // attempt to locate an existing jar for the class.
884     String jar = findContainingJar(my_class, packagedClasses);
885     if (null == jar || jar.isEmpty()) {
886       jar = getJar(my_class);
887       updateMap(jar, packagedClasses);
888     }
889 
890     if (null == jar || jar.isEmpty()) {
891       return null;
892     }
893 
894     LOG.debug(String.format("For class %s, using jar %s", my_class.getName(), jar));
895     return new Path(jar).makeQualified(fs);
896   }
897 
898   /**
899    * Add entries to <code>packagedClasses</code> corresponding to class files
900    * contained in <code>jar</code>.
901    * @param jar The jar who's content to list.
902    * @param packagedClasses map[class -> jar]
903    */
904   private static void updateMap(String jar, Map<String, String> packagedClasses) throws IOException {
905     if (null == jar || jar.isEmpty()) {
906       return;
907     }
908     ZipFile zip = null;
909     try {
910       zip = new ZipFile(jar);
911       for (Enumeration<? extends ZipEntry> iter = zip.entries(); iter.hasMoreElements();) {
912         ZipEntry entry = iter.nextElement();
913         if (entry.getName().endsWith("class")) {
914           packagedClasses.put(entry.getName(), jar);
915         }
916       }
917     } finally {
918       if (null != zip) zip.close();
919     }
920   }
921 
922   /**
923    * Find a jar that contains a class of the same name, if any. It will return
924    * a jar file, even if that is not the first thing on the class path that
925    * has a class with the same name. Looks first on the classpath and then in
926    * the <code>packagedClasses</code> map.
927    * @param my_class the class to find.
928    * @return a jar file that contains the class, or null.
929    * @throws IOException
930    */
931   private static String findContainingJar(Class<?> my_class, Map<String, String> packagedClasses)
932       throws IOException {
933     ClassLoader loader = my_class.getClassLoader();
934 
935     String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
936 
937     if (loader != null) {
938       // first search the classpath
939       for (Enumeration<URL> itr = loader.getResources(class_file); itr.hasMoreElements();) {
940         URL url = itr.nextElement();
941         if ("jar".equals(url.getProtocol())) {
942           String toReturn = url.getPath();
943           if (toReturn.startsWith("file:")) {
944             toReturn = toReturn.substring("file:".length());
945           }
946           // URLDecoder is a misnamed class, since it actually decodes
947           // x-www-form-urlencoded MIME type rather than actual
948           // URL encoding (which the file path has). Therefore it would
949           // decode +s to ' 's which is incorrect (spaces are actually
950           // either unencoded or encoded as "%20"). Replace +s first, so
951           // that they are kept sacred during the decoding process.
952           toReturn = toReturn.replaceAll("\\+", "%2B");
953           toReturn = URLDecoder.decode(toReturn, "UTF-8");
954           return toReturn.replaceAll("!.*$", "");
955         }
956       }
957     }
958 
959     // now look in any jars we've packaged using JarFinder. Returns null when
960     // no jar is found.
961     return packagedClasses.get(class_file);
962   }
963 
964   /**
965    * Invoke 'getJar' on a JarFinder implementation. Useful for some job
966    * configuration contexts (HBASE-8140) and also for testing on MRv2. First
967    * check if we have HADOOP-9426. Lacking that, fall back to the backport.
968    * @param my_class the class to find.
969    * @return a jar file that contains the class, or null.
970    */
971   private static String getJar(Class<?> my_class) {
972     String ret = null;
973     String hadoopJarFinder = "org.apache.hadoop.util.JarFinder";
974     Class<?> jarFinder = null;
975     try {
976       LOG.debug("Looking for " + hadoopJarFinder + ".");
977       jarFinder = Class.forName(hadoopJarFinder);
978       LOG.debug(hadoopJarFinder + " found.");
979       Method getJar = jarFinder.getMethod("getJar", Class.class);
980       ret = (String) getJar.invoke(null, my_class);
981     } catch (ClassNotFoundException e) {
982       LOG.debug("Using backported JarFinder.");
983       ret = JarFinder.getJar(my_class);
984     } catch (InvocationTargetException e) {
985       // function was properly called, but threw it's own exception. Unwrap it
986       // and pass it on.
987       throw new RuntimeException(e.getCause());
988     } catch (Exception e) {
989       // toss all other exceptions, related to reflection failure
990       throw new RuntimeException("getJar invocation failed.", e);
991     }
992 
993     return ret;
994   }
995 }