View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.mapreduce;
20  
21  import java.io.File;
22  import java.io.IOException;
23  import java.lang.reflect.InvocationTargetException;
24  import java.lang.reflect.Method;
25  import java.net.URL;
26  import java.net.URLDecoder;
27  import java.util.ArrayList;
28  import java.util.Enumeration;
29  import java.util.HashMap;
30  import java.util.HashSet;
31  import java.util.List;
32  import java.util.Map;
33  import java.util.Set;
34  import java.util.zip.ZipEntry;
35  import java.util.zip.ZipFile;
36  
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.hadoop.hbase.classification.InterfaceAudience;
40  import org.apache.hadoop.hbase.classification.InterfaceStability;
41  import org.apache.hadoop.conf.Configuration;
42  import org.apache.hadoop.fs.FileSystem;
43  import org.apache.hadoop.fs.Path;
44  import org.apache.hadoop.hbase.HBaseConfiguration;
45  import org.apache.hadoop.hbase.HConstants;
46  import org.apache.hadoop.hbase.catalog.MetaReader;
47  import org.apache.hadoop.hbase.client.HConnection;
48  import org.apache.hadoop.hbase.client.HConnectionManager;
49  import org.apache.hadoop.hbase.client.Put;
50  import org.apache.hadoop.hbase.client.Scan;
51  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
52  import org.apache.hadoop.hbase.mapreduce.hadoopbackport.JarFinder;
53  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
54  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
55  import org.apache.hadoop.hbase.security.User;
56  import org.apache.hadoop.hbase.security.UserProvider;
57  import org.apache.hadoop.hbase.security.token.TokenUtil;
58  import org.apache.hadoop.hbase.util.Base64;
59  import org.apache.hadoop.hbase.util.Bytes;
60  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
61  import org.apache.hadoop.io.Writable;
62  import org.apache.hadoop.io.WritableComparable;
63  import org.apache.hadoop.mapreduce.InputFormat;
64  import org.apache.hadoop.mapreduce.Job;
65  import org.apache.hadoop.util.StringUtils;
66  import com.google.protobuf.InvalidProtocolBufferException;
67  
68  /**
69   * Utility for {@link TableMapper} and {@link TableReducer}
70   */
71  @SuppressWarnings({ "rawtypes", "unchecked" })
72  @InterfaceAudience.Public
73  @InterfaceStability.Stable
74  public class TableMapReduceUtil {
75    static Log LOG = LogFactory.getLog(TableMapReduceUtil.class);
76  
77    /**
78     * Use this before submitting a TableMap job. It will appropriately set up
79     * the job.
80     *
81     * @param table  The table name to read from.
82     * @param scan  The scan instance with the columns, time range etc.
83     * @param mapper  The mapper class to use.
84     * @param outputKeyClass  The class of the output key.
85     * @param outputValueClass  The class of the output value.
86     * @param job  The current job to adjust.  Make sure the passed job is
87     * carrying all necessary HBase configuration.
88     * @throws IOException When setting up the details fails.
89     */
90    public static void initTableMapperJob(String table, Scan scan,
91        Class<? extends TableMapper> mapper,
92        Class<?> outputKeyClass,
93        Class<?> outputValueClass, Job job)
94    throws IOException {
95      initTableMapperJob(table, scan, mapper, outputKeyClass, outputValueClass,
96          job, true);
97    }
98  
99    /**
100    * Use this before submitting a TableMap job. It will appropriately set up
101    * the job.
102    *
103    * @param table Binary representation of the table name to read from.
104    * @param scan  The scan instance with the columns, time range etc.
105    * @param mapper  The mapper class to use.
106    * @param outputKeyClass  The class of the output key.
107    * @param outputValueClass  The class of the output value.
108    * @param job  The current job to adjust.  Make sure the passed job is
109    * carrying all necessary HBase configuration.
110    * @throws IOException When setting up the details fails.
111    */
112    public static void initTableMapperJob(byte[] table, Scan scan,
113       Class<? extends TableMapper> mapper,
114       Class<?> outputKeyClass,
115       Class<?> outputValueClass, Job job)
116   throws IOException {
117       initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass, outputValueClass,
118               job, true);
119   }
120 
121    /**
122     * Use this before submitting a TableMap job. It will appropriately set up
123     * the job.
124     *
125     * @param table  The table name to read from.
126     * @param scan  The scan instance with the columns, time range etc.
127     * @param mapper  The mapper class to use.
128     * @param outputKeyClass  The class of the output key.
129     * @param outputValueClass  The class of the output value.
130     * @param job  The current job to adjust.  Make sure the passed job is
131     * carrying all necessary HBase configuration.
132     * @param addDependencyJars upload HBase jars and jars for any of the configured
133     *           job classes via the distributed cache (tmpjars).
134     * @throws IOException When setting up the details fails.
135     */
136    public static void initTableMapperJob(String table, Scan scan,
137        Class<? extends TableMapper> mapper,
138        Class<?> outputKeyClass,
139        Class<?> outputValueClass, Job job,
140        boolean addDependencyJars, Class<? extends InputFormat> inputFormatClass)
141    throws IOException {
142      initTableMapperJob(table, scan, mapper, outputKeyClass, outputValueClass, job,
143          addDependencyJars, true, inputFormatClass);
144    }
145 
146 
147   /**
148    * Use this before submitting a TableMap job. It will appropriately set up
149    * the job.
150    *
151    * @param table  The table name to read from.
152    * @param scan  The scan instance with the columns, time range etc.
153    * @param mapper  The mapper class to use.
154    * @param outputKeyClass  The class of the output key.
155    * @param outputValueClass  The class of the output value.
156    * @param job  The current job to adjust.  Make sure the passed job is
157    * carrying all necessary HBase configuration.
158    * @param addDependencyJars upload HBase jars and jars for any of the configured
159    *           job classes via the distributed cache (tmpjars).
160    * @param initCredentials whether to initialize hbase auth credentials for the job
161    * @param inputFormatClass the input format
162    * @throws IOException When setting up the details fails.
163    */
164   public static void initTableMapperJob(String table, Scan scan,
165       Class<? extends TableMapper> mapper,
166       Class<?> outputKeyClass,
167       Class<?> outputValueClass, Job job,
168       boolean addDependencyJars, boolean initCredentials,
169       Class<? extends InputFormat> inputFormatClass)
170   throws IOException {
171     job.setInputFormatClass(inputFormatClass);
172     if (outputValueClass != null) job.setMapOutputValueClass(outputValueClass);
173     if (outputKeyClass != null) job.setMapOutputKeyClass(outputKeyClass);
174     job.setMapperClass(mapper);
175     if (Put.class.equals(outputValueClass)) {
176       job.setCombinerClass(PutCombiner.class);
177     }
178     Configuration conf = job.getConfiguration();
179     HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
180     conf.set(TableInputFormat.INPUT_TABLE, table);
181     conf.set(TableInputFormat.SCAN, convertScanToString(scan));
182     conf.setStrings("io.serializations", conf.get("io.serializations"),
183         MutationSerialization.class.getName(), ResultSerialization.class.getName(),
184         KeyValueSerialization.class.getName());
185     if (addDependencyJars) {
186       addDependencyJars(job);
187     }
188     if (initCredentials) {
189       initCredentials(job);
190     }
191   }
192 
193   /**
194    * Use this before submitting a TableMap job. It will appropriately set up
195    * the job.
196    *
197    * @param table Binary representation of the table name to read from.
198    * @param scan  The scan instance with the columns, time range etc.
199    * @param mapper  The mapper class to use.
200    * @param outputKeyClass  The class of the output key.
201    * @param outputValueClass  The class of the output value.
202    * @param job  The current job to adjust.  Make sure the passed job is
203    * carrying all necessary HBase configuration.
204    * @param addDependencyJars upload HBase jars and jars for any of the configured
205    *           job classes via the distributed cache (tmpjars).
206    * @param inputFormatClass The class of the input format
207    * @throws IOException When setting up the details fails.
208    */
209   public static void initTableMapperJob(byte[] table, Scan scan,
210       Class<? extends TableMapper> mapper,
211       Class<?> outputKeyClass,
212       Class<?> outputValueClass, Job job,
213       boolean addDependencyJars, Class<? extends InputFormat> inputFormatClass)
214   throws IOException {
215       initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass,
216               outputValueClass, job, addDependencyJars, inputFormatClass);
217   }
218 
219   /**
220    * Use this before submitting a TableMap job. It will appropriately set up
221    * the job.
222    *
223    * @param table Binary representation of the table name to read from.
224    * @param scan  The scan instance with the columns, time range etc.
225    * @param mapper  The mapper class to use.
226    * @param outputKeyClass  The class of the output key.
227    * @param outputValueClass  The class of the output value.
228    * @param job  The current job to adjust.  Make sure the passed job is
229    * carrying all necessary HBase configuration.
230    * @param addDependencyJars upload HBase jars and jars for any of the configured
231    *           job classes via the distributed cache (tmpjars).
232    * @throws IOException When setting up the details fails.
233    */
234   public static void initTableMapperJob(byte[] table, Scan scan,
235       Class<? extends TableMapper> mapper,
236       Class<?> outputKeyClass,
237       Class<?> outputValueClass, Job job,
238       boolean addDependencyJars)
239   throws IOException {
240       initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass,
241               outputValueClass, job, addDependencyJars, TableInputFormat.class);
242   }
243 
244   /**
245    * Use this before submitting a TableMap job. It will appropriately set up
246    * the job.
247    *
248    * @param table The table name to read from.
249    * @param scan  The scan instance with the columns, time range etc.
250    * @param mapper  The mapper class to use.
251    * @param outputKeyClass  The class of the output key.
252    * @param outputValueClass  The class of the output value.
253    * @param job  The current job to adjust.  Make sure the passed job is
254    * carrying all necessary HBase configuration.
255    * @param addDependencyJars upload HBase jars and jars for any of the configured
256    *           job classes via the distributed cache (tmpjars).
257    * @throws IOException When setting up the details fails.
258    */
259   public static void initTableMapperJob(String table, Scan scan,
260       Class<? extends TableMapper> mapper,
261       Class<?> outputKeyClass,
262       Class<?> outputValueClass, Job job,
263       boolean addDependencyJars)
264   throws IOException {
265       initTableMapperJob(table, scan, mapper, outputKeyClass,
266               outputValueClass, job, addDependencyJars, TableInputFormat.class);
267   }
268 
269   /**
270    * Enable a basic on-heap cache for these jobs. Any BlockCache implementation based on
271    * direct memory will likely cause the map tasks to OOM when opening the region. This
272    * is done here instead of in TableSnapshotRegionRecordReader in case an advanced user
273    * wants to override this behavior in their job.
274    */
275   public static void resetCacheConfig(Configuration conf) {
276     conf.setFloat(
277       HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
278     conf.setFloat("hbase.offheapcache.percentage", 0f);
279     conf.setFloat("hbase.bucketcache.size", 0f);
280     conf.unset("hbase.bucketcache.ioengine");
281   }
282 
283   /**
284    * Sets up the job for reading from a table snapshot. It bypasses hbase servers
285    * and read directly from snapshot files.
286    *
287    * @param snapshotName The name of the snapshot (of a table) to read from.
288    * @param scan  The scan instance with the columns, time range etc.
289    * @param mapper  The mapper class to use.
290    * @param outputKeyClass  The class of the output key.
291    * @param outputValueClass  The class of the output value.
292    * @param job  The current job to adjust.  Make sure the passed job is
293    * carrying all necessary HBase configuration.
294    * @param addDependencyJars upload HBase jars and jars for any of the configured
295    *           job classes via the distributed cache (tmpjars).
296    *
297    * @param tmpRestoreDir a temporary directory to copy the snapshot files into. Current user should
298    * have write permissions to this directory, and this should not be a subdirectory of rootdir.
299    * After the job is finished, restore directory can be deleted.
300    * @throws IOException When setting up the details fails.
301    * @see TableSnapshotInputFormat
302    */
303   public static void initTableSnapshotMapperJob(String snapshotName, Scan scan,
304       Class<? extends TableMapper> mapper,
305       Class<?> outputKeyClass,
306       Class<?> outputValueClass, Job job,
307       boolean addDependencyJars, Path tmpRestoreDir)
308   throws IOException {
309     TableSnapshotInputFormat.setInput(job, snapshotName, tmpRestoreDir);
310     initTableMapperJob(snapshotName, scan, mapper, outputKeyClass,
311         outputValueClass, job, addDependencyJars, false, TableSnapshotInputFormat.class);
312     resetCacheConfig(job.getConfiguration());
313   }
314 
315   /**
316    * Use this before submitting a Multi TableMap job. It will appropriately set
317    * up the job.
318    *
319    * @param scans The list of {@link Scan} objects to read from.
320    * @param mapper The mapper class to use.
321    * @param outputKeyClass The class of the output key.
322    * @param outputValueClass The class of the output value.
323    * @param job The current job to adjust. Make sure the passed job is carrying
324    *          all necessary HBase configuration.
325    * @throws IOException When setting up the details fails.
326    */
327   public static void initTableMapperJob(List<Scan> scans,
328       Class<? extends TableMapper> mapper,
329       Class<? extends WritableComparable> outputKeyClass,
330       Class<? extends Writable> outputValueClass, Job job) throws IOException {
331     initTableMapperJob(scans, mapper, outputKeyClass, outputValueClass, job,
332         true);
333   }
334 
335   /**
336    * Use this before submitting a Multi TableMap job. It will appropriately set
337    * up the job.
338    *
339    * @param scans The list of {@link Scan} objects to read from.
340    * @param mapper The mapper class to use.
341    * @param outputKeyClass The class of the output key.
342    * @param outputValueClass The class of the output value.
343    * @param job The current job to adjust. Make sure the passed job is carrying
344    *          all necessary HBase configuration.
345    * @param addDependencyJars upload HBase jars and jars for any of the
346    *          configured job classes via the distributed cache (tmpjars).
347    * @throws IOException When setting up the details fails.
348    */
349   public static void initTableMapperJob(List<Scan> scans,
350       Class<? extends TableMapper> mapper,
351       Class<? extends WritableComparable> outputKeyClass,
352       Class<? extends Writable> outputValueClass, Job job,
353       boolean addDependencyJars) throws IOException {
354     initTableMapperJob(scans, mapper, outputKeyClass, outputValueClass, job,
355       addDependencyJars, true);
356   }
357 
358   /**
359    * Use this before submitting a Multi TableMap job. It will appropriately set
360    * up the job.
361    *
362    * @param scans The list of {@link Scan} objects to read from.
363    * @param mapper The mapper class to use.
364    * @param outputKeyClass The class of the output key.
365    * @param outputValueClass The class of the output value.
366    * @param job The current job to adjust. Make sure the passed job is carrying
367    *          all necessary HBase configuration.
368    * @param addDependencyJars upload HBase jars and jars for any of the
369    *          configured job classes via the distributed cache (tmpjars).
370    * @param initCredentials whether to initialize hbase auth credentials for the job
371    * @throws IOException When setting up the details fails.
372    */
373   public static void initTableMapperJob(List<Scan> scans,
374       Class<? extends TableMapper> mapper,
375       Class<? extends WritableComparable> outputKeyClass,
376       Class<? extends Writable> outputValueClass, Job job,
377       boolean addDependencyJars,
378       boolean initCredentials) throws IOException {
379     job.setInputFormatClass(MultiTableInputFormat.class);
380     if (outputValueClass != null) {
381       job.setMapOutputValueClass(outputValueClass);
382     }
383     if (outputKeyClass != null) {
384       job.setMapOutputKeyClass(outputKeyClass);
385     }
386     job.setMapperClass(mapper);
387     Configuration conf = job.getConfiguration();
388     HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
389     List<String> scanStrings = new ArrayList<String>();
390 
391     for (Scan scan : scans) {
392       scanStrings.add(convertScanToString(scan));
393     }
394     job.getConfiguration().setStrings(MultiTableInputFormat.SCANS,
395       scanStrings.toArray(new String[scanStrings.size()]));
396 
397     if (addDependencyJars) {
398       addDependencyJars(job);
399     }
400 
401     if (initCredentials) {
402       initCredentials(job);
403     }
404   }
405 
406   public static void initCredentials(Job job) throws IOException {
407     UserProvider userProvider = UserProvider.instantiate(job.getConfiguration());
408     if (userProvider.isHadoopSecurityEnabled()) {
409       // propagate delegation related props from launcher job to MR job
410       if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
411         job.getConfiguration().set("mapreduce.job.credentials.binary",
412                                    System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
413       }
414     }
415 
416     if (userProvider.isHBaseSecurityEnabled()) {
417       try {
418         // init credentials for remote cluster
419         String quorumAddress = job.getConfiguration().get(TableOutputFormat.QUORUM_ADDRESS);
420         User user = userProvider.getCurrent();
421         if (quorumAddress != null) {
422           Configuration peerConf = HBaseConfiguration.create(job.getConfiguration());
423           ZKUtil.applyClusterKeyToConf(peerConf, quorumAddress);
424           HConnection peerConn = HConnectionManager.createConnection(peerConf);
425           try {
426             TokenUtil.addTokenForJob(peerConn, user, job);
427           } finally {
428             peerConn.close();
429           }
430         }
431 
432         HConnection conn = HConnectionManager.createConnection(job.getConfiguration());
433         try {
434           TokenUtil.addTokenForJob(conn, user, job);
435         } finally {
436           conn.close();
437         }
438       } catch (InterruptedException ie) {
439         LOG.info("Interrupted obtaining user authentication token");
440         Thread.currentThread().interrupt();
441       }
442     }
443   }
444 
445   /**
446    * Obtain an authentication token, for the specified cluster, on behalf of the current user
447    * and add it to the credentials for the given map reduce job.
448    *
449    * The quorumAddress is the key to the ZK ensemble, which contains:
450    * hbase.zookeeper.quorum, hbase.zookeeper.client.port and zookeeper.znode.parent
451    *
452    * @param job The job that requires the permission.
453    * @param quorumAddress string that contains the 3 required configuratins
454    * @throws IOException When the authentication token cannot be obtained.
455    */
456   public static void initCredentialsForCluster(Job job, String quorumAddress)
457       throws IOException {
458     UserProvider userProvider = UserProvider.instantiate(job.getConfiguration());
459     if (userProvider.isHBaseSecurityEnabled()) {
460       try {
461         Configuration peerConf = HBaseConfiguration.create(job.getConfiguration());
462         ZKUtil.applyClusterKeyToConf(peerConf, quorumAddress);
463         HConnection peerConn = HConnectionManager.createConnection(peerConf);
464         try {
465           TokenUtil.addTokenForJob(peerConn, userProvider.getCurrent(), job);
466         } finally {
467           peerConn.close();
468         }
469       } catch (InterruptedException e) {
470         LOG.info("Interrupted obtaining user authentication token");
471         Thread.interrupted();
472       }
473     }
474   }
475 
476   /**
477    * Writes the given scan into a Base64 encoded string.
478    *
479    * @param scan  The scan to write out.
480    * @return The scan saved in a Base64 encoded string.
481    * @throws IOException When writing the scan fails.
482    */
483   static String convertScanToString(Scan scan) throws IOException {
484     ClientProtos.Scan proto = ProtobufUtil.toScan(scan);
485     return Base64.encodeBytes(proto.toByteArray());
486   }
487 
488   /**
489    * Converts the given Base64 string back into a Scan instance.
490    *
491    * @param base64  The scan details.
492    * @return The newly created Scan instance.
493    * @throws IOException When reading the scan instance fails.
494    */
495   static Scan convertStringToScan(String base64) throws IOException {
496     byte [] decoded = Base64.decode(base64);
497     ClientProtos.Scan scan;
498     try {
499       scan = ClientProtos.Scan.parseFrom(decoded);
500     } catch (InvalidProtocolBufferException ipbe) {
501       throw new IOException(ipbe);
502     }
503 
504     return ProtobufUtil.toScan(scan);
505   }
506 
507   /**
508    * Use this before submitting a TableReduce job. It will
509    * appropriately set up the JobConf.
510    *
511    * @param table  The output table.
512    * @param reducer  The reducer class to use.
513    * @param job  The current job to adjust.
514    * @throws IOException When determining the region count fails.
515    */
516   public static void initTableReducerJob(String table,
517     Class<? extends TableReducer> reducer, Job job)
518   throws IOException {
519     initTableReducerJob(table, reducer, job, null);
520   }
521 
522   /**
523    * Use this before submitting a TableReduce job. It will
524    * appropriately set up the JobConf.
525    *
526    * @param table  The output table.
527    * @param reducer  The reducer class to use.
528    * @param job  The current job to adjust.
529    * @param partitioner  Partitioner to use. Pass <code>null</code> to use
530    * default partitioner.
531    * @throws IOException When determining the region count fails.
532    */
533   public static void initTableReducerJob(String table,
534     Class<? extends TableReducer> reducer, Job job,
535     Class partitioner) throws IOException {
536     initTableReducerJob(table, reducer, job, partitioner, null, null, null);
537   }
538 
539   /**
540    * Use this before submitting a TableReduce job. It will
541    * appropriately set up the JobConf.
542    *
543    * @param table  The output table.
544    * @param reducer  The reducer class to use.
545    * @param job  The current job to adjust.  Make sure the passed job is
546    * carrying all necessary HBase configuration.
547    * @param partitioner  Partitioner to use. Pass <code>null</code> to use
548    * default partitioner.
549    * @param quorumAddress Distant cluster to write to; default is null for
550    * output to the cluster that is designated in <code>hbase-site.xml</code>.
551    * Set this String to the zookeeper ensemble of an alternate remote cluster
552    * when you would have the reduce write a cluster that is other than the
553    * default; e.g. copying tables between clusters, the source would be
554    * designated by <code>hbase-site.xml</code> and this param would have the
555    * ensemble address of the remote cluster.  The format to pass is particular.
556    * Pass <code> &lt;hbase.zookeeper.quorum>:&lt;hbase.zookeeper.client.port>:&lt;zookeeper.znode.parent>
557    * </code> such as <code>server,server2,server3:2181:/hbase</code>.
558    * @param serverClass redefined hbase.regionserver.class
559    * @param serverImpl redefined hbase.regionserver.impl
560    * @throws IOException When determining the region count fails.
561    */
562   public static void initTableReducerJob(String table,
563     Class<? extends TableReducer> reducer, Job job,
564     Class partitioner, String quorumAddress, String serverClass,
565     String serverImpl) throws IOException {
566     initTableReducerJob(table, reducer, job, partitioner, quorumAddress,
567         serverClass, serverImpl, true);
568   }
569 
570   /**
571    * Use this before submitting a TableReduce job. It will
572    * appropriately set up the JobConf.
573    *
574    * @param table  The output table.
575    * @param reducer  The reducer class to use.
576    * @param job  The current job to adjust.  Make sure the passed job is
577    * carrying all necessary HBase configuration.
578    * @param partitioner  Partitioner to use. Pass <code>null</code> to use
579    * default partitioner.
580    * @param quorumAddress Distant cluster to write to; default is null for
581    * output to the cluster that is designated in <code>hbase-site.xml</code>.
582    * Set this String to the zookeeper ensemble of an alternate remote cluster
583    * when you would have the reduce write a cluster that is other than the
584    * default; e.g. copying tables between clusters, the source would be
585    * designated by <code>hbase-site.xml</code> and this param would have the
586    * ensemble address of the remote cluster.  The format to pass is particular.
587    * Pass <code> &lt;hbase.zookeeper.quorum>:&lt;hbase.zookeeper.client.port>:&lt;zookeeper.znode.parent>
588    * </code> such as <code>server,server2,server3:2181:/hbase</code>.
589    * @param serverClass redefined hbase.regionserver.class
590    * @param serverImpl redefined hbase.regionserver.impl
591    * @param addDependencyJars upload HBase jars and jars for any of the configured
592    *           job classes via the distributed cache (tmpjars).
593    * @throws IOException When determining the region count fails.
594    */
595   public static void initTableReducerJob(String table,
596     Class<? extends TableReducer> reducer, Job job,
597     Class partitioner, String quorumAddress, String serverClass,
598     String serverImpl, boolean addDependencyJars) throws IOException {
599 
600     Configuration conf = job.getConfiguration();
601     HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
602     job.setOutputFormatClass(TableOutputFormat.class);
603     if (reducer != null) job.setReducerClass(reducer);
604     conf.set(TableOutputFormat.OUTPUT_TABLE, table);
605     conf.setStrings("io.serializations", conf.get("io.serializations"),
606         MutationSerialization.class.getName(), ResultSerialization.class.getName());
607     // If passed a quorum/ensemble address, pass it on to TableOutputFormat.
608     if (quorumAddress != null) {
609       // Calling this will validate the format
610       ZKUtil.transformClusterKey(quorumAddress);
611       conf.set(TableOutputFormat.QUORUM_ADDRESS,quorumAddress);
612     }
613     if (serverClass != null && serverImpl != null) {
614       conf.set(TableOutputFormat.REGION_SERVER_CLASS, serverClass);
615       conf.set(TableOutputFormat.REGION_SERVER_IMPL, serverImpl);
616     }
617     job.setOutputKeyClass(ImmutableBytesWritable.class);
618     job.setOutputValueClass(Writable.class);
619     if (partitioner == HRegionPartitioner.class) {
620       job.setPartitionerClass(HRegionPartitioner.class);
621       int regions = MetaReader.getRegionCount(conf, table);
622       if (job.getNumReduceTasks() > regions) {
623         job.setNumReduceTasks(regions);
624       }
625     } else if (partitioner != null) {
626       job.setPartitionerClass(partitioner);
627     }
628 
629     if (addDependencyJars) {
630       addDependencyJars(job);
631     }
632 
633     initCredentials(job);
634   }
635 
636   /**
637    * Ensures that the given number of reduce tasks for the given job
638    * configuration does not exceed the number of regions for the given table.
639    *
640    * @param table  The table to get the region count for.
641    * @param job  The current job to adjust.
642    * @throws IOException When retrieving the table details fails.
643    */
644   public static void limitNumReduceTasks(String table, Job job)
645   throws IOException {
646     int regions = MetaReader.getRegionCount(job.getConfiguration(), table);
647     if (job.getNumReduceTasks() > regions)
648       job.setNumReduceTasks(regions);
649   }
650 
651   /**
652    * Sets the number of reduce tasks for the given job configuration to the
653    * number of regions the given table has.
654    *
655    * @param table  The table to get the region count for.
656    * @param job  The current job to adjust.
657    * @throws IOException When retrieving the table details fails.
658    */
659   public static void setNumReduceTasks(String table, Job job)
660   throws IOException {
661     job.setNumReduceTasks(MetaReader.getRegionCount(job.getConfiguration(), table));
662   }
663 
664   /**
665    * Sets the number of rows to return and cache with each scanner iteration.
666    * Higher caching values will enable faster mapreduce jobs at the expense of
667    * requiring more heap to contain the cached rows.
668    *
669    * @param job The current job to adjust.
670    * @param batchSize The number of rows to return in batch with each scanner
671    * iteration.
672    */
673   public static void setScannerCaching(Job job, int batchSize) {
674     job.getConfiguration().setInt("hbase.client.scanner.caching", batchSize);
675   }
676 
677   /**
678    * Add HBase and its dependencies (only) to the job configuration.
679    * <p>
680    * This is intended as a low-level API, facilitating code reuse between this
681    * class and its mapred counterpart. It also of use to extenral tools that
682    * need to build a MapReduce job that interacts with HBase but want
683    * fine-grained control over the jars shipped to the cluster.
684    * </p>
685    * @param conf The Configuration object to extend with dependencies.
686    * @see org.apache.hadoop.hbase.mapred.TableMapReduceUtil
687    * @see <a href="https://issues.apache.org/jira/browse/PIG-3285">PIG-3285</a>
688    */
689   public static void addHBaseDependencyJars(Configuration conf) throws IOException {
690     addDependencyJars(conf,
691       // explicitly pull a class from each module
692       org.apache.hadoop.hbase.HConstants.class,                      // hbase-common
693       org.apache.hadoop.hbase.protobuf.generated.ClientProtos.class, // hbase-protocol
694       org.apache.hadoop.hbase.client.Put.class,                      // hbase-client
695       org.apache.hadoop.hbase.CompatibilityFactory.class,            // hbase-hadoop-compat
696       org.apache.hadoop.hbase.mapreduce.TableMapper.class,           // hbase-server
697       // pull necessary dependencies
698       org.apache.zookeeper.ZooKeeper.class,
699       org.jboss.netty.channel.ChannelFactory.class,
700       com.google.protobuf.Message.class,
701       com.google.common.collect.Lists.class,
702       org.cloudera.htrace.Trace.class,
703       org.cliffc.high_scale_lib.Counter.class,
704       com.yammer.metrics.core.MetricsRegistry.class); // needed for mapred over snapshots
705   }
706 
707   /**
708    * Returns a classpath string built from the content of the "tmpjars" value in {@code conf}.
709    * Also exposed to shell scripts via `bin/hbase mapredcp`.
710    */
711   public static String buildDependencyClasspath(Configuration conf) {
712     if (conf == null) {
713       throw new IllegalArgumentException("Must provide a configuration object.");
714     }
715     Set<String> paths = new HashSet<String>(conf.getStringCollection("tmpjars"));
716     if (paths.size() == 0) {
717       throw new IllegalArgumentException("Configuration contains no tmpjars.");
718     }
719     StringBuilder sb = new StringBuilder();
720     for (String s : paths) {
721       // entries can take the form 'file:/path/to/file.jar'.
722       int idx = s.indexOf(":");
723       if (idx != -1) s = s.substring(idx + 1);
724       if (sb.length() > 0) sb.append(File.pathSeparator);
725       sb.append(s);
726     }
727     return sb.toString();
728   }
729 
730   /**
731    * Add the HBase dependency jars as well as jars for any of the configured
732    * job classes to the job configuration, so that JobClient will ship them
733    * to the cluster and add them to the DistributedCache.
734    */
735   public static void addDependencyJars(Job job) throws IOException {
736     addHBaseDependencyJars(job.getConfiguration());
737     try {
738       addDependencyJars(job.getConfiguration(),
739           // when making changes here, consider also mapred.TableMapReduceUtil
740           // pull job classes
741           job.getMapOutputKeyClass(),
742           job.getMapOutputValueClass(),
743           job.getInputFormatClass(),
744           job.getOutputKeyClass(),
745           job.getOutputValueClass(),
746           job.getOutputFormatClass(),
747           job.getPartitionerClass(),
748           job.getCombinerClass());
749     } catch (ClassNotFoundException e) {
750       throw new IOException(e);
751     }
752   }
753 
754   /**
755    * Add the jars containing the given classes to the job's configuration
756    * such that JobClient will ship them to the cluster and add them to
757    * the DistributedCache.
758    */
759   public static void addDependencyJars(Configuration conf,
760       Class<?>... classes) throws IOException {
761 
762     FileSystem localFs = FileSystem.getLocal(conf);
763     Set<String> jars = new HashSet<String>();
764     // Add jars that are already in the tmpjars variable
765     jars.addAll(conf.getStringCollection("tmpjars"));
766 
767     // add jars as we find them to a map of contents jar name so that we can avoid
768     // creating new jars for classes that have already been packaged.
769     Map<String, String> packagedClasses = new HashMap<String, String>();
770 
771     // Add jars containing the specified classes
772     for (Class<?> clazz : classes) {
773       if (clazz == null) continue;
774 
775       Path path = findOrCreateJar(clazz, localFs, packagedClasses);
776       if (path == null) {
777         LOG.warn("Could not find jar for class " + clazz +
778                  " in order to ship it to the cluster.");
779         continue;
780       }
781       if (!localFs.exists(path)) {
782         LOG.warn("Could not validate jar file " + path + " for class "
783                  + clazz);
784         continue;
785       }
786       jars.add(path.toString());
787     }
788     if (jars.isEmpty()) return;
789 
790     conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[jars.size()])));
791   }
792 
793   /**
794    * If org.apache.hadoop.util.JarFinder is available (0.23+ hadoop), finds
795    * the Jar for a class or creates it if it doesn't exist. If the class is in
796    * a directory in the classpath, it creates a Jar on the fly with the
797    * contents of the directory and returns the path to that Jar. If a Jar is
798    * created, it is created in the system temporary directory. Otherwise,
799    * returns an existing jar that contains a class of the same name. Maintains
800    * a mapping from jar contents to the tmp jar created.
801    * @param my_class the class to find.
802    * @param fs the FileSystem with which to qualify the returned path.
803    * @param packagedClasses a map of class name to path.
804    * @return a jar file that contains the class.
805    * @throws IOException
806    */
807   private static Path findOrCreateJar(Class<?> my_class, FileSystem fs,
808       Map<String, String> packagedClasses)
809   throws IOException {
810     // attempt to locate an existing jar for the class.
811     String jar = findContainingJar(my_class, packagedClasses);
812     if (null == jar || jar.isEmpty()) {
813       jar = getJar(my_class);
814       updateMap(jar, packagedClasses);
815     }
816 
817     if (null == jar || jar.isEmpty()) {
818       return null;
819     }
820 
821     LOG.debug(String.format("For class %s, using jar %s", my_class.getName(), jar));
822     return new Path(jar).makeQualified(fs);
823   }
824 
825   /**
826    * Add entries to <code>packagedClasses</code> corresponding to class files
827    * contained in <code>jar</code>.
828    * @param jar The jar who's content to list.
829    * @param packagedClasses map[class -> jar]
830    */
831   private static void updateMap(String jar, Map<String, String> packagedClasses) throws IOException {
832     if (null == jar || jar.isEmpty()) {
833       return;
834     }
835     ZipFile zip = null;
836     try {
837       zip = new ZipFile(jar);
838       for (Enumeration<? extends ZipEntry> iter = zip.entries(); iter.hasMoreElements();) {
839         ZipEntry entry = iter.nextElement();
840         if (entry.getName().endsWith("class")) {
841           packagedClasses.put(entry.getName(), jar);
842         }
843       }
844     } finally {
845       if (null != zip) zip.close();
846     }
847   }
848 
849   /**
850    * Find a jar that contains a class of the same name, if any. It will return
851    * a jar file, even if that is not the first thing on the class path that
852    * has a class with the same name. Looks first on the classpath and then in
853    * the <code>packagedClasses</code> map.
854    * @param my_class the class to find.
855    * @return a jar file that contains the class, or null.
856    * @throws IOException
857    */
858   private static String findContainingJar(Class<?> my_class, Map<String, String> packagedClasses)
859       throws IOException {
860     ClassLoader loader = my_class.getClassLoader();
861 
862     String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
863 
864     if (loader != null) {
865       // first search the classpath
866       for (Enumeration<URL> itr = loader.getResources(class_file); itr.hasMoreElements();) {
867         URL url = itr.nextElement();
868         if ("jar".equals(url.getProtocol())) {
869           String toReturn = url.getPath();
870           if (toReturn.startsWith("file:")) {
871             toReturn = toReturn.substring("file:".length());
872           }
873           // URLDecoder is a misnamed class, since it actually decodes
874           // x-www-form-urlencoded MIME type rather than actual
875           // URL encoding (which the file path has). Therefore it would
876           // decode +s to ' 's which is incorrect (spaces are actually
877           // either unencoded or encoded as "%20"). Replace +s first, so
878           // that they are kept sacred during the decoding process.
879           toReturn = toReturn.replaceAll("\\+", "%2B");
880           toReturn = URLDecoder.decode(toReturn, "UTF-8");
881           return toReturn.replaceAll("!.*$", "");
882         }
883       }
884     }
885 
886     // now look in any jars we've packaged using JarFinder. Returns null when
887     // no jar is found.
888     return packagedClasses.get(class_file);
889   }
890 
891   /**
892    * Invoke 'getJar' on a JarFinder implementation. Useful for some job
893    * configuration contexts (HBASE-8140) and also for testing on MRv2. First
894    * check if we have HADOOP-9426. Lacking that, fall back to the backport.
895    * @param my_class the class to find.
896    * @return a jar file that contains the class, or null.
897    */
898   private static String getJar(Class<?> my_class) {
899     String ret = null;
900     String hadoopJarFinder = "org.apache.hadoop.util.JarFinder";
901     Class<?> jarFinder = null;
902     try {
903       LOG.debug("Looking for " + hadoopJarFinder + ".");
904       jarFinder = Class.forName(hadoopJarFinder);
905       LOG.debug(hadoopJarFinder + " found.");
906       Method getJar = jarFinder.getMethod("getJar", Class.class);
907       ret = (String) getJar.invoke(null, my_class);
908     } catch (ClassNotFoundException e) {
909       LOG.debug("Using backported JarFinder.");
910       ret = JarFinder.getJar(my_class);
911     } catch (InvocationTargetException e) {
912       // function was properly called, but threw it's own exception. Unwrap it
913       // and pass it on.
914       throw new RuntimeException(e.getCause());
915     } catch (Exception e) {
916       // toss all other exceptions, related to reflection failure
917       throw new RuntimeException("getJar invocation failed.", e);
918     }
919 
920     return ret;
921   }
922 }