View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.mapreduce;
20  
21  import java.io.IOException;
22  import java.lang.reflect.InvocationTargetException;
23  import java.lang.reflect.Method;
24  import java.net.URL;
25  import java.net.URLDecoder;
26  import java.util.ArrayList;
27  import java.util.Enumeration;
28  import java.util.HashSet;
29  import java.util.List;
30  import java.util.Set;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.classification.InterfaceAudience;
35  import org.apache.hadoop.classification.InterfaceStability;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FileSystem;
38  import org.apache.hadoop.fs.Path;
39  import org.apache.hadoop.hbase.HBaseConfiguration;
40  import org.apache.hadoop.hbase.catalog.MetaReader;
41  import org.apache.hadoop.hbase.client.Put;
42  import org.apache.hadoop.hbase.client.Scan;
43  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
44  import org.apache.hadoop.hbase.mapreduce.hadoopbackport.JarFinder;
45  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
46  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
47  import org.apache.hadoop.hbase.security.User;
48  import org.apache.hadoop.hbase.util.Base64;
49  import org.apache.hadoop.hbase.util.Bytes;
50  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
51  import org.apache.hadoop.io.Writable;
52  import org.apache.hadoop.io.WritableComparable;
53  import org.apache.hadoop.mapreduce.InputFormat;
54  import org.apache.hadoop.mapreduce.Job;
55  import org.apache.hadoop.util.StringUtils;
56  
57  import com.google.protobuf.InvalidProtocolBufferException;
58  
59  /**
60   * Utility for {@link TableMapper} and {@link TableReducer}
61   */
62  @SuppressWarnings("unchecked")
63  @InterfaceAudience.Public
64  @InterfaceStability.Stable
65  public class TableMapReduceUtil {
66    static Log LOG = LogFactory.getLog(TableMapReduceUtil.class);
67  
68    /**
69     * Use this before submitting a TableMap job. It will appropriately set up
70     * the job.
71     *
72     * @param table  The table name to read from.
73     * @param scan  The scan instance with the columns, time range etc.
74     * @param mapper  The mapper class to use.
75     * @param outputKeyClass  The class of the output key.
76     * @param outputValueClass  The class of the output value.
77     * @param job  The current job to adjust.  Make sure the passed job is
78     * carrying all necessary HBase configuration.
79     * @throws IOException When setting up the details fails.
80     */
81    public static void initTableMapperJob(String table, Scan scan,
82        Class<? extends TableMapper> mapper,
83        Class<?> outputKeyClass,
84        Class<?> outputValueClass, Job job)
85    throws IOException {
86      initTableMapperJob(table, scan, mapper, outputKeyClass, outputValueClass,
87          job, true);
88    }
89  
90  
91    /**
92     * Use this before submitting a TableMap job. It will appropriately set up
93     * the job.
94     *
95     * @param table Binary representation of the table name to read from.
96     * @param scan  The scan instance with the columns, time range etc.
97     * @param mapper  The mapper class to use.
98     * @param outputKeyClass  The class of the output key.
99     * @param outputValueClass  The class of the output value.
100    * @param job  The current job to adjust.  Make sure the passed job is
101    * carrying all necessary HBase configuration.
102    * @throws IOException When setting up the details fails.
103    */
104    public static void initTableMapperJob(byte[] table, Scan scan,
105       Class<? extends TableMapper> mapper,
106       Class<?> outputKeyClass,
107       Class<?> outputValueClass, Job job)
108   throws IOException {
109       initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass, outputValueClass,
110               job, true);
111   }
112 
113   /**
114    * Use this before submitting a TableMap job. It will appropriately set up
115    * the job.
116    *
117    * @param table  The table name to read from.
118    * @param scan  The scan instance with the columns, time range etc.
119    * @param mapper  The mapper class to use.
120    * @param outputKeyClass  The class of the output key.
121    * @param outputValueClass  The class of the output value.
122    * @param job  The current job to adjust.  Make sure the passed job is
123    * carrying all necessary HBase configuration.
124    * @param addDependencyJars upload HBase jars and jars for any of the configured
125    *           job classes via the distributed cache (tmpjars).
126    * @throws IOException When setting up the details fails.
127    */
128   public static void initTableMapperJob(String table, Scan scan,
129       Class<? extends TableMapper> mapper,
130       Class<?> outputKeyClass,
131       Class<?> outputValueClass, Job job,
132       boolean addDependencyJars, Class<? extends InputFormat> inputFormatClass)
133   throws IOException {
134     job.setInputFormatClass(inputFormatClass);
135     if (outputValueClass != null) job.setMapOutputValueClass(outputValueClass);
136     if (outputKeyClass != null) job.setMapOutputKeyClass(outputKeyClass);
137     job.setMapperClass(mapper);
138     if (Put.class.equals(outputValueClass)) {
139       job.setCombinerClass(PutCombiner.class);
140     }
141     Configuration conf = job.getConfiguration();
142     HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
143     conf.set(TableInputFormat.INPUT_TABLE, table);
144     conf.set(TableInputFormat.SCAN, convertScanToString(scan));
145     conf.setStrings("io.serializations", conf.get("io.serializations"),
146         MutationSerialization.class.getName(), ResultSerialization.class.getName(),
147         KeyValueSerialization.class.getName());
148     if (addDependencyJars) {
149       addDependencyJars(job);
150     }
151     initCredentials(job);
152   }
153 
154   /**
155    * Use this before submitting a TableMap job. It will appropriately set up
156    * the job.
157    *
158    * @param table Binary representation of the table name to read from.
159    * @param scan  The scan instance with the columns, time range etc.
160    * @param mapper  The mapper class to use.
161    * @param outputKeyClass  The class of the output key.
162    * @param outputValueClass  The class of the output value.
163    * @param job  The current job to adjust.  Make sure the passed job is
164    * carrying all necessary HBase configuration.
165    * @param addDependencyJars upload HBase jars and jars for any of the configured
166    *           job classes via the distributed cache (tmpjars).
167    * @param inputFormatClass The class of the input format
168    * @throws IOException When setting up the details fails.
169    */
170   public static void initTableMapperJob(byte[] table, Scan scan,
171       Class<? extends TableMapper> mapper,
172       Class<?> outputKeyClass,
173       Class<?> outputValueClass, Job job,
174       boolean addDependencyJars, Class<? extends InputFormat> inputFormatClass)
175   throws IOException {
176       initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass,
177               outputValueClass, job, addDependencyJars, inputFormatClass);
178   }
179 
180   /**
181    * Use this before submitting a TableMap job. It will appropriately set up
182    * the job.
183    *
184    * @param table Binary representation of the table name to read from.
185    * @param scan  The scan instance with the columns, time range etc.
186    * @param mapper  The mapper class to use.
187    * @param outputKeyClass  The class of the output key.
188    * @param outputValueClass  The class of the output value.
189    * @param job  The current job to adjust.  Make sure the passed job is
190    * carrying all necessary HBase configuration.
191    * @param addDependencyJars upload HBase jars and jars for any of the configured
192    *           job classes via the distributed cache (tmpjars).
193    * @throws IOException When setting up the details fails.
194    */
195   public static void initTableMapperJob(byte[] table, Scan scan,
196       Class<? extends TableMapper> mapper,
197       Class<?> outputKeyClass,
198       Class<?> outputValueClass, Job job,
199       boolean addDependencyJars)
200   throws IOException {
201       initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass,
202               outputValueClass, job, addDependencyJars, TableInputFormat.class);
203   }
204 
205   /**
206    * Use this before submitting a TableMap job. It will appropriately set up
207    * the job.
208    *
209    * @param table The table name to read from.
210    * @param scan  The scan instance with the columns, time range etc.
211    * @param mapper  The mapper class to use.
212    * @param outputKeyClass  The class of the output key.
213    * @param outputValueClass  The class of the output value.
214    * @param job  The current job to adjust.  Make sure the passed job is
215    * carrying all necessary HBase configuration.
216    * @param addDependencyJars upload HBase jars and jars for any of the configured
217    *           job classes via the distributed cache (tmpjars).
218    * @throws IOException When setting up the details fails.
219    */
220   public static void initTableMapperJob(String table, Scan scan,
221       Class<? extends TableMapper> mapper,
222       Class<?> outputKeyClass,
223       Class<?> outputValueClass, Job job,
224       boolean addDependencyJars)
225   throws IOException {
226       initTableMapperJob(table, scan, mapper, outputKeyClass,
227               outputValueClass, job, addDependencyJars, TableInputFormat.class);
228   }
229 
230   /**
231    * Use this before submitting a Multi TableMap job. It will appropriately set
232    * up the job.
233    *
234    * @param scans The list of {@link Scan} objects to read from.
235    * @param mapper The mapper class to use.
236    * @param outputKeyClass The class of the output key.
237    * @param outputValueClass The class of the output value.
238    * @param job The current job to adjust. Make sure the passed job is carrying
239    *          all necessary HBase configuration.
240    * @throws IOException When setting up the details fails.
241    */
242   public static void initTableMapperJob(List<Scan> scans,
243       Class<? extends TableMapper> mapper,
244       Class<? extends WritableComparable> outputKeyClass,
245       Class<? extends Writable> outputValueClass, Job job) throws IOException {
246     initTableMapperJob(scans, mapper, outputKeyClass, outputValueClass, job,
247         true);
248   }
249 
250   /**
251    * Use this before submitting a Multi TableMap job. It will appropriately set
252    * up the job.
253    *
254    * @param scans The list of {@link Scan} objects to read from.
255    * @param mapper The mapper class to use.
256    * @param outputKeyClass The class of the output key.
257    * @param outputValueClass The class of the output value.
258    * @param job The current job to adjust. Make sure the passed job is carrying
259    *          all necessary HBase configuration.
260    * @param addDependencyJars upload HBase jars and jars for any of the
261    *          configured job classes via the distributed cache (tmpjars).
262    * @throws IOException When setting up the details fails.
263    */
264   public static void initTableMapperJob(List<Scan> scans,
265       Class<? extends TableMapper> mapper,
266       Class<? extends WritableComparable> outputKeyClass,
267       Class<? extends Writable> outputValueClass, Job job,
268       boolean addDependencyJars) throws IOException {
269     job.setInputFormatClass(MultiTableInputFormat.class);
270     if (outputValueClass != null) {
271       job.setMapOutputValueClass(outputValueClass);
272     }
273     if (outputKeyClass != null) {
274       job.setMapOutputKeyClass(outputKeyClass);
275     }
276     job.setMapperClass(mapper);
277     HBaseConfiguration.addHbaseResources(job.getConfiguration());
278     List<String> scanStrings = new ArrayList<String>();
279 
280     for (Scan scan : scans) {
281       scanStrings.add(convertScanToString(scan));
282     }
283     job.getConfiguration().setStrings(MultiTableInputFormat.SCANS,
284       scanStrings.toArray(new String[scanStrings.size()]));
285 
286     if (addDependencyJars) {
287       addDependencyJars(job);
288     }
289   }
290 
291   public static void initCredentials(Job job) throws IOException {
292     if (User.isHBaseSecurityEnabled(job.getConfiguration())) {
293       try {
294         // init credentials for remote cluster
295         String quorumAddress = job.getConfiguration().get(TableOutputFormat.QUORUM_ADDRESS);
296         if (quorumAddress != null) {
297           Configuration peerConf = HBaseConfiguration.create(job.getConfiguration());
298           ZKUtil.applyClusterKeyToConf(peerConf, quorumAddress);
299           User.getCurrent().obtainAuthTokenForJob(peerConf, job);
300         }
301         User.getCurrent().obtainAuthTokenForJob(job.getConfiguration(), job);
302       } catch (InterruptedException ie) {
303         LOG.info("Interrupted obtaining user authentication token");
304         Thread.interrupted();
305       }
306     }
307   }
308 
309   /**
310    * Writes the given scan into a Base64 encoded string.
311    *
312    * @param scan  The scan to write out.
313    * @return The scan saved in a Base64 encoded string.
314    * @throws IOException When writing the scan fails.
315    */
316   static String convertScanToString(Scan scan) throws IOException {
317     ClientProtos.Scan proto = ProtobufUtil.toScan(scan);
318     return Base64.encodeBytes(proto.toByteArray());
319   }
320 
321   /**
322    * Converts the given Base64 string back into a Scan instance.
323    *
324    * @param base64  The scan details.
325    * @return The newly created Scan instance.
326    * @throws IOException When reading the scan instance fails.
327    */
328   static Scan convertStringToScan(String base64) throws IOException {
329     byte [] decoded = Base64.decode(base64);
330     ClientProtos.Scan scan;
331     try {
332       scan = ClientProtos.Scan.parseFrom(decoded);
333     } catch (InvalidProtocolBufferException ipbe) {
334       throw new IOException(ipbe);
335     }
336 
337     return ProtobufUtil.toScan(scan);
338   }
339 
340   /**
341    * Use this before submitting a TableReduce job. It will
342    * appropriately set up the JobConf.
343    *
344    * @param table  The output table.
345    * @param reducer  The reducer class to use.
346    * @param job  The current job to adjust.
347    * @throws IOException When determining the region count fails.
348    */
349   public static void initTableReducerJob(String table,
350     Class<? extends TableReducer> reducer, Job job)
351   throws IOException {
352     initTableReducerJob(table, reducer, job, null);
353   }
354 
355   /**
356    * Use this before submitting a TableReduce job. It will
357    * appropriately set up the JobConf.
358    *
359    * @param table  The output table.
360    * @param reducer  The reducer class to use.
361    * @param job  The current job to adjust.
362    * @param partitioner  Partitioner to use. Pass <code>null</code> to use
363    * default partitioner.
364    * @throws IOException When determining the region count fails.
365    */
366   public static void initTableReducerJob(String table,
367     Class<? extends TableReducer> reducer, Job job,
368     Class partitioner) throws IOException {
369     initTableReducerJob(table, reducer, job, partitioner, null, null, null);
370   }
371 
372   /**
373    * Use this before submitting a TableReduce job. It will
374    * appropriately set up the JobConf.
375    *
376    * @param table  The output table.
377    * @param reducer  The reducer class to use.
378    * @param job  The current job to adjust.  Make sure the passed job is
379    * carrying all necessary HBase configuration.
380    * @param partitioner  Partitioner to use. Pass <code>null</code> to use
381    * default partitioner.
382    * @param quorumAddress Distant cluster to write to; default is null for
383    * output to the cluster that is designated in <code>hbase-site.xml</code>.
384    * Set this String to the zookeeper ensemble of an alternate remote cluster
385    * when you would have the reduce write a cluster that is other than the
386    * default; e.g. copying tables between clusters, the source would be
387    * designated by <code>hbase-site.xml</code> and this param would have the
388    * ensemble address of the remote cluster.  The format to pass is particular.
389    * Pass <code> &lt;hbase.zookeeper.quorum>:&lt;hbase.zookeeper.client.port>:&lt;zookeeper.znode.parent>
390    * </code> such as <code>server,server2,server3:2181:/hbase</code>.
391    * @param serverClass redefined hbase.regionserver.class
392    * @param serverImpl redefined hbase.regionserver.impl
393    * @throws IOException When determining the region count fails.
394    */
395   public static void initTableReducerJob(String table,
396     Class<? extends TableReducer> reducer, Job job,
397     Class partitioner, String quorumAddress, String serverClass,
398     String serverImpl) throws IOException {
399     initTableReducerJob(table, reducer, job, partitioner, quorumAddress,
400         serverClass, serverImpl, true);
401   }
402 
403   /**
404    * Use this before submitting a TableReduce job. It will
405    * appropriately set up the JobConf.
406    *
407    * @param table  The output table.
408    * @param reducer  The reducer class to use.
409    * @param job  The current job to adjust.  Make sure the passed job is
410    * carrying all necessary HBase configuration.
411    * @param partitioner  Partitioner to use. Pass <code>null</code> to use
412    * default partitioner.
413    * @param quorumAddress Distant cluster to write to; default is null for
414    * output to the cluster that is designated in <code>hbase-site.xml</code>.
415    * Set this String to the zookeeper ensemble of an alternate remote cluster
416    * when you would have the reduce write a cluster that is other than the
417    * default; e.g. copying tables between clusters, the source would be
418    * designated by <code>hbase-site.xml</code> and this param would have the
419    * ensemble address of the remote cluster.  The format to pass is particular.
420    * Pass <code> &lt;hbase.zookeeper.quorum>:&lt;hbase.zookeeper.client.port>:&lt;zookeeper.znode.parent>
421    * </code> such as <code>server,server2,server3:2181:/hbase</code>.
422    * @param serverClass redefined hbase.regionserver.class
423    * @param serverImpl redefined hbase.regionserver.impl
424    * @param addDependencyJars upload HBase jars and jars for any of the configured
425    *           job classes via the distributed cache (tmpjars).
426    * @throws IOException When determining the region count fails.
427    */
428   public static void initTableReducerJob(String table,
429     Class<? extends TableReducer> reducer, Job job,
430     Class partitioner, String quorumAddress, String serverClass,
431     String serverImpl, boolean addDependencyJars) throws IOException {
432 
433     Configuration conf = job.getConfiguration();
434     HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
435     job.setOutputFormatClass(TableOutputFormat.class);
436     if (reducer != null) job.setReducerClass(reducer);
437     conf.set(TableOutputFormat.OUTPUT_TABLE, table);
438     conf.setStrings("io.serializations", conf.get("io.serializations"),
439         MutationSerialization.class.getName(), ResultSerialization.class.getName());
440     // If passed a quorum/ensemble address, pass it on to TableOutputFormat.
441     if (quorumAddress != null) {
442       // Calling this will validate the format
443       ZKUtil.transformClusterKey(quorumAddress);
444       conf.set(TableOutputFormat.QUORUM_ADDRESS,quorumAddress);
445     }
446     if (serverClass != null && serverImpl != null) {
447       conf.set(TableOutputFormat.REGION_SERVER_CLASS, serverClass);
448       conf.set(TableOutputFormat.REGION_SERVER_IMPL, serverImpl);
449     }
450     job.setOutputKeyClass(ImmutableBytesWritable.class);
451     job.setOutputValueClass(Writable.class);
452     if (partitioner == HRegionPartitioner.class) {
453       job.setPartitionerClass(HRegionPartitioner.class);
454       int regions = MetaReader.getRegionCount(conf, table);
455       if (job.getNumReduceTasks() > regions) {
456         job.setNumReduceTasks(regions);
457       }
458     } else if (partitioner != null) {
459       job.setPartitionerClass(partitioner);
460     }
461 
462     if (addDependencyJars) {
463       addDependencyJars(job);
464     }
465 
466     initCredentials(job);
467   }
468 
469   /**
470    * Ensures that the given number of reduce tasks for the given job
471    * configuration does not exceed the number of regions for the given table.
472    *
473    * @param table  The table to get the region count for.
474    * @param job  The current job to adjust.
475    * @throws IOException When retrieving the table details fails.
476    */
477   public static void limitNumReduceTasks(String table, Job job)
478   throws IOException {
479     int regions = MetaReader.getRegionCount(job.getConfiguration(), table);
480     if (job.getNumReduceTasks() > regions)
481       job.setNumReduceTasks(regions);
482   }
483 
484   /**
485    * Sets the number of reduce tasks for the given job configuration to the
486    * number of regions the given table has.
487    *
488    * @param table  The table to get the region count for.
489    * @param job  The current job to adjust.
490    * @throws IOException When retrieving the table details fails.
491    */
492   public static void setNumReduceTasks(String table, Job job)
493   throws IOException {
494     job.setNumReduceTasks(MetaReader.getRegionCount(job.getConfiguration(), table));
495   }
496 
497   /**
498    * Sets the number of rows to return and cache with each scanner iteration.
499    * Higher caching values will enable faster mapreduce jobs at the expense of
500    * requiring more heap to contain the cached rows.
501    *
502    * @param job The current job to adjust.
503    * @param batchSize The number of rows to return in batch with each scanner
504    * iteration.
505    */
506   public static void setScannerCaching(Job job, int batchSize) {
507     job.getConfiguration().setInt("hbase.client.scanner.caching", batchSize);
508   }
509 
510   /**
511    * Add the HBase dependency jars as well as jars for any of the configured
512    * job classes to the job configuration, so that JobClient will ship them
513    * to the cluster and add them to the DistributedCache.
514    */
515   public static void addDependencyJars(Job job) throws IOException {
516     try {
517       addDependencyJars(job.getConfiguration(),
518           // explicitly pull a class from each module
519           org.apache.hadoop.hbase.HConstants.class,                      // hbase-common
520           org.apache.hadoop.hbase.protobuf.generated.ClientProtos.class, // hbase-protocol
521           org.apache.hadoop.hbase.client.Put.class,                      // hbase-client
522           org.apache.hadoop.hbase.CompatibilityFactory.class,            // hbase-hadoop-compat
523           // pull necessary dependencies
524           org.apache.zookeeper.ZooKeeper.class,
525           com.google.protobuf.Message.class,
526           com.google.common.collect.Lists.class,
527           org.cloudera.htrace.Trace.class,
528           // pull job classes
529           job.getMapOutputKeyClass(),
530           job.getMapOutputValueClass(),
531           job.getInputFormatClass(),
532           job.getOutputKeyClass(),
533           job.getOutputValueClass(),
534           job.getOutputFormatClass(),
535           job.getPartitionerClass(),
536           job.getCombinerClass());
537     } catch (ClassNotFoundException e) {
538       throw new IOException(e);
539     }
540   }
541 
542   /**
543    * Add the jars containing the given classes to the job's configuration
544    * such that JobClient will ship them to the cluster and add them to
545    * the DistributedCache.
546    */
547   public static void addDependencyJars(Configuration conf,
548       Class... classes) throws IOException {
549 
550     FileSystem localFs = FileSystem.getLocal(conf);
551 
552     Set<String> jars = new HashSet<String>();
553 
554     // Add jars that are already in the tmpjars variable
555     jars.addAll( conf.getStringCollection("tmpjars") );
556 
557     // Add jars containing the specified classes
558     for (Class clazz : classes) {
559       if (clazz == null) continue;
560 
561       String pathStr = findOrCreateJar(clazz);
562       if (pathStr == null) {
563         LOG.warn("Could not find jar for class " + clazz +
564                  " in order to ship it to the cluster.");
565         continue;
566       }
567       Path path = new Path(pathStr);
568       if (!localFs.exists(path)) {
569         LOG.warn("Could not validate jar file " + path + " for class "
570                  + clazz);
571         continue;
572       }
573       jars.add(path.makeQualified(localFs).toString());
574     }
575     if (jars.isEmpty()) return;
576 
577     conf.set("tmpjars",
578              StringUtils.arrayToString(jars.toArray(new String[0])));
579   }
580 
581   /**
582    * If org.apache.hadoop.util.JarFinder is available (0.23+ hadoop),
583    * finds the Jar for a class or creates it if it doesn't exist. If
584    * the class is in a directory in the classpath, it creates a Jar
585    * on the fly with the contents of the directory and returns the path
586    * to that Jar. If a Jar is created, it is created in
587    * the system temporary directory.
588    *
589    * Otherwise, returns an existing jar that contains a class of the
590    * same name.
591    *
592    * @param my_class the class to find.
593    * @return a jar file that contains the class, or null.
594    * @throws IOException
595    */
596   private static String findOrCreateJar(Class<?> my_class)
597   throws IOException {
598     try {
599       Class<?> jarFinder = Class.forName("org.apache.hadoop.util.JarFinder");
600       // hadoop-0.23 has a JarFinder class that will create the jar
601       // if it doesn't exist.  Note that this is needed to run the mapreduce
602       // unit tests post-0.23, because mapreduce v2 requires the relevant jars
603       // to be in the mr cluster to do output, split, etc.  At unit test time,
604       // the hbase jars do not exist, so we need to create some.
605       Method m = jarFinder.getMethod("getJar", Class.class);
606       return (String)m.invoke(null,my_class);
607     } catch (InvocationTargetException ite) {
608       // function was properly called, but threw it's own exception
609       throw new IOException(ite.getCause());
610     } catch (Exception e) {
611       // ignore all other exceptions. related to reflection failure
612     }
613 
614     LOG.debug("New JarFinder: org.apache.hadoop.util.JarFinder.getJar " +
615         "not available. Falling back to backported JarFinder");
616     // Use JarFinder because it will construct a jar from class files when
617     // one does not exist. This is relevant for cases when an HBase MR job
618     // is created in the context of another MR job (particularly common for
619     // tools consuming the bulk import APIs). In that case, the dependency
620     // jars have already been shipped to and expanded in the job's working
621     // directory, so it has no jars to package. We could just construct a
622     // classpath from those class files, but we don't know the context: are
623     // they on the local filesystem, are they are ephemeral tmp files, &c.
624     // Better to package them up and ship them via the normal means.
625     return JarFinder.getJar(my_class);
626   }
627 }