1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.mapreduce;
21
22 import java.io.ByteArrayInputStream;
23 import java.io.ByteArrayOutputStream;
24 import java.io.DataInputStream;
25 import java.io.DataOutputStream;
26 import java.io.File;
27 import java.io.IOException;
28 import java.lang.reflect.InvocationTargetException;
29 import java.lang.reflect.Method;
30 import java.net.URL;
31 import java.net.URLDecoder;
32 import java.util.ArrayList;
33 import java.util.Enumeration;
34 import java.util.HashMap;
35 import java.util.HashSet;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.Set;
39 import java.util.zip.ZipEntry;
40 import java.util.zip.ZipFile;
41
42 import org.apache.commons.logging.Log;
43 import org.apache.commons.logging.LogFactory;
44 import org.apache.hadoop.conf.Configuration;
45 import org.apache.hadoop.fs.FileSystem;
46 import org.apache.hadoop.fs.Path;
47 import org.apache.hadoop.hbase.HBaseConfiguration;
48 import org.apache.hadoop.hbase.HConstants;
49 import org.apache.hadoop.hbase.client.HTable;
50 import org.apache.hadoop.hbase.client.Scan;
51 import org.apache.hadoop.hbase.client.UserProvider;
52 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
53 import org.apache.hadoop.hbase.mapreduce.hadoopbackport.JarFinder;
54 import org.apache.hadoop.hbase.security.User;
55 import org.apache.hadoop.hbase.util.Base64;
56 import org.apache.hadoop.hbase.util.Bytes;
57 import org.apache.hadoop.hbase.zookeeper.ClusterId;
58 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
59 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
60 import org.apache.hadoop.io.Writable;
61 import org.apache.hadoop.io.WritableComparable;
62 import org.apache.hadoop.mapreduce.InputFormat;
63 import org.apache.hadoop.mapreduce.Job;
64 import org.apache.hadoop.util.StringUtils;
65 import org.apache.hadoop.security.token.Token;
66 import org.apache.zookeeper.KeeperException;
67
68
69
70
71 @SuppressWarnings("unchecked")
72 public class TableMapReduceUtil {
73 static Log LOG = LogFactory.getLog(TableMapReduceUtil.class);
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88 public static void initTableMapperJob(String table, Scan scan,
89 Class<? extends TableMapper> mapper,
90 Class<? extends WritableComparable> outputKeyClass,
91 Class<? extends Writable> outputValueClass, Job job)
92 throws IOException {
93 initTableMapperJob(table, scan, mapper, outputKeyClass, outputValueClass,
94 job, true);
95 }
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111 public static void initTableMapperJob(byte[] table, Scan scan,
112 Class<? extends TableMapper> mapper,
113 Class<? extends WritableComparable> outputKeyClass,
114 Class<? extends Writable> outputValueClass, Job job)
115 throws IOException {
116 initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass, outputValueClass,
117 job, true);
118 }
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135 public static void initTableMapperJob(String table, Scan scan,
136 Class<? extends TableMapper> mapper,
137 Class<? extends WritableComparable> outputKeyClass,
138 Class<? extends Writable> outputValueClass, Job job,
139 boolean addDependencyJars, Class<? extends InputFormat> inputFormatClass)
140 throws IOException {
141 job.setInputFormatClass(inputFormatClass);
142 if (outputValueClass != null) job.setMapOutputValueClass(outputValueClass);
143 if (outputKeyClass != null) job.setMapOutputKeyClass(outputKeyClass);
144 job.setMapperClass(mapper);
145 Configuration conf = job.getConfiguration();
146 HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
147 conf.set(TableInputFormat.INPUT_TABLE, table);
148 conf.set(TableInputFormat.SCAN, convertScanToString(scan));
149 if (addDependencyJars) {
150 addDependencyJars(job);
151 }
152 initCredentials(job);
153 }
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171 public static void initTableMapperJob(byte[] table, Scan scan,
172 Class<? extends TableMapper> mapper,
173 Class<? extends WritableComparable> outputKeyClass,
174 Class<? extends Writable> outputValueClass, Job job,
175 boolean addDependencyJars, Class<? extends InputFormat> inputFormatClass)
176 throws IOException {
177 initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass,
178 outputValueClass, job, addDependencyJars, inputFormatClass);
179 }
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196 public static void initTableMapperJob(byte[] table, Scan scan,
197 Class<? extends TableMapper> mapper,
198 Class<? extends WritableComparable> outputKeyClass,
199 Class<? extends Writable> outputValueClass, Job job,
200 boolean addDependencyJars)
201 throws IOException {
202 initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass,
203 outputValueClass, job, addDependencyJars, TableInputFormat.class);
204 }
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221 public static void initTableMapperJob(String table, Scan scan,
222 Class<? extends TableMapper> mapper,
223 Class<? extends WritableComparable> outputKeyClass,
224 Class<? extends Writable> outputValueClass, Job job,
225 boolean addDependencyJars)
226 throws IOException {
227 initTableMapperJob(table, scan, mapper, outputKeyClass,
228 outputValueClass, job, addDependencyJars, TableInputFormat.class);
229 }
230
231
232
233
234
235
236
237
238
239
240
241
242
243 public static void initTableMapperJob(List<Scan> scans,
244 Class<? extends TableMapper> mapper,
245 Class<? extends WritableComparable> outputKeyClass,
246 Class<? extends Writable> outputValueClass, Job job) throws IOException {
247 initTableMapperJob(scans, mapper, outputKeyClass, outputValueClass, job,
248 true);
249 }
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265 public static void initTableMapperJob(List<Scan> scans,
266 Class<? extends TableMapper> mapper,
267 Class<? extends WritableComparable> outputKeyClass,
268 Class<? extends Writable> outputValueClass, Job job,
269 boolean addDependencyJars) throws IOException {
270 job.setInputFormatClass(MultiTableInputFormat.class);
271 if (outputValueClass != null) {
272 job.setMapOutputValueClass(outputValueClass);
273 }
274 if (outputKeyClass != null) {
275 job.setMapOutputKeyClass(outputKeyClass);
276 }
277 job.setMapperClass(mapper);
278 HBaseConfiguration.addHbaseResources(job.getConfiguration());
279 List<String> scanStrings = new ArrayList<String>();
280
281 for (Scan scan : scans) {
282 scanStrings.add(convertScanToString(scan));
283 }
284 job.getConfiguration().setStrings(MultiTableInputFormat.SCANS,
285 scanStrings.toArray(new String[scanStrings.size()]));
286
287 if (addDependencyJars) {
288 addDependencyJars(job);
289 }
290 }
291
292 public static void initCredentials(Job job) throws IOException {
293 UserProvider provider = UserProvider.instantiate(job.getConfiguration());
294
295 if (provider.isHadoopSecurityEnabled()) {
296
297 if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
298 job.getConfiguration().set("mapreduce.job.credentials.binary",
299 System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
300 }
301 }
302
303 if (provider.isHBaseSecurityEnabled()) {
304 try {
305
306 String quorumAddress = job.getConfiguration().get(
307 TableOutputFormat.QUORUM_ADDRESS);
308 User user = provider.getCurrent();
309 if (quorumAddress != null) {
310 String[] parts = ZKUtil.transformClusterKey(quorumAddress);
311 Configuration peerConf = HBaseConfiguration.create(job
312 .getConfiguration());
313 peerConf.set(HConstants.ZOOKEEPER_QUORUM, parts[0]);
314 peerConf.set("hbase.zookeeper.client.port", parts[1]);
315 peerConf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parts[2]);
316 obtainAuthTokenForJob(job, peerConf, user);
317 }
318
319 obtainAuthTokenForJob(job, job.getConfiguration(), user);
320 } catch (InterruptedException ie) {
321 LOG.info("Interrupted obtaining user authentication token");
322 Thread.interrupted();
323 }
324 }
325 }
326
327
328
329
330
331
332
333
334
335
336
337
338 public static void initCredentialsForCluster(Job job, String quorumAddress)
339 throws IOException {
340 UserProvider userProvider = UserProvider.instantiate(job.getConfiguration());
341 if (userProvider.isHBaseSecurityEnabled()) {
342 try {
343 Configuration peerConf = HBaseConfiguration.create(job.getConfiguration());
344 ZKUtil.applyClusterKeyToConf(peerConf, quorumAddress);
345 obtainAuthTokenForJob(job, peerConf, userProvider.getCurrent());
346 } catch (InterruptedException e) {
347 LOG.info("Interrupted obtaining user authentication token");
348 Thread.interrupted();
349 }
350 }
351 }
352
353 private static void obtainAuthTokenForJob(Job job, Configuration conf, User user)
354 throws IOException, InterruptedException {
355 Token<?> authToken = getAuthToken(conf, user);
356 if (authToken == null) {
357 user.obtainAuthTokenForJob(conf, job);
358 } else {
359 job.getCredentials().addToken(authToken.getService(), authToken);
360 }
361 }
362
363
364
365
366
367 private static Token<?> getAuthToken(Configuration conf, User user)
368 throws IOException, InterruptedException {
369 ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "mr-init-credentials", null);
370 try {
371 String clusterId = ClusterId.readClusterIdZNode(zkw);
372 return user.getToken("HBASE_AUTH_TOKEN", clusterId);
373 } catch (KeeperException e) {
374 throw new IOException(e);
375 } finally {
376 zkw.close();
377 }
378 }
379
380
381
382
383
384
385
386
387 static String convertScanToString(Scan scan) throws IOException {
388 ByteArrayOutputStream out = new ByteArrayOutputStream();
389 DataOutputStream dos = new DataOutputStream(out);
390 scan.write(dos);
391 return Base64.encodeBytes(out.toByteArray());
392 }
393
394
395
396
397
398
399
400
401 static Scan convertStringToScan(String base64) throws IOException {
402 ByteArrayInputStream bis = new ByteArrayInputStream(Base64.decode(base64));
403 DataInputStream dis = new DataInputStream(bis);
404 Scan scan = new Scan();
405 scan.readFields(dis);
406 return scan;
407 }
408
409
410
411
412
413
414
415
416
417
418 public static void initTableReducerJob(String table,
419 Class<? extends TableReducer> reducer, Job job)
420 throws IOException {
421 initTableReducerJob(table, reducer, job, null);
422 }
423
424
425
426
427
428
429
430
431
432
433
434
435 public static void initTableReducerJob(String table,
436 Class<? extends TableReducer> reducer, Job job,
437 Class partitioner) throws IOException {
438 initTableReducerJob(table, reducer, job, partitioner, null, null, null);
439 }
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464 public static void initTableReducerJob(String table,
465 Class<? extends TableReducer> reducer, Job job,
466 Class partitioner, String quorumAddress, String serverClass,
467 String serverImpl) throws IOException {
468 initTableReducerJob(table, reducer, job, partitioner, quorumAddress,
469 serverClass, serverImpl, true);
470 }
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497 public static void initTableReducerJob(String table,
498 Class<? extends TableReducer> reducer, Job job,
499 Class partitioner, String quorumAddress, String serverClass,
500 String serverImpl, boolean addDependencyJars) throws IOException {
501
502 Configuration conf = job.getConfiguration();
503 HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
504 job.setOutputFormatClass(TableOutputFormat.class);
505 if (reducer != null) job.setReducerClass(reducer);
506 conf.set(TableOutputFormat.OUTPUT_TABLE, table);
507
508 if (quorumAddress != null) {
509
510 ZKUtil.transformClusterKey(quorumAddress);
511 conf.set(TableOutputFormat.QUORUM_ADDRESS,quorumAddress);
512 }
513 if (serverClass != null && serverImpl != null) {
514 conf.set(TableOutputFormat.REGION_SERVER_CLASS, serverClass);
515 conf.set(TableOutputFormat.REGION_SERVER_IMPL, serverImpl);
516 }
517 job.setOutputKeyClass(ImmutableBytesWritable.class);
518 job.setOutputValueClass(Writable.class);
519 if (partitioner == HRegionPartitioner.class) {
520 job.setPartitionerClass(HRegionPartitioner.class);
521 HTable outputTable = new HTable(conf, table);
522 int regions = outputTable.getRegionsInfo().size();
523 if (job.getNumReduceTasks() > regions) {
524 job.setNumReduceTasks(outputTable.getRegionsInfo().size());
525 }
526 } else if (partitioner != null) {
527 job.setPartitionerClass(partitioner);
528 }
529
530 if (addDependencyJars) {
531 addDependencyJars(job);
532 }
533
534 initCredentials(job);
535 }
536
537
538
539
540
541
542
543
544
545 public static void limitNumReduceTasks(String table, Job job)
546 throws IOException {
547 HTable outputTable = new HTable(job.getConfiguration(), table);
548 int regions = outputTable.getRegionsInfo().size();
549 if (job.getNumReduceTasks() > regions)
550 job.setNumReduceTasks(regions);
551 }
552
553
554
555
556
557
558
559
560
561 public static void setNumReduceTasks(String table, Job job)
562 throws IOException {
563 HTable outputTable = new HTable(job.getConfiguration(), table);
564 int regions = outputTable.getRegionsInfo().size();
565 job.setNumReduceTasks(regions);
566 }
567
568
569
570
571
572
573
574
575
576
577 public static void setScannerCaching(Job job, int batchSize) {
578 job.getConfiguration().setInt("hbase.client.scanner.caching", batchSize);
579 }
580
581
582
583
584
585
586
587
588
589
590
591
592
593 public static void addHBaseDependencyJars(Configuration conf) throws IOException {
594 addDependencyJars(conf,
595 org.apache.zookeeper.ZooKeeper.class,
596 com.google.protobuf.Message.class,
597 com.google.common.base.Function.class,
598 com.google.common.collect.ImmutableSet.class,
599 org.apache.hadoop.hbase.util.Bytes.class);
600 }
601
602
603
604
605
606 public static String buildDependencyClasspath(Configuration conf) {
607 if (conf == null) {
608 throw new IllegalArgumentException("Must provide a configuration object.");
609 }
610 Set<String> paths = new HashSet<String>(conf.getStringCollection("tmpjars"));
611 if (paths.size() == 0) {
612 throw new IllegalArgumentException("Configuration contains no tmpjars.");
613 }
614 StringBuilder sb = new StringBuilder();
615 for (String s : paths) {
616
617 int idx = s.indexOf(":");
618 if (idx != -1) s = s.substring(idx + 1);
619 if (sb.length() > 0) sb.append(File.pathSeparator);
620 sb.append(s);
621 }
622 return sb.toString();
623 }
624
625
626
627
628
629
630 public static void addDependencyJars(Job job) throws IOException {
631 addHBaseDependencyJars(job.getConfiguration());
632 try {
633 addDependencyJars(job.getConfiguration(),
634
635 job.getMapOutputKeyClass(),
636 job.getMapOutputValueClass(),
637 job.getInputFormatClass(),
638 job.getOutputKeyClass(),
639 job.getOutputValueClass(),
640 job.getOutputFormatClass(),
641 job.getPartitionerClass(),
642 job.getCombinerClass());
643 } catch (ClassNotFoundException e) {
644 throw new IOException(e);
645 }
646 }
647
648
649
650
651
652
653 public static void addDependencyJars(Configuration conf,
654 Class<?>... classes) throws IOException {
655
656 FileSystem localFs = FileSystem.getLocal(conf);
657 Set<String> jars = new HashSet<String>();
658
659 jars.addAll(conf.getStringCollection("tmpjars"));
660
661
662
663 Map<String, String> packagedClasses = new HashMap<String, String>();
664
665
666 for (Class<?> clazz : classes) {
667 if (clazz == null) continue;
668
669 Path path = findOrCreateJar(clazz, localFs, packagedClasses);
670 if (path == null) {
671 LOG.warn("Could not find jar for class " + clazz +
672 " in order to ship it to the cluster.");
673 continue;
674 }
675 if (!localFs.exists(path)) {
676 LOG.warn("Could not validate jar file " + path + " for class "
677 + clazz);
678 continue;
679 }
680 jars.add(path.toString());
681 }
682 if (jars.isEmpty()) return;
683
684 conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[0])));
685 }
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701 private static Path findOrCreateJar(Class<?> my_class, FileSystem fs,
702 Map<String, String> packagedClasses)
703 throws IOException {
704
705 String jar = findContainingJar(my_class, packagedClasses);
706 if (null == jar || jar.isEmpty()) {
707 jar = getJar(my_class);
708 updateMap(jar, packagedClasses);
709 }
710
711 if (null == jar || jar.isEmpty()) {
712 return null;
713 }
714
715 LOG.debug(String.format("For class %s, using jar %s", my_class.getName(), jar));
716 return new Path(jar).makeQualified(fs);
717 }
718
719
720
721
722
723
724
725 private static void updateMap(String jar, Map<String, String> packagedClasses) throws IOException {
726 if (null == jar || jar.isEmpty()) {
727 return;
728 }
729 ZipFile zip = null;
730 try {
731 zip = new ZipFile(jar);
732 for (Enumeration<? extends ZipEntry> iter = zip.entries(); iter.hasMoreElements();) {
733 ZipEntry entry = iter.nextElement();
734 if (entry.getName().endsWith("class")) {
735 packagedClasses.put(entry.getName(), jar);
736 }
737 }
738 } finally {
739 if (null != zip) zip.close();
740 }
741 }
742
743
744
745
746
747
748
749
750
751
752 private static String findContainingJar(Class<?> my_class, Map<String, String> packagedClasses)
753 throws IOException {
754 ClassLoader loader = my_class.getClassLoader();
755 String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
756
757
758 for (Enumeration<URL> itr = loader.getResources(class_file); itr.hasMoreElements();) {
759 URL url = itr.nextElement();
760 if ("jar".equals(url.getProtocol())) {
761 String toReturn = url.getPath();
762 if (toReturn.startsWith("file:")) {
763 toReturn = toReturn.substring("file:".length());
764 }
765
766
767
768
769
770
771 toReturn = toReturn.replaceAll("\\+", "%2B");
772 toReturn = URLDecoder.decode(toReturn, "UTF-8");
773 return toReturn.replaceAll("!.*$", "");
774 }
775 }
776
777
778
779 return packagedClasses.get(class_file);
780 }
781
782
783
784
785
786
787
788
789 private static String getJar(Class<?> my_class) {
790 String ret = null;
791 String hadoopJarFinder = "org.apache.hadoop.util.JarFinder";
792 Class<?> jarFinder = null;
793 try {
794 LOG.debug("Looking for " + hadoopJarFinder + ".");
795 jarFinder = Class.forName(hadoopJarFinder);
796 LOG.debug(hadoopJarFinder + " found.");
797 Method getJar = jarFinder.getMethod("getJar", Class.class);
798 ret = (String) getJar.invoke(null, my_class);
799 } catch (ClassNotFoundException e) {
800 LOG.debug("Using backported JarFinder.");
801 ret = JarFinder.getJar(my_class);
802 } catch (InvocationTargetException e) {
803
804
805 throw new RuntimeException(e.getCause());
806 } catch (Exception e) {
807
808 throw new RuntimeException("getJar invocation failed.", e);
809 }
810
811 return ret;
812 }
813 }