1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.coprocessor;
21
22 import com.google.common.collect.MapMaker;
23 import com.google.protobuf.Service;
24 import com.google.protobuf.ServiceException;
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.classification.InterfaceAudience;
28 import org.apache.hadoop.classification.InterfaceStability;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.FileSystem;
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.hbase.Coprocessor;
33 import org.apache.hadoop.hbase.CoprocessorEnvironment;
34 import org.apache.hadoop.hbase.exceptions.DoNotRetryIOException;
35 import org.apache.hadoop.hbase.HTableDescriptor;
36 import org.apache.hadoop.hbase.client.*;
37 import org.apache.hadoop.hbase.client.coprocessor.Batch;
38 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
39 import org.apache.hadoop.hbase.util.Bytes;
40 import org.apache.hadoop.hbase.util.SortedCopyOnWriteSet;
41 import org.apache.hadoop.hbase.util.VersionInfo;
42 import org.apache.hadoop.hbase.Server;
43 import org.apache.hadoop.io.IOUtils;
44
45 import java.io.File;
46 import java.io.FileOutputStream;
47 import java.io.IOException;
48 import java.net.URL;
49 import java.security.AccessController;
50 import java.security.PrivilegedAction;
51 import java.util.*;
52 import java.util.concurrent.ConcurrentMap;
53 import java.util.jar.JarEntry;
54 import java.util.jar.JarFile;
55
56
57
58
59
60
61
62 @InterfaceAudience.Public
63 @InterfaceStability.Evolving
64 public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
65 public static final String REGION_COPROCESSOR_CONF_KEY =
66 "hbase.coprocessor.region.classes";
67 public static final String REGIONSERVER_COPROCESSOR_CONF_KEY =
68 "hbase.coprocessor.regionserver.classes";
69 public static final String USER_REGION_COPROCESSOR_CONF_KEY =
70 "hbase.coprocessor.user.region.classes";
71 public static final String MASTER_COPROCESSOR_CONF_KEY =
72 "hbase.coprocessor.master.classes";
73 public static final String WAL_COPROCESSOR_CONF_KEY =
74 "hbase.coprocessor.wal.classes";
75
76
77 private static final String COPROCESSOR_JARS_DIR = File.separator
78 + "coprocessor" + File.separator + "jars" + File.separator;
79
80 private static final Log LOG = LogFactory.getLog(CoprocessorHost.class);
81
82 protected SortedSet<E> coprocessors =
83 new SortedCopyOnWriteSet<E>(new EnvironmentPriorityComparator());
84 protected Configuration conf;
85
86 protected String pathPrefix;
87 protected volatile int loadSequence;
88
89
90
91
92
93
94
95 static ConcurrentMap<Path, ClassLoader> classLoadersCache =
96 new MapMaker().concurrencyLevel(3).weakValues().makeMap();
97
98 public CoprocessorHost() {
99 pathPrefix = UUID.randomUUID().toString();
100 }
101
102
103
104
105
106
107
108
109
110 private static Set<String> coprocessorNames =
111 Collections.synchronizedSet(new HashSet<String>());
112 public static Set<String> getLoadedCoprocessors() {
113 return coprocessorNames;
114 }
115
116
117
118
119
120
121
122
123 public Set<String> getCoprocessors() {
124 Set<String> returnValue = new TreeSet<String>();
125 for(CoprocessorEnvironment e: coprocessors) {
126 returnValue.add(e.getInstance().getClass().getSimpleName());
127 }
128 return returnValue;
129 }
130
131
132
133
134
135 protected void loadSystemCoprocessors(Configuration conf, String confKey) {
136 Class<?> implClass = null;
137
138
139 String[] defaultCPClasses = conf.getStrings(confKey);
140 if (defaultCPClasses == null || defaultCPClasses.length == 0)
141 return;
142
143 int priority = Coprocessor.PRIORITY_SYSTEM;
144 List<E> configured = new ArrayList<E>();
145 for (String className : defaultCPClasses) {
146 className = className.trim();
147 if (findCoprocessor(className) != null) {
148 continue;
149 }
150 ClassLoader cl = this.getClass().getClassLoader();
151 Thread.currentThread().setContextClassLoader(cl);
152 try {
153 implClass = cl.loadClass(className);
154 configured.add(loadInstance(implClass, Coprocessor.PRIORITY_SYSTEM, conf));
155 LOG.info("System coprocessor " + className + " was loaded " +
156 "successfully with priority (" + priority++ + ").");
157 } catch (ClassNotFoundException e) {
158 LOG.warn("Class " + className + " cannot be found. " +
159 e.getMessage());
160 } catch (IOException e) {
161 LOG.warn("Load coprocessor " + className + " failed. " +
162 e.getMessage());
163 }
164 }
165
166
167 coprocessors.addAll(configured);
168 }
169
170
171
172
173
174
175
176
177
178 @SuppressWarnings("deprecation")
179 public E load(Path path, String className, int priority,
180 Configuration conf) throws IOException {
181 Class<?> implClass = null;
182 LOG.debug("Loading coprocessor class " + className + " with path " +
183 path + " and priority " + priority);
184
185 ClassLoader cl = null;
186 if (path == null) {
187 try {
188 implClass = getClass().getClassLoader().loadClass(className);
189 } catch (ClassNotFoundException e) {
190 throw new IOException("No jar path specified for " + className);
191 }
192 } else {
193
194
195 cl = classLoadersCache.get(path);
196 if (cl != null){
197 LOG.debug("Found classloader "+ cl + "for "+path.toString());
198 try {
199 implClass = cl.loadClass(className);
200 } catch (ClassNotFoundException e) {
201 LOG.info("Class " + className + " needs to be loaded from a file - " +
202 path + ".");
203
204 }
205 }
206 }
207
208
209 if (implClass == null) {
210 if (path == null) {
211 throw new IOException("No jar path specified for " + className);
212 }
213
214 if (!path.toString().endsWith(".jar")) {
215 throw new IOException(path.toString() + ": not a jar file?");
216 }
217 FileSystem fs = path.getFileSystem(this.conf);
218 File parentDir = new File(this.conf.get("hbase.local.dir") + COPROCESSOR_JARS_DIR);
219 parentDir.mkdirs();
220 File dst = new File(parentDir, "." + pathPrefix +
221 "." + className + "." + System.currentTimeMillis() + ".jar");
222 fs.copyToLocalFile(path, new Path(dst.toString()));
223 dst.deleteOnExit();
224
225
226
227
228
229
230
231
232
233
234
235
236 final List<URL> paths = new ArrayList<URL>();
237 URL url = dst.getCanonicalFile().toURL();
238 paths.add(url);
239
240 JarFile jarFile = new JarFile(dst.toString());
241 Enumeration<JarEntry> entries = jarFile.entries();
242 while (entries.hasMoreElements()) {
243 JarEntry entry = entries.nextElement();
244 if (entry.getName().matches("/lib/[^/]+\\.jar")) {
245 File file = new File(parentDir, "." + pathPrefix +
246 "." + className + "." + System.currentTimeMillis() + "." + entry.getName().substring(5));
247 IOUtils.copyBytes(jarFile.getInputStream(entry), new FileOutputStream(file), conf, true);
248 file.deleteOnExit();
249 paths.add(file.toURL());
250 }
251 }
252 jarFile.close();
253
254 cl = AccessController.doPrivileged(new PrivilegedAction<CoprocessorClassLoader>() {
255 @Override
256 public CoprocessorClassLoader run() {
257 return new CoprocessorClassLoader(paths, this.getClass().getClassLoader());
258 }
259 });
260
261
262 ClassLoader prev = classLoadersCache.putIfAbsent(path, cl);
263 if (prev != null) {
264
265 cl = prev;
266 }
267
268 try {
269 implClass = cl.loadClass(className);
270 } catch (ClassNotFoundException e) {
271 throw new IOException("Cannot load external coprocessor class " + className, e);
272 }
273 }
274
275
276 Thread currentThread = Thread.currentThread();
277 ClassLoader hostClassLoader = currentThread.getContextClassLoader();
278 try{
279
280 currentThread.setContextClassLoader(cl);
281 E cpInstance = loadInstance(implClass, priority, conf);
282 return cpInstance;
283 } finally {
284
285 currentThread.setContextClassLoader(hostClassLoader);
286 }
287 }
288
289
290
291
292
293
294
295 public void load(Class<?> implClass, int priority, Configuration conf)
296 throws IOException {
297 E env = loadInstance(implClass, priority, conf);
298 coprocessors.add(env);
299 }
300
301
302
303
304
305
306
307 public E loadInstance(Class<?> implClass, int priority, Configuration conf)
308 throws IOException {
309 if (!Coprocessor.class.isAssignableFrom(implClass)) {
310 throw new IOException("Configured class " + implClass.getName() + " must implement "
311 + Coprocessor.class.getName() + " interface ");
312 }
313
314
315 Coprocessor impl;
316 Object o = null;
317 try {
318 o = implClass.newInstance();
319 impl = (Coprocessor)o;
320 } catch (InstantiationException e) {
321 throw new IOException(e);
322 } catch (IllegalAccessException e) {
323 throw new IOException(e);
324 }
325
326 E env = createEnvironment(implClass, impl, priority, ++loadSequence, conf);
327 if (env instanceof Environment) {
328 ((Environment)env).startup();
329 }
330
331
332 coprocessorNames.add(implClass.getName());
333 return env;
334 }
335
336
337
338
339 public abstract E createEnvironment(Class<?> implClass, Coprocessor instance,
340 int priority, int sequence, Configuration conf);
341
342 public void shutdown(CoprocessorEnvironment e) {
343 if (e instanceof Environment) {
344 ((Environment)e).shutdown();
345 } else {
346 LOG.warn("Shutdown called on unknown environment: "+
347 e.getClass().getName());
348 }
349 }
350
351
352
353
354
355
356 public Coprocessor findCoprocessor(String className) {
357
358 for (E env: coprocessors) {
359 if (env.getInstance().getClass().getName().equals(className) ||
360 env.getInstance().getClass().getSimpleName().equals(className)) {
361 return env.getInstance();
362 }
363 }
364 return null;
365 }
366
367
368
369
370
371
372 Set<ClassLoader> getExternalClassLoaders() {
373 Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>();
374 final ClassLoader systemClassLoader = this.getClass().getClassLoader();
375 for (E env : coprocessors) {
376 ClassLoader cl = env.getInstance().getClass().getClassLoader();
377 if (cl != systemClassLoader ){
378
379 externalClassLoaders.add(cl);
380 }
381 }
382 return externalClassLoaders;
383 }
384
385
386
387
388
389
390 public CoprocessorEnvironment findCoprocessorEnvironment(String className) {
391
392 for (E env: coprocessors) {
393 if (env.getInstance().getClass().getName().equals(className) ||
394 env.getInstance().getClass().getSimpleName().equals(className)) {
395 return env;
396 }
397 }
398 return null;
399 }
400
401
402
403
404
405 static class EnvironmentPriorityComparator
406 implements Comparator<CoprocessorEnvironment> {
407 public int compare(final CoprocessorEnvironment env1,
408 final CoprocessorEnvironment env2) {
409 if (env1.getPriority() < env2.getPriority()) {
410 return -1;
411 } else if (env1.getPriority() > env2.getPriority()) {
412 return 1;
413 }
414 if (env1.getLoadSequence() < env2.getLoadSequence()) {
415 return -1;
416 } else if (env1.getLoadSequence() > env2.getLoadSequence()) {
417 return 1;
418 }
419 return 0;
420 }
421 }
422
423
424
425
426 public static class Environment implements CoprocessorEnvironment {
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442 class HTableWrapper implements HTableInterface {
443
444 private byte[] tableName;
445 private HTable table;
446
447 public HTableWrapper(byte[] tableName) throws IOException {
448 this.tableName = tableName;
449 this.table = new HTable(conf, tableName);
450 openTables.add(this);
451 }
452
453 void internalClose() throws IOException {
454 table.close();
455 }
456
457 public Configuration getConfiguration() {
458 return table.getConfiguration();
459 }
460
461 public void close() throws IOException {
462 try {
463 internalClose();
464 } finally {
465 openTables.remove(this);
466 }
467 }
468
469 public Result getRowOrBefore(byte[] row, byte[] family)
470 throws IOException {
471 return table.getRowOrBefore(row, family);
472 }
473
474 public Result get(Get get) throws IOException {
475 return table.get(get);
476 }
477
478 public boolean exists(Get get) throws IOException {
479 return table.exists(get);
480 }
481
482 public Boolean[] exists(List<Get> gets) throws IOException{
483 return table.exists(gets);
484 }
485
486 public void put(Put put) throws IOException {
487 table.put(put);
488 }
489
490 public void put(List<Put> puts) throws IOException {
491 table.put(puts);
492 }
493
494 public void delete(Delete delete) throws IOException {
495 table.delete(delete);
496 }
497
498 public void delete(List<Delete> deletes) throws IOException {
499 table.delete(deletes);
500 }
501
502 public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier,
503 byte[] value, Put put) throws IOException {
504 return table.checkAndPut(row, family, qualifier, value, put);
505 }
506
507 public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier,
508 byte[] value, Delete delete) throws IOException {
509 return table.checkAndDelete(row, family, qualifier, value, delete);
510 }
511
512 public long incrementColumnValue(byte[] row, byte[] family,
513 byte[] qualifier, long amount) throws IOException {
514 return table.incrementColumnValue(row, family, qualifier, amount);
515 }
516
517 public long incrementColumnValue(byte[] row, byte[] family,
518 byte[] qualifier, long amount, boolean writeToWAL)
519 throws IOException {
520 return table.incrementColumnValue(row, family, qualifier, amount,
521 writeToWAL);
522 }
523
524 @Override
525 public Result append(Append append) throws IOException {
526 return table.append(append);
527 }
528
529 @Override
530 public Result increment(Increment increment) throws IOException {
531 return table.increment(increment);
532 }
533
534 public void flushCommits() throws IOException {
535 table.flushCommits();
536 }
537
538 public boolean isAutoFlush() {
539 return table.isAutoFlush();
540 }
541
542 public ResultScanner getScanner(Scan scan) throws IOException {
543 return table.getScanner(scan);
544 }
545
546 public ResultScanner getScanner(byte[] family) throws IOException {
547 return table.getScanner(family);
548 }
549
550 public ResultScanner getScanner(byte[] family, byte[] qualifier)
551 throws IOException {
552 return table.getScanner(family, qualifier);
553 }
554
555 public HTableDescriptor getTableDescriptor() throws IOException {
556 return table.getTableDescriptor();
557 }
558
559 public byte[] getTableName() {
560 return tableName;
561 }
562
563 @Override
564 public void batch(List<? extends Row> actions, Object[] results)
565 throws IOException, InterruptedException {
566 table.batch(actions, results);
567 }
568
569 @Override
570 public Object[] batch(List<? extends Row> actions)
571 throws IOException, InterruptedException {
572 return table.batch(actions);
573 }
574
575 @Override
576 public <R> void batchCallback(List<? extends Row> actions, Object[] results,
577 Batch.Callback<R> callback) throws IOException, InterruptedException {
578 table.batchCallback(actions, results, callback);
579 }
580
581 @Override
582 public <R> Object[] batchCallback(List<? extends Row> actions,
583 Batch.Callback<R> callback) throws IOException, InterruptedException {
584 return table.batchCallback(actions, callback);
585 }
586
587 @Override
588 public Result[] get(List<Get> gets) throws IOException {
589 return table.get(gets);
590 }
591
592 @Override
593 public CoprocessorRpcChannel coprocessorService(byte[] row) {
594 return table.coprocessorService(row);
595 }
596
597 @Override
598 public <T extends Service, R> Map<byte[], R> coprocessorService(Class<T> service,
599 byte[] startKey, byte[] endKey, Batch.Call<T, R> callable)
600 throws ServiceException, Throwable {
601 return table.coprocessorService(service, startKey, endKey, callable);
602 }
603
604 @Override
605 public <T extends Service, R> void coprocessorService(Class<T> service,
606 byte[] startKey, byte[] endKey, Batch.Call<T, R> callable, Batch.Callback<R> callback)
607 throws ServiceException, Throwable {
608 table.coprocessorService(service, startKey, endKey, callable, callback);
609 }
610
611 @Override
612 public void mutateRow(RowMutations rm) throws IOException {
613 table.mutateRow(rm);
614 }
615
616 @Override
617 public void setAutoFlush(boolean autoFlush) {
618 table.setAutoFlush(autoFlush);
619 }
620
621 @Override
622 public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
623 table.setAutoFlush(autoFlush, clearBufferOnFail);
624 }
625
626 @Override
627 public long getWriteBufferSize() {
628 return table.getWriteBufferSize();
629 }
630
631 @Override
632 public void setWriteBufferSize(long writeBufferSize) throws IOException {
633 table.setWriteBufferSize(writeBufferSize);
634 }
635 }
636
637
638 public Coprocessor impl;
639
640 protected int priority = Coprocessor.PRIORITY_USER;
641
642 Coprocessor.State state = Coprocessor.State.UNINSTALLED;
643
644 protected List<HTableInterface> openTables =
645 Collections.synchronizedList(new ArrayList<HTableInterface>());
646 private int seq;
647 private Configuration conf;
648
649
650
651
652
653
654 public Environment(final Coprocessor impl, final int priority,
655 final int seq, final Configuration conf) {
656 this.impl = impl;
657 this.priority = priority;
658 this.state = Coprocessor.State.INSTALLED;
659 this.seq = seq;
660 this.conf = conf;
661 }
662
663
664 public void startup() {
665 if (state == Coprocessor.State.INSTALLED ||
666 state == Coprocessor.State.STOPPED) {
667 state = Coprocessor.State.STARTING;
668 try {
669 impl.start(this);
670 state = Coprocessor.State.ACTIVE;
671 } catch (IOException ioe) {
672 LOG.error("Error starting coprocessor "+impl.getClass().getName(), ioe);
673 }
674 } else {
675 LOG.warn("Not starting coprocessor "+impl.getClass().getName()+
676 " because not inactive (state="+state.toString()+")");
677 }
678 }
679
680
681 protected void shutdown() {
682 if (state == Coprocessor.State.ACTIVE) {
683 state = Coprocessor.State.STOPPING;
684 try {
685 impl.stop(this);
686 state = Coprocessor.State.STOPPED;
687 } catch (IOException ioe) {
688 LOG.error("Error stopping coprocessor "+impl.getClass().getName(), ioe);
689 }
690 } else {
691 LOG.warn("Not stopping coprocessor "+impl.getClass().getName()+
692 " because not active (state="+state.toString()+")");
693 }
694
695 for (HTableInterface table: openTables) {
696 try {
697 ((HTableWrapper)table).internalClose();
698 } catch (IOException e) {
699
700 LOG.warn("Failed to close " +
701 Bytes.toStringBinary(table.getTableName()), e);
702 }
703 }
704 }
705
706 @Override
707 public Coprocessor getInstance() {
708 return impl;
709 }
710
711 @Override
712 public int getPriority() {
713 return priority;
714 }
715
716 @Override
717 public int getLoadSequence() {
718 return seq;
719 }
720
721
722 @Override
723 public int getVersion() {
724 return Coprocessor.VERSION;
725 }
726
727
728 @Override
729 public String getHBaseVersion() {
730 return VersionInfo.getVersion();
731 }
732
733 @Override
734 public Configuration getConfiguration() {
735 return conf;
736 }
737
738
739
740
741
742
743
744 @Override
745 public HTableInterface getTable(byte[] tableName) throws IOException {
746 return new HTableWrapper(tableName);
747 }
748 }
749
750 protected void abortServer(final String service,
751 final Server server,
752 final CoprocessorEnvironment environment,
753 final Throwable e) {
754 String coprocessorName = (environment.getInstance()).toString();
755 server.abort("Aborting service: " + service + " running on : "
756 + server.getServerName() + " because coprocessor: "
757 + coprocessorName + " threw an exception.", e);
758 }
759
760 protected void abortServer(final CoprocessorEnvironment environment,
761 final Throwable e) {
762 String coprocessorName = (environment.getInstance()).toString();
763 LOG.error("The coprocessor: " + coprocessorName + " threw an unexpected " +
764 "exception: " + e + ", but there's no specific implementation of " +
765 " abortServer() for this coprocessor's environment.");
766 }
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784 protected void handleCoprocessorThrowable(final CoprocessorEnvironment env,
785 final Throwable e)
786 throws IOException {
787 if (e instanceof IOException) {
788 throw (IOException)e;
789 }
790
791
792
793
794
795
796 if (env.getConfiguration().getBoolean("hbase.coprocessor.abortonerror",false)) {
797
798 abortServer(env, e);
799 } else {
800 LOG.error("Removing coprocessor '" + env.toString() + "' from " +
801 "environment because it threw: " + e,e);
802 coprocessors.remove(env);
803 throw new DoNotRetryIOException("Coprocessor: '" + env.toString() +
804 "' threw: '" + e + "' and has been removed" + "from the active " +
805 "coprocessor set.", e);
806 }
807 }
808 }
809
810