1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.coprocessor;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.Collections;
25 import java.util.Comparator;
26 import java.util.HashSet;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Set;
30 import java.util.TreeSet;
31 import java.util.UUID;
32 import java.util.concurrent.ExecutorService;
33 import java.util.concurrent.atomic.AtomicInteger;
34
35 import org.apache.commons.logging.Log;
36 import org.apache.commons.logging.LogFactory;
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.fs.Path;
39 import org.apache.hadoop.hbase.Abortable;
40 import org.apache.hadoop.hbase.Coprocessor;
41 import org.apache.hadoop.hbase.CoprocessorEnvironment;
42 import org.apache.hadoop.hbase.DoNotRetryIOException;
43 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
44 import org.apache.hadoop.hbase.HTableDescriptor;
45 import org.apache.hadoop.hbase.TableName;
46 import org.apache.hadoop.hbase.classification.InterfaceAudience;
47 import org.apache.hadoop.hbase.classification.InterfaceStability;
48 import org.apache.hadoop.hbase.client.Append;
49 import org.apache.hadoop.hbase.client.CoprocessorHConnection;
50 import org.apache.hadoop.hbase.client.Delete;
51 import org.apache.hadoop.hbase.client.Durability;
52 import org.apache.hadoop.hbase.client.Get;
53 import org.apache.hadoop.hbase.client.HConnection;
54 import org.apache.hadoop.hbase.client.HTable;
55 import org.apache.hadoop.hbase.client.HTableInterface;
56 import org.apache.hadoop.hbase.client.Increment;
57 import org.apache.hadoop.hbase.client.Put;
58 import org.apache.hadoop.hbase.client.Result;
59 import org.apache.hadoop.hbase.client.ResultScanner;
60 import org.apache.hadoop.hbase.client.Row;
61 import org.apache.hadoop.hbase.client.RowMutations;
62 import org.apache.hadoop.hbase.client.Scan;
63 import org.apache.hadoop.hbase.client.coprocessor.Batch;
64 import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback;
65 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
66 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
67 import org.apache.hadoop.hbase.util.Bytes;
68 import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
69 import org.apache.hadoop.hbase.util.SortedList;
70 import org.apache.hadoop.hbase.util.VersionInfo;
71 import org.apache.hadoop.io.MultipleIOException;
72
73 import com.google.protobuf.Descriptors;
74 import com.google.protobuf.Message;
75 import com.google.protobuf.Service;
76 import com.google.protobuf.ServiceException;
77
78
79
80
81
82
83
84 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
85 @InterfaceStability.Evolving
86 public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
87 public static final String REGION_COPROCESSOR_CONF_KEY =
88 "hbase.coprocessor.region.classes";
89 public static final String REGIONSERVER_COPROCESSOR_CONF_KEY =
90 "hbase.coprocessor.regionserver.classes";
91 public static final String USER_REGION_COPROCESSOR_CONF_KEY =
92 "hbase.coprocessor.user.region.classes";
93 public static final String MASTER_COPROCESSOR_CONF_KEY =
94 "hbase.coprocessor.master.classes";
95 public static final String WAL_COPROCESSOR_CONF_KEY =
96 "hbase.coprocessor.wal.classes";
97 public static final String ABORT_ON_ERROR_KEY = "hbase.coprocessor.abortonerror";
98 public static final boolean DEFAULT_ABORT_ON_ERROR = true;
99 public static final String COPROCESSORS_ENABLED_CONF_KEY = "hbase.coprocessor.enabled";
100 public static final boolean DEFAULT_COPROCESSORS_ENABLED = true;
101 public static final String USER_COPROCESSORS_ENABLED_CONF_KEY =
102 "hbase.coprocessor.user.enabled";
103 public static final boolean DEFAULT_USER_COPROCESSORS_ENABLED = true;
104
105 protected static final Log LOG = LogFactory.getLog(CoprocessorHost.class);
106 protected Abortable abortable;
107
108 protected SortedList<E> coprocessors =
109 new SortedList<E>(new EnvironmentPriorityComparator());
110 protected Configuration conf;
111
112 protected String pathPrefix;
113 protected AtomicInteger loadSequence = new AtomicInteger();
114
115 public CoprocessorHost(Abortable abortable) {
116 this.abortable = abortable;
117 this.pathPrefix = UUID.randomUUID().toString();
118 }
119
120
121
122
123
124
125
126
127
128 private static Set<String> coprocessorNames =
129 Collections.synchronizedSet(new HashSet<String>());
130
131 public static Set<String> getLoadedCoprocessors() {
132 synchronized (coprocessorNames) {
133 return new HashSet(coprocessorNames);
134 }
135 }
136
137
138
139
140
141
142
143
144 public Set<String> getCoprocessors() {
145 Set<String> returnValue = new TreeSet<String>();
146 for (CoprocessorEnvironment e: coprocessors) {
147 returnValue.add(e.getInstance().getClass().getSimpleName());
148 }
149 return returnValue;
150 }
151
152
153
154
155
156 protected void loadSystemCoprocessors(Configuration conf, String confKey) {
157 boolean coprocessorsEnabled = conf.getBoolean(COPROCESSORS_ENABLED_CONF_KEY,
158 DEFAULT_COPROCESSORS_ENABLED);
159 if (!coprocessorsEnabled) {
160 return;
161 }
162
163 Class<?> implClass = null;
164
165
166 String[] defaultCPClasses = conf.getStrings(confKey);
167 if (defaultCPClasses == null || defaultCPClasses.length == 0)
168 return;
169
170 int priority = Coprocessor.PRIORITY_SYSTEM;
171 for (String className : defaultCPClasses) {
172 className = className.trim();
173 if (findCoprocessor(className) != null) {
174
175 LOG.warn("Attempted duplicate loading of " + className + "; skipped");
176 continue;
177 }
178 ClassLoader cl = this.getClass().getClassLoader();
179 Thread.currentThread().setContextClassLoader(cl);
180 try {
181 implClass = cl.loadClass(className);
182
183
184 this.coprocessors.add(loadInstance(implClass, Coprocessor.PRIORITY_SYSTEM, conf));
185 LOG.info("System coprocessor " + className + " was loaded " +
186 "successfully with priority (" + priority++ + ").");
187 } catch (Throwable t) {
188
189 abortServer(className, t);
190 }
191 }
192 }
193
194
195
196
197
198
199
200
201
202 public E load(Path path, String className, int priority,
203 Configuration conf) throws IOException {
204 Class<?> implClass = null;
205 LOG.debug("Loading coprocessor class " + className + " with path " +
206 path + " and priority " + priority);
207
208 ClassLoader cl = null;
209 if (path == null) {
210 try {
211 implClass = getClass().getClassLoader().loadClass(className);
212 } catch (ClassNotFoundException e) {
213 throw new IOException("No jar path specified for " + className);
214 }
215 } else {
216 cl = CoprocessorClassLoader.getClassLoader(
217 path, getClass().getClassLoader(), pathPrefix, conf);
218 try {
219 implClass = cl.loadClass(className);
220 } catch (ClassNotFoundException e) {
221 throw new IOException("Cannot load external coprocessor class " + className, e);
222 }
223 }
224
225
226 Thread currentThread = Thread.currentThread();
227 ClassLoader hostClassLoader = currentThread.getContextClassLoader();
228 try{
229
230 currentThread.setContextClassLoader(cl);
231 E cpInstance = loadInstance(implClass, priority, conf);
232 return cpInstance;
233 } finally {
234
235 currentThread.setContextClassLoader(hostClassLoader);
236 }
237 }
238
239
240
241
242
243
244
245 public void load(Class<?> implClass, int priority, Configuration conf)
246 throws IOException {
247 E env = loadInstance(implClass, priority, conf);
248 coprocessors.add(env);
249 }
250
251
252
253
254
255
256
257 public E loadInstance(Class<?> implClass, int priority, Configuration conf)
258 throws IOException {
259 if (!Coprocessor.class.isAssignableFrom(implClass)) {
260 throw new IOException("Configured class " + implClass.getName() + " must implement "
261 + Coprocessor.class.getName() + " interface ");
262 }
263
264
265 Coprocessor impl;
266 Object o = null;
267 try {
268 o = implClass.newInstance();
269 impl = (Coprocessor)o;
270 } catch (InstantiationException e) {
271 throw new IOException(e);
272 } catch (IllegalAccessException e) {
273 throw new IOException(e);
274 }
275
276 E env = createEnvironment(implClass, impl, priority, loadSequence.incrementAndGet(), conf);
277 if (env instanceof Environment) {
278 ((Environment)env).startup();
279 }
280
281
282 coprocessorNames.add(implClass.getName());
283 return env;
284 }
285
286
287
288
289 public abstract E createEnvironment(Class<?> implClass, Coprocessor instance,
290 int priority, int sequence, Configuration conf);
291
292 public void shutdown(CoprocessorEnvironment e) {
293 if (e instanceof Environment) {
294 if (LOG.isDebugEnabled()) {
295 LOG.debug("Stop coprocessor " + e.getInstance().getClass().getName());
296 }
297 ((Environment)e).shutdown();
298 } else {
299 LOG.warn("Shutdown called on unknown environment: "+
300 e.getClass().getName());
301 }
302 }
303
304
305
306
307
308
309 public Coprocessor findCoprocessor(String className) {
310 for (E env: coprocessors) {
311 if (env.getInstance().getClass().getName().equals(className) ||
312 env.getInstance().getClass().getSimpleName().equals(className)) {
313 return env.getInstance();
314 }
315 }
316 return null;
317 }
318
319
320
321
322
323
324 public <T extends Coprocessor> List<T> findCoprocessors(Class<T> cls) {
325 ArrayList<T> ret = new ArrayList<T>();
326
327 for (E env: coprocessors) {
328 Coprocessor cp = env.getInstance();
329
330 if(cp != null) {
331 if (cls.isAssignableFrom(cp.getClass())) {
332 ret.add((T)cp);
333 }
334 }
335 }
336 return ret;
337 }
338
339
340
341
342
343
344 public CoprocessorEnvironment findCoprocessorEnvironment(String className) {
345 for (E env: coprocessors) {
346 if (env.getInstance().getClass().getName().equals(className) ||
347 env.getInstance().getClass().getSimpleName().equals(className)) {
348 return env;
349 }
350 }
351 return null;
352 }
353
354
355
356
357
358
359 Set<ClassLoader> getExternalClassLoaders() {
360 Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>();
361 final ClassLoader systemClassLoader = this.getClass().getClassLoader();
362 for (E env : coprocessors) {
363 ClassLoader cl = env.getInstance().getClass().getClassLoader();
364 if (cl != systemClassLoader ){
365
366 externalClassLoaders.add(cl);
367 }
368 }
369 return externalClassLoaders;
370 }
371
372
373
374
375
376 static class EnvironmentPriorityComparator
377 implements Comparator<CoprocessorEnvironment> {
378 @Override
379 public int compare(final CoprocessorEnvironment env1,
380 final CoprocessorEnvironment env2) {
381 if (env1.getPriority() < env2.getPriority()) {
382 return -1;
383 } else if (env1.getPriority() > env2.getPriority()) {
384 return 1;
385 }
386 if (env1.getLoadSequence() < env2.getLoadSequence()) {
387 return -1;
388 } else if (env1.getLoadSequence() > env2.getLoadSequence()) {
389 return 1;
390 }
391 return 0;
392 }
393 }
394
395
396
397
398 public static class Environment implements CoprocessorEnvironment {
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414 class HTableWrapper implements HTableInterface {
415
416 private TableName tableName;
417 private HTable table;
418 private HConnection connection;
419
420 public HTableWrapper(TableName tableName, HConnection connection, ExecutorService pool)
421 throws IOException {
422 this.tableName = tableName;
423 this.table = new HTable(tableName, connection, pool);
424 this.connection = connection;
425 openTables.add(this);
426 }
427
428 void internalClose() throws IOException {
429 List<IOException> exceptions = new ArrayList<IOException>(2);
430 try {
431 table.close();
432 } catch (IOException e) {
433 exceptions.add(e);
434 }
435 try {
436
437 if (this.connection != null) {
438 this.connection.close();
439 }
440 } catch (IOException e) {
441 exceptions.add(e);
442 }
443 if (!exceptions.isEmpty()) {
444 throw MultipleIOException.createIOException(exceptions);
445 }
446 }
447
448 public Configuration getConfiguration() {
449 return table.getConfiguration();
450 }
451
452 public void close() throws IOException {
453 try {
454 internalClose();
455 } finally {
456 openTables.remove(this);
457 }
458 }
459
460 public Result getRowOrBefore(byte[] row, byte[] family)
461 throws IOException {
462 return table.getRowOrBefore(row, family);
463 }
464
465 public Result get(Get get) throws IOException {
466 return table.get(get);
467 }
468
469 public boolean exists(Get get) throws IOException {
470 return table.exists(get);
471 }
472
473 public Boolean[] exists(List<Get> gets) throws IOException{
474 return table.exists(gets);
475 }
476
477 public void put(Put put) throws IOException {
478 table.put(put);
479 }
480
481 public void put(List<Put> puts) throws IOException {
482 table.put(puts);
483 }
484
485 public void delete(Delete delete) throws IOException {
486 table.delete(delete);
487 }
488
489 public void delete(List<Delete> deletes) throws IOException {
490 table.delete(deletes);
491 }
492
493 public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier,
494 byte[] value, Put put) throws IOException {
495 return table.checkAndPut(row, family, qualifier, value, put);
496 }
497
498 public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier,
499 byte[] value, Delete delete) throws IOException {
500 return table.checkAndDelete(row, family, qualifier, value, delete);
501 }
502
503 public long incrementColumnValue(byte[] row, byte[] family,
504 byte[] qualifier, long amount) throws IOException {
505 return table.incrementColumnValue(row, family, qualifier, amount);
506 }
507
508 public long incrementColumnValue(byte[] row, byte[] family,
509 byte[] qualifier, long amount, Durability durability)
510 throws IOException {
511 return table.incrementColumnValue(row, family, qualifier, amount,
512 durability);
513 }
514
515 @Override
516 public Result append(Append append) throws IOException {
517 return table.append(append);
518 }
519
520 @Override
521 public Result increment(Increment increment) throws IOException {
522 return table.increment(increment);
523 }
524
525 public void flushCommits() throws IOException {
526 table.flushCommits();
527 }
528
529 public boolean isAutoFlush() {
530 return table.isAutoFlush();
531 }
532
533 public ResultScanner getScanner(Scan scan) throws IOException {
534 return table.getScanner(scan);
535 }
536
537 public ResultScanner getScanner(byte[] family) throws IOException {
538 return table.getScanner(family);
539 }
540
541 public ResultScanner getScanner(byte[] family, byte[] qualifier)
542 throws IOException {
543 return table.getScanner(family, qualifier);
544 }
545
546 public HTableDescriptor getTableDescriptor() throws IOException {
547 return table.getTableDescriptor();
548 }
549
550 @Override
551 public byte[] getTableName() {
552 return tableName.getName();
553 }
554
555 @Override
556 public TableName getName() {
557 return table.getName();
558 }
559
560 @Override
561 public void batch(List<? extends Row> actions, Object[] results)
562 throws IOException, InterruptedException {
563 table.batch(actions, results);
564 }
565
566
567
568
569
570
571 @Override
572 public Object[] batch(List<? extends Row> actions)
573 throws IOException, InterruptedException {
574 return table.batch(actions);
575 }
576
577 @Override
578 public <R> void batchCallback(List<? extends Row> actions, Object[] results,
579 Batch.Callback<R> callback) throws IOException, InterruptedException {
580 table.batchCallback(actions, results, callback);
581 }
582
583
584
585
586
587
588
589
590 @Override
591 public <R> Object[] batchCallback(List<? extends Row> actions,
592 Batch.Callback<R> callback) throws IOException, InterruptedException {
593 return table.batchCallback(actions, callback);
594 }
595
596 @Override
597 public Result[] get(List<Get> gets) throws IOException {
598 return table.get(gets);
599 }
600
601 @Override
602 public CoprocessorRpcChannel coprocessorService(byte[] row) {
603 return table.coprocessorService(row);
604 }
605
606 @Override
607 public <T extends Service, R> Map<byte[], R> coprocessorService(Class<T> service,
608 byte[] startKey, byte[] endKey, Batch.Call<T, R> callable)
609 throws ServiceException, Throwable {
610 return table.coprocessorService(service, startKey, endKey, callable);
611 }
612
613 @Override
614 public <T extends Service, R> void coprocessorService(Class<T> service,
615 byte[] startKey, byte[] endKey, Batch.Call<T, R> callable, Batch.Callback<R> callback)
616 throws ServiceException, Throwable {
617 table.coprocessorService(service, startKey, endKey, callable, callback);
618 }
619
620 @Override
621 public void mutateRow(RowMutations rm) throws IOException {
622 table.mutateRow(rm);
623 }
624
625 @Override
626 public void setAutoFlush(boolean autoFlush) {
627 table.setAutoFlush(autoFlush, autoFlush);
628 }
629
630 @Override
631 public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
632 table.setAutoFlush(autoFlush, clearBufferOnFail);
633 }
634
635 @Override
636 public void setAutoFlushTo(boolean autoFlush) {
637 table.setAutoFlushTo(autoFlush);
638 }
639
640 @Override
641 public long getWriteBufferSize() {
642 return table.getWriteBufferSize();
643 }
644
645 @Override
646 public void setWriteBufferSize(long writeBufferSize) throws IOException {
647 table.setWriteBufferSize(writeBufferSize);
648 }
649
650 @Override
651 public long incrementColumnValue(byte[] row, byte[] family,
652 byte[] qualifier, long amount, boolean writeToWAL) throws IOException {
653 return table.incrementColumnValue(row, family, qualifier, amount, writeToWAL);
654 }
655
656 @Override
657 public <R extends Message> Map<byte[], R> batchCoprocessorService(
658 Descriptors.MethodDescriptor method, Message request, byte[] startKey,
659 byte[] endKey, R responsePrototype) throws ServiceException, Throwable {
660 return table.batchCoprocessorService(method, request, startKey, endKey, responsePrototype);
661 }
662
663 @Override
664 public <R extends Message> void batchCoprocessorService(Descriptors.MethodDescriptor method,
665 Message request, byte[] startKey, byte[] endKey, R responsePrototype,
666 Callback<R> callback) throws ServiceException, Throwable {
667 table.batchCoprocessorService(method, request, startKey, endKey, responsePrototype,
668 callback);
669 }
670
671 @Override
672 public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier,
673 CompareOp compareOp, byte[] value, RowMutations mutation) throws IOException {
674 return table.checkAndMutate(row, family, qualifier, compareOp, value, mutation);
675 }
676 }
677
678
679 public Coprocessor impl;
680
681 protected int priority = Coprocessor.PRIORITY_USER;
682
683 Coprocessor.State state = Coprocessor.State.UNINSTALLED;
684
685 protected List<HTableInterface> openTables =
686 Collections.synchronizedList(new ArrayList<HTableInterface>());
687 private int seq;
688 private Configuration conf;
689 private ClassLoader classLoader;
690
691
692
693
694
695
696 public Environment(final Coprocessor impl, final int priority,
697 final int seq, final Configuration conf) {
698 this.impl = impl;
699 this.classLoader = impl.getClass().getClassLoader();
700 this.priority = priority;
701 this.state = Coprocessor.State.INSTALLED;
702 this.seq = seq;
703 this.conf = conf;
704 }
705
706
707 public void startup() throws IOException {
708 if (state == Coprocessor.State.INSTALLED ||
709 state == Coprocessor.State.STOPPED) {
710 state = Coprocessor.State.STARTING;
711 Thread currentThread = Thread.currentThread();
712 ClassLoader hostClassLoader = currentThread.getContextClassLoader();
713 try {
714 currentThread.setContextClassLoader(this.getClassLoader());
715 impl.start(this);
716 state = Coprocessor.State.ACTIVE;
717 } finally {
718 currentThread.setContextClassLoader(hostClassLoader);
719 }
720 } else {
721 LOG.warn("Not starting coprocessor "+impl.getClass().getName()+
722 " because not inactive (state="+state.toString()+")");
723 }
724 }
725
726
727 protected void shutdown() {
728 if (state == Coprocessor.State.ACTIVE) {
729 state = Coprocessor.State.STOPPING;
730 Thread currentThread = Thread.currentThread();
731 ClassLoader hostClassLoader = currentThread.getContextClassLoader();
732 try {
733 currentThread.setContextClassLoader(this.getClassLoader());
734 impl.stop(this);
735 state = Coprocessor.State.STOPPED;
736 } catch (IOException ioe) {
737 LOG.error("Error stopping coprocessor "+impl.getClass().getName(), ioe);
738 } finally {
739 currentThread.setContextClassLoader(hostClassLoader);
740 }
741 } else {
742 LOG.warn("Not stopping coprocessor "+impl.getClass().getName()+
743 " because not active (state="+state.toString()+")");
744 }
745 synchronized (openTables) {
746
747 for (HTableInterface table: openTables) {
748 try {
749 ((HTableWrapper)table).internalClose();
750 } catch (IOException e) {
751
752 LOG.warn("Failed to close " +
753 Bytes.toStringBinary(table.getTableName()), e);
754 }
755 }
756 }
757 }
758
759 @Override
760 public Coprocessor getInstance() {
761 return impl;
762 }
763
764 @Override
765 public ClassLoader getClassLoader() {
766 return classLoader;
767 }
768
769 @Override
770 public int getPriority() {
771 return priority;
772 }
773
774 @Override
775 public int getLoadSequence() {
776 return seq;
777 }
778
779
780 @Override
781 public int getVersion() {
782 return Coprocessor.VERSION;
783 }
784
785
786 @Override
787 public String getHBaseVersion() {
788 return VersionInfo.getVersion();
789 }
790
791 @Override
792 public Configuration getConfiguration() {
793 return conf;
794 }
795
796
797
798
799
800
801
802 @Override
803 public HTableInterface getTable(TableName tableName) throws IOException {
804 return this.getTable(tableName, HTable.getDefaultExecutor(getConfiguration()));
805 }
806
807
808
809
810
811
812
813 @Override
814 public HTableInterface getTable(TableName tableName, ExecutorService pool) throws IOException {
815 return new HTableWrapper(tableName, CoprocessorHConnection.getConnectionForEnvironment(this),
816 pool);
817 }
818 }
819
820 protected void abortServer(final CoprocessorEnvironment environment, final Throwable e) {
821 abortServer(environment.getInstance().getClass().getName(), e);
822 }
823
824 protected void abortServer(final String coprocessorName, final Throwable e) {
825 String message = "The coprocessor " + coprocessorName + " threw " + e.toString();
826 LOG.error(message, e);
827 if (abortable != null) {
828 abortable.abort(message, e);
829 } else {
830 LOG.warn("No available Abortable, process was not aborted");
831 }
832 }
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849 protected void handleCoprocessorThrowable(final CoprocessorEnvironment env, final Throwable e)
850 throws IOException {
851 if (e instanceof IOException) {
852 throw (IOException)e;
853 }
854
855
856
857
858
859
860 if (env.getConfiguration().getBoolean(ABORT_ON_ERROR_KEY, DEFAULT_ABORT_ON_ERROR)) {
861
862 abortServer(env, e);
863 } else {
864 LOG.error("Removing coprocessor '" + env.toString() + "' from " +
865 "environment because it threw: " + e,e);
866 coprocessors.remove(env);
867 try {
868 shutdown(env);
869 } catch (Exception x) {
870 LOG.error("Uncaught exception when shutting down coprocessor '"
871 + env.toString() + "'", x);
872 }
873 throw new DoNotRetryIOException("Coprocessor: '" + env.toString() +
874 "' threw: '" + e + "' and has been removed from the active " +
875 "coprocessor set.", e);
876 }
877 }
878 }