View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   * http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.coprocessor;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.Collections;
25  import java.util.Comparator;
26  import java.util.HashSet;
27  import java.util.List;
28  import java.util.Map;
29  import java.util.Set;
30  import java.util.SortedSet;
31  import java.util.TreeSet;
32  import java.util.UUID;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.classification.InterfaceAudience;
37  import org.apache.hadoop.classification.InterfaceStability;
38  import org.apache.hadoop.conf.Configuration;
39  import org.apache.hadoop.fs.Path;
40  import org.apache.hadoop.hbase.Coprocessor;
41  import org.apache.hadoop.hbase.CoprocessorEnvironment;
42  import org.apache.hadoop.hbase.TableName;
43  import org.apache.hadoop.hbase.DoNotRetryIOException;
44  import org.apache.hadoop.hbase.HTableDescriptor;
45  import org.apache.hadoop.hbase.Server;
46  import org.apache.hadoop.hbase.client.Append;
47  import org.apache.hadoop.hbase.client.Delete;
48  import org.apache.hadoop.hbase.client.Durability;
49  import org.apache.hadoop.hbase.client.Get;
50  import org.apache.hadoop.hbase.client.HTable;
51  import org.apache.hadoop.hbase.client.HTableInterface;
52  import org.apache.hadoop.hbase.client.Increment;
53  import org.apache.hadoop.hbase.client.Put;
54  import org.apache.hadoop.hbase.client.Result;
55  import org.apache.hadoop.hbase.client.ResultScanner;
56  import org.apache.hadoop.hbase.client.Row;
57  import org.apache.hadoop.hbase.client.RowMutations;
58  import org.apache.hadoop.hbase.client.Scan;
59  import org.apache.hadoop.hbase.client.coprocessor.Batch;
60  import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
61  import org.apache.hadoop.hbase.util.Bytes;
62  import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
63  import org.apache.hadoop.hbase.util.SortedCopyOnWriteSet;
64  import org.apache.hadoop.hbase.util.VersionInfo;
65  
66  import com.google.protobuf.Service;
67  import com.google.protobuf.ServiceException;
68  
69  /**
70   * Provides the common setup framework and runtime services for coprocessor
71   * invocation from HBase services.
72   * @param <E> the specific environment extension that a concrete implementation
73   * provides
74   */
75  @InterfaceAudience.Public
76  @InterfaceStability.Evolving
77  public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
78    public static final String REGION_COPROCESSOR_CONF_KEY =
79        "hbase.coprocessor.region.classes";
80    public static final String REGIONSERVER_COPROCESSOR_CONF_KEY =
81        "hbase.coprocessor.regionserver.classes";
82    public static final String USER_REGION_COPROCESSOR_CONF_KEY =
83        "hbase.coprocessor.user.region.classes";
84    public static final String MASTER_COPROCESSOR_CONF_KEY =
85        "hbase.coprocessor.master.classes";
86    public static final String WAL_COPROCESSOR_CONF_KEY =
87      "hbase.coprocessor.wal.classes";
88  
89    private static final Log LOG = LogFactory.getLog(CoprocessorHost.class);
90    /** Ordered set of loaded coprocessors with lock */
91    protected SortedSet<E> coprocessors =
92        new SortedCopyOnWriteSet<E>(new EnvironmentPriorityComparator());
93    protected Configuration conf;
94    // unique file prefix to use for local copies of jars when classloading
95    protected String pathPrefix;
96    protected volatile int loadSequence;
97  
98    public CoprocessorHost() {
99      pathPrefix = UUID.randomUUID().toString();
100   }
101 
102   /**
103    * Not to be confused with the per-object _coprocessors_ (above),
104    * coprocessorNames is static and stores the set of all coprocessors ever
105    * loaded by any thread in this JVM. It is strictly additive: coprocessors are
106    * added to coprocessorNames, by loadInstance() but are never removed, since
107    * the intention is to preserve a history of all loaded coprocessors for
108    * diagnosis in case of server crash (HBASE-4014).
109    */
110   private static Set<String> coprocessorNames =
111       Collections.synchronizedSet(new HashSet<String>());
112   public static Set<String> getLoadedCoprocessors() {
113       return coprocessorNames;
114   }
115 
116   /**
117    * Used to create a parameter to the HServerLoad constructor so that
118    * HServerLoad can provide information about the coprocessors loaded by this
119    * regionserver.
120    * (HBASE-4070: Improve region server metrics to report loaded coprocessors
121    * to master).
122    */
123   public Set<String> getCoprocessors() {
124     Set<String> returnValue = new TreeSet<String>();
125     for(CoprocessorEnvironment e: coprocessors) {
126       returnValue.add(e.getInstance().getClass().getSimpleName());
127     }
128     return returnValue;
129   }
130 
131   /**
132    * Load system coprocessors. Read the class names from configuration.
133    * Called by constructor.
134    */
135   protected void loadSystemCoprocessors(Configuration conf, String confKey) {
136     Class<?> implClass = null;
137 
138     // load default coprocessors from configure file
139     String[] defaultCPClasses = conf.getStrings(confKey);
140     if (defaultCPClasses == null || defaultCPClasses.length == 0)
141       return;
142 
143     int priority = Coprocessor.PRIORITY_SYSTEM;
144     List<E> configured = new ArrayList<E>();
145     for (String className : defaultCPClasses) {
146       className = className.trim();
147       if (findCoprocessor(className) != null) {
148         continue;
149       }
150       ClassLoader cl = this.getClass().getClassLoader();
151       Thread.currentThread().setContextClassLoader(cl);
152       try {
153         implClass = cl.loadClass(className);
154         configured.add(loadInstance(implClass, Coprocessor.PRIORITY_SYSTEM, conf));
155         LOG.info("System coprocessor " + className + " was loaded " +
156             "successfully with priority (" + priority++ + ").");
157       } catch (ClassNotFoundException e) {
158         LOG.warn("Class " + className + " cannot be found. " +
159             e.getMessage());
160       } catch (IOException e) {
161         LOG.warn("Load coprocessor " + className + " failed. " +
162             e.getMessage());
163       }
164     }
165 
166     // add entire set to the collection for COW efficiency
167     coprocessors.addAll(configured);
168   }
169 
170   /**
171    * Load a coprocessor implementation into the host
172    * @param path path to implementation jar
173    * @param className the main class name
174    * @param priority chaining priority
175    * @param conf configuration for coprocessor
176    * @throws java.io.IOException Exception
177    */
178   public E load(Path path, String className, int priority,
179       Configuration conf) throws IOException {
180     Class<?> implClass = null;
181     LOG.debug("Loading coprocessor class " + className + " with path " +
182         path + " and priority " + priority);
183 
184     ClassLoader cl = null;
185     if (path == null) {
186       try {
187         implClass = getClass().getClassLoader().loadClass(className);
188       } catch (ClassNotFoundException e) {
189         throw new IOException("No jar path specified for " + className);
190       }
191     } else {
192       cl = CoprocessorClassLoader.getClassLoader(
193         path, getClass().getClassLoader(), pathPrefix, conf);
194       try {
195         implClass = cl.loadClass(className);
196       } catch (ClassNotFoundException e) {
197         throw new IOException("Cannot load external coprocessor class " + className, e);
198       }
199     }
200 
201     //load custom code for coprocessor
202     Thread currentThread = Thread.currentThread();
203     ClassLoader hostClassLoader = currentThread.getContextClassLoader();
204     try{
205       // switch temporarily to the thread classloader for custom CP
206       currentThread.setContextClassLoader(cl);
207       E cpInstance = loadInstance(implClass, priority, conf);
208       return cpInstance;
209     } finally {
210       // restore the fresh (host) classloader
211       currentThread.setContextClassLoader(hostClassLoader);
212     }
213   }
214 
215   /**
216    * @param implClass Implementation class
217    * @param priority priority
218    * @param conf configuration
219    * @throws java.io.IOException Exception
220    */
221   public void load(Class<?> implClass, int priority, Configuration conf)
222       throws IOException {
223     E env = loadInstance(implClass, priority, conf);
224     coprocessors.add(env);
225   }
226 
227   /**
228    * @param implClass Implementation class
229    * @param priority priority
230    * @param conf configuration
231    * @throws java.io.IOException Exception
232    */
233   public E loadInstance(Class<?> implClass, int priority, Configuration conf)
234       throws IOException {
235     if (!Coprocessor.class.isAssignableFrom(implClass)) {
236       throw new IOException("Configured class " + implClass.getName() + " must implement "
237           + Coprocessor.class.getName() + " interface ");
238     }
239 
240     // create the instance
241     Coprocessor impl;
242     Object o = null;
243     try {
244       o = implClass.newInstance();
245       impl = (Coprocessor)o;
246     } catch (InstantiationException e) {
247       throw new IOException(e);
248     } catch (IllegalAccessException e) {
249       throw new IOException(e);
250     }
251     // create the environment
252     E env = createEnvironment(implClass, impl, priority, ++loadSequence, conf);
253     if (env instanceof Environment) {
254       ((Environment)env).startup();
255     }
256     // HBASE-4014: maintain list of loaded coprocessors for later crash analysis
257     // if server (master or regionserver) aborts.
258     coprocessorNames.add(implClass.getName());
259     return env;
260   }
261 
262   /**
263    * Called when a new Coprocessor class is loaded
264    */
265   public abstract E createEnvironment(Class<?> implClass, Coprocessor instance,
266       int priority, int sequence, Configuration conf);
267 
268   public void shutdown(CoprocessorEnvironment e) {
269     if (e instanceof Environment) {
270       ((Environment)e).shutdown();
271     } else {
272       LOG.warn("Shutdown called on unknown environment: "+
273           e.getClass().getName());
274     }
275   }
276 
277   /**
278    * Find a coprocessor implementation by class name
279    * @param className the class name
280    * @return the coprocessor, or null if not found
281    */
282   public Coprocessor findCoprocessor(String className) {
283     // initialize the coprocessors
284     for (E env: coprocessors) {
285       if (env.getInstance().getClass().getName().equals(className) ||
286           env.getInstance().getClass().getSimpleName().equals(className)) {
287         return env.getInstance();
288       }
289     }
290     return null;
291   }
292 
293   /**
294    * Retrieves the set of classloaders used to instantiate Coprocessor classes defined in external
295    * jar files.
296    * @return A set of ClassLoader instances
297    */
298   Set<ClassLoader> getExternalClassLoaders() {
299     Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>();
300     final ClassLoader systemClassLoader = this.getClass().getClassLoader();
301     for (E env : coprocessors) {
302       ClassLoader cl = env.getInstance().getClass().getClassLoader();
303       if (cl != systemClassLoader ){
304         //do not include system classloader
305         externalClassLoaders.add(cl);
306       }
307     }
308     return externalClassLoaders;
309   }
310 
311   /**
312    * Find a coprocessor environment by class name
313    * @param className the class name
314    * @return the coprocessor, or null if not found
315    */
316   public CoprocessorEnvironment findCoprocessorEnvironment(String className) {
317     // initialize the coprocessors
318     for (E env: coprocessors) {
319       if (env.getInstance().getClass().getName().equals(className) ||
320           env.getInstance().getClass().getSimpleName().equals(className)) {
321         return env;
322       }
323     }
324     return null;
325   }
326 
327   /**
328    * Environment priority comparator.
329    * Coprocessors are chained in sorted order.
330    */
331   static class EnvironmentPriorityComparator
332       implements Comparator<CoprocessorEnvironment> {
333     public int compare(final CoprocessorEnvironment env1,
334         final CoprocessorEnvironment env2) {
335       if (env1.getPriority() < env2.getPriority()) {
336         return -1;
337       } else if (env1.getPriority() > env2.getPriority()) {
338         return 1;
339       }
340       if (env1.getLoadSequence() < env2.getLoadSequence()) {
341         return -1;
342       } else if (env1.getLoadSequence() > env2.getLoadSequence()) {
343         return 1;
344       }
345       return 0;
346     }
347   }
348 
349   /**
350    * Encapsulation of the environment of each coprocessor
351    */
352   public static class Environment implements CoprocessorEnvironment {
353 
354     /**
355      * A wrapper for HTable. Can be used to restrict privilege.
356      *
357      * Currently it just helps to track tables opened by a Coprocessor and
358      * facilitate close of them if it is aborted.
359      *
360      * We also disallow row locking.
361      *
362      * There is nothing now that will stop a coprocessor from using HTable
363      * objects directly instead of this API, but in the future we intend to
364      * analyze coprocessor implementations as they are loaded and reject those
365      * which attempt to use objects and methods outside the Environment
366      * sandbox.
367      */
368     class HTableWrapper implements HTableInterface {
369 
370       private TableName tableName;
371       private HTable table;
372 
373       public HTableWrapper(TableName tableName) throws IOException {
374         this.tableName = tableName;
375         this.table = new HTable(conf, tableName);
376         openTables.add(this);
377       }
378 
379       void internalClose() throws IOException {
380         table.close();
381       }
382 
383       public Configuration getConfiguration() {
384         return table.getConfiguration();
385       }
386 
387       public void close() throws IOException {
388         try {
389           internalClose();
390         } finally {
391           openTables.remove(this);
392         }
393       }
394 
395       public Result getRowOrBefore(byte[] row, byte[] family)
396           throws IOException {
397         return table.getRowOrBefore(row, family);
398       }
399 
400       public Result get(Get get) throws IOException {
401         return table.get(get);
402       }
403 
404       public boolean exists(Get get) throws IOException {
405         return table.exists(get);
406       }
407 
408       public Boolean[] exists(List<Get> gets) throws IOException{
409         return table.exists(gets);
410       }
411 
412       public void put(Put put) throws IOException {
413         table.put(put);
414       }
415 
416       public void put(List<Put> puts) throws IOException {
417         table.put(puts);
418       }
419 
420       public void delete(Delete delete) throws IOException {
421         table.delete(delete);
422       }
423 
424       public void delete(List<Delete> deletes) throws IOException {
425         table.delete(deletes);
426       }
427 
428       public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier,
429           byte[] value, Put put) throws IOException {
430         return table.checkAndPut(row, family, qualifier, value, put);
431       }
432 
433       public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier,
434           byte[] value, Delete delete) throws IOException {
435         return table.checkAndDelete(row, family, qualifier, value, delete);
436       }
437 
438       public long incrementColumnValue(byte[] row, byte[] family,
439           byte[] qualifier, long amount) throws IOException {
440         return table.incrementColumnValue(row, family, qualifier, amount);
441       }
442 
443       public long incrementColumnValue(byte[] row, byte[] family,
444           byte[] qualifier, long amount, Durability durability)
445           throws IOException {
446         return table.incrementColumnValue(row, family, qualifier, amount,
447             durability);
448       }
449 
450       @Override
451       public Result append(Append append) throws IOException {
452         return table.append(append);
453       }
454 
455       @Override
456       public Result increment(Increment increment) throws IOException {
457         return table.increment(increment);
458       }
459 
460       public void flushCommits() throws IOException {
461         table.flushCommits();
462       }
463 
464       public boolean isAutoFlush() {
465         return table.isAutoFlush();
466       }
467 
468       public ResultScanner getScanner(Scan scan) throws IOException {
469         return table.getScanner(scan);
470       }
471 
472       public ResultScanner getScanner(byte[] family) throws IOException {
473         return table.getScanner(family);
474       }
475 
476       public ResultScanner getScanner(byte[] family, byte[] qualifier)
477           throws IOException {
478         return table.getScanner(family, qualifier);
479       }
480 
481       public HTableDescriptor getTableDescriptor() throws IOException {
482         return table.getTableDescriptor();
483       }
484 
485       @Override
486       public byte[] getTableName() {
487         return tableName.getName();
488       }
489 
490       @Override
491       public TableName getName() {
492         return table.getName();
493       }
494 
495       @Override
496       public void batch(List<? extends Row> actions, Object[] results)
497           throws IOException, InterruptedException {
498         table.batch(actions, results);
499       }
500 
501       @Override
502       public Object[] batch(List<? extends Row> actions)
503           throws IOException, InterruptedException {
504         return table.batch(actions);
505       }
506 
507       @Override
508       public <R> void batchCallback(List<? extends Row> actions, Object[] results,
509           Batch.Callback<R> callback) throws IOException, InterruptedException {
510         table.batchCallback(actions, results, callback);
511       }
512 
513       @Override
514       public <R> Object[] batchCallback(List<? extends Row> actions,
515           Batch.Callback<R> callback) throws IOException, InterruptedException {
516         return table.batchCallback(actions, callback);
517       }
518 
519       @Override
520       public Result[] get(List<Get> gets) throws IOException {
521         return table.get(gets);
522       }
523 
524       @Override
525       public CoprocessorRpcChannel coprocessorService(byte[] row) {
526         return table.coprocessorService(row);
527       }
528 
529       @Override
530       public <T extends Service, R> Map<byte[], R> coprocessorService(Class<T> service,
531           byte[] startKey, byte[] endKey, Batch.Call<T, R> callable)
532           throws ServiceException, Throwable {
533         return table.coprocessorService(service, startKey, endKey, callable);
534       }
535 
536       @Override
537       public <T extends Service, R> void coprocessorService(Class<T> service,
538           byte[] startKey, byte[] endKey, Batch.Call<T, R> callable, Batch.Callback<R> callback)
539           throws ServiceException, Throwable {
540         table.coprocessorService(service, startKey, endKey, callable, callback);
541       }
542 
543       @Override
544       public void mutateRow(RowMutations rm) throws IOException {
545         table.mutateRow(rm);
546       }
547 
548       @Override
549       public void setAutoFlush(boolean autoFlush) {
550         table.setAutoFlush(autoFlush);
551       }
552 
553       @Override
554       public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
555         table.setAutoFlush(autoFlush, clearBufferOnFail);
556       }
557 
558       @Override
559       public long getWriteBufferSize() {
560          return table.getWriteBufferSize();
561       }
562 
563       @Override
564       public void setWriteBufferSize(long writeBufferSize) throws IOException {
565         table.setWriteBufferSize(writeBufferSize);
566       }
567     }
568 
569     /** The coprocessor */
570     public Coprocessor impl;
571     /** Chaining priority */
572     protected int priority = Coprocessor.PRIORITY_USER;
573     /** Current coprocessor state */
574     Coprocessor.State state = Coprocessor.State.UNINSTALLED;
575     /** Accounting for tables opened by the coprocessor */
576     protected List<HTableInterface> openTables =
577       Collections.synchronizedList(new ArrayList<HTableInterface>());
578     private int seq;
579     private Configuration conf;
580 
581     /**
582      * Constructor
583      * @param impl the coprocessor instance
584      * @param priority chaining priority
585      */
586     public Environment(final Coprocessor impl, final int priority,
587         final int seq, final Configuration conf) {
588       this.impl = impl;
589       this.priority = priority;
590       this.state = Coprocessor.State.INSTALLED;
591       this.seq = seq;
592       this.conf = conf;
593     }
594 
595     /** Initialize the environment */
596     public void startup() {
597       if (state == Coprocessor.State.INSTALLED ||
598           state == Coprocessor.State.STOPPED) {
599         state = Coprocessor.State.STARTING;
600         try {
601           impl.start(this);
602           state = Coprocessor.State.ACTIVE;
603         } catch (IOException ioe) {
604           LOG.error("Error starting coprocessor "+impl.getClass().getName(), ioe);
605         }
606       } else {
607         LOG.warn("Not starting coprocessor "+impl.getClass().getName()+
608             " because not inactive (state="+state.toString()+")");
609       }
610     }
611 
612     /** Clean up the environment */
613     protected void shutdown() {
614       if (state == Coprocessor.State.ACTIVE) {
615         state = Coprocessor.State.STOPPING;
616         try {
617           impl.stop(this);
618           state = Coprocessor.State.STOPPED;
619         } catch (IOException ioe) {
620           LOG.error("Error stopping coprocessor "+impl.getClass().getName(), ioe);
621         }
622       } else {
623         LOG.warn("Not stopping coprocessor "+impl.getClass().getName()+
624             " because not active (state="+state.toString()+")");
625       }
626       // clean up any table references
627       for (HTableInterface table: openTables) {
628         try {
629           ((HTableWrapper)table).internalClose();
630         } catch (IOException e) {
631           // nothing can be done here
632           LOG.warn("Failed to close " +
633               Bytes.toStringBinary(table.getTableName()), e);
634         }
635       }
636     }
637 
638     @Override
639     public Coprocessor getInstance() {
640       return impl;
641     }
642 
643     @Override
644     public int getPriority() {
645       return priority;
646     }
647 
648     @Override
649     public int getLoadSequence() {
650       return seq;
651     }
652 
653     /** @return the coprocessor environment version */
654     @Override
655     public int getVersion() {
656       return Coprocessor.VERSION;
657     }
658 
659     /** @return the HBase release */
660     @Override
661     public String getHBaseVersion() {
662       return VersionInfo.getVersion();
663     }
664 
665     @Override
666     public Configuration getConfiguration() {
667       return conf;
668     }
669 
670     /**
671      * Open a table from within the Coprocessor environment
672      * @param tableName the table name
673      * @return an interface for manipulating the table
674      * @exception java.io.IOException Exception
675      */
676     @Override
677     public HTableInterface getTable(TableName tableName) throws IOException {
678       return new HTableWrapper(tableName);
679     }
680   }
681 
682   protected void abortServer(final String service,
683       final Server server,
684       final CoprocessorEnvironment environment,
685       final Throwable e) {
686     String coprocessorName = (environment.getInstance()).toString();
687     server.abort("Aborting service: " + service + " running on : "
688             + server.getServerName() + " because coprocessor: "
689             + coprocessorName + " threw an exception.", e);
690   }
691 
692   protected void abortServer(final CoprocessorEnvironment environment,
693                              final Throwable e) {
694     String coprocessorName = (environment.getInstance()).toString();
695     LOG.error("The coprocessor: " + coprocessorName + " threw an unexpected " +
696         "exception: " + e + ", but there's no specific implementation of " +
697         " abortServer() for this coprocessor's environment.");
698   }
699 
700 
701   /**
702    * This is used by coprocessor hooks which are declared to throw IOException
703    * (or its subtypes). For such hooks, we should handle throwable objects
704    * depending on the Throwable's type. Those which are instances of
705    * IOException should be passed on to the client. This is in conformance with
706    * the HBase idiom regarding IOException: that it represents a circumstance
707    * that should be passed along to the client for its own handling. For
708    * example, a coprocessor that implements access controls would throw a
709    * subclass of IOException, such as AccessDeniedException, in its preGet()
710    * method to prevent an unauthorized client's performing a Get on a particular
711    * table.
712    * @param env Coprocessor Environment
713    * @param e Throwable object thrown by coprocessor.
714    * @exception IOException Exception
715    */
716   protected void handleCoprocessorThrowable(final CoprocessorEnvironment env,
717                                             final Throwable e)
718       throws IOException {
719     if (e instanceof IOException) {
720       throw (IOException)e;
721     }
722     // If we got here, e is not an IOException. A loaded coprocessor has a
723     // fatal bug, and the server (master or regionserver) should remove the
724     // faulty coprocessor from its set of active coprocessors. Setting
725     // 'hbase.coprocessor.abortonerror' to true will cause abortServer(),
726     // which may be useful in development and testing environments where
727     // 'failing fast' for error analysis is desired.
728     if (env.getConfiguration().getBoolean("hbase.coprocessor.abortonerror",false)) {
729       // server is configured to abort.
730       abortServer(env, e);
731     } else {
732       LOG.error("Removing coprocessor '" + env.toString() + "' from " +
733           "environment because it threw:  " + e,e);
734       coprocessors.remove(env);
735       throw new DoNotRetryIOException("Coprocessor: '" + env.toString() +
736           "' threw: '" + e + "' and has been removed" + "from the active " +
737           "coprocessor set.", e);
738     }
739   }
740 }
741 
742