View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   * http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.coprocessor;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.Collections;
25  import java.util.Comparator;
26  import java.util.HashSet;
27  import java.util.List;
28  import java.util.Set;
29  import java.util.SortedSet;
30  import java.util.TreeSet;
31  import java.util.UUID;
32  import java.util.concurrent.ConcurrentSkipListSet;
33  import java.util.concurrent.ExecutorService;
34  import java.util.concurrent.atomic.AtomicInteger;
35  
36  import org.apache.commons.logging.Log;
37  import org.apache.commons.logging.LogFactory;
38  import org.apache.hadoop.hbase.classification.InterfaceAudience;
39  import org.apache.hadoop.hbase.classification.InterfaceStability;
40  import org.apache.hadoop.conf.Configuration;
41  import org.apache.hadoop.fs.Path;
42  import org.apache.hadoop.hbase.Abortable;
43  import org.apache.hadoop.hbase.Coprocessor;
44  import org.apache.hadoop.hbase.CoprocessorEnvironment;
45  import org.apache.hadoop.hbase.DoNotRetryIOException;
46  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
47  import org.apache.hadoop.hbase.TableName;
48  import org.apache.hadoop.hbase.client.HTable;
49  import org.apache.hadoop.hbase.client.HTableInterface;
50  import org.apache.hadoop.hbase.client.HTableWrapper;
51  import org.apache.hadoop.hbase.util.Bytes;
52  import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
53  import org.apache.hadoop.hbase.util.SortedCopyOnWriteSet;
54  import org.apache.hadoop.hbase.util.VersionInfo;
55  
56  /**
57   * Provides the common setup framework and runtime services for coprocessor
58   * invocation from HBase services.
59   * @param <E> the specific environment extension that a concrete implementation
60   * provides
61   */
62  @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
63  @InterfaceStability.Evolving
64  public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
65    public static final String REGION_COPROCESSOR_CONF_KEY =
66        "hbase.coprocessor.region.classes";
67    public static final String REGIONSERVER_COPROCESSOR_CONF_KEY =
68        "hbase.coprocessor.regionserver.classes";
69    public static final String USER_REGION_COPROCESSOR_CONF_KEY =
70        "hbase.coprocessor.user.region.classes";
71    public static final String MASTER_COPROCESSOR_CONF_KEY =
72        "hbase.coprocessor.master.classes";
73    public static final String WAL_COPROCESSOR_CONF_KEY =
74      "hbase.coprocessor.wal.classes";
75    public static final String ABORT_ON_ERROR_KEY = "hbase.coprocessor.abortonerror";
76    public static final boolean DEFAULT_ABORT_ON_ERROR = true;
77    public static final String COPROCESSORS_ENABLED_CONF_KEY = "hbase.coprocessor.enabled";
78    public static final boolean DEFAULT_COPROCESSORS_ENABLED = true;
79    public static final String USER_COPROCESSORS_ENABLED_CONF_KEY =
80      "hbase.coprocessor.user.enabled";
81    public static final boolean DEFAULT_USER_COPROCESSORS_ENABLED = true;
82  
83    private static final Log LOG = LogFactory.getLog(CoprocessorHost.class);
84    protected Abortable abortable;
85    /** Ordered set of loaded coprocessors with lock */
86    protected SortedSet<E> coprocessors =
87        new SortedCopyOnWriteSet<E>(new EnvironmentPriorityComparator());
88    protected Configuration conf;
89    // unique file prefix to use for local copies of jars when classloading
90    protected String pathPrefix;
91    protected AtomicInteger loadSequence = new AtomicInteger();
92  
93    public CoprocessorHost(Abortable abortable) {
94      this.abortable = abortable;
95      this.pathPrefix = UUID.randomUUID().toString();
96    }
97  
98    /**
99     * Not to be confused with the per-object _coprocessors_ (above),
100    * coprocessorNames is static and stores the set of all coprocessors ever
101    * loaded by any thread in this JVM. It is strictly additive: coprocessors are
102    * added to coprocessorNames, by loadInstance() but are never removed, since
103    * the intention is to preserve a history of all loaded coprocessors for
104    * diagnosis in case of server crash (HBASE-4014).
105    */
106   private static Set<String> coprocessorNames =
107       Collections.synchronizedSet(new HashSet<String>());
108   public static Set<String> getLoadedCoprocessors() {
109       return coprocessorNames;
110   }
111 
112   /**
113    * Used to create a parameter to the HServerLoad constructor so that
114    * HServerLoad can provide information about the coprocessors loaded by this
115    * regionserver.
116    * (HBASE-4070: Improve region server metrics to report loaded coprocessors
117    * to master).
118    */
119   public Set<String> getCoprocessors() {
120     Set<String> returnValue = new TreeSet<String>();
121     for(CoprocessorEnvironment e: coprocessors) {
122       returnValue.add(e.getInstance().getClass().getSimpleName());
123     }
124     return returnValue;
125   }
126 
127   /**
128    * Load system coprocessors. Read the class names from configuration.
129    * Called by constructor.
130    */
131   protected void loadSystemCoprocessors(Configuration conf, String confKey) {
132     boolean coprocessorsEnabled = conf.getBoolean(COPROCESSORS_ENABLED_CONF_KEY,
133       DEFAULT_COPROCESSORS_ENABLED);
134     if (!coprocessorsEnabled) {
135       return;
136     }
137 
138     Class<?> implClass = null;
139 
140     // load default coprocessors from configure file
141     String[] defaultCPClasses = conf.getStrings(confKey);
142     if (defaultCPClasses == null || defaultCPClasses.length == 0)
143       return;
144 
145     int priority = Coprocessor.PRIORITY_SYSTEM;
146     List<E> configured = new ArrayList<E>();
147     for (String className : defaultCPClasses) {
148       className = className.trim();
149       if (findCoprocessor(className) != null) {
150         continue;
151       }
152       ClassLoader cl = this.getClass().getClassLoader();
153       Thread.currentThread().setContextClassLoader(cl);
154       try {
155         implClass = cl.loadClass(className);
156         configured.add(loadInstance(implClass, Coprocessor.PRIORITY_SYSTEM, conf));
157         LOG.info("System coprocessor " + className + " was loaded " +
158             "successfully with priority (" + priority++ + ").");
159       } catch (Throwable t) {
160         // We always abort if system coprocessors cannot be loaded
161         abortServer(className, t);
162       }
163     }
164 
165     // add entire set to the collection for COW efficiency
166     coprocessors.addAll(configured);
167   }
168 
169   /**
170    * Load a coprocessor implementation into the host
171    * @param path path to implementation jar
172    * @param className the main class name
173    * @param priority chaining priority
174    * @param conf configuration for coprocessor
175    * @throws java.io.IOException Exception
176    */
177   public E load(Path path, String className, int priority,
178       Configuration conf) throws IOException {
179     Class<?> implClass = null;
180     LOG.debug("Loading coprocessor class " + className + " with path " +
181         path + " and priority " + priority);
182 
183     ClassLoader cl = null;
184     if (path == null) {
185       try {
186         implClass = getClass().getClassLoader().loadClass(className);
187       } catch (ClassNotFoundException e) {
188         throw new IOException("No jar path specified for " + className);
189       }
190     } else {
191       cl = CoprocessorClassLoader.getClassLoader(
192         path, getClass().getClassLoader(), pathPrefix, conf);
193       try {
194         implClass = cl.loadClass(className);
195       } catch (ClassNotFoundException e) {
196         throw new IOException("Cannot load external coprocessor class " + className, e);
197       }
198     }
199 
200     //load custom code for coprocessor
201     Thread currentThread = Thread.currentThread();
202     ClassLoader hostClassLoader = currentThread.getContextClassLoader();
203     try{
204       // switch temporarily to the thread classloader for custom CP
205       currentThread.setContextClassLoader(cl);
206       E cpInstance = loadInstance(implClass, priority, conf);
207       return cpInstance;
208     } finally {
209       // restore the fresh (host) classloader
210       currentThread.setContextClassLoader(hostClassLoader);
211     }
212   }
213 
214   /**
215    * @param implClass Implementation class
216    * @param priority priority
217    * @param conf configuration
218    * @throws java.io.IOException Exception
219    */
220   public void load(Class<?> implClass, int priority, Configuration conf)
221       throws IOException {
222     E env = loadInstance(implClass, priority, conf);
223     coprocessors.add(env);
224   }
225 
226   /**
227    * @param implClass Implementation class
228    * @param priority priority
229    * @param conf configuration
230    * @throws java.io.IOException Exception
231    */
232   public E loadInstance(Class<?> implClass, int priority, Configuration conf)
233       throws IOException {
234     if (!Coprocessor.class.isAssignableFrom(implClass)) {
235       throw new IOException("Configured class " + implClass.getName() + " must implement "
236           + Coprocessor.class.getName() + " interface ");
237     }
238 
239     // create the instance
240     Coprocessor impl;
241     Object o = null;
242     try {
243       o = implClass.newInstance();
244       impl = (Coprocessor)o;
245     } catch (InstantiationException e) {
246       throw new IOException(e);
247     } catch (IllegalAccessException e) {
248       throw new IOException(e);
249     }
250     // create the environment
251     E env = createEnvironment(implClass, impl, priority, loadSequence.incrementAndGet(), conf);
252     if (env instanceof Environment) {
253       ((Environment)env).startup();
254     }
255     // HBASE-4014: maintain list of loaded coprocessors for later crash analysis
256     // if server (master or regionserver) aborts.
257     coprocessorNames.add(implClass.getName());
258     return env;
259   }
260 
261   /**
262    * Called when a new Coprocessor class is loaded
263    */
264   public abstract E createEnvironment(Class<?> implClass, Coprocessor instance,
265       int priority, int sequence, Configuration conf);
266 
267   public void shutdown(CoprocessorEnvironment e) {
268     if (e instanceof Environment) {
269       if (LOG.isDebugEnabled()) {
270         LOG.debug("Stop coprocessor " + e.getInstance().getClass().getName());
271       }
272       ((Environment)e).shutdown();
273     } else {
274       LOG.warn("Shutdown called on unknown environment: "+
275           e.getClass().getName());
276     }
277   }
278 
279   /**
280    * Find a coprocessor implementation by class name
281    * @param className the class name
282    * @return the coprocessor, or null if not found
283    */
284   public Coprocessor findCoprocessor(String className) {
285     for (E env: coprocessors) {
286       if (env.getInstance().getClass().getName().equals(className) ||
287           env.getInstance().getClass().getSimpleName().equals(className)) {
288         return env.getInstance();
289       }
290     }
291     return null;
292   }
293 
294   /**
295    * Find list of coprocessors that extend/implement the given class/interface
296    * @param cls the class/interface to look for
297    * @return the list of coprocessors, or null if not found
298    */
299   public <T extends Coprocessor> List<T> findCoprocessors(Class<T> cls) {
300     ArrayList<T> ret = new ArrayList<T>();
301 
302     for (E env: coprocessors) {
303       Coprocessor cp = env.getInstance();
304 
305       if(cp != null) {
306         if (cls.isAssignableFrom(cp.getClass())) {
307           ret.add((T)cp);
308         }
309       }
310     }
311     return ret;
312   }
313 
314   /**
315    * Find a coprocessor environment by class name
316    * @param className the class name
317    * @return the coprocessor, or null if not found
318    */
319   public CoprocessorEnvironment findCoprocessorEnvironment(String className) {
320     for (E env: coprocessors) {
321       if (env.getInstance().getClass().getName().equals(className) ||
322           env.getInstance().getClass().getSimpleName().equals(className)) {
323         return env;
324       }
325     }
326     return null;
327   }
328 
329   /**
330    * Retrieves the set of classloaders used to instantiate Coprocessor classes defined in external
331    * jar files.
332    * @return A set of ClassLoader instances
333    */
334   Set<ClassLoader> getExternalClassLoaders() {
335     Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>();
336     final ClassLoader systemClassLoader = this.getClass().getClassLoader();
337     for (E env : coprocessors) {
338       ClassLoader cl = env.getInstance().getClass().getClassLoader();
339       if (cl != systemClassLoader ){
340         //do not include system classloader
341         externalClassLoaders.add(cl);
342       }
343     }
344     return externalClassLoaders;
345   }
346 
347   /**
348    * Environment priority comparator.
349    * Coprocessors are chained in sorted order.
350    */
351   static class EnvironmentPriorityComparator
352       implements Comparator<CoprocessorEnvironment> {
353     public int compare(final CoprocessorEnvironment env1,
354         final CoprocessorEnvironment env2) {
355       if (env1.getPriority() < env2.getPriority()) {
356         return -1;
357       } else if (env1.getPriority() > env2.getPriority()) {
358         return 1;
359       }
360       if (env1.getLoadSequence() < env2.getLoadSequence()) {
361         return -1;
362       } else if (env1.getLoadSequence() > env2.getLoadSequence()) {
363         return 1;
364       }
365       return 0;
366     }
367   }
368 
369   /**
370    * Encapsulation of the environment of each coprocessor
371    */
372   public static class Environment implements CoprocessorEnvironment {
373 
374     /** The coprocessor */
375     public Coprocessor impl;
376     /** Chaining priority */
377     protected int priority = Coprocessor.PRIORITY_USER;
378     /** Current coprocessor state */
379     Coprocessor.State state = Coprocessor.State.UNINSTALLED;
380     /** Accounting for tables opened by the coprocessor */
381     protected List<HTableInterface> openTables =
382       Collections.synchronizedList(new ArrayList<HTableInterface>());
383     private int seq;
384     private Configuration conf;
385     private ClassLoader classLoader;
386 
387     /**
388      * Constructor
389      * @param impl the coprocessor instance
390      * @param priority chaining priority
391      */
392     public Environment(final Coprocessor impl, final int priority,
393         final int seq, final Configuration conf) {
394       this.impl = impl;
395       this.classLoader = impl.getClass().getClassLoader();
396       this.priority = priority;
397       this.state = Coprocessor.State.INSTALLED;
398       this.seq = seq;
399       this.conf = conf;
400     }
401 
402     /** Initialize the environment */
403     public void startup() throws IOException {
404       if (state == Coprocessor.State.INSTALLED ||
405           state == Coprocessor.State.STOPPED) {
406         state = Coprocessor.State.STARTING;
407         Thread currentThread = Thread.currentThread();
408         ClassLoader hostClassLoader = currentThread.getContextClassLoader();
409         try {
410           currentThread.setContextClassLoader(this.getClassLoader());
411           impl.start(this);
412           state = Coprocessor.State.ACTIVE;
413         } finally {
414           currentThread.setContextClassLoader(hostClassLoader);
415         }
416       } else {
417         LOG.warn("Not starting coprocessor "+impl.getClass().getName()+
418             " because not inactive (state="+state.toString()+")");
419       }
420     }
421 
422     /** Clean up the environment */
423     protected void shutdown() {
424       if (state == Coprocessor.State.ACTIVE) {
425         state = Coprocessor.State.STOPPING;
426         Thread currentThread = Thread.currentThread();
427         ClassLoader hostClassLoader = currentThread.getContextClassLoader();
428         try {
429           currentThread.setContextClassLoader(this.getClassLoader());
430           impl.stop(this);
431           state = Coprocessor.State.STOPPED;
432         } catch (IOException ioe) {
433           LOG.error("Error stopping coprocessor "+impl.getClass().getName(), ioe);
434         } finally {
435           currentThread.setContextClassLoader(hostClassLoader);
436         }
437       } else {
438         LOG.warn("Not stopping coprocessor "+impl.getClass().getName()+
439             " because not active (state="+state.toString()+")");
440       }
441       // clean up any table references
442       for (HTableInterface table: openTables) {
443         try {
444           ((HTableWrapper)table).internalClose();
445         } catch (IOException e) {
446           // nothing can be done here
447           LOG.warn("Failed to close " +
448               Bytes.toStringBinary(table.getTableName()), e);
449         }
450       }
451     }
452 
453     @Override
454     public Coprocessor getInstance() {
455       return impl;
456     }
457 
458     @Override
459     public ClassLoader getClassLoader() {
460       return classLoader;
461     }
462 
463     @Override
464     public int getPriority() {
465       return priority;
466     }
467 
468     @Override
469     public int getLoadSequence() {
470       return seq;
471     }
472 
473     /** @return the coprocessor environment version */
474     @Override
475     public int getVersion() {
476       return Coprocessor.VERSION;
477     }
478 
479     /** @return the HBase release */
480     @Override
481     public String getHBaseVersion() {
482       return VersionInfo.getVersion();
483     }
484 
485     @Override
486     public Configuration getConfiguration() {
487       return conf;
488     }
489 
490     /**
491      * Open a table from within the Coprocessor environment
492      * @param tableName the table name
493      * @return an interface for manipulating the table
494      * @exception java.io.IOException Exception
495      */
496     @Override
497     public HTableInterface getTable(TableName tableName) throws IOException {
498       return this.getTable(tableName, HTable.getDefaultExecutor(getConfiguration()));
499     }
500 
501     /**
502      * Open a table from within the Coprocessor environment
503      * @param tableName the table name
504      * @return an interface for manipulating the table
505      * @exception java.io.IOException Exception
506      */
507     @Override
508     public HTableInterface getTable(TableName tableName, ExecutorService pool) throws IOException {
509       return HTableWrapper.createWrapper(openTables, tableName, this, pool);
510     }
511   }
512 
513   protected void abortServer(final CoprocessorEnvironment environment, final Throwable e) {
514     abortServer(environment.getInstance().getClass().getName(), e);
515   }
516 
517   protected void abortServer(final String coprocessorName, final Throwable e) {
518     String message = "The coprocessor " + coprocessorName + " threw " + e.toString();
519     LOG.error(message, e);
520     if (abortable != null) {
521       abortable.abort(message, e);
522     } else {
523       LOG.warn("No available Abortable, process was not aborted");
524     }
525   }
526 
527   /**
528    * This is used by coprocessor hooks which are declared to throw IOException
529    * (or its subtypes). For such hooks, we should handle throwable objects
530    * depending on the Throwable's type. Those which are instances of
531    * IOException should be passed on to the client. This is in conformance with
532    * the HBase idiom regarding IOException: that it represents a circumstance
533    * that should be passed along to the client for its own handling. For
534    * example, a coprocessor that implements access controls would throw a
535    * subclass of IOException, such as AccessDeniedException, in its preGet()
536    * method to prevent an unauthorized client's performing a Get on a particular
537    * table.
538    * @param env Coprocessor Environment
539    * @param e Throwable object thrown by coprocessor.
540    * @exception IOException Exception
541    */
542   protected void handleCoprocessorThrowable(final CoprocessorEnvironment env, final Throwable e)
543       throws IOException {
544     if (e instanceof IOException) {
545       throw (IOException)e;
546     }
547     // If we got here, e is not an IOException. A loaded coprocessor has a
548     // fatal bug, and the server (master or regionserver) should remove the
549     // faulty coprocessor from its set of active coprocessors. Setting
550     // 'hbase.coprocessor.abortonerror' to true will cause abortServer(),
551     // which may be useful in development and testing environments where
552     // 'failing fast' for error analysis is desired.
553     if (env.getConfiguration().getBoolean(ABORT_ON_ERROR_KEY, DEFAULT_ABORT_ON_ERROR)) {
554       // server is configured to abort.
555       abortServer(env, e);
556     } else {
557       LOG.error("Removing coprocessor '" + env.toString() + "' from " +
558           "environment because it threw:  " + e,e);
559       coprocessors.remove(env);
560       try {
561         shutdown(env);
562       } catch (Exception x) {
563         LOG.error("Uncaught exception when shutting down coprocessor '"
564             + env.toString() + "'", x);
565       }
566       throw new DoNotRetryIOException("Coprocessor: '" + env.toString() +
567           "' threw: '" + e + "' and has been removed from the active " +
568           "coprocessor set.", e);
569     }
570   }
571 
572   /**
573    * Used to gracefully handle fallback to deprecated methods when we
574    * evolve coprocessor APIs.
575    *
576    * When a particular Coprocessor API is updated to change methods, hosts can support fallback
577    * to the deprecated API by using this method to determine if an instance implements the new API.
578    * In the event that said support is partial, then in the face of a runtime issue that prevents
579    * proper operation {@link #legacyWarning(Class, String)} should be used to let operators know.
580    *
581    * For examples of this in action, see the implementation of
582    * <ul>
583    *   <li>{@link org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost}
584    *   <li>{@link org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost}
585    * </ul>
586    *
587    * @param clazz Coprocessor you wish to evaluate
588    * @param methodName the name of the non-deprecated method version
589    * @param parameterTypes the Class of the non-deprecated method's arguments in the order they are
590    *     declared.
591    */
592   @InterfaceAudience.Private
593   protected static boolean useLegacyMethod(final Class<? extends Coprocessor> clazz,
594       final String methodName, final Class<?>... parameterTypes) {
595     boolean useLegacy;
596     // Use reflection to see if they implement the non-deprecated version
597     try {
598       clazz.getDeclaredMethod(methodName, parameterTypes);
599       LOG.debug("Found an implementation of '" + methodName + "' that uses updated method " +
600           "signature. Skipping legacy support for invocations in '" + clazz +"'.");
601       useLegacy = false;
602     } catch (NoSuchMethodException exception) {
603       useLegacy = true;
604     } catch (SecurityException exception) {
605       LOG.warn("The Security Manager denied our attempt to detect if the coprocessor '" + clazz +
606           "' requires legacy support; assuming it does. If you get later errors about legacy " +
607           "coprocessor use, consider updating your security policy to allow access to the package" +
608           " and declared members of your implementation.");
609       LOG.debug("Details of Security Manager rejection.", exception);
610       useLegacy = true;
611     }
612     return useLegacy;
613   }
614 
615   /**
616    * Used to limit legacy handling to once per Coprocessor class per classloader.
617    */
618   private static final Set<Class<? extends Coprocessor>> legacyWarning =
619       new ConcurrentSkipListSet<Class<? extends Coprocessor>>(
620           new Comparator<Class<? extends Coprocessor>>() {
621             @Override
622             public int compare(Class<? extends Coprocessor> c1, Class<? extends Coprocessor> c2) {
623               if (c1.equals(c2)) {
624                 return 0;
625               }
626               return c1.getName().compareTo(c2.getName());
627             }
628           });
629 
630   /**
631    * limits the amount of logging to once per coprocessor class.
632    * Used in concert with {@link #useLegacyMethod(Class, String, Class[])} when a runtime issue
633    * prevents properly supporting the legacy version of a coprocessor API.
634    * Since coprocessors can be in tight loops this serves to limit the amount of log spam we create.
635    */
636   @InterfaceAudience.Private
637   protected void legacyWarning(final Class<? extends Coprocessor> clazz, final String message) {
638     if(legacyWarning.add(clazz)) {
639       LOG.error("You have a legacy coprocessor loaded and there are events we can't map to the " +
640           " deprecated API. Your coprocessor will not see these events.  Please update '" + clazz +
641           "'. Details of the problem: " + message);
642     }
643   }
644 }