View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   * http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.Collection;
25  import java.util.HashMap;
26  import java.util.List;
27  import java.util.Map;
28  import java.util.NavigableSet;
29  import java.util.concurrent.ArrayBlockingQueue;
30  import java.util.concurrent.BlockingQueue;
31  import java.util.concurrent.ConcurrentHashMap;
32  import java.util.concurrent.ConcurrentMap;
33  import java.util.regex.Matcher;
34  
35  import org.apache.commons.collections.map.AbstractReferenceMap;
36  import org.apache.commons.collections.map.ReferenceMap;
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
40  import org.apache.hadoop.hbase.classification.InterfaceAudience;
41  import org.apache.hadoop.hbase.classification.InterfaceStability;
42  import org.apache.hadoop.conf.Configuration;
43  import org.apache.hadoop.fs.FileSystem;
44  import org.apache.hadoop.fs.Path;
45  import org.apache.hadoop.hbase.Cell;
46  import org.apache.hadoop.hbase.Coprocessor;
47  import org.apache.hadoop.hbase.CoprocessorEnvironment;
48  import org.apache.hadoop.hbase.HBaseConfiguration;
49  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
50  import org.apache.hadoop.hbase.HConstants;
51  import org.apache.hadoop.hbase.HRegionInfo;
52  import org.apache.hadoop.hbase.client.Append;
53  import org.apache.hadoop.hbase.client.Delete;
54  import org.apache.hadoop.hbase.client.Durability;
55  import org.apache.hadoop.hbase.client.Get;
56  import org.apache.hadoop.hbase.client.Increment;
57  import org.apache.hadoop.hbase.client.Mutation;
58  import org.apache.hadoop.hbase.client.Put;
59  import org.apache.hadoop.hbase.client.Result;
60  import org.apache.hadoop.hbase.client.Scan;
61  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
62  import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
63  import org.apache.hadoop.hbase.coprocessor.EndpointObserver;
64  import org.apache.hadoop.hbase.coprocessor.ObserverContext;
65  import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
66  import org.apache.hadoop.hbase.coprocessor.RegionObserver;
67  import org.apache.hadoop.hbase.coprocessor.RegionObserver.MutationType;
68  import org.apache.hadoop.hbase.filter.ByteArrayComparable;
69  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
70  import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
71  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
72  import org.apache.hadoop.hbase.io.Reference;
73  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
74  import org.apache.hadoop.hbase.regionserver.HRegion.Operation;
75  import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
76  import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
77  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
78  import org.apache.hadoop.hbase.util.Bytes;
79  import org.apache.hadoop.hbase.util.Pair;
80  
81  import com.google.common.collect.ImmutableList;
82  import com.google.common.collect.Lists;
83  import com.google.protobuf.Message;
84  import com.google.protobuf.Service;
85  
86  /**
87   * Implements the coprocessor environment and runtime support for coprocessors
88   * loaded within a {@link HRegion}.
89   */
90  @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
91  @InterfaceStability.Evolving
92  public class RegionCoprocessorHost
93      extends CoprocessorHost<RegionCoprocessorHost.RegionEnvironment> {
94  
95    private static final Log LOG = LogFactory.getLog(RegionCoprocessorHost.class);
96    // The shared data map
97    private static ReferenceMap sharedDataMap =
98        new ReferenceMap(AbstractReferenceMap.HARD, AbstractReferenceMap.WEAK);
99  
100   /**
101    * Encapsulation of the environment of each coprocessor
102    */
103   static class RegionEnvironment extends CoprocessorHost.Environment
104       implements RegionCoprocessorEnvironment {
105 
106     private HRegion region;
107     private RegionServerServices rsServices;
108     ConcurrentMap<String, Object> sharedData;
109     private static final int LATENCY_BUFFER_SIZE = 100;
110     private final BlockingQueue<Long> coprocessorTimeNanos = new ArrayBlockingQueue<Long>(
111         LATENCY_BUFFER_SIZE);
112 
113     /**
114      * Constructor
115      * @param impl the coprocessor instance
116      * @param priority chaining priority
117      */
118     public RegionEnvironment(final Coprocessor impl, final int priority,
119         final int seq, final Configuration conf, final HRegion region,
120         final RegionServerServices services, final ConcurrentMap<String, Object> sharedData) {
121       super(impl, priority, seq, conf);
122       this.region = region;
123       this.rsServices = services;
124       this.sharedData = sharedData;
125     }
126 
127     /** @return the region */
128     @Override
129     public HRegion getRegion() {
130       return region;
131     }
132 
133     /** @return reference to the region server services */
134     @Override
135     public RegionServerServices getRegionServerServices() {
136       return rsServices;
137     }
138 
139     public void shutdown() {
140       super.shutdown();
141     }
142 
143     @Override
144     public ConcurrentMap<String, Object> getSharedData() {
145       return sharedData;
146     }
147 
148     public void offerExecutionLatency(long latencyNanos) {
149       coprocessorTimeNanos.offer(latencyNanos);
150     }
151 
152     public Collection<Long> getExecutionLatenciesNanos() {
153       final List<Long> latencies = Lists.newArrayListWithCapacity(coprocessorTimeNanos.size());
154       coprocessorTimeNanos.drainTo(latencies);
155       return latencies;
156     }
157 
158   }
159 
160   /** The region server services */
161   RegionServerServices rsServices;
162   /** The region */
163   HRegion region;
164 
165   /**
166    * Constructor
167    * @param region the region
168    * @param rsServices interface to available region server functionality
169    * @param conf the configuration
170    */
171   public RegionCoprocessorHost(final HRegion region,
172       final RegionServerServices rsServices, final Configuration conf) {
173     super(rsServices);
174     this.conf = conf;
175     this.rsServices = rsServices;
176     this.region = region;
177     this.pathPrefix = Integer.toString(this.region.getRegionInfo().hashCode());
178 
179     // load system default cp's from configuration.
180     loadSystemCoprocessors(conf, REGION_COPROCESSOR_CONF_KEY);
181 
182     // load system default cp's for user tables from configuration.
183     if (!region.getRegionInfo().getTable().isSystemTable()) {
184       loadSystemCoprocessors(conf, USER_REGION_COPROCESSOR_CONF_KEY);
185     }
186 
187     // load Coprocessor From HDFS
188     loadTableCoprocessors(conf);
189   }
190 
191   void loadTableCoprocessors(final Configuration conf) {
192     // scan the table attributes for coprocessor load specifications
193     // initialize the coprocessors
194     List<RegionEnvironment> configured = new ArrayList<RegionEnvironment>();
195     for (Map.Entry<ImmutableBytesWritable,ImmutableBytesWritable> e:
196         region.getTableDesc().getValues().entrySet()) {
197       String key = Bytes.toString(e.getKey().get()).trim();
198       String spec = Bytes.toString(e.getValue().get()).trim();
199       if (HConstants.CP_HTD_ATTR_KEY_PATTERN.matcher(key).matches()) {
200         // found one
201         try {
202           Matcher matcher = HConstants.CP_HTD_ATTR_VALUE_PATTERN.matcher(spec);
203           if (matcher.matches()) {
204             // jar file path can be empty if the cp class can be loaded
205             // from class loader.
206             Path path = matcher.group(1).trim().isEmpty() ?
207                 null : new Path(matcher.group(1).trim());
208             String className = matcher.group(2).trim();
209             int priority = matcher.group(3).trim().isEmpty() ?
210                 Coprocessor.PRIORITY_USER : Integer.valueOf(matcher.group(3));
211             String cfgSpec = null;
212             try {
213               cfgSpec = matcher.group(4);
214             } catch (IndexOutOfBoundsException ex) {
215               // ignore
216             }
217             Configuration ourConf;
218             if (cfgSpec != null) {
219               cfgSpec = cfgSpec.substring(cfgSpec.indexOf('|') + 1);
220               // do an explicit deep copy of the passed configuration
221               ourConf = new Configuration(false);
222               HBaseConfiguration.merge(ourConf, conf);
223               Matcher m = HConstants.CP_HTD_ATTR_VALUE_PARAM_PATTERN.matcher(cfgSpec);
224               while (m.find()) {
225                 ourConf.set(m.group(1), m.group(2));
226               }
227             } else {
228               ourConf = conf;
229             }
230             // Load encompasses classloading and coprocessor initialization
231             try {
232               RegionEnvironment env = load(path, className, priority, ourConf);
233               configured.add(env);
234               LOG.info("Loaded coprocessor " + className + " from HTD of " +
235                 region.getTableDesc().getTableName().getNameAsString() + " successfully.");
236             } catch (Throwable t) {
237               // Coprocessor failed to load, do we abort on error?
238               if (conf.getBoolean(ABORT_ON_ERROR_KEY, DEFAULT_ABORT_ON_ERROR)) {
239                 abortServer(className, t);
240               } else {
241                 LOG.error("Failed to load coprocessor " + className, t);
242               }
243             }
244           } else {
245             LOG.error("Malformed table coprocessor specification: key=" + key +
246               ", spec: " + spec);
247           }
248         } catch (Exception ioe) {
249           LOG.error("Malformed table coprocessor specification: key=" + key +
250             ", spec: " + spec);
251         }
252       }
253     }
254     // add together to coprocessor set for COW efficiency
255     coprocessors.addAll(configured);
256   }
257 
258   @Override
259   public RegionEnvironment createEnvironment(Class<?> implClass,
260       Coprocessor instance, int priority, int seq, Configuration conf) {
261     // Check if it's an Endpoint.
262     // Due to current dynamic protocol design, Endpoint
263     // uses a different way to be registered and executed.
264     // It uses a visitor pattern to invoke registered Endpoint
265     // method.
266     for (Class<?> c : implClass.getInterfaces()) {
267       if (CoprocessorService.class.isAssignableFrom(c)) {
268         region.registerService( ((CoprocessorService)instance).getService() );
269       }
270     }
271     ConcurrentMap<String, Object> classData;
272     // make sure only one thread can add maps
273     synchronized (sharedDataMap) {
274       // as long as at least one RegionEnvironment holds on to its classData it will
275       // remain in this map
276       classData = (ConcurrentMap<String, Object>)sharedDataMap.get(implClass.getName());
277       if (classData == null) {
278         classData = new ConcurrentHashMap<String, Object>();
279         sharedDataMap.put(implClass.getName(), classData);
280       }
281     }
282     return new RegionEnvironment(instance, priority, seq, conf, region,
283         rsServices, classData);
284   }
285 
286   /**
287    * HBASE-4014 : This is used by coprocessor hooks which are not declared to throw exceptions.
288    *
289    * For example, {@link
290    * org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost#preOpen()} and
291    * {@link org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost#postOpen()} are such hooks.
292    *
293    * See also
294    * {@link org.apache.hadoop.hbase.master.MasterCoprocessorHost#handleCoprocessorThrowable(
295    *    CoprocessorEnvironment, Throwable)}
296    * @param env The coprocessor that threw the exception.
297    * @param e The exception that was thrown.
298    */
299   private void handleCoprocessorThrowableNoRethrow(
300       final CoprocessorEnvironment env, final Throwable e) {
301     try {
302       handleCoprocessorThrowable(env,e);
303     } catch (IOException ioe) {
304       // We cannot throw exceptions from the caller hook, so ignore.
305       LOG.warn(
306         "handleCoprocessorThrowable() threw an IOException while attempting to handle Throwable " +
307         e + ". Ignoring.",e);
308     }
309   }
310 
311   /**
312    * Invoked before a region open.
313    *
314    * @throws IOException Signals that an I/O exception has occurred.
315    */
316   public void preOpen() throws IOException {
317     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
318       @Override
319       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
320           throws IOException {
321         oserver.preOpen(ctx);
322       }
323     });
324   }
325 
326   /**
327    * Invoked after a region open
328    */
329   public void postOpen() {
330     try {
331       execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
332         @Override
333         public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
334             throws IOException {
335           oserver.postOpen(ctx);
336         }
337       });
338     } catch (IOException e) {
339       LOG.warn(e);
340     }
341   }
342 
343   /**
344    * Invoked after log replay on region
345    */
346   public void postLogReplay() {
347     try {
348       execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
349         @Override
350         public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
351             throws IOException {
352           oserver.postLogReplay(ctx);
353         }
354       });
355     } catch (IOException e) {
356       LOG.warn(e);
357     }
358   }
359 
360   /**
361    * Invoked before a region is closed
362    * @param abortRequested true if the server is aborting
363    */
364   public void preClose(final boolean abortRequested) throws IOException {
365     execOperation(false, new RegionOperation() {
366       @Override
367       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
368           throws IOException {
369         oserver.preClose(ctx, abortRequested);
370       }
371     });
372   }
373 
374   /**
375    * Invoked after a region is closed
376    * @param abortRequested true if the server is aborting
377    */
378   public void postClose(final boolean abortRequested) {
379     try {
380       execOperation(false, new RegionOperation() {
381         @Override
382         public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
383             throws IOException {
384           oserver.postClose(ctx, abortRequested);
385         }
386         public void postEnvCall(RegionEnvironment env) {
387           shutdown(env);
388         }
389       });
390     } catch (IOException e) {
391       LOG.warn(e);
392     }
393   }
394 
395   /**
396    * See
397    * {@link RegionObserver#preCompactScannerOpen(ObserverContext, Store, List, ScanType, long, InternalScanner, CompactionRequest)}
398    */
399   public InternalScanner preCompactScannerOpen(final Store store,
400       final List<StoreFileScanner> scanners, final ScanType scanType, final long earliestPutTs,
401       final CompactionRequest request) throws IOException {
402     return execOperationWithResult(null,
403         coprocessors.isEmpty() ? null : new RegionOperationWithResult<InternalScanner>() {
404       @Override
405       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
406           throws IOException {
407         setResult(oserver.preCompactScannerOpen(ctx, store, scanners, scanType,
408           earliestPutTs, getResult(), request));
409       }
410     });
411   }
412 
413   /**
414    * Called prior to selecting the {@link StoreFile}s for compaction from the list of currently
415    * available candidates.
416    * @param store The store where compaction is being requested
417    * @param candidates The currently available store files
418    * @param request custom compaction request
419    * @return If {@code true}, skip the normal selection process and use the current list
420    * @throws IOException
421    */
422   public boolean preCompactSelection(final Store store, final List<StoreFile> candidates,
423       final CompactionRequest request) throws IOException {
424     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
425       @Override
426       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
427           throws IOException {
428         oserver.preCompactSelection(ctx, store, candidates, request);
429       }
430     });
431   }
432 
433   /**
434    * Called after the {@link StoreFile}s to be compacted have been selected from the available
435    * candidates.
436    * @param store The store where compaction is being requested
437    * @param selected The store files selected to compact
438    * @param request custom compaction
439    */
440   public void postCompactSelection(final Store store, final ImmutableList<StoreFile> selected,
441       final CompactionRequest request) {
442     try {
443       execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
444         @Override
445         public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
446             throws IOException {
447           oserver.postCompactSelection(ctx, store, selected, request);
448         }
449       });
450     } catch (IOException e) {
451       LOG.warn(e);
452     }
453   }
454 
455   /**
456    * Called prior to rewriting the store files selected for compaction
457    * @param store the store being compacted
458    * @param scanner the scanner used to read store data during compaction
459    * @param scanType type of Scan
460    * @param request the compaction that will be executed
461    * @throws IOException
462    */
463   public InternalScanner preCompact(final Store store, final InternalScanner scanner,
464       final ScanType scanType, final CompactionRequest request) throws IOException {
465     return execOperationWithResult(false, scanner,
466         coprocessors.isEmpty() ? null : new RegionOperationWithResult<InternalScanner>() {
467       @Override
468       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
469           throws IOException {
470         setResult(oserver.preCompact(ctx, store, getResult(), scanType, request));
471       }
472     });
473   }
474 
475   /**
476    * Called after the store compaction has completed.
477    * @param store the store being compacted
478    * @param resultFile the new store file written during compaction
479    * @param request the compaction that is being executed
480    * @throws IOException
481    */
482   public void postCompact(final Store store, final StoreFile resultFile,
483       final CompactionRequest request) throws IOException {
484     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
485       @Override
486       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
487           throws IOException {
488         oserver.postCompact(ctx, store, resultFile, request);
489       }
490     });
491   }
492 
493   /**
494    * Invoked before a memstore flush
495    * @throws IOException
496    */
497   public InternalScanner preFlush(final Store store, final InternalScanner scanner)
498       throws IOException {
499     return execOperationWithResult(false, scanner,
500         coprocessors.isEmpty() ? null : new RegionOperationWithResult<InternalScanner>() {
501       @Override
502       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
503           throws IOException {
504         setResult(oserver.preFlush(ctx, store, getResult()));
505       }
506     });
507   }
508 
509   /**
510    * Invoked before a memstore flush
511    * @throws IOException
512    */
513   public void preFlush() throws IOException {
514     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
515       @Override
516       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
517           throws IOException {
518         oserver.preFlush(ctx);
519       }
520     });
521   }
522 
523   /**
524    * See
525    * {@link RegionObserver#preFlushScannerOpen(ObserverContext,
526    *    Store, KeyValueScanner, InternalScanner)}
527    */
528   public InternalScanner preFlushScannerOpen(final Store store,
529       final KeyValueScanner memstoreScanner) throws IOException {
530     return execOperationWithResult(null,
531         coprocessors.isEmpty() ? null : new RegionOperationWithResult<InternalScanner>() {
532       @Override
533       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
534           throws IOException {
535         setResult(oserver.preFlushScannerOpen(ctx, store, memstoreScanner, getResult()));
536       }
537     });
538   }
539 
540   /**
541    * Invoked after a memstore flush
542    * @throws IOException
543    */
544   public void postFlush() throws IOException {
545     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
546       @Override
547       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
548           throws IOException {
549         oserver.postFlush(ctx);
550       }
551     });
552   }
553 
554   /**
555    * Invoked after a memstore flush
556    * @throws IOException
557    */
558   public void postFlush(final Store store, final StoreFile storeFile) throws IOException {
559     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
560       @Override
561       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
562           throws IOException {
563         oserver.postFlush(ctx, store, storeFile);
564       }
565     });
566   }
567 
568   /**
569    * Invoked just before a split
570    * @throws IOException
571    */
572   public void preSplit() throws IOException {
573     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
574       @Override
575       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
576           throws IOException {
577         oserver.preSplit(ctx);
578       }
579     });
580   }
581 
582   /**
583    * Invoked just before a split
584    * @throws IOException
585    */
586   public void preSplit(final byte[] splitRow) throws IOException {
587     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
588       @Override
589       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
590           throws IOException {
591         oserver.preSplit(ctx, splitRow);
592       }
593     });
594   }
595 
596   /**
597    * Invoked just after a split
598    * @param l the new left-hand daughter region
599    * @param r the new right-hand daughter region
600    * @throws IOException
601    */
602   public void postSplit(final HRegion l, final HRegion r) throws IOException {
603     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
604       @Override
605       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
606           throws IOException {
607         oserver.postSplit(ctx, l, r);
608       }
609     });
610   }
611 
612   public boolean preSplitBeforePONR(final byte[] splitKey,
613       final List<Mutation> metaEntries) throws IOException {
614     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
615       @Override
616       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
617           throws IOException {
618         oserver.preSplitBeforePONR(ctx, splitKey, metaEntries);
619       }
620     });
621   }
622 
623   public void preSplitAfterPONR() throws IOException {
624     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
625       @Override
626       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
627           throws IOException {
628         oserver.preSplitAfterPONR(ctx);
629       }
630     });
631   }
632 
633   /**
634    * Invoked just before the rollback of a failed split is started
635    * @throws IOException
636    */
637   public void preRollBackSplit() throws IOException {
638     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
639       @Override
640       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
641           throws IOException {
642         oserver.preRollBackSplit(ctx);
643       }
644     });
645   }
646 
647   /**
648    * Invoked just after the rollback of a failed split is done
649    * @throws IOException
650    */
651   public void postRollBackSplit() throws IOException {
652     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
653       @Override
654       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
655           throws IOException {
656         oserver.postRollBackSplit(ctx);
657       }
658     });
659   }
660 
661   /**
662    * Invoked after a split is completed irrespective of a failure or success.
663    * @throws IOException
664    */
665   public void postCompleteSplit() throws IOException {
666     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
667       @Override
668       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
669           throws IOException {
670         oserver.postCompleteSplit(ctx);
671       }
672     });
673   }
674 
675   // RegionObserver support
676 
677   /**
678    * @param row the row key
679    * @param family the family
680    * @param result the result set from the region
681    * @return true if default processing should be bypassed
682    * @exception IOException Exception
683    */
684   public boolean preGetClosestRowBefore(final byte[] row, final byte[] family,
685       final Result result) throws IOException {
686     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
687       @Override
688       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
689           throws IOException {
690         oserver.preGetClosestRowBefore(ctx, row, family, result);
691       }
692     });
693   }
694 
695   /**
696    * @param row the row key
697    * @param family the family
698    * @param result the result set from the region
699    * @exception IOException Exception
700    */
701   public void postGetClosestRowBefore(final byte[] row, final byte[] family,
702       final Result result) throws IOException {
703     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
704       @Override
705       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
706           throws IOException {
707         oserver.postGetClosestRowBefore(ctx, row, family, result);
708       }
709     });
710   }
711 
712   /**
713    * @param get the Get request
714    * @return true if default processing should be bypassed
715    * @exception IOException Exception
716    */
717   public boolean preGet(final Get get, final List<Cell> results)
718       throws IOException {
719     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
720       @Override
721       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
722           throws IOException {
723         oserver.preGetOp(ctx, get, results);
724       }
725     });
726   }
727 
728   /**
729    * @param get the Get request
730    * @param results the result sett
731    * @exception IOException Exception
732    */
733   public void postGet(final Get get, final List<Cell> results)
734       throws IOException {
735     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
736       @Override
737       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
738           throws IOException {
739         oserver.postGetOp(ctx, get, results);
740       }
741     });
742   }
743 
744   /**
745    * @param get the Get request
746    * @return true or false to return to client if bypassing normal operation,
747    * or null otherwise
748    * @exception IOException Exception
749    */
750   public Boolean preExists(final Get get) throws IOException {
751     return execOperationWithResult(true, false,
752         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
753       @Override
754       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
755           throws IOException {
756         setResult(oserver.preExists(ctx, get, getResult()));
757       }
758     });
759   }
760 
761   /**
762    * @param get the Get request
763    * @param exists the result returned by the region server
764    * @return the result to return to the client
765    * @exception IOException Exception
766    */
767   public boolean postExists(final Get get, boolean exists)
768       throws IOException {
769     return execOperationWithResult(exists,
770         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
771       @Override
772       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
773           throws IOException {
774         setResult(oserver.postExists(ctx, get, getResult()));
775       }
776     });
777   }
778 
779   /**
780    * @param put The Put object
781    * @param edit The WALEdit object.
782    * @param durability The durability used
783    * @return true if default processing should be bypassed
784    * @exception IOException Exception
785    */
786   public boolean prePut(final Put put, final WALEdit edit, final Durability durability)
787       throws IOException {
788     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
789       @Override
790       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
791           throws IOException {
792         oserver.prePut(ctx, put, edit, durability);
793       }
794     });
795   }
796 
797   /**
798    * @param mutation - the current mutation
799    * @param kv - the current cell
800    * @param byteNow - current timestamp in bytes
801    * @param get - the get that could be used
802    * Note that the get only does not specify the family and qualifier that should be used
803    * @return true if default processing should be bypassed
804    * @exception IOException
805    *              Exception
806    */
807   public boolean prePrepareTimeStampForDeleteVersion(final Mutation mutation,
808       final Cell kv, final byte[] byteNow, final Get get) throws IOException {
809     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
810       @Override
811       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
812           throws IOException {
813         oserver.prePrepareTimeStampForDeleteVersion(ctx, mutation, kv, byteNow, get);
814       }
815     });
816   }
817 
818   /**
819    * @param put The Put object
820    * @param edit The WALEdit object.
821    * @param durability The durability used
822    * @exception IOException Exception
823    */
824   public void postPut(final Put put, final WALEdit edit, final Durability durability)
825       throws IOException {
826     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
827       @Override
828       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
829           throws IOException {
830         oserver.postPut(ctx, put, edit, durability);
831       }
832     });
833   }
834 
835   /**
836    * @param delete The Delete object
837    * @param edit The WALEdit object.
838    * @param durability The durability used
839    * @return true if default processing should be bypassed
840    * @exception IOException Exception
841    */
842   public boolean preDelete(final Delete delete, final WALEdit edit, final Durability durability)
843       throws IOException {
844     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
845       @Override
846       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
847           throws IOException {
848         oserver.preDelete(ctx, delete, edit, durability);
849       }
850     });
851   }
852 
853   /**
854    * @param delete The Delete object
855    * @param edit The WALEdit object.
856    * @param durability The durability used
857    * @exception IOException Exception
858    */
859   public void postDelete(final Delete delete, final WALEdit edit, final Durability durability)
860       throws IOException {
861     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
862       @Override
863       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
864           throws IOException {
865         oserver.postDelete(ctx, delete, edit, durability);
866       }
867     });
868   }
869 
870   /**
871    * @param miniBatchOp
872    * @return true if default processing should be bypassed
873    * @throws IOException
874    */
875   public boolean preBatchMutate(
876       final MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {
877     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
878       @Override
879       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
880           throws IOException {
881         oserver.preBatchMutate(ctx, miniBatchOp);
882       }
883     });
884   }
885 
886   /**
887    * @param miniBatchOp
888    * @throws IOException
889    */
890   public void postBatchMutate(
891       final MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {
892     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
893       @Override
894       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
895           throws IOException {
896         oserver.postBatchMutate(ctx, miniBatchOp);
897       }
898     });
899   }
900 
901   public void postBatchMutateIndispensably(
902       final MiniBatchOperationInProgress<Mutation> miniBatchOp, final boolean success)
903       throws IOException {
904     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
905       @Override
906       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
907           throws IOException {
908         oserver.postBatchMutateIndispensably(ctx, miniBatchOp, success);
909       }
910     });
911   }
912 
913   /**
914    * @param row row to check
915    * @param family column family
916    * @param qualifier column qualifier
917    * @param compareOp the comparison operation
918    * @param comparator the comparator
919    * @param put data to put if check succeeds
920    * @return true or false to return to client if default processing should
921    * be bypassed, or null otherwise
922    * @throws IOException e
923    */
924   public Boolean preCheckAndPut(final byte [] row, final byte [] family,
925       final byte [] qualifier, final CompareOp compareOp,
926       final ByteArrayComparable comparator, final Put put)
927       throws IOException {
928     return execOperationWithResult(true, false,
929         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
930       @Override
931       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
932           throws IOException {
933         setResult(oserver.preCheckAndPut(ctx, row, family, qualifier,
934           compareOp, comparator, put, getResult()));
935       }
936     });
937   }
938 
939   /**
940    * @param row row to check
941    * @param family column family
942    * @param qualifier column qualifier
943    * @param compareOp the comparison operation
944    * @param comparator the comparator
945    * @param put data to put if check succeeds
946    * @return true or false to return to client if default processing should
947    * be bypassed, or null otherwise
948    * @throws IOException e
949    */
950   public Boolean preCheckAndPutAfterRowLock(final byte[] row, final byte[] family,
951       final byte[] qualifier, final CompareOp compareOp, final ByteArrayComparable comparator,
952       final Put put) throws IOException {
953     return execOperationWithResult(true, false,
954         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
955       @Override
956       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
957           throws IOException {
958         setResult(oserver.preCheckAndPutAfterRowLock(ctx, row, family, qualifier,
959           compareOp, comparator, put, getResult()));
960       }
961     });
962   }
963 
964   /**
965    * @param row row to check
966    * @param family column family
967    * @param qualifier column qualifier
968    * @param compareOp the comparison operation
969    * @param comparator the comparator
970    * @param put data to put if check succeeds
971    * @throws IOException e
972    */
973   public boolean postCheckAndPut(final byte [] row, final byte [] family,
974       final byte [] qualifier, final CompareOp compareOp,
975       final ByteArrayComparable comparator, final Put put,
976       boolean result) throws IOException {
977     return execOperationWithResult(result,
978         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
979       @Override
980       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
981           throws IOException {
982         setResult(oserver.postCheckAndPut(ctx, row, family, qualifier,
983           compareOp, comparator, put, getResult()));
984       }
985     });
986   }
987 
988   /**
989    * @param row row to check
990    * @param family column family
991    * @param qualifier column qualifier
992    * @param compareOp the comparison operation
993    * @param comparator the comparator
994    * @param delete delete to commit if check succeeds
995    * @return true or false to return to client if default processing should
996    * be bypassed, or null otherwise
997    * @throws IOException e
998    */
999   public Boolean preCheckAndDelete(final byte [] row, final byte [] family,
1000       final byte [] qualifier, final CompareOp compareOp,
1001       final ByteArrayComparable comparator, final Delete delete)
1002       throws IOException {
1003     return execOperationWithResult(true, false,
1004         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
1005       @Override
1006       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1007           throws IOException {
1008         setResult(oserver.preCheckAndDelete(ctx, row, family,
1009             qualifier, compareOp, comparator, delete, getResult()));
1010       }
1011     });
1012   }
1013 
1014   /**
1015    * @param row row to check
1016    * @param family column family
1017    * @param qualifier column qualifier
1018    * @param compareOp the comparison operation
1019    * @param comparator the comparator
1020    * @param delete delete to commit if check succeeds
1021    * @return true or false to return to client if default processing should
1022    * be bypassed, or null otherwise
1023    * @throws IOException e
1024    */
1025   public Boolean preCheckAndDeleteAfterRowLock(final byte[] row, final byte[] family,
1026       final byte[] qualifier, final CompareOp compareOp, final ByteArrayComparable comparator,
1027       final Delete delete) throws IOException {
1028     return execOperationWithResult(true, false,
1029         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
1030       @Override
1031       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1032           throws IOException {
1033         setResult(oserver.preCheckAndDeleteAfterRowLock(ctx, row,
1034               family, qualifier, compareOp, comparator, delete, getResult()));
1035       }
1036     });
1037   }
1038 
1039   /**
1040    * @param row row to check
1041    * @param family column family
1042    * @param qualifier column qualifier
1043    * @param compareOp the comparison operation
1044    * @param comparator the comparator
1045    * @param delete delete to commit if check succeeds
1046    * @throws IOException e
1047    */
1048   public boolean postCheckAndDelete(final byte [] row, final byte [] family,
1049       final byte [] qualifier, final CompareOp compareOp,
1050       final ByteArrayComparable comparator, final Delete delete,
1051       boolean result) throws IOException {
1052     return execOperationWithResult(result,
1053         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
1054       @Override
1055       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1056           throws IOException {
1057         setResult(oserver.postCheckAndDelete(ctx, row, family,
1058             qualifier, compareOp, comparator, delete, getResult()));
1059       }
1060     });
1061   }
1062 
1063   /**
1064    * @param append append object
1065    * @return result to return to client if default operation should be
1066    * bypassed, null otherwise
1067    * @throws IOException if an error occurred on the coprocessor
1068    */
1069   public Result preAppend(final Append append) throws IOException {
1070     return execOperationWithResult(true, null,
1071         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Result>() {
1072       @Override
1073       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1074           throws IOException {
1075         setResult(oserver.preAppend(ctx, append));
1076       }
1077     });
1078   }
1079 
1080   /**
1081    * @param append append object
1082    * @return result to return to client if default operation should be
1083    * bypassed, null otherwise
1084    * @throws IOException if an error occurred on the coprocessor
1085    */
1086   public Result preAppendAfterRowLock(final Append append) throws IOException {
1087     return execOperationWithResult(true, null,
1088         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Result>() {
1089       @Override
1090       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1091           throws IOException {
1092         setResult(oserver.preAppendAfterRowLock(ctx, append));
1093       }
1094     });
1095   }
1096 
1097   /**
1098    * @param increment increment object
1099    * @return result to return to client if default operation should be
1100    * bypassed, null otherwise
1101    * @throws IOException if an error occurred on the coprocessor
1102    */
1103   public Result preIncrement(final Increment increment) throws IOException {
1104     return execOperationWithResult(true, null,
1105         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Result>() {
1106       @Override
1107       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1108           throws IOException {
1109         setResult(oserver.preIncrement(ctx, increment));
1110       }
1111     });
1112   }
1113 
1114   /**
1115    * @param increment increment object
1116    * @return result to return to client if default operation should be
1117    * bypassed, null otherwise
1118    * @throws IOException if an error occurred on the coprocessor
1119    */
1120   public Result preIncrementAfterRowLock(final Increment increment) throws IOException {
1121     return execOperationWithResult(true, null,
1122         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Result>() {
1123       @Override
1124       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1125           throws IOException {
1126         setResult(oserver.preIncrementAfterRowLock(ctx, increment));
1127       }
1128     });
1129   }
1130 
1131   /**
1132    * @param append Append object
1133    * @param result the result returned by the append
1134    * @throws IOException if an error occurred on the coprocessor
1135    */
1136   public void postAppend(final Append append, final Result result) throws IOException {
1137     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
1138       @Override
1139       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1140           throws IOException {
1141         oserver.postAppend(ctx, append, result);
1142       }
1143     });
1144   }
1145 
1146   /**
1147    * @param increment increment object
1148    * @param result the result returned by postIncrement
1149    * @throws IOException if an error occurred on the coprocessor
1150    */
1151   public Result postIncrement(final Increment increment, Result result) throws IOException {
1152     return execOperationWithResult(result,
1153         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Result>() {
1154       @Override
1155       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1156           throws IOException {
1157         setResult(oserver.postIncrement(ctx, increment, getResult()));
1158       }
1159     });
1160   }
1161 
1162   /**
1163    * @param scan the Scan specification
1164    * @return scanner id to return to client if default operation should be
1165    * bypassed, false otherwise
1166    * @exception IOException Exception
1167    */
1168   public RegionScanner preScannerOpen(final Scan scan) throws IOException {
1169     return execOperationWithResult(true, null,
1170         coprocessors.isEmpty() ? null : new RegionOperationWithResult<RegionScanner>() {
1171       @Override
1172       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1173           throws IOException {
1174         setResult(oserver.preScannerOpen(ctx, scan, getResult()));
1175       }
1176     });
1177   }
1178 
1179   /**
1180    * See
1181    * {@link RegionObserver#preStoreScannerOpen(ObserverContext,
1182    *    Store, Scan, NavigableSet, KeyValueScanner)}
1183    */
1184   public KeyValueScanner preStoreScannerOpen(final Store store, final Scan scan,
1185       final NavigableSet<byte[]> targetCols) throws IOException {
1186     return execOperationWithResult(null,
1187         coprocessors.isEmpty() ? null : new RegionOperationWithResult<KeyValueScanner>() {
1188       @Override
1189       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1190           throws IOException {
1191         setResult(oserver.preStoreScannerOpen(ctx, store, scan, targetCols, getResult()));
1192       }
1193     });
1194   }
1195 
1196   /**
1197    * @param scan the Scan specification
1198    * @param s the scanner
1199    * @return the scanner instance to use
1200    * @exception IOException Exception
1201    */
1202   public RegionScanner postScannerOpen(final Scan scan, RegionScanner s) throws IOException {
1203     return execOperationWithResult(s,
1204         coprocessors.isEmpty() ? null : new RegionOperationWithResult<RegionScanner>() {
1205       @Override
1206       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1207           throws IOException {
1208         setResult(oserver.postScannerOpen(ctx, scan, getResult()));
1209       }
1210     });
1211   }
1212 
1213   /**
1214    * @param s the scanner
1215    * @param results the result set returned by the region server
1216    * @param limit the maximum number of results to return
1217    * @return 'has next' indication to client if bypassing default behavior, or
1218    * null otherwise
1219    * @exception IOException Exception
1220    */
1221   public Boolean preScannerNext(final InternalScanner s,
1222       final List<Result> results, final int limit) throws IOException {
1223     return execOperationWithResult(true, false,
1224         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
1225       @Override
1226       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1227           throws IOException {
1228         setResult(oserver.preScannerNext(ctx, s, results, limit, getResult()));
1229       }
1230     });
1231   }
1232 
1233   /**
1234    * @param s the scanner
1235    * @param results the result set returned by the region server
1236    * @param limit the maximum number of results to return
1237    * @param hasMore
1238    * @return 'has more' indication to give to client
1239    * @exception IOException Exception
1240    */
1241   public boolean postScannerNext(final InternalScanner s,
1242       final List<Result> results, final int limit, boolean hasMore)
1243       throws IOException {
1244     return execOperationWithResult(hasMore,
1245         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
1246       @Override
1247       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1248           throws IOException {
1249         setResult(oserver.postScannerNext(ctx, s, results, limit, getResult()));
1250       }
1251     });
1252   }
1253 
1254   /**
1255    * This will be called by the scan flow when the current scanned row is being filtered out by the
1256    * filter.
1257    * @param s the scanner
1258    * @param currentRow The current rowkey which got filtered out
1259    * @param offset offset to rowkey
1260    * @param length length of rowkey
1261    * @return whether more rows are available for the scanner or not
1262    * @throws IOException
1263    */
1264   public boolean postScannerFilterRow(final InternalScanner s, final byte[] currentRow,
1265       final int offset, final short length) throws IOException {
1266     return execOperationWithResult(true,
1267         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
1268       @Override
1269       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1270           throws IOException {
1271         setResult(oserver.postScannerFilterRow(ctx, s, currentRow, offset,length, getResult()));
1272       }
1273     });
1274   }
1275 
1276   /**
1277    * @param s the scanner
1278    * @return true if default behavior should be bypassed, false otherwise
1279    * @exception IOException Exception
1280    */
1281   public boolean preScannerClose(final InternalScanner s) throws IOException {
1282     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
1283       @Override
1284       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1285           throws IOException {
1286         oserver.preScannerClose(ctx, s);
1287       }
1288     });
1289   }
1290 
1291   /**
1292    * @exception IOException Exception
1293    */
1294   public void postScannerClose(final InternalScanner s) throws IOException {
1295     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
1296       @Override
1297       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1298           throws IOException {
1299         oserver.postScannerClose(ctx, s);
1300       }
1301     });
1302   }
1303 
1304   /**
1305    * @param info
1306    * @param logKey
1307    * @param logEdit
1308    * @return true if default behavior should be bypassed, false otherwise
1309    * @throws IOException
1310    */
1311   public boolean preWALRestore(final HRegionInfo info, final HLogKey logKey,
1312       final WALEdit logEdit) throws IOException {
1313     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
1314       @Override
1315       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1316           throws IOException {
1317         oserver.preWALRestore(ctx, info, logKey, logEdit);
1318       }
1319     });
1320   }
1321 
1322   /**
1323    * @param info
1324    * @param logKey
1325    * @param logEdit
1326    * @throws IOException
1327    */
1328   public void postWALRestore(final HRegionInfo info, final HLogKey logKey, final WALEdit logEdit)
1329       throws IOException {
1330     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
1331       @Override
1332       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1333           throws IOException {
1334         oserver.postWALRestore(ctx, info, logKey, logEdit);
1335       }
1336     });
1337   }
1338 
1339   /**
1340    * @param familyPaths pairs of { CF, file path } submitted for bulk load
1341    * @return true if the default operation should be bypassed
1342    * @throws IOException
1343    */
1344   public boolean preBulkLoadHFile(final List<Pair<byte[], String>> familyPaths) throws IOException {
1345     return execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
1346       @Override
1347       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1348           throws IOException {
1349         oserver.preBulkLoadHFile(ctx, familyPaths);
1350       }
1351     });
1352   }
1353 
1354   /**
1355    * @param familyPaths pairs of { CF, file path } submitted for bulk load
1356    * @param hasLoaded whether load was successful or not
1357    * @return the possibly modified value of hasLoaded
1358    * @throws IOException
1359    */
1360   public boolean postBulkLoadHFile(final List<Pair<byte[], String>> familyPaths,
1361       boolean hasLoaded) throws IOException {
1362     return execOperationWithResult(hasLoaded,
1363         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() {
1364       @Override
1365       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1366           throws IOException {
1367         setResult(oserver.postBulkLoadHFile(ctx, familyPaths, getResult()));
1368       }
1369     });
1370   }
1371 
1372   public void postStartRegionOperation(final Operation op) throws IOException {
1373     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
1374       @Override
1375       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1376           throws IOException {
1377         oserver.postStartRegionOperation(ctx, op);
1378       }
1379     });
1380   }
1381 
1382   public void postCloseRegionOperation(final Operation op) throws IOException {
1383     execOperation(coprocessors.isEmpty() ? null : new RegionOperation() {
1384       @Override
1385       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1386           throws IOException {
1387         oserver.postCloseRegionOperation(ctx, op);
1388       }
1389     });
1390   }
1391 
1392   /**
1393    * @param fs fileystem to read from
1394    * @param p path to the file
1395    * @param in {@link FSDataInputStreamWrapper}
1396    * @param size Full size of the file
1397    * @param cacheConf
1398    * @param r original reference file. This will be not null only when reading a split file.
1399    * @return a Reader instance to use instead of the base reader if overriding
1400    * default behavior, null otherwise
1401    * @throws IOException
1402    */
1403   public StoreFile.Reader preStoreFileReaderOpen(final FileSystem fs, final Path p,
1404       final FSDataInputStreamWrapper in, final long size, final CacheConfig cacheConf,
1405       final Reference r) throws IOException {
1406     return execOperationWithResult(null,
1407         coprocessors.isEmpty() ? null : new RegionOperationWithResult<StoreFile.Reader>() {
1408       @Override
1409       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1410           throws IOException {
1411         setResult(oserver.preStoreFileReaderOpen(ctx, fs, p, in, size, cacheConf, r, getResult()));
1412       }
1413     });
1414   }
1415 
1416   /**
1417    * @param fs fileystem to read from
1418    * @param p path to the file
1419    * @param in {@link FSDataInputStreamWrapper}
1420    * @param size Full size of the file
1421    * @param cacheConf
1422    * @param r original reference file. This will be not null only when reading a split file.
1423    * @param reader the base reader instance
1424    * @return The reader to use
1425    * @throws IOException
1426    */
1427   public StoreFile.Reader postStoreFileReaderOpen(final FileSystem fs, final Path p,
1428       final FSDataInputStreamWrapper in, final long size, final CacheConfig cacheConf,
1429       final Reference r, final StoreFile.Reader reader) throws IOException {
1430     return execOperationWithResult(reader,
1431         coprocessors.isEmpty() ? null : new RegionOperationWithResult<StoreFile.Reader>() {
1432       @Override
1433       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1434           throws IOException {
1435         setResult(oserver.postStoreFileReaderOpen(ctx, fs, p, in, size, cacheConf, r, getResult()));
1436       }
1437     });
1438   }
1439 
1440   public Cell postMutationBeforeWAL(final MutationType opType, final Mutation mutation,
1441       final Cell oldCell, Cell newCell) throws IOException {
1442     return execOperationWithResult(newCell,
1443         coprocessors.isEmpty() ? null : new RegionOperationWithResult<Cell>() {
1444       @Override
1445       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1446           throws IOException {
1447         setResult(oserver.postMutationBeforeWAL(ctx, opType, mutation, oldCell, getResult()));
1448       }
1449     });
1450   }
1451 
1452   public Message preEndpointInvocation(final Service service, final String methodName,
1453       Message request) throws IOException {
1454     return execOperationWithResult(request,
1455         coprocessors.isEmpty() ? null : new EndpointOperationWithResult<Message>() {
1456       @Override
1457       public void call(EndpointObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1458           throws IOException {
1459         setResult(oserver.preEndpointInvocation(ctx, service, methodName, getResult()));
1460       }
1461     });
1462   }
1463 
1464   public void postEndpointInvocation(final Service service, final String methodName,
1465       final Message request, final Message.Builder responseBuilder) throws IOException {
1466     execOperation(coprocessors.isEmpty() ? null : new EndpointOperation() {
1467       @Override
1468       public void call(EndpointObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1469           throws IOException {
1470         oserver.postEndpointInvocation(ctx, service, methodName, request, responseBuilder);
1471       }
1472     });
1473   }
1474 
1475   public DeleteTracker postInstantiateDeleteTracker(DeleteTracker tracker) throws IOException {
1476     return execOperationWithResult(tracker,
1477         coprocessors.isEmpty() ? null : new RegionOperationWithResult<DeleteTracker>() {
1478       @Override
1479       public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
1480           throws IOException {
1481         setResult(oserver.postInstantiateDeleteTracker(ctx, getResult()));
1482       }
1483     });
1484   }
1485 
1486   public Map<String, DescriptiveStatistics> getCoprocessorExecutionStatistics() {
1487     Map<String, DescriptiveStatistics> results = new HashMap<String, DescriptiveStatistics>();
1488     for (RegionEnvironment env : coprocessors) {
1489       DescriptiveStatistics ds = new DescriptiveStatistics();
1490       if (env.getInstance() instanceof RegionObserver) {
1491         for (Long time : env.getExecutionLatenciesNanos()) {
1492           ds.addValue(time);
1493         }
1494         // Ensures that web ui circumvents the display of NaN values when there are zero samples.
1495         if (ds.getN() == 0) {
1496           ds.addValue(0);
1497         }
1498         results.put(env.getInstance().getClass().getSimpleName(), ds);
1499       }
1500     }
1501     return results;
1502   }
1503 
1504   private static abstract class CoprocessorOperation
1505       extends ObserverContext<RegionCoprocessorEnvironment> {
1506     public abstract void call(Coprocessor observer,
1507         ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException;
1508     public abstract boolean hasCall(Coprocessor observer);
1509     public void postEnvCall(RegionEnvironment env) { }
1510   }
1511 
1512   private static abstract class RegionOperation extends CoprocessorOperation {
1513     public abstract void call(RegionObserver observer,
1514         ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException;
1515 
1516     public boolean hasCall(Coprocessor observer) {
1517       return observer instanceof RegionObserver;
1518     }
1519 
1520     public void call(Coprocessor observer, ObserverContext<RegionCoprocessorEnvironment> ctx)
1521         throws IOException {
1522       call((RegionObserver)observer, ctx);
1523     }
1524   }
1525 
1526   private static abstract class RegionOperationWithResult<T> extends RegionOperation {
1527     private T result = null;
1528     public void setResult(final T result) { this.result = result; }
1529     public T getResult() { return this.result; }
1530   }
1531 
1532   private static abstract class EndpointOperation extends CoprocessorOperation {
1533     public abstract void call(EndpointObserver observer,
1534         ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException;
1535 
1536     public boolean hasCall(Coprocessor observer) {
1537       return observer instanceof EndpointObserver;
1538     }
1539 
1540     public void call(Coprocessor observer, ObserverContext<RegionCoprocessorEnvironment> ctx)
1541         throws IOException {
1542       call((EndpointObserver)observer, ctx);
1543     }
1544   }
1545 
1546   private static abstract class EndpointOperationWithResult<T> extends EndpointOperation {
1547     private T result = null;
1548     public void setResult(final T result) { this.result = result; }
1549     public T getResult() { return this.result; }
1550   }
1551 
1552   private boolean execOperation(final CoprocessorOperation ctx)
1553       throws IOException {
1554     return execOperation(true, ctx);
1555   }
1556 
1557   private <T> T execOperationWithResult(final T defaultValue,
1558       final RegionOperationWithResult<T> ctx) throws IOException {
1559     if (ctx == null) return defaultValue;
1560     ctx.setResult(defaultValue);
1561     execOperation(true, ctx);
1562     return ctx.getResult();
1563   }
1564 
1565   private <T> T execOperationWithResult(final boolean ifBypass, final T defaultValue,
1566       final RegionOperationWithResult<T> ctx) throws IOException {
1567     boolean bypass = false;
1568     T result = defaultValue;
1569     if (ctx != null) {
1570       ctx.setResult(defaultValue);
1571       bypass = execOperation(true, ctx);
1572       result = ctx.getResult();
1573     }
1574     return bypass == ifBypass ? result : null;
1575   }
1576 
1577   private <T> T execOperationWithResult(final T defaultValue,
1578       final EndpointOperationWithResult<T> ctx) throws IOException {
1579     if (ctx == null) return defaultValue;
1580     ctx.setResult(defaultValue);
1581     execOperation(true, ctx);
1582     return ctx.getResult();
1583   }
1584 
1585   private boolean execOperation(final boolean earlyExit, final CoprocessorOperation ctx)
1586       throws IOException {
1587     boolean bypass = false;
1588     for (RegionEnvironment env: coprocessors) {
1589       Coprocessor observer = env.getInstance();
1590       if (ctx.hasCall(observer)) {
1591         long startTime = System.nanoTime();
1592         ctx.prepare(env);
1593         Thread currentThread = Thread.currentThread();
1594         ClassLoader cl = currentThread.getContextClassLoader();
1595         try {
1596           currentThread.setContextClassLoader(env.getClassLoader());
1597           ctx.call(observer, ctx);
1598         } catch (Throwable e) {
1599           handleCoprocessorThrowable(env, e);
1600         } finally {
1601           currentThread.setContextClassLoader(cl);
1602         }
1603         env.offerExecutionLatency(System.nanoTime() - startTime);
1604         bypass |= ctx.shouldBypass();
1605         if (earlyExit && ctx.shouldComplete()) {
1606           break;
1607         }
1608       }
1609 
1610       ctx.postEnvCall(env);
1611     }
1612     return bypass;
1613   }
1614 }