View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.procedure;
19  
20  import java.io.IOException;
21  import java.util.Collection;
22  import java.util.HashSet;
23  import java.util.List;
24  import java.util.Set;
25  import java.util.concurrent.ConcurrentMap;
26  import java.util.concurrent.ExecutorService;
27  import java.util.concurrent.Future;
28  import java.util.concurrent.RejectedExecutionException;
29  import java.util.concurrent.SynchronousQueue;
30  import java.util.concurrent.ThreadPoolExecutor;
31  import java.util.concurrent.TimeUnit;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.classification.InterfaceAudience;
36  import org.apache.hadoop.classification.InterfaceStability;
37  import org.apache.hadoop.hbase.DaemonThreadFactory;
38  import org.apache.hadoop.hbase.errorhandling.ForeignException;
39  import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
40  
41  import com.google.common.collect.MapMaker;
42  
43  /**
44   * This is the master side of a distributed complex procedure execution.
45   * <p>
46   * The {@link Procedure} is generic and subclassing or customization shouldn't be
47   * necessary -- any customization should happen just in {@link Subprocedure}s.
48   */
49  @InterfaceAudience.Public
50  @InterfaceStability.Evolving
51  public class ProcedureCoordinator {
52    private static final Log LOG = LogFactory.getLog(ProcedureCoordinator.class);
53  
54    final static long TIMEOUT_MILLIS_DEFAULT = 60000;
55    final static long WAKE_MILLIS_DEFAULT = 500;
56  
57    private final ProcedureCoordinatorRpcs rpcs;
58    private final ExecutorService pool;
59  
60    // Running procedure table.  Maps procedure name to running procedure reference
61    private final ConcurrentMap<String, Procedure> procedures =
62        new MapMaker().concurrencyLevel(4).weakValues().makeMap();
63  
64    /**
65     * Create and start a ProcedureCoordinator.
66     *
67     * The rpc object registers the ProcedureCoordinator and starts any threads in this
68     * constructor.
69     *
70     * @param rpcs
71     * @param pool Used for executing procedures.
72     */
73    public ProcedureCoordinator(ProcedureCoordinatorRpcs rpcs, ThreadPoolExecutor pool) {
74      this.rpcs = rpcs;
75      this.pool = pool;
76      this.rpcs.start(this);
77    }
78  
79    /**
80     * Default thread pool for the procedure
81     */
82    public static ThreadPoolExecutor defaultPool(String coordName, long keepAliveTime, int opThreads,
83        long wakeFrequency) {
84      return new ThreadPoolExecutor(1, opThreads, keepAliveTime, TimeUnit.SECONDS,
85          new SynchronousQueue<Runnable>(),
86          new DaemonThreadFactory("(" + coordName + ")-proc-coordinator-pool"));
87    }
88  
89    /**
90     * Shutdown the thread pools and release rpc resources
91     * @throws IOException
92     */
93    public void close() throws IOException {
94      // have to use shutdown now to break any latch waiting
95      pool.shutdownNow();
96      rpcs.close();
97    }
98  
99    /**
100    * Submit an procedure to kick off its dependent subprocedures.
101    * @param proc Procedure to execute
102    * @return <tt>true</tt> if the procedure was started correctly, <tt>false</tt> if the
103    *         procedure or any subprocedures could not be started.  Failure could be due to
104    *         submitting a procedure multiple times (or one with the same name), or some sort
105    *         of IO problem.  On errors, the procedure's monitor holds a reference to the exception
106    *         that caused the failure.
107    */
108   boolean submitProcedure(Procedure proc) {
109     // if the submitted procedure was null, then we don't want to run it
110     if (proc == null) {
111       return false;
112     }
113     String procName = proc.getName();
114 
115     // make sure we aren't already running a procedure of that name
116     synchronized (procedures) {
117       Procedure oldProc = procedures.get(procName);
118       if (oldProc != null) {
119         // procedures are always eventually completed on both successful and failed execution
120         if (oldProc.completedLatch.getCount() != 0) {
121           LOG.warn("Procedure " + procName + " currently running.  Rejecting new request");
122           return false;
123         }
124         LOG.debug("Procedure " + procName + " was in running list but was completed.  Accepting new attempt.");
125         procedures.remove(procName);
126       }
127     }
128 
129     // kick off the procedure's execution in a separate thread
130     Future<Void> f = null;
131     try {
132       synchronized (procedures) {
133         f = this.pool.submit(proc);
134         // if everything got started properly, we can add it known running procedures
135         this.procedures.put(procName, proc);
136       }
137       return true;
138     } catch (RejectedExecutionException e) {
139       LOG.warn("Procedure " + procName + " rejected by execution pool.  Propagating error and " +
140           "cancelling operation.", e);
141       // the thread pool is full and we can't run the procedure
142       proc.receive(new ForeignException(procName, e));
143 
144       // cancel procedure proactively
145       if (f != null) {
146         f.cancel(true);
147       }
148     }
149     return false;
150   }
151 
152   /**
153    * The connection to the rest of the procedure group (members and coordinator) has been
154    * broken/lost/failed. This should fail any interested procedures, but not attempt to notify other
155    * members since we cannot reach them anymore.
156    * @param message description of the error
157    * @param cause the actual cause of the failure
158    */
159   void rpcConnectionFailure(final String message, final IOException cause) {
160     Collection<Procedure> toNotify = procedures.values();
161 
162     for (Procedure proc : toNotify) {
163       if (proc == null) {
164         continue;
165       }
166       // notify the elements, if they aren't null
167       proc.receive(new ForeignException(proc.getName(), cause));
168     }
169   }
170 
171   /**
172    * Abort the procedure with the given name
173    * @param procName name of the procedure to abort
174    * @param reason serialized information about the abort
175    */
176   public void abortProcedure(String procName, ForeignException reason) {
177     // if we know about the Procedure, notify it
178     synchronized(procedures) {
179       Procedure proc = procedures.get(procName);
180       if (proc == null) {
181         return;
182       }
183       proc.receive(reason);
184     }
185   }
186 
187   /**
188    * Exposed for hooking with unit tests.
189    * @param procName
190    * @param procArgs
191    * @param expectedMembers
192    * @return
193    */
194   Procedure createProcedure(ForeignExceptionDispatcher fed, String procName, byte[] procArgs,
195       List<String> expectedMembers) {
196     // build the procedure
197     return new Procedure(this, fed, WAKE_MILLIS_DEFAULT, TIMEOUT_MILLIS_DEFAULT,
198         procName, procArgs, expectedMembers);
199   }
200 
201   /**
202    * Kick off the named procedure
203    * @param procName name of the procedure to start
204    * @param procArgs arguments for the procedure
205    * @param expectedMembers expected members to start
206    * @return handle to the running procedure, if it was started correctly, <tt>null</tt> otherwise
207    * @throws RejectedExecutionException if there are no more available threads to run the procedure
208    */
209   public Procedure startProcedure(ForeignExceptionDispatcher fed, String procName, byte[] procArgs,
210       List<String> expectedMembers) throws RejectedExecutionException {
211     Procedure proc = createProcedure(fed, procName, procArgs, expectedMembers);
212     if (!this.submitProcedure(proc)) {
213       LOG.error("Failed to submit procedure '" + procName + "'");
214       return null;
215     }
216     return proc;
217   }
218 
219   /**
220    * Notification that the procedure had the specified member acquired its part of the barrier
221    * via {@link Subprocedure#acquireBarrier()}.
222    * @param procName name of the procedure that acquired
223    * @param member name of the member that acquired
224    */
225   void memberAcquiredBarrier(String procName, final String member) {
226     Procedure proc = procedures.get(procName);
227     if (proc != null) {
228       proc.barrierAcquiredByMember(member);
229     }
230   }
231 
232   /**
233    * Notification that the procedure had another member finished executing its in-barrier subproc
234    * via {@link Subprocedure#insideBarrier()}.
235    * @param procName name of the subprocedure that finished
236    * @param member name of the member that executed and released its barrier
237    */
238   void memberFinishedBarrier(String procName, final String member) {
239     Procedure proc = procedures.get(procName);
240     if (proc != null) {
241       proc.barrierReleasedByMember(member);
242     }
243   }
244 
245   /**
246    * @return the rpcs implementation for all current procedures
247    */
248   ProcedureCoordinatorRpcs getRpcs() {
249     return rpcs;
250   }
251 
252   /**
253    * Returns the procedure.  This Procedure is a live instance so should not be modified but can
254    * be inspected.
255    * @param name Name of the procedure
256    * @return Procedure or null if not present any more
257    */
258   public Procedure getProcedure(String name) {
259     return procedures.get(name);
260   }
261 
262   /**
263    * @return Return set of all procedure names.
264    */
265   public Set<String> getProcedureNames() {
266     return new HashSet<String>(procedures.keySet());
267   }
268 }