1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.procedure;
19  
20  import static org.junit.Assert.assertNull;
21  import static org.junit.Assert.assertTrue;
22  import static org.mockito.Matchers.any;
23  import static org.mockito.Matchers.anyListOf;
24  import static org.mockito.Matchers.anyString;
25  import static org.mockito.Matchers.eq;
26  import static org.mockito.Mockito.atLeastOnce;
27  import static org.mockito.Mockito.doAnswer;
28  import static org.mockito.Mockito.doThrow;
29  import static org.mockito.Mockito.inOrder;
30  import static org.mockito.Mockito.mock;
31  import static org.mockito.Mockito.never;
32  import static org.mockito.Mockito.reset;
33  import static org.mockito.Mockito.spy;
34  import static org.mockito.Mockito.times;
35  import static org.mockito.Mockito.verify;
36  import static org.mockito.Mockito.when;
37  
38  import java.io.IOException;
39  import java.util.Arrays;
40  import java.util.List;
41  import java.util.concurrent.ThreadPoolExecutor;
42  
43  import org.apache.hadoop.hbase.SmallTests;
44  import org.apache.hadoop.hbase.errorhandling.ForeignException;
45  import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
46  import org.junit.After;
47  import org.junit.Test;
48  import org.junit.experimental.categories.Category;
49  import org.mockito.InOrder;
50  import org.mockito.invocation.InvocationOnMock;
51  import org.mockito.stubbing.Answer;
52  
53  import com.google.common.collect.Lists;
54  
55  /**
56   * Test Procedure coordinator operation.
57   * <p>
58   * This only works correctly when we do <i>class level parallelization</i> of tests. If we do method
59   * level serialization this class will likely throw all kinds of errors.
60   */
61  @Category(SmallTests.class)
62  public class TestProcedureCoordinator {
63    // general test constants
64    private static final long WAKE_FREQUENCY = 1000;
65    private static final long TIMEOUT = 100000;
66    private static final long POOL_KEEP_ALIVE = 1;
67    private static final String nodeName = "node";
68    private static final String procName = "some op";
69    private static final byte[] procData = new byte[0];
70    private static final List<String> expected = Lists.newArrayList("remote1", "remote2");
71  
72    // setup the mocks
73    private final ProcedureCoordinatorRpcs controller = mock(ProcedureCoordinatorRpcs.class);
74    private final Procedure task = mock(Procedure.class);
75    private final ForeignExceptionDispatcher monitor = mock(ForeignExceptionDispatcher.class);
76  
77    // handle to the coordinator for each test
78    private ProcedureCoordinator coordinator;
79  
80    @After
81    public void resetTest() throws IOException {
82      // reset all the mocks used for the tests
83      reset(controller, task, monitor);
84      // close the open coordinator, if it was used
85      if (coordinator != null) coordinator.close();
86    }
87  
88    private ProcedureCoordinator buildNewCoordinator() {
89      ThreadPoolExecutor pool = ProcedureCoordinator.defaultPool(nodeName, POOL_KEEP_ALIVE, 1, WAKE_FREQUENCY);
90      return spy(new ProcedureCoordinator(controller, pool));
91    }
92  
93    /**
94     * Currently we can only handle one procedure at a time.  This makes sure we handle that and
95     * reject submitting more.
96     */
97    @Test
98    public void testThreadPoolSize() throws Exception {
99      ProcedureCoordinator coordinator = buildNewCoordinator();
100     Procedure proc = new Procedure(coordinator,  monitor,
101         WAKE_FREQUENCY, TIMEOUT, procName, procData, expected);
102     Procedure procSpy = spy(proc);
103 
104     Procedure proc2 = new Procedure(coordinator,  monitor,
105         WAKE_FREQUENCY, TIMEOUT, procName +"2", procData, expected);
106     Procedure procSpy2 = spy(proc2);
107     when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(procName), eq(procData), anyListOf(String.class)))
108     .thenReturn(procSpy, procSpy2);
109 
110     coordinator.startProcedure(procSpy.getErrorMonitor(), procName, procData, expected);
111     // null here means second procedure failed to start.
112     assertNull("Coordinator successfully ran two tasks at once with a single thread pool.",
113       coordinator.startProcedure(proc2.getErrorMonitor(), "another op", procData, expected));
114   }
115 
116   /**
117    * Check handling a connection failure correctly if we get it during the acquiring phase
118    */
119   @Test(timeout = 5000)
120   public void testUnreachableControllerDuringPrepare() throws Exception {
121     coordinator = buildNewCoordinator();
122     // setup the proc
123     List<String> expected = Arrays.asList("cohort");
124     Procedure proc = new Procedure(coordinator, WAKE_FREQUENCY,
125         TIMEOUT, procName, procData, expected);
126     final Procedure procSpy = spy(proc);
127 
128     when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(procName), eq(procData), anyListOf(String.class)))
129         .thenReturn(procSpy);
130 
131     // use the passed controller responses
132     IOException cause = new IOException("Failed to reach comms during acquire");
133     doThrow(cause).when(controller)
134         .sendGlobalBarrierAcquire(eq(procSpy), eq(procData), anyListOf(String.class));
135 
136     // run the operation
137     proc = coordinator.startProcedure(proc.getErrorMonitor(), procName, procData, expected);
138     // and wait for it to finish
139     proc.waitForCompleted();
140     verify(procSpy, atLeastOnce()).receive(any(ForeignException.class));
141     verify(coordinator, times(1)).rpcConnectionFailure(anyString(), eq(cause));
142     verify(controller, times(1)).sendGlobalBarrierAcquire(procSpy, procData, expected);
143     verify(controller, never()).sendGlobalBarrierReached(any(Procedure.class),
144         anyListOf(String.class));
145   }
146 
147   /**
148    * Check handling a connection failure correctly if we get it during the barrier phase
149    */
150   @Test(timeout = 5000)
151   public void testUnreachableControllerDuringCommit() throws Exception {
152     coordinator = buildNewCoordinator();
153 
154     // setup the task and spy on it
155     List<String> expected = Arrays.asList("cohort");
156     final Procedure spy = spy(new Procedure(coordinator,
157         WAKE_FREQUENCY, TIMEOUT, procName, procData, expected));
158 
159     when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(procName), eq(procData), anyListOf(String.class)))
160     .thenReturn(spy);
161 
162     // use the passed controller responses
163     IOException cause = new IOException("Failed to reach controller during prepare");
164     doAnswer(new AcquireBarrierAnswer(procName, new String[] { "cohort" }))
165         .when(controller).sendGlobalBarrierAcquire(eq(spy), eq(procData), anyListOf(String.class));
166     doThrow(cause).when(controller).sendGlobalBarrierReached(eq(spy), anyListOf(String.class));
167 
168     // run the operation
169     Procedure task = coordinator.startProcedure(spy.getErrorMonitor(), procName, procData, expected);
170     // and wait for it to finish
171     task.waitForCompleted();
172     verify(spy, atLeastOnce()).receive(any(ForeignException.class));
173     verify(coordinator, times(1)).rpcConnectionFailure(anyString(), eq(cause));
174     verify(controller, times(1)).sendGlobalBarrierAcquire(eq(spy),
175         eq(procData), anyListOf(String.class));
176     verify(controller, times(1)).sendGlobalBarrierReached(any(Procedure.class),
177         anyListOf(String.class));
178   }
179 
180   @Test(timeout = 1000)
181   public void testNoCohort() throws Exception {
182     runSimpleProcedure();
183   }
184 
185   @Test(timeout = 1000)
186   public void testSingleCohortOrchestration() throws Exception {
187     runSimpleProcedure("one");
188   }
189 
190   @Test(timeout = 1000)
191   public void testMultipleCohortOrchestration() throws Exception {
192     runSimpleProcedure("one", "two", "three", "four");
193   }
194 
195   public void runSimpleProcedure(String... members) throws Exception {
196     coordinator = buildNewCoordinator();
197     Procedure task = new Procedure(coordinator, monitor, WAKE_FREQUENCY,
198         TIMEOUT, procName, procData, Arrays.asList(members));
199     final Procedure spy = spy(task);
200     runCoordinatedProcedure(spy, members);
201   }
202 
203   /**
204    * Test that if nodes join the barrier early we still correctly handle the progress
205    */
206   @Test(timeout = 1000)
207   public void testEarlyJoiningBarrier() throws Exception {
208     final String[] cohort = new String[] { "one", "two", "three", "four" };
209     coordinator = buildNewCoordinator();
210     final ProcedureCoordinator ref = coordinator;
211     Procedure task = new Procedure(coordinator, monitor, WAKE_FREQUENCY,
212         TIMEOUT, procName, procData, Arrays.asList(cohort));
213     final Procedure spy = spy(task);
214 
215     AcquireBarrierAnswer prepare = new AcquireBarrierAnswer(procName, cohort) {
216       public void doWork() {
217         // then do some fun where we commit before all nodes have prepared
218         // "one" commits before anyone else is done
219         ref.memberAcquiredBarrier(this.opName, this.cohort[0]);
220         ref.memberFinishedBarrier(this.opName, this.cohort[0]);
221         // but "two" takes a while
222         ref.memberAcquiredBarrier(this.opName, this.cohort[1]);
223         // "three"jumps ahead
224         ref.memberAcquiredBarrier(this.opName, this.cohort[2]);
225         ref.memberFinishedBarrier(this.opName, this.cohort[2]);
226         // and "four" takes a while
227         ref.memberAcquiredBarrier(this.opName, this.cohort[3]);
228       }
229     };
230 
231     BarrierAnswer commit = new BarrierAnswer(procName, cohort) {
232       @Override
233       public void doWork() {
234         ref.memberFinishedBarrier(opName, this.cohort[1]);
235         ref.memberFinishedBarrier(opName, this.cohort[3]);
236       }
237     };
238     runCoordinatedOperation(spy, prepare, commit, cohort);
239   }
240 
241   /**
242    * Just run a procedure with the standard name and data, with not special task for the mock
243    * coordinator (it works just like a regular coordinator). For custom behavior see
244    * {@link #runCoordinatedOperation(Procedure, AcquireBarrierAnswer, BarrierAnswer, String[])}
245    * .
246    * @param spy Spy on a real {@link Procedure}
247    * @param cohort expected cohort members
248    * @throws Exception on failure
249    */
250   public void runCoordinatedProcedure(Procedure spy, String... cohort) throws Exception {
251     runCoordinatedOperation(spy, new AcquireBarrierAnswer(procName, cohort),
252       new BarrierAnswer(procName, cohort), cohort);
253   }
254 
255   public void runCoordinatedOperation(Procedure spy, AcquireBarrierAnswer prepare,
256       String... cohort) throws Exception {
257     runCoordinatedOperation(spy, prepare, new BarrierAnswer(procName, cohort), cohort);
258   }
259 
260   public void runCoordinatedOperation(Procedure spy, BarrierAnswer commit,
261       String... cohort) throws Exception {
262     runCoordinatedOperation(spy, new AcquireBarrierAnswer(procName, cohort), commit, cohort);
263   }
264 
265   public void runCoordinatedOperation(Procedure spy, AcquireBarrierAnswer prepareOperation,
266       BarrierAnswer commitOperation, String... cohort) throws Exception {
267     List<String> expected = Arrays.asList(cohort);
268     when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(procName), eq(procData), anyListOf(String.class)))
269       .thenReturn(spy);
270 
271     // use the passed controller responses
272     doAnswer(prepareOperation).when(controller).sendGlobalBarrierAcquire(spy, procData, expected);
273     doAnswer(commitOperation).when(controller)
274         .sendGlobalBarrierReached(eq(spy), anyListOf(String.class));
275 
276     // run the operation
277     Procedure task = coordinator.startProcedure(spy.getErrorMonitor(), procName, procData, expected);
278     // and wait for it to finish
279     task.waitForCompleted();
280 
281     // make sure we mocked correctly
282     prepareOperation.ensureRan();
283     // we never got an exception
284     InOrder inorder = inOrder(spy, controller);
285     inorder.verify(spy).sendGlobalBarrierStart();
286     inorder.verify(controller).sendGlobalBarrierAcquire(task, procData, expected);
287     inorder.verify(spy).sendGlobalBarrierReached();
288     inorder.verify(controller).sendGlobalBarrierReached(eq(task), anyListOf(String.class));
289   }
290 
291   private abstract class OperationAnswer implements Answer<Void> {
292     private boolean ran = false;
293 
294     public void ensureRan() {
295       assertTrue("Prepare mocking didn't actually run!", ran);
296     }
297 
298     @Override
299     public final Void answer(InvocationOnMock invocation) throws Throwable {
300       this.ran = true;
301       doWork();
302       return null;
303     }
304 
305     protected abstract void doWork() throws Throwable;
306   }
307 
308   /**
309    * Just tell the current coordinator that each of the nodes has prepared
310    */
311   private class AcquireBarrierAnswer extends OperationAnswer {
312     protected final String[] cohort;
313     protected final String opName;
314 
315     public AcquireBarrierAnswer(String opName, String... cohort) {
316       this.cohort = cohort;
317       this.opName = opName;
318     }
319 
320     @Override
321     public void doWork() {
322       if (cohort == null) return;
323       for (String member : cohort) {
324         TestProcedureCoordinator.this.coordinator.memberAcquiredBarrier(opName, member);
325       }
326     }
327   }
328 
329   /**
330    * Just tell the current coordinator that each of the nodes has committed
331    */
332   private class BarrierAnswer extends OperationAnswer {
333     protected final String[] cohort;
334     protected final String opName;
335 
336     public BarrierAnswer(String opName, String... cohort) {
337       this.cohort = cohort;
338       this.opName = opName;
339     }
340 
341     @Override
342     public void doWork() {
343       if (cohort == null) return;
344       for (String member : cohort) {
345         TestProcedureCoordinator.this.coordinator.memberFinishedBarrier(opName, member);
346       }
347     }
348   }
349 }