1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertNotSame;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.util.ArrayList;
28  import java.util.HashMap;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.concurrent.atomic.AtomicBoolean;
32  
33  import org.apache.hadoop.hbase.HBaseConfiguration;
34  import org.apache.hadoop.hbase.HBaseTestingUtility;
35  import org.apache.hadoop.hbase.HConstants;
36  import org.apache.hadoop.hbase.HRegionInfo;
37  import org.apache.hadoop.hbase.HServerLoad;
38  import org.apache.hadoop.hbase.KeyValue;
39  import org.apache.hadoop.hbase.MediumTests;
40  import org.apache.hadoop.hbase.Server;
41  import org.apache.hadoop.hbase.ServerName;
42  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
43  import org.apache.hadoop.hbase.catalog.CatalogTracker;
44  import org.apache.hadoop.hbase.client.Get;
45  import org.apache.hadoop.hbase.client.HConnection;
46  import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
47  import org.apache.hadoop.hbase.client.Result;
48  import org.apache.hadoop.hbase.client.Scan;
49  import org.apache.hadoop.hbase.executor.EventHandler.EventType;
50  import org.apache.hadoop.hbase.executor.ExecutorService;
51  import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
52  import org.apache.hadoop.hbase.executor.RegionTransitionData;
53  import org.apache.hadoop.hbase.ipc.HRegionInterface;
54  import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
55  import org.apache.hadoop.hbase.master.AssignmentManager.RegionState.State;
56  import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
57  import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
58  import org.apache.hadoop.hbase.util.Bytes;
59  import org.apache.hadoop.hbase.util.Pair;
60  import org.apache.hadoop.hbase.util.Threads;
61  import org.apache.hadoop.hbase.util.Writables;
62  import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
63  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
64  import org.apache.hadoop.hbase.zookeeper.ZKTable.TableState;
65  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
66  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
67  import org.apache.zookeeper.KeeperException;
68  import org.apache.zookeeper.KeeperException.NodeExistsException;
69  import org.apache.zookeeper.Watcher;
70  import org.junit.After;
71  import org.junit.AfterClass;
72  import org.junit.Before;
73  import org.junit.BeforeClass;
74  import org.junit.FixMethodOrder;
75  import org.junit.Test;
76  import org.junit.experimental.categories.Category;
77  import org.junit.runners.MethodSorters;
78  import org.mockito.Mockito;
79  import org.mockito.internal.util.reflection.Whitebox;
80  
81  import com.google.protobuf.ServiceException;
82  
83  
84  /**
85   * Test {@link AssignmentManager}
86   */
87  @FixMethodOrder(MethodSorters.NAME_ASCENDING)
88  @Category(MediumTests.class)
89  public class TestAssignmentManager {
90    private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
91    private static final ServerName SERVERNAME_A =
92      new ServerName("example.org", 1234, 5678);
93    private static final ServerName SERVERNAME_B =
94      new ServerName("example.org", 0, 5678);
95    private static final HRegionInfo REGIONINFO =
96      new HRegionInfo(Bytes.toBytes("t"),
97        HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
98    private static final HRegionInfo REGIONINFO_2 = new HRegionInfo(Bytes.toBytes("t"),
99        Bytes.toBytes("a"),Bytes.toBytes( "b"));
100   private static int assignmentCount;
101   private static boolean enabling = false;  
102 
103   // Mocked objects or; get redone for each test.
104   private Server server;
105   private ServerManager serverManager;
106   private ZooKeeperWatcher watcher;
107   private LoadBalancer balancer;
108 
109   @BeforeClass
110   public static void beforeClass() throws Exception {
111     HTU.startMiniZKCluster();
112   }
113 
114   @AfterClass
115   public static void afterClass() throws IOException {
116     HTU.shutdownMiniZKCluster();
117   }
118 
119   @Before
120   public void before() throws ZooKeeperConnectionException, IOException {
121     // TODO: Make generic versions of what we do below and put up in a mocking
122     // utility class or move up into HBaseTestingUtility.
123 
124     // Mock a Server.  Have it return a legit Configuration and ZooKeeperWatcher.
125     // If abort is called, be sure to fail the test (don't just swallow it
126     // silently as is mockito default).
127     this.server = Mockito.mock(Server.class);
128     Mockito.when(server.getConfiguration()).thenReturn(HTU.getConfiguration());
129     this.watcher =
130       new ZooKeeperWatcher(HTU.getConfiguration(), "mockedServer", this.server, true);
131     Mockito.when(server.getZooKeeper()).thenReturn(this.watcher);
132     Mockito.doThrow(new RuntimeException("Aborted")).
133       when(server).abort(Mockito.anyString(), (Throwable)Mockito.anyObject());
134 
135     // Mock a ServerManager.  Say server SERVERNAME_{A,B} are online.  Also
136     // make it so if close or open, we return 'success'.
137     this.serverManager = Mockito.mock(ServerManager.class);
138     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
139     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true);
140     final Map<ServerName, HServerLoad> onlineServers = new HashMap<ServerName, HServerLoad>();
141     onlineServers.put(SERVERNAME_B, new HServerLoad());
142     onlineServers.put(SERVERNAME_A, new HServerLoad());
143     Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
144         new ArrayList<ServerName>(onlineServers.keySet()));
145     Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);
146     Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)).
147       thenReturn(true);
148     Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)).
149       thenReturn(true);
150     // Ditto on open.
151     Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_A, REGIONINFO, -1)).
152       thenReturn(RegionOpeningState.OPENED);
153     Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_B, REGIONINFO, -1)).
154     thenReturn(RegionOpeningState.OPENED);
155   }
156 
157   @After
158     public void after() throws KeeperException {
159     if (this.watcher != null) {
160       // Clean up all znodes
161       ZKAssign.deleteAllNodes(this.watcher);
162       this.watcher.close();
163     }
164   }
165 
166   /**
167    * Test a balance going on at same time as a master failover
168    *
169    * @throws IOException
170    * @throws KeeperException
171    * @throws InterruptedException
172    */
173   @Test(timeout = 60000)
174   public void testBalanceOnMasterFailoverScenarioWithOpenedNode()
175       throws IOException, KeeperException, InterruptedException {
176     AssignmentManagerWithExtrasForTesting am =
177       setUpMockedAssignmentManager(this.server, this.serverManager);
178     try {
179       createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
180       startFakeFailedOverMasterAssignmentManager(am, this.watcher);
181       while (!am.processRITInvoked) Thread.sleep(1);
182       // Now fake the region closing successfully over on the regionserver; the
183       // regionserver will have set the region in CLOSED state. This will
184       // trigger callback into AM. The below zk close call is from the RS close
185       // region handler duplicated here because its down deep in a private
186       // method hard to expose.
187       int versionid =
188         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
189       assertNotSame(versionid, -1);
190       Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
191 
192       // Get the OFFLINE version id.  May have to wait some for it to happen.
193       // OPENING below
194       while (true) {
195         int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
196         if (vid != versionid) {
197           versionid = vid;
198           break;
199         }
200       }
201       assertNotSame(-1, versionid);
202       // This uglyness below is what the openregionhandler on RS side does.
203       versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
204         SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
205         EventType.RS_ZK_REGION_OPENING, versionid);
206       assertNotSame(-1, versionid);
207       // Move znode from OPENING to OPENED as RS does on successful open.
208       versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
209         SERVERNAME_B, versionid);
210       assertNotSame(-1, versionid);
211       am.gate.set(false);
212       // Block here until our znode is cleared or until this test times out.
213       ZKAssign.blockUntilNoRIT(watcher);
214     } finally {
215       am.getExecutorService().shutdown();
216       am.shutdown();
217     }
218   }
219 
220   @Test(timeout = 60000)
221   public void testBalanceOnMasterFailoverScenarioWithClosedNode()
222       throws IOException, KeeperException, InterruptedException {
223     AssignmentManagerWithExtrasForTesting am =
224       setUpMockedAssignmentManager(this.server, this.serverManager);
225     try {
226       createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
227       startFakeFailedOverMasterAssignmentManager(am, this.watcher);
228       while (!am.processRITInvoked) Thread.sleep(1);
229       // Now fake the region closing successfully over on the regionserver; the
230       // regionserver will have set the region in CLOSED state. This will
231       // trigger callback into AM. The below zk close call is from the RS close
232       // region handler duplicated here because its down deep in a private
233       // method hard to expose.
234       int versionid =
235         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
236       assertNotSame(versionid, -1);
237       am.gate.set(false);
238       Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
239 
240       // Get current versionid else will fail on transition from OFFLINE to
241       // OPENING below
242       while (true) {
243         int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
244         if (vid != versionid) {
245           versionid = vid;
246           break;
247         }
248       }
249       assertNotSame(-1, versionid);
250       // This uglyness below is what the openregionhandler on RS side does.
251       versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
252           SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
253           EventType.RS_ZK_REGION_OPENING, versionid);
254       assertNotSame(-1, versionid);
255       // Move znode from OPENING to OPENED as RS does on successful open.
256       versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
257           SERVERNAME_B, versionid);
258       assertNotSame(-1, versionid);
259 
260       // Block here until our znode is cleared or until this test timesout.
261       ZKAssign.blockUntilNoRIT(watcher);
262     } finally {
263       am.getExecutorService().shutdown();
264       am.shutdown();
265     }
266   }
267 
268   @Test(timeout = 60000)
269   public void testBalanceOnMasterFailoverScenarioWithOfflineNode()
270       throws IOException, KeeperException, InterruptedException {
271     AssignmentManagerWithExtrasForTesting am =
272       setUpMockedAssignmentManager(this.server, this.serverManager);
273     try {
274       createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
275       startFakeFailedOverMasterAssignmentManager(am, this.watcher);
276       while (!am.processRITInvoked) Thread.sleep(1);
277       // Now fake the region closing successfully over on the regionserver; the
278       // regionserver will have set the region in CLOSED state. This will
279       // trigger callback into AM. The below zk close call is from the RS close
280       // region handler duplicated here because its down deep in a private
281       // method hard to expose.
282       int versionid =
283         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
284       assertNotSame(versionid, -1);
285       Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
286 
287       am.gate.set(false);
288       // Get current versionid else will fail on transition from OFFLINE to
289       // OPENING below
290       while (true) {
291         int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
292         if (vid != versionid) {
293           versionid = vid;
294           break;
295         }
296       }
297       assertNotSame(-1, versionid);
298       // This uglyness below is what the openregionhandler on RS side does.
299       versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
300           SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
301           EventType.RS_ZK_REGION_OPENING, versionid);
302       assertNotSame(-1, versionid);
303       // Move znode from OPENING to OPENED as RS does on successful open.
304       versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
305           SERVERNAME_B, versionid);
306       assertNotSame(-1, versionid);
307       // Block here until our znode is cleared or until this test timesout.
308       ZKAssign.blockUntilNoRIT(watcher);
309     } finally {
310       am.getExecutorService().shutdown();
311       am.shutdown();
312     }
313   }
314 
315   private void createRegionPlanAndBalance(final AssignmentManager am,
316       final ServerName from, final ServerName to, final HRegionInfo hri) {
317     // Call the balance function but fake the region being online first at
318     // servername from.
319     am.regionOnline(hri, from);
320     // Balance region from 'from' to 'to'. It calls unassign setting CLOSING state
321     // up in zk.  Create a plan and balance
322     am.balance(new RegionPlan(hri, from, to));
323   }
324 
325 
326   /**
327    * Tests AssignmentManager balance function.  Runs a balance moving a region
328    * from one server to another mocking regionserver responding over zk.
329    * @throws IOException
330    * @throws KeeperException
331    * @throws InterruptedException
332    */
333   @Test(timeout = 60000)
334   public void testBalance()
335   throws IOException, KeeperException, InterruptedException {
336     // Create and startup an executor.  This is used by AssignmentManager
337     // handling zk callbacks.
338     ExecutorService executor = startupMasterExecutor("testBalanceExecutor");
339 
340     // We need a mocked catalog tracker.
341     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
342     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
343         .getConfiguration());
344     // Create an AM.
345     AssignmentManager am = new AssignmentManager(this.server,
346         this.serverManager, ct, balancer, executor);
347     try {
348       // Make sure our new AM gets callbacks; once registered, can't unregister.
349       // Thats ok because we make a new zk watcher for each test.
350       this.watcher.registerListenerFirst(am);
351       // Call the balance function but fake the region being online first at
352       // SERVERNAME_A.  Create a balance plan.
353       am.regionOnline(REGIONINFO, SERVERNAME_A);
354       // Balance region from A to B.
355       RegionPlan plan = new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B);
356       am.balance(plan);
357 
358       // Now fake the region closing successfully over on the regionserver; the
359       // regionserver will have set the region in CLOSED state.  This will
360       // trigger callback into AM. The below zk close call is from the RS close
361       // region handler duplicated here because its down deep in a private
362       // method hard to expose.
363       int versionid =
364         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
365       assertNotSame(versionid, -1);
366       // AM is going to notice above CLOSED and queue up a new assign.  The
367       // assign will go to open the region in the new location set by the
368       // balancer.  The zk node will be OFFLINE waiting for regionserver to
369       // transition it through OPENING, OPENED.  Wait till we see the RIT
370       // before we proceed.
371       Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
372       // Get current versionid else will fail on transition from OFFLINE to OPENING below
373       while (true) {
374         int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
375         if (vid != versionid) {
376           versionid = vid;
377           break;
378         }
379       }
380       assertNotSame(-1, versionid);
381       // This uglyness below is what the openregionhandler on RS side does.
382       versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
383         SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
384         EventType.RS_ZK_REGION_OPENING, versionid);
385       assertNotSame(-1, versionid);
386       // Move znode from OPENING to OPENED as RS does on successful open.
387       versionid =
388         ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, SERVERNAME_B, versionid);
389       assertNotSame(-1, versionid);
390       // Wait on the handler removing the OPENED znode.
391       while(am.isRegionInTransition(REGIONINFO) != null) Threads.sleep(1);
392     } finally {
393       executor.shutdown();
394       am.shutdown();
395       // Clean up all znodes
396       ZKAssign.deleteAllNodes(this.watcher);
397     }
398   }
399 
400   /**
401    * Run a simple server shutdown handler.
402    * @throws KeeperException
403    * @throws IOException
404    */
405   @Test
406   public void testShutdownHandler() throws KeeperException, IOException {
407     // Create and startup an executor.  This is used by AssignmentManager
408     // handling zk callbacks.
409     ExecutorService executor = startupMasterExecutor("testShutdownHandler");
410 
411     // We need a mocked catalog tracker.
412     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
413     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
414         .getConfiguration());
415     // Create an AM.
416     AssignmentManager am =
417       new AssignmentManager(this.server, this.serverManager, ct, balancer, executor);
418     try {
419       processServerShutdownHandler(ct, am, false, null);
420     } finally {
421       executor.shutdown();
422       am.shutdown();
423       // Clean up all znodes
424       ZKAssign.deleteAllNodes(this.watcher);
425     }
426   }
427 
428   /**
429    * To test closed region handler to remove rit and delete corresponding znode if region in pending
430    * close or closing while processing shutdown of a region server.(HBASE-5927).
431    * @throws KeeperException
432    * @throws IOException
433    */
434   @Test
435   public void testSSHWhenDisableTableInProgress()
436       throws KeeperException, IOException {
437     testCaseWithPartiallyDisabledState(TableState.DISABLING, false);
438     testCaseWithPartiallyDisabledState(TableState.DISABLED, false);
439   }
440 
441   @Test
442   public void testSSHWhenDisablingTableRegionsInOpeningState()
443       throws KeeperException, IOException {
444     testCaseWithPartiallyDisabledState(TableState.DISABLING, true);
445     testCaseWithPartiallyDisabledState(TableState.DISABLED, true);
446   }
447 
448   
449   /**
450    * To test if the split region is removed from RIT if the region was in SPLITTING state
451    * but the RS has actually completed the splitting in META but went down. See HBASE-6070
452    * and also HBASE-5806
453    * @throws KeeperException
454    * @throws IOException
455    */
456   @Test
457   public void testSSHWhenSplitRegionInProgress()
458       throws KeeperException, IOException, Exception {
459     // true indicates the region is split but still in RIT
460     testCaseWithSplitRegionPartial(true);
461     // false indicate the region is not split
462     testCaseWithSplitRegionPartial(false);
463 
464   }
465 
466   private void testCaseWithSplitRegionPartial(boolean regionSplitDone) throws KeeperException, IOException,
467       NodeExistsException, InterruptedException {
468     // Create and startup an executor. This is used by AssignmentManager
469     // handling zk callbacks.
470     ExecutorService executor = startupMasterExecutor("testSSHWhenSplitRegionInProgress");
471 
472     // We need a mocked catalog tracker.
473     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
474     // Create an AM.
475     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server, this.serverManager);
476     // adding region to regions and servers maps.
477     am.regionOnline(REGIONINFO, SERVERNAME_A);
478     // adding region in pending close.
479     am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
480         State.SPLITTING, System.currentTimeMillis(), SERVERNAME_A));
481     am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
482 
483     RegionTransitionData data = new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING,
484         REGIONINFO.getRegionName(), SERVERNAME_A);
485     String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
486     // create znode in M_ZK_REGION_CLOSING state.
487     ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
488 
489     try {
490       processServerShutdownHandler(ct, am, regionSplitDone, null);
491       // check znode deleted or not.
492       // In both cases the znode should be deleted.
493 
494       if(regionSplitDone){
495         assertTrue("Region state of region in SPLITTING should be removed from rit.",
496             am.regionsInTransition.isEmpty());
497       }
498       else{
499         while (!am.assignInvoked) {
500           Thread.sleep(1);
501         }
502         assertTrue("Assign should be invoked.", am.assignInvoked);
503       }
504     } finally {
505       REGIONINFO.setOffline(false);
506       REGIONINFO.setSplit(false);
507       executor.shutdown();
508       am.shutdown();
509       // Clean up all znodes
510       ZKAssign.deleteAllNodes(this.watcher);
511     }
512   }
513 
514   private void testCaseWithPartiallyDisabledState(TableState state, boolean opening)
515       throws KeeperException, IOException, NodeExistsException {
516     // Create and startup an executor. This is used by AssignmentManager
517     // handling zk callbacks.
518     ExecutorService executor = startupMasterExecutor("testSSHWhenDisableTableInProgress");
519 
520     // We need a mocked catalog tracker.
521     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
522     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
523     // Create an AM.
524     AssignmentManager am = new AssignmentManager(this.server, this.serverManager, ct, balancer,
525         executor);
526     if (opening) {
527       am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
528           State.OPENING, System.currentTimeMillis(), SERVERNAME_A));
529     } else {
530       // adding region to regions and servers maps.
531       am.regionOnline(REGIONINFO, SERVERNAME_A);
532       // adding region in pending close.
533       am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
534           State.PENDING_CLOSE, System.currentTimeMillis(), SERVERNAME_A));
535     }
536     if (state == TableState.DISABLING) {
537       am.getZKTable().setDisablingTable(REGIONINFO.getTableNameAsString());
538     } else {
539       am.getZKTable().setDisabledTable(REGIONINFO.getTableNameAsString());
540     }
541     RegionTransitionData data = null;
542     if (opening) {
543       data =
544           new RegionTransitionData(EventType.RS_ZK_REGION_OPENING, REGIONINFO.getRegionName(),
545               SERVERNAME_A);
546 
547     } else {
548       data =
549           new RegionTransitionData(EventType.M_ZK_REGION_CLOSING, REGIONINFO.getRegionName(),
550               SERVERNAME_A);
551     }
552     String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
553     // create znode in M_ZK_REGION_CLOSING state.
554     ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
555 
556     try {
557       processServerShutdownHandler(ct, am, false, null);
558       // check znode deleted or not.
559       // In both cases the znode should be deleted.
560       assertTrue("The znode should be deleted.",ZKUtil.checkExists(this.watcher, node) == -1);
561       assertTrue("Region state of region in pending close should be removed from rit.",
562         am.regionsInTransition.isEmpty());
563     } finally {
564       executor.shutdown();
565       am.shutdown();
566       // Clean up all znodes
567       ZKAssign.deleteAllNodes(this.watcher);
568     }
569   }
570 
571   private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager am,
572     boolean splitRegion, ServerName sn)
573       throws IOException {
574     // Make sure our new AM gets callbacks; once registered, can't unregister.
575     // Thats ok because we make a new zk watcher for each test.
576     this.watcher.registerListenerFirst(am);
577     // Need to set up a fake scan of meta for the servershutdown handler
578     // Make an RS Interface implementation.  Make it so a scanner can go against it.
579     HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
580     // Get a meta row result that has region up on SERVERNAME_A
581 
582     Result r = null;
583     if (sn == null) {
584       if (splitRegion) {
585         r = getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A);
586       } else {
587         r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
588       }
589     } else {
590       if (sn.equals(SERVERNAME_A)) {
591         r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
592       } else if (sn.equals(SERVERNAME_B)) {
593         r = new Result(new KeyValue[0]);
594       }
595     }
596 
597     Mockito.when(implementation.openScanner((byte [])Mockito.any(), (Scan)Mockito.any())).
598       thenReturn(System.currentTimeMillis());
599     // Return a good result first and then return null to indicate end of scan
600     Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt())).
601       thenReturn(new Result [] {r}, (Result [])null);
602 
603     // Get a connection w/ mocked up common methods.
604     HConnection connection =
605       HConnectionTestingUtility.getMockedConnectionAndDecorate(HTU.getConfiguration(),
606         implementation, SERVERNAME_B, REGIONINFO);
607 
608     // Make it so we can get a catalogtracker from servermanager.. .needed
609     // down in guts of server shutdown handler.
610     Mockito.when(ct.getConnection()).thenReturn(connection);
611     Mockito.when(this.server.getCatalogTracker()).thenReturn(ct);
612 
613     // Now make a server shutdown handler instance and invoke process.
614     // Have it that SERVERNAME_A died.
615     DeadServer deadServers = new DeadServer();
616     deadServers.add(SERVERNAME_A);
617     // I need a services instance that will return the AM
618     MasterServices services = Mockito.mock(MasterServices.class);
619     Mockito.when(services.getAssignmentManager()).thenReturn(am);
620     Mockito.when(services.getZooKeeper()).thenReturn(this.watcher);
621     ServerShutdownHandler handler = null;
622     if (sn != null) {
623       handler = new ServerShutdownHandler(this.server, services, deadServers, sn, false);
624     } else {
625       handler = new ServerShutdownHandler(this.server, services, deadServers, SERVERNAME_A, false);
626     }
627     handler.process();
628     // The region in r will have been assigned.  It'll be up in zk as unassigned.
629   }
630 
631   /**
632    * @param sn ServerName to use making startcode and server in meta
633    * @param hri Region to serialize into HRegionInfo
634    * @return A mocked up Result that fakes a Get on a row in the
635    * <code>.META.</code> table.
636    * @throws IOException
637    */
638   private Result getMetaTableRowResult(final HRegionInfo hri,
639       final ServerName sn)
640   throws IOException {
641     // TODO: Move to a utilities class.  More than one test case can make use
642     // of this facility.
643     List<KeyValue> kvs = new ArrayList<KeyValue>();
644     kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
645       HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
646       Writables.getBytes(hri)));
647     kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
648       HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
649       Bytes.toBytes(sn.getHostAndPort())));
650     kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
651       HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
652       Bytes.toBytes(sn.getStartcode())));
653     return new Result(kvs);
654   }
655 
656   /**
657    * @param sn ServerName to use making startcode and server in meta
658    * @param hri Region to serialize into HRegionInfo
659    * @return A mocked up Result that fakes a Get on a row in the
660    * <code>.META.</code> table.
661    * @throws IOException
662    */
663   private Result getMetaTableRowResultAsSplitRegion(final HRegionInfo hri, final ServerName sn)
664       throws IOException {
665     hri.setOffline(true);
666     hri.setSplit(true);
667     return getMetaTableRowResult(hri, sn);
668   }
669 
670   /**
671    * Create and startup executor pools. Start same set as master does (just
672    * run a few less).
673    * @param name Name to give our executor
674    * @return Created executor (be sure to call shutdown when done).
675    */
676   private ExecutorService startupMasterExecutor(final String name) {
677     // TODO: Move up into HBaseTestingUtility?  Generally useful.
678     ExecutorService executor = new ExecutorService(name);
679     executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
680     executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
681     executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
682     executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
683     return executor;
684   }
685 
686   @Test
687   public void testUnassignWithSplitAtSameTime() throws KeeperException, IOException {
688     // Region to use in test.
689     final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
690     // First amend the servermanager mock so that when we do send close of the
691     // first meta region on SERVERNAME_A, it will return true rather than
692     // default null.
693     Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, hri, -1)).thenReturn(true);
694     // Need a mocked catalog tracker.
695     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
696     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
697         .getConfiguration());
698     // Create an AM.
699     AssignmentManager am =
700       new AssignmentManager(this.server, this.serverManager, ct, balancer, null);
701     try {
702       // First make sure my mock up basically works.  Unassign a region.
703       unassign(am, SERVERNAME_A, hri);
704       // This delete will fail if the previous unassign did wrong thing.
705       ZKAssign.deleteClosingNode(this.watcher, hri);
706       // Now put a SPLITTING region in the way.  I don't have to assert it
707       // go put in place.  This method puts it in place then asserts it still
708       // owns it by moving state from SPLITTING to SPLITTING.
709       int version = createNodeSplitting(this.watcher, hri, SERVERNAME_A);
710       // Now, retry the unassign with the SPLTTING in place.  It should just
711       // complete without fail; a sort of 'silent' recognition that the
712       // region to unassign has been split and no longer exists: TOOD: what if
713       // the split fails and the parent region comes back to life?
714       unassign(am, SERVERNAME_A, hri);
715       // This transition should fail if the znode has been messed with.
716       ZKAssign.transitionNode(this.watcher, hri, SERVERNAME_A,
717         EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
718       assertTrue(am.isRegionInTransition(hri) == null);
719     } finally {
720       am.shutdown();
721     }
722   }
723 
724   /**
725    * Tests the processDeadServersAndRegionsInTransition should not fail with NPE
726    * when it failed to get the children. Let's abort the system in this
727    * situation
728    * @throws ServiceException
729    */
730   @Test(timeout = 60000)
731   public void testProcessDeadServersAndRegionsInTransitionShouldNotFailWithNPE()
732       throws IOException, KeeperException, InterruptedException, ServiceException {
733     final RecoverableZooKeeper recoverableZk = Mockito
734         .mock(RecoverableZooKeeper.class);
735     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
736         this.server, this.serverManager);
737     Watcher zkw = new ZooKeeperWatcher(HBaseConfiguration.create(), "unittest",
738         null) {
739       public RecoverableZooKeeper getRecoverableZooKeeper() {
740         return recoverableZk;
741       }
742     };
743     ((ZooKeeperWatcher) zkw).registerListener(am);
744     Mockito.doThrow(new InterruptedException()).when(recoverableZk)
745         .getChildren("/hbase/unassigned", zkw);
746     am.setWatcher((ZooKeeperWatcher) zkw);
747     try {
748       am.processDeadServersAndRegionsInTransition();
749       fail("Expected to abort");
750     } catch (NullPointerException e) {
751       fail("Should not throw NPE");
752     } catch (RuntimeException e) {
753       assertEquals("Aborted", e.getLocalizedMessage());
754     }
755   }
756 
757   /**
758    * Creates a new ephemeral node in the SPLITTING state for the specified region.
759    * Create it ephemeral in case regionserver dies mid-split.
760    *
761    * <p>Does not transition nodes from other states.  If a node already exists
762    * for this region, a {@link NodeExistsException} will be thrown.
763    *
764    * @param zkw zk reference
765    * @param region region to be created as offline
766    * @param serverName server event originates from
767    * @return Version of znode created.
768    * @throws KeeperException
769    * @throws IOException
770    */
771   // Copied from SplitTransaction rather than open the method over there in
772   // the regionserver package.
773   private static int createNodeSplitting(final ZooKeeperWatcher zkw,
774       final HRegionInfo region, final ServerName serverName)
775   throws KeeperException, IOException {
776     RegionTransitionData data =
777       new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING,
778         region.getRegionName(), serverName);
779 
780     String node = ZKAssign.getNodeName(zkw, region.getEncodedName());
781     if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, data.getBytes())) {
782       throw new IOException("Failed create of ephemeral " + node);
783     }
784     // Transition node from SPLITTING to SPLITTING and pick up version so we
785     // can be sure this znode is ours; version is needed deleting.
786     return transitionNodeSplitting(zkw, region, serverName, -1);
787   }
788 
789   // Copied from SplitTransaction rather than open the method over there in
790   // the regionserver package.
791   private static int transitionNodeSplitting(final ZooKeeperWatcher zkw,
792       final HRegionInfo parent,
793       final ServerName serverName, final int version)
794   throws KeeperException, IOException {
795     return ZKAssign.transitionNode(zkw, parent, serverName,
796       EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
797   }
798 
799   private void unassign(final AssignmentManager am, final ServerName sn,
800       final HRegionInfo hri) {
801     // Before I can unassign a region, I need to set it online.
802     am.regionOnline(hri, sn);
803     // Unassign region.
804     am.unassign(hri);
805   }
806 
807   /**
808    * Create an {@link AssignmentManagerWithExtrasForTesting} that has mocked
809    * {@link CatalogTracker} etc.
810    * @param server
811    * @param manager
812    * @return An AssignmentManagerWithExtras with mock connections, etc.
813    * @throws IOException
814    * @throws KeeperException
815    */
816   private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final Server server,
817       final ServerManager manager)
818   throws IOException, KeeperException {
819     // We need a mocked catalog tracker. Its used by our AM instance.
820     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
821     // Make an RS Interface implementation. Make it so a scanner can go against
822     // it and a get to return the single region, REGIONINFO, this test is
823     // messing with. Needed when "new master" joins cluster. AM will try and
824     // rebuild its list of user regions and it will also get the HRI that goes
825     // with an encoded name by doing a Get on .META.
826     HRegionInterface ri = Mockito.mock(HRegionInterface.class);
827     // Get a meta row result that has region up on SERVERNAME_A for REGIONINFO
828     Result[] result = null;
829     if (enabling) {
830       result = new Result[2];
831       result[0] = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
832       result[1] = getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A);
833     }
834     Result r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
835     Mockito.when(ri .openScanner((byte[]) Mockito.any(), (Scan) Mockito.any())).
836       thenReturn(System.currentTimeMillis());
837    if (enabling) {
838       Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(result, result, result,
839           (Result[]) null);
840       // If a get, return the above result too for REGIONINFO_2
841       Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(
842           getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A));
843     } else {
844       // Return good result 'r' first and then return null to indicate end of scan
845       Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(new Result[] { r });
846       // If a get, return the above result too for REGIONINFO
847       Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(r);
848     }
849     // Get a connection w/ mocked up common methods.
850     HConnection connection = HConnectionTestingUtility.
851       getMockedConnectionAndDecorate(HTU.getConfiguration(), ri, SERVERNAME_B,
852         REGIONINFO);
853     // Make it so we can get the connection from our mocked catalogtracker
854     Mockito.when(ct.getConnection()).thenReturn(connection);
855     // Create and startup an executor. Used by AM handling zk callbacks.
856     ExecutorService executor = startupMasterExecutor("mockedAMExecutor");
857     this.balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
858     AssignmentManagerWithExtrasForTesting am = new AssignmentManagerWithExtrasForTesting(
859         server, manager, ct, balancer, executor);
860     return am;
861   }
862 
863   /**
864    * TestCase verifies that the regionPlan is updated whenever a region fails to open
865    * and the master tries to process RS_ZK_FAILED_OPEN state.(HBASE-5546).
866    */
867   @Test
868   public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, KeeperException,
869       ServiceException, InterruptedException {
870     this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
871         MockedLoadBalancer.class, LoadBalancer.class);
872     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server,
873         this.serverManager);
874     try {
875       // Boolean variable used for waiting until randomAssignment is called and new
876       // plan is generated.
877       AtomicBoolean gate = new AtomicBoolean(false);
878       if (balancer instanceof MockedLoadBalancer) {
879         ((MockedLoadBalancer) balancer).setGateVariable(gate);
880       }
881       ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
882       int v = ZKAssign.getVersion(this.watcher, REGIONINFO);
883       ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
884           EventType.RS_ZK_REGION_FAILED_OPEN, v);
885       String path = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
886       RegionState state = new RegionState(REGIONINFO, State.OPENING, System.currentTimeMillis(),
887           SERVERNAME_A);
888       am.regionsInTransition.put(REGIONINFO.getEncodedName(), state);
889       // a dummy plan inserted into the regionPlans. This plan is cleared and new one is formed
890       am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));
891       RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
892       List<ServerName> serverList = new ArrayList<ServerName>(2);
893       serverList.add(SERVERNAME_B);
894       Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(serverList);
895       am.nodeDataChanged(path);
896       // here we are waiting until the random assignment in the load balancer is called.
897       while (!gate.get()) {
898         Thread.sleep(10);
899       }
900       // new region plan may take some time to get updated after random assignment is called and
901       // gate is set to true.
902       RegionPlan newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
903       while (newRegionPlan == null) {
904         Thread.sleep(10);
905         newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
906       }
907       // the new region plan created may contain the same RS as destination but it should
908       // be new plan.
909       assertNotSame("Same region plan should not come", regionPlan, newRegionPlan);
910       assertTrue("Destnation servers should be different.", !(regionPlan.getDestination().equals(
911         newRegionPlan.getDestination())));
912       Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
913     } finally {
914       this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
915         DefaultLoadBalancer.class, LoadBalancer.class);
916       am.shutdown();
917     }
918   }
919 
920   /**
921    * Test verifies whether assignment is skipped for regions of tables in DISABLING state during
922    * clean cluster startup. See HBASE-6281.
923    *
924    * @throws KeeperException
925    * @throws IOException
926    * @throws Exception
927    */
928   @Test
929   public void testDisablingTableRegionsAssignmentDuringCleanClusterStartup()
930       throws KeeperException, IOException, Exception {
931     this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
932         MockedLoadBalancer.class, LoadBalancer.class);
933     Mockito.when(this.serverManager.getOnlineServers()).thenReturn(
934         new HashMap<ServerName, HServerLoad>(0));
935     List<ServerName> destServers = new ArrayList<ServerName>(1);
936     destServers.add(SERVERNAME_A);
937     Mockito.when(this.serverManager.getDrainingServersList()).thenReturn(destServers);
938     // To avoid cast exception in DisableTableHandler process.
939     //Server server = new HMaster(HTU.getConfiguration());
940     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
941         this.serverManager);
942     AtomicBoolean gate = new AtomicBoolean(false);
943     if (balancer instanceof MockedLoadBalancer) {
944       ((MockedLoadBalancer) balancer).setGateVariable(gate);
945     }
946     try{
947       // set table in disabling state.
948       am.getZKTable().setDisablingTable(REGIONINFO.getTableNameAsString());
949       am.joinCluster();
950       // should not call retainAssignment if we get empty regions in assignAllUserRegions.
951       assertFalse(
952           "Assign should not be invoked for disabling table regions during clean cluster startup.",
953           gate.get());
954       // need to change table state from disabling to disabled.
955       assertTrue("Table should be disabled.",
956           am.getZKTable().isDisabledTable(REGIONINFO.getTableNameAsString()));
957     } finally {
958       this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
959         DefaultLoadBalancer.class, LoadBalancer.class);
960       am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
961       am.shutdown();
962     }
963   }
964 
965   /**
966    * Test verifies whether stale znodes of unknown tables as for the hbase:meta will be removed or
967    * not.
968    * @throws KeeperException
969    * @throws IOException
970    * @throws Exception
971    */
972   @Test
973   public void testMasterRestartShouldRemoveStaleZnodesOfUnknownTableAsForMeta()
974       throws KeeperException, IOException, Exception {
975     List<ServerName> destServers = new ArrayList<ServerName>(1);
976     destServers.add(SERVERNAME_A);
977     Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(destServers);
978     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
979     HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
980     Server server = new HMaster(HTU.getConfiguration());
981     Whitebox.setInternalState(server, "serverManager", this.serverManager);
982     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
983         this.serverManager);
984     try {
985       String tableName = "dummyTable";
986       am.enablingTables.put(tableName, null);
987       // set table in enabling state.
988       am.getZKTable().setEnablingTable(tableName);
989       am.joinCluster();
990       assertFalse("Table should not be present in zookeeper.",
991         am.getZKTable().isTablePresent(tableName));
992     } finally {
993     }
994   }
995 
996   /**
997    * Test verifies whether all the enabling table regions assigned only once during master startup.
998    * 
999    * @throws KeeperException
1000    * @throws IOException
1001    * @throws Exception
1002    */
1003   @Test
1004   public void testMasterRestartWhenTableInEnabling() throws KeeperException, IOException, Exception {
1005     enabling = true;
1006     this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
1007         DefaultLoadBalancer.class, LoadBalancer.class);
1008     Map<ServerName, HServerLoad> serverAndLoad = new HashMap<ServerName, HServerLoad>();
1009     serverAndLoad.put(SERVERNAME_A, null);
1010     Mockito.when(this.serverManager.getOnlineServers()).thenReturn(serverAndLoad);
1011     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(false);
1012     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
1013     HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
1014     Server server = new HMaster(HTU.getConfiguration());
1015     Whitebox.setInternalState(server, "serverManager", this.serverManager);
1016     assignmentCount = 0;
1017     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
1018         this.serverManager);
1019     am.regionOnline(new HRegionInfo("t1".getBytes(), HConstants.EMPTY_START_ROW,
1020         HConstants.EMPTY_END_ROW), SERVERNAME_A);
1021     am.gate.set(false);
1022     try {
1023       // set table in enabling state.
1024       am.getZKTable().setEnablingTable(REGIONINFO.getTableNameAsString());
1025       ZKAssign.createNodeOffline(this.watcher, REGIONINFO_2, SERVERNAME_B);
1026 
1027       am.joinCluster();
1028       while (!am.getZKTable().isEnabledTable(REGIONINFO.getTableNameAsString())) {
1029         Thread.sleep(10);
1030       }
1031       assertEquals("Number of assignments should be equal.", 2, assignmentCount);
1032       assertTrue("Table should be enabled.",
1033           am.getZKTable().isEnabledTable(REGIONINFO.getTableNameAsString()));
1034     } finally {
1035       enabling = false;
1036       am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
1037       am.shutdown();
1038       ZKAssign.deleteAllNodes(this.watcher);
1039       assignmentCount = 0;
1040     }
1041   }
1042 
1043 
1044 
1045   /**
1046    * When region in transition if region server opening the region gone down then region assignment
1047    * taking long time(Waiting for timeout monitor to trigger assign). HBASE-5396(HBASE-6060) fixes this
1048    * scenario. This test case verifies whether SSH calling assign for the region in transition or not.
1049    *
1050    * @throws KeeperException
1051    * @throws IOException
1052    * @throws ServiceException
1053    */
1054   @Test
1055   public void testSSHWhenSourceRSandDestRSInRegionPlanGoneDown() throws KeeperException, IOException,
1056       ServiceException {
1057     testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(true);
1058     testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(false);
1059   }
1060 
1061   private void testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(boolean regionInOffline)
1062       throws IOException, KeeperException, ServiceException {
1063     // We need a mocked catalog tracker.
1064     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1065     // Create an AM.
1066     AssignmentManagerWithExtrasForTesting am =
1067         setUpMockedAssignmentManager(this.server, this.serverManager);
1068     // adding region in pending open.
1069     if (regionInOffline) {
1070       ServerName MASTER_SERVERNAME = new ServerName("example.org", 1111, 1111);
1071       am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
1072           State.OFFLINE, System.currentTimeMillis(), MASTER_SERVERNAME));
1073     } else {
1074       am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
1075           State.OPENING, System.currentTimeMillis(), SERVERNAME_B));
1076     }
1077     // adding region plan
1078     am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B));
1079     am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
1080 
1081     try {
1082       processServerShutdownHandler(ct, am, false, SERVERNAME_A);
1083       processServerShutdownHandler(ct, am, false, SERVERNAME_B);
1084       if(regionInOffline){
1085         assertFalse("Assign should not be invoked.", am.assignInvoked);
1086       } else {
1087         assertTrue("Assign should be invoked.", am.assignInvoked);
1088       }
1089     } finally {
1090       am.regionsInTransition.remove(REGIONINFO.getEncodedName());
1091       am.regionPlans.remove(REGIONINFO.getEncodedName());
1092     }
1093   }
1094 
1095   /**
1096    * Mocked load balancer class used in the testcase to make sure that the testcase waits until
1097    * random assignment is called and the gate variable is set to true.
1098    */
1099   public static class MockedLoadBalancer extends DefaultLoadBalancer {
1100     private AtomicBoolean gate;
1101 
1102     public void setGateVariable(AtomicBoolean gate) {
1103       this.gate = gate;
1104     }
1105 
1106     @Override
1107     public ServerName randomAssignment(List<ServerName> servers) {
1108       ServerName randomServerName = super.randomAssignment(servers);
1109       this.gate.set(true);
1110       return randomServerName;
1111     }
1112 
1113     @Override
1114     public Map<ServerName, List<HRegionInfo>> retainAssignment(
1115         Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
1116       this.gate.set(true);
1117       return super.retainAssignment(regions, servers);
1118     }
1119 
1120   }
1121 
1122   /**
1123    * An {@link AssignmentManager} with some extra facility used testing
1124    */
1125   class AssignmentManagerWithExtrasForTesting extends AssignmentManager {
1126     // Keep a reference so can give it out below in {@link #getExecutorService}
1127     private final ExecutorService es;
1128     // Ditto for ct
1129     private final CatalogTracker ct;
1130     boolean processRITInvoked = false;
1131     boolean assignInvoked = false;
1132     AtomicBoolean gate = new AtomicBoolean(true);
1133 
1134     public AssignmentManagerWithExtrasForTesting(final Server master,
1135         final ServerManager serverManager, final CatalogTracker catalogTracker,
1136         final LoadBalancer balancer, final ExecutorService service)
1137     throws KeeperException, IOException {
1138       super(master, serverManager, catalogTracker, balancer, service);
1139       this.es = service;
1140       this.ct = catalogTracker;
1141     }
1142 
1143     @Override
1144     boolean processRegionInTransition(String encodedRegionName,
1145         HRegionInfo regionInfo,
1146         Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers)
1147         throws KeeperException, IOException {
1148       this.processRITInvoked = true;
1149       return super.processRegionInTransition(encodedRegionName, regionInfo,
1150           deadServers);
1151     }
1152     @Override
1153     void processRegionsInTransition(final RegionTransitionData data,
1154         final HRegionInfo regionInfo,
1155         final Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers,
1156         final int expectedVersion) throws KeeperException {
1157       while (this.gate.get()) Threads.sleep(1);
1158       super.processRegionsInTransition(data, regionInfo, deadServers, expectedVersion);
1159     }
1160     
1161     @Override
1162     public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan,
1163         boolean hijack) {
1164       if (enabling) {
1165         assignmentCount++;
1166         this.regionOnline(region, SERVERNAME_A);
1167       } else {
1168         assignInvoked = true;
1169         super.assign(region, setOfflineInZK, forceNewPlan, hijack);
1170       }
1171     }
1172     
1173     @Override
1174     public ServerName getRegionServerOfRegion(HRegionInfo hri) {
1175       return SERVERNAME_A;
1176     }
1177     
1178     /** reset the watcher */
1179     void setWatcher(ZooKeeperWatcher watcher) {
1180       this.watcher = watcher;
1181     }
1182 
1183     /**
1184      * @return ExecutorService used by this instance.
1185      */
1186     ExecutorService getExecutorService() {
1187       return this.es;
1188     }
1189 
1190     /**
1191      * @return CatalogTracker used by this AM (Its a mock).
1192      */
1193     CatalogTracker getCatalogTracker() {
1194       return this.ct;
1195     }
1196   }
1197 
1198   /**
1199    * Call joinCluster on the passed AssignmentManager.  Do it in a thread
1200    * so it runs independent of what all else is going on.  Try to simulate
1201    * an AM running insided a failed over master by clearing all in-memory
1202    * AM state first.
1203   */
1204   private void startFakeFailedOverMasterAssignmentManager(final AssignmentManager am,
1205       final ZooKeeperWatcher watcher) {
1206     // Make sure our new AM gets callbacks; once registered, we can't unregister.
1207     // Thats ok because we make a new zk watcher for each test.
1208     watcher.registerListenerFirst(am);
1209     Thread t = new Thread("RunAmJoinCluster") {
1210       public void run() {
1211         // Call the joinCluster function as though we were doing a master
1212         // failover at this point. It will stall just before we go to add
1213         // the RIT region to our RIT Map in AM at processRegionsInTransition.
1214         // First clear any inmemory state from AM so it acts like a new master
1215         // coming on line.
1216         am.regionsInTransition.clear();
1217         am.regionPlans.clear();
1218         try {
1219           am.joinCluster();
1220         } catch (IOException e) {
1221           throw new RuntimeException(e);
1222         } catch (KeeperException e) {
1223           throw new RuntimeException(e);
1224         } catch (InterruptedException e) {
1225           throw new RuntimeException(e);
1226         }
1227       };
1228     };
1229     t.start();
1230     while (!t.isAlive()) Threads.sleep(1);
1231   }
1232 
1233   @org.junit.Rule
1234   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
1235     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
1236 }