1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertNotSame;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.util.ArrayList;
28  import java.util.HashMap;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.concurrent.atomic.AtomicBoolean;
32  
33  import org.apache.hadoop.hbase.HBaseConfiguration;
34  import org.apache.hadoop.hbase.HBaseTestingUtility;
35  import org.apache.hadoop.hbase.HConstants;
36  import org.apache.hadoop.hbase.HRegionInfo;
37  import org.apache.hadoop.hbase.HServerLoad;
38  import org.apache.hadoop.hbase.KeyValue;
39  import org.apache.hadoop.hbase.MediumTests;
40  import org.apache.hadoop.hbase.Server;
41  import org.apache.hadoop.hbase.ServerName;
42  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
43  import org.apache.hadoop.hbase.catalog.CatalogTracker;
44  import org.apache.hadoop.hbase.client.Get;
45  import org.apache.hadoop.hbase.client.HConnection;
46  import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
47  import org.apache.hadoop.hbase.client.Result;
48  import org.apache.hadoop.hbase.client.Scan;
49  import org.apache.hadoop.hbase.executor.EventHandler.EventType;
50  import org.apache.hadoop.hbase.executor.ExecutorService;
51  import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
52  import org.apache.hadoop.hbase.executor.RegionTransitionData;
53  import org.apache.hadoop.hbase.ipc.HRegionInterface;
54  import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
55  import org.apache.hadoop.hbase.master.AssignmentManager.RegionState.State;
56  import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
57  import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
58  import org.apache.hadoop.hbase.util.Bytes;
59  import org.apache.hadoop.hbase.util.Pair;
60  import org.apache.hadoop.hbase.util.Threads;
61  import org.apache.hadoop.hbase.util.Writables;
62  import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
63  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
64  import org.apache.hadoop.hbase.zookeeper.ZKTable.TableState;
65  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
66  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
67  import org.apache.zookeeper.KeeperException;
68  import org.apache.zookeeper.KeeperException.NodeExistsException;
69  import org.apache.zookeeper.Watcher;
70  import org.junit.After;
71  import org.junit.AfterClass;
72  import org.junit.Before;
73  import org.junit.BeforeClass;
74  import org.junit.Test;
75  import org.junit.experimental.categories.Category;
76  import org.mockito.Mockito;
77  import org.mockito.internal.util.reflection.Whitebox;
78  
79  import com.google.protobuf.ServiceException;
80  
81  
82  /**
83   * Test {@link AssignmentManager}
84   */
85  @Category(MediumTests.class)
86  public class TestAssignmentManager {
87    private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
88    private static final ServerName SERVERNAME_A =
89      new ServerName("example.org", 1234, 5678);
90    private static final ServerName SERVERNAME_B =
91      new ServerName("example.org", 0, 5678);
92    private static final HRegionInfo REGIONINFO =
93      new HRegionInfo(Bytes.toBytes("t"),
94        HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
95    private static final HRegionInfo REGIONINFO_2 = new HRegionInfo(Bytes.toBytes("t"),
96        Bytes.toBytes("a"),Bytes.toBytes( "b"));
97    private static int assignmentCount;
98    private static boolean enabling = false;  
99  
100   // Mocked objects or; get redone for each test.
101   private Server server;
102   private ServerManager serverManager;
103   private ZooKeeperWatcher watcher;
104   private LoadBalancer balancer;
105 
106   @BeforeClass
107   public static void beforeClass() throws Exception {
108     HTU.startMiniZKCluster();
109   }
110 
111   @AfterClass
112   public static void afterClass() throws IOException {
113     HTU.shutdownMiniZKCluster();
114   }
115 
116   @Before
117   public void before() throws ZooKeeperConnectionException, IOException {
118     // TODO: Make generic versions of what we do below and put up in a mocking
119     // utility class or move up into HBaseTestingUtility.
120 
121     // Mock a Server.  Have it return a legit Configuration and ZooKeeperWatcher.
122     // If abort is called, be sure to fail the test (don't just swallow it
123     // silently as is mockito default).
124     this.server = Mockito.mock(Server.class);
125     Mockito.when(server.getConfiguration()).thenReturn(HTU.getConfiguration());
126     this.watcher =
127       new ZooKeeperWatcher(HTU.getConfiguration(), "mockedServer", this.server, true);
128     Mockito.when(server.getZooKeeper()).thenReturn(this.watcher);
129     Mockito.doThrow(new RuntimeException("Aborted")).
130       when(server).abort(Mockito.anyString(), (Throwable)Mockito.anyObject());
131 
132     // Mock a ServerManager.  Say server SERVERNAME_{A,B} are online.  Also
133     // make it so if close or open, we return 'success'.
134     this.serverManager = Mockito.mock(ServerManager.class);
135     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
136     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true);
137     final Map<ServerName, HServerLoad> onlineServers = new HashMap<ServerName, HServerLoad>();
138     onlineServers.put(SERVERNAME_B, new HServerLoad());
139     onlineServers.put(SERVERNAME_A, new HServerLoad());
140     Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
141         new ArrayList<ServerName>(onlineServers.keySet()));
142     Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);
143     Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)).
144       thenReturn(true);
145     Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)).
146       thenReturn(true);
147     // Ditto on open.
148     Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_A, REGIONINFO, -1)).
149       thenReturn(RegionOpeningState.OPENED);
150     Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_B, REGIONINFO, -1)).
151     thenReturn(RegionOpeningState.OPENED);
152   }
153 
154   @After
155     public void after() throws KeeperException {
156     if (this.watcher != null) {
157       // Clean up all znodes
158       ZKAssign.deleteAllNodes(this.watcher);
159       this.watcher.close();
160     }
161   }
162 
163   /**
164    * Test a balance going on at same time as a master failover
165    *
166    * @throws IOException
167    * @throws KeeperException
168    * @throws InterruptedException
169    */
170   @Test(timeout = 60000)
171   public void testBalanceOnMasterFailoverScenarioWithOpenedNode()
172       throws IOException, KeeperException, InterruptedException {
173     AssignmentManagerWithExtrasForTesting am =
174       setUpMockedAssignmentManager(this.server, this.serverManager);
175     try {
176       createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
177       startFakeFailedOverMasterAssignmentManager(am, this.watcher);
178       while (!am.processRITInvoked) Thread.sleep(1);
179       // Now fake the region closing successfully over on the regionserver; the
180       // regionserver will have set the region in CLOSED state. This will
181       // trigger callback into AM. The below zk close call is from the RS close
182       // region handler duplicated here because its down deep in a private
183       // method hard to expose.
184       int versionid =
185         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
186       assertNotSame(versionid, -1);
187       Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
188 
189       // Get the OFFLINE version id.  May have to wait some for it to happen.
190       // OPENING below
191       while (true) {
192         int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
193         if (vid != versionid) {
194           versionid = vid;
195           break;
196         }
197       }
198       assertNotSame(-1, versionid);
199       // This uglyness below is what the openregionhandler on RS side does.
200       versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
201         SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
202         EventType.RS_ZK_REGION_OPENING, versionid);
203       assertNotSame(-1, versionid);
204       // Move znode from OPENING to OPENED as RS does on successful open.
205       versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
206         SERVERNAME_B, versionid);
207       assertNotSame(-1, versionid);
208       am.gate.set(false);
209       // Block here until our znode is cleared or until this test times out.
210       ZKAssign.blockUntilNoRIT(watcher);
211     } finally {
212       am.getExecutorService().shutdown();
213       am.shutdown();
214     }
215   }
216 
217   @Test(timeout = 60000)
218   public void testBalanceOnMasterFailoverScenarioWithClosedNode()
219       throws IOException, KeeperException, InterruptedException {
220     AssignmentManagerWithExtrasForTesting am =
221       setUpMockedAssignmentManager(this.server, this.serverManager);
222     try {
223       createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
224       startFakeFailedOverMasterAssignmentManager(am, this.watcher);
225       while (!am.processRITInvoked) Thread.sleep(1);
226       // Now fake the region closing successfully over on the regionserver; the
227       // regionserver will have set the region in CLOSED state. This will
228       // trigger callback into AM. The below zk close call is from the RS close
229       // region handler duplicated here because its down deep in a private
230       // method hard to expose.
231       int versionid =
232         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
233       assertNotSame(versionid, -1);
234       am.gate.set(false);
235       Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
236 
237       // Get current versionid else will fail on transition from OFFLINE to
238       // OPENING below
239       while (true) {
240         int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
241         if (vid != versionid) {
242           versionid = vid;
243           break;
244         }
245       }
246       assertNotSame(-1, versionid);
247       // This uglyness below is what the openregionhandler on RS side does.
248       versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
249           SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
250           EventType.RS_ZK_REGION_OPENING, versionid);
251       assertNotSame(-1, versionid);
252       // Move znode from OPENING to OPENED as RS does on successful open.
253       versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
254           SERVERNAME_B, versionid);
255       assertNotSame(-1, versionid);
256 
257       // Block here until our znode is cleared or until this test timesout.
258       ZKAssign.blockUntilNoRIT(watcher);
259     } finally {
260       am.getExecutorService().shutdown();
261       am.shutdown();
262     }
263   }
264 
265   @Test(timeout = 60000)
266   public void testBalanceOnMasterFailoverScenarioWithOfflineNode()
267       throws IOException, KeeperException, InterruptedException {
268     AssignmentManagerWithExtrasForTesting am =
269       setUpMockedAssignmentManager(this.server, this.serverManager);
270     try {
271       createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
272       startFakeFailedOverMasterAssignmentManager(am, this.watcher);
273       while (!am.processRITInvoked) Thread.sleep(1);
274       // Now fake the region closing successfully over on the regionserver; the
275       // regionserver will have set the region in CLOSED state. This will
276       // trigger callback into AM. The below zk close call is from the RS close
277       // region handler duplicated here because its down deep in a private
278       // method hard to expose.
279       int versionid =
280         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
281       assertNotSame(versionid, -1);
282       Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
283 
284       am.gate.set(false);
285       // Get current versionid else will fail on transition from OFFLINE to
286       // OPENING below
287       while (true) {
288         int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
289         if (vid != versionid) {
290           versionid = vid;
291           break;
292         }
293       }
294       assertNotSame(-1, versionid);
295       // This uglyness below is what the openregionhandler on RS side does.
296       versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
297           SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
298           EventType.RS_ZK_REGION_OPENING, versionid);
299       assertNotSame(-1, versionid);
300       // Move znode from OPENING to OPENED as RS does on successful open.
301       versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
302           SERVERNAME_B, versionid);
303       assertNotSame(-1, versionid);
304       // Block here until our znode is cleared or until this test timesout.
305       ZKAssign.blockUntilNoRIT(watcher);
306     } finally {
307       am.getExecutorService().shutdown();
308       am.shutdown();
309     }
310   }
311 
312   private void createRegionPlanAndBalance(final AssignmentManager am,
313       final ServerName from, final ServerName to, final HRegionInfo hri) {
314     // Call the balance function but fake the region being online first at
315     // servername from.
316     am.regionOnline(hri, from);
317     // Balance region from 'from' to 'to'. It calls unassign setting CLOSING state
318     // up in zk.  Create a plan and balance
319     am.balance(new RegionPlan(hri, from, to));
320   }
321 
322 
323   /**
324    * Tests AssignmentManager balance function.  Runs a balance moving a region
325    * from one server to another mocking regionserver responding over zk.
326    * @throws IOException
327    * @throws KeeperException
328    * @throws InterruptedException
329    */
330   @Test(timeout = 60000)
331   public void testBalance()
332   throws IOException, KeeperException, InterruptedException {
333     // Create and startup an executor.  This is used by AssignmentManager
334     // handling zk callbacks.
335     ExecutorService executor = startupMasterExecutor("testBalanceExecutor");
336 
337     // We need a mocked catalog tracker.
338     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
339     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
340         .getConfiguration());
341     // Create an AM.
342     AssignmentManager am = new AssignmentManager(this.server,
343         this.serverManager, ct, balancer, executor);
344     try {
345       // Make sure our new AM gets callbacks; once registered, can't unregister.
346       // Thats ok because we make a new zk watcher for each test.
347       this.watcher.registerListenerFirst(am);
348       // Call the balance function but fake the region being online first at
349       // SERVERNAME_A.  Create a balance plan.
350       am.regionOnline(REGIONINFO, SERVERNAME_A);
351       // Balance region from A to B.
352       RegionPlan plan = new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B);
353       am.balance(plan);
354 
355       // Now fake the region closing successfully over on the regionserver; the
356       // regionserver will have set the region in CLOSED state.  This will
357       // trigger callback into AM. The below zk close call is from the RS close
358       // region handler duplicated here because its down deep in a private
359       // method hard to expose.
360       int versionid =
361         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
362       assertNotSame(versionid, -1);
363       // AM is going to notice above CLOSED and queue up a new assign.  The
364       // assign will go to open the region in the new location set by the
365       // balancer.  The zk node will be OFFLINE waiting for regionserver to
366       // transition it through OPENING, OPENED.  Wait till we see the RIT
367       // before we proceed.
368       Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
369       // Get current versionid else will fail on transition from OFFLINE to OPENING below
370       while (true) {
371         int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
372         if (vid != versionid) {
373           versionid = vid;
374           break;
375         }
376       }
377       assertNotSame(-1, versionid);
378       // This uglyness below is what the openregionhandler on RS side does.
379       versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
380         SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
381         EventType.RS_ZK_REGION_OPENING, versionid);
382       assertNotSame(-1, versionid);
383       // Move znode from OPENING to OPENED as RS does on successful open.
384       versionid =
385         ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, SERVERNAME_B, versionid);
386       assertNotSame(-1, versionid);
387       // Wait on the handler removing the OPENED znode.
388       while(am.isRegionInTransition(REGIONINFO) != null) Threads.sleep(1);
389     } finally {
390       executor.shutdown();
391       am.shutdown();
392       // Clean up all znodes
393       ZKAssign.deleteAllNodes(this.watcher);
394     }
395   }
396 
397   /**
398    * Run a simple server shutdown handler.
399    * @throws KeeperException
400    * @throws IOException
401    */
402   @Test
403   public void testShutdownHandler() throws KeeperException, IOException {
404     // Create and startup an executor.  This is used by AssignmentManager
405     // handling zk callbacks.
406     ExecutorService executor = startupMasterExecutor("testShutdownHandler");
407 
408     // We need a mocked catalog tracker.
409     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
410     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
411         .getConfiguration());
412     // Create an AM.
413     AssignmentManager am =
414       new AssignmentManager(this.server, this.serverManager, ct, balancer, executor);
415     try {
416       processServerShutdownHandler(ct, am, false, null);
417     } finally {
418       executor.shutdown();
419       am.shutdown();
420       // Clean up all znodes
421       ZKAssign.deleteAllNodes(this.watcher);
422     }
423   }
424 
425   /**
426    * To test closed region handler to remove rit and delete corresponding znode if region in pending
427    * close or closing while processing shutdown of a region server.(HBASE-5927).
428    * @throws KeeperException
429    * @throws IOException
430    */
431   @Test
432   public void testSSHWhenDisableTableInProgress()
433       throws KeeperException, IOException {
434     testCaseWithPartiallyDisabledState(TableState.DISABLING, false);
435     testCaseWithPartiallyDisabledState(TableState.DISABLED, false);
436   }
437 
438   @Test
439   public void testSSHWhenDisablingTableRegionsInOpeningState()
440       throws KeeperException, IOException {
441     testCaseWithPartiallyDisabledState(TableState.DISABLING, true);
442     testCaseWithPartiallyDisabledState(TableState.DISABLED, true);
443   }
444 
445   
446   /**
447    * To test if the split region is removed from RIT if the region was in SPLITTING state
448    * but the RS has actually completed the splitting in META but went down. See HBASE-6070
449    * and also HBASE-5806
450    * @throws KeeperException
451    * @throws IOException
452    */
453   @Test
454   public void testSSHWhenSplitRegionInProgress()
455       throws KeeperException, IOException, Exception {
456     // true indicates the region is split but still in RIT
457     testCaseWithSplitRegionPartial(true);
458     // false indicate the region is not split
459     testCaseWithSplitRegionPartial(false);
460 
461   }
462 
463   private void testCaseWithSplitRegionPartial(boolean regionSplitDone) throws KeeperException, IOException,
464       NodeExistsException, InterruptedException {
465     // Create and startup an executor. This is used by AssignmentManager
466     // handling zk callbacks.
467     ExecutorService executor = startupMasterExecutor("testSSHWhenSplitRegionInProgress");
468 
469     // We need a mocked catalog tracker.
470     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
471     // Create an AM.
472     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server, this.serverManager);
473     // adding region to regions and servers maps.
474     am.regionOnline(REGIONINFO, SERVERNAME_A);
475     // adding region in pending close.
476     am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
477         State.SPLITTING, System.currentTimeMillis(), SERVERNAME_A));
478     am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
479 
480     RegionTransitionData data = new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING,
481         REGIONINFO.getRegionName(), SERVERNAME_A);
482     String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
483     // create znode in M_ZK_REGION_CLOSING state.
484     ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
485 
486     try {
487       processServerShutdownHandler(ct, am, regionSplitDone, null);
488       // check znode deleted or not.
489       // In both cases the znode should be deleted.
490 
491       if(regionSplitDone){
492         assertTrue("Region state of region in SPLITTING should be removed from rit.",
493             am.regionsInTransition.isEmpty());
494       }
495       else{
496         while (!am.assignInvoked) {
497           Thread.sleep(1);
498         }
499         assertTrue("Assign should be invoked.", am.assignInvoked);
500       }
501     } finally {
502       REGIONINFO.setOffline(false);
503       REGIONINFO.setSplit(false);
504       executor.shutdown();
505       am.shutdown();
506       // Clean up all znodes
507       ZKAssign.deleteAllNodes(this.watcher);
508     }
509   }
510 
511   private void testCaseWithPartiallyDisabledState(TableState state, boolean opening)
512       throws KeeperException, IOException, NodeExistsException {
513     // Create and startup an executor. This is used by AssignmentManager
514     // handling zk callbacks.
515     ExecutorService executor = startupMasterExecutor("testSSHWhenDisableTableInProgress");
516 
517     // We need a mocked catalog tracker.
518     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
519     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
520     // Create an AM.
521     AssignmentManager am = new AssignmentManager(this.server, this.serverManager, ct, balancer,
522         executor);
523     if (opening) {
524       am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
525           State.OPENING, System.currentTimeMillis(), SERVERNAME_A));
526     } else {
527       // adding region to regions and servers maps.
528       am.regionOnline(REGIONINFO, SERVERNAME_A);
529       // adding region in pending close.
530       am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
531           State.PENDING_CLOSE, System.currentTimeMillis(), SERVERNAME_A));
532     }
533     if (state == TableState.DISABLING) {
534       am.getZKTable().setDisablingTable(REGIONINFO.getTableNameAsString());
535     } else {
536       am.getZKTable().setDisabledTable(REGIONINFO.getTableNameAsString());
537     }
538     RegionTransitionData data = null;
539     if (opening) {
540       data =
541           new RegionTransitionData(EventType.RS_ZK_REGION_OPENING, REGIONINFO.getRegionName(),
542               SERVERNAME_A);
543 
544     } else {
545       data =
546           new RegionTransitionData(EventType.M_ZK_REGION_CLOSING, REGIONINFO.getRegionName(),
547               SERVERNAME_A);
548     }
549     String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
550     // create znode in M_ZK_REGION_CLOSING state.
551     ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
552 
553     try {
554       processServerShutdownHandler(ct, am, false, null);
555       // check znode deleted or not.
556       // In both cases the znode should be deleted.
557       assertTrue("The znode should be deleted.",ZKUtil.checkExists(this.watcher, node) == -1);
558       assertTrue("Region state of region in pending close should be removed from rit.",
559         am.regionsInTransition.isEmpty());
560     } finally {
561       executor.shutdown();
562       am.shutdown();
563       // Clean up all znodes
564       ZKAssign.deleteAllNodes(this.watcher);
565     }
566   }
567 
568   private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager am,
569     boolean splitRegion, ServerName sn)
570       throws IOException {
571     // Make sure our new AM gets callbacks; once registered, can't unregister.
572     // Thats ok because we make a new zk watcher for each test.
573     this.watcher.registerListenerFirst(am);
574     // Need to set up a fake scan of meta for the servershutdown handler
575     // Make an RS Interface implementation.  Make it so a scanner can go against it.
576     HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
577     // Get a meta row result that has region up on SERVERNAME_A
578 
579     Result r = null;
580     if (sn == null) {
581       if (splitRegion) {
582         r = getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A);
583       } else {
584         r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
585       }
586     } else {
587       if (sn.equals(SERVERNAME_A)) {
588         r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
589       } else if (sn.equals(SERVERNAME_B)) {
590         r = new Result(new KeyValue[0]);
591       }
592     }
593 
594     Mockito.when(implementation.openScanner((byte [])Mockito.any(), (Scan)Mockito.any())).
595       thenReturn(System.currentTimeMillis());
596     // Return a good result first and then return null to indicate end of scan
597     Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt())).
598       thenReturn(new Result [] {r}, (Result [])null);
599 
600     // Get a connection w/ mocked up common methods.
601     HConnection connection =
602       HConnectionTestingUtility.getMockedConnectionAndDecorate(HTU.getConfiguration(),
603         implementation, SERVERNAME_B, REGIONINFO);
604 
605     // Make it so we can get a catalogtracker from servermanager.. .needed
606     // down in guts of server shutdown handler.
607     Mockito.when(ct.getConnection()).thenReturn(connection);
608     Mockito.when(this.server.getCatalogTracker()).thenReturn(ct);
609 
610     // Now make a server shutdown handler instance and invoke process.
611     // Have it that SERVERNAME_A died.
612     DeadServer deadServers = new DeadServer();
613     deadServers.add(SERVERNAME_A);
614     // I need a services instance that will return the AM
615     MasterServices services = Mockito.mock(MasterServices.class);
616     Mockito.when(services.getAssignmentManager()).thenReturn(am);
617     Mockito.when(services.getZooKeeper()).thenReturn(this.watcher);
618     ServerShutdownHandler handler = null;
619     if (sn != null) {
620       handler = new ServerShutdownHandler(this.server, services, deadServers, sn, false);
621     } else {
622       handler = new ServerShutdownHandler(this.server, services, deadServers, SERVERNAME_A, false);
623     }
624     handler.process();
625     // The region in r will have been assigned.  It'll be up in zk as unassigned.
626   }
627 
628   /**
629    * @param sn ServerName to use making startcode and server in meta
630    * @param hri Region to serialize into HRegionInfo
631    * @return A mocked up Result that fakes a Get on a row in the
632    * <code>.META.</code> table.
633    * @throws IOException
634    */
635   private Result getMetaTableRowResult(final HRegionInfo hri,
636       final ServerName sn)
637   throws IOException {
638     // TODO: Move to a utilities class.  More than one test case can make use
639     // of this facility.
640     List<KeyValue> kvs = new ArrayList<KeyValue>();
641     kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
642       HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
643       Writables.getBytes(hri)));
644     kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
645       HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
646       Bytes.toBytes(sn.getHostAndPort())));
647     kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
648       HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
649       Bytes.toBytes(sn.getStartcode())));
650     return new Result(kvs);
651   }
652 
653   /**
654    * @param sn ServerName to use making startcode and server in meta
655    * @param hri Region to serialize into HRegionInfo
656    * @return A mocked up Result that fakes a Get on a row in the
657    * <code>.META.</code> table.
658    * @throws IOException
659    */
660   private Result getMetaTableRowResultAsSplitRegion(final HRegionInfo hri, final ServerName sn)
661       throws IOException {
662     hri.setOffline(true);
663     hri.setSplit(true);
664     return getMetaTableRowResult(hri, sn);
665   }
666 
667   /**
668    * Create and startup executor pools. Start same set as master does (just
669    * run a few less).
670    * @param name Name to give our executor
671    * @return Created executor (be sure to call shutdown when done).
672    */
673   private ExecutorService startupMasterExecutor(final String name) {
674     // TODO: Move up into HBaseTestingUtility?  Generally useful.
675     ExecutorService executor = new ExecutorService(name);
676     executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
677     executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
678     executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
679     executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
680     return executor;
681   }
682 
683   @Test
684   public void testUnassignWithSplitAtSameTime() throws KeeperException, IOException {
685     // Region to use in test.
686     final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
687     // First amend the servermanager mock so that when we do send close of the
688     // first meta region on SERVERNAME_A, it will return true rather than
689     // default null.
690     Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, hri, -1)).thenReturn(true);
691     // Need a mocked catalog tracker.
692     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
693     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
694         .getConfiguration());
695     // Create an AM.
696     AssignmentManager am =
697       new AssignmentManager(this.server, this.serverManager, ct, balancer, null);
698     try {
699       // First make sure my mock up basically works.  Unassign a region.
700       unassign(am, SERVERNAME_A, hri);
701       // This delete will fail if the previous unassign did wrong thing.
702       ZKAssign.deleteClosingNode(this.watcher, hri);
703       // Now put a SPLITTING region in the way.  I don't have to assert it
704       // go put in place.  This method puts it in place then asserts it still
705       // owns it by moving state from SPLITTING to SPLITTING.
706       int version = createNodeSplitting(this.watcher, hri, SERVERNAME_A);
707       // Now, retry the unassign with the SPLTTING in place.  It should just
708       // complete without fail; a sort of 'silent' recognition that the
709       // region to unassign has been split and no longer exists: TOOD: what if
710       // the split fails and the parent region comes back to life?
711       unassign(am, SERVERNAME_A, hri);
712       // This transition should fail if the znode has been messed with.
713       ZKAssign.transitionNode(this.watcher, hri, SERVERNAME_A,
714         EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
715       assertTrue(am.isRegionInTransition(hri) == null);
716     } finally {
717       am.shutdown();
718     }
719   }
720 
721   /**
722    * Tests the processDeadServersAndRegionsInTransition should not fail with NPE
723    * when it failed to get the children. Let's abort the system in this
724    * situation
725    * @throws ServiceException
726    */
727   @Test(timeout = 60000)
728   public void testProcessDeadServersAndRegionsInTransitionShouldNotFailWithNPE()
729       throws IOException, KeeperException, InterruptedException, ServiceException {
730     final RecoverableZooKeeper recoverableZk = Mockito
731         .mock(RecoverableZooKeeper.class);
732     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
733         this.server, this.serverManager);
734     Watcher zkw = new ZooKeeperWatcher(HBaseConfiguration.create(), "unittest",
735         null) {
736       public RecoverableZooKeeper getRecoverableZooKeeper() {
737         return recoverableZk;
738       }
739     };
740     ((ZooKeeperWatcher) zkw).registerListener(am);
741     Mockito.doThrow(new InterruptedException()).when(recoverableZk)
742         .getChildren("/hbase/unassigned", zkw);
743     am.setWatcher((ZooKeeperWatcher) zkw);
744     try {
745       am.processDeadServersAndRegionsInTransition();
746       fail("Expected to abort");
747     } catch (NullPointerException e) {
748       fail("Should not throw NPE");
749     } catch (RuntimeException e) {
750       assertEquals("Aborted", e.getLocalizedMessage());
751     }
752   }
753 
754   /**
755    * Creates a new ephemeral node in the SPLITTING state for the specified region.
756    * Create it ephemeral in case regionserver dies mid-split.
757    *
758    * <p>Does not transition nodes from other states.  If a node already exists
759    * for this region, a {@link NodeExistsException} will be thrown.
760    *
761    * @param zkw zk reference
762    * @param region region to be created as offline
763    * @param serverName server event originates from
764    * @return Version of znode created.
765    * @throws KeeperException
766    * @throws IOException
767    */
768   // Copied from SplitTransaction rather than open the method over there in
769   // the regionserver package.
770   private static int createNodeSplitting(final ZooKeeperWatcher zkw,
771       final HRegionInfo region, final ServerName serverName)
772   throws KeeperException, IOException {
773     RegionTransitionData data =
774       new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING,
775         region.getRegionName(), serverName);
776 
777     String node = ZKAssign.getNodeName(zkw, region.getEncodedName());
778     if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, data.getBytes())) {
779       throw new IOException("Failed create of ephemeral " + node);
780     }
781     // Transition node from SPLITTING to SPLITTING and pick up version so we
782     // can be sure this znode is ours; version is needed deleting.
783     return transitionNodeSplitting(zkw, region, serverName, -1);
784   }
785 
786   // Copied from SplitTransaction rather than open the method over there in
787   // the regionserver package.
788   private static int transitionNodeSplitting(final ZooKeeperWatcher zkw,
789       final HRegionInfo parent,
790       final ServerName serverName, final int version)
791   throws KeeperException, IOException {
792     return ZKAssign.transitionNode(zkw, parent, serverName,
793       EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
794   }
795 
796   private void unassign(final AssignmentManager am, final ServerName sn,
797       final HRegionInfo hri) {
798     // Before I can unassign a region, I need to set it online.
799     am.regionOnline(hri, sn);
800     // Unassign region.
801     am.unassign(hri);
802   }
803 
804   /**
805    * Create an {@link AssignmentManagerWithExtrasForTesting} that has mocked
806    * {@link CatalogTracker} etc.
807    * @param server
808    * @param manager
809    * @return An AssignmentManagerWithExtras with mock connections, etc.
810    * @throws IOException
811    * @throws KeeperException
812    */
813   private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final Server server,
814       final ServerManager manager)
815   throws IOException, KeeperException {
816     // We need a mocked catalog tracker. Its used by our AM instance.
817     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
818     // Make an RS Interface implementation. Make it so a scanner can go against
819     // it and a get to return the single region, REGIONINFO, this test is
820     // messing with. Needed when "new master" joins cluster. AM will try and
821     // rebuild its list of user regions and it will also get the HRI that goes
822     // with an encoded name by doing a Get on .META.
823     HRegionInterface ri = Mockito.mock(HRegionInterface.class);
824     // Get a meta row result that has region up on SERVERNAME_A for REGIONINFO
825     Result[] result = null;
826     if (enabling) {
827       result = new Result[2];
828       result[0] = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
829       result[1] = getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A);
830     }
831     Result r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
832     Mockito.when(ri .openScanner((byte[]) Mockito.any(), (Scan) Mockito.any())).
833       thenReturn(System.currentTimeMillis());
834    if (enabling) {
835       Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(result, result, result,
836           (Result[]) null);
837       // If a get, return the above result too for REGIONINFO_2
838       Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(
839           getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A));
840     } else {
841       // Return good result 'r' first and then return null to indicate end of scan
842       Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(new Result[] { r });
843       // If a get, return the above result too for REGIONINFO
844       Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(r);
845     }
846     // Get a connection w/ mocked up common methods.
847     HConnection connection = HConnectionTestingUtility.
848       getMockedConnectionAndDecorate(HTU.getConfiguration(), ri, SERVERNAME_B,
849         REGIONINFO);
850     // Make it so we can get the connection from our mocked catalogtracker
851     Mockito.when(ct.getConnection()).thenReturn(connection);
852     // Create and startup an executor. Used by AM handling zk callbacks.
853     ExecutorService executor = startupMasterExecutor("mockedAMExecutor");
854     this.balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
855     AssignmentManagerWithExtrasForTesting am = new AssignmentManagerWithExtrasForTesting(
856         server, manager, ct, balancer, executor);
857     return am;
858   }
859 
860   /**
861    * TestCase verifies that the regionPlan is updated whenever a region fails to open
862    * and the master tries to process RS_ZK_FAILED_OPEN state.(HBASE-5546).
863    */
864   @Test
865   public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, KeeperException,
866       ServiceException, InterruptedException {
867     this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
868         MockedLoadBalancer.class, LoadBalancer.class);
869     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server,
870         this.serverManager);
871     try {
872       // Boolean variable used for waiting until randomAssignment is called and new
873       // plan is generated.
874       AtomicBoolean gate = new AtomicBoolean(false);
875       if (balancer instanceof MockedLoadBalancer) {
876         ((MockedLoadBalancer) balancer).setGateVariable(gate);
877       }
878       ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
879       int v = ZKAssign.getVersion(this.watcher, REGIONINFO);
880       ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
881           EventType.RS_ZK_REGION_FAILED_OPEN, v);
882       String path = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
883       RegionState state = new RegionState(REGIONINFO, State.OPENING, System.currentTimeMillis(),
884           SERVERNAME_A);
885       am.regionsInTransition.put(REGIONINFO.getEncodedName(), state);
886       // a dummy plan inserted into the regionPlans. This plan is cleared and new one is formed
887       am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));
888       RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
889       List<ServerName> serverList = new ArrayList<ServerName>(2);
890       serverList.add(SERVERNAME_B);
891       Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(serverList);
892       am.nodeDataChanged(path);
893       // here we are waiting until the random assignment in the load balancer is called.
894       while (!gate.get()) {
895         Thread.sleep(10);
896       }
897       // new region plan may take some time to get updated after random assignment is called and
898       // gate is set to true.
899       RegionPlan newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
900       while (newRegionPlan == null) {
901         Thread.sleep(10);
902         newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
903       }
904       // the new region plan created may contain the same RS as destination but it should
905       // be new plan.
906       assertNotSame("Same region plan should not come", regionPlan, newRegionPlan);
907       assertTrue("Destnation servers should be different.", !(regionPlan.getDestination().equals(
908         newRegionPlan.getDestination())));
909       Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
910     } finally {
911       this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
912         DefaultLoadBalancer.class, LoadBalancer.class);
913       am.shutdown();
914     }
915   }
916 
917   /**
918    * Test verifies whether assignment is skipped for regions of tables in DISABLING state during
919    * clean cluster startup. See HBASE-6281.
920    *
921    * @throws KeeperException
922    * @throws IOException
923    * @throws Exception
924    */
925   @Test
926   public void testDisablingTableRegionsAssignmentDuringCleanClusterStartup()
927       throws KeeperException, IOException, Exception {
928     this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
929         MockedLoadBalancer.class, LoadBalancer.class);
930     Mockito.when(this.serverManager.getOnlineServers()).thenReturn(
931         new HashMap<ServerName, HServerLoad>(0));
932     List<ServerName> destServers = new ArrayList<ServerName>(1);
933     destServers.add(SERVERNAME_A);
934     Mockito.when(this.serverManager.getDrainingServersList()).thenReturn(destServers);
935     // To avoid cast exception in DisableTableHandler process.
936     //Server server = new HMaster(HTU.getConfiguration());
937     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
938         this.serverManager);
939     AtomicBoolean gate = new AtomicBoolean(false);
940     if (balancer instanceof MockedLoadBalancer) {
941       ((MockedLoadBalancer) balancer).setGateVariable(gate);
942     }
943     try{
944       // set table in disabling state.
945       am.getZKTable().setDisablingTable(REGIONINFO.getTableNameAsString());
946       am.joinCluster();
947       // should not call retainAssignment if we get empty regions in assignAllUserRegions.
948       assertFalse(
949           "Assign should not be invoked for disabling table regions during clean cluster startup.",
950           gate.get());
951       // need to change table state from disabling to disabled.
952       assertTrue("Table should be disabled.",
953           am.getZKTable().isDisabledTable(REGIONINFO.getTableNameAsString()));
954     } finally {
955       this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
956         DefaultLoadBalancer.class, LoadBalancer.class);
957       am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
958       am.shutdown();
959     }
960   }
961 
962   /**
963    * Test verifies whether all the enabling table regions assigned only once during master startup.
964    * 
965    * @throws KeeperException
966    * @throws IOException
967    * @throws Exception
968    */
969   @Test
970   public void testMasterRestartWhenTableInEnabling() throws KeeperException, IOException, Exception {
971     enabling = true;
972     this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
973         DefaultLoadBalancer.class, LoadBalancer.class);
974     Map<ServerName, HServerLoad> serverAndLoad = new HashMap<ServerName, HServerLoad>();
975     serverAndLoad.put(SERVERNAME_A, null);
976     Mockito.when(this.serverManager.getOnlineServers()).thenReturn(serverAndLoad);
977     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(false);
978     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
979     HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
980     Server server = new HMaster(HTU.getConfiguration());
981     Whitebox.setInternalState(server, "serverManager", this.serverManager);
982     assignmentCount = 0;
983     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
984         this.serverManager);
985     am.regionOnline(new HRegionInfo("t1".getBytes(), HConstants.EMPTY_START_ROW,
986         HConstants.EMPTY_END_ROW), SERVERNAME_A);
987     am.gate.set(false);
988     try {
989       // set table in enabling state.
990       am.getZKTable().setEnablingTable(REGIONINFO.getTableNameAsString());
991       ZKAssign.createNodeOffline(this.watcher, REGIONINFO_2, SERVERNAME_B);
992 
993       am.joinCluster();
994       while (!am.getZKTable().isEnabledTable(REGIONINFO.getTableNameAsString())) {
995         Thread.sleep(10);
996       }
997       assertEquals("Number of assignments should be equal.", 2, assignmentCount);
998       assertTrue("Table should be enabled.",
999           am.getZKTable().isEnabledTable(REGIONINFO.getTableNameAsString()));
1000     } finally {
1001       enabling = false;
1002       am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
1003       am.shutdown();
1004       ZKAssign.deleteAllNodes(this.watcher);
1005       assignmentCount = 0;
1006     }
1007   }
1008 
1009 
1010 
1011   /**
1012    * When region in transition if region server opening the region gone down then region assignment
1013    * taking long time(Waiting for timeout monitor to trigger assign). HBASE-5396(HBASE-6060) fixes this
1014    * scenario. This test case verifies whether SSH calling assign for the region in transition or not.
1015    *
1016    * @throws KeeperException
1017    * @throws IOException
1018    * @throws ServiceException
1019    */
1020   @Test
1021   public void testSSHWhenSourceRSandDestRSInRegionPlanGoneDown() throws KeeperException, IOException,
1022       ServiceException {
1023     testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(true);
1024     testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(false);
1025   }
1026 
1027   private void testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(boolean regionInOffline)
1028       throws IOException, KeeperException, ServiceException {
1029     // We need a mocked catalog tracker.
1030     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1031     // Create an AM.
1032     AssignmentManagerWithExtrasForTesting am =
1033         setUpMockedAssignmentManager(this.server, this.serverManager);
1034     // adding region in pending open.
1035     if (regionInOffline) {
1036       ServerName MASTER_SERVERNAME = new ServerName("example.org", 1111, 1111);
1037       am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
1038           State.OFFLINE, System.currentTimeMillis(), MASTER_SERVERNAME));
1039     } else {
1040       am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
1041           State.OPENING, System.currentTimeMillis(), SERVERNAME_B));
1042     }
1043     // adding region plan
1044     am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B));
1045     am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
1046 
1047     try {
1048       processServerShutdownHandler(ct, am, false, SERVERNAME_A);
1049       processServerShutdownHandler(ct, am, false, SERVERNAME_B);
1050       if(regionInOffline){
1051         assertFalse("Assign should not be invoked.", am.assignInvoked);
1052       } else {
1053         assertTrue("Assign should be invoked.", am.assignInvoked);
1054       }
1055     } finally {
1056       am.regionsInTransition.remove(REGIONINFO.getEncodedName());
1057       am.regionPlans.remove(REGIONINFO.getEncodedName());
1058     }
1059   }
1060 
1061   /**
1062    * Mocked load balancer class used in the testcase to make sure that the testcase waits until
1063    * random assignment is called and the gate variable is set to true.
1064    */
1065   public static class MockedLoadBalancer extends DefaultLoadBalancer {
1066     private AtomicBoolean gate;
1067 
1068     public void setGateVariable(AtomicBoolean gate) {
1069       this.gate = gate;
1070     }
1071 
1072     @Override
1073     public ServerName randomAssignment(List<ServerName> servers) {
1074       ServerName randomServerName = super.randomAssignment(servers);
1075       this.gate.set(true);
1076       return randomServerName;
1077     }
1078 
1079     @Override
1080     public Map<ServerName, List<HRegionInfo>> retainAssignment(
1081         Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
1082       this.gate.set(true);
1083       return super.retainAssignment(regions, servers);
1084     }
1085 
1086   }
1087 
1088   /**
1089    * An {@link AssignmentManager} with some extra facility used testing
1090    */
1091   class AssignmentManagerWithExtrasForTesting extends AssignmentManager {
1092     // Keep a reference so can give it out below in {@link #getExecutorService}
1093     private final ExecutorService es;
1094     // Ditto for ct
1095     private final CatalogTracker ct;
1096     boolean processRITInvoked = false;
1097     boolean assignInvoked = false;
1098     AtomicBoolean gate = new AtomicBoolean(true);
1099 
1100     public AssignmentManagerWithExtrasForTesting(final Server master,
1101         final ServerManager serverManager, final CatalogTracker catalogTracker,
1102         final LoadBalancer balancer, final ExecutorService service)
1103     throws KeeperException, IOException {
1104       super(master, serverManager, catalogTracker, balancer, service);
1105       this.es = service;
1106       this.ct = catalogTracker;
1107     }
1108 
1109     @Override
1110     boolean processRegionInTransition(String encodedRegionName,
1111         HRegionInfo regionInfo,
1112         Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers)
1113         throws KeeperException, IOException {
1114       this.processRITInvoked = true;
1115       return super.processRegionInTransition(encodedRegionName, regionInfo,
1116           deadServers);
1117     }
1118     @Override
1119     void processRegionsInTransition(final RegionTransitionData data,
1120         final HRegionInfo regionInfo,
1121         final Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers,
1122         final int expectedVersion) throws KeeperException {
1123       while (this.gate.get()) Threads.sleep(1);
1124       super.processRegionsInTransition(data, regionInfo, deadServers, expectedVersion);
1125     }
1126     
1127     @Override
1128     public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan,
1129         boolean hijack) {
1130       if (enabling) {
1131         assignmentCount++;
1132         this.regionOnline(region, SERVERNAME_A);
1133       } else {
1134         assignInvoked = true;
1135         super.assign(region, setOfflineInZK, forceNewPlan, hijack);
1136       }
1137     }
1138     
1139     @Override
1140     public ServerName getRegionServerOfRegion(HRegionInfo hri) {
1141       return SERVERNAME_A;
1142     }
1143     
1144     /** reset the watcher */
1145     void setWatcher(ZooKeeperWatcher watcher) {
1146       this.watcher = watcher;
1147     }
1148 
1149     /**
1150      * @return ExecutorService used by this instance.
1151      */
1152     ExecutorService getExecutorService() {
1153       return this.es;
1154     }
1155 
1156     /**
1157      * @return CatalogTracker used by this AM (Its a mock).
1158      */
1159     CatalogTracker getCatalogTracker() {
1160       return this.ct;
1161     }
1162   }
1163 
1164   /**
1165    * Call joinCluster on the passed AssignmentManager.  Do it in a thread
1166    * so it runs independent of what all else is going on.  Try to simulate
1167    * an AM running insided a failed over master by clearing all in-memory
1168    * AM state first.
1169   */
1170   private void startFakeFailedOverMasterAssignmentManager(final AssignmentManager am,
1171       final ZooKeeperWatcher watcher) {
1172     // Make sure our new AM gets callbacks; once registered, we can't unregister.
1173     // Thats ok because we make a new zk watcher for each test.
1174     watcher.registerListenerFirst(am);
1175     Thread t = new Thread("RunAmJoinCluster") {
1176       public void run() {
1177         // Call the joinCluster function as though we were doing a master
1178         // failover at this point. It will stall just before we go to add
1179         // the RIT region to our RIT Map in AM at processRegionsInTransition.
1180         // First clear any inmemory state from AM so it acts like a new master
1181         // coming on line.
1182         am.regionsInTransition.clear();
1183         am.regionPlans.clear();
1184         try {
1185           am.joinCluster();
1186         } catch (IOException e) {
1187           throw new RuntimeException(e);
1188         } catch (KeeperException e) {
1189           throw new RuntimeException(e);
1190         } catch (InterruptedException e) {
1191           throw new RuntimeException(e);
1192         }
1193       };
1194     };
1195     t.start();
1196     while (!t.isAlive()) Threads.sleep(1);
1197   }
1198 
1199   @org.junit.Rule
1200   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
1201     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
1202 }