1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertNotSame;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.util.*;
28  import java.util.concurrent.atomic.AtomicBoolean;
29  
30  import com.google.common.collect.Lists;
31  import org.apache.hadoop.hbase.HBaseConfiguration;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.HRegionInfo;
35  import org.apache.hadoop.hbase.HServerLoad;
36  import org.apache.hadoop.hbase.KeyValue;
37  import org.apache.hadoop.hbase.MediumTests;
38  import org.apache.hadoop.hbase.Server;
39  import org.apache.hadoop.hbase.ServerName;
40  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
41  import org.apache.hadoop.hbase.catalog.CatalogTracker;
42  import org.apache.hadoop.hbase.client.Get;
43  import org.apache.hadoop.hbase.client.HConnection;
44  import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
45  import org.apache.hadoop.hbase.client.Result;
46  import org.apache.hadoop.hbase.client.Scan;
47  import org.apache.hadoop.hbase.executor.EventHandler.EventType;
48  import org.apache.hadoop.hbase.executor.ExecutorService;
49  import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
50  import org.apache.hadoop.hbase.executor.RegionTransitionData;
51  import org.apache.hadoop.hbase.ipc.HRegionInterface;
52  import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
53  import org.apache.hadoop.hbase.master.AssignmentManager.RegionState.State;
54  import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
55  import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
56  import org.apache.hadoop.hbase.util.Bytes;
57  import org.apache.hadoop.hbase.util.Pair;
58  import org.apache.hadoop.hbase.util.Threads;
59  import org.apache.hadoop.hbase.util.Writables;
60  import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
61  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
62  import org.apache.hadoop.hbase.zookeeper.ZKTable.TableState;
63  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
64  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
65  import org.apache.zookeeper.KeeperException;
66  import org.apache.zookeeper.KeeperException.NodeExistsException;
67  import org.apache.zookeeper.Watcher;
68  import org.junit.After;
69  import org.junit.AfterClass;
70  import org.junit.Before;
71  import org.junit.BeforeClass;
72  import org.junit.FixMethodOrder;
73  import org.junit.Test;
74  import org.junit.experimental.categories.Category;
75  import org.junit.runners.MethodSorters;
76  import org.mockito.Mockito;
77  import org.mockito.internal.util.reflection.Whitebox;
78  
79  import com.google.protobuf.ServiceException;
80  
81  
82  /**
83   * Test {@link AssignmentManager}
84   */
85  @FixMethodOrder(MethodSorters.NAME_ASCENDING)
86  @Category(MediumTests.class)
87  public class TestAssignmentManager {
88    private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
89    private static final ServerName SERVERNAME_A =
90      new ServerName("example.org", 1234, 5678);
91    private static final ServerName SERVERNAME_B =
92      new ServerName("example.org", 0, 5678);
93    private static final ServerName SERVERNAME_C =
94        new ServerName("example.org", 123, 5678);
95    private static final HRegionInfo REGIONINFO =
96      new HRegionInfo(Bytes.toBytes("t"),
97        HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
98    private static final HRegionInfo REGIONINFO_2 = new HRegionInfo(Bytes.toBytes("t"),
99        Bytes.toBytes("a"),Bytes.toBytes( "b"));
100   private static int assignmentCount;
101   private static boolean enabling = false;  
102 
103   // Mocked objects or; get redone for each test.
104   private Server server;
105   private ServerManager serverManager;
106   private ZooKeeperWatcher watcher;
107   private LoadBalancer balancer;
108 
109   @BeforeClass
110   public static void beforeClass() throws Exception {
111     HTU.startMiniZKCluster();
112   }
113 
114   @AfterClass
115   public static void afterClass() throws IOException {
116     HTU.shutdownMiniZKCluster();
117   }
118 
119   @Before
120   public void before() throws ZooKeeperConnectionException, IOException {
121     // TODO: Make generic versions of what we do below and put up in a mocking
122     // utility class or move up into HBaseTestingUtility.
123 
124     // Mock a Server.  Have it return a legit Configuration and ZooKeeperWatcher.
125     // If abort is called, be sure to fail the test (don't just swallow it
126     // silently as is mockito default).
127     this.server = Mockito.mock(Server.class);
128     Mockito.when(server.getConfiguration()).thenReturn(HTU.getConfiguration());
129     this.watcher =
130       new ZooKeeperWatcher(HTU.getConfiguration(), "mockedServer", this.server, true);
131     Mockito.when(server.getZooKeeper()).thenReturn(this.watcher);
132     Mockito.doThrow(new RuntimeException("Aborted")).
133       when(server).abort(Mockito.anyString(), (Throwable)Mockito.anyObject());
134 
135     // Mock a ServerManager.  Say server SERVERNAME_{A,B} are online.  Also
136     // make it so if close or open, we return 'success'.
137     this.serverManager = mockManager(SERVERNAME_A, SERVERNAME_B);
138   }
139 
140   private ServerManager mockManager(ServerName... servers) throws IOException {
141     ServerManager serverManager = Mockito.mock(ServerManager.class);
142     final Map<ServerName, HServerLoad> onlineServers = new HashMap<ServerName, HServerLoad>();
143     for (ServerName server : servers) {
144       Mockito.when(serverManager.isServerOnline(server)).thenReturn(true);
145       onlineServers.put(server, new HServerLoad());
146       Mockito.when(serverManager.sendRegionClose(server, REGIONINFO, -1)).thenReturn(true);
147       Mockito.when(serverManager.sendRegionOpen(server, REGIONINFO, -1)).
148           thenReturn(RegionOpeningState.OPENED);
149     }
150     Mockito.when(serverManager.getOnlineServersList()).thenReturn(
151         new ArrayList<ServerName>(onlineServers.keySet()));
152     Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
153     return serverManager;
154   }
155 
156   @After
157     public void after() throws KeeperException {
158     if (this.watcher != null) {
159       // Clean up all znodes
160       ZKAssign.deleteAllNodes(this.watcher);
161       this.watcher.close();
162     }
163   }
164 
165   /**
166    * Test a balance going on at same time as a master failover
167    *
168    * @throws IOException
169    * @throws KeeperException
170    * @throws InterruptedException
171    */
172   @Test(timeout = 60000)
173   public void testBalanceOnMasterFailoverScenarioWithOpenedNode()
174       throws IOException, KeeperException, InterruptedException {
175     AssignmentManagerWithExtrasForTesting am =
176       setUpMockedAssignmentManager(this.server, this.serverManager);
177     try {
178       createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
179       startFakeFailedOverMasterAssignmentManager(am, this.watcher);
180       while (!am.processRITInvoked) Thread.sleep(1);
181       // Now fake the region closing successfully over on the regionserver; the
182       // regionserver will have set the region in CLOSED state. This will
183       // trigger callback into AM. The below zk close call is from the RS close
184       // region handler duplicated here because its down deep in a private
185       // method hard to expose.
186       int versionid =
187         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
188       assertNotSame(versionid, -1);
189       Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
190 
191       // Get the OFFLINE version id.  May have to wait some for it to happen.
192       // OPENING below
193       while (true) {
194         int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
195         if (vid != versionid) {
196           versionid = vid;
197           break;
198         }
199       }
200       assertNotSame(-1, versionid);
201       // This uglyness below is what the openregionhandler on RS side does.
202       versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
203         SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
204         EventType.RS_ZK_REGION_OPENING, versionid);
205       assertNotSame(-1, versionid);
206       // Move znode from OPENING to OPENED as RS does on successful open.
207       versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
208         SERVERNAME_B, versionid);
209       assertNotSame(-1, versionid);
210       am.gate.set(false);
211       // Block here until our znode is cleared or until this test times out.
212       ZKAssign.blockUntilNoRIT(watcher);
213     } finally {
214       am.getExecutorService().shutdown();
215       am.shutdown();
216     }
217   }
218 
219   @Test(timeout = 60000)
220   public void testBalanceOnMasterFailoverScenarioWithClosedNode()
221       throws IOException, KeeperException, InterruptedException {
222     AssignmentManagerWithExtrasForTesting am =
223       setUpMockedAssignmentManager(this.server, this.serverManager);
224     try {
225       createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
226       startFakeFailedOverMasterAssignmentManager(am, this.watcher);
227       while (!am.processRITInvoked) Thread.sleep(1);
228       // Now fake the region closing successfully over on the regionserver; the
229       // regionserver will have set the region in CLOSED state. This will
230       // trigger callback into AM. The below zk close call is from the RS close
231       // region handler duplicated here because its down deep in a private
232       // method hard to expose.
233       int versionid =
234         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
235       assertNotSame(versionid, -1);
236       am.gate.set(false);
237       Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
238 
239       // Get current versionid else will fail on transition from OFFLINE to
240       // OPENING below
241       while (true) {
242         int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
243         if (vid != versionid) {
244           versionid = vid;
245           break;
246         }
247       }
248       assertNotSame(-1, versionid);
249       // This uglyness below is what the openregionhandler on RS side does.
250       versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
251           SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
252           EventType.RS_ZK_REGION_OPENING, versionid);
253       assertNotSame(-1, versionid);
254       // Move znode from OPENING to OPENED as RS does on successful open.
255       versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
256           SERVERNAME_B, versionid);
257       assertNotSame(-1, versionid);
258 
259       // Block here until our znode is cleared or until this test timesout.
260       ZKAssign.blockUntilNoRIT(watcher);
261     } finally {
262       am.getExecutorService().shutdown();
263       am.shutdown();
264     }
265   }
266 
267   @Test(timeout = 60000)
268   public void testBalanceOnMasterFailoverScenarioWithOfflineNode()
269       throws IOException, KeeperException, InterruptedException {
270     AssignmentManagerWithExtrasForTesting am =
271       setUpMockedAssignmentManager(this.server, this.serverManager);
272     try {
273       createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
274       startFakeFailedOverMasterAssignmentManager(am, this.watcher);
275       while (!am.processRITInvoked) Thread.sleep(1);
276       // Now fake the region closing successfully over on the regionserver; the
277       // regionserver will have set the region in CLOSED state. This will
278       // trigger callback into AM. The below zk close call is from the RS close
279       // region handler duplicated here because its down deep in a private
280       // method hard to expose.
281       int versionid =
282         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
283       assertNotSame(versionid, -1);
284       Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
285 
286       am.gate.set(false);
287       // Get current versionid else will fail on transition from OFFLINE to
288       // OPENING below
289       while (true) {
290         int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
291         if (vid != versionid) {
292           versionid = vid;
293           break;
294         }
295       }
296       assertNotSame(-1, versionid);
297       // This uglyness below is what the openregionhandler on RS side does.
298       versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
299           SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
300           EventType.RS_ZK_REGION_OPENING, versionid);
301       assertNotSame(-1, versionid);
302       // Move znode from OPENING to OPENED as RS does on successful open.
303       versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
304           SERVERNAME_B, versionid);
305       assertNotSame(-1, versionid);
306       // Block here until our znode is cleared or until this test timesout.
307       ZKAssign.blockUntilNoRIT(watcher);
308     } finally {
309       am.getExecutorService().shutdown();
310       am.shutdown();
311     }
312   }
313 
314   private void createRegionPlanAndBalance(final AssignmentManager am,
315       final ServerName from, final ServerName to, final HRegionInfo hri) {
316     // Call the balance function but fake the region being online first at
317     // servername from.
318     am.regionOnline(hri, from);
319     // Balance region from 'from' to 'to'. It calls unassign setting CLOSING state
320     // up in zk.  Create a plan and balance
321     am.balance(new RegionPlan(hri, from, to));
322   }
323 
324 
325   /**
326    * Tests AssignmentManager balance function.  Runs a balance moving a region
327    * from one server to another mocking regionserver responding over zk.
328    * @throws IOException
329    * @throws KeeperException
330    * @throws InterruptedException
331    */
332   @Test(timeout = 60000)
333   public void testBalance()
334   throws IOException, KeeperException, InterruptedException {
335     // Create and startup an executor.  This is used by AssignmentManager
336     // handling zk callbacks.
337     ExecutorService executor = startupMasterExecutor("testBalanceExecutor");
338 
339     // We need a mocked catalog tracker.
340     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
341     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
342         .getConfiguration());
343     // Create an AM.
344     AssignmentManager am = new AssignmentManager(this.server,
345         this.serverManager, ct, balancer, executor);
346     try {
347       // Make sure our new AM gets callbacks; once registered, can't unregister.
348       // Thats ok because we make a new zk watcher for each test.
349       this.watcher.registerListenerFirst(am);
350       // Call the balance function but fake the region being online first at
351       // SERVERNAME_A.  Create a balance plan.
352       am.regionOnline(REGIONINFO, SERVERNAME_A);
353       // Balance region from A to B.
354       RegionPlan plan = new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B);
355       am.balance(plan);
356 
357       // Now fake the region closing successfully over on the regionserver; the
358       // regionserver will have set the region in CLOSED state.  This will
359       // trigger callback into AM. The below zk close call is from the RS close
360       // region handler duplicated here because its down deep in a private
361       // method hard to expose.
362       int versionid =
363         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
364       assertNotSame(versionid, -1);
365       // AM is going to notice above CLOSED and queue up a new assign.  The
366       // assign will go to open the region in the new location set by the
367       // balancer.  The zk node will be OFFLINE waiting for regionserver to
368       // transition it through OPENING, OPENED.  Wait till we see the RIT
369       // before we proceed.
370       Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
371       // Get current versionid else will fail on transition from OFFLINE to OPENING below
372       while (true) {
373         int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
374         if (vid != versionid) {
375           versionid = vid;
376           break;
377         }
378       }
379       assertNotSame(-1, versionid);
380       // This uglyness below is what the openregionhandler on RS side does.
381       versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
382         SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
383         EventType.RS_ZK_REGION_OPENING, versionid);
384       assertNotSame(-1, versionid);
385       // Move znode from OPENING to OPENED as RS does on successful open.
386       versionid =
387         ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, SERVERNAME_B, versionid);
388       assertNotSame(-1, versionid);
389       // Wait on the handler removing the OPENED znode.
390       while(am.isRegionInTransition(REGIONINFO) != null) Threads.sleep(1);
391     } finally {
392       executor.shutdown();
393       am.shutdown();
394       // Clean up all znodes
395       ZKAssign.deleteAllNodes(this.watcher);
396     }
397   }
398 
399   @Test
400   public void testGettingAssignmentsExcludesDrainingServers() throws Exception {
401     List<ServerName> availableServers =
402         Lists.newArrayList(SERVERNAME_A, SERVERNAME_B, SERVERNAME_C);
403     ServerManager serverManager = mockManager(availableServers.toArray(new ServerName[0]));
404 
405 
406     ExecutorService executor = startupMasterExecutor("testAssignmentsWithRSInDraining");
407     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
408 
409     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
410 
411     Mockito.when(serverManager.getDrainingServersList()).thenReturn(
412         Lists.newArrayList(SERVERNAME_C));
413     AssignmentManager am = new AssignmentManager(this.server, serverManager, ct, balancer, executor);
414 
415     for (ServerName availableServer : availableServers) {
416       HRegionInfo info = Mockito.mock(HRegionInfo.class);
417       Mockito.when(info.getEncodedName()).thenReturn(UUID.randomUUID().toString());
418       am.regionOnline(info, availableServer);
419     }
420 
421     Map<String, Map<ServerName, List<HRegionInfo>>> result = am.getAssignmentsByTable();
422     for (Map<ServerName, List<HRegionInfo>> map : result.values()) {
423       System.out.println(map.keySet());
424       assertFalse(map.containsKey(SERVERNAME_C));
425     }
426   }
427 
428   /**
429    * Run a simple server shutdown handler.
430    * @throws KeeperException
431    * @throws IOException
432    */
433   @Test
434   public void testShutdownHandler() throws KeeperException, IOException {
435     // Create and startup an executor.  This is used by AssignmentManager
436     // handling zk callbacks.
437     ExecutorService executor = startupMasterExecutor("testShutdownHandler");
438 
439     // We need a mocked catalog tracker.
440     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
441     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
442         .getConfiguration());
443     // Create an AM.
444     AssignmentManager am =
445       new AssignmentManager(this.server, this.serverManager, ct, balancer, executor);
446     try {
447       processServerShutdownHandler(ct, am, false, null);
448     } finally {
449       executor.shutdown();
450       am.shutdown();
451       // Clean up all znodes
452       ZKAssign.deleteAllNodes(this.watcher);
453     }
454   }
455 
456   /**
457    * To test closed region handler to remove rit and delete corresponding znode if region in pending
458    * close or closing while processing shutdown of a region server.(HBASE-5927).
459    * @throws KeeperException
460    * @throws IOException
461    */
462   @Test
463   public void testSSHWhenDisableTableInProgress()
464       throws KeeperException, IOException {
465     testCaseWithPartiallyDisabledState(TableState.DISABLING, false);
466     testCaseWithPartiallyDisabledState(TableState.DISABLED, false);
467   }
468 
469   @Test
470   public void testSSHWhenDisablingTableRegionsInOpeningState()
471       throws KeeperException, IOException {
472     testCaseWithPartiallyDisabledState(TableState.DISABLING, true);
473     testCaseWithPartiallyDisabledState(TableState.DISABLED, true);
474   }
475 
476   
477   /**
478    * To test if the split region is removed from RIT if the region was in SPLITTING state
479    * but the RS has actually completed the splitting in META but went down. See HBASE-6070
480    * and also HBASE-5806
481    * @throws KeeperException
482    * @throws IOException
483    */
484   @Test
485   public void testSSHWhenSplitRegionInProgress()
486       throws KeeperException, IOException, Exception {
487     // true indicates the region is split but still in RIT
488     testCaseWithSplitRegionPartial(true);
489     // false indicate the region is not split
490     testCaseWithSplitRegionPartial(false);
491 
492   }
493 
494   private void testCaseWithSplitRegionPartial(boolean regionSplitDone) throws KeeperException, IOException,
495       NodeExistsException, InterruptedException {
496     // Create and startup an executor. This is used by AssignmentManager
497     // handling zk callbacks.
498     ExecutorService executor = startupMasterExecutor("testSSHWhenSplitRegionInProgress");
499 
500     // We need a mocked catalog tracker.
501     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
502     // Create an AM.
503     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server, this.serverManager);
504     // adding region to regions and servers maps.
505     am.regionOnline(REGIONINFO, SERVERNAME_A);
506     // adding region in pending close.
507     am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
508         State.SPLITTING, System.currentTimeMillis(), SERVERNAME_A));
509     am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
510 
511     RegionTransitionData data = new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING,
512         REGIONINFO.getRegionName(), SERVERNAME_A);
513     String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
514     // create znode in M_ZK_REGION_CLOSING state.
515     ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
516 
517     try {
518       processServerShutdownHandler(ct, am, regionSplitDone, null);
519       // check znode deleted or not.
520       // In both cases the znode should be deleted.
521 
522       if(regionSplitDone){
523         assertTrue("Region state of region in SPLITTING should be removed from rit.",
524             am.regionsInTransition.isEmpty());
525       }
526       else{
527         while (!am.assignInvoked) {
528           Thread.sleep(1);
529         }
530         assertTrue("Assign should be invoked.", am.assignInvoked);
531       }
532     } finally {
533       REGIONINFO.setOffline(false);
534       REGIONINFO.setSplit(false);
535       executor.shutdown();
536       am.shutdown();
537       // Clean up all znodes
538       ZKAssign.deleteAllNodes(this.watcher);
539     }
540   }
541 
542   private void testCaseWithPartiallyDisabledState(TableState state, boolean opening)
543       throws KeeperException, IOException, NodeExistsException {
544     // Create and startup an executor. This is used by AssignmentManager
545     // handling zk callbacks.
546     ExecutorService executor = startupMasterExecutor("testSSHWhenDisableTableInProgress");
547 
548     // We need a mocked catalog tracker.
549     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
550     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
551     // Create an AM.
552     AssignmentManager am = new AssignmentManager(this.server, this.serverManager, ct, balancer,
553         executor);
554     if (opening) {
555       am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
556           State.OPENING, System.currentTimeMillis(), SERVERNAME_A));
557     } else {
558       // adding region to regions and servers maps.
559       am.regionOnline(REGIONINFO, SERVERNAME_A);
560       // adding region in pending close.
561       am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
562           State.PENDING_CLOSE, System.currentTimeMillis(), SERVERNAME_A));
563     }
564     if (state == TableState.DISABLING) {
565       am.getZKTable().setDisablingTable(REGIONINFO.getTableNameAsString());
566     } else {
567       am.getZKTable().setDisabledTable(REGIONINFO.getTableNameAsString());
568     }
569     RegionTransitionData data = null;
570     if (opening) {
571       data =
572           new RegionTransitionData(EventType.RS_ZK_REGION_OPENING, REGIONINFO.getRegionName(),
573               SERVERNAME_A);
574 
575     } else {
576       data =
577           new RegionTransitionData(EventType.M_ZK_REGION_CLOSING, REGIONINFO.getRegionName(),
578               SERVERNAME_A);
579     }
580     String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
581     // create znode in M_ZK_REGION_CLOSING state.
582     ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
583 
584     try {
585       processServerShutdownHandler(ct, am, false, null);
586       // check znode deleted or not.
587       // In both cases the znode should be deleted.
588       assertTrue("The znode should be deleted.",ZKUtil.checkExists(this.watcher, node) == -1);
589       assertTrue("Region state of region in pending close should be removed from rit.",
590         am.regionsInTransition.isEmpty());
591     } finally {
592       executor.shutdown();
593       am.shutdown();
594       // Clean up all znodes
595       ZKAssign.deleteAllNodes(this.watcher);
596     }
597   }
598 
599   private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager am,
600     boolean splitRegion, ServerName sn)
601       throws IOException {
602     // Make sure our new AM gets callbacks; once registered, can't unregister.
603     // Thats ok because we make a new zk watcher for each test.
604     this.watcher.registerListenerFirst(am);
605     // Need to set up a fake scan of meta for the servershutdown handler
606     // Make an RS Interface implementation.  Make it so a scanner can go against it.
607     HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
608     // Get a meta row result that has region up on SERVERNAME_A
609 
610     Result r = null;
611     if (sn == null) {
612       if (splitRegion) {
613         r = getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A);
614       } else {
615         r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
616       }
617     } else {
618       if (sn.equals(SERVERNAME_A)) {
619         r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
620       } else if (sn.equals(SERVERNAME_B)) {
621         r = new Result(new KeyValue[0]);
622       }
623     }
624 
625     Mockito.when(implementation.openScanner((byte [])Mockito.any(), (Scan)Mockito.any())).
626       thenReturn(System.currentTimeMillis());
627     // Return a good result first and then return null to indicate end of scan
628     Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt())).
629       thenReturn(new Result [] {r}, (Result [])null);
630 
631     // Get a connection w/ mocked up common methods.
632     HConnection connection =
633       HConnectionTestingUtility.getMockedConnectionAndDecorate(HTU.getConfiguration(),
634         implementation, SERVERNAME_B, REGIONINFO);
635 
636     // Make it so we can get a catalogtracker from servermanager.. .needed
637     // down in guts of server shutdown handler.
638     Mockito.when(ct.getConnection()).thenReturn(connection);
639     Mockito.when(this.server.getCatalogTracker()).thenReturn(ct);
640 
641     // Now make a server shutdown handler instance and invoke process.
642     // Have it that SERVERNAME_A died.
643     DeadServer deadServers = new DeadServer();
644     deadServers.add(SERVERNAME_A);
645     // I need a services instance that will return the AM
646     MasterServices services = Mockito.mock(MasterServices.class);
647     Mockito.when(services.getAssignmentManager()).thenReturn(am);
648     Mockito.when(services.getZooKeeper()).thenReturn(this.watcher);
649     ServerShutdownHandler handler = null;
650     if (sn != null) {
651       handler = new ServerShutdownHandler(this.server, services, deadServers, sn, false);
652     } else {
653       handler = new ServerShutdownHandler(this.server, services, deadServers, SERVERNAME_A, false);
654     }
655     handler.process();
656     // The region in r will have been assigned.  It'll be up in zk as unassigned.
657   }
658 
659   /**
660    * @param sn ServerName to use making startcode and server in meta
661    * @param hri Region to serialize into HRegionInfo
662    * @return A mocked up Result that fakes a Get on a row in the
663    * <code>.META.</code> table.
664    * @throws IOException
665    */
666   private Result getMetaTableRowResult(final HRegionInfo hri,
667       final ServerName sn)
668   throws IOException {
669     // TODO: Move to a utilities class.  More than one test case can make use
670     // of this facility.
671     List<KeyValue> kvs = new ArrayList<KeyValue>();
672     kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
673       HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
674       Writables.getBytes(hri)));
675     kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
676       HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
677       Bytes.toBytes(sn.getHostAndPort())));
678     kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
679       HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
680       Bytes.toBytes(sn.getStartcode())));
681     return new Result(kvs);
682   }
683 
684   /**
685    * @param sn ServerName to use making startcode and server in meta
686    * @param hri Region to serialize into HRegionInfo
687    * @return A mocked up Result that fakes a Get on a row in the
688    * <code>.META.</code> table.
689    * @throws IOException
690    */
691   private Result getMetaTableRowResultAsSplitRegion(final HRegionInfo hri, final ServerName sn)
692       throws IOException {
693     hri.setOffline(true);
694     hri.setSplit(true);
695     return getMetaTableRowResult(hri, sn);
696   }
697 
698   /**
699    * Create and startup executor pools. Start same set as master does (just
700    * run a few less).
701    * @param name Name to give our executor
702    * @return Created executor (be sure to call shutdown when done).
703    */
704   private ExecutorService startupMasterExecutor(final String name) {
705     // TODO: Move up into HBaseTestingUtility?  Generally useful.
706     ExecutorService executor = new ExecutorService(name);
707     executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
708     executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
709     executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
710     executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
711     return executor;
712   }
713 
714   @Test
715   public void testUnassignWithSplitAtSameTime() throws KeeperException, IOException {
716     // Region to use in test.
717     final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
718     // First amend the servermanager mock so that when we do send close of the
719     // first meta region on SERVERNAME_A, it will return true rather than
720     // default null.
721     Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, hri, -1)).thenReturn(true);
722     // Need a mocked catalog tracker.
723     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
724     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
725         .getConfiguration());
726     // Create an AM.
727     AssignmentManager am =
728       new AssignmentManager(this.server, this.serverManager, ct, balancer, null);
729     try {
730       // First make sure my mock up basically works.  Unassign a region.
731       unassign(am, SERVERNAME_A, hri);
732       // This delete will fail if the previous unassign did wrong thing.
733       ZKAssign.deleteClosingNode(this.watcher, hri);
734       // Now put a SPLITTING region in the way.  I don't have to assert it
735       // go put in place.  This method puts it in place then asserts it still
736       // owns it by moving state from SPLITTING to SPLITTING.
737       int version = createNodeSplitting(this.watcher, hri, SERVERNAME_A);
738       // Now, retry the unassign with the SPLTTING in place.  It should just
739       // complete without fail; a sort of 'silent' recognition that the
740       // region to unassign has been split and no longer exists: TOOD: what if
741       // the split fails and the parent region comes back to life?
742       unassign(am, SERVERNAME_A, hri);
743       // This transition should fail if the znode has been messed with.
744       ZKAssign.transitionNode(this.watcher, hri, SERVERNAME_A,
745         EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
746       assertTrue(am.isRegionInTransition(hri) == null);
747     } finally {
748       am.shutdown();
749     }
750   }
751 
752   /**
753    * Tests the processDeadServersAndRegionsInTransition should not fail with NPE
754    * when it failed to get the children. Let's abort the system in this
755    * situation
756    * @throws ServiceException
757    */
758   @Test(timeout = 60000)
759   public void testProcessDeadServersAndRegionsInTransitionShouldNotFailWithNPE()
760       throws IOException, KeeperException, InterruptedException, ServiceException {
761     final RecoverableZooKeeper recoverableZk = Mockito
762         .mock(RecoverableZooKeeper.class);
763     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
764         this.server, this.serverManager);
765     Watcher zkw = new ZooKeeperWatcher(HBaseConfiguration.create(), "unittest",
766         null) {
767       public RecoverableZooKeeper getRecoverableZooKeeper() {
768         return recoverableZk;
769       }
770     };
771     ((ZooKeeperWatcher) zkw).registerListener(am);
772     Mockito.doThrow(new InterruptedException()).when(recoverableZk)
773         .getChildren("/hbase/unassigned", zkw);
774     am.setWatcher((ZooKeeperWatcher) zkw);
775     try {
776       am.processDeadServersAndRegionsInTransition();
777       fail("Expected to abort");
778     } catch (NullPointerException e) {
779       fail("Should not throw NPE");
780     } catch (RuntimeException e) {
781       assertEquals("Aborted", e.getLocalizedMessage());
782     }
783   }
784 
785   /**
786    * Creates a new ephemeral node in the SPLITTING state for the specified region.
787    * Create it ephemeral in case regionserver dies mid-split.
788    *
789    * <p>Does not transition nodes from other states.  If a node already exists
790    * for this region, a {@link NodeExistsException} will be thrown.
791    *
792    * @param zkw zk reference
793    * @param region region to be created as offline
794    * @param serverName server event originates from
795    * @return Version of znode created.
796    * @throws KeeperException
797    * @throws IOException
798    */
799   // Copied from SplitTransaction rather than open the method over there in
800   // the regionserver package.
801   private static int createNodeSplitting(final ZooKeeperWatcher zkw,
802       final HRegionInfo region, final ServerName serverName)
803   throws KeeperException, IOException {
804     RegionTransitionData data =
805       new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING,
806         region.getRegionName(), serverName);
807 
808     String node = ZKAssign.getNodeName(zkw, region.getEncodedName());
809     if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, data.getBytes())) {
810       throw new IOException("Failed create of ephemeral " + node);
811     }
812     // Transition node from SPLITTING to SPLITTING and pick up version so we
813     // can be sure this znode is ours; version is needed deleting.
814     return transitionNodeSplitting(zkw, region, serverName, -1);
815   }
816 
817   // Copied from SplitTransaction rather than open the method over there in
818   // the regionserver package.
819   private static int transitionNodeSplitting(final ZooKeeperWatcher zkw,
820       final HRegionInfo parent,
821       final ServerName serverName, final int version)
822   throws KeeperException, IOException {
823     return ZKAssign.transitionNode(zkw, parent, serverName,
824       EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
825   }
826 
827   private void unassign(final AssignmentManager am, final ServerName sn,
828       final HRegionInfo hri) {
829     // Before I can unassign a region, I need to set it online.
830     am.regionOnline(hri, sn);
831     // Unassign region.
832     am.unassign(hri);
833   }
834 
835   /**
836    * Create an {@link AssignmentManagerWithExtrasForTesting} that has mocked
837    * {@link CatalogTracker} etc.
838    * @param server
839    * @param manager
840    * @return An AssignmentManagerWithExtras with mock connections, etc.
841    * @throws IOException
842    * @throws KeeperException
843    */
844   private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final Server server,
845       final ServerManager manager)
846   throws IOException, KeeperException {
847     // We need a mocked catalog tracker. Its used by our AM instance.
848     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
849     // Make an RS Interface implementation. Make it so a scanner can go against
850     // it and a get to return the single region, REGIONINFO, this test is
851     // messing with. Needed when "new master" joins cluster. AM will try and
852     // rebuild its list of user regions and it will also get the HRI that goes
853     // with an encoded name by doing a Get on .META.
854     HRegionInterface ri = Mockito.mock(HRegionInterface.class);
855     // Get a meta row result that has region up on SERVERNAME_A for REGIONINFO
856     Result[] result = null;
857     if (enabling) {
858       result = new Result[2];
859       result[0] = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
860       result[1] = getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A);
861     }
862     Result r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
863     Mockito.when(ri .openScanner((byte[]) Mockito.any(), (Scan) Mockito.any())).
864       thenReturn(System.currentTimeMillis());
865    if (enabling) {
866       Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt())).thenReturn(result, result, result,
867           (Result[]) null);
868       // If a get, return the above result too for REGIONINFO_2
869       Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(
870           getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A));
871     } else {
872       // Return good result 'r' first and then return null to indicate end of scan
873       Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt())).thenReturn(new Result[] { r });
874       // If a get, return the above result too for REGIONINFO
875       Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(r);
876     }
877     // Get a connection w/ mocked up common methods.
878     HConnection connection = HConnectionTestingUtility.
879       getMockedConnectionAndDecorate(HTU.getConfiguration(), ri, SERVERNAME_B,
880         REGIONINFO);
881     // Make it so we can get the connection from our mocked catalogtracker
882     Mockito.when(ct.getConnection()).thenReturn(connection);
883     // Create and startup an executor. Used by AM handling zk callbacks.
884     ExecutorService executor = startupMasterExecutor("mockedAMExecutor");
885     this.balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
886     AssignmentManagerWithExtrasForTesting am = new AssignmentManagerWithExtrasForTesting(
887         server, manager, ct, balancer, executor);
888     return am;
889   }
890 
891   /**
892    * TestCase verifies that the regionPlan is updated whenever a region fails to open
893    * and the master tries to process RS_ZK_FAILED_OPEN state.(HBASE-5546).
894    */
895   @Test
896   public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, KeeperException,
897       ServiceException, InterruptedException {
898     this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
899         MockedLoadBalancer.class, LoadBalancer.class);
900     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server,
901         this.serverManager);
902     try {
903       // Boolean variable used for waiting until randomAssignment is called and new
904       // plan is generated.
905       AtomicBoolean gate = new AtomicBoolean(false);
906       if (balancer instanceof MockedLoadBalancer) {
907         ((MockedLoadBalancer) balancer).setGateVariable(gate);
908       }
909       ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
910       int v = ZKAssign.getVersion(this.watcher, REGIONINFO);
911       ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
912           EventType.RS_ZK_REGION_FAILED_OPEN, v);
913       String path = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
914       RegionState state = new RegionState(REGIONINFO, State.OPENING, System.currentTimeMillis(),
915           SERVERNAME_A);
916       am.regionsInTransition.put(REGIONINFO.getEncodedName(), state);
917       // a dummy plan inserted into the regionPlans. This plan is cleared and new one is formed
918       am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));
919       RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
920       List<ServerName> serverList = new ArrayList<ServerName>(2);
921       serverList.add(SERVERNAME_B);
922       Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(serverList);
923       am.nodeDataChanged(path);
924       // here we are waiting until the random assignment in the load balancer is called.
925       while (!gate.get()) {
926         Thread.sleep(10);
927       }
928       // new region plan may take some time to get updated after random assignment is called and
929       // gate is set to true.
930       RegionPlan newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
931       while (newRegionPlan == null) {
932         Thread.sleep(10);
933         newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
934       }
935       // the new region plan created may contain the same RS as destination but it should
936       // be new plan.
937       assertNotSame("Same region plan should not come", regionPlan, newRegionPlan);
938       assertTrue("Destnation servers should be different.", !(regionPlan.getDestination().equals(
939         newRegionPlan.getDestination())));
940       Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
941     } finally {
942       this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
943         DefaultLoadBalancer.class, LoadBalancer.class);
944       am.shutdown();
945     }
946   }
947 
948   /**
949    * Test verifies whether assignment is skipped for regions of tables in DISABLING state during
950    * clean cluster startup. See HBASE-6281.
951    *
952    * @throws KeeperException
953    * @throws IOException
954    * @throws Exception
955    */
956   @Test
957   public void testDisablingTableRegionsAssignmentDuringCleanClusterStartup()
958       throws KeeperException, IOException, Exception {
959     this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
960         MockedLoadBalancer.class, LoadBalancer.class);
961     Mockito.when(this.serverManager.getOnlineServers()).thenReturn(
962         new HashMap<ServerName, HServerLoad>(0));
963     List<ServerName> destServers = new ArrayList<ServerName>(1);
964     destServers.add(SERVERNAME_A);
965     Mockito.when(this.serverManager.getDrainingServersList()).thenReturn(destServers);
966     // To avoid cast exception in DisableTableHandler process.
967     //Server server = new HMaster(HTU.getConfiguration());
968     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
969         this.serverManager);
970     AtomicBoolean gate = new AtomicBoolean(false);
971     if (balancer instanceof MockedLoadBalancer) {
972       ((MockedLoadBalancer) balancer).setGateVariable(gate);
973     }
974     try{
975       // set table in disabling state.
976       am.getZKTable().setDisablingTable(REGIONINFO.getTableNameAsString());
977       am.joinCluster();
978       // should not call retainAssignment if we get empty regions in assignAllUserRegions.
979       assertFalse(
980           "Assign should not be invoked for disabling table regions during clean cluster startup.",
981           gate.get());
982       // need to change table state from disabling to disabled.
983       assertTrue("Table should be disabled.",
984           am.getZKTable().isDisabledTable(REGIONINFO.getTableNameAsString()));
985     } finally {
986       this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
987         DefaultLoadBalancer.class, LoadBalancer.class);
988       am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
989       am.shutdown();
990     }
991   }
992 
993   /**
994    * Test verifies whether stale znodes of unknown tables as for the hbase:meta will be removed or
995    * not.
996    * @throws KeeperException
997    * @throws IOException
998    * @throws Exception
999    */
1000   @Test
1001   public void testMasterRestartShouldRemoveStaleZnodesOfUnknownTableAsForMeta()
1002       throws KeeperException, IOException, Exception {
1003     List<ServerName> destServers = new ArrayList<ServerName>(1);
1004     destServers.add(SERVERNAME_A);
1005     Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(destServers);
1006     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
1007     HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
1008     Server server = new HMaster(HTU.getConfiguration());
1009     Whitebox.setInternalState(server, "serverManager", this.serverManager);
1010     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
1011         this.serverManager);
1012     try {
1013       String tableName = "dummyTable";
1014       am.enablingTables.put(tableName, null);
1015       // set table in enabling state.
1016       am.getZKTable().setEnablingTable(tableName);
1017       am.joinCluster();
1018       assertFalse("Table should not be present in zookeeper.",
1019         am.getZKTable().isTablePresent(tableName));
1020     } finally {
1021     }
1022   }
1023 
1024   /**
1025    * Test verifies whether all the enabling table regions assigned only once during master startup.
1026    * 
1027    * @throws KeeperException
1028    * @throws IOException
1029    * @throws Exception
1030    */
1031   @Test
1032   public void testMasterRestartWhenTableInEnabling() throws KeeperException, IOException, Exception {
1033     enabling = true;
1034     this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
1035         DefaultLoadBalancer.class, LoadBalancer.class);
1036     Map<ServerName, HServerLoad> serverAndLoad = new HashMap<ServerName, HServerLoad>();
1037     serverAndLoad.put(SERVERNAME_A, null);
1038     Mockito.when(this.serverManager.getOnlineServers()).thenReturn(serverAndLoad);
1039     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(false);
1040     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
1041     HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
1042     Server server = new HMaster(HTU.getConfiguration());
1043     Whitebox.setInternalState(server, "serverManager", this.serverManager);
1044     assignmentCount = 0;
1045     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
1046         this.serverManager);
1047     am.regionOnline(new HRegionInfo("t1".getBytes(), HConstants.EMPTY_START_ROW,
1048         HConstants.EMPTY_END_ROW), SERVERNAME_A);
1049     am.gate.set(false);
1050     try {
1051       // set table in enabling state.
1052       am.getZKTable().setEnablingTable(REGIONINFO.getTableNameAsString());
1053       ZKAssign.createNodeOffline(this.watcher, REGIONINFO_2, SERVERNAME_B);
1054 
1055       am.joinCluster();
1056       while (!am.getZKTable().isEnabledTable(REGIONINFO.getTableNameAsString())) {
1057         Thread.sleep(10);
1058       }
1059       assertEquals("Number of assignments should be equal.", 2, assignmentCount);
1060       assertTrue("Table should be enabled.",
1061           am.getZKTable().isEnabledTable(REGIONINFO.getTableNameAsString()));
1062     } finally {
1063       enabling = false;
1064       am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
1065       am.shutdown();
1066       ZKAssign.deleteAllNodes(this.watcher);
1067       assignmentCount = 0;
1068     }
1069   }
1070 
1071 
1072 
1073   /**
1074    * When region in transition if region server opening the region gone down then region assignment
1075    * taking long time(Waiting for timeout monitor to trigger assign). HBASE-5396(HBASE-6060) fixes this
1076    * scenario. This test case verifies whether SSH calling assign for the region in transition or not.
1077    *
1078    * @throws KeeperException
1079    * @throws IOException
1080    * @throws ServiceException
1081    */
1082   @Test
1083   public void testSSHWhenSourceRSandDestRSInRegionPlanGoneDown() throws KeeperException, IOException,
1084       ServiceException {
1085     testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(true);
1086     testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(false);
1087   }
1088 
1089   private void testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(boolean regionInOffline)
1090       throws IOException, KeeperException, ServiceException {
1091     // We need a mocked catalog tracker.
1092     CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1093     // Create an AM.
1094     AssignmentManagerWithExtrasForTesting am =
1095         setUpMockedAssignmentManager(this.server, this.serverManager);
1096     // adding region in pending open.
1097     if (regionInOffline) {
1098       ServerName MASTER_SERVERNAME = new ServerName("example.org", 1111, 1111);
1099       am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
1100           State.OFFLINE, System.currentTimeMillis(), MASTER_SERVERNAME));
1101     } else {
1102       am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
1103           State.OPENING, System.currentTimeMillis(), SERVERNAME_B));
1104     }
1105     // adding region plan
1106     am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B));
1107     am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
1108 
1109     try {
1110       processServerShutdownHandler(ct, am, false, SERVERNAME_A);
1111       processServerShutdownHandler(ct, am, false, SERVERNAME_B);
1112       if(regionInOffline){
1113         assertFalse("Assign should not be invoked.", am.assignInvoked);
1114       } else {
1115         assertTrue("Assign should be invoked.", am.assignInvoked);
1116       }
1117     } finally {
1118       am.regionsInTransition.remove(REGIONINFO.getEncodedName());
1119       am.regionPlans.remove(REGIONINFO.getEncodedName());
1120     }
1121   }
1122 
1123   /**
1124    * Mocked load balancer class used in the testcase to make sure that the testcase waits until
1125    * random assignment is called and the gate variable is set to true.
1126    */
1127   public static class MockedLoadBalancer extends DefaultLoadBalancer {
1128     private AtomicBoolean gate;
1129 
1130     public void setGateVariable(AtomicBoolean gate) {
1131       this.gate = gate;
1132     }
1133 
1134     @Override
1135     public ServerName randomAssignment(List<ServerName> servers) {
1136       ServerName randomServerName = super.randomAssignment(servers);
1137       this.gate.set(true);
1138       return randomServerName;
1139     }
1140 
1141     @Override
1142     public Map<ServerName, List<HRegionInfo>> retainAssignment(
1143         Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
1144       this.gate.set(true);
1145       return super.retainAssignment(regions, servers);
1146     }
1147 
1148   }
1149 
1150   /**
1151    * An {@link AssignmentManager} with some extra facility used testing
1152    */
1153   class AssignmentManagerWithExtrasForTesting extends AssignmentManager {
1154     // Keep a reference so can give it out below in {@link #getExecutorService}
1155     private final ExecutorService es;
1156     // Ditto for ct
1157     private final CatalogTracker ct;
1158     boolean processRITInvoked = false;
1159     boolean assignInvoked = false;
1160     AtomicBoolean gate = new AtomicBoolean(true);
1161 
1162     public AssignmentManagerWithExtrasForTesting(final Server master,
1163         final ServerManager serverManager, final CatalogTracker catalogTracker,
1164         final LoadBalancer balancer, final ExecutorService service)
1165     throws KeeperException, IOException {
1166       super(master, serverManager, catalogTracker, balancer, service);
1167       this.es = service;
1168       this.ct = catalogTracker;
1169     }
1170 
1171     @Override
1172     boolean processRegionInTransition(String encodedRegionName,
1173         HRegionInfo regionInfo,
1174         Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers)
1175         throws KeeperException, IOException {
1176       this.processRITInvoked = true;
1177       return super.processRegionInTransition(encodedRegionName, regionInfo,
1178           deadServers);
1179     }
1180     @Override
1181     void processRegionsInTransition(final RegionTransitionData data,
1182         final HRegionInfo regionInfo,
1183         final Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers,
1184         final int expectedVersion) throws KeeperException {
1185       while (this.gate.get()) Threads.sleep(1);
1186       super.processRegionsInTransition(data, regionInfo, deadServers, expectedVersion);
1187     }
1188     
1189     @Override
1190     public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan,
1191         boolean hijack) {
1192       if (enabling) {
1193         assignmentCount++;
1194         this.regionOnline(region, SERVERNAME_A);
1195       } else {
1196         assignInvoked = true;
1197         super.assign(region, setOfflineInZK, forceNewPlan, hijack);
1198       }
1199     }
1200     
1201     @Override
1202     public ServerName getRegionServerOfRegion(HRegionInfo hri) {
1203       return SERVERNAME_A;
1204     }
1205     
1206     /** reset the watcher */
1207     void setWatcher(ZooKeeperWatcher watcher) {
1208       this.watcher = watcher;
1209     }
1210 
1211     /**
1212      * @return ExecutorService used by this instance.
1213      */
1214     ExecutorService getExecutorService() {
1215       return this.es;
1216     }
1217 
1218     /**
1219      * @return CatalogTracker used by this AM (Its a mock).
1220      */
1221     CatalogTracker getCatalogTracker() {
1222       return this.ct;
1223     }
1224   }
1225 
1226   /**
1227    * Call joinCluster on the passed AssignmentManager.  Do it in a thread
1228    * so it runs independent of what all else is going on.  Try to simulate
1229    * an AM running insided a failed over master by clearing all in-memory
1230    * AM state first.
1231   */
1232   private void startFakeFailedOverMasterAssignmentManager(final AssignmentManager am,
1233       final ZooKeeperWatcher watcher) {
1234     // Make sure our new AM gets callbacks; once registered, we can't unregister.
1235     // Thats ok because we make a new zk watcher for each test.
1236     watcher.registerListenerFirst(am);
1237     Thread t = new Thread("RunAmJoinCluster") {
1238       public void run() {
1239         // Call the joinCluster function as though we were doing a master
1240         // failover at this point. It will stall just before we go to add
1241         // the RIT region to our RIT Map in AM at processRegionsInTransition.
1242         // First clear any inmemory state from AM so it acts like a new master
1243         // coming on line.
1244         am.regionsInTransition.clear();
1245         am.regionPlans.clear();
1246         try {
1247           am.joinCluster();
1248         } catch (IOException e) {
1249           throw new RuntimeException(e);
1250         } catch (KeeperException e) {
1251           throw new RuntimeException(e);
1252         } catch (InterruptedException e) {
1253           throw new RuntimeException(e);
1254         }
1255       };
1256     };
1257     t.start();
1258     while (!t.isAlive()) Threads.sleep(1);
1259   }
1260 
1261   @org.junit.Rule
1262   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
1263     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
1264 }