1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertFalse;
24  import static org.junit.Assert.assertNotNull;
25  import static org.junit.Assert.assertTrue;
26  
27  import java.io.IOException;
28  import java.util.ArrayList;
29  import java.util.HashSet;
30  import java.util.Iterator;
31  import java.util.List;
32  import java.util.Set;
33  import java.util.TreeSet;
34  
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.fs.FileSystem;
39  import org.apache.hadoop.fs.Path;
40  import org.apache.hadoop.hbase.Abortable;
41  import org.apache.hadoop.hbase.ClusterStatus;
42  import org.apache.hadoop.hbase.HBaseConfiguration;
43  import org.apache.hadoop.hbase.HBaseTestingUtility;
44  import org.apache.hadoop.hbase.HColumnDescriptor;
45  import org.apache.hadoop.hbase.HConstants;
46  import org.apache.hadoop.hbase.HRegionInfo;
47  import org.apache.hadoop.hbase.HTableDescriptor;
48  import org.apache.hadoop.hbase.LargeTests;
49  import org.apache.hadoop.hbase.MasterNotRunningException;
50  import org.apache.hadoop.hbase.MiniHBaseCluster;
51  import org.apache.hadoop.hbase.ServerName;
52  import org.apache.hadoop.hbase.client.HTable;
53  import org.apache.hadoop.hbase.executor.EventHandler.EventType;
54  import org.apache.hadoop.hbase.executor.RegionTransitionData;
55  import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
56  import org.apache.hadoop.hbase.regionserver.HRegion;
57  import org.apache.hadoop.hbase.regionserver.HRegionServer;
58  import org.apache.hadoop.hbase.util.Bytes;
59  import org.apache.hadoop.hbase.util.FSTableDescriptors;
60  import org.apache.hadoop.hbase.util.JVMClusterUtil;
61  import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
62  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
63  import org.apache.hadoop.hbase.util.Threads;
64  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
65  import org.apache.hadoop.hbase.zookeeper.ZKTable;
66  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
67  import org.junit.Test;
68  import org.junit.experimental.categories.Category;
69  
70  @Category(LargeTests.class)
71  public class TestMasterFailover {
72    private static final Log LOG = LogFactory.getLog(TestMasterFailover.class);
73  
74    @Test (timeout=180000)
75    public void testShouldCheckMasterFailOverWhenMETAIsInOpenedState()
76        throws Exception {
77      LOG.info("Starting testShouldCheckMasterFailOverWhenMETAIsInOpenedState");
78      final int NUM_MASTERS = 1;
79      final int NUM_RS = 2;
80  
81      Configuration conf = HBaseConfiguration.create();
82      conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000);
83      conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 8000);
84      // Start the cluster
85      HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
86  
87      TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
88      MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
89  
90      // Find regionserver carrying meta.
91      List<RegionServerThread> regionServerThreads =
92        cluster.getRegionServerThreads();
93      int count = -1;
94      HRegion metaRegion = null;
95      for (RegionServerThread regionServerThread : regionServerThreads) {
96        HRegionServer regionServer = regionServerThread.getRegionServer();
97        metaRegion = regionServer.getOnlineRegion(HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
98        count++;
99        regionServer.abort("");
100       if (null != metaRegion) break;
101     }
102     HRegionServer regionServer = cluster.getRegionServer(count);
103 
104     TEST_UTIL.shutdownMiniHBaseCluster();
105 
106     // Create a ZKW to use in the test
107     ZooKeeperWatcher zkw = 
108       HBaseTestingUtility.createAndForceNodeToOpenedState(TEST_UTIL, 
109           metaRegion, regionServer.getServerName());
110 
111     LOG.info("Staring cluster for second time");
112     TEST_UTIL.startMiniHBaseCluster(NUM_MASTERS, NUM_RS);
113 
114     // Failover should be completed, now wait for no RIT
115     log("Waiting for no more RIT");
116     ZKAssign.blockUntilNoRIT(zkw);
117 
118     zkw.close();
119     // Stop the cluster
120     TEST_UTIL.shutdownMiniCluster();
121   }
122 
123   /**
124    * Simple test of master failover.
125    * <p>
126    * Starts with three masters.  Kills a backup master.  Then kills the active
127    * master.  Ensures the final master becomes active and we can still contact
128    * the cluster.
129    * @throws Exception
130    */
131   @Test (timeout=240000)
132   public void testSimpleMasterFailover() throws Exception {
133 
134     final int NUM_MASTERS = 3;
135     final int NUM_RS = 3;
136 
137     // Create config to use for this cluster
138     Configuration conf = HBaseConfiguration.create();
139 
140     // Start the cluster
141     HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
142     TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
143     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
144 
145     // get all the master threads
146     List<MasterThread> masterThreads = cluster.getMasterThreads();
147 
148     // wait for each to come online
149     for (MasterThread mt : masterThreads) {
150       assertTrue(mt.isAlive());
151     }
152 
153     // verify only one is the active master and we have right number
154     int numActive = 0;
155     int activeIndex = -1;
156     ServerName activeName = null;
157     HMaster active = null;
158     for (int i = 0; i < masterThreads.size(); i++) {
159       if (masterThreads.get(i).getMaster().isActiveMaster()) {
160         numActive++;
161         activeIndex = i;
162         active = masterThreads.get(activeIndex).getMaster();
163         activeName = active.getServerName();
164       }
165     }
166     assertEquals(1, numActive);
167     assertEquals(NUM_MASTERS, masterThreads.size());
168     LOG.info("Active master " + activeName);
169 
170     // Check that ClusterStatus reports the correct active and backup masters
171     assertNotNull(active);
172     ClusterStatus status = active.getClusterStatus();
173     assertTrue(status.getMaster().equals(activeName));
174     assertEquals(2, status.getBackupMastersSize());
175     assertEquals(2, status.getBackupMasters().size());
176 
177     // attempt to stop one of the inactive masters
178     int backupIndex = (activeIndex == 0 ? 1 : activeIndex - 1);
179     HMaster master = cluster.getMaster(backupIndex);
180     LOG.debug("\n\nStopping a backup master: " + master.getServerName() + "\n");
181     cluster.stopMaster(backupIndex, false);
182     cluster.waitOnMaster(backupIndex);
183 
184     // Verify still one active master and it's the same
185     for (int i = 0; i < masterThreads.size(); i++) {
186       if (masterThreads.get(i).getMaster().isActiveMaster()) {
187         assertTrue(activeName.equals(masterThreads.get(i).getMaster().getServerName()));
188         activeIndex = i;
189         active = masterThreads.get(activeIndex).getMaster();
190       }
191     }
192     assertEquals(1, numActive);
193     assertEquals(2, masterThreads.size());
194     int rsCount = masterThreads.get(activeIndex).getMaster().getClusterStatus().getServersSize();
195     LOG.info("Active master " + active.getServerName() + " managing " + rsCount +  " regions servers");
196     assertEquals(3, rsCount);
197 
198     // Check that ClusterStatus reports the correct active and backup masters
199     assertNotNull(active);
200     status = active.getClusterStatus();
201     assertTrue(status.getMaster().equals(activeName));
202     assertEquals(1, status.getBackupMastersSize());
203     assertEquals(1, status.getBackupMasters().size());
204 
205     // kill the active master
206     LOG.debug("\n\nStopping the active master " + active.getServerName() + "\n");
207     cluster.stopMaster(activeIndex, false);
208     cluster.waitOnMaster(activeIndex);
209 
210     // wait for an active master to show up and be ready
211     assertTrue(cluster.waitForActiveAndReadyMaster());
212 
213     LOG.debug("\n\nVerifying backup master is now active\n");
214     // should only have one master now
215     assertEquals(1, masterThreads.size());
216 
217     // and he should be active
218     active = masterThreads.get(0).getMaster();
219     assertNotNull(active);
220     status = active.getClusterStatus();
221     ServerName mastername = status.getMaster();
222     assertTrue(mastername.equals(active.getServerName()));
223     assertTrue(active.isActiveMaster());
224     assertEquals(0, status.getBackupMastersSize());
225     assertEquals(0, status.getBackupMasters().size());
226     int rss = status.getServersSize();
227     LOG.info("Active master " + mastername.getServerName() + " managing " +
228       rss +  " region servers");
229     assertEquals(3, rss);
230 
231     // Stop the cluster
232     TEST_UTIL.shutdownMiniCluster();
233   }
234 
235   /**
236    * Complex test of master failover that tests as many permutations of the
237    * different possible states that regions in transition could be in within ZK.
238    * <p>
239    * This tests the proper handling of these states by the failed-over master
240    * and includes a thorough testing of the timeout code as well.
241    * <p>
242    * Starts with a single master and three regionservers.
243    * <p>
244    * Creates two tables, enabledTable and disabledTable, each containing 5
245    * regions.  The disabledTable is then disabled.
246    * <p>
247    * After reaching steady-state, the master is killed.  We then mock several
248    * states in ZK.
249    * <p>
250    * After mocking them, we will startup a new master which should become the
251    * active master and also detect that it is a failover.  The primary test
252    * passing condition will be that all regions of the enabled table are
253    * assigned and all the regions of the disabled table are not assigned.
254    * <p>
255    * The different scenarios to be tested are below:
256    * <p>
257    * <b>ZK State:  OFFLINE</b>
258    * <p>A node can get into OFFLINE state if</p>
259    * <ul>
260    * <li>An RS fails to open a region, so it reverts the state back to OFFLINE
261    * <li>The Master is assigning the region to a RS before it sends RPC
262    * </ul>
263    * <p>We will mock the scenarios</p>
264    * <ul>
265    * <li>Master has assigned an enabled region but RS failed so a region is
266    *     not assigned anywhere and is sitting in ZK as OFFLINE</li>
267    * <li>This seems to cover both cases?</li>
268    * </ul>
269    * <p>
270    * <b>ZK State:  CLOSING</b>
271    * <p>A node can get into CLOSING state if</p>
272    * <ul>
273    * <li>An RS has begun to close a region
274    * </ul>
275    * <p>We will mock the scenarios</p>
276    * <ul>
277    * <li>Region of enabled table was being closed but did not complete
278    * <li>Region of disabled table was being closed but did not complete
279    * </ul>
280    * <p>
281    * <b>ZK State:  CLOSED</b>
282    * <p>A node can get into CLOSED state if</p>
283    * <ul>
284    * <li>An RS has completed closing a region but not acknowledged by master yet
285    * </ul>
286    * <p>We will mock the scenarios</p>
287    * <ul>
288    * <li>Region of a table that should be enabled was closed on an RS
289    * <li>Region of a table that should be disabled was closed on an RS
290    * </ul>
291    * <p>
292    * <b>ZK State:  OPENING</b>
293    * <p>A node can get into OPENING state if</p>
294    * <ul>
295    * <li>An RS has begun to open a region
296    * </ul>
297    * <p>We will mock the scenarios</p>
298    * <ul>
299    * <li>RS was opening a region of enabled table but never finishes
300    * </ul>
301    * <p>
302    * <b>ZK State:  OPENED</b>
303    * <p>A node can get into OPENED state if</p>
304    * <ul>
305    * <li>An RS has finished opening a region but not acknowledged by master yet
306    * </ul>
307    * <p>We will mock the scenarios</p>
308    * <ul>
309    * <li>Region of a table that should be enabled was opened on an RS
310    * <li>Region of a table that should be disabled was opened on an RS
311    * </ul>
312    * @throws Exception
313    */
314   @Test (timeout=180000)
315   public void testMasterFailoverWithMockedRIT() throws Exception {
316 
317     final int NUM_MASTERS = 1;
318     final int NUM_RS = 3;
319 
320     // Create config to use for this cluster
321     Configuration conf = HBaseConfiguration.create();
322     // Need to drop the timeout much lower
323     conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000);
324     conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 4000);
325     conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 3);
326     conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 3);
327 
328     // Start the cluster
329     HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
330     TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
331     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
332     log("Cluster started");
333 
334     // Create a ZKW to use in the test
335     ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
336 
337     // get all the master threads
338     List<MasterThread> masterThreads = cluster.getMasterThreads();
339     assertEquals(1, masterThreads.size());
340 
341     // only one master thread, let's wait for it to be initialized
342     assertTrue(cluster.waitForActiveAndReadyMaster());
343     HMaster master = masterThreads.get(0).getMaster();
344     assertTrue(master.isActiveMaster());
345     assertTrue(master.isInitialized());
346 
347     // disable load balancing on this master
348     master.balanceSwitch(false);
349 
350     // create two tables in META, each with 10 regions
351     byte [] FAMILY = Bytes.toBytes("family");
352     byte [][] SPLIT_KEYS = new byte [][] {
353         new byte[0], Bytes.toBytes("aaa"), Bytes.toBytes("bbb"),
354         Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"),
355         Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
356         Bytes.toBytes("iii"), Bytes.toBytes("jjj")
357     };
358 
359     byte [] enabledTable = Bytes.toBytes("enabledTable");
360     HTableDescriptor htdEnabled = new HTableDescriptor(enabledTable);
361     htdEnabled.addFamily(new HColumnDescriptor(FAMILY));
362 
363     FileSystem filesystem = FileSystem.get(conf);
364     Path rootdir = filesystem.makeQualified(
365         new Path(conf.get(HConstants.HBASE_DIR)));
366     // Write the .tableinfo
367     FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdEnabled);
368 
369     HRegionInfo hriEnabled = new HRegionInfo(htdEnabled.getName(), null, null);
370     createRegion(hriEnabled, rootdir, conf, htdEnabled);
371 
372     List<HRegionInfo> enabledRegions = TEST_UTIL.createMultiRegionsInMeta(
373         TEST_UTIL.getConfiguration(), htdEnabled, SPLIT_KEYS);
374 
375     byte [] disabledTable = Bytes.toBytes("disabledTable");
376     HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable);
377     htdDisabled.addFamily(new HColumnDescriptor(FAMILY));
378     // Write the .tableinfo
379     FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdDisabled);
380     HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getName(), null, null);
381     createRegion(hriDisabled, rootdir, conf, htdDisabled);
382     List<HRegionInfo> disabledRegions = TEST_UTIL.createMultiRegionsInMeta(
383         TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS);
384 
385     log("Regions in META have been created");
386 
387     // at this point we only expect 2 regions to be assigned out (catalogs)
388     assertEquals(2, cluster.countServedRegions());
389 
390     // Let's just assign everything to first RS
391     HRegionServer hrs = cluster.getRegionServer(0);
392     ServerName serverName = hrs.getServerName();
393     HRegionInfo closingRegion = enabledRegions.remove(0);
394     // we'll need some regions to already be assigned out properly on live RS
395     List<HRegionInfo> enabledAndAssignedRegions = new ArrayList<HRegionInfo>();
396     enabledAndAssignedRegions.add(enabledRegions.remove(0));
397     enabledAndAssignedRegions.add(enabledRegions.remove(0));
398     enabledAndAssignedRegions.add(closingRegion);
399 
400     List<HRegionInfo> disabledAndAssignedRegions = new ArrayList<HRegionInfo>();
401     disabledAndAssignedRegions.add(disabledRegions.remove(0));
402     disabledAndAssignedRegions.add(disabledRegions.remove(0));
403 
404     // now actually assign them
405     for (HRegionInfo hri : enabledAndAssignedRegions) {
406       master.assignmentManager.regionPlans.put(hri.getEncodedName(),
407           new RegionPlan(hri, null, serverName));
408       master.assignRegion(hri);
409     }
410     for (HRegionInfo hri : disabledAndAssignedRegions) {
411       master.assignmentManager.regionPlans.put(hri.getEncodedName(),
412           new RegionPlan(hri, null, serverName));
413       master.assignRegion(hri);
414     }
415 
416     // wait for no more RIT
417     log("Waiting for assignment to finish");
418     ZKAssign.blockUntilNoRIT(zkw);
419     log("Assignment completed");
420 
421     // Stop the master
422     log("Aborting master");
423     cluster.abortMaster(0);
424     cluster.waitOnMaster(0);
425     log("Master has aborted");
426 
427     /*
428      * Now, let's start mocking up some weird states as described in the method
429      * javadoc.
430      */
431 
432     List<HRegionInfo> regionsThatShouldBeOnline = new ArrayList<HRegionInfo>();
433     List<HRegionInfo> regionsThatShouldBeOffline = new ArrayList<HRegionInfo>();
434 
435     log("Beginning to mock scenarios");
436 
437     // Disable the disabledTable in ZK
438     ZKTable zktable = new ZKTable(zkw);
439     zktable.setDisabledTable(Bytes.toString(disabledTable));
440 
441     /*
442      *  ZK = OFFLINE
443      */
444 
445     // Region that should be assigned but is not and is in ZK as OFFLINE
446     HRegionInfo region = enabledRegions.remove(0);
447     regionsThatShouldBeOnline.add(region);
448     ZKAssign.createNodeOffline(zkw, region, serverName);
449 
450     /*
451      * ZK = CLOSING
452      */
453     regionsThatShouldBeOnline.add(closingRegion);
454     ZKAssign.createNodeClosing(zkw, closingRegion, serverName);
455 
456     /*
457      * ZK = CLOSED
458      */
459 
460     // Region of enabled table closed but not ack
461     region = enabledRegions.remove(0);
462     regionsThatShouldBeOnline.add(region);
463     int version = ZKAssign.createNodeClosing(zkw, region, serverName);
464     ZKAssign.transitionNodeClosed(zkw, region, serverName, version);
465 
466     // Region of disabled table closed but not ack
467     region = disabledRegions.remove(0);
468     regionsThatShouldBeOffline.add(region);
469     version = ZKAssign.createNodeClosing(zkw, region, serverName);
470     ZKAssign.transitionNodeClosed(zkw, region, serverName, version);
471 
472     /*
473      * ZK = OPENING
474      */
475 
476     // RS was opening a region of enabled table but never finishes
477     region = enabledRegions.remove(0);
478     regionsThatShouldBeOnline.add(region);
479     ZKAssign.createNodeOffline(zkw, region, serverName);
480     ZKAssign.transitionNodeOpening(zkw, region, serverName);
481 
482     /*
483      * ZK = OPENED
484      */
485 
486     // Region of enabled table was opened on RS
487     region = enabledRegions.remove(0);
488     regionsThatShouldBeOnline.add(region);
489     ZKAssign.createNodeOffline(zkw, region, serverName);
490     hrs.openRegion(region);
491     while (true) {
492       RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName());
493       if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) {
494         break;
495       }
496       Thread.sleep(100);
497     }
498 
499     // Region of disable table was opened on RS
500     region = disabledRegions.remove(0);
501     regionsThatShouldBeOffline.add(region);
502     ZKAssign.createNodeOffline(zkw, region, serverName);
503     hrs.openRegion(region);
504     while (true) {
505       RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName());
506       if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) {
507         break;
508       }
509       Thread.sleep(100);
510     }
511 
512     /*
513      * ZK = NONE
514      */
515 
516     /*
517      * DONE MOCKING
518      */
519 
520     log("Done mocking data up in ZK");
521 
522     // Start up a new master
523     log("Starting up a new master");
524     master = cluster.startMaster().getMaster();
525     log("Waiting for master to be ready");
526     cluster.waitForActiveAndReadyMaster();
527     log("Master is ready");
528 
529     // Failover should be completed, now wait for no RIT
530     log("Waiting for no more RIT");
531     ZKAssign.blockUntilNoRIT(zkw);
532     log("No more RIT in ZK, now doing final test verification");
533 
534     // Grab all the regions that are online across RSs
535     Set<HRegionInfo> onlineRegions = new TreeSet<HRegionInfo>();
536     for (JVMClusterUtil.RegionServerThread rst :
537       cluster.getRegionServerThreads()) {
538       onlineRegions.addAll(rst.getRegionServer().getOnlineRegions());
539     }
540 
541     // Now, everything that should be online should be online
542     for (HRegionInfo hri : regionsThatShouldBeOnline) {
543       assertTrue(onlineRegions.contains(hri));
544     }
545 
546     // Everything that should be offline should not be online
547     for (HRegionInfo hri : regionsThatShouldBeOffline) {
548       assertFalse(onlineRegions.contains(hri));
549     }
550 
551     log("Done with verification, all passed, shutting down cluster");
552 
553     // Done, shutdown the cluster
554     TEST_UTIL.shutdownMiniCluster();
555   }
556 
557 
558   /**
559    * Complex test of master failover that tests as many permutations of the
560    * different possible states that regions in transition could be in within ZK
561    * pointing to an RS that has died while no master is around to process it.
562    * <p>
563    * This tests the proper handling of these states by the failed-over master
564    * and includes a thorough testing of the timeout code as well.
565    * <p>
566    * Starts with a single master and two regionservers.
567    * <p>
568    * Creates two tables, enabledTable and disabledTable, each containing 5
569    * regions.  The disabledTable is then disabled.
570    * <p>
571    * After reaching steady-state, the master is killed.  We then mock several
572    * states in ZK.  And one of the RS will be killed.
573    * <p>
574    * After mocking them and killing an RS, we will startup a new master which
575    * should become the active master and also detect that it is a failover.  The
576    * primary test passing condition will be that all regions of the enabled
577    * table are assigned and all the regions of the disabled table are not
578    * assigned.
579    * <p>
580    * The different scenarios to be tested are below:
581    * <p>
582    * <b>ZK State:  CLOSING</b>
583    * <p>A node can get into CLOSING state if</p>
584    * <ul>
585    * <li>An RS has begun to close a region
586    * </ul>
587    * <p>We will mock the scenarios</p>
588    * <ul>
589    * <li>Region was being closed but the RS died before finishing the close
590    * </ul>
591    * <b>ZK State:  OPENED</b>
592    * <p>A node can get into OPENED state if</p>
593    * <ul>
594    * <li>An RS has finished opening a region but not acknowledged by master yet
595    * </ul>
596    * <p>We will mock the scenarios</p>
597    * <ul>
598    * <li>Region of a table that should be enabled was opened by a now-dead RS
599    * <li>Region of a table that should be disabled was opened by a now-dead RS
600    * </ul>
601    * <p>
602    * <b>ZK State:  NONE</b>
603    * <p>A region could not have a transition node if</p>
604    * <ul>
605    * <li>The server hosting the region died and no master processed it
606    * </ul>
607    * <p>We will mock the scenarios</p>
608    * <ul>
609    * <li>Region of enabled table was on a dead RS that was not yet processed
610    * <li>Region of disabled table was on a dead RS that was not yet processed
611    * </ul>
612    * @throws Exception
613    */
614   @Test(timeout = 180000)
615   public void testMasterFailoverWithMockedRITOnDeadRS() throws Exception {
616 
617     final int NUM_MASTERS = 1;
618     final int NUM_RS = 2;
619 
620     // Create config to use for this cluster
621     Configuration conf = HBaseConfiguration.create();
622     // Need to drop the timeout much lower
623     conf.setInt("hbase.master.assignment.timeoutmonitor.period", 4000);
624     conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 8000);
625     conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
626     conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 2);
627 
628     // Create and start the cluster
629     HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
630     TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
631     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
632     log("Cluster started");
633 
634     // Create a ZKW to use in the test
635     ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
636         "unittest", new Abortable() {
637 
638           @Override
639           public void abort(String why, Throwable e) {
640             LOG.error("Fatal ZK Error: " + why, e);
641             org.junit.Assert.assertFalse("Fatal ZK error", true);
642           }
643 
644           @Override
645           public boolean isAborted() {
646             return false;
647           }
648 
649     });
650 
651     // get all the master threads
652     List<MasterThread> masterThreads = cluster.getMasterThreads();
653     assertEquals(1, masterThreads.size());
654 
655     // only one master thread, let's wait for it to be initialized
656     assertTrue(cluster.waitForActiveAndReadyMaster());
657     HMaster master = masterThreads.get(0).getMaster();
658     assertTrue(master.isActiveMaster());
659     assertTrue(master.isInitialized());
660 
661     // disable load balancing on this master
662     master.balanceSwitch(false);
663 
664     // create two tables in META, each with 30 regions
665     byte [] FAMILY = Bytes.toBytes("family");
666     byte[][] SPLIT_KEYS =
667         TEST_UTIL.getRegionSplitStartKeys(Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 30);
668 
669     byte [] enabledTable = Bytes.toBytes("enabledTable");
670     HTableDescriptor htdEnabled = new HTableDescriptor(enabledTable);
671     htdEnabled.addFamily(new HColumnDescriptor(FAMILY));
672     FileSystem filesystem = FileSystem.get(conf);
673     Path rootdir = filesystem.makeQualified(
674            new Path(conf.get(HConstants.HBASE_DIR)));
675     // Write the .tableinfo
676     FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdEnabled);
677     HRegionInfo hriEnabled = new HRegionInfo(htdEnabled.getName(),
678         null, null);
679     createRegion(hriEnabled, rootdir, conf, htdEnabled);
680 
681     List<HRegionInfo> enabledRegions = TEST_UTIL.createMultiRegionsInMeta(
682         TEST_UTIL.getConfiguration(), htdEnabled, SPLIT_KEYS);
683 
684     byte [] disabledTable = Bytes.toBytes("disabledTable");
685     HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable);
686     htdDisabled.addFamily(new HColumnDescriptor(FAMILY));
687     // Write the .tableinfo
688     FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdDisabled);
689     HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getName(), null, null);
690     createRegion(hriDisabled, rootdir, conf, htdDisabled);
691 
692     List<HRegionInfo> disabledRegions = TEST_UTIL.createMultiRegionsInMeta(
693         TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS);
694 
695     log("Regions in META have been created");
696 
697     // at this point we only expect 2 regions to be assigned out (catalogs)
698     assertEquals(2, cluster.countServedRegions());
699 
700     // The first RS will stay online
701     List<RegionServerThread> regionservers =
702       cluster.getRegionServerThreads();
703     HRegionServer hrs = regionservers.get(0).getRegionServer();
704 
705     // The second RS is going to be hard-killed
706     RegionServerThread hrsDeadThread = regionservers.get(1);
707     HRegionServer hrsDead = hrsDeadThread.getRegionServer();
708     ServerName deadServerName = hrsDead.getServerName();
709 
710     // we'll need some regions to already be assigned out properly on live RS
711     List<HRegionInfo> enabledAndAssignedRegions = new ArrayList<HRegionInfo>();
712     enabledAndAssignedRegions.addAll(enabledRegions.subList(0, 6));
713     enabledRegions.removeAll(enabledAndAssignedRegions);
714     List<HRegionInfo> disabledAndAssignedRegions = new ArrayList<HRegionInfo>();
715     disabledAndAssignedRegions.addAll(disabledRegions.subList(0, 6));
716     disabledRegions.removeAll(disabledAndAssignedRegions);
717 
718     // now actually assign them
719     for (HRegionInfo hri : enabledAndAssignedRegions) {
720       master.assignmentManager.regionPlans.put(hri.getEncodedName(),
721           new RegionPlan(hri, null, hrs.getServerName()));
722       master.assignRegion(hri);
723     }
724     for (HRegionInfo hri : disabledAndAssignedRegions) {
725       master.assignmentManager.regionPlans.put(hri.getEncodedName(),
726           new RegionPlan(hri, null, hrs.getServerName()));
727       master.assignRegion(hri);
728     }
729 
730     log("Waiting for assignment to finish");
731     ZKAssign.blockUntilNoRIT(zkw);
732     master.assignmentManager.waitUntilNoRegionsInTransition(60000);
733     log("Assignment completed");
734 
735     assertTrue(" Table must be enabled.", master.getAssignmentManager()
736         .getZKTable().isEnabledTable("enabledTable"));
737     // we also need regions assigned out on the dead server
738     List<HRegionInfo> enabledAndOnDeadRegions = new ArrayList<HRegionInfo>();
739     enabledAndOnDeadRegions.addAll(enabledRegions.subList(0, 6));
740     enabledRegions.removeAll(enabledAndOnDeadRegions);
741     List<HRegionInfo> disabledAndOnDeadRegions = new ArrayList<HRegionInfo>();
742     disabledAndOnDeadRegions.addAll(disabledRegions.subList(0, 6));
743     disabledRegions.removeAll(disabledAndOnDeadRegions);
744 
745     // set region plan to server to be killed and trigger assign
746     for (HRegionInfo hri : enabledAndOnDeadRegions) {
747       master.assignmentManager.regionPlans.put(hri.getEncodedName(),
748           new RegionPlan(hri, null, deadServerName));
749       master.assignRegion(hri);
750     }
751     for (HRegionInfo hri : disabledAndOnDeadRegions) {
752       master.assignmentManager.regionPlans.put(hri.getEncodedName(),
753           new RegionPlan(hri, null, deadServerName));
754       master.assignRegion(hri);
755     }
756 
757     // wait for no more RIT
758     log("Waiting for assignment to finish");
759     ZKAssign.blockUntilNoRIT(zkw);
760     master.assignmentManager.waitUntilNoRegionsInTransition(60000);
761     log("Assignment completed");
762 
763     // Due to master.assignRegion(hri) could fail to assign a region to a specified RS
764     // therefore, we need make sure that regions are in the expected RS
765     verifyRegionLocation(hrs, enabledAndAssignedRegions);
766     verifyRegionLocation(hrs, disabledAndAssignedRegions);
767     verifyRegionLocation(hrsDead, enabledAndOnDeadRegions);
768     verifyRegionLocation(hrsDead, disabledAndOnDeadRegions);
769 
770     assertTrue(" Didn't get enough regions of enabledTalbe on live rs.",
771       enabledAndAssignedRegions.size() >= 2);
772     assertTrue(" Didn't get enough regions of disalbedTable on live rs.",
773       disabledAndAssignedRegions.size() >= 2);
774     assertTrue(" Didn't get enough regions of enabledTalbe on dead rs.",
775       enabledAndOnDeadRegions.size() >= 2);
776     assertTrue(" Didn't get enough regions of disalbedTable on dead rs.",
777       disabledAndOnDeadRegions.size() >= 2);
778 
779     // Stop the master
780     log("Aborting master");
781     cluster.abortMaster(0);
782     cluster.waitOnMaster(0);
783     log("Master has aborted");
784 
785     /*
786      * Now, let's start mocking up some weird states as described in the method
787      * javadoc.
788      */
789 
790     List<HRegionInfo> regionsThatShouldBeOnline = new ArrayList<HRegionInfo>();
791     List<HRegionInfo> regionsThatShouldBeOffline = new ArrayList<HRegionInfo>();
792 
793     log("Beginning to mock scenarios");
794 
795     // Disable the disabledTable in ZK
796     ZKTable zktable = new ZKTable(zkw);
797     zktable.setDisabledTable(Bytes.toString(disabledTable));
798 
799     assertTrue(" The enabled table should be identified on master fail over.",
800         zktable.isEnabledTable("enabledTable"));
801 
802     /*
803      * ZK = CLOSING
804      */
805 
806     // Region of enabled table being closed on dead RS but not finished
807     HRegionInfo region = enabledAndOnDeadRegions.remove(0);
808     regionsThatShouldBeOnline.add(region);
809     ZKAssign.createNodeClosing(zkw, region, deadServerName);
810     LOG.debug("\n\nRegion of enabled table was CLOSING on dead RS\n" +
811         region + "\n\n");
812 
813     // Region of disabled table being closed on dead RS but not finished
814     region = disabledAndOnDeadRegions.remove(0);
815     regionsThatShouldBeOffline.add(region);
816     ZKAssign.createNodeClosing(zkw, region, deadServerName);
817     LOG.debug("\n\nRegion of disabled table was CLOSING on dead RS\n" +
818         region + "\n\n");
819 
820     /*
821      * ZK = CLOSED
822      */
823 
824     // Region of enabled on dead server gets closed but not ack'd by master
825     region = enabledAndOnDeadRegions.remove(0);
826     regionsThatShouldBeOnline.add(region);
827     int version = ZKAssign.createNodeClosing(zkw, region, deadServerName);
828     ZKAssign.transitionNodeClosed(zkw, region, deadServerName, version);
829     LOG.debug("\n\nRegion of enabled table was CLOSED on dead RS\n" +
830         region + "\n\n");
831 
832     // Region of disabled on dead server gets closed but not ack'd by master
833     region = disabledAndOnDeadRegions.remove(0);
834     regionsThatShouldBeOffline.add(region);
835     version = ZKAssign.createNodeClosing(zkw, region, deadServerName);
836     ZKAssign.transitionNodeClosed(zkw, region, deadServerName, version);
837     LOG.debug("\n\nRegion of disabled table was CLOSED on dead RS\n" +
838         region + "\n\n");
839 
840     /*
841      * ZK = OPENING
842      */
843 
844     // RS was opening a region of enabled table then died
845     region = enabledRegions.remove(0);
846     regionsThatShouldBeOnline.add(region);
847     ZKAssign.createNodeOffline(zkw, region, deadServerName);
848     ZKAssign.transitionNodeOpening(zkw, region, deadServerName);
849     LOG.debug("\n\nRegion of enabled table was OPENING on dead RS\n" +
850         region + "\n\n");
851 
852     // RS was opening a region of disabled table then died
853     region = disabledRegions.remove(0);
854     regionsThatShouldBeOffline.add(region);
855     ZKAssign.createNodeOffline(zkw, region, deadServerName);
856     ZKAssign.transitionNodeOpening(zkw, region, deadServerName);
857     LOG.debug("\n\nRegion of disabled table was OPENING on dead RS\n" +
858         region + "\n\n");
859 
860     /*
861      * ZK = OPENED
862      */
863 
864     // Region of enabled table was opened on dead RS
865     region = enabledRegions.remove(0);
866     regionsThatShouldBeOnline.add(region);
867     ZKAssign.createNodeOffline(zkw, region, deadServerName);
868     hrsDead.openRegion(region);
869     while (true) {
870       RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName());
871       if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) {
872         break;
873       }
874       Thread.sleep(100);
875     }
876     LOG.debug("\n\nRegion of enabled table was OPENED on dead RS\n" +
877         region + "\n\n");
878 
879     // Region of disabled table was opened on dead RS
880     region = disabledRegions.remove(0);
881     regionsThatShouldBeOffline.add(region);
882     ZKAssign.createNodeOffline(zkw, region, deadServerName);
883     hrsDead.openRegion(region);
884     while (true) {
885       RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName());
886       if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) {
887         break;
888       }
889       Thread.sleep(100);
890     }
891     LOG.debug("\n\nRegion of disabled table was OPENED on dead RS\n" +
892         region + "\n\n");
893 
894     /*
895      * ZK = NONE
896      */
897 
898     // Region of enabled table was open at steady-state on dead RS
899     region = enabledRegions.remove(0);
900     regionsThatShouldBeOnline.add(region);
901     ZKAssign.createNodeOffline(zkw, region, deadServerName);
902     hrsDead.openRegion(region);
903     while (true) {
904       RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName());
905       if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) {
906         ZKAssign.deleteOpenedNode(zkw, region.getEncodedName());
907         break;
908       }
909       Thread.sleep(100);
910     }
911     LOG.debug("\n\nRegion of enabled table was open at steady-state on dead RS"
912         + "\n" + region + "\n\n");
913 
914     // Region of disabled table was open at steady-state on dead RS
915     region = disabledRegions.remove(0);
916     regionsThatShouldBeOffline.add(region);
917     ZKAssign.createNodeOffline(zkw, region, deadServerName);
918     hrsDead.openRegion(region);
919     while (true) {
920       RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName());
921       if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) {
922         ZKAssign.deleteOpenedNode(zkw, region.getEncodedName());
923         break;
924       }
925       Thread.sleep(100);
926     }
927     LOG.debug("\n\nRegion of disabled table was open at steady-state on dead RS"
928       + "\n" + region + "\n\n");
929 
930     /*
931      * DONE MOCKING
932      */
933 
934     log("Done mocking data up in ZK");
935 
936     // Kill the RS that had a hard death
937     log("Killing RS " + deadServerName);
938     hrsDead.abort("Killing for unit test");
939     log("RS " + deadServerName + " killed");
940 
941     // Start up a new master.  Wait until regionserver is completely down
942     // before starting new master because of hbase-4511.
943     while (hrsDeadThread.isAlive()) {
944       Threads.sleep(10);
945     }
946     log("Starting up a new master");
947     master = cluster.startMaster().getMaster();
948     log("Waiting for master to be ready");
949     assertTrue(cluster.waitForActiveAndReadyMaster());
950     log("Master is ready");
951 
952     // Let's add some weird states to master in-memory state
953 
954     // After HBASE-3181, we need to have some ZK state if we're PENDING_OPEN
955     // b/c it is impossible for us to get into this state w/o a zk node
956     // this is not true of PENDING_CLOSE
957 
958     // PENDING_OPEN and enabled
959     region = enabledRegions.remove(0);
960     regionsThatShouldBeOnline.add(region);
961     master.assignmentManager.regionsInTransition.put(region.getEncodedName(),
962         new RegionState(region, RegionState.State.PENDING_OPEN, 0, null));
963     ZKAssign.createNodeOffline(zkw, region, master.getServerName());
964     // PENDING_OPEN and disabled
965     region = disabledRegions.remove(0);
966     regionsThatShouldBeOffline.add(region);
967     master.assignmentManager.regionsInTransition.put(region.getEncodedName(),
968         new RegionState(region, RegionState.State.PENDING_OPEN, 0, null));
969     ZKAssign.createNodeOffline(zkw, region, master.getServerName());
970     // This test is bad.  It puts up a PENDING_CLOSE but doesn't say what
971     // server we were PENDING_CLOSE against -- i.e. an entry in
972     // AssignmentManager#regions.  W/o a server, we NPE trying to resend close.
973     // In past, there was wonky logic that had us reassign region if no server
974     // at tail of the unassign.  This was removed.  Commenting out for now.
975     // TODO: Remove completely.
976     /*
977     // PENDING_CLOSE and enabled
978     region = enabledRegions.remove(0);
979     LOG.info("Setting PENDING_CLOSE enabled " + region.getEncodedName());
980     regionsThatShouldBeOnline.add(region);
981     master.assignmentManager.regionsInTransition.put(region.getEncodedName(),
982       new RegionState(region, RegionState.State.PENDING_CLOSE, 0));
983     // PENDING_CLOSE and disabled
984     region = disabledRegions.remove(0);
985     LOG.info("Setting PENDING_CLOSE disabled " + region.getEncodedName());
986     regionsThatShouldBeOffline.add(region);
987     master.assignmentManager.regionsInTransition.put(region.getEncodedName(),
988       new RegionState(region, RegionState.State.PENDING_CLOSE, 0));
989       */
990 
991     // Failover should be completed, now wait for no RIT
992     log("Waiting for no more RIT");
993     ZKAssign.blockUntilNoRIT(zkw);
994     log("No more RIT in ZK");
995     long now = System.currentTimeMillis();
996     final long maxTime = 120000;
997     boolean done = master.assignmentManager.waitUntilNoRegionsInTransition(maxTime);
998     if (!done) {
999       LOG.info("rit=" + master.assignmentManager.getRegionsInTransition());
1000     }
1001     long elapsed = System.currentTimeMillis() - now;
1002     assertTrue("Elapsed=" + elapsed + ", maxTime=" + maxTime + ", done=" + done,
1003       elapsed < maxTime);
1004     log("No more RIT in RIT map, doing final test verification");
1005 
1006     // Grab all the regions that are online across RSs
1007     Set<HRegionInfo> onlineRegions = new TreeSet<HRegionInfo>();
1008     for (JVMClusterUtil.RegionServerThread rst :
1009         cluster.getRegionServerThreads()) {
1010       try {
1011         onlineRegions.addAll(rst.getRegionServer().getOnlineRegions());
1012       } catch (org.apache.hadoop.hbase.regionserver.RegionServerStoppedException e) {
1013         LOG.info("Got RegionServerStoppedException", e);
1014       }
1015     }
1016 
1017     // Now, everything that should be online should be online
1018     for (HRegionInfo hri : regionsThatShouldBeOnline) {
1019       assertTrue("region=" + hri.getRegionNameAsString(), onlineRegions.contains(hri));
1020     }
1021 
1022     // Everything that should be offline should not be online
1023     for (HRegionInfo hri : regionsThatShouldBeOffline) {
1024       assertFalse(onlineRegions.contains(hri));
1025     }
1026 
1027     log("Done with verification, all passed, shutting down cluster");
1028 
1029     // Done, shutdown the cluster
1030     TEST_UTIL.shutdownMiniCluster();
1031   }
1032 
1033   @Test(timeout = 180000)
1034   public void testRSKilledWithMockedOpeningRITGoingToDeadRS() throws Exception {
1035     final int NUM_MASTERS = 1;
1036     final int NUM_RS = 2;
1037 
1038     // Create config to use for this cluster
1039     Configuration conf = HBaseConfiguration.create();
1040     // Need to drop the timeout much lower
1041     conf.setInt("hbase.master.assignment.timeoutmonitor.period", 10000);
1042     conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 30000);
1043     conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
1044     conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 2);
1045 
1046     // Create and start the cluster
1047     HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
1048     TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
1049     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1050     log("Cluster started");
1051 
1052     // Create a ZKW to use in the test
1053     ZooKeeperWatcher zkw =
1054         new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), "unittest", new Abortable() {
1055 
1056           @Override
1057           public void abort(String why, Throwable e) {
1058             LOG.error("Fatal ZK Error: " + why, e);
1059             org.junit.Assert.assertFalse("Fatal ZK error", true);
1060           }
1061 
1062           @Override
1063           public boolean isAborted() {
1064             return false;
1065           }
1066 
1067         });
1068 
1069     // get all the master threads
1070     List<MasterThread> masterThreads = cluster.getMasterThreads();
1071     assertEquals(1, masterThreads.size());
1072 
1073     // only one master thread, let's wait for it to be initialized
1074     assertTrue(cluster.waitForActiveAndReadyMaster());
1075     HMaster master = masterThreads.get(0).getMaster();
1076     assertTrue(master.isActiveMaster());
1077     assertTrue(master.isInitialized());
1078 
1079     // disable load balancing on this master
1080     master.balanceSwitch(false);
1081 
1082     // create two tables in META, each with 30 regions
1083     byte[] FAMILY = Bytes.toBytes("family");
1084     byte[][] SPLIT_KEYS =
1085         TEST_UTIL.getRegionSplitStartKeys(Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 15);
1086 
1087     FileSystem filesystem = FileSystem.get(conf);
1088     Path rootdir = filesystem.makeQualified(new Path(conf.get(HConstants.HBASE_DIR)));
1089 
1090     byte[] disabledTable = Bytes.toBytes("disabledTable");
1091     HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable);
1092     htdDisabled.addFamily(new HColumnDescriptor(FAMILY));
1093     // Write the .tableinfo
1094     FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdDisabled);
1095     HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getName(), null, null);
1096     createRegion(hriDisabled, rootdir, conf, htdDisabled);
1097 
1098     List<HRegionInfo> tableRegions =
1099         TEST_UTIL.createMultiRegionsInMeta(TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS);
1100 
1101     log("Regions in META have been created");
1102 
1103     // at this point we only expect 2 regions to be assigned out (catalogs)
1104     assertEquals(2, cluster.countServedRegions());
1105 
1106     // The first RS will stay online
1107     List<RegionServerThread> regionservers = cluster.getRegionServerThreads();
1108     HRegionServer hrs = regionservers.get(0).getRegionServer();
1109 
1110     // The second RS is going to be hard-killed
1111     RegionServerThread hrsDeadThread = regionservers.get(1);
1112     HRegionServer hrsDead = hrsDeadThread.getRegionServer();
1113     ServerName deadServerName = hrsDead.getServerName();
1114 
1115     // we'll need some regions to already be assigned out properly on live RS
1116     List<HRegionInfo> assignedRegionsOnLiveRS = new ArrayList<HRegionInfo>();
1117     assignedRegionsOnLiveRS.addAll(tableRegions.subList(0, 3));
1118     tableRegions.removeAll(assignedRegionsOnLiveRS);
1119 
1120     // now actually assign them
1121     for (HRegionInfo hri : assignedRegionsOnLiveRS) {
1122       master.assignmentManager.regionPlans.put(hri.getEncodedName(),
1123         new RegionPlan(hri, null, hrs.getServerName()));
1124       master.assignRegion(hri);
1125     }
1126 
1127     log("Waiting for assignment to finish");
1128     ZKAssign.blockUntilNoRIT(zkw);
1129     master.assignmentManager.waitUntilNoRegionsInTransition(60000);
1130     log("Assignment completed");
1131 
1132     // Due to master.assignRegion(hri) could fail to assign a region to a specified RS
1133     // therefore, we need make sure that regions are in the expected RS
1134     verifyRegionLocation(hrs, assignedRegionsOnLiveRS);
1135 
1136     assertTrue(" Table must be enabled.", master.getAssignmentManager().getZKTable()
1137         .isEnabledTable("disabledTable"));
1138 
1139     assertTrue(" Didn't get enough regions of enabledTalbe on live rs.",
1140       assignedRegionsOnLiveRS.size() >= 1);
1141 
1142     // Disable the disabledTable in ZK
1143     ZKTable zktable = master.assignmentManager.getZKTable();
1144     zktable.setDisablingTable("disabledTable");
1145 
1146     // RS was opening a region of disabled table then died
1147     HRegionInfo region = assignedRegionsOnLiveRS.remove(0);
1148     master.assignmentManager.regionOffline(region);
1149     master.assignmentManager.regionsInTransition.put(region.getEncodedName(), new RegionState(
1150         region, RegionState.State.OPENING, System.currentTimeMillis(), deadServerName));
1151     ZKAssign.createNodeOffline(zkw, region, deadServerName);
1152     ZKAssign.transitionNodeOpening(zkw, region, deadServerName);
1153 
1154     // Kill the RS that had a hard death
1155     log("Killing RS " + deadServerName);
1156     hrsDead.abort("Killing for unit test");
1157     while (hrsDeadThread.isAlive()) {
1158       Threads.sleep(10);
1159     }
1160     log("RS " + deadServerName + " killed");
1161 
1162     log("Waiting for no more RIT");
1163     ZKAssign.blockUntilNoRIT(zkw);
1164     log("No more RIT in ZK");
1165     assertTrue(master.assignmentManager.waitUntilNoRegionsInTransition(120000));
1166   }
1167 
1168   /**
1169    * Verify regions are on the expected region server
1170    */
1171   private void verifyRegionLocation(HRegionServer hrs, List<HRegionInfo> regions)
1172       throws IOException {
1173     List<HRegionInfo> tmpOnlineRegions = hrs.getOnlineRegions();
1174     Iterator<HRegionInfo> itr = regions.iterator();
1175     while (itr.hasNext()) {
1176       HRegionInfo tmp = itr.next();
1177       if (!tmpOnlineRegions.contains(tmp)) {
1178         itr.remove();
1179       }
1180     }
1181   }
1182 
1183   HRegion createRegion(final HRegionInfo  hri, final Path rootdir, final Configuration c,
1184       final HTableDescriptor htd)
1185   throws IOException {
1186     HRegion r = HRegion.createHRegion(hri, rootdir, c, htd);
1187     // The above call to create a region will create an hlog file.  Each
1188     // log file create will also create a running thread to do syncing.  We need
1189     // to close out this log else we will have a running thread trying to sync
1190     // the file system continuously which is ugly when dfs is taken away at the
1191     // end of the test.
1192     HRegion.closeHRegion(r);
1193     return r;
1194   }
1195 
1196   // TODO: Next test to add is with testing permutations of the RIT or the RS
1197   //       killed are hosting ROOT and META regions.
1198 
1199   private void log(String string) {
1200     LOG.info("\n\n" + string + " \n\n");
1201   }
1202 
1203   @org.junit.Rule
1204   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
1205     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
1206 }
1207