View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertTrue;
25  
26  import java.io.IOException;
27  import java.util.ArrayList;
28  import java.util.Iterator;
29  import java.util.List;
30  import java.util.Set;
31  import java.util.TreeSet;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.fs.Path;
38  import org.apache.hadoop.hbase.Abortable;
39  import org.apache.hadoop.hbase.ClusterStatus;
40  import org.apache.hadoop.hbase.HBaseConfiguration;
41  import org.apache.hadoop.hbase.HBaseTestingUtility;
42  import org.apache.hadoop.hbase.HColumnDescriptor;
43  import org.apache.hadoop.hbase.HConstants;
44  import org.apache.hadoop.hbase.HRegionInfo;
45  import org.apache.hadoop.hbase.HTableDescriptor;
46  import org.apache.hadoop.hbase.LargeTests;
47  import org.apache.hadoop.hbase.MiniHBaseCluster;
48  import org.apache.hadoop.hbase.RegionTransition;
49  import org.apache.hadoop.hbase.ServerName;
50  import org.apache.hadoop.hbase.TableName;
51  import org.apache.hadoop.hbase.catalog.MetaEditor;
52  import org.apache.hadoop.hbase.client.HTable;
53  import org.apache.hadoop.hbase.executor.EventType;
54  import org.apache.hadoop.hbase.master.RegionState.State;
55  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
56  import org.apache.hadoop.hbase.regionserver.HRegion;
57  import org.apache.hadoop.hbase.regionserver.HRegionServer;
58  import org.apache.hadoop.hbase.regionserver.RegionMergeTransaction;
59  import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
60  import org.apache.hadoop.hbase.util.Bytes;
61  import org.apache.hadoop.hbase.util.FSTableDescriptors;
62  import org.apache.hadoop.hbase.util.FSUtils;
63  import org.apache.hadoop.hbase.util.JVMClusterUtil;
64  import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
65  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
66  import org.apache.hadoop.hbase.util.Threads;
67  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
68  import org.apache.hadoop.hbase.zookeeper.ZKTable;
69  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
70  import org.apache.zookeeper.data.Stat;
71  import org.junit.Test;
72  import org.junit.experimental.categories.Category;
73  
74  @Category(LargeTests.class)
75  public class TestMasterFailover {
76    private static final Log LOG = LogFactory.getLog(TestMasterFailover.class);
77  
78    /**
79     * Complex test of master failover that tests as many permutations of the
80     * different possible states that regions in transition could be in within ZK.
81     * <p>
82     * This tests the proper handling of these states by the failed-over master
83     * and includes a thorough testing of the timeout code as well.
84     * <p>
85     * Starts with a single master and three regionservers.
86     * <p>
87     * Creates two tables, enabledTable and disabledTable, each containing 5
88     * regions.  The disabledTable is then disabled.
89     * <p>
90     * After reaching steady-state, the master is killed.  We then mock several
91     * states in ZK.
92     * <p>
93     * After mocking them, we will startup a new master which should become the
94     * active master and also detect that it is a failover.  The primary test
95     * passing condition will be that all regions of the enabled table are
96     * assigned and all the regions of the disabled table are not assigned.
97     * <p>
98     * The different scenarios to be tested are below:
99     * <p>
100    * <b>ZK State:  OFFLINE</b>
101    * <p>A node can get into OFFLINE state if</p>
102    * <ul>
103    * <li>An RS fails to open a region, so it reverts the state back to OFFLINE
104    * <li>The Master is assigning the region to a RS before it sends RPC
105    * </ul>
106    * <p>We will mock the scenarios</p>
107    * <ul>
108    * <li>Master has assigned an enabled region but RS failed so a region is
109    *     not assigned anywhere and is sitting in ZK as OFFLINE</li>
110    * <li>This seems to cover both cases?</li>
111    * </ul>
112    * <p>
113    * <b>ZK State:  CLOSING</b>
114    * <p>A node can get into CLOSING state if</p>
115    * <ul>
116    * <li>An RS has begun to close a region
117    * </ul>
118    * <p>We will mock the scenarios</p>
119    * <ul>
120    * <li>Region of enabled table was being closed but did not complete
121    * <li>Region of disabled table was being closed but did not complete
122    * </ul>
123    * <p>
124    * <b>ZK State:  CLOSED</b>
125    * <p>A node can get into CLOSED state if</p>
126    * <ul>
127    * <li>An RS has completed closing a region but not acknowledged by master yet
128    * </ul>
129    * <p>We will mock the scenarios</p>
130    * <ul>
131    * <li>Region of a table that should be enabled was closed on an RS
132    * <li>Region of a table that should be disabled was closed on an RS
133    * </ul>
134    * <p>
135    * <b>ZK State:  OPENING</b>
136    * <p>A node can get into OPENING state if</p>
137    * <ul>
138    * <li>An RS has begun to open a region
139    * </ul>
140    * <p>We will mock the scenarios</p>
141    * <ul>
142    * <li>RS was opening a region of enabled table but never finishes
143    * </ul>
144    * <p>
145    * <b>ZK State:  OPENED</b>
146    * <p>A node can get into OPENED state if</p>
147    * <ul>
148    * <li>An RS has finished opening a region but not acknowledged by master yet
149    * </ul>
150    * <p>We will mock the scenarios</p>
151    * <ul>
152    * <li>Region of a table that should be enabled was opened on an RS
153    * <li>Region of a table that should be disabled was opened on an RS
154    * </ul>
155    * @throws Exception
156    */
157   @Test (timeout=240000)
158   public void testMasterFailoverWithMockedRIT() throws Exception {
159 
160     final int NUM_MASTERS = 1;
161     final int NUM_RS = 3;
162 
163     // Create config to use for this cluster
164     Configuration conf = HBaseConfiguration.create();
165     conf.setBoolean("hbase.assignment.usezk", true);
166 
167     // Start the cluster
168     HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
169     TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
170     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
171     log("Cluster started");
172 
173     // Create a ZKW to use in the test
174     ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
175 
176     // get all the master threads
177     List<MasterThread> masterThreads = cluster.getMasterThreads();
178     assertEquals(1, masterThreads.size());
179 
180     // only one master thread, let's wait for it to be initialized
181     assertTrue(cluster.waitForActiveAndReadyMaster());
182     HMaster master = masterThreads.get(0).getMaster();
183     assertTrue(master.isActiveMaster());
184     assertTrue(master.isInitialized());
185 
186     // disable load balancing on this master
187     master.balanceSwitch(false);
188 
189     // create two tables in META, each with 10 regions
190     byte [] FAMILY = Bytes.toBytes("family");
191     byte [][] SPLIT_KEYS = new byte [][] {
192         new byte[0], Bytes.toBytes("aaa"), Bytes.toBytes("bbb"),
193         Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"),
194         Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
195         Bytes.toBytes("iii"), Bytes.toBytes("jjj")
196     };
197 
198     byte [] enabledTable = Bytes.toBytes("enabledTable");
199     HTableDescriptor htdEnabled = new HTableDescriptor(TableName.valueOf(enabledTable));
200     htdEnabled.addFamily(new HColumnDescriptor(FAMILY));
201 
202     FileSystem filesystem = FileSystem.get(conf);
203     Path rootdir = FSUtils.getRootDir(conf);
204     FSTableDescriptors fstd = new FSTableDescriptors(conf, filesystem, rootdir);
205     // Write the .tableinfo
206     fstd.createTableDescriptor(htdEnabled);
207 
208     HRegionInfo hriEnabled = new HRegionInfo(htdEnabled.getTableName(), null, null);
209     createRegion(hriEnabled, rootdir, conf, htdEnabled);
210 
211     List<HRegionInfo> enabledRegions = TEST_UTIL.createMultiRegionsInMeta(
212         TEST_UTIL.getConfiguration(), htdEnabled, SPLIT_KEYS);
213 
214     TableName disabledTable = TableName.valueOf("disabledTable");
215     HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable);
216     htdDisabled.addFamily(new HColumnDescriptor(FAMILY));
217     // Write the .tableinfo
218     fstd.createTableDescriptor(htdDisabled);
219     HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getTableName(), null, null);
220     createRegion(hriDisabled, rootdir, conf, htdDisabled);
221     List<HRegionInfo> disabledRegions = TEST_UTIL.createMultiRegionsInMeta(
222         TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS);
223 
224     TableName tableWithMergingRegions = TableName.valueOf("tableWithMergingRegions");
225     TEST_UTIL.createTable(tableWithMergingRegions, FAMILY, new byte [][] {Bytes.toBytes("m")});
226 
227     log("Regions in hbase:meta and namespace have been created");
228 
229     // at this point we only expect 4 regions to be assigned out
230     // (catalogs and namespace, + 2 merging regions)
231     assertEquals(4, cluster.countServedRegions());
232 
233     // Move merging regions to the same region server
234     AssignmentManager am = master.getAssignmentManager();
235     RegionStates regionStates = am.getRegionStates();
236     List<HRegionInfo> mergingRegions = regionStates.getRegionsOfTable(tableWithMergingRegions);
237     assertEquals(2, mergingRegions.size());
238     HRegionInfo a = mergingRegions.get(0);
239     HRegionInfo b = mergingRegions.get(1);
240     HRegionInfo newRegion = RegionMergeTransaction.getMergedRegionInfo(a, b);
241     ServerName mergingServer = regionStates.getRegionServerOfRegion(a);
242     ServerName serverB = regionStates.getRegionServerOfRegion(b);
243     if (!serverB.equals(mergingServer)) {
244       RegionPlan plan = new RegionPlan(b, serverB, mergingServer);
245       am.balance(plan);
246       assertTrue(am.waitForAssignment(b));
247     }
248 
249     // Let's just assign everything to first RS
250     HRegionServer hrs = cluster.getRegionServer(0);
251     ServerName serverName = hrs.getServerName();
252     HRegionInfo closingRegion = enabledRegions.remove(0);
253     // we'll need some regions to already be assigned out properly on live RS
254     List<HRegionInfo> enabledAndAssignedRegions = new ArrayList<HRegionInfo>();
255     enabledAndAssignedRegions.add(enabledRegions.remove(0));
256     enabledAndAssignedRegions.add(enabledRegions.remove(0));
257     enabledAndAssignedRegions.add(closingRegion);
258 
259     List<HRegionInfo> disabledAndAssignedRegions = new ArrayList<HRegionInfo>();
260     disabledAndAssignedRegions.add(disabledRegions.remove(0));
261     disabledAndAssignedRegions.add(disabledRegions.remove(0));
262 
263     // now actually assign them
264     for (HRegionInfo hri : enabledAndAssignedRegions) {
265       master.assignmentManager.addPlan(hri.getEncodedName(),
266           new RegionPlan(hri, null, serverName));
267       master.assignRegion(hri);
268     }
269 
270     for (HRegionInfo hri : disabledAndAssignedRegions) {
271       master.assignmentManager.addPlan(hri.getEncodedName(),
272           new RegionPlan(hri, null, serverName));
273       master.assignRegion(hri);
274     }
275 
276     // wait for no more RIT
277     log("Waiting for assignment to finish");
278     ZKAssign.blockUntilNoRIT(zkw);
279     log("Assignment completed");
280 
281     // Stop the master
282     log("Aborting master");
283     cluster.abortMaster(0);
284     cluster.waitOnMaster(0);
285     log("Master has aborted");
286 
287     /*
288      * Now, let's start mocking up some weird states as described in the method
289      * javadoc.
290      */
291 
292     List<HRegionInfo> regionsThatShouldBeOnline = new ArrayList<HRegionInfo>();
293     List<HRegionInfo> regionsThatShouldBeOffline = new ArrayList<HRegionInfo>();
294 
295     log("Beginning to mock scenarios");
296 
297     // Disable the disabledTable in ZK
298     ZKTable zktable = new ZKTable(zkw);
299     zktable.setDisabledTable(disabledTable);
300 
301     /*
302      *  ZK = OFFLINE
303      */
304 
305     // Region that should be assigned but is not and is in ZK as OFFLINE
306     // Cause: This can happen if the master crashed after creating the znode but before sending the
307     //  request to the region server
308     HRegionInfo region = enabledRegions.remove(0);
309     regionsThatShouldBeOnline.add(region);
310     ZKAssign.createNodeOffline(zkw, region, serverName);
311 
312     /*
313      * ZK = CLOSING
314      */
315     // Cause: Same as offline.
316     regionsThatShouldBeOnline.add(closingRegion);
317     ZKAssign.createNodeClosing(zkw, closingRegion, serverName);
318 
319     /*
320      * ZK = CLOSED
321      */
322 
323     // Region of enabled table closed but not ack
324     //Cause: Master was down while the region server updated the ZK status.
325     region = enabledRegions.remove(0);
326     regionsThatShouldBeOnline.add(region);
327     int version = ZKAssign.createNodeClosing(zkw, region, serverName);
328     ZKAssign.transitionNodeClosed(zkw, region, serverName, version);
329 
330     // Region of disabled table closed but not ack
331     region = disabledRegions.remove(0);
332     regionsThatShouldBeOffline.add(region);
333     version = ZKAssign.createNodeClosing(zkw, region, serverName);
334     ZKAssign.transitionNodeClosed(zkw, region, serverName, version);
335 
336     /*
337      * ZK = OPENED
338      */
339 
340     // Region of enabled table was opened on RS
341     // Cause: as offline
342     region = enabledRegions.remove(0);
343     regionsThatShouldBeOnline.add(region);
344     ZKAssign.createNodeOffline(zkw, region, serverName);
345     ProtobufUtil.openRegion(hrs, hrs.getServerName(), region);
346     while (true) {
347       byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
348       RegionTransition rt = RegionTransition.parseFrom(bytes);
349       if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
350         break;
351       }
352       Thread.sleep(100);
353     }
354 
355     // Region of disable table was opened on RS
356     // Cause: Master failed while updating the status for this region server.
357     region = disabledRegions.remove(0);
358     regionsThatShouldBeOffline.add(region);
359     ZKAssign.createNodeOffline(zkw, region, serverName);
360     ProtobufUtil.openRegion(hrs, hrs.getServerName(), region);
361     while (true) {
362       byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
363       RegionTransition rt = RegionTransition.parseFrom(bytes);
364       if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
365         break;
366       }
367       Thread.sleep(100);
368     }
369 
370     /*
371      * ZK = MERGING
372      */
373 
374     // Regions of table of merging regions
375     // Cause: Master was down while merging was going on
376     RegionMergeTransaction.createNodeMerging(
377       zkw, newRegion, mergingServer, a, b);
378 
379     /*
380      * ZK = NONE
381      */
382 
383     /*
384      * DONE MOCKING
385      */
386 
387     log("Done mocking data up in ZK");
388 
389     // Start up a new master
390     log("Starting up a new master");
391     master = cluster.startMaster().getMaster();
392     log("Waiting for master to be ready");
393     cluster.waitForActiveAndReadyMaster();
394     log("Master is ready");
395 
396     // Get new region states since master restarted
397     regionStates = master.getAssignmentManager().getRegionStates();
398     // Merging region should remain merging
399     assertTrue(regionStates.isRegionInState(a, State.MERGING));
400     assertTrue(regionStates.isRegionInState(b, State.MERGING));
401     assertTrue(regionStates.isRegionInState(newRegion, State.MERGING_NEW));
402     // Now remove the faked merging znode, merging regions should be
403     // offlined automatically, otherwise it is a bug in AM.
404     ZKAssign.deleteNodeFailSilent(zkw, newRegion);
405 
406     // Failover should be completed, now wait for no RIT
407     log("Waiting for no more RIT");
408     ZKAssign.blockUntilNoRIT(zkw);
409     log("No more RIT in ZK, now doing final test verification");
410 
411     // Grab all the regions that are online across RSs
412     Set<HRegionInfo> onlineRegions = new TreeSet<HRegionInfo>();
413     for (JVMClusterUtil.RegionServerThread rst :
414       cluster.getRegionServerThreads()) {
415       onlineRegions.addAll(ProtobufUtil.getOnlineRegions(rst.getRegionServer()));
416     }
417 
418     // Now, everything that should be online should be online
419     for (HRegionInfo hri : regionsThatShouldBeOnline) {
420       assertTrue(onlineRegions.contains(hri));
421     }
422 
423     // Everything that should be offline should not be online
424     for (HRegionInfo hri : regionsThatShouldBeOffline) {
425       if (onlineRegions.contains(hri)) {
426        LOG.debug(hri);
427       }
428       assertFalse(onlineRegions.contains(hri));
429     }
430 
431     log("Done with verification, all passed, shutting down cluster");
432 
433     // Done, shutdown the cluster
434     TEST_UTIL.shutdownMiniCluster();
435   }
436 
437   /**
438    * Complex test of master failover that tests as many permutations of the
439    * different possible states that regions in transition could be in within ZK
440    * pointing to an RS that has died while no master is around to process it.
441    * <p>
442    * This tests the proper handling of these states by the failed-over master
443    * and includes a thorough testing of the timeout code as well.
444    * <p>
445    * Starts with a single master and two regionservers.
446    * <p>
447    * Creates two tables, enabledTable and disabledTable, each containing 5
448    * regions.  The disabledTable is then disabled.
449    * <p>
450    * After reaching steady-state, the master is killed.  We then mock several
451    * states in ZK.  And one of the RS will be killed.
452    * <p>
453    * After mocking them and killing an RS, we will startup a new master which
454    * should become the active master and also detect that it is a failover.  The
455    * primary test passing condition will be that all regions of the enabled
456    * table are assigned and all the regions of the disabled table are not
457    * assigned.
458    * <p>
459    * The different scenarios to be tested are below:
460    * <p>
461    * <b>ZK State:  CLOSING</b>
462    * <p>A node can get into CLOSING state if</p>
463    * <ul>
464    * <li>An RS has begun to close a region
465    * </ul>
466    * <p>We will mock the scenarios</p>
467    * <ul>
468    * <li>Region was being closed but the RS died before finishing the close
469    * </ul>
470    * <b>ZK State:  OPENED</b>
471    * <p>A node can get into OPENED state if</p>
472    * <ul>
473    * <li>An RS has finished opening a region but not acknowledged by master yet
474    * </ul>
475    * <p>We will mock the scenarios</p>
476    * <ul>
477    * <li>Region of a table that should be enabled was opened by a now-dead RS
478    * <li>Region of a table that should be disabled was opened by a now-dead RS
479    * </ul>
480    * <p>
481    * <b>ZK State:  NONE</b>
482    * <p>A region could not have a transition node if</p>
483    * <ul>
484    * <li>The server hosting the region died and no master processed it
485    * </ul>
486    * <p>We will mock the scenarios</p>
487    * <ul>
488    * <li>Region of enabled table was on a dead RS that was not yet processed
489    * <li>Region of disabled table was on a dead RS that was not yet processed
490    * </ul>
491    * @throws Exception
492    */
493   @Test (timeout=180000)
494   public void testMasterFailoverWithMockedRITOnDeadRS() throws Exception {
495 
496     final int NUM_MASTERS = 1;
497     final int NUM_RS = 2;
498 
499     // Create and start the cluster
500     HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
501     Configuration conf = TEST_UTIL.getConfiguration();
502     conf.setBoolean("hbase.assignment.usezk", true);
503 
504     conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
505     conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 2);
506     TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
507     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
508     log("Cluster started");
509 
510     // Create a ZKW to use in the test
511     ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
512         "unittest", new Abortable() {
513 
514           @Override
515           public void abort(String why, Throwable e) {
516             LOG.error("Fatal ZK Error: " + why, e);
517             org.junit.Assert.assertFalse("Fatal ZK error", true);
518           }
519 
520           @Override
521           public boolean isAborted() {
522             return false;
523           }
524 
525     });
526 
527     // get all the master threads
528     List<MasterThread> masterThreads = cluster.getMasterThreads();
529     assertEquals(1, masterThreads.size());
530 
531     // only one master thread, let's wait for it to be initialized
532     assertTrue(cluster.waitForActiveAndReadyMaster());
533     HMaster master = masterThreads.get(0).getMaster();
534     assertTrue(master.isActiveMaster());
535     assertTrue(master.isInitialized());
536 
537     // disable load balancing on this master
538     master.balanceSwitch(false);
539 
540     // create two tables in META, each with 30 regions
541     byte [] FAMILY = Bytes.toBytes("family");
542     byte[][] SPLIT_KEYS =
543         TEST_UTIL.getRegionSplitStartKeys(Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 30);
544 
545     byte [] enabledTable = Bytes.toBytes("enabledTable");
546     HTableDescriptor htdEnabled = new HTableDescriptor(TableName.valueOf(enabledTable));
547     htdEnabled.addFamily(new HColumnDescriptor(FAMILY));
548     FileSystem filesystem = FileSystem.get(conf);
549     Path rootdir = FSUtils.getRootDir(conf);
550     FSTableDescriptors fstd = new FSTableDescriptors(conf, filesystem, rootdir);
551     // Write the .tableinfo
552     fstd.createTableDescriptor(htdEnabled);
553     HRegionInfo hriEnabled = new HRegionInfo(htdEnabled.getTableName(),
554         null, null);
555     createRegion(hriEnabled, rootdir, conf, htdEnabled);
556 
557     List<HRegionInfo> enabledRegions = TEST_UTIL.createMultiRegionsInMeta(
558         TEST_UTIL.getConfiguration(), htdEnabled, SPLIT_KEYS);
559 
560     TableName disabledTable =
561         TableName.valueOf("disabledTable");
562     HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable);
563     htdDisabled.addFamily(new HColumnDescriptor(FAMILY));
564     // Write the .tableinfo
565     fstd.createTableDescriptor(htdDisabled);
566     HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getTableName(), null, null);
567     createRegion(hriDisabled, rootdir, conf, htdDisabled);
568 
569     List<HRegionInfo> disabledRegions = TEST_UTIL.createMultiRegionsInMeta(
570         TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS);
571 
572     log("Regions in hbase:meta and Namespace have been created");
573 
574     // at this point we only expect 2 regions to be assigned out (catalogs and namespace  )
575     assertEquals(2, cluster.countServedRegions());
576 
577     // The first RS will stay online
578     List<RegionServerThread> regionservers =
579       cluster.getRegionServerThreads();
580     HRegionServer hrs = regionservers.get(0).getRegionServer();
581 
582     // The second RS is going to be hard-killed
583     RegionServerThread hrsDeadThread = regionservers.get(1);
584     HRegionServer hrsDead = hrsDeadThread.getRegionServer();
585     ServerName deadServerName = hrsDead.getServerName();
586 
587     // we'll need some regions to already be assigned out properly on live RS
588     List<HRegionInfo> enabledAndAssignedRegions = new ArrayList<HRegionInfo>();
589     enabledAndAssignedRegions.addAll(enabledRegions.subList(0, 6));
590     enabledRegions.removeAll(enabledAndAssignedRegions);
591     List<HRegionInfo> disabledAndAssignedRegions = new ArrayList<HRegionInfo>();
592     disabledAndAssignedRegions.addAll(disabledRegions.subList(0, 6));
593     disabledRegions.removeAll(disabledAndAssignedRegions);
594 
595     // now actually assign them
596     for (HRegionInfo hri : enabledAndAssignedRegions) {
597       master.assignmentManager.addPlan(hri.getEncodedName(),
598           new RegionPlan(hri, null, hrs.getServerName()));
599       master.assignRegion(hri);
600     }
601     for (HRegionInfo hri : disabledAndAssignedRegions) {
602       master.assignmentManager.addPlan(hri.getEncodedName(),
603           new RegionPlan(hri, null, hrs.getServerName()));
604       master.assignRegion(hri);
605     }
606 
607     log("Waiting for assignment to finish");
608     ZKAssign.blockUntilNoRIT(zkw);
609     master.assignmentManager.waitUntilNoRegionsInTransition(60000);
610     log("Assignment completed");
611 
612     assertTrue(" Table must be enabled.", master.getAssignmentManager()
613         .getZKTable().isEnabledTable(TableName.valueOf("enabledTable")));
614     // we also need regions assigned out on the dead server
615     List<HRegionInfo> enabledAndOnDeadRegions = new ArrayList<HRegionInfo>();
616     enabledAndOnDeadRegions.addAll(enabledRegions.subList(0, 6));
617     enabledRegions.removeAll(enabledAndOnDeadRegions);
618     List<HRegionInfo> disabledAndOnDeadRegions = new ArrayList<HRegionInfo>();
619     disabledAndOnDeadRegions.addAll(disabledRegions.subList(0, 6));
620     disabledRegions.removeAll(disabledAndOnDeadRegions);
621 
622     // set region plan to server to be killed and trigger assign
623     for (HRegionInfo hri : enabledAndOnDeadRegions) {
624       master.assignmentManager.addPlan(hri.getEncodedName(),
625           new RegionPlan(hri, null, deadServerName));
626       master.assignRegion(hri);
627     }
628     for (HRegionInfo hri : disabledAndOnDeadRegions) {
629       master.assignmentManager.addPlan(hri.getEncodedName(),
630           new RegionPlan(hri, null, deadServerName));
631       master.assignRegion(hri);
632     }
633 
634     // wait for no more RIT
635     log("Waiting for assignment to finish");
636     ZKAssign.blockUntilNoRIT(zkw);
637     master.assignmentManager.waitUntilNoRegionsInTransition(60000);
638     log("Assignment completed");
639 
640     // Due to master.assignRegion(hri) could fail to assign a region to a specified RS
641     // therefore, we need make sure that regions are in the expected RS
642     verifyRegionLocation(hrs, enabledAndAssignedRegions);
643     verifyRegionLocation(hrs, disabledAndAssignedRegions);
644     verifyRegionLocation(hrsDead, enabledAndOnDeadRegions);
645     verifyRegionLocation(hrsDead, disabledAndOnDeadRegions);
646 
647     assertTrue(" Didn't get enough regions of enabledTalbe on live rs.",
648       enabledAndAssignedRegions.size() >= 2);
649     assertTrue(" Didn't get enough regions of disalbedTable on live rs.",
650       disabledAndAssignedRegions.size() >= 2);
651     assertTrue(" Didn't get enough regions of enabledTalbe on dead rs.",
652       enabledAndOnDeadRegions.size() >= 2);
653     assertTrue(" Didn't get enough regions of disalbedTable on dead rs.",
654       disabledAndOnDeadRegions.size() >= 2);
655 
656     // Stop the master
657     log("Aborting master");
658     cluster.abortMaster(0);
659     cluster.waitOnMaster(0);
660     log("Master has aborted");
661 
662     /*
663      * Now, let's start mocking up some weird states as described in the method
664      * javadoc.
665      */
666 
667     List<HRegionInfo> regionsThatShouldBeOnline = new ArrayList<HRegionInfo>();
668     List<HRegionInfo> regionsThatShouldBeOffline = new ArrayList<HRegionInfo>();
669 
670     log("Beginning to mock scenarios");
671 
672     // Disable the disabledTable in ZK
673     ZKTable zktable = new ZKTable(zkw);
674     zktable.setDisabledTable(disabledTable);
675 
676     assertTrue(" The enabled table should be identified on master fail over.",
677         zktable.isEnabledTable(TableName.valueOf("enabledTable")));
678 
679     /*
680      * ZK = CLOSING
681      */
682 
683     // Region of enabled table being closed on dead RS but not finished
684     HRegionInfo region = enabledAndOnDeadRegions.remove(0);
685     regionsThatShouldBeOnline.add(region);
686     ZKAssign.createNodeClosing(zkw, region, deadServerName);
687     LOG.debug("\n\nRegion of enabled table was CLOSING on dead RS\n" +
688         region + "\n\n");
689 
690     // Region of disabled table being closed on dead RS but not finished
691     region = disabledAndOnDeadRegions.remove(0);
692     regionsThatShouldBeOffline.add(region);
693     ZKAssign.createNodeClosing(zkw, region, deadServerName);
694     LOG.debug("\n\nRegion of disabled table was CLOSING on dead RS\n" +
695         region + "\n\n");
696 
697     /*
698      * ZK = CLOSED
699      */
700 
701     // Region of enabled on dead server gets closed but not ack'd by master
702     region = enabledAndOnDeadRegions.remove(0);
703     regionsThatShouldBeOnline.add(region);
704     int version = ZKAssign.createNodeClosing(zkw, region, deadServerName);
705     ZKAssign.transitionNodeClosed(zkw, region, deadServerName, version);
706     LOG.debug("\n\nRegion of enabled table was CLOSED on dead RS\n" +
707         region + "\n\n");
708 
709     // Region of disabled on dead server gets closed but not ack'd by master
710     region = disabledAndOnDeadRegions.remove(0);
711     regionsThatShouldBeOffline.add(region);
712     version = ZKAssign.createNodeClosing(zkw, region, deadServerName);
713     ZKAssign.transitionNodeClosed(zkw, region, deadServerName, version);
714     LOG.debug("\n\nRegion of disabled table was CLOSED on dead RS\n" +
715         region + "\n\n");
716 
717     /*
718      * ZK = OPENING
719      */
720 
721     // RS was opening a region of enabled table then died
722     region = enabledRegions.remove(0);
723     regionsThatShouldBeOnline.add(region);
724     ZKAssign.createNodeOffline(zkw, region, deadServerName);
725     ZKAssign.transitionNodeOpening(zkw, region, deadServerName);
726     LOG.debug("\n\nRegion of enabled table was OPENING on dead RS\n" +
727         region + "\n\n");
728 
729     // RS was opening a region of disabled table then died
730     region = disabledRegions.remove(0);
731     regionsThatShouldBeOffline.add(region);
732     ZKAssign.createNodeOffline(zkw, region, deadServerName);
733     ZKAssign.transitionNodeOpening(zkw, region, deadServerName);
734     LOG.debug("\n\nRegion of disabled table was OPENING on dead RS\n" +
735         region + "\n\n");
736 
737     /*
738      * ZK = OPENED
739      */
740 
741     // Region of enabled table was opened on dead RS
742     region = enabledRegions.remove(0);
743     regionsThatShouldBeOnline.add(region);
744     ZKAssign.createNodeOffline(zkw, region, deadServerName);
745     ProtobufUtil.openRegion(hrsDead, hrsDead.getServerName(), region);
746     while (true) {
747       byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
748       RegionTransition rt = RegionTransition.parseFrom(bytes);
749       if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
750         break;
751       }
752       Thread.sleep(100);
753     }
754     LOG.debug("\n\nRegion of enabled table was OPENED on dead RS\n" +
755         region + "\n\n");
756 
757     // Region of disabled table was opened on dead RS
758     region = disabledRegions.remove(0);
759     regionsThatShouldBeOffline.add(region);
760     ZKAssign.createNodeOffline(zkw, region, deadServerName);
761     ProtobufUtil.openRegion(hrsDead, hrsDead.getServerName(), region);
762     while (true) {
763       byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
764       RegionTransition rt = RegionTransition.parseFrom(bytes);
765       if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
766         break;
767       }
768       Thread.sleep(100);
769     }
770     LOG.debug("\n\nRegion of disabled table was OPENED on dead RS\n" +
771         region + "\n\n");
772 
773     /*
774      * ZK = NONE
775      */
776 
777     // Region of enabled table was open at steady-state on dead RS
778     region = enabledRegions.remove(0);
779     regionsThatShouldBeOnline.add(region);
780     ZKAssign.createNodeOffline(zkw, region, deadServerName);
781     ProtobufUtil.openRegion(hrsDead, hrsDead.getServerName(), region);
782     while (true) {
783       byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
784       RegionTransition rt = RegionTransition.parseFrom(bytes);
785       if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
786         ZKAssign.deleteOpenedNode(zkw, region.getEncodedName(), rt.getServerName());
787         LOG.debug("DELETED " + rt);
788         break;
789       }
790       Thread.sleep(100);
791     }
792     LOG.debug("\n\nRegion of enabled table was open at steady-state on dead RS"
793         + "\n" + region + "\n\n");
794 
795     // Region of disabled table was open at steady-state on dead RS
796     region = disabledRegions.remove(0);
797     regionsThatShouldBeOffline.add(region);
798     ZKAssign.createNodeOffline(zkw, region, deadServerName);
799     ProtobufUtil.openRegion(hrsDead, hrsDead.getServerName(), region);
800     while (true) {
801       byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
802       RegionTransition rt = RegionTransition.parseFrom(bytes);
803       if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
804         ZKAssign.deleteOpenedNode(zkw, region.getEncodedName(), rt.getServerName());
805         break;
806       }
807       Thread.sleep(100);
808     }
809     LOG.debug("\n\nRegion of disabled table was open at steady-state on dead RS"
810       + "\n" + region + "\n\n");
811 
812     /*
813      * DONE MOCKING
814      */
815 
816     log("Done mocking data up in ZK");
817 
818     // Kill the RS that had a hard death
819     log("Killing RS " + deadServerName);
820     hrsDead.abort("Killing for unit test");
821     log("RS " + deadServerName + " killed");
822 
823     // Start up a new master.  Wait until regionserver is completely down
824     // before starting new master because of hbase-4511.
825     while (hrsDeadThread.isAlive()) {
826       Threads.sleep(10);
827     }
828     log("Starting up a new master");
829     master = cluster.startMaster().getMaster();
830     log("Waiting for master to be ready");
831     assertTrue(cluster.waitForActiveAndReadyMaster());
832     log("Master is ready");
833 
834     // Wait until SSH processing completed for dead server.
835     while (master.getServerManager().areDeadServersInProgress()) {
836       Thread.sleep(10);
837     }
838 
839     // Failover should be completed, now wait for no RIT
840     log("Waiting for no more RIT");
841     ZKAssign.blockUntilNoRIT(zkw);
842     log("No more RIT in ZK");
843     long now = System.currentTimeMillis();
844     long maxTime = 120000;
845     boolean done = master.assignmentManager.waitUntilNoRegionsInTransition(maxTime);
846     if (!done) {
847       RegionStates regionStates = master.getAssignmentManager().getRegionStates();
848       LOG.info("rit=" + regionStates.getRegionsInTransition());
849     }
850     long elapsed = System.currentTimeMillis() - now;
851     assertTrue("Elapsed=" + elapsed + ", maxTime=" + maxTime + ", done=" + done,
852       elapsed < maxTime);
853     log("No more RIT in RIT map, doing final test verification");
854 
855     // Grab all the regions that are online across RSs
856     Set<HRegionInfo> onlineRegions = new TreeSet<HRegionInfo>();
857     now = System.currentTimeMillis();
858     maxTime = 30000;
859     for (JVMClusterUtil.RegionServerThread rst :
860         cluster.getRegionServerThreads()) {
861       try {
862         HRegionServer rs = rst.getRegionServer();
863         while (!rs.getRegionsInTransitionInRS().isEmpty()) {
864           elapsed = System.currentTimeMillis() - now;
865           assertTrue("Test timed out in getting online regions", elapsed < maxTime);
866           if (rs.isAborted() || rs.isStopped()) {
867             // This region server is stopped, skip it.
868             break;
869           }
870           Thread.sleep(100);
871         }
872         onlineRegions.addAll(ProtobufUtil.getOnlineRegions(rs));
873       } catch (RegionServerStoppedException e) {
874         LOG.info("Got RegionServerStoppedException", e);
875       }
876     }
877 
878     // Now, everything that should be online should be online
879     for (HRegionInfo hri : regionsThatShouldBeOnline) {
880       assertTrue("region=" + hri.getRegionNameAsString() + ", " + onlineRegions.toString(),
881         onlineRegions.contains(hri));
882     }
883 
884     // Everything that should be offline should not be online
885     for (HRegionInfo hri : regionsThatShouldBeOffline) {
886       assertFalse(onlineRegions.contains(hri));
887     }
888 
889     log("Done with verification, all passed, shutting down cluster");
890 
891     // Done, shutdown the cluster
892     TEST_UTIL.shutdownMiniCluster();
893   }
894 
895   /**
896    * Verify regions are on the expected region server
897    */
898   private void verifyRegionLocation(HRegionServer hrs, List<HRegionInfo> regions)
899       throws IOException {
900     List<HRegionInfo> tmpOnlineRegions = ProtobufUtil.getOnlineRegions(hrs);
901     Iterator<HRegionInfo> itr = regions.iterator();
902     while (itr.hasNext()) {
903       HRegionInfo tmp = itr.next();
904       if (!tmpOnlineRegions.contains(tmp)) {
905         itr.remove();
906       }
907     }
908   }
909 
910   HRegion createRegion(final HRegionInfo  hri, final Path rootdir, final Configuration c,
911       final HTableDescriptor htd)
912   throws IOException {
913     HRegion r = HRegion.createHRegion(hri, rootdir, c, htd);
914     // The above call to create a region will create an hlog file.  Each
915     // log file create will also create a running thread to do syncing.  We need
916     // to close out this log else we will have a running thread trying to sync
917     // the file system continuously which is ugly when dfs is taken away at the
918     // end of the test.
919     HRegion.closeHRegion(r);
920     return r;
921   }
922 
923   // TODO: Next test to add is with testing permutations of the RIT or the RS
924   //       killed are hosting ROOT and hbase:meta regions.
925 
926   private void log(String string) {
927     LOG.info("\n\n" + string + " \n\n");
928   }
929 
930   @Test (timeout=180000)
931   public void testShouldCheckMasterFailOverWhenMETAIsInOpenedState()
932       throws Exception {
933     LOG.info("Starting testShouldCheckMasterFailOverWhenMETAIsInOpenedState");
934     final int NUM_MASTERS = 1;
935     final int NUM_RS = 2;
936 
937     // Start the cluster
938     HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
939     Configuration conf = TEST_UTIL.getConfiguration();
940     conf.setInt("hbase.master.info.port", -1);
941     conf.setBoolean("hbase.assignment.usezk", true);
942 
943     TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
944     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
945 
946     // Find regionserver carrying meta.
947     List<RegionServerThread> regionServerThreads =
948       cluster.getRegionServerThreads();
949     int count = -1;
950     HRegion metaRegion = null;
951     for (RegionServerThread regionServerThread : regionServerThreads) {
952       HRegionServer regionServer = regionServerThread.getRegionServer();
953       metaRegion = regionServer.getOnlineRegion(HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
954       count++;
955       regionServer.abort("");
956       if (null != metaRegion) break;
957     }
958     HRegionServer regionServer = cluster.getRegionServer(count);
959 
960     TEST_UTIL.shutdownMiniHBaseCluster();
961 
962     // Create a ZKW to use in the test
963     ZooKeeperWatcher zkw =
964       HBaseTestingUtility.createAndForceNodeToOpenedState(TEST_UTIL,
965           metaRegion, regionServer.getServerName());
966 
967     LOG.info("Staring cluster for second time");
968     TEST_UTIL.startMiniHBaseCluster(NUM_MASTERS, NUM_RS);
969 
970     HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
971     while (!master.isInitialized()) {
972       Thread.sleep(100);
973     }
974     // Failover should be completed, now wait for no RIT
975     log("Waiting for no more RIT");
976     ZKAssign.blockUntilNoRIT(zkw);
977 
978     zkw.close();
979     // Stop the cluster
980     TEST_UTIL.shutdownMiniCluster();
981   }
982 
983   /**
984    * This tests a RIT in offline state will get re-assigned after a master restart
985    */
986   @Test(timeout=240000)
987   public void testOfflineRegionReAssginedAfterMasterRestart() throws Exception {
988     final TableName table = TableName.valueOf("testOfflineRegionReAssginedAfterMasterRestart");
989     final int NUM_MASTERS = 1;
990     final int NUM_RS = 2;
991 
992     // Create config to use for this cluster
993     Configuration conf = HBaseConfiguration.create();
994     conf.setBoolean("hbase.assignment.usezk", true);
995 
996     // Start the cluster
997     final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
998     TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
999     log("Cluster started");
1000 
1001     TEST_UTIL.createTable(table, Bytes.toBytes("family"));
1002     HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
1003     RegionStates regionStates = master.getAssignmentManager().getRegionStates();
1004     HRegionInfo hri = regionStates.getRegionsOfTable(table).get(0);
1005     ServerName serverName = regionStates.getRegionServerOfRegion(hri);
1006     TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
1007 
1008     ServerName dstName = null;
1009     for (ServerName tmpServer : master.serverManager.getOnlineServers().keySet()) {
1010       if (!tmpServer.equals(serverName)) {
1011         dstName = tmpServer;
1012         break;
1013       }
1014     }
1015     // find a different server
1016     assertTrue(dstName != null);
1017     // shutdown HBase cluster
1018     TEST_UTIL.shutdownMiniHBaseCluster();
1019     // create a RIT node in offline state
1020     ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
1021     ZKAssign.createNodeOffline(zkw, hri, dstName);
1022     Stat stat = new Stat();
1023     byte[] data =
1024         ZKAssign.getDataNoWatch(zkw, hri.getEncodedName(), stat);
1025     assertTrue(data != null);
1026     RegionTransition rt = RegionTransition.parseFrom(data);
1027     assertTrue(rt.getEventType() == EventType.M_ZK_REGION_OFFLINE);
1028 
1029     LOG.info(hri.getEncodedName() + " region is in offline state with source server=" + serverName
1030         + " and dst server=" + dstName);
1031 
1032     // start HBase cluster
1033     TEST_UTIL.startMiniHBaseCluster(NUM_MASTERS, NUM_RS);
1034 
1035     while (true) {
1036       master = TEST_UTIL.getHBaseCluster().getMaster();
1037       if (master != null && master.isInitialized()) {
1038         ServerManager serverManager = master.getServerManager();
1039         if (!serverManager.areDeadServersInProgress()) {
1040           break;
1041         }
1042       }
1043       Thread.sleep(200);
1044     }
1045 
1046     // verify the region is assigned
1047     master = TEST_UTIL.getHBaseCluster().getMaster();
1048     master.getAssignmentManager().waitForAssignment(hri);
1049     regionStates = master.getAssignmentManager().getRegionStates();
1050     RegionState newState = regionStates.getRegionState(hri);
1051     assertTrue(newState.isOpened());
1052   }
1053 
1054   /**
1055    * Simple test of master failover.
1056    * <p>
1057    * Starts with three masters.  Kills a backup master.  Then kills the active
1058    * master.  Ensures the final master becomes active and we can still contact
1059    * the cluster.
1060    * @throws Exception
1061    */
1062   @Test (timeout=240000)
1063   public void testSimpleMasterFailover() throws Exception {
1064 
1065     final int NUM_MASTERS = 3;
1066     final int NUM_RS = 3;
1067 
1068     // Start the cluster
1069     HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
1070 
1071     TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
1072     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1073 
1074     // get all the master threads
1075     List<MasterThread> masterThreads = cluster.getMasterThreads();
1076 
1077     // wait for each to come online
1078     for (MasterThread mt : masterThreads) {
1079       assertTrue(mt.isAlive());
1080     }
1081 
1082     // verify only one is the active master and we have right number
1083     int numActive = 0;
1084     int activeIndex = -1;
1085     ServerName activeName = null;
1086     HMaster active = null;
1087     for (int i = 0; i < masterThreads.size(); i++) {
1088       if (masterThreads.get(i).getMaster().isActiveMaster()) {
1089         numActive++;
1090         activeIndex = i;
1091         active = masterThreads.get(activeIndex).getMaster();
1092         activeName = active.getServerName();
1093       }
1094     }
1095     assertEquals(1, numActive);
1096     assertEquals(NUM_MASTERS, masterThreads.size());
1097     LOG.info("Active master " + activeName);
1098 
1099     // Check that ClusterStatus reports the correct active and backup masters
1100     assertNotNull(active);
1101     ClusterStatus status = active.getClusterStatus();
1102     assertTrue(status.getMaster().equals(activeName));
1103     assertEquals(2, status.getBackupMastersSize());
1104     assertEquals(2, status.getBackupMasters().size());
1105 
1106     // attempt to stop one of the inactive masters
1107     int backupIndex = (activeIndex == 0 ? 1 : activeIndex - 1);
1108     HMaster master = cluster.getMaster(backupIndex);
1109     LOG.debug("\n\nStopping a backup master: " + master.getServerName() + "\n");
1110     cluster.stopMaster(backupIndex, false);
1111     cluster.waitOnMaster(backupIndex);
1112 
1113     // Verify still one active master and it's the same
1114     for (int i = 0; i < masterThreads.size(); i++) {
1115       if (masterThreads.get(i).getMaster().isActiveMaster()) {
1116         assertTrue(activeName.equals(masterThreads.get(i).getMaster().getServerName()));
1117         activeIndex = i;
1118         active = masterThreads.get(activeIndex).getMaster();
1119       }
1120     }
1121     assertEquals(1, numActive);
1122     assertEquals(2, masterThreads.size());
1123     int rsCount = masterThreads.get(activeIndex).getMaster().getClusterStatus().getServersSize();
1124     LOG.info("Active master " + active.getServerName() + " managing " + rsCount +  " regions servers");
1125     assertEquals(3, rsCount);
1126 
1127     // Check that ClusterStatus reports the correct active and backup masters
1128     assertNotNull(active);
1129     status = active.getClusterStatus();
1130     assertTrue(status.getMaster().equals(activeName));
1131     assertEquals(1, status.getBackupMastersSize());
1132     assertEquals(1, status.getBackupMasters().size());
1133 
1134     // kill the active master
1135     LOG.debug("\n\nStopping the active master " + active.getServerName() + "\n");
1136     cluster.stopMaster(activeIndex, false);
1137     cluster.waitOnMaster(activeIndex);
1138 
1139     // wait for an active master to show up and be ready
1140     assertTrue(cluster.waitForActiveAndReadyMaster());
1141 
1142     LOG.debug("\n\nVerifying backup master is now active\n");
1143     // should only have one master now
1144     assertEquals(1, masterThreads.size());
1145 
1146     // and he should be active
1147     active = masterThreads.get(0).getMaster();
1148     assertNotNull(active);
1149     status = active.getClusterStatus();
1150     ServerName mastername = status.getMaster();
1151     assertTrue(mastername.equals(active.getServerName()));
1152     assertTrue(active.isActiveMaster());
1153     assertEquals(0, status.getBackupMastersSize());
1154     assertEquals(0, status.getBackupMasters().size());
1155     int rss = status.getServersSize();
1156     LOG.info("Active master " + mastername.getServerName() + " managing " +
1157       rss +  " region servers");
1158     assertEquals(3, rss);
1159 
1160     // Stop the cluster
1161     TEST_UTIL.shutdownMiniCluster();
1162   }
1163 
1164   /**
1165    * Test region in pending_open/close when master failover
1166    */
1167   @Test (timeout=180000)
1168   public void testPendingOpenOrCloseWhenMasterFailover() throws Exception {
1169     final int NUM_MASTERS = 1;
1170     final int NUM_RS = 1;
1171 
1172     // Create config to use for this cluster
1173     Configuration conf = HBaseConfiguration.create();
1174     conf.setBoolean("hbase.assignment.usezk", false);
1175 
1176     // Start the cluster
1177     HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
1178     TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
1179     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1180     log("Cluster started");
1181 
1182     // get all the master threads
1183     List<MasterThread> masterThreads = cluster.getMasterThreads();
1184     assertEquals(1, masterThreads.size());
1185 
1186     // only one master thread, let's wait for it to be initialized
1187     assertTrue(cluster.waitForActiveAndReadyMaster());
1188     HMaster master = masterThreads.get(0).getMaster();
1189     assertTrue(master.isActiveMaster());
1190     assertTrue(master.isInitialized());
1191 
1192     // Create a table with a region online
1193     HTable onlineTable = TEST_UTIL.createTable("onlineTable", "family");
1194 
1195     // Create a table in META, so it has a region offline
1196     HTableDescriptor offlineTable = new HTableDescriptor(
1197       TableName.valueOf(Bytes.toBytes("offlineTable")));
1198     offlineTable.addFamily(new HColumnDescriptor(Bytes.toBytes("family")));
1199 
1200     FileSystem filesystem = FileSystem.get(conf);
1201     Path rootdir = FSUtils.getRootDir(conf);
1202     FSTableDescriptors fstd = new FSTableDescriptors(conf, filesystem, rootdir);
1203     fstd.createTableDescriptor(offlineTable);
1204 
1205     HRegionInfo hriOffline = new HRegionInfo(offlineTable.getTableName(), null, null);
1206     createRegion(hriOffline, rootdir, conf, offlineTable);
1207     MetaEditor.addRegionToMeta(master.getCatalogTracker(), hriOffline);
1208 
1209     log("Regions in hbase:meta and namespace have been created");
1210 
1211     // at this point we only expect 3 regions to be assigned out
1212     // (catalogs and namespace, + 1 online region)
1213     assertEquals(3, cluster.countServedRegions());
1214     HRegionInfo hriOnline = onlineTable.getRegionLocation("").getRegionInfo();
1215 
1216     RegionStates regionStates = master.getAssignmentManager().getRegionStates();
1217     RegionStateStore stateStore = master.getAssignmentManager().getRegionStateStore();
1218 
1219     // Put the online region in pending_close. It is actually already opened.
1220     // This is to simulate that the region close RPC is not sent out before failover
1221     RegionState oldState = regionStates.getRegionState(hriOnline);
1222     RegionState newState = new RegionState(hriOnline, State.PENDING_CLOSE, oldState.getServerName());
1223     stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState);
1224 
1225     // Put the offline region in pending_open. It is actually not opened yet.
1226     // This is to simulate that the region open RPC is not sent out before failover
1227     oldState = new RegionState(hriOffline, State.OFFLINE);
1228     newState = new RegionState(hriOffline, State.PENDING_OPEN, newState.getServerName());
1229     stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState);
1230 
1231     // Stop the master
1232     log("Aborting master");
1233     cluster.abortMaster(0);
1234     cluster.waitOnMaster(0);
1235     log("Master has aborted");
1236 
1237     // Start up a new master
1238     log("Starting up a new master");
1239     master = cluster.startMaster().getMaster();
1240     log("Waiting for master to be ready");
1241     cluster.waitForActiveAndReadyMaster();
1242     log("Master is ready");
1243 
1244     // Wait till no region in transition any more
1245     master.getAssignmentManager().waitUntilNoRegionsInTransition(60000);
1246 
1247     // Get new region states since master restarted
1248     regionStates = master.getAssignmentManager().getRegionStates();
1249 
1250     // Both pending_open (RPC sent/not yet) regions should be online
1251     assertTrue(regionStates.isRegionOnline(hriOffline));
1252     assertTrue(regionStates.isRegionOnline(hriOnline));
1253 
1254     log("Done with verification, shutting down cluster");
1255 
1256     // Done, shutdown the cluster
1257     TEST_UTIL.shutdownMiniCluster();
1258   }
1259 }
1260