View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertTrue;
25  
26  import java.io.IOException;
27  import java.util.ArrayList;
28  import java.util.Iterator;
29  import java.util.List;
30  import java.util.Set;
31  import java.util.TreeSet;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.fs.Path;
38  import org.apache.hadoop.hbase.Abortable;
39  import org.apache.hadoop.hbase.ClusterStatus;
40  import org.apache.hadoop.hbase.HBaseConfiguration;
41  import org.apache.hadoop.hbase.HBaseTestingUtility;
42  import org.apache.hadoop.hbase.HColumnDescriptor;
43  import org.apache.hadoop.hbase.HConstants;
44  import org.apache.hadoop.hbase.HRegionInfo;
45  import org.apache.hadoop.hbase.HTableDescriptor;
46  import org.apache.hadoop.hbase.LargeTests;
47  import org.apache.hadoop.hbase.MiniHBaseCluster;
48  import org.apache.hadoop.hbase.RegionTransition;
49  import org.apache.hadoop.hbase.ServerName;
50  import org.apache.hadoop.hbase.TableName;
51  import org.apache.hadoop.hbase.catalog.MetaEditor;
52  import org.apache.hadoop.hbase.client.HTable;
53  import org.apache.hadoop.hbase.executor.EventType;
54  import org.apache.hadoop.hbase.master.RegionState.State;
55  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
56  import org.apache.hadoop.hbase.regionserver.HRegion;
57  import org.apache.hadoop.hbase.regionserver.HRegionServer;
58  import org.apache.hadoop.hbase.regionserver.RegionMergeTransaction;
59  import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
60  import org.apache.hadoop.hbase.util.Bytes;
61  import org.apache.hadoop.hbase.util.FSTableDescriptors;
62  import org.apache.hadoop.hbase.util.FSUtils;
63  import org.apache.hadoop.hbase.util.JVMClusterUtil;
64  import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
65  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
66  import org.apache.hadoop.hbase.util.Threads;
67  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
68  import org.apache.hadoop.hbase.zookeeper.ZKTable;
69  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
70  import org.apache.zookeeper.data.Stat;
71  import org.junit.Test;
72  import org.junit.experimental.categories.Category;
73  
74  @Category(LargeTests.class)
75  public class TestMasterFailover {
76    private static final Log LOG = LogFactory.getLog(TestMasterFailover.class);
77  
78    /**
79     * Complex test of master failover that tests as many permutations of the
80     * different possible states that regions in transition could be in within ZK.
81     * <p>
82     * This tests the proper handling of these states by the failed-over master
83     * and includes a thorough testing of the timeout code as well.
84     * <p>
85     * Starts with a single master and three regionservers.
86     * <p>
87     * Creates two tables, enabledTable and disabledTable, each containing 5
88     * regions.  The disabledTable is then disabled.
89     * <p>
90     * After reaching steady-state, the master is killed.  We then mock several
91     * states in ZK.
92     * <p>
93     * After mocking them, we will startup a new master which should become the
94     * active master and also detect that it is a failover.  The primary test
95     * passing condition will be that all regions of the enabled table are
96     * assigned and all the regions of the disabled table are not assigned.
97     * <p>
98     * The different scenarios to be tested are below:
99     * <p>
100    * <b>ZK State:  OFFLINE</b>
101    * <p>A node can get into OFFLINE state if</p>
102    * <ul>
103    * <li>An RS fails to open a region, so it reverts the state back to OFFLINE
104    * <li>The Master is assigning the region to a RS before it sends RPC
105    * </ul>
106    * <p>We will mock the scenarios</p>
107    * <ul>
108    * <li>Master has assigned an enabled region but RS failed so a region is
109    *     not assigned anywhere and is sitting in ZK as OFFLINE</li>
110    * <li>This seems to cover both cases?</li>
111    * </ul>
112    * <p>
113    * <b>ZK State:  CLOSING</b>
114    * <p>A node can get into CLOSING state if</p>
115    * <ul>
116    * <li>An RS has begun to close a region
117    * </ul>
118    * <p>We will mock the scenarios</p>
119    * <ul>
120    * <li>Region of enabled table was being closed but did not complete
121    * <li>Region of disabled table was being closed but did not complete
122    * </ul>
123    * <p>
124    * <b>ZK State:  CLOSED</b>
125    * <p>A node can get into CLOSED state if</p>
126    * <ul>
127    * <li>An RS has completed closing a region but not acknowledged by master yet
128    * </ul>
129    * <p>We will mock the scenarios</p>
130    * <ul>
131    * <li>Region of a table that should be enabled was closed on an RS
132    * <li>Region of a table that should be disabled was closed on an RS
133    * </ul>
134    * <p>
135    * <b>ZK State:  OPENING</b>
136    * <p>A node can get into OPENING state if</p>
137    * <ul>
138    * <li>An RS has begun to open a region
139    * </ul>
140    * <p>We will mock the scenarios</p>
141    * <ul>
142    * <li>RS was opening a region of enabled table but never finishes
143    * </ul>
144    * <p>
145    * <b>ZK State:  OPENED</b>
146    * <p>A node can get into OPENED state if</p>
147    * <ul>
148    * <li>An RS has finished opening a region but not acknowledged by master yet
149    * </ul>
150    * <p>We will mock the scenarios</p>
151    * <ul>
152    * <li>Region of a table that should be enabled was opened on an RS
153    * <li>Region of a table that should be disabled was opened on an RS
154    * </ul>
155    * @throws Exception
156    */
157   @Test (timeout=240000)
158   public void testMasterFailoverWithMockedRIT() throws Exception {
159 
160     final int NUM_MASTERS = 1;
161     final int NUM_RS = 3;
162 
163     // Create config to use for this cluster
164     Configuration conf = HBaseConfiguration.create();
165     conf.setBoolean("hbase.assignment.usezk", true);
166 
167     // Start the cluster
168     HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
169     TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
170     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
171     log("Cluster started");
172 
173     // Create a ZKW to use in the test
174     ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
175 
176     // get all the master threads
177     List<MasterThread> masterThreads = cluster.getMasterThreads();
178     assertEquals(1, masterThreads.size());
179 
180     // only one master thread, let's wait for it to be initialized
181     assertTrue(cluster.waitForActiveAndReadyMaster());
182     HMaster master = masterThreads.get(0).getMaster();
183     assertTrue(master.isActiveMaster());
184     assertTrue(master.isInitialized());
185 
186     // disable load balancing on this master
187     master.balanceSwitch(false);
188 
189     // create two tables in META, each with 10 regions
190     byte [] FAMILY = Bytes.toBytes("family");
191     byte [][] SPLIT_KEYS = new byte [][] {
192         new byte[0], Bytes.toBytes("aaa"), Bytes.toBytes("bbb"),
193         Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"),
194         Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
195         Bytes.toBytes("iii"), Bytes.toBytes("jjj")
196     };
197 
198     byte [] enabledTable = Bytes.toBytes("enabledTable");
199     HTableDescriptor htdEnabled = new HTableDescriptor(TableName.valueOf(enabledTable));
200     htdEnabled.addFamily(new HColumnDescriptor(FAMILY));
201 
202     FileSystem filesystem = FileSystem.get(conf);
203     Path rootdir = FSUtils.getRootDir(conf);
204     FSTableDescriptors fstd = new FSTableDescriptors(filesystem, rootdir);
205     // Write the .tableinfo
206     fstd.createTableDescriptor(htdEnabled);
207 
208     HRegionInfo hriEnabled = new HRegionInfo(htdEnabled.getTableName(), null, null);
209     createRegion(hriEnabled, rootdir, conf, htdEnabled);
210 
211     List<HRegionInfo> enabledRegions = TEST_UTIL.createMultiRegionsInMeta(
212         TEST_UTIL.getConfiguration(), htdEnabled, SPLIT_KEYS);
213 
214     TableName disabledTable = TableName.valueOf("disabledTable");
215     HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable);
216     htdDisabled.addFamily(new HColumnDescriptor(FAMILY));
217     // Write the .tableinfo
218     fstd.createTableDescriptor(htdDisabled);
219     HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getTableName(), null, null);
220     createRegion(hriDisabled, rootdir, conf, htdDisabled);
221     List<HRegionInfo> disabledRegions = TEST_UTIL.createMultiRegionsInMeta(
222         TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS);
223 
224     TableName tableWithMergingRegions = TableName.valueOf("tableWithMergingRegions");
225     TEST_UTIL.createTable(tableWithMergingRegions, FAMILY, new byte [][] {Bytes.toBytes("m")});
226 
227     log("Regions in hbase:meta and namespace have been created");
228 
229     // at this point we only expect 4 regions to be assigned out
230     // (catalogs and namespace, + 2 merging regions)
231     assertEquals(4, cluster.countServedRegions());
232 
233     // Move merging regions to the same region server
234     AssignmentManager am = master.getAssignmentManager();
235     RegionStates regionStates = am.getRegionStates();
236     List<HRegionInfo> mergingRegions = regionStates.getRegionsOfTable(tableWithMergingRegions);
237     assertEquals(2, mergingRegions.size());
238     HRegionInfo a = mergingRegions.get(0);
239     HRegionInfo b = mergingRegions.get(1);
240     HRegionInfo newRegion = RegionMergeTransaction.getMergedRegionInfo(a, b);
241     ServerName mergingServer = regionStates.getRegionServerOfRegion(a);
242     ServerName serverB = regionStates.getRegionServerOfRegion(b);
243     if (!serverB.equals(mergingServer)) {
244       RegionPlan plan = new RegionPlan(b, serverB, mergingServer);
245       am.balance(plan);
246       assertTrue(am.waitForAssignment(b));
247     }
248 
249     // Let's just assign everything to first RS
250     HRegionServer hrs = cluster.getRegionServer(0);
251     ServerName serverName = hrs.getServerName();
252     HRegionInfo closingRegion = enabledRegions.remove(0);
253     // we'll need some regions to already be assigned out properly on live RS
254     List<HRegionInfo> enabledAndAssignedRegions = new ArrayList<HRegionInfo>();
255     enabledAndAssignedRegions.add(enabledRegions.remove(0));
256     enabledAndAssignedRegions.add(enabledRegions.remove(0));
257     enabledAndAssignedRegions.add(closingRegion);
258 
259     List<HRegionInfo> disabledAndAssignedRegions = new ArrayList<HRegionInfo>();
260     disabledAndAssignedRegions.add(disabledRegions.remove(0));
261     disabledAndAssignedRegions.add(disabledRegions.remove(0));
262 
263     // now actually assign them
264     for (HRegionInfo hri : enabledAndAssignedRegions) {
265       master.assignmentManager.regionPlans.put(hri.getEncodedName(),
266           new RegionPlan(hri, null, serverName));
267       master.assignRegion(hri);
268     }
269     for (HRegionInfo hri : disabledAndAssignedRegions) {
270       master.assignmentManager.regionPlans.put(hri.getEncodedName(),
271           new RegionPlan(hri, null, serverName));
272       master.assignRegion(hri);
273     }
274 
275     // wait for no more RIT
276     log("Waiting for assignment to finish");
277     ZKAssign.blockUntilNoRIT(zkw);
278     log("Assignment completed");
279 
280     // Stop the master
281     log("Aborting master");
282     cluster.abortMaster(0);
283     cluster.waitOnMaster(0);
284     log("Master has aborted");
285 
286     /*
287      * Now, let's start mocking up some weird states as described in the method
288      * javadoc.
289      */
290 
291     List<HRegionInfo> regionsThatShouldBeOnline = new ArrayList<HRegionInfo>();
292     List<HRegionInfo> regionsThatShouldBeOffline = new ArrayList<HRegionInfo>();
293 
294     log("Beginning to mock scenarios");
295 
296     // Disable the disabledTable in ZK
297     ZKTable zktable = new ZKTable(zkw);
298     zktable.setDisabledTable(disabledTable);
299 
300     /*
301      *  ZK = OFFLINE
302      */
303 
304     // Region that should be assigned but is not and is in ZK as OFFLINE
305     // Cause: This can happen if the master crashed after creating the znode but before sending the
306     //  request to the region server
307     HRegionInfo region = enabledRegions.remove(0);
308     regionsThatShouldBeOnline.add(region);
309     ZKAssign.createNodeOffline(zkw, region, serverName);
310 
311     /*
312      * ZK = CLOSING
313      */
314     // Cause: Same as offline.
315     regionsThatShouldBeOnline.add(closingRegion);
316     ZKAssign.createNodeClosing(zkw, closingRegion, serverName);
317 
318     /*
319      * ZK = CLOSED
320      */
321 
322     // Region of enabled table closed but not ack
323     //Cause: Master was down while the region server updated the ZK status.
324     region = enabledRegions.remove(0);
325     regionsThatShouldBeOnline.add(region);
326     int version = ZKAssign.createNodeClosing(zkw, region, serverName);
327     ZKAssign.transitionNodeClosed(zkw, region, serverName, version);
328 
329     // Region of disabled table closed but not ack
330     region = disabledRegions.remove(0);
331     regionsThatShouldBeOffline.add(region);
332     version = ZKAssign.createNodeClosing(zkw, region, serverName);
333     ZKAssign.transitionNodeClosed(zkw, region, serverName, version);
334 
335     /*
336      * ZK = OPENED
337      */
338 
339     // Region of enabled table was opened on RS
340     // Cause: as offline
341     region = enabledRegions.remove(0);
342     regionsThatShouldBeOnline.add(region);
343     ZKAssign.createNodeOffline(zkw, region, serverName);
344     ProtobufUtil.openRegion(hrs, hrs.getServerName(), region);
345     while (true) {
346       byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
347       RegionTransition rt = RegionTransition.parseFrom(bytes);
348       if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
349         break;
350       }
351       Thread.sleep(100);
352     }
353 
354     // Region of disable table was opened on RS
355     // Cause: Master failed while updating the status for this region server.
356     region = disabledRegions.remove(0);
357     regionsThatShouldBeOffline.add(region);
358     ZKAssign.createNodeOffline(zkw, region, serverName);
359     ProtobufUtil.openRegion(hrs, hrs.getServerName(), region);
360     while (true) {
361       byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
362       RegionTransition rt = RegionTransition.parseFrom(bytes);
363       if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
364         break;
365       }
366       Thread.sleep(100);
367     }
368 
369     /*
370      * ZK = MERGING
371      */
372 
373     // Regions of table of merging regions
374     // Cause: Master was down while merging was going on
375     RegionMergeTransaction.createNodeMerging(
376       zkw, newRegion, mergingServer, a, b);
377 
378     /*
379      * ZK = NONE
380      */
381 
382     /*
383      * DONE MOCKING
384      */
385 
386     log("Done mocking data up in ZK");
387 
388     // Start up a new master
389     log("Starting up a new master");
390     master = cluster.startMaster().getMaster();
391     log("Waiting for master to be ready");
392     cluster.waitForActiveAndReadyMaster();
393     log("Master is ready");
394 
395     // Get new region states since master restarted
396     regionStates = master.getAssignmentManager().getRegionStates();
397     // Merging region should remain merging
398     assertTrue(regionStates.isRegionInState(a, State.MERGING));
399     assertTrue(regionStates.isRegionInState(b, State.MERGING));
400     assertTrue(regionStates.isRegionInState(newRegion, State.MERGING_NEW));
401     // Now remove the faked merging znode, merging regions should be
402     // offlined automatically, otherwise it is a bug in AM.
403     ZKAssign.deleteNodeFailSilent(zkw, newRegion);
404 
405     // Failover should be completed, now wait for no RIT
406     log("Waiting for no more RIT");
407     ZKAssign.blockUntilNoRIT(zkw);
408     log("No more RIT in ZK, now doing final test verification");
409 
410     // Grab all the regions that are online across RSs
411     Set<HRegionInfo> onlineRegions = new TreeSet<HRegionInfo>();
412     for (JVMClusterUtil.RegionServerThread rst :
413       cluster.getRegionServerThreads()) {
414       onlineRegions.addAll(ProtobufUtil.getOnlineRegions(rst.getRegionServer()));
415     }
416 
417     // Now, everything that should be online should be online
418     for (HRegionInfo hri : regionsThatShouldBeOnline) {
419       assertTrue(onlineRegions.contains(hri));
420     }
421 
422     // Everything that should be offline should not be online
423     for (HRegionInfo hri : regionsThatShouldBeOffline) {
424       if (onlineRegions.contains(hri)) {
425        LOG.debug(hri);
426       }
427       assertFalse(onlineRegions.contains(hri));
428     }
429 
430     log("Done with verification, all passed, shutting down cluster");
431 
432     // Done, shutdown the cluster
433     TEST_UTIL.shutdownMiniCluster();
434   }
435 
436   /**
437    * Complex test of master failover that tests as many permutations of the
438    * different possible states that regions in transition could be in within ZK
439    * pointing to an RS that has died while no master is around to process it.
440    * <p>
441    * This tests the proper handling of these states by the failed-over master
442    * and includes a thorough testing of the timeout code as well.
443    * <p>
444    * Starts with a single master and two regionservers.
445    * <p>
446    * Creates two tables, enabledTable and disabledTable, each containing 5
447    * regions.  The disabledTable is then disabled.
448    * <p>
449    * After reaching steady-state, the master is killed.  We then mock several
450    * states in ZK.  And one of the RS will be killed.
451    * <p>
452    * After mocking them and killing an RS, we will startup a new master which
453    * should become the active master and also detect that it is a failover.  The
454    * primary test passing condition will be that all regions of the enabled
455    * table are assigned and all the regions of the disabled table are not
456    * assigned.
457    * <p>
458    * The different scenarios to be tested are below:
459    * <p>
460    * <b>ZK State:  CLOSING</b>
461    * <p>A node can get into CLOSING state if</p>
462    * <ul>
463    * <li>An RS has begun to close a region
464    * </ul>
465    * <p>We will mock the scenarios</p>
466    * <ul>
467    * <li>Region was being closed but the RS died before finishing the close
468    * </ul>
469    * <b>ZK State:  OPENED</b>
470    * <p>A node can get into OPENED state if</p>
471    * <ul>
472    * <li>An RS has finished opening a region but not acknowledged by master yet
473    * </ul>
474    * <p>We will mock the scenarios</p>
475    * <ul>
476    * <li>Region of a table that should be enabled was opened by a now-dead RS
477    * <li>Region of a table that should be disabled was opened by a now-dead RS
478    * </ul>
479    * <p>
480    * <b>ZK State:  NONE</b>
481    * <p>A region could not have a transition node if</p>
482    * <ul>
483    * <li>The server hosting the region died and no master processed it
484    * </ul>
485    * <p>We will mock the scenarios</p>
486    * <ul>
487    * <li>Region of enabled table was on a dead RS that was not yet processed
488    * <li>Region of disabled table was on a dead RS that was not yet processed
489    * </ul>
490    * @throws Exception
491    */
492   @Test (timeout=180000)
493   public void testMasterFailoverWithMockedRITOnDeadRS() throws Exception {
494 
495     final int NUM_MASTERS = 1;
496     final int NUM_RS = 2;
497 
498     // Create and start the cluster
499     HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
500     Configuration conf = TEST_UTIL.getConfiguration();
501     conf.setBoolean("hbase.assignment.usezk", true);
502 
503     conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
504     conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 2);
505     TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
506     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
507     log("Cluster started");
508 
509     // Create a ZKW to use in the test
510     ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
511         "unittest", new Abortable() {
512 
513           @Override
514           public void abort(String why, Throwable e) {
515             LOG.error("Fatal ZK Error: " + why, e);
516             org.junit.Assert.assertFalse("Fatal ZK error", true);
517           }
518 
519           @Override
520           public boolean isAborted() {
521             return false;
522           }
523 
524     });
525 
526     // get all the master threads
527     List<MasterThread> masterThreads = cluster.getMasterThreads();
528     assertEquals(1, masterThreads.size());
529 
530     // only one master thread, let's wait for it to be initialized
531     assertTrue(cluster.waitForActiveAndReadyMaster());
532     HMaster master = masterThreads.get(0).getMaster();
533     assertTrue(master.isActiveMaster());
534     assertTrue(master.isInitialized());
535 
536     // disable load balancing on this master
537     master.balanceSwitch(false);
538 
539     // create two tables in META, each with 30 regions
540     byte [] FAMILY = Bytes.toBytes("family");
541     byte[][] SPLIT_KEYS =
542         TEST_UTIL.getRegionSplitStartKeys(Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 30);
543 
544     byte [] enabledTable = Bytes.toBytes("enabledTable");
545     HTableDescriptor htdEnabled = new HTableDescriptor(TableName.valueOf(enabledTable));
546     htdEnabled.addFamily(new HColumnDescriptor(FAMILY));
547     FileSystem filesystem = FileSystem.get(conf);
548     Path rootdir = FSUtils.getRootDir(conf);
549     FSTableDescriptors fstd = new FSTableDescriptors(filesystem, rootdir);
550     // Write the .tableinfo
551     fstd.createTableDescriptor(htdEnabled);
552     HRegionInfo hriEnabled = new HRegionInfo(htdEnabled.getTableName(),
553         null, null);
554     createRegion(hriEnabled, rootdir, conf, htdEnabled);
555 
556     List<HRegionInfo> enabledRegions = TEST_UTIL.createMultiRegionsInMeta(
557         TEST_UTIL.getConfiguration(), htdEnabled, SPLIT_KEYS);
558 
559     TableName disabledTable =
560         TableName.valueOf("disabledTable");
561     HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable);
562     htdDisabled.addFamily(new HColumnDescriptor(FAMILY));
563     // Write the .tableinfo
564     fstd.createTableDescriptor(htdDisabled);
565     HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getTableName(), null, null);
566     createRegion(hriDisabled, rootdir, conf, htdDisabled);
567 
568     List<HRegionInfo> disabledRegions = TEST_UTIL.createMultiRegionsInMeta(
569         TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS);
570 
571     log("Regions in hbase:meta and Namespace have been created");
572 
573     // at this point we only expect 2 regions to be assigned out (catalogs and namespace  )
574     assertEquals(2, cluster.countServedRegions());
575 
576     // The first RS will stay online
577     List<RegionServerThread> regionservers =
578       cluster.getRegionServerThreads();
579     HRegionServer hrs = regionservers.get(0).getRegionServer();
580 
581     // The second RS is going to be hard-killed
582     RegionServerThread hrsDeadThread = regionservers.get(1);
583     HRegionServer hrsDead = hrsDeadThread.getRegionServer();
584     ServerName deadServerName = hrsDead.getServerName();
585 
586     // we'll need some regions to already be assigned out properly on live RS
587     List<HRegionInfo> enabledAndAssignedRegions = new ArrayList<HRegionInfo>();
588     enabledAndAssignedRegions.addAll(enabledRegions.subList(0, 6));
589     enabledRegions.removeAll(enabledAndAssignedRegions);
590     List<HRegionInfo> disabledAndAssignedRegions = new ArrayList<HRegionInfo>();
591     disabledAndAssignedRegions.addAll(disabledRegions.subList(0, 6));
592     disabledRegions.removeAll(disabledAndAssignedRegions);
593 
594     // now actually assign them
595     for (HRegionInfo hri : enabledAndAssignedRegions) {
596       master.assignmentManager.regionPlans.put(hri.getEncodedName(),
597           new RegionPlan(hri, null, hrs.getServerName()));
598       master.assignRegion(hri);
599     }
600     for (HRegionInfo hri : disabledAndAssignedRegions) {
601       master.assignmentManager.regionPlans.put(hri.getEncodedName(),
602           new RegionPlan(hri, null, hrs.getServerName()));
603       master.assignRegion(hri);
604     }
605 
606     log("Waiting for assignment to finish");
607     ZKAssign.blockUntilNoRIT(zkw);
608     master.assignmentManager.waitUntilNoRegionsInTransition(60000);
609     log("Assignment completed");
610 
611     assertTrue(" Table must be enabled.", master.getAssignmentManager()
612         .getZKTable().isEnabledTable(TableName.valueOf("enabledTable")));
613     // we also need regions assigned out on the dead server
614     List<HRegionInfo> enabledAndOnDeadRegions = new ArrayList<HRegionInfo>();
615     enabledAndOnDeadRegions.addAll(enabledRegions.subList(0, 6));
616     enabledRegions.removeAll(enabledAndOnDeadRegions);
617     List<HRegionInfo> disabledAndOnDeadRegions = new ArrayList<HRegionInfo>();
618     disabledAndOnDeadRegions.addAll(disabledRegions.subList(0, 6));
619     disabledRegions.removeAll(disabledAndOnDeadRegions);
620 
621     // set region plan to server to be killed and trigger assign
622     for (HRegionInfo hri : enabledAndOnDeadRegions) {
623       master.assignmentManager.regionPlans.put(hri.getEncodedName(),
624           new RegionPlan(hri, null, deadServerName));
625       master.assignRegion(hri);
626     }
627     for (HRegionInfo hri : disabledAndOnDeadRegions) {
628       master.assignmentManager.regionPlans.put(hri.getEncodedName(),
629           new RegionPlan(hri, null, deadServerName));
630       master.assignRegion(hri);
631     }
632 
633     // wait for no more RIT
634     log("Waiting for assignment to finish");
635     ZKAssign.blockUntilNoRIT(zkw);
636     master.assignmentManager.waitUntilNoRegionsInTransition(60000);
637     log("Assignment completed");
638 
639     // Due to master.assignRegion(hri) could fail to assign a region to a specified RS
640     // therefore, we need make sure that regions are in the expected RS
641     verifyRegionLocation(hrs, enabledAndAssignedRegions);
642     verifyRegionLocation(hrs, disabledAndAssignedRegions);
643     verifyRegionLocation(hrsDead, enabledAndOnDeadRegions);
644     verifyRegionLocation(hrsDead, disabledAndOnDeadRegions);
645 
646     assertTrue(" Didn't get enough regions of enabledTalbe on live rs.",
647       enabledAndAssignedRegions.size() >= 2);
648     assertTrue(" Didn't get enough regions of disalbedTable on live rs.",
649       disabledAndAssignedRegions.size() >= 2);
650     assertTrue(" Didn't get enough regions of enabledTalbe on dead rs.",
651       enabledAndOnDeadRegions.size() >= 2);
652     assertTrue(" Didn't get enough regions of disalbedTable on dead rs.",
653       disabledAndOnDeadRegions.size() >= 2);
654 
655     // Stop the master
656     log("Aborting master");
657     cluster.abortMaster(0);
658     cluster.waitOnMaster(0);
659     log("Master has aborted");
660 
661     /*
662      * Now, let's start mocking up some weird states as described in the method
663      * javadoc.
664      */
665 
666     List<HRegionInfo> regionsThatShouldBeOnline = new ArrayList<HRegionInfo>();
667     List<HRegionInfo> regionsThatShouldBeOffline = new ArrayList<HRegionInfo>();
668 
669     log("Beginning to mock scenarios");
670 
671     // Disable the disabledTable in ZK
672     ZKTable zktable = new ZKTable(zkw);
673     zktable.setDisabledTable(disabledTable);
674 
675     assertTrue(" The enabled table should be identified on master fail over.",
676         zktable.isEnabledTable(TableName.valueOf("enabledTable")));
677 
678     /*
679      * ZK = CLOSING
680      */
681 
682     // Region of enabled table being closed on dead RS but not finished
683     HRegionInfo region = enabledAndOnDeadRegions.remove(0);
684     regionsThatShouldBeOnline.add(region);
685     ZKAssign.createNodeClosing(zkw, region, deadServerName);
686     LOG.debug("\n\nRegion of enabled table was CLOSING on dead RS\n" +
687         region + "\n\n");
688 
689     // Region of disabled table being closed on dead RS but not finished
690     region = disabledAndOnDeadRegions.remove(0);
691     regionsThatShouldBeOffline.add(region);
692     ZKAssign.createNodeClosing(zkw, region, deadServerName);
693     LOG.debug("\n\nRegion of disabled table was CLOSING on dead RS\n" +
694         region + "\n\n");
695 
696     /*
697      * ZK = CLOSED
698      */
699 
700     // Region of enabled on dead server gets closed but not ack'd by master
701     region = enabledAndOnDeadRegions.remove(0);
702     regionsThatShouldBeOnline.add(region);
703     int version = ZKAssign.createNodeClosing(zkw, region, deadServerName);
704     ZKAssign.transitionNodeClosed(zkw, region, deadServerName, version);
705     LOG.debug("\n\nRegion of enabled table was CLOSED on dead RS\n" +
706         region + "\n\n");
707 
708     // Region of disabled on dead server gets closed but not ack'd by master
709     region = disabledAndOnDeadRegions.remove(0);
710     regionsThatShouldBeOffline.add(region);
711     version = ZKAssign.createNodeClosing(zkw, region, deadServerName);
712     ZKAssign.transitionNodeClosed(zkw, region, deadServerName, version);
713     LOG.debug("\n\nRegion of disabled table was CLOSED on dead RS\n" +
714         region + "\n\n");
715 
716     /*
717      * ZK = OPENING
718      */
719 
720     // RS was opening a region of enabled table then died
721     region = enabledRegions.remove(0);
722     regionsThatShouldBeOnline.add(region);
723     ZKAssign.createNodeOffline(zkw, region, deadServerName);
724     ZKAssign.transitionNodeOpening(zkw, region, deadServerName);
725     LOG.debug("\n\nRegion of enabled table was OPENING on dead RS\n" +
726         region + "\n\n");
727 
728     // RS was opening a region of disabled table then died
729     region = disabledRegions.remove(0);
730     regionsThatShouldBeOffline.add(region);
731     ZKAssign.createNodeOffline(zkw, region, deadServerName);
732     ZKAssign.transitionNodeOpening(zkw, region, deadServerName);
733     LOG.debug("\n\nRegion of disabled table was OPENING on dead RS\n" +
734         region + "\n\n");
735 
736     /*
737      * ZK = OPENED
738      */
739 
740     // Region of enabled table was opened on dead RS
741     region = enabledRegions.remove(0);
742     regionsThatShouldBeOnline.add(region);
743     ZKAssign.createNodeOffline(zkw, region, deadServerName);
744     ProtobufUtil.openRegion(hrsDead, hrsDead.getServerName(), region);
745     while (true) {
746       byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
747       RegionTransition rt = RegionTransition.parseFrom(bytes);
748       if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
749         break;
750       }
751       Thread.sleep(100);
752     }
753     LOG.debug("\n\nRegion of enabled table was OPENED on dead RS\n" +
754         region + "\n\n");
755 
756     // Region of disabled table was opened on dead RS
757     region = disabledRegions.remove(0);
758     regionsThatShouldBeOffline.add(region);
759     ZKAssign.createNodeOffline(zkw, region, deadServerName);
760     ProtobufUtil.openRegion(hrsDead, hrsDead.getServerName(), region);
761     while (true) {
762       byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
763       RegionTransition rt = RegionTransition.parseFrom(bytes);
764       if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
765         break;
766       }
767       Thread.sleep(100);
768     }
769     LOG.debug("\n\nRegion of disabled table was OPENED on dead RS\n" +
770         region + "\n\n");
771 
772     /*
773      * ZK = NONE
774      */
775 
776     // Region of enabled table was open at steady-state on dead RS
777     region = enabledRegions.remove(0);
778     regionsThatShouldBeOnline.add(region);
779     ZKAssign.createNodeOffline(zkw, region, deadServerName);
780     ProtobufUtil.openRegion(hrsDead, hrsDead.getServerName(), region);
781     while (true) {
782       byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
783       RegionTransition rt = RegionTransition.parseFrom(bytes);
784       if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
785         ZKAssign.deleteOpenedNode(zkw, region.getEncodedName(), rt.getServerName());
786         LOG.debug("DELETED " + rt);
787         break;
788       }
789       Thread.sleep(100);
790     }
791     LOG.debug("\n\nRegion of enabled table was open at steady-state on dead RS"
792         + "\n" + region + "\n\n");
793 
794     // Region of disabled table was open at steady-state on dead RS
795     region = disabledRegions.remove(0);
796     regionsThatShouldBeOffline.add(region);
797     ZKAssign.createNodeOffline(zkw, region, deadServerName);
798     ProtobufUtil.openRegion(hrsDead, hrsDead.getServerName(), region);
799     while (true) {
800       byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
801       RegionTransition rt = RegionTransition.parseFrom(bytes);
802       if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
803         ZKAssign.deleteOpenedNode(zkw, region.getEncodedName(), rt.getServerName());
804         break;
805       }
806       Thread.sleep(100);
807     }
808     LOG.debug("\n\nRegion of disabled table was open at steady-state on dead RS"
809       + "\n" + region + "\n\n");
810 
811     /*
812      * DONE MOCKING
813      */
814 
815     log("Done mocking data up in ZK");
816 
817     // Kill the RS that had a hard death
818     log("Killing RS " + deadServerName);
819     hrsDead.abort("Killing for unit test");
820     log("RS " + deadServerName + " killed");
821 
822     // Start up a new master.  Wait until regionserver is completely down
823     // before starting new master because of hbase-4511.
824     while (hrsDeadThread.isAlive()) {
825       Threads.sleep(10);
826     }
827     log("Starting up a new master");
828     master = cluster.startMaster().getMaster();
829     log("Waiting for master to be ready");
830     assertTrue(cluster.waitForActiveAndReadyMaster());
831     log("Master is ready");
832 
833     // Wait until SSH processing completed for dead server.
834     while (master.getServerManager().areDeadServersInProgress()) {
835       Thread.sleep(10);
836     }
837 
838     // Failover should be completed, now wait for no RIT
839     log("Waiting for no more RIT");
840     ZKAssign.blockUntilNoRIT(zkw);
841     log("No more RIT in ZK");
842     long now = System.currentTimeMillis();
843     long maxTime = 120000;
844     boolean done = master.assignmentManager.waitUntilNoRegionsInTransition(maxTime);
845     if (!done) {
846       RegionStates regionStates = master.getAssignmentManager().getRegionStates();
847       LOG.info("rit=" + regionStates.getRegionsInTransition());
848     }
849     long elapsed = System.currentTimeMillis() - now;
850     assertTrue("Elapsed=" + elapsed + ", maxTime=" + maxTime + ", done=" + done,
851       elapsed < maxTime);
852     log("No more RIT in RIT map, doing final test verification");
853 
854     // Grab all the regions that are online across RSs
855     Set<HRegionInfo> onlineRegions = new TreeSet<HRegionInfo>();
856     now = System.currentTimeMillis();
857     maxTime = 30000;
858     for (JVMClusterUtil.RegionServerThread rst :
859         cluster.getRegionServerThreads()) {
860       try {
861         HRegionServer rs = rst.getRegionServer();
862         while (!rs.getRegionsInTransitionInRS().isEmpty()) {
863           elapsed = System.currentTimeMillis() - now;
864           assertTrue("Test timed out in getting online regions", elapsed < maxTime);
865           if (rs.isAborted() || rs.isStopped()) {
866             // This region server is stopped, skip it.
867             break;
868           }
869           Thread.sleep(100);
870         }
871         onlineRegions.addAll(ProtobufUtil.getOnlineRegions(rs));
872       } catch (RegionServerStoppedException e) {
873         LOG.info("Got RegionServerStoppedException", e);
874       }
875     }
876 
877     // Now, everything that should be online should be online
878     for (HRegionInfo hri : regionsThatShouldBeOnline) {
879       assertTrue("region=" + hri.getRegionNameAsString() + ", " + onlineRegions.toString(),
880         onlineRegions.contains(hri));
881     }
882 
883     // Everything that should be offline should not be online
884     for (HRegionInfo hri : regionsThatShouldBeOffline) {
885       assertFalse(onlineRegions.contains(hri));
886     }
887 
888     log("Done with verification, all passed, shutting down cluster");
889 
890     // Done, shutdown the cluster
891     TEST_UTIL.shutdownMiniCluster();
892   }
893 
894   /**
895    * Verify regions are on the expected region server
896    */
897   private void verifyRegionLocation(HRegionServer hrs, List<HRegionInfo> regions)
898       throws IOException {
899     List<HRegionInfo> tmpOnlineRegions = ProtobufUtil.getOnlineRegions(hrs);
900     Iterator<HRegionInfo> itr = regions.iterator();
901     while (itr.hasNext()) {
902       HRegionInfo tmp = itr.next();
903       if (!tmpOnlineRegions.contains(tmp)) {
904         itr.remove();
905       }
906     }
907   }
908 
909   HRegion createRegion(final HRegionInfo  hri, final Path rootdir, final Configuration c,
910       final HTableDescriptor htd)
911   throws IOException {
912     HRegion r = HRegion.createHRegion(hri, rootdir, c, htd);
913     // The above call to create a region will create an hlog file.  Each
914     // log file create will also create a running thread to do syncing.  We need
915     // to close out this log else we will have a running thread trying to sync
916     // the file system continuously which is ugly when dfs is taken away at the
917     // end of the test.
918     HRegion.closeHRegion(r);
919     return r;
920   }
921 
922   // TODO: Next test to add is with testing permutations of the RIT or the RS
923   //       killed are hosting ROOT and hbase:meta regions.
924 
925   private void log(String string) {
926     LOG.info("\n\n" + string + " \n\n");
927   }
928 
929   @Test (timeout=180000)
930   public void testShouldCheckMasterFailOverWhenMETAIsInOpenedState()
931       throws Exception {
932     LOG.info("Starting testShouldCheckMasterFailOverWhenMETAIsInOpenedState");
933     final int NUM_MASTERS = 1;
934     final int NUM_RS = 2;
935 
936     // Start the cluster
937     HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
938     Configuration conf = TEST_UTIL.getConfiguration();
939     conf.setInt("hbase.master.info.port", -1);
940     conf.setBoolean("hbase.assignment.usezk", true);
941 
942     TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
943     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
944 
945     // Find regionserver carrying meta.
946     List<RegionServerThread> regionServerThreads =
947       cluster.getRegionServerThreads();
948     int count = -1;
949     HRegion metaRegion = null;
950     for (RegionServerThread regionServerThread : regionServerThreads) {
951       HRegionServer regionServer = regionServerThread.getRegionServer();
952       metaRegion = regionServer.getOnlineRegion(HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
953       count++;
954       regionServer.abort("");
955       if (null != metaRegion) break;
956     }
957     HRegionServer regionServer = cluster.getRegionServer(count);
958 
959     TEST_UTIL.shutdownMiniHBaseCluster();
960 
961     // Create a ZKW to use in the test
962     ZooKeeperWatcher zkw =
963       HBaseTestingUtility.createAndForceNodeToOpenedState(TEST_UTIL,
964           metaRegion, regionServer.getServerName());
965 
966     LOG.info("Staring cluster for second time");
967     TEST_UTIL.startMiniHBaseCluster(NUM_MASTERS, NUM_RS);
968 
969     HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
970     while (!master.isInitialized()) {
971       Thread.sleep(100);
972     }
973     // Failover should be completed, now wait for no RIT
974     log("Waiting for no more RIT");
975     ZKAssign.blockUntilNoRIT(zkw);
976 
977     zkw.close();
978     // Stop the cluster
979     TEST_UTIL.shutdownMiniCluster();
980   }
981 
982   /**
983    * This tests a RIT in offline state will get re-assigned after a master restart
984    */
985   @Test(timeout=240000)
986   public void testOfflineRegionReAssginedAfterMasterRestart() throws Exception {
987     final TableName table = TableName.valueOf("testOfflineRegionReAssginedAfterMasterRestart");
988     final int NUM_MASTERS = 1;
989     final int NUM_RS = 2;
990 
991     // Create config to use for this cluster
992     Configuration conf = HBaseConfiguration.create();
993     conf.setBoolean("hbase.assignment.usezk", true);
994 
995     // Start the cluster
996     final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
997     TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
998     log("Cluster started");
999 
1000     TEST_UTIL.createTable(table, Bytes.toBytes("family"));
1001     HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
1002     RegionStates regionStates = master.getAssignmentManager().getRegionStates();
1003     HRegionInfo hri = regionStates.getRegionsOfTable(table).get(0);
1004     ServerName serverName = regionStates.getRegionServerOfRegion(hri);
1005     TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
1006 
1007     ServerName dstName = null;
1008     for (ServerName tmpServer : master.serverManager.getOnlineServers().keySet()) {
1009       if (!tmpServer.equals(serverName)) {
1010         dstName = tmpServer;
1011         break;
1012       }
1013     }
1014     // find a different server
1015     assertTrue(dstName != null);
1016     // shutdown HBase cluster
1017     TEST_UTIL.shutdownMiniHBaseCluster();
1018     // create a RIT node in offline state
1019     ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
1020     ZKAssign.createNodeOffline(zkw, hri, dstName);
1021     Stat stat = new Stat();
1022     byte[] data =
1023         ZKAssign.getDataNoWatch(zkw, hri.getEncodedName(), stat);
1024     assertTrue(data != null);
1025     RegionTransition rt = RegionTransition.parseFrom(data);
1026     assertTrue(rt.getEventType() == EventType.M_ZK_REGION_OFFLINE);
1027 
1028     LOG.info(hri.getEncodedName() + " region is in offline state with source server=" + serverName
1029         + " and dst server=" + dstName);
1030 
1031     // start HBase cluster
1032     TEST_UTIL.startMiniHBaseCluster(NUM_MASTERS, NUM_RS);
1033 
1034     while (true) {
1035       master = TEST_UTIL.getHBaseCluster().getMaster();
1036       if (master != null && master.isInitialized()) {
1037         ServerManager serverManager = master.getServerManager();
1038         if (!serverManager.areDeadServersInProgress()) {
1039           break;
1040         }
1041       }
1042       Thread.sleep(200);
1043     }
1044 
1045     // verify the region is assigned
1046     master = TEST_UTIL.getHBaseCluster().getMaster();
1047     master.getAssignmentManager().waitForAssignment(hri);
1048     regionStates = master.getAssignmentManager().getRegionStates();
1049     RegionState newState = regionStates.getRegionState(hri);
1050     assertTrue(newState.isOpened());
1051   }
1052 
1053   /**
1054    * Simple test of master failover.
1055    * <p>
1056    * Starts with three masters.  Kills a backup master.  Then kills the active
1057    * master.  Ensures the final master becomes active and we can still contact
1058    * the cluster.
1059    * @throws Exception
1060    */
1061   @Test (timeout=240000)
1062   public void testSimpleMasterFailover() throws Exception {
1063 
1064     final int NUM_MASTERS = 3;
1065     final int NUM_RS = 3;
1066 
1067     // Start the cluster
1068     HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
1069 
1070     TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
1071     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1072 
1073     // get all the master threads
1074     List<MasterThread> masterThreads = cluster.getMasterThreads();
1075 
1076     // wait for each to come online
1077     for (MasterThread mt : masterThreads) {
1078       assertTrue(mt.isAlive());
1079     }
1080 
1081     // verify only one is the active master and we have right number
1082     int numActive = 0;
1083     int activeIndex = -1;
1084     ServerName activeName = null;
1085     HMaster active = null;
1086     for (int i = 0; i < masterThreads.size(); i++) {
1087       if (masterThreads.get(i).getMaster().isActiveMaster()) {
1088         numActive++;
1089         activeIndex = i;
1090         active = masterThreads.get(activeIndex).getMaster();
1091         activeName = active.getServerName();
1092       }
1093     }
1094     assertEquals(1, numActive);
1095     assertEquals(NUM_MASTERS, masterThreads.size());
1096     LOG.info("Active master " + activeName);
1097 
1098     // Check that ClusterStatus reports the correct active and backup masters
1099     assertNotNull(active);
1100     ClusterStatus status = active.getClusterStatus();
1101     assertTrue(status.getMaster().equals(activeName));
1102     assertEquals(2, status.getBackupMastersSize());
1103     assertEquals(2, status.getBackupMasters().size());
1104 
1105     // attempt to stop one of the inactive masters
1106     int backupIndex = (activeIndex == 0 ? 1 : activeIndex - 1);
1107     HMaster master = cluster.getMaster(backupIndex);
1108     LOG.debug("\n\nStopping a backup master: " + master.getServerName() + "\n");
1109     cluster.stopMaster(backupIndex, false);
1110     cluster.waitOnMaster(backupIndex);
1111 
1112     // Verify still one active master and it's the same
1113     for (int i = 0; i < masterThreads.size(); i++) {
1114       if (masterThreads.get(i).getMaster().isActiveMaster()) {
1115         assertTrue(activeName.equals(masterThreads.get(i).getMaster().getServerName()));
1116         activeIndex = i;
1117         active = masterThreads.get(activeIndex).getMaster();
1118       }
1119     }
1120     assertEquals(1, numActive);
1121     assertEquals(2, masterThreads.size());
1122     int rsCount = masterThreads.get(activeIndex).getMaster().getClusterStatus().getServersSize();
1123     LOG.info("Active master " + active.getServerName() + " managing " + rsCount +  " regions servers");
1124     assertEquals(3, rsCount);
1125 
1126     // Check that ClusterStatus reports the correct active and backup masters
1127     assertNotNull(active);
1128     status = active.getClusterStatus();
1129     assertTrue(status.getMaster().equals(activeName));
1130     assertEquals(1, status.getBackupMastersSize());
1131     assertEquals(1, status.getBackupMasters().size());
1132 
1133     // kill the active master
1134     LOG.debug("\n\nStopping the active master " + active.getServerName() + "\n");
1135     cluster.stopMaster(activeIndex, false);
1136     cluster.waitOnMaster(activeIndex);
1137 
1138     // wait for an active master to show up and be ready
1139     assertTrue(cluster.waitForActiveAndReadyMaster());
1140 
1141     LOG.debug("\n\nVerifying backup master is now active\n");
1142     // should only have one master now
1143     assertEquals(1, masterThreads.size());
1144 
1145     // and he should be active
1146     active = masterThreads.get(0).getMaster();
1147     assertNotNull(active);
1148     status = active.getClusterStatus();
1149     ServerName mastername = status.getMaster();
1150     assertTrue(mastername.equals(active.getServerName()));
1151     assertTrue(active.isActiveMaster());
1152     assertEquals(0, status.getBackupMastersSize());
1153     assertEquals(0, status.getBackupMasters().size());
1154     int rss = status.getServersSize();
1155     LOG.info("Active master " + mastername.getServerName() + " managing " +
1156       rss +  " region servers");
1157     assertEquals(3, rss);
1158 
1159     // Stop the cluster
1160     TEST_UTIL.shutdownMiniCluster();
1161   }
1162 
1163   /**
1164    * Test region in pending_open/close when master failover
1165    */
1166   @Test (timeout=180000)
1167   public void testPendingOpenOrCloseWhenMasterFailover() throws Exception {
1168     final int NUM_MASTERS = 1;
1169     final int NUM_RS = 1;
1170 
1171     // Create config to use for this cluster
1172     Configuration conf = HBaseConfiguration.create();
1173     conf.setBoolean("hbase.assignment.usezk", false);
1174 
1175     // Start the cluster
1176     HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
1177     TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
1178     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1179     log("Cluster started");
1180 
1181     // get all the master threads
1182     List<MasterThread> masterThreads = cluster.getMasterThreads();
1183     assertEquals(1, masterThreads.size());
1184 
1185     // only one master thread, let's wait for it to be initialized
1186     assertTrue(cluster.waitForActiveAndReadyMaster());
1187     HMaster master = masterThreads.get(0).getMaster();
1188     assertTrue(master.isActiveMaster());
1189     assertTrue(master.isInitialized());
1190 
1191     // Create a table with a region online
1192     HTable onlineTable = TEST_UTIL.createTable("onlineTable", "family");
1193 
1194     // Create a table in META, so it has a region offline
1195     HTableDescriptor offlineTable = new HTableDescriptor(
1196       TableName.valueOf(Bytes.toBytes("offlineTable")));
1197     offlineTable.addFamily(new HColumnDescriptor(Bytes.toBytes("family")));
1198 
1199     FileSystem filesystem = FileSystem.get(conf);
1200     Path rootdir = FSUtils.getRootDir(conf);
1201     FSTableDescriptors fstd = new FSTableDescriptors(filesystem, rootdir);
1202     fstd.createTableDescriptor(offlineTable);
1203 
1204     HRegionInfo hriOffline = new HRegionInfo(offlineTable.getTableName(), null, null);
1205     createRegion(hriOffline, rootdir, conf, offlineTable);
1206     MetaEditor.addRegionToMeta(master.getCatalogTracker(), hriOffline);
1207 
1208     log("Regions in hbase:meta and namespace have been created");
1209 
1210     // at this point we only expect 3 regions to be assigned out
1211     // (catalogs and namespace, + 1 online region)
1212     assertEquals(3, cluster.countServedRegions());
1213     HRegionInfo hriOnline = onlineTable.getRegionLocation("").getRegionInfo();
1214 
1215     RegionStates regionStates = master.getAssignmentManager().getRegionStates();
1216     RegionStateStore stateStore = master.getAssignmentManager().getRegionStateStore();
1217 
1218     // Put the online region in pending_close. It is actually already opened.
1219     // This is to simulate that the region close RPC is not sent out before failover
1220     RegionState oldState = regionStates.getRegionState(hriOnline);
1221     RegionState newState = new RegionState(hriOnline, State.PENDING_CLOSE, oldState.getServerName());
1222     stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState);
1223 
1224     // Put the offline region in pending_open. It is actually not opened yet.
1225     // This is to simulate that the region open RPC is not sent out before failover
1226     oldState = new RegionState(hriOffline, State.OFFLINE);
1227     newState = new RegionState(hriOffline, State.PENDING_OPEN, newState.getServerName());
1228     stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState);
1229 
1230     // Stop the master
1231     log("Aborting master");
1232     cluster.abortMaster(0);
1233     cluster.waitOnMaster(0);
1234     log("Master has aborted");
1235 
1236     // Start up a new master
1237     log("Starting up a new master");
1238     master = cluster.startMaster().getMaster();
1239     log("Waiting for master to be ready");
1240     cluster.waitForActiveAndReadyMaster();
1241     log("Master is ready");
1242 
1243     // Wait till no region in transition any more
1244     master.getAssignmentManager().waitUntilNoRegionsInTransition(60000);
1245 
1246     // Get new region states since master restarted
1247     regionStates = master.getAssignmentManager().getRegionStates();
1248 
1249     // Both pending_open (RPC sent/not yet) regions should be online
1250     assertTrue(regionStates.isRegionOnline(hriOffline));
1251     assertTrue(regionStates.isRegionOnline(hriOnline));
1252 
1253     log("Done with verification, shutting down cluster");
1254 
1255     // Done, shutdown the cluster
1256     TEST_UTIL.shutdownMiniCluster();
1257   }
1258 }
1259