1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertTrue;
25  
26  import java.io.IOException;
27  import java.util.concurrent.Semaphore;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.hbase.*;
33  import org.apache.hadoop.hbase.catalog.CatalogTracker;
34  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
35  import org.apache.hadoop.hbase.util.Bytes;
36  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
37  import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
38  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
39  import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
40  import org.apache.zookeeper.KeeperException;
41  import org.junit.AfterClass;
42  import org.junit.BeforeClass;
43  import org.junit.Test;
44  import org.junit.experimental.categories.Category;
45  import org.mockito.Mockito;
46  
47  /**
48   * Test the {@link ActiveMasterManager}.
49   */
50  @Category(MediumTests.class)
51  public class TestActiveMasterManager {
52    private final static Log LOG = LogFactory.getLog(TestActiveMasterManager.class);
53    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
54  
55    @BeforeClass
56    public static void setUpBeforeClass() throws Exception {
57      TEST_UTIL.startMiniZKCluster();
58    }
59  
60    @AfterClass
61    public static void tearDownAfterClass() throws Exception {
62      TEST_UTIL.shutdownMiniZKCluster();
63    }
64  
65    @Test public void testRestartMaster() throws IOException, KeeperException {
66      ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
67        "testActiveMasterManagerFromZK", null, true);
68      try {
69        ZKUtil.deleteNode(zk, zk.masterAddressZNode);
70        ZKUtil.deleteNode(zk, zk.clusterStateZNode);
71      } catch(KeeperException.NoNodeException nne) {}
72  
73      // Create the master node with a dummy address
74      ServerName master = new ServerName("localhost", 1, System.currentTimeMillis());
75      // Should not have a master yet
76      DummyMaster dummyMaster = new DummyMaster(zk,master);
77      ClusterStatusTracker clusterStatusTracker =
78        dummyMaster.getClusterStatusTracker();
79      ActiveMasterManager activeMasterManager =
80        dummyMaster.getActiveMasterManager();
81      assertFalse(activeMasterManager.clusterHasActiveMaster.get());
82  
83      // First test becoming the active master uninterrupted
84      MonitoredTask status = Mockito.mock(MonitoredTask.class);
85      clusterStatusTracker.setClusterUp();
86  
87      activeMasterManager.blockUntilBecomingActiveMaster(status);
88      assertTrue(activeMasterManager.clusterHasActiveMaster.get());
89      assertMaster(zk, master);
90  
91      // Now pretend master restart
92      DummyMaster secondDummyMaster = new DummyMaster(zk,master);
93      ActiveMasterManager secondActiveMasterManager =
94        secondDummyMaster.getActiveMasterManager();
95      assertFalse(secondActiveMasterManager.clusterHasActiveMaster.get());
96      activeMasterManager.blockUntilBecomingActiveMaster(status);
97      assertTrue(activeMasterManager.clusterHasActiveMaster.get());
98      assertMaster(zk, master);
99    }
100 
101   /**
102    * Unit tests that uses ZooKeeper but does not use the master-side methods
103    * but rather acts directly on ZK.
104    * @throws Exception
105    */
106   @Test
107   public void testActiveMasterManagerFromZK() throws Exception {
108     ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
109       "testActiveMasterManagerFromZK", null, true);
110     try {
111       ZKUtil.deleteNode(zk, zk.masterAddressZNode);
112       ZKUtil.deleteNode(zk, zk.clusterStateZNode);
113     } catch(KeeperException.NoNodeException nne) {}
114 
115     // Create the master node with a dummy address
116     ServerName firstMasterAddress =
117       new ServerName("localhost", 1, System.currentTimeMillis());
118     ServerName secondMasterAddress =
119       new ServerName("localhost", 2, System.currentTimeMillis());
120 
121     // Should not have a master yet
122     DummyMaster ms1 = new DummyMaster(zk,firstMasterAddress);
123     ActiveMasterManager activeMasterManager =
124       ms1.getActiveMasterManager();
125     assertFalse(activeMasterManager.clusterHasActiveMaster.get());
126 
127     // First test becoming the active master uninterrupted
128     ClusterStatusTracker clusterStatusTracker =
129       ms1.getClusterStatusTracker();
130     clusterStatusTracker.setClusterUp();
131     activeMasterManager.blockUntilBecomingActiveMaster(
132         Mockito.mock(MonitoredTask.class));
133     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
134     assertMaster(zk, firstMasterAddress);
135 
136     // New manager will now try to become the active master in another thread
137     WaitToBeMasterThread t = new WaitToBeMasterThread(zk, secondMasterAddress);
138     t.start();
139     // Wait for this guy to figure out there is another active master
140     // Wait for 1 second at most
141     int sleeps = 0;
142     while(!t.manager.clusterHasActiveMaster.get() && sleeps < 100) {
143       Thread.sleep(10);
144       sleeps++;
145     }
146 
147     // Both should see that there is an active master
148     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
149     assertTrue(t.manager.clusterHasActiveMaster.get());
150     // But secondary one should not be the active master
151     assertFalse(t.isActiveMaster);
152 
153     // Close the first server and delete it's master node
154     ms1.stop("stopping first server");
155 
156     // Use a listener to capture when the node is actually deleted
157     NodeDeletionListener listener = new NodeDeletionListener(zk, zk.masterAddressZNode);
158     zk.registerListener(listener);
159 
160     LOG.info("Deleting master node");
161     ZKUtil.deleteNode(zk, zk.masterAddressZNode);
162 
163     // Wait for the node to be deleted
164     LOG.info("Waiting for active master manager to be notified");
165     listener.waitForDeletion();
166     LOG.info("Master node deleted");
167 
168     // Now we expect the secondary manager to have and be the active master
169     // Wait for 1 second at most
170     sleeps = 0;
171     while(!t.isActiveMaster && sleeps < 100) {
172       Thread.sleep(10);
173       sleeps++;
174     }
175     LOG.debug("Slept " + sleeps + " times");
176 
177     assertTrue(t.manager.clusterHasActiveMaster.get());
178     assertTrue(t.isActiveMaster);
179 
180     LOG.info("Deleting master node");
181     ZKUtil.deleteNode(zk, zk.masterAddressZNode);
182   }
183 
184   /**
185    * Assert there is an active master and that it has the specified address.
186    * @param zk
187    * @param thisMasterAddress
188    * @throws KeeperException
189    */
190   private void assertMaster(ZooKeeperWatcher zk,
191       ServerName expectedAddress)
192   throws KeeperException {
193     ServerName readAddress =
194       ServerName.parseVersionedServerName(ZKUtil.getData(zk, zk.masterAddressZNode));
195     assertNotNull(readAddress);
196     assertTrue(expectedAddress.equals(readAddress));
197   }
198 
199   public static class WaitToBeMasterThread extends Thread {
200 
201     ActiveMasterManager manager;
202     DummyMaster dummyMaster;
203     boolean isActiveMaster;
204 
205     public WaitToBeMasterThread(ZooKeeperWatcher zk, ServerName address) {
206       this.dummyMaster = new DummyMaster(zk,address);
207       this.manager = this.dummyMaster.getActiveMasterManager();
208       isActiveMaster = false;
209     }
210 
211     @Override
212     public void run() {
213       manager.blockUntilBecomingActiveMaster(
214           Mockito.mock(MonitoredTask.class));
215       LOG.info("Second master has become the active master!");
216       isActiveMaster = true;
217     }
218   }
219 
220   public static class NodeDeletionListener extends ZooKeeperListener {
221     private static final Log LOG = LogFactory.getLog(NodeDeletionListener.class);
222 
223     private Semaphore lock;
224     private String node;
225 
226     public NodeDeletionListener(ZooKeeperWatcher watcher, String node) {
227       super(watcher);
228       lock = new Semaphore(0);
229       this.node = node;
230     }
231 
232     @Override
233     public void nodeDeleted(String path) {
234       if(path.equals(node)) {
235         LOG.debug("nodeDeleted(" + path + ")");
236         lock.release();
237       }
238     }
239 
240     public void waitForDeletion() throws InterruptedException {
241       lock.acquire();
242     }
243   }
244 
245   /**
246    * Dummy Master Implementation.
247    */
248   public static class DummyMaster implements Server {
249     private volatile boolean stopped;
250     private ClusterStatusTracker clusterStatusTracker;
251     private ActiveMasterManager activeMasterManager;
252 
253     public DummyMaster(ZooKeeperWatcher zk, ServerName master) {
254       this.clusterStatusTracker =
255         new ClusterStatusTracker(zk, this);
256       clusterStatusTracker.start();
257       
258       this.activeMasterManager =
259         new ActiveMasterManager(zk, master, this);
260       zk.registerListener(activeMasterManager);
261     }
262 
263     @Override
264     public void abort(final String msg, final Throwable t) {}
265     
266     @Override
267     public boolean isAborted() {
268       return false;
269     }
270 
271     @Override
272     public Configuration getConfiguration() {
273       return null;
274     }
275 
276     @Override
277     public ZooKeeperWatcher getZooKeeper() {
278       return null;
279     }
280 
281     @Override
282     public ServerName getServerName() {
283       return null;
284     }
285 
286     @Override
287     public boolean isStopped() {
288       return this.stopped;
289     }
290 
291     @Override
292     public void stop(String why) {
293       this.stopped = true;
294     }
295 
296     @Override
297     public CatalogTracker getCatalogTracker() {
298       return null;
299     }
300 
301     public ClusterStatusTracker getClusterStatusTracker() {
302       return clusterStatusTracker;
303     }
304 
305     public ActiveMasterManager getActiveMasterManager() {
306       return activeMasterManager;
307     }
308   }
309 
310   @org.junit.Rule
311   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
312     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
313 }
314