View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertNotNull;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.io.IOException;
26  import java.util.concurrent.Semaphore;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.CoordinatedStateManager;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.testclassification.MediumTests;
34  import org.apache.hadoop.hbase.Server;
35  import org.apache.hadoop.hbase.ServerName;
36  import org.apache.hadoop.hbase.client.ClusterConnection;
37  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
38  import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
39  import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
40  import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
41  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
42  import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
43  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
44  import org.apache.zookeeper.KeeperException;
45  import org.junit.AfterClass;
46  import org.junit.BeforeClass;
47  import org.junit.Test;
48  import org.junit.experimental.categories.Category;
49  import org.mockito.Mockito;
50  
51  /**
52   * Test the {@link ActiveMasterManager}.
53   */
54  @Category(MediumTests.class)
55  public class TestActiveMasterManager {
56    private final static Log LOG = LogFactory.getLog(TestActiveMasterManager.class);
57    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
58  
59    @BeforeClass
60    public static void setUpBeforeClass() throws Exception {
61      TEST_UTIL.startMiniZKCluster();
62    }
63  
64    @AfterClass
65    public static void tearDownAfterClass() throws Exception {
66      TEST_UTIL.shutdownMiniZKCluster();
67    }
68  
69    @Test public void testRestartMaster() throws IOException, KeeperException {
70      ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
71        "testActiveMasterManagerFromZK", null, true);
72      try {
73        ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
74        ZKUtil.deleteNode(zk, zk.clusterStateZNode);
75      } catch(KeeperException.NoNodeException nne) {}
76  
77      // Create the master node with a dummy address
78      ServerName master = ServerName.valueOf("localhost", 1, System.currentTimeMillis());
79      // Should not have a master yet
80      DummyMaster dummyMaster = new DummyMaster(zk,master);
81      ClusterStatusTracker clusterStatusTracker =
82        dummyMaster.getClusterStatusTracker();
83      ActiveMasterManager activeMasterManager =
84        dummyMaster.getActiveMasterManager();
85      assertFalse(activeMasterManager.clusterHasActiveMaster.get());
86  
87      // First test becoming the active master uninterrupted
88      MonitoredTask status = Mockito.mock(MonitoredTask.class);
89      clusterStatusTracker.setClusterUp();
90  
91      activeMasterManager.blockUntilBecomingActiveMaster(100, status);
92      assertTrue(activeMasterManager.clusterHasActiveMaster.get());
93      assertMaster(zk, master);
94  
95      // Now pretend master restart
96      DummyMaster secondDummyMaster = new DummyMaster(zk,master);
97      ActiveMasterManager secondActiveMasterManager =
98        secondDummyMaster.getActiveMasterManager();
99      assertFalse(secondActiveMasterManager.clusterHasActiveMaster.get());
100     activeMasterManager.blockUntilBecomingActiveMaster(100, status);
101     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
102     assertMaster(zk, master);
103   }
104 
105   /**
106    * Unit tests that uses ZooKeeper but does not use the master-side methods
107    * but rather acts directly on ZK.
108    * @throws Exception
109    */
110   @Test
111   public void testActiveMasterManagerFromZK() throws Exception {
112     ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
113       "testActiveMasterManagerFromZK", null, true);
114     try {
115       ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
116       ZKUtil.deleteNode(zk, zk.clusterStateZNode);
117     } catch(KeeperException.NoNodeException nne) {}
118 
119     // Create the master node with a dummy address
120     ServerName firstMasterAddress =
121         ServerName.valueOf("localhost", 1, System.currentTimeMillis());
122     ServerName secondMasterAddress =
123         ServerName.valueOf("localhost", 2, System.currentTimeMillis());
124 
125     // Should not have a master yet
126     DummyMaster ms1 = new DummyMaster(zk,firstMasterAddress);
127     ActiveMasterManager activeMasterManager =
128       ms1.getActiveMasterManager();
129     assertFalse(activeMasterManager.clusterHasActiveMaster.get());
130 
131     // First test becoming the active master uninterrupted
132     ClusterStatusTracker clusterStatusTracker =
133       ms1.getClusterStatusTracker();
134     clusterStatusTracker.setClusterUp();
135     activeMasterManager.blockUntilBecomingActiveMaster(100, 
136         Mockito.mock(MonitoredTask.class));
137     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
138     assertMaster(zk, firstMasterAddress);
139 
140     // New manager will now try to become the active master in another thread
141     WaitToBeMasterThread t = new WaitToBeMasterThread(zk, secondMasterAddress);
142     t.start();
143     // Wait for this guy to figure out there is another active master
144     // Wait for 1 second at most
145     int sleeps = 0;
146     while(!t.manager.clusterHasActiveMaster.get() && sleeps < 100) {
147       Thread.sleep(10);
148       sleeps++;
149     }
150 
151     // Both should see that there is an active master
152     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
153     assertTrue(t.manager.clusterHasActiveMaster.get());
154     // But secondary one should not be the active master
155     assertFalse(t.isActiveMaster);
156 
157     // Close the first server and delete it's master node
158     ms1.stop("stopping first server");
159 
160     // Use a listener to capture when the node is actually deleted
161     NodeDeletionListener listener = new NodeDeletionListener(zk, zk.getMasterAddressZNode());
162     zk.registerListener(listener);
163 
164     LOG.info("Deleting master node");
165     ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
166 
167     // Wait for the node to be deleted
168     LOG.info("Waiting for active master manager to be notified");
169     listener.waitForDeletion();
170     LOG.info("Master node deleted");
171 
172     // Now we expect the secondary manager to have and be the active master
173     // Wait for 1 second at most
174     sleeps = 0;
175     while(!t.isActiveMaster && sleeps < 100) {
176       Thread.sleep(10);
177       sleeps++;
178     }
179     LOG.debug("Slept " + sleeps + " times");
180 
181     assertTrue(t.manager.clusterHasActiveMaster.get());
182     assertTrue(t.isActiveMaster);
183 
184     LOG.info("Deleting master node");
185     ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
186   }
187 
188   /**
189    * Assert there is an active master and that it has the specified address.
190    * @param zk
191    * @param thisMasterAddress
192    * @throws KeeperException
193    * @throws IOException 
194    */
195   private void assertMaster(ZooKeeperWatcher zk,
196       ServerName expectedAddress)
197   throws KeeperException, IOException {
198     ServerName readAddress = MasterAddressTracker.getMasterAddress(zk);
199     assertNotNull(readAddress);
200     assertTrue(expectedAddress.equals(readAddress));
201   }
202 
203   public static class WaitToBeMasterThread extends Thread {
204 
205     ActiveMasterManager manager;
206     DummyMaster dummyMaster;
207     boolean isActiveMaster;
208 
209     public WaitToBeMasterThread(ZooKeeperWatcher zk, ServerName address) {
210       this.dummyMaster = new DummyMaster(zk,address);
211       this.manager = this.dummyMaster.getActiveMasterManager();
212       isActiveMaster = false;
213     }
214 
215     @Override
216     public void run() {
217       manager.blockUntilBecomingActiveMaster(100,
218           Mockito.mock(MonitoredTask.class));
219       LOG.info("Second master has become the active master!");
220       isActiveMaster = true;
221     }
222   }
223 
224   public static class NodeDeletionListener extends ZooKeeperListener {
225     private static final Log LOG = LogFactory.getLog(NodeDeletionListener.class);
226 
227     private Semaphore lock;
228     private String node;
229 
230     public NodeDeletionListener(ZooKeeperWatcher watcher, String node) {
231       super(watcher);
232       lock = new Semaphore(0);
233       this.node = node;
234     }
235 
236     @Override
237     public void nodeDeleted(String path) {
238       if(path.equals(node)) {
239         LOG.debug("nodeDeleted(" + path + ")");
240         lock.release();
241       }
242     }
243 
244     public void waitForDeletion() throws InterruptedException {
245       lock.acquire();
246     }
247   }
248 
249   /**
250    * Dummy Master Implementation.
251    */
252   public static class DummyMaster implements Server {
253     private volatile boolean stopped;
254     private ClusterStatusTracker clusterStatusTracker;
255     private ActiveMasterManager activeMasterManager;
256 
257     public DummyMaster(ZooKeeperWatcher zk, ServerName master) {
258       this.clusterStatusTracker =
259         new ClusterStatusTracker(zk, this);
260       clusterStatusTracker.start();
261 
262       this.activeMasterManager =
263         new ActiveMasterManager(zk, master, this);
264       zk.registerListener(activeMasterManager);
265     }
266 
267     @Override
268     public void abort(final String msg, final Throwable t) {}
269     
270     @Override
271     public boolean isAborted() {
272       return false;
273     }
274 
275     @Override
276     public Configuration getConfiguration() {
277       return null;
278     }
279 
280     @Override
281     public ZooKeeperWatcher getZooKeeper() {
282       return null;
283     }
284 
285     @Override
286     public CoordinatedStateManager getCoordinatedStateManager() {
287       return null;
288     }
289 
290     @Override
291     public ServerName getServerName() {
292       return null;
293     }
294 
295     @Override
296     public boolean isStopped() {
297       return this.stopped;
298     }
299 
300     @Override
301     public void stop(String why) {
302       this.stopped = true;
303     }
304 
305     @Override
306     public ClusterConnection getConnection() {
307       return null;
308     }
309 
310     @Override
311     public MetaTableLocator getMetaTableLocator() {
312       return null;
313     }
314 
315     public ClusterStatusTracker getClusterStatusTracker() {
316       return clusterStatusTracker;
317     }
318 
319     public ActiveMasterManager getActiveMasterManager() {
320       return activeMasterManager;
321     }
322   }
323 }