1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertTrue;
25  
26  import java.io.IOException;
27  import java.util.concurrent.Semaphore;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.hbase.*;
33  import org.apache.hadoop.hbase.catalog.CatalogTracker;
34  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
35  import org.apache.hadoop.hbase.util.Bytes;
36  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
37  import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
38  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
39  import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
40  import org.apache.zookeeper.KeeperException;
41  import org.junit.AfterClass;
42  import org.junit.BeforeClass;
43  import org.junit.Test;
44  import org.junit.experimental.categories.Category;
45  import org.mockito.Mockito;
46  
47  /**
48   * Test the {@link ActiveMasterManager}.
49   */
50  @Category(MediumTests.class)
51  public class TestActiveMasterManager {
52    private final static Log LOG = LogFactory.getLog(TestActiveMasterManager.class);
53    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
54  
55    @BeforeClass
56    public static void setUpBeforeClass() throws Exception {
57      TEST_UTIL.startMiniZKCluster();
58    }
59  
60    @AfterClass
61    public static void tearDownAfterClass() throws Exception {
62      TEST_UTIL.shutdownMiniZKCluster();
63    }
64  
65    @Test public void testRestartMaster() throws IOException, KeeperException {
66      ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
67        "testActiveMasterManagerFromZK", null, true);
68      try {
69        ZKUtil.deleteNode(zk, zk.masterAddressZNode);
70        ZKUtil.deleteNode(zk, zk.clusterStateZNode);
71      } catch(KeeperException.NoNodeException nne) {}
72  
73      // Create the master node with a dummy address
74      ServerName master = new ServerName("localhost", 1, System.currentTimeMillis());
75      // Should not have a master yet
76      DummyMaster dummyMaster = new DummyMaster(zk,master);
77      ClusterStatusTracker clusterStatusTracker =
78        dummyMaster.getClusterStatusTracker();
79      ActiveMasterManager activeMasterManager =
80        dummyMaster.getActiveMasterManager();
81      assertFalse(activeMasterManager.clusterHasActiveMaster.get());
82  
83      // First test becoming the active master uninterrupted
84      MonitoredTask status = Mockito.mock(MonitoredTask.class);
85      clusterStatusTracker.setClusterUp();
86  
87      activeMasterManager.blockUntilBecomingActiveMaster(status,clusterStatusTracker);
88      assertTrue(activeMasterManager.clusterHasActiveMaster.get());
89      assertMaster(zk, master);
90  
91      // Now pretend master restart
92      DummyMaster secondDummyMaster = new DummyMaster(zk,master);
93      ActiveMasterManager secondActiveMasterManager =
94        secondDummyMaster.getActiveMasterManager();
95      assertFalse(secondActiveMasterManager.clusterHasActiveMaster.get());
96      activeMasterManager.blockUntilBecomingActiveMaster(status,clusterStatusTracker);
97      assertTrue(activeMasterManager.clusterHasActiveMaster.get());
98      assertMaster(zk, master);
99    }
100 
101   /**
102    * Unit tests that uses ZooKeeper but does not use the master-side methods
103    * but rather acts directly on ZK.
104    * @throws Exception
105    */
106   @Test
107   public void testActiveMasterManagerFromZK() throws Exception {
108     ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
109       "testActiveMasterManagerFromZK", null, true);
110     try {
111       ZKUtil.deleteNode(zk, zk.masterAddressZNode);
112       ZKUtil.deleteNode(zk, zk.clusterStateZNode);
113     } catch(KeeperException.NoNodeException nne) {}
114 
115     // Create the master node with a dummy address
116     ServerName firstMasterAddress =
117       new ServerName("localhost", 1, System.currentTimeMillis());
118     ServerName secondMasterAddress =
119       new ServerName("localhost", 2, System.currentTimeMillis());
120 
121     // Should not have a master yet
122     DummyMaster ms1 = new DummyMaster(zk,firstMasterAddress);
123     ActiveMasterManager activeMasterManager =
124       ms1.getActiveMasterManager();
125     assertFalse(activeMasterManager.clusterHasActiveMaster.get());
126 
127     // First test becoming the active master uninterrupted
128     ClusterStatusTracker clusterStatusTracker =
129       ms1.getClusterStatusTracker();
130     clusterStatusTracker.setClusterUp();
131     activeMasterManager.blockUntilBecomingActiveMaster(
132         Mockito.mock(MonitoredTask.class),clusterStatusTracker);
133     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
134     assertMaster(zk, firstMasterAddress);
135 
136     // New manager will now try to become the active master in another thread
137     WaitToBeMasterThread t = new WaitToBeMasterThread(zk, secondMasterAddress);
138     t.start();
139     // Wait for this guy to figure out there is another active master
140     // Wait for 1 second at most
141     int sleeps = 0;
142     while(!t.manager.clusterHasActiveMaster.get() && sleeps < 100) {
143       Thread.sleep(10);
144       sleeps++;
145     }
146 
147     // Both should see that there is an active master
148     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
149     assertTrue(t.manager.clusterHasActiveMaster.get());
150     // But secondary one should not be the active master
151     assertFalse(t.isActiveMaster);
152 
153     // Close the first server and delete it's master node
154     ms1.stop("stopping first server");
155 
156     // Use a listener to capture when the node is actually deleted
157     NodeDeletionListener listener = new NodeDeletionListener(zk, zk.masterAddressZNode);
158     zk.registerListener(listener);
159 
160     LOG.info("Deleting master node");
161     ZKUtil.deleteNode(zk, zk.masterAddressZNode);
162 
163     // Wait for the node to be deleted
164     LOG.info("Waiting for active master manager to be notified");
165     listener.waitForDeletion();
166     LOG.info("Master node deleted");
167 
168     // Now we expect the secondary manager to have and be the active master
169     // Wait for 1 second at most
170     sleeps = 0;
171     while(!t.isActiveMaster && sleeps < 100) {
172       Thread.sleep(10);
173       sleeps++;
174     }
175     LOG.debug("Slept " + sleeps + " times");
176 
177     assertTrue(t.manager.clusterHasActiveMaster.get());
178     assertTrue(t.isActiveMaster);
179 
180     LOG.info("Deleting master node");
181     ZKUtil.deleteNode(zk, zk.masterAddressZNode);
182   }
183 
184   /**
185    * Assert there is an active master and that it has the specified address.
186    * @param zk
187    * @param thisMasterAddress
188    * @throws KeeperException
189    */
190   private void assertMaster(ZooKeeperWatcher zk,
191       ServerName expectedAddress)
192   throws KeeperException {
193     ServerName readAddress =
194       ServerName.parseVersionedServerName(ZKUtil.getData(zk, zk.masterAddressZNode));
195     assertNotNull(readAddress);
196     assertTrue(expectedAddress.equals(readAddress));
197   }
198 
199   public static class WaitToBeMasterThread extends Thread {
200 
201     ActiveMasterManager manager;
202     DummyMaster dummyMaster;
203     boolean isActiveMaster;
204 
205     public WaitToBeMasterThread(ZooKeeperWatcher zk, ServerName address) {
206       this.dummyMaster = new DummyMaster(zk,address);
207       this.manager = this.dummyMaster.getActiveMasterManager();
208       isActiveMaster = false;
209     }
210 
211     @Override
212     public void run() {
213       manager.blockUntilBecomingActiveMaster(
214           Mockito.mock(MonitoredTask.class),
215           this.dummyMaster.getClusterStatusTracker());
216       LOG.info("Second master has become the active master!");
217       isActiveMaster = true;
218     }
219   }
220 
221   public static class NodeDeletionListener extends ZooKeeperListener {
222     private static final Log LOG = LogFactory.getLog(NodeDeletionListener.class);
223 
224     private Semaphore lock;
225     private String node;
226 
227     public NodeDeletionListener(ZooKeeperWatcher watcher, String node) {
228       super(watcher);
229       lock = new Semaphore(0);
230       this.node = node;
231     }
232 
233     @Override
234     public void nodeDeleted(String path) {
235       if(path.equals(node)) {
236         LOG.debug("nodeDeleted(" + path + ")");
237         lock.release();
238       }
239     }
240 
241     public void waitForDeletion() throws InterruptedException {
242       lock.acquire();
243     }
244   }
245 
246   /**
247    * Dummy Master Implementation.
248    */
249   public static class DummyMaster implements Server {
250     private volatile boolean stopped;
251     private ClusterStatusTracker clusterStatusTracker;
252     private ActiveMasterManager activeMasterManager;
253 
254     public DummyMaster(ZooKeeperWatcher zk, ServerName master) {
255       this.clusterStatusTracker =
256         new ClusterStatusTracker(zk, this);
257       clusterStatusTracker.start();
258       
259       this.activeMasterManager =
260         new ActiveMasterManager(zk, master, this);
261       zk.registerListener(activeMasterManager);
262     }
263 
264     @Override
265     public void abort(final String msg, final Throwable t) {}
266     
267     @Override
268     public boolean isAborted() {
269       return false;
270     }
271 
272     @Override
273     public Configuration getConfiguration() {
274       return null;
275     }
276 
277     @Override
278     public ZooKeeperWatcher getZooKeeper() {
279       return null;
280     }
281 
282     @Override
283     public ServerName getServerName() {
284       return null;
285     }
286 
287     @Override
288     public boolean isStopped() {
289       return this.stopped;
290     }
291 
292     @Override
293     public void stop(String why) {
294       this.stopped = true;
295     }
296 
297     @Override
298     public CatalogTracker getCatalogTracker() {
299       return null;
300     }
301 
302     public ClusterStatusTracker getClusterStatusTracker() {
303       return clusterStatusTracker;
304     }
305 
306     public ActiveMasterManager getActiveMasterManager() {
307       return activeMasterManager;
308     }
309   }
310 
311   @org.junit.Rule
312   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
313     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
314 }
315