1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master;
21
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertTrue;
25
26 import java.io.IOException;
27 import java.util.concurrent.Semaphore;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.hbase.*;
33 import org.apache.hadoop.hbase.catalog.CatalogTracker;
34 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
35 import org.apache.hadoop.hbase.util.Bytes;
36 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
37 import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
38 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
39 import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
40 import org.apache.zookeeper.KeeperException;
41 import org.junit.AfterClass;
42 import org.junit.BeforeClass;
43 import org.junit.Test;
44 import org.junit.experimental.categories.Category;
45 import org.mockito.Mockito;
46
47
48
49
50 @Category(MediumTests.class)
51 public class TestActiveMasterManager {
52 private final static Log LOG = LogFactory.getLog(TestActiveMasterManager.class);
53 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
54
55 @BeforeClass
56 public static void setUpBeforeClass() throws Exception {
57 TEST_UTIL.startMiniZKCluster();
58 }
59
60 @AfterClass
61 public static void tearDownAfterClass() throws Exception {
62 TEST_UTIL.shutdownMiniZKCluster();
63 }
64
65 @Test public void testRestartMaster() throws IOException, KeeperException {
66 ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
67 "testActiveMasterManagerFromZK", null, true);
68 try {
69 ZKUtil.deleteNode(zk, zk.masterAddressZNode);
70 ZKUtil.deleteNode(zk, zk.clusterStateZNode);
71 } catch(KeeperException.NoNodeException nne) {}
72
73
74 ServerName master = new ServerName("localhost", 1, System.currentTimeMillis());
75
76 DummyMaster dummyMaster = new DummyMaster(zk,master);
77 ClusterStatusTracker clusterStatusTracker =
78 dummyMaster.getClusterStatusTracker();
79 ActiveMasterManager activeMasterManager =
80 dummyMaster.getActiveMasterManager();
81 assertFalse(activeMasterManager.clusterHasActiveMaster.get());
82
83
84 MonitoredTask status = Mockito.mock(MonitoredTask.class);
85 clusterStatusTracker.setClusterUp();
86
87 activeMasterManager.blockUntilBecomingActiveMaster(status);
88 assertTrue(activeMasterManager.clusterHasActiveMaster.get());
89 assertMaster(zk, master);
90
91
92 DummyMaster secondDummyMaster = new DummyMaster(zk,master);
93 ActiveMasterManager secondActiveMasterManager =
94 secondDummyMaster.getActiveMasterManager();
95 assertFalse(secondActiveMasterManager.clusterHasActiveMaster.get());
96 activeMasterManager.blockUntilBecomingActiveMaster(status);
97 assertTrue(activeMasterManager.clusterHasActiveMaster.get());
98 assertMaster(zk, master);
99 }
100
101
102
103
104
105
106 @Test
107 public void testActiveMasterManagerFromZK() throws Exception {
108 ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
109 "testActiveMasterManagerFromZK", null, true);
110 try {
111 ZKUtil.deleteNode(zk, zk.masterAddressZNode);
112 ZKUtil.deleteNode(zk, zk.clusterStateZNode);
113 } catch(KeeperException.NoNodeException nne) {}
114
115
116 ServerName firstMasterAddress =
117 new ServerName("localhost", 1, System.currentTimeMillis());
118 ServerName secondMasterAddress =
119 new ServerName("localhost", 2, System.currentTimeMillis());
120
121
122 DummyMaster ms1 = new DummyMaster(zk,firstMasterAddress);
123 ActiveMasterManager activeMasterManager =
124 ms1.getActiveMasterManager();
125 assertFalse(activeMasterManager.clusterHasActiveMaster.get());
126
127
128 ClusterStatusTracker clusterStatusTracker =
129 ms1.getClusterStatusTracker();
130 clusterStatusTracker.setClusterUp();
131 activeMasterManager.blockUntilBecomingActiveMaster(
132 Mockito.mock(MonitoredTask.class));
133 assertTrue(activeMasterManager.clusterHasActiveMaster.get());
134 assertMaster(zk, firstMasterAddress);
135
136
137 WaitToBeMasterThread t = new WaitToBeMasterThread(zk, secondMasterAddress);
138 t.start();
139
140
141 int sleeps = 0;
142 while(!t.manager.clusterHasActiveMaster.get() && sleeps < 100) {
143 Thread.sleep(10);
144 sleeps++;
145 }
146
147
148 assertTrue(activeMasterManager.clusterHasActiveMaster.get());
149 assertTrue(t.manager.clusterHasActiveMaster.get());
150
151 assertFalse(t.isActiveMaster);
152
153
154 ms1.stop("stopping first server");
155
156
157 NodeDeletionListener listener = new NodeDeletionListener(zk, zk.masterAddressZNode);
158 zk.registerListener(listener);
159
160 LOG.info("Deleting master node");
161 ZKUtil.deleteNode(zk, zk.masterAddressZNode);
162
163
164 LOG.info("Waiting for active master manager to be notified");
165 listener.waitForDeletion();
166 LOG.info("Master node deleted");
167
168
169
170 sleeps = 0;
171 while(!t.isActiveMaster && sleeps < 100) {
172 Thread.sleep(10);
173 sleeps++;
174 }
175 LOG.debug("Slept " + sleeps + " times");
176
177 assertTrue(t.manager.clusterHasActiveMaster.get());
178 assertTrue(t.isActiveMaster);
179
180 LOG.info("Deleting master node");
181 ZKUtil.deleteNode(zk, zk.masterAddressZNode);
182 }
183
184
185
186
187
188
189
190 private void assertMaster(ZooKeeperWatcher zk,
191 ServerName expectedAddress)
192 throws KeeperException {
193 ServerName readAddress =
194 ServerName.parseVersionedServerName(ZKUtil.getData(zk, zk.masterAddressZNode));
195 assertNotNull(readAddress);
196 assertTrue(expectedAddress.equals(readAddress));
197 }
198
199 public static class WaitToBeMasterThread extends Thread {
200
201 ActiveMasterManager manager;
202 DummyMaster dummyMaster;
203 boolean isActiveMaster;
204
205 public WaitToBeMasterThread(ZooKeeperWatcher zk, ServerName address) {
206 this.dummyMaster = new DummyMaster(zk,address);
207 this.manager = this.dummyMaster.getActiveMasterManager();
208 isActiveMaster = false;
209 }
210
211 @Override
212 public void run() {
213 manager.blockUntilBecomingActiveMaster(
214 Mockito.mock(MonitoredTask.class));
215 LOG.info("Second master has become the active master!");
216 isActiveMaster = true;
217 }
218 }
219
220 public static class NodeDeletionListener extends ZooKeeperListener {
221 private static final Log LOG = LogFactory.getLog(NodeDeletionListener.class);
222
223 private Semaphore lock;
224 private String node;
225
226 public NodeDeletionListener(ZooKeeperWatcher watcher, String node) {
227 super(watcher);
228 lock = new Semaphore(0);
229 this.node = node;
230 }
231
232 @Override
233 public void nodeDeleted(String path) {
234 if(path.equals(node)) {
235 LOG.debug("nodeDeleted(" + path + ")");
236 lock.release();
237 }
238 }
239
240 public void waitForDeletion() throws InterruptedException {
241 lock.acquire();
242 }
243 }
244
245
246
247
248 public static class DummyMaster implements Server {
249 private volatile boolean stopped;
250 private ClusterStatusTracker clusterStatusTracker;
251 private ActiveMasterManager activeMasterManager;
252
253 public DummyMaster(ZooKeeperWatcher zk, ServerName master) {
254 this.clusterStatusTracker =
255 new ClusterStatusTracker(zk, this);
256 clusterStatusTracker.start();
257
258 this.activeMasterManager =
259 new ActiveMasterManager(zk, master, this);
260 zk.registerListener(activeMasterManager);
261 }
262
263 @Override
264 public void abort(final String msg, final Throwable t) {}
265
266 @Override
267 public boolean isAborted() {
268 return false;
269 }
270
271 @Override
272 public Configuration getConfiguration() {
273 return null;
274 }
275
276 @Override
277 public ZooKeeperWatcher getZooKeeper() {
278 return null;
279 }
280
281 @Override
282 public ServerName getServerName() {
283 return null;
284 }
285
286 @Override
287 public boolean isStopped() {
288 return this.stopped;
289 }
290
291 @Override
292 public void stop(String why) {
293 this.stopped = true;
294 }
295
296 @Override
297 public CatalogTracker getCatalogTracker() {
298 return null;
299 }
300
301 public ClusterStatusTracker getClusterStatusTracker() {
302 return clusterStatusTracker;
303 }
304
305 public ActiveMasterManager getActiveMasterManager() {
306 return activeMasterManager;
307 }
308 }
309
310 @org.junit.Rule
311 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
312 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
313 }
314