1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertNotNull;
23 import static org.junit.Assert.assertTrue;
24
25 import java.io.IOException;
26 import java.util.concurrent.Semaphore;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.hbase.CoordinatedStateManager;
32 import org.apache.hadoop.hbase.HBaseTestingUtility;
33 import org.apache.hadoop.hbase.testclassification.MediumTests;
34 import org.apache.hadoop.hbase.Server;
35 import org.apache.hadoop.hbase.ServerName;
36 import org.apache.hadoop.hbase.client.ClusterConnection;
37 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
38 import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
39 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
40 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
41 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
42 import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
43 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
44 import org.apache.zookeeper.KeeperException;
45 import org.junit.AfterClass;
46 import org.junit.BeforeClass;
47 import org.junit.Test;
48 import org.junit.experimental.categories.Category;
49 import org.mockito.Mockito;
50
51
52
53
54 @Category(MediumTests.class)
55 public class TestActiveMasterManager {
56 private final static Log LOG = LogFactory.getLog(TestActiveMasterManager.class);
57 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
58
59 @BeforeClass
60 public static void setUpBeforeClass() throws Exception {
61 TEST_UTIL.startMiniZKCluster();
62 }
63
64 @AfterClass
65 public static void tearDownAfterClass() throws Exception {
66 TEST_UTIL.shutdownMiniZKCluster();
67 }
68
69 @Test public void testRestartMaster() throws IOException, KeeperException {
70 ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
71 "testActiveMasterManagerFromZK", null, true);
72 try {
73 ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
74 ZKUtil.deleteNode(zk, zk.clusterStateZNode);
75 } catch(KeeperException.NoNodeException nne) {}
76
77
78 ServerName master = ServerName.valueOf("localhost", 1, System.currentTimeMillis());
79
80 DummyMaster dummyMaster = new DummyMaster(zk,master);
81 ClusterStatusTracker clusterStatusTracker =
82 dummyMaster.getClusterStatusTracker();
83 ActiveMasterManager activeMasterManager =
84 dummyMaster.getActiveMasterManager();
85 assertFalse(activeMasterManager.clusterHasActiveMaster.get());
86
87
88 MonitoredTask status = Mockito.mock(MonitoredTask.class);
89 clusterStatusTracker.setClusterUp();
90
91 activeMasterManager.blockUntilBecomingActiveMaster(100, status);
92 assertTrue(activeMasterManager.clusterHasActiveMaster.get());
93 assertMaster(zk, master);
94
95
96 DummyMaster secondDummyMaster = new DummyMaster(zk,master);
97 ActiveMasterManager secondActiveMasterManager =
98 secondDummyMaster.getActiveMasterManager();
99 assertFalse(secondActiveMasterManager.clusterHasActiveMaster.get());
100 activeMasterManager.blockUntilBecomingActiveMaster(100, status);
101 assertTrue(activeMasterManager.clusterHasActiveMaster.get());
102 assertMaster(zk, master);
103 }
104
105
106
107
108
109
110 @Test
111 public void testActiveMasterManagerFromZK() throws Exception {
112 ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
113 "testActiveMasterManagerFromZK", null, true);
114 try {
115 ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
116 ZKUtil.deleteNode(zk, zk.clusterStateZNode);
117 } catch(KeeperException.NoNodeException nne) {}
118
119
120 ServerName firstMasterAddress =
121 ServerName.valueOf("localhost", 1, System.currentTimeMillis());
122 ServerName secondMasterAddress =
123 ServerName.valueOf("localhost", 2, System.currentTimeMillis());
124
125
126 DummyMaster ms1 = new DummyMaster(zk,firstMasterAddress);
127 ActiveMasterManager activeMasterManager =
128 ms1.getActiveMasterManager();
129 assertFalse(activeMasterManager.clusterHasActiveMaster.get());
130
131
132 ClusterStatusTracker clusterStatusTracker =
133 ms1.getClusterStatusTracker();
134 clusterStatusTracker.setClusterUp();
135 activeMasterManager.blockUntilBecomingActiveMaster(100,
136 Mockito.mock(MonitoredTask.class));
137 assertTrue(activeMasterManager.clusterHasActiveMaster.get());
138 assertMaster(zk, firstMasterAddress);
139
140
141 WaitToBeMasterThread t = new WaitToBeMasterThread(zk, secondMasterAddress);
142 t.start();
143
144
145 int sleeps = 0;
146 while(!t.manager.clusterHasActiveMaster.get() && sleeps < 100) {
147 Thread.sleep(10);
148 sleeps++;
149 }
150
151
152 assertTrue(activeMasterManager.clusterHasActiveMaster.get());
153 assertTrue(t.manager.clusterHasActiveMaster.get());
154
155 assertFalse(t.isActiveMaster);
156
157
158 ms1.stop("stopping first server");
159
160
161 NodeDeletionListener listener = new NodeDeletionListener(zk, zk.getMasterAddressZNode());
162 zk.registerListener(listener);
163
164 LOG.info("Deleting master node");
165 ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
166
167
168 LOG.info("Waiting for active master manager to be notified");
169 listener.waitForDeletion();
170 LOG.info("Master node deleted");
171
172
173
174 sleeps = 0;
175 while(!t.isActiveMaster && sleeps < 100) {
176 Thread.sleep(10);
177 sleeps++;
178 }
179 LOG.debug("Slept " + sleeps + " times");
180
181 assertTrue(t.manager.clusterHasActiveMaster.get());
182 assertTrue(t.isActiveMaster);
183
184 LOG.info("Deleting master node");
185 ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
186 }
187
188
189
190
191
192
193
194
195 private void assertMaster(ZooKeeperWatcher zk,
196 ServerName expectedAddress)
197 throws KeeperException, IOException {
198 ServerName readAddress = MasterAddressTracker.getMasterAddress(zk);
199 assertNotNull(readAddress);
200 assertTrue(expectedAddress.equals(readAddress));
201 }
202
203 public static class WaitToBeMasterThread extends Thread {
204
205 ActiveMasterManager manager;
206 DummyMaster dummyMaster;
207 boolean isActiveMaster;
208
209 public WaitToBeMasterThread(ZooKeeperWatcher zk, ServerName address) {
210 this.dummyMaster = new DummyMaster(zk,address);
211 this.manager = this.dummyMaster.getActiveMasterManager();
212 isActiveMaster = false;
213 }
214
215 @Override
216 public void run() {
217 manager.blockUntilBecomingActiveMaster(100,
218 Mockito.mock(MonitoredTask.class));
219 LOG.info("Second master has become the active master!");
220 isActiveMaster = true;
221 }
222 }
223
224 public static class NodeDeletionListener extends ZooKeeperListener {
225 private static final Log LOG = LogFactory.getLog(NodeDeletionListener.class);
226
227 private Semaphore lock;
228 private String node;
229
230 public NodeDeletionListener(ZooKeeperWatcher watcher, String node) {
231 super(watcher);
232 lock = new Semaphore(0);
233 this.node = node;
234 }
235
236 @Override
237 public void nodeDeleted(String path) {
238 if(path.equals(node)) {
239 LOG.debug("nodeDeleted(" + path + ")");
240 lock.release();
241 }
242 }
243
244 public void waitForDeletion() throws InterruptedException {
245 lock.acquire();
246 }
247 }
248
249
250
251
252 public static class DummyMaster implements Server {
253 private volatile boolean stopped;
254 private ClusterStatusTracker clusterStatusTracker;
255 private ActiveMasterManager activeMasterManager;
256
257 public DummyMaster(ZooKeeperWatcher zk, ServerName master) {
258 this.clusterStatusTracker =
259 new ClusterStatusTracker(zk, this);
260 clusterStatusTracker.start();
261
262 this.activeMasterManager =
263 new ActiveMasterManager(zk, master, this);
264 zk.registerListener(activeMasterManager);
265 }
266
267 @Override
268 public void abort(final String msg, final Throwable t) {}
269
270 @Override
271 public boolean isAborted() {
272 return false;
273 }
274
275 @Override
276 public Configuration getConfiguration() {
277 return null;
278 }
279
280 @Override
281 public ZooKeeperWatcher getZooKeeper() {
282 return null;
283 }
284
285 @Override
286 public CoordinatedStateManager getCoordinatedStateManager() {
287 return null;
288 }
289
290 @Override
291 public ServerName getServerName() {
292 return null;
293 }
294
295 @Override
296 public boolean isStopped() {
297 return this.stopped;
298 }
299
300 @Override
301 public void stop(String why) {
302 this.stopped = true;
303 }
304
305 @Override
306 public ClusterConnection getConnection() {
307 return null;
308 }
309
310 @Override
311 public MetaTableLocator getMetaTableLocator() {
312 return null;
313 }
314
315 public ClusterStatusTracker getClusterStatusTracker() {
316 return clusterStatusTracker;
317 }
318
319 public ActiveMasterManager getActiveMasterManager() {
320 return activeMasterManager;
321 }
322 }
323 }