1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.zookeeper;
20
21 import java.io.BufferedReader;
22 import java.io.File;
23 import java.io.IOException;
24 import java.io.InputStreamReader;
25 import java.io.OutputStream;
26 import java.io.Reader;
27 import java.net.BindException;
28 import java.net.InetSocketAddress;
29 import java.net.Socket;
30 import java.util.ArrayList;
31 import java.util.List;
32 import java.util.Random;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.classification.InterfaceAudience;
37 import org.apache.hadoop.classification.InterfaceStability;
38 import org.apache.hadoop.conf.Configuration;
39 import org.apache.hadoop.fs.FileUtil;
40 import org.apache.hadoop.hbase.HConstants;
41 import org.apache.zookeeper.server.NIOServerCnxnFactory;
42 import org.apache.zookeeper.server.ZooKeeperServer;
43 import org.apache.zookeeper.server.persistence.FileTxnLog;
44
45
46
47
48
49
50 @InterfaceAudience.Public
51 @InterfaceStability.Evolving
52 public class MiniZooKeeperCluster {
53 private static final Log LOG = LogFactory.getLog(MiniZooKeeperCluster.class);
54
55 private static final int TICK_TIME = 2000;
56 private static final int CONNECTION_TIMEOUT = 30000;
57
58 private boolean started;
59
60
61 private int defaultClientPort = 0;
62
63 private int clientPort;
64
65 private List<NIOServerCnxnFactory> standaloneServerFactoryList;
66 private List<ZooKeeperServer> zooKeeperServers;
67 private List<Integer> clientPortList;
68
69 private int activeZKServerIndex;
70 private int tickTime = 0;
71
72 private Configuration configuration;
73
74 public MiniZooKeeperCluster() {
75 this(new Configuration());
76 }
77
78 public MiniZooKeeperCluster(Configuration configuration) {
79 this.started = false;
80 this.configuration = configuration;
81 activeZKServerIndex = -1;
82 zooKeeperServers = new ArrayList<ZooKeeperServer>();
83 clientPortList = new ArrayList<Integer>();
84 standaloneServerFactoryList = new ArrayList<NIOServerCnxnFactory>();
85 }
86
87 public void setDefaultClientPort(int clientPort) {
88 if (clientPort <= 0) {
89 throw new IllegalArgumentException("Invalid default ZK client port: "
90 + clientPort);
91 }
92 this.defaultClientPort = clientPort;
93 }
94
95
96
97
98
99
100
101 private int selectClientPort() {
102 if (defaultClientPort > 0) {
103 return defaultClientPort;
104 }
105 return 0xc000 + new Random().nextInt(0x3f00);
106 }
107
108 public void setTickTime(int tickTime) {
109 this.tickTime = tickTime;
110 }
111
112 public int getBackupZooKeeperServerNum() {
113 return zooKeeperServers.size()-1;
114 }
115
116 public int getZooKeeperServerNum() {
117 return zooKeeperServers.size();
118 }
119
120
121 private static void setupTestEnv() {
122
123
124
125
126 System.setProperty("zookeeper.preAllocSize", "100");
127 FileTxnLog.setPreallocSize(100 * 1024);
128 }
129
130 public int startup(File baseDir) throws IOException, InterruptedException {
131 return startup(baseDir,1);
132 }
133
134
135
136
137
138
139
140
141
142 public int startup(File baseDir, int numZooKeeperServers) throws IOException,
143 InterruptedException {
144 if (numZooKeeperServers <= 0)
145 return -1;
146
147 setupTestEnv();
148 shutdown();
149
150 int tentativePort = selectClientPort();
151
152
153 for (int i = 0; i < numZooKeeperServers; i++) {
154 File dir = new File(baseDir, "zookeeper_"+i).getAbsoluteFile();
155 recreateDir(dir);
156 int tickTimeToUse;
157 if (this.tickTime > 0) {
158 tickTimeToUse = this.tickTime;
159 } else {
160 tickTimeToUse = TICK_TIME;
161 }
162 ZooKeeperServer server = new ZooKeeperServer(dir, dir, tickTimeToUse);
163 NIOServerCnxnFactory standaloneServerFactory;
164 while (true) {
165 try {
166 standaloneServerFactory = new NIOServerCnxnFactory();
167 standaloneServerFactory.configure(
168 new InetSocketAddress(tentativePort),
169 configuration.getInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS,
170 1000));
171 } catch (BindException e) {
172 LOG.debug("Failed binding ZK Server to client port: " +
173 tentativePort, e);
174
175 if (defaultClientPort > 0) return -1;
176
177 tentativePort = selectClientPort();
178 continue;
179 }
180 break;
181 }
182
183
184 standaloneServerFactory.startup(server);
185 if (!waitForServerUp(tentativePort, CONNECTION_TIMEOUT)) {
186 throw new IOException("Waiting for startup of standalone server");
187 }
188
189
190 clientPortList.add(tentativePort);
191 standaloneServerFactoryList.add(standaloneServerFactory);
192 zooKeeperServers.add(server);
193 tentativePort++;
194 }
195
196
197 activeZKServerIndex = 0;
198 started = true;
199 clientPort = clientPortList.get(activeZKServerIndex);
200 LOG.info("Started MiniZK Cluster and connect 1 ZK server " +
201 "on client port: " + clientPort);
202 return clientPort;
203 }
204
205 private void recreateDir(File dir) throws IOException {
206 if (dir.exists()) {
207 if(!FileUtil.fullyDelete(dir)) {
208 throw new IOException("Could not delete zk base directory: " + dir);
209 }
210 }
211 try {
212 dir.mkdirs();
213 } catch (SecurityException e) {
214 throw new IOException("creating dir: " + dir, e);
215 }
216 }
217
218
219
220
221 public void shutdown() throws IOException {
222 if (!started) {
223 return;
224 }
225
226
227 for (int i = 0; i < standaloneServerFactoryList.size(); i++) {
228 NIOServerCnxnFactory standaloneServerFactory =
229 standaloneServerFactoryList.get(i);
230 int clientPort = clientPortList.get(i);
231
232 standaloneServerFactory.shutdown();
233 if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
234 throw new IOException("Waiting for shutdown of standalone server");
235 }
236 }
237 for (ZooKeeperServer zkServer: zooKeeperServers) {
238
239 zkServer.getZKDatabase().close();
240 }
241
242
243 started = false;
244 activeZKServerIndex = 0;
245 standaloneServerFactoryList.clear();
246 clientPortList.clear();
247 zooKeeperServers.clear();
248
249 LOG.info("Shutdown MiniZK cluster with all ZK servers");
250 }
251
252
253
254
255
256
257 public int killCurrentActiveZooKeeperServer() throws IOException,
258 InterruptedException {
259 if (!started || activeZKServerIndex < 0 ) {
260 return -1;
261 }
262
263
264 NIOServerCnxnFactory standaloneServerFactory =
265 standaloneServerFactoryList.get(activeZKServerIndex);
266 int clientPort = clientPortList.get(activeZKServerIndex);
267
268 standaloneServerFactory.shutdown();
269 if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
270 throw new IOException("Waiting for shutdown of standalone server");
271 }
272
273
274 standaloneServerFactoryList.remove(activeZKServerIndex);
275 clientPortList.remove(activeZKServerIndex);
276 zooKeeperServers.remove(activeZKServerIndex);
277 LOG.info("Kill the current active ZK servers in the cluster " +
278 "on client port: " + clientPort);
279
280 if (standaloneServerFactoryList.size() == 0) {
281
282 return -1;
283 }
284 clientPort = clientPortList.get(activeZKServerIndex);
285 LOG.info("Activate a backup zk server in the cluster " +
286 "on client port: " + clientPort);
287
288 return clientPort;
289 }
290
291
292
293
294
295
296 public void killOneBackupZooKeeperServer() throws IOException,
297 InterruptedException {
298 if (!started || activeZKServerIndex < 0 ||
299 standaloneServerFactoryList.size() <= 1) {
300 return ;
301 }
302
303 int backupZKServerIndex = activeZKServerIndex+1;
304
305 NIOServerCnxnFactory standaloneServerFactory =
306 standaloneServerFactoryList.get(backupZKServerIndex);
307 int clientPort = clientPortList.get(backupZKServerIndex);
308
309 standaloneServerFactory.shutdown();
310 if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
311 throw new IOException("Waiting for shutdown of standalone server");
312 }
313
314
315 standaloneServerFactoryList.remove(backupZKServerIndex);
316 clientPortList.remove(backupZKServerIndex);
317 zooKeeperServers.remove(backupZKServerIndex);
318 LOG.info("Kill one backup ZK servers in the cluster " +
319 "on client port: " + clientPort);
320 }
321
322
323 private static boolean waitForServerDown(int port, long timeout) {
324 long start = System.currentTimeMillis();
325 while (true) {
326 try {
327 Socket sock = new Socket("localhost", port);
328 try {
329 OutputStream outstream = sock.getOutputStream();
330 outstream.write("stat".getBytes());
331 outstream.flush();
332 } finally {
333 sock.close();
334 }
335 } catch (IOException e) {
336 return true;
337 }
338
339 if (System.currentTimeMillis() > start + timeout) {
340 break;
341 }
342 try {
343 Thread.sleep(250);
344 } catch (InterruptedException e) {
345
346 }
347 }
348 return false;
349 }
350
351
352 private static boolean waitForServerUp(int port, long timeout) {
353 long start = System.currentTimeMillis();
354 while (true) {
355 try {
356 Socket sock = new Socket("localhost", port);
357 BufferedReader reader = null;
358 try {
359 OutputStream outstream = sock.getOutputStream();
360 outstream.write("stat".getBytes());
361 outstream.flush();
362
363 Reader isr = new InputStreamReader(sock.getInputStream());
364 reader = new BufferedReader(isr);
365 String line = reader.readLine();
366 if (line != null && line.startsWith("Zookeeper version:")) {
367 return true;
368 }
369 } finally {
370 sock.close();
371 if (reader != null) {
372 reader.close();
373 }
374 }
375 } catch (IOException e) {
376
377 LOG.info("server localhost:" + port + " not up " + e);
378 }
379
380 if (System.currentTimeMillis() > start + timeout) {
381 break;
382 }
383 try {
384 Thread.sleep(250);
385 } catch (InterruptedException e) {
386
387 }
388 }
389 return false;
390 }
391
392 public int getClientPort() {
393 return clientPort;
394 }
395 }