1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.zookeeper;
20
21 import java.io.BufferedReader;
22 import java.io.File;
23 import java.io.IOException;
24 import java.io.InputStreamReader;
25 import java.io.OutputStream;
26 import java.io.Reader;
27 import java.net.BindException;
28 import java.net.InetSocketAddress;
29 import java.net.Socket;
30 import java.util.ArrayList;
31 import java.util.List;
32 import java.util.Random;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.classification.InterfaceAudience;
37 import org.apache.hadoop.classification.InterfaceStability;
38 import org.apache.hadoop.conf.Configuration;
39 import org.apache.hadoop.fs.FileUtil;
40 import org.apache.hadoop.hbase.HConstants;
41 import org.apache.zookeeper.server.NIOServerCnxnFactory;
42 import org.apache.zookeeper.server.ZooKeeperServer;
43 import org.apache.zookeeper.server.persistence.FileTxnLog;
44
45
46
47
48
49
50 @InterfaceAudience.Public
51 @InterfaceStability.Evolving
52 public class MiniZooKeeperCluster {
53 private static final Log LOG = LogFactory.getLog(MiniZooKeeperCluster.class);
54
55 private static final int TICK_TIME = 2000;
56 private static final int CONNECTION_TIMEOUT = 30000;
57
58 private boolean started;
59
60
61 private int defaultClientPort = 0;
62
63 private int clientPort;
64
65 private List<NIOServerCnxnFactory> standaloneServerFactoryList;
66 private List<ZooKeeperServer> zooKeeperServers;
67 private List<Integer> clientPortList;
68
69 private int activeZKServerIndex;
70 private int tickTime = 0;
71
72 private Configuration configuration;
73
74 public MiniZooKeeperCluster() {
75 this(new Configuration());
76 }
77
78 public MiniZooKeeperCluster(Configuration configuration) {
79 this.started = false;
80 this.configuration = configuration;
81 activeZKServerIndex = -1;
82 zooKeeperServers = new ArrayList<ZooKeeperServer>();
83 clientPortList = new ArrayList<Integer>();
84 standaloneServerFactoryList = new ArrayList<NIOServerCnxnFactory>();
85 }
86
87 public void setDefaultClientPort(int clientPort) {
88 if (clientPort <= 0) {
89 throw new IllegalArgumentException("Invalid default ZK client port: "
90 + clientPort);
91 }
92 this.defaultClientPort = clientPort;
93 }
94
95
96
97
98
99
100
101 private int selectClientPort() {
102 if (defaultClientPort > 0) {
103 return defaultClientPort;
104 }
105 return 0xc000 + new Random().nextInt(0x3f00);
106 }
107
108 public void setTickTime(int tickTime) {
109 this.tickTime = tickTime;
110 }
111
112 public int getBackupZooKeeperServerNum() {
113 return zooKeeperServers.size()-1;
114 }
115
116 public int getZooKeeperServerNum() {
117 return zooKeeperServers.size();
118 }
119
120
121 private static void setupTestEnv() {
122
123
124
125
126 System.setProperty("zookeeper.preAllocSize", "100");
127 FileTxnLog.setPreallocSize(100 * 1024);
128 }
129
130 public int startup(File baseDir) throws IOException, InterruptedException {
131 return startup(baseDir,1);
132 }
133
134
135
136
137
138
139
140
141
142 public int startup(File baseDir, int numZooKeeperServers) throws IOException,
143 InterruptedException {
144 if (numZooKeeperServers <= 0)
145 return -1;
146
147 setupTestEnv();
148 shutdown();
149
150 int tentativePort = selectClientPort();
151
152
153 for (int i = 0; i < numZooKeeperServers; i++) {
154 File dir = new File(baseDir, "zookeeper_"+i).getAbsoluteFile();
155 recreateDir(dir);
156 int tickTimeToUse;
157 if (this.tickTime > 0) {
158 tickTimeToUse = this.tickTime;
159 } else {
160 tickTimeToUse = TICK_TIME;
161 }
162 ZooKeeperServer server = new ZooKeeperServer(dir, dir, tickTimeToUse);
163 NIOServerCnxnFactory standaloneServerFactory;
164 while (true) {
165 try {
166 standaloneServerFactory = new NIOServerCnxnFactory();
167 standaloneServerFactory.configure(
168 new InetSocketAddress(tentativePort),
169 configuration.getInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS,
170 1000));
171 } catch (BindException e) {
172 LOG.debug("Failed binding ZK Server to client port: " +
173 tentativePort, e);
174
175 if (defaultClientPort > 0) return -1;
176
177 tentativePort = selectClientPort();
178 continue;
179 }
180 break;
181 }
182
183
184 standaloneServerFactory.startup(server);
185 if (!waitForServerUp(tentativePort, CONNECTION_TIMEOUT)) {
186 throw new IOException("Waiting for startup of standalone server");
187 }
188
189
190 clientPortList.add(tentativePort);
191 standaloneServerFactoryList.add(standaloneServerFactory);
192 zooKeeperServers.add(server);
193 tentativePort++;
194 }
195
196
197 activeZKServerIndex = 0;
198 started = true;
199 clientPort = clientPortList.get(activeZKServerIndex);
200 LOG.info("Started MiniZK Cluster and connect 1 ZK server " +
201 "on client port: " + clientPort);
202 return clientPort;
203 }
204
205 private void recreateDir(File dir) throws IOException {
206 if (dir.exists()) {
207 if(!FileUtil.fullyDelete(dir)) {
208 throw new IOException("Could not delete zk base directory: " + dir);
209 }
210 }
211 try {
212 dir.mkdirs();
213 } catch (SecurityException e) {
214 throw new IOException("creating dir: " + dir, e);
215 }
216 }
217
218
219
220
221 public void shutdown() throws IOException {
222 if (!started) {
223 return;
224 }
225
226
227 for (int i = 0; i < standaloneServerFactoryList.size(); i++) {
228 NIOServerCnxnFactory standaloneServerFactory =
229 standaloneServerFactoryList.get(i);
230 int clientPort = clientPortList.get(i);
231
232 standaloneServerFactory.shutdown();
233 if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
234 throw new IOException("Waiting for shutdown of standalone server");
235 }
236 }
237 for (ZooKeeperServer zkServer: zooKeeperServers) {
238
239 zkServer.getZKDatabase().close();
240 }
241
242
243 started = false;
244 activeZKServerIndex = 0;
245 standaloneServerFactoryList.clear();
246 clientPortList.clear();
247 zooKeeperServers.clear();
248
249 LOG.info("Shutdown MiniZK cluster with all ZK servers");
250 }
251
252
253
254
255
256
257 public int killCurrentActiveZooKeeperServer() throws IOException,
258 InterruptedException {
259 if (!started || activeZKServerIndex < 0 ) {
260 return -1;
261 }
262
263
264 NIOServerCnxnFactory standaloneServerFactory =
265 standaloneServerFactoryList.get(activeZKServerIndex);
266 int clientPort = clientPortList.get(activeZKServerIndex);
267
268 standaloneServerFactory.shutdown();
269 if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
270 throw new IOException("Waiting for shutdown of standalone server");
271 }
272
273 zooKeeperServers.get(activeZKServerIndex).getZKDatabase().close();
274
275
276 standaloneServerFactoryList.remove(activeZKServerIndex);
277 clientPortList.remove(activeZKServerIndex);
278 zooKeeperServers.remove(activeZKServerIndex);
279 LOG.info("Kill the current active ZK servers in the cluster " +
280 "on client port: " + clientPort);
281
282 if (standaloneServerFactoryList.size() == 0) {
283
284 return -1;
285 }
286 clientPort = clientPortList.get(activeZKServerIndex);
287 LOG.info("Activate a backup zk server in the cluster " +
288 "on client port: " + clientPort);
289
290 return clientPort;
291 }
292
293
294
295
296
297
298 public void killOneBackupZooKeeperServer() throws IOException,
299 InterruptedException {
300 if (!started || activeZKServerIndex < 0 ||
301 standaloneServerFactoryList.size() <= 1) {
302 return ;
303 }
304
305 int backupZKServerIndex = activeZKServerIndex+1;
306
307 NIOServerCnxnFactory standaloneServerFactory =
308 standaloneServerFactoryList.get(backupZKServerIndex);
309 int clientPort = clientPortList.get(backupZKServerIndex);
310
311 standaloneServerFactory.shutdown();
312 if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
313 throw new IOException("Waiting for shutdown of standalone server");
314 }
315
316 zooKeeperServers.get(backupZKServerIndex).getZKDatabase().close();
317
318
319 standaloneServerFactoryList.remove(backupZKServerIndex);
320 clientPortList.remove(backupZKServerIndex);
321 zooKeeperServers.remove(backupZKServerIndex);
322 LOG.info("Kill one backup ZK servers in the cluster " +
323 "on client port: " + clientPort);
324 }
325
326
327 private static boolean waitForServerDown(int port, long timeout) {
328 long start = System.currentTimeMillis();
329 while (true) {
330 try {
331 Socket sock = new Socket("localhost", port);
332 try {
333 OutputStream outstream = sock.getOutputStream();
334 outstream.write("stat".getBytes());
335 outstream.flush();
336 } finally {
337 sock.close();
338 }
339 } catch (IOException e) {
340 return true;
341 }
342
343 if (System.currentTimeMillis() > start + timeout) {
344 break;
345 }
346 try {
347 Thread.sleep(250);
348 } catch (InterruptedException e) {
349
350 }
351 }
352 return false;
353 }
354
355
356 private static boolean waitForServerUp(int port, long timeout) {
357 long start = System.currentTimeMillis();
358 while (true) {
359 try {
360 Socket sock = new Socket("localhost", port);
361 BufferedReader reader = null;
362 try {
363 OutputStream outstream = sock.getOutputStream();
364 outstream.write("stat".getBytes());
365 outstream.flush();
366
367 Reader isr = new InputStreamReader(sock.getInputStream());
368 reader = new BufferedReader(isr);
369 String line = reader.readLine();
370 if (line != null && line.startsWith("Zookeeper version:")) {
371 return true;
372 }
373 } finally {
374 sock.close();
375 if (reader != null) {
376 reader.close();
377 }
378 }
379 } catch (IOException e) {
380
381 LOG.info("server localhost:" + port + " not up " + e);
382 }
383
384 if (System.currentTimeMillis() > start + timeout) {
385 break;
386 }
387 try {
388 Thread.sleep(250);
389 } catch (InterruptedException e) {
390
391 }
392 }
393 return false;
394 }
395
396 public int getClientPort() {
397 return clientPort;
398 }
399 }