View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.zookeeper;
20  
21  import java.io.BufferedReader;
22  import java.io.File;
23  import java.io.IOException;
24  import java.io.InputStreamReader;
25  import java.io.OutputStream;
26  import java.io.Reader;
27  import java.net.BindException;
28  import java.net.InetSocketAddress;
29  import java.net.Socket;
30  import java.util.ArrayList;
31  import java.util.List;
32  import java.util.Random;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.classification.InterfaceAudience;
37  import org.apache.hadoop.classification.InterfaceStability;
38  import org.apache.hadoop.conf.Configuration;
39  import org.apache.hadoop.fs.FileUtil;
40  import org.apache.hadoop.hbase.HConstants;
41  import org.apache.zookeeper.server.NIOServerCnxnFactory;
42  import org.apache.zookeeper.server.ZooKeeperServer;
43  import org.apache.zookeeper.server.persistence.FileTxnLog;
44  
45  /**
46   * TODO: Most of the code in this class is ripped from ZooKeeper tests. Instead
47   * of redoing it, we should contribute updates to their code which let us more
48   * easily access testing helper objects.
49   */
50  @InterfaceAudience.Public
51  @InterfaceStability.Evolving
52  public class MiniZooKeeperCluster {
53    private static final Log LOG = LogFactory.getLog(MiniZooKeeperCluster.class);
54  
55    private static final int TICK_TIME = 2000;
56    private static final int CONNECTION_TIMEOUT = 30000;
57  
58    private boolean started;
59  
60    /** The default port. If zero, we use a random port. */
61    private int defaultClientPort = 0;
62  
63    private int clientPort;
64  
65    private List<NIOServerCnxnFactory> standaloneServerFactoryList;
66    private List<ZooKeeperServer> zooKeeperServers;
67    private List<Integer> clientPortList;
68  
69    private int activeZKServerIndex;
70    private int tickTime = 0;
71  
72    private Configuration configuration;
73  
74    public MiniZooKeeperCluster() {
75      this(new Configuration());
76    }
77  
78    public MiniZooKeeperCluster(Configuration configuration) {
79      this.started = false;
80      this.configuration = configuration;
81      activeZKServerIndex = -1;
82      zooKeeperServers = new ArrayList<ZooKeeperServer>();
83      clientPortList = new ArrayList<Integer>();
84      standaloneServerFactoryList = new ArrayList<NIOServerCnxnFactory>();
85    }
86  
87    public void setDefaultClientPort(int clientPort) {
88      if (clientPort <= 0) {
89        throw new IllegalArgumentException("Invalid default ZK client port: "
90            + clientPort);
91      }
92      this.defaultClientPort = clientPort;
93    }
94  
95    /**
96     * Selects a ZK client port. Returns the default port if specified.
97     * Otherwise, returns a random port. The random port is selected from the
98     * range between 49152 to 65535. These ports cannot be registered with IANA
99     * and are intended for dynamic allocation (see http://bit.ly/dynports).
100    */
101   private int selectClientPort() {
102     if (defaultClientPort > 0) {
103       return defaultClientPort;
104     }
105     return 0xc000 + new Random().nextInt(0x3f00);
106   }
107 
108   public void setTickTime(int tickTime) {
109     this.tickTime = tickTime;
110   }
111 
112   public int getBackupZooKeeperServerNum() {
113     return zooKeeperServers.size()-1;
114   }
115 
116   public int getZooKeeperServerNum() {
117     return zooKeeperServers.size();
118   }
119 
120   // / XXX: From o.a.zk.t.ClientBase
121   private static void setupTestEnv() {
122     // during the tests we run with 100K prealloc in the logs.
123     // on windows systems prealloc of 64M was seen to take ~15seconds
124     // resulting in test failure (client timeout on first session).
125     // set env and directly in order to handle static init/gc issues
126     System.setProperty("zookeeper.preAllocSize", "100");
127     FileTxnLog.setPreallocSize(100 * 1024);
128   }
129 
130   public int startup(File baseDir) throws IOException, InterruptedException {
131     return startup(baseDir,1);
132   }
133 
134   /**
135    * @param baseDir
136    * @param numZooKeeperServers
137    * @return ClientPort server bound to, -1 if there was a
138    *         binding problem and we couldn't pick another port.
139    * @throws IOException
140    * @throws InterruptedException
141    */
142   public int startup(File baseDir, int numZooKeeperServers) throws IOException,
143       InterruptedException {
144     if (numZooKeeperServers <= 0)
145       return -1;
146 
147     setupTestEnv();
148     shutdown();
149 
150     int tentativePort = selectClientPort();
151 
152     // running all the ZK servers
153     for (int i = 0; i < numZooKeeperServers; i++) {
154       File dir = new File(baseDir, "zookeeper_"+i).getAbsoluteFile();
155       recreateDir(dir);
156       int tickTimeToUse;
157       if (this.tickTime > 0) {
158         tickTimeToUse = this.tickTime;
159       } else {
160         tickTimeToUse = TICK_TIME;
161       }
162       ZooKeeperServer server = new ZooKeeperServer(dir, dir, tickTimeToUse);
163       NIOServerCnxnFactory standaloneServerFactory;
164       while (true) {
165         try {
166           standaloneServerFactory = new NIOServerCnxnFactory();
167           standaloneServerFactory.configure(
168             new InetSocketAddress(tentativePort),
169             configuration.getInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS,
170               1000));
171         } catch (BindException e) {
172           LOG.debug("Failed binding ZK Server to client port: " +
173               tentativePort, e);
174           // We're told to use some port but it's occupied, fail
175           if (defaultClientPort > 0) return -1;
176           // This port is already in use, try to use another.
177           tentativePort = selectClientPort();
178           continue;
179         }
180         break;
181       }
182 
183       // Start up this ZK server
184       standaloneServerFactory.startup(server);
185       if (!waitForServerUp(tentativePort, CONNECTION_TIMEOUT)) {
186         throw new IOException("Waiting for startup of standalone server");
187       }
188 
189       // We have selected this port as a client port.
190       clientPortList.add(tentativePort);
191       standaloneServerFactoryList.add(standaloneServerFactory);
192       zooKeeperServers.add(server);
193       tentativePort++; //for the next server
194     }
195 
196     // set the first one to be active ZK; Others are backups
197     activeZKServerIndex = 0;
198     started = true;
199     clientPort = clientPortList.get(activeZKServerIndex);
200     LOG.info("Started MiniZK Cluster and connect 1 ZK server " +
201         "on client port: " + clientPort);
202     return clientPort;
203   }
204 
205   private void recreateDir(File dir) throws IOException {
206     if (dir.exists()) {
207       if(!FileUtil.fullyDelete(dir)) {
208         throw new IOException("Could not delete zk base directory: " + dir);
209       }
210     }
211     try {
212       dir.mkdirs();
213     } catch (SecurityException e) {
214       throw new IOException("creating dir: " + dir, e);
215     }
216   }
217 
218   /**
219    * @throws IOException
220    */
221   public void shutdown() throws IOException {
222     if (!started) {
223       return;
224     }
225 
226     // shut down all the zk servers
227     for (int i = 0; i < standaloneServerFactoryList.size(); i++) {
228       NIOServerCnxnFactory standaloneServerFactory =
229         standaloneServerFactoryList.get(i);
230       int clientPort = clientPortList.get(i);
231 
232       standaloneServerFactory.shutdown();
233       if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
234         throw new IOException("Waiting for shutdown of standalone server");
235       }
236     }
237     for (ZooKeeperServer zkServer: zooKeeperServers) {
238       //explicitly close ZKDatabase since ZookeeperServer does not close them
239       zkServer.getZKDatabase().close();
240     }
241 
242     // clear everything
243     started = false;
244     activeZKServerIndex = 0;
245     standaloneServerFactoryList.clear();
246     clientPortList.clear();
247     zooKeeperServers.clear();
248 
249     LOG.info("Shutdown MiniZK cluster with all ZK servers");
250   }
251 
252   /**@return clientPort return clientPort if there is another ZK backup can run
253    *         when killing the current active; return -1, if there is no backups.
254    * @throws IOException
255    * @throws InterruptedException
256    */
257   public int killCurrentActiveZooKeeperServer() throws IOException,
258                                         InterruptedException {
259     if (!started || activeZKServerIndex < 0 ) {
260       return -1;
261     }
262 
263     // Shutdown the current active one
264     NIOServerCnxnFactory standaloneServerFactory =
265       standaloneServerFactoryList.get(activeZKServerIndex);
266     int clientPort = clientPortList.get(activeZKServerIndex);
267 
268     standaloneServerFactory.shutdown();
269     if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
270       throw new IOException("Waiting for shutdown of standalone server");
271     }
272 
273     zooKeeperServers.get(activeZKServerIndex).getZKDatabase().close();
274 
275     // remove the current active zk server
276     standaloneServerFactoryList.remove(activeZKServerIndex);
277     clientPortList.remove(activeZKServerIndex);
278     zooKeeperServers.remove(activeZKServerIndex);
279     LOG.info("Kill the current active ZK servers in the cluster " +
280         "on client port: " + clientPort);
281 
282     if (standaloneServerFactoryList.size() == 0) {
283       // there is no backup servers;
284       return -1;
285     }
286     clientPort = clientPortList.get(activeZKServerIndex);
287     LOG.info("Activate a backup zk server in the cluster " +
288         "on client port: " + clientPort);
289     // return the next back zk server's port
290     return clientPort;
291   }
292 
293   /**
294    * Kill one back up ZK servers
295    * @throws IOException
296    * @throws InterruptedException
297    */
298   public void killOneBackupZooKeeperServer() throws IOException,
299                                         InterruptedException {
300     if (!started || activeZKServerIndex < 0 ||
301         standaloneServerFactoryList.size() <= 1) {
302       return ;
303     }
304 
305     int backupZKServerIndex = activeZKServerIndex+1;
306     // Shutdown the current active one
307     NIOServerCnxnFactory standaloneServerFactory =
308       standaloneServerFactoryList.get(backupZKServerIndex);
309     int clientPort = clientPortList.get(backupZKServerIndex);
310 
311     standaloneServerFactory.shutdown();
312     if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
313       throw new IOException("Waiting for shutdown of standalone server");
314     }
315 
316     zooKeeperServers.get(backupZKServerIndex).getZKDatabase().close();
317 
318     // remove this backup zk server
319     standaloneServerFactoryList.remove(backupZKServerIndex);
320     clientPortList.remove(backupZKServerIndex);
321     zooKeeperServers.remove(backupZKServerIndex);
322     LOG.info("Kill one backup ZK servers in the cluster " +
323         "on client port: " + clientPort);
324   }
325 
326   // XXX: From o.a.zk.t.ClientBase
327   private static boolean waitForServerDown(int port, long timeout) {
328     long start = System.currentTimeMillis();
329     while (true) {
330       try {
331         Socket sock = new Socket("localhost", port);
332         try {
333           OutputStream outstream = sock.getOutputStream();
334           outstream.write("stat".getBytes());
335           outstream.flush();
336         } finally {
337           sock.close();
338         }
339       } catch (IOException e) {
340         return true;
341       }
342 
343       if (System.currentTimeMillis() > start + timeout) {
344         break;
345       }
346       try {
347         Thread.sleep(250);
348       } catch (InterruptedException e) {
349         // ignore
350       }
351     }
352     return false;
353   }
354 
355   // XXX: From o.a.zk.t.ClientBase
356   private static boolean waitForServerUp(int port, long timeout) {
357     long start = System.currentTimeMillis();
358     while (true) {
359       try {
360         Socket sock = new Socket("localhost", port);
361         BufferedReader reader = null;
362         try {
363           OutputStream outstream = sock.getOutputStream();
364           outstream.write("stat".getBytes());
365           outstream.flush();
366 
367           Reader isr = new InputStreamReader(sock.getInputStream());
368           reader = new BufferedReader(isr);
369           String line = reader.readLine();
370           if (line != null && line.startsWith("Zookeeper version:")) {
371             return true;
372           }
373         } finally {
374           sock.close();
375           if (reader != null) {
376             reader.close();
377           }
378         }
379       } catch (IOException e) {
380         // ignore as this is expected
381         LOG.info("server localhost:" + port + " not up " + e);
382       }
383 
384       if (System.currentTimeMillis() > start + timeout) {
385         break;
386       }
387       try {
388         Thread.sleep(250);
389       } catch (InterruptedException e) {
390         // ignore
391       }
392     }
393     return false;
394   }
395 
396   public int getClientPort() {
397     return clientPort;
398   }
399 }