View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.zookeeper;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.HashSet;
25  import java.util.List;
26  import java.util.Set;
27  import java.util.concurrent.CopyOnWriteArrayList;
28  import java.util.concurrent.CountDownLatch;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.hbase.Abortable;
34  import org.apache.hadoop.hbase.HConstants;
35  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
36  import org.apache.hadoop.hbase.util.Threads;
37  import org.apache.zookeeper.KeeperException;
38  import org.apache.zookeeper.WatchedEvent;
39  import org.apache.zookeeper.Watcher;
40  import org.apache.zookeeper.ZooDefs;
41  import org.apache.zookeeper.data.ACL;
42  
43  /**
44   * Acts as the single ZooKeeper Watcher.  One instance of this is instantiated
45   * for each Master, RegionServer, and client process.
46   *
47   * <p>This is the only class that implements {@link Watcher}.  Other internal
48   * classes which need to be notified of ZooKeeper events must register with
49   * the local instance of this watcher via {@link #registerListener}.
50   *
51   * <p>This class also holds and manages the connection to ZooKeeper.  Code to
52   * deal with connection related events and exceptions are handled here.
53   */
54  public class ZooKeeperWatcher implements Watcher, Abortable {
55    private static final Log LOG = LogFactory.getLog(ZooKeeperWatcher.class);
56  
57    // Identifier for this watcher (for logging only).  It is made of the prefix
58    // passed on construction and the zookeeper sessionid.
59    private String identifier;
60  
61    // zookeeper quorum
62    private String quorum;
63  
64    // zookeeper connection
65    private RecoverableZooKeeper recoverableZooKeeper;
66  
67    // abortable in case of zk failure
68    private Abortable abortable;
69  
70    // listeners to be notified
71    private final List<ZooKeeperListener> listeners =
72      new CopyOnWriteArrayList<ZooKeeperListener>();
73  
74    // set of unassigned nodes watched
75    private Set<String> unassignedNodes = new HashSet<String>();
76  
77    // node names
78  
79    // base znode for this cluster
80    public String baseZNode;
81    // znode containing location of server hosting root region
82    public String rootServerZNode;
83    // znode containing ephemeral nodes of the regionservers
84    public String rsZNode;
85    // znode containing ephemeral nodes of the draining regionservers
86    public String drainingZNode;
87    // znode of currently active master
88    public String masterAddressZNode;
89    // znode of this master in backup master directory, if not the active master
90    public String backupMasterAddressesZNode;
91    // znode containing the current cluster state
92    public String clusterStateZNode;
93    // znode used for region transitioning and assignment
94    public String assignmentZNode;
95    // znode that the master uses for reading/writing the table disabling/enabling states
96    public String masterTableZNode;
97    // znode where the client reads table enabling/disabling states.
98    public String clientTableZNode;
99    // znode where the master writes table disabling/enabling states in the format expected
100   // by 0.92.0/0.92.1 clients for backwards compatibility.  See HBASE-6710 for details.
101   public String masterTableZNode92;
102   // znode containing the unique cluster ID
103   public String clusterIdZNode;
104   // znode used for log splitting work assignment
105   public String splitLogZNode;
106 
107   // Certain ZooKeeper nodes need to be world-readable
108   public static final ArrayList<ACL> CREATOR_ALL_AND_WORLD_READABLE =
109     new ArrayList<ACL>() { {
110       add(new ACL(ZooDefs.Perms.READ,ZooDefs.Ids.ANYONE_ID_UNSAFE));
111       add(new ACL(ZooDefs.Perms.ALL,ZooDefs.Ids.AUTH_IDS));
112     }};
113 
114   private final Configuration conf;
115 
116   private final Exception constructorCaller;
117 
118   /**
119    * Instantiate a ZooKeeper connection and watcher.
120    * @param descriptor Descriptive string that is added to zookeeper sessionid
121    * and used as identifier for this instance.
122    * @throws IOException
123    * @throws ZooKeeperConnectionException
124    */
125   public ZooKeeperWatcher(Configuration conf, String descriptor,
126       Abortable abortable) throws ZooKeeperConnectionException, IOException {
127     this(conf, descriptor, abortable, false);
128   }
129   /**
130    * Instantiate a ZooKeeper connection and watcher.
131    * @param descriptor Descriptive string that is added to zookeeper sessionid
132    * and used as identifier for this instance.
133    * @throws IOException
134    * @throws ZooKeeperConnectionException
135    */
136   public ZooKeeperWatcher(Configuration conf, String descriptor,
137       Abortable abortable, boolean canCreateBaseZNode)
138   throws IOException, ZooKeeperConnectionException {
139     this.conf = conf;
140     // Capture a stack trace now.  Will print it out later if problem so we can
141     // distingush amongst the myriad ZKWs.
142     try {
143       throw new Exception("ZKW CONSTRUCTOR STACK TRACE FOR DEBUGGING");
144     } catch (Exception e) {
145       this.constructorCaller = e;
146     }
147     this.quorum = ZKConfig.getZKQuorumServersString(conf);
148     // Identifier will get the sessionid appended later below down when we
149     // handle the syncconnect event.
150     this.identifier = descriptor;
151     this.abortable = abortable;
152     setNodeNames(conf);
153     this.recoverableZooKeeper = ZKUtil.connect(conf, quorum, this, descriptor);
154     if (canCreateBaseZNode) {
155       createBaseZNodes();
156     }
157   }
158 
159   private void createBaseZNodes() throws ZooKeeperConnectionException {
160     try {
161       // Create all the necessary "directories" of znodes
162       ZKUtil.createAndFailSilent(this, baseZNode);
163       ZKUtil.createAndFailSilent(this, assignmentZNode);
164       ZKUtil.createAndFailSilent(this, rsZNode);
165       ZKUtil.createAndFailSilent(this, drainingZNode);
166       ZKUtil.createAndFailSilent(this, masterTableZNode);
167       ZKUtil.createAndFailSilent(this, masterTableZNode92);
168       ZKUtil.createAndFailSilent(this, splitLogZNode);
169       ZKUtil.createAndFailSilent(this, backupMasterAddressesZNode);
170     } catch (KeeperException e) {
171       throw new ZooKeeperConnectionException(
172           prefix("Unexpected KeeperException creating base node"), e);
173     }
174   }
175 
176   private boolean isFinishedRetryingRecoverable(final long finished) {
177     return System.currentTimeMillis() < finished;
178   }
179 
180   @Override
181   public String toString() {
182     return this.identifier;
183   }
184 
185   /**
186    * Adds this instance's identifier as a prefix to the passed <code>str</code>
187    * @param str String to amend.
188    * @return A new string with this instance's identifier as prefix: e.g.
189    * if passed 'hello world', the returned string could be
190    */
191   public String prefix(final String str) {
192     return this.toString() + " " + str;
193   }
194 
195   /**
196    * Set the local variable node names using the specified configuration.
197    */
198   private void setNodeNames(Configuration conf) {
199     baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
200         HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
201     rootServerZNode = ZKUtil.joinZNode(baseZNode,
202         conf.get("zookeeper.znode.rootserver", "root-region-server"));
203     rsZNode = ZKUtil.joinZNode(baseZNode,
204         conf.get("zookeeper.znode.rs", "rs"));
205     drainingZNode = ZKUtil.joinZNode(baseZNode,
206         conf.get("zookeeper.znode.draining.rs", "draining"));
207     masterAddressZNode = ZKUtil.joinZNode(baseZNode,
208         conf.get("zookeeper.znode.master", "master"));
209     backupMasterAddressesZNode = ZKUtil.joinZNode(baseZNode,
210         conf.get("zookeeper.znode.backup.masters", "backup-masters"));
211     clusterStateZNode = ZKUtil.joinZNode(baseZNode,
212         conf.get("zookeeper.znode.state", "shutdown"));
213     assignmentZNode = ZKUtil.joinZNode(baseZNode,
214         conf.get("zookeeper.znode.unassigned", "unassigned"));
215     String tableZNodeDefault = "table";
216     masterTableZNode = ZKUtil.joinZNode(baseZNode,
217         conf.get("zookeeper.znode.masterTableEnableDisable", tableZNodeDefault));
218     clientTableZNode = ZKUtil.joinZNode(baseZNode,
219             conf.get("zookeeper.znode.clientTableEnableDisable", tableZNodeDefault));
220     masterTableZNode92 = ZKUtil.joinZNode(baseZNode,
221         conf.get("zookeeper.znode.masterTableEnableDisable92", "table92"));
222     clusterIdZNode = ZKUtil.joinZNode(baseZNode,
223         conf.get("zookeeper.znode.clusterId", "hbaseid"));
224     splitLogZNode = ZKUtil.joinZNode(baseZNode,
225         conf.get("zookeeper.znode.splitlog", HConstants.SPLIT_LOGDIR_NAME));
226   }
227 
228   /**
229    * Register the specified listener to receive ZooKeeper events.
230    * @param listener
231    */
232   public void registerListener(ZooKeeperListener listener) {
233     listeners.add(listener);
234   }
235 
236   /**
237    * Register the specified listener to receive ZooKeeper events and add it as
238    * the first in the list of current listeners.
239    * @param listener
240    */
241   public void registerListenerFirst(ZooKeeperListener listener) {
242     listeners.add(0, listener);
243   }
244 
245   /**
246    * Clean all existing listeners
247    */
248   public void unregisterAllListeners() {
249     listeners.clear();
250   }
251 
252   /**
253    * Get a copy of current registered listeners
254    */
255   public List<ZooKeeperListener> getListeners() {
256     return new ArrayList<ZooKeeperListener>(listeners);
257   }
258 
259   /**
260    * @return The number of currently registered listeners
261    */
262   public int getNumberOfListeners() {
263     return listeners.size();
264   }
265 
266   /**
267    * Get the connection to ZooKeeper.
268    * @return connection reference to zookeeper
269    */
270   public RecoverableZooKeeper getRecoverableZooKeeper() {
271     return recoverableZooKeeper;
272   }
273 
274   public void reconnectAfterExpiration() throws IOException, KeeperException, InterruptedException {
275     recoverableZooKeeper.reconnectAfterExpiration();
276   }
277 
278   /**
279    * Get the quorum address of this instance.
280    * @return quorum string of this zookeeper connection instance
281    */
282   public String getQuorum() {
283     return quorum;
284   }
285 
286   /**
287    * Method called from ZooKeeper for events and connection status.
288    * <p>
289    * Valid events are passed along to listeners.  Connection status changes
290    * are dealt with locally.
291    */
292   @Override
293   public void process(WatchedEvent event) {
294     LOG.debug(prefix("Received ZooKeeper Event, " +
295         "type=" + event.getType() + ", " +
296         "state=" + event.getState() + ", " +
297         "path=" + event.getPath()));
298 
299     switch(event.getType()) {
300 
301       // If event type is NONE, this is a connection status change
302       case None: {
303         connectionEvent(event);
304         break;
305       }
306 
307       // Otherwise pass along to the listeners
308 
309       case NodeCreated: {
310         for(ZooKeeperListener listener : listeners) {
311           listener.nodeCreated(event.getPath());
312         }
313         break;
314       }
315 
316       case NodeDeleted: {
317         for(ZooKeeperListener listener : listeners) {
318           listener.nodeDeleted(event.getPath());
319         }
320         break;
321       }
322 
323       case NodeDataChanged: {
324         for(ZooKeeperListener listener : listeners) {
325           listener.nodeDataChanged(event.getPath());
326         }
327         break;
328       }
329 
330       case NodeChildrenChanged: {
331         for(ZooKeeperListener listener : listeners) {
332           listener.nodeChildrenChanged(event.getPath());
333         }
334         break;
335       }
336     }
337   }
338 
339   // Connection management
340 
341   /**
342    * Called when there is a connection-related event via the Watcher callback.
343    * <p>
344    * If Disconnected or Expired, this should shutdown the cluster. But, since
345    * we send a KeeperException.SessionExpiredException along with the abort
346    * call, it's possible for the Abortable to catch it and try to create a new
347    * session with ZooKeeper. This is what the client does in HCM.
348    * <p>
349    * @param event
350    */
351   private void connectionEvent(WatchedEvent event) {
352     switch(event.getState()) {
353       case SyncConnected:
354         // Now, this callback can be invoked before the this.zookeeper is set.
355         // Wait a little while.
356         long finished = System.currentTimeMillis() +
357           this.conf.getLong("hbase.zookeeper.watcher.sync.connected.wait", 2000);
358         while (System.currentTimeMillis() < finished) {
359           Threads.sleep(1);
360           if (this.recoverableZooKeeper != null) break;
361         }
362         if (this.recoverableZooKeeper == null) {
363           LOG.error("ZK is null on connection event -- see stack trace " +
364             "for the stack trace when constructor was called on this zkw",
365             this.constructorCaller);
366           throw new NullPointerException("ZK is null");
367         }
368         this.identifier = this.identifier + "-0x" +
369           Long.toHexString(this.recoverableZooKeeper.getSessionId());
370         // Update our identifier.  Otherwise ignore.
371         LOG.debug(this.identifier + " connected");
372         break;
373 
374       // Abort the server if Disconnected or Expired
375       case Disconnected:
376         LOG.debug(prefix("Received Disconnected from ZooKeeper, ignoring"));
377         break;
378 
379       case Expired:
380         String msg = prefix(this.identifier + " received expired from " +
381           "ZooKeeper, aborting");
382         // TODO: One thought is to add call to ZooKeeperListener so say,
383         // ZooKeeperNodeTracker can zero out its data values.
384         if (this.abortable != null) this.abortable.abort(msg,
385             new KeeperException.SessionExpiredException());
386         break;
387     }
388   }
389 
390   /**
391    * Forces a synchronization of this ZooKeeper client connection.
392    * <p>
393    * Executing this method before running other methods will ensure that the
394    * subsequent operations are up-to-date and consistent as of the time that
395    * the sync is complete.
396    * <p>
397    * This is used for compareAndSwap type operations where we need to read the
398    * data of an existing node and delete or transition that node, utilizing the
399    * previously read version and data.  We want to ensure that the version read
400    * is up-to-date from when we begin the operation.
401    */
402   public void sync(String path) throws KeeperException {
403     this.recoverableZooKeeper.sync(path, null, null);
404   }
405 
406   /**
407    * Handles KeeperExceptions in client calls.
408    * <p>
409    * This may be temporary but for now this gives one place to deal with these.
410    * <p>
411    * TODO: Currently this method rethrows the exception to let the caller handle
412    * <p>
413    * @param ke
414    * @throws KeeperException
415    */
416   public void keeperException(KeeperException ke)
417   throws KeeperException {
418     LOG.error(prefix("Received unexpected KeeperException, re-throwing exception"), ke);
419     throw ke;
420   }
421 
422   /**
423    * Handles InterruptedExceptions in client calls.
424    * <p>
425    * This may be temporary but for now this gives one place to deal with these.
426    * <p>
427    * TODO: Currently, this method does nothing.
428    *       Is this ever expected to happen?  Do we abort or can we let it run?
429    *       Maybe this should be logged as WARN?  It shouldn't happen?
430    * <p>
431    * @param ie
432    */
433   public void interruptedException(InterruptedException ie) {
434     LOG.debug(prefix("Received InterruptedException, doing nothing here"), ie);
435     // At least preserver interrupt.
436     Thread.currentThread().interrupt();
437     // no-op
438   }
439 
440   /**
441    * Close the connection to ZooKeeper.
442    * @throws InterruptedException
443    */
444   public void close() {
445     try {
446       if (recoverableZooKeeper != null) {
447         recoverableZooKeeper.close();
448 //        super.close();
449       }
450     } catch (InterruptedException e) {
451     }
452   }
453 
454   public Configuration getConfiguration() {
455     return conf;
456   }
457 
458   @Override
459   public void abort(String why, Throwable e) {
460     this.abortable.abort(why, e);
461   }
462   
463   @Override
464   public boolean isAborted() {
465     return this.abortable.isAborted();
466   }
467 }