View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.zookeeper;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.HashSet;
25  import java.util.List;
26  import java.util.Set;
27  import java.util.concurrent.CopyOnWriteArrayList;
28  import java.util.concurrent.CountDownLatch;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.hbase.Abortable;
34  import org.apache.hadoop.hbase.HConstants;
35  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
36  import org.apache.hadoop.hbase.util.Threads;
37  import org.apache.zookeeper.KeeperException;
38  import org.apache.zookeeper.WatchedEvent;
39  import org.apache.zookeeper.Watcher;
40  import org.apache.zookeeper.ZooDefs;
41  import org.apache.zookeeper.data.ACL;
42  
43  /**
44   * Acts as the single ZooKeeper Watcher.  One instance of this is instantiated
45   * for each Master, RegionServer, and client process.
46   *
47   * <p>This is the only class that implements {@link Watcher}.  Other internal
48   * classes which need to be notified of ZooKeeper events must register with
49   * the local instance of this watcher via {@link #registerListener}.
50   *
51   * <p>This class also holds and manages the connection to ZooKeeper.  Code to
52   * deal with connection related events and exceptions are handled here.
53   */
54  public class ZooKeeperWatcher implements Watcher, Abortable {
55    private static final Log LOG = LogFactory.getLog(ZooKeeperWatcher.class);
56  
57    // Identifier for this watcher (for logging only).  It is made of the prefix
58    // passed on construction and the zookeeper sessionid.
59    private String identifier;
60  
61    // zookeeper quorum
62    private String quorum;
63  
64    // zookeeper connection
65    private RecoverableZooKeeper recoverableZooKeeper;
66  
67    // abortable in case of zk failure
68    private Abortable abortable;
69  
70    // listeners to be notified
71    private final List<ZooKeeperListener> listeners =
72      new CopyOnWriteArrayList<ZooKeeperListener>();
73  
74    // set of unassigned nodes watched
75    private Set<String> unassignedNodes = new HashSet<String>();
76  
77    // node names
78  
79    // base znode for this cluster
80    public String baseZNode;
81    // znode containing location of server hosting root region
82    public String rootServerZNode;
83    // znode containing ephemeral nodes of the regionservers
84    public String rsZNode;
85    // znode containing ephemeral nodes of the draining regionservers
86    public String drainingZNode;
87    // znode of currently active master
88    public String masterAddressZNode;
89    // znode of this master in backup master directory, if not the active master
90    public String backupMasterAddressesZNode;
91    // znode containing the current cluster state
92    public String clusterStateZNode;
93    // znode used for region transitioning and assignment
94    public String assignmentZNode;
95    // znode that the master uses for reading/writing the table disabling/enabling states
96    public String masterTableZNode;
97    // znode where the client reads table enabling/disabling states.
98    public String clientTableZNode;
99    // znode where the master writes table disabling/enabling states in the format expected
100   // by 0.92.0/0.92.1 clients for backwards compatibility.  See HBASE-6710 for details.
101   public String masterTableZNode92;
102   // znode containing the unique cluster ID
103   public String clusterIdZNode;
104   // znode used for log splitting work assignment
105   public String splitLogZNode;
106 
107   // Certain ZooKeeper nodes need to be world-readable
108   public static final ArrayList<ACL> CREATOR_ALL_AND_WORLD_READABLE =
109     new ArrayList<ACL>() { {
110       add(new ACL(ZooDefs.Perms.READ,ZooDefs.Ids.ANYONE_ID_UNSAFE));
111       add(new ACL(ZooDefs.Perms.ALL,ZooDefs.Ids.AUTH_IDS));
112     }};
113 
114   private final Configuration conf;
115 
116   private final Exception constructorCaller;
117 
118   /**
119    * Instantiate a ZooKeeper connection and watcher.
120    * @param descriptor Descriptive string that is added to zookeeper sessionid
121    * and used as identifier for this instance.
122    * @throws IOException
123    * @throws ZooKeeperConnectionException
124    */
125   public ZooKeeperWatcher(Configuration conf, String descriptor,
126       Abortable abortable) throws ZooKeeperConnectionException, IOException {
127     this(conf, descriptor, abortable, false);
128   }
129   /**
130    * Instantiate a ZooKeeper connection and watcher.
131    * @param descriptor Descriptive string that is added to zookeeper sessionid
132    * and used as identifier for this instance.
133    * @throws IOException
134    * @throws ZooKeeperConnectionException
135    */
136   public ZooKeeperWatcher(Configuration conf, String descriptor,
137       Abortable abortable, boolean canCreateBaseZNode)
138   throws IOException, ZooKeeperConnectionException {
139     this.conf = conf;
140     // Capture a stack trace now.  Will print it out later if problem so we can
141     // distingush amongst the myriad ZKWs.
142     try {
143       throw new Exception("ZKW CONSTRUCTOR STACK TRACE FOR DEBUGGING");
144     } catch (Exception e) {
145       this.constructorCaller = e;
146     }
147     this.quorum = ZKConfig.getZKQuorumServersString(conf);
148     // Identifier will get the sessionid appended later below down when we
149     // handle the syncconnect event.
150     this.identifier = descriptor;
151     this.abortable = abortable;
152     setNodeNames(conf);
153     this.recoverableZooKeeper = ZKUtil.connect(conf, quorum, this, descriptor);
154     if (canCreateBaseZNode) {
155       createBaseZNodes();
156     }
157   }
158 
159   private void createBaseZNodes() throws ZooKeeperConnectionException {
160     try {
161       // Create all the necessary "directories" of znodes
162       ZKUtil.createAndFailSilent(this, baseZNode);
163       ZKUtil.createAndFailSilent(this, assignmentZNode);
164       ZKUtil.createAndFailSilent(this, rsZNode);
165       ZKUtil.createAndFailSilent(this, drainingZNode);
166       ZKUtil.createAndFailSilent(this, masterTableZNode);
167       ZKUtil.createAndFailSilent(this, masterTableZNode92);
168       ZKUtil.createAndFailSilent(this, splitLogZNode);
169       ZKUtil.createAndFailSilent(this, backupMasterAddressesZNode);
170     } catch (KeeperException e) {
171       throw new ZooKeeperConnectionException(
172           prefix("Unexpected KeeperException creating base node"), e);
173     }
174   }
175 
176   private boolean isFinishedRetryingRecoverable(final long finished) {
177     return System.currentTimeMillis() < finished;
178   }
179 
180   @Override
181   public String toString() {
182     return this.identifier;
183   }
184 
185   /**
186    * Adds this instance's identifier as a prefix to the passed <code>str</code>
187    * @param str String to amend.
188    * @return A new string with this instance's identifier as prefix: e.g.
189    * if passed 'hello world', the returned string could be
190    */
191   public String prefix(final String str) {
192     return this.toString() + " " + str;
193   }
194 
195   /**
196    * Set the local variable node names using the specified configuration.
197    */
198   private void setNodeNames(Configuration conf) {
199     baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
200         HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
201     rootServerZNode = ZKUtil.joinZNode(baseZNode,
202         conf.get("zookeeper.znode.rootserver", "root-region-server"));
203     rsZNode = ZKUtil.joinZNode(baseZNode,
204         conf.get("zookeeper.znode.rs", "rs"));
205     drainingZNode = ZKUtil.joinZNode(baseZNode,
206         conf.get("zookeeper.znode.draining.rs", "draining"));
207     masterAddressZNode = ZKUtil.joinZNode(baseZNode,
208         conf.get("zookeeper.znode.master", "master"));
209     backupMasterAddressesZNode = ZKUtil.joinZNode(baseZNode,
210         conf.get("zookeeper.znode.backup.masters", "backup-masters"));
211     clusterStateZNode = ZKUtil.joinZNode(baseZNode,
212         conf.get("zookeeper.znode.state", "shutdown"));
213     assignmentZNode = ZKUtil.joinZNode(baseZNode,
214         conf.get("zookeeper.znode.unassigned", "unassigned"));
215     String tableZNodeDefault = "table";
216     masterTableZNode = ZKUtil.joinZNode(baseZNode,
217         conf.get("zookeeper.znode.masterTableEnableDisable", tableZNodeDefault));
218     clientTableZNode = ZKUtil.joinZNode(baseZNode,
219             conf.get("zookeeper.znode.clientTableEnableDisable", tableZNodeDefault));
220     masterTableZNode92 = ZKUtil.joinZNode(baseZNode,
221         conf.get("zookeeper.znode.masterTableEnableDisable92", "table92"));
222     clusterIdZNode = ZKUtil.joinZNode(baseZNode,
223         conf.get("zookeeper.znode.clusterId", "hbaseid"));
224     splitLogZNode = ZKUtil.joinZNode(baseZNode,
225         conf.get("zookeeper.znode.splitlog", HConstants.SPLIT_LOGDIR_NAME));
226   }
227 
228   /**
229    * Register the specified listener to receive ZooKeeper events.
230    * @param listener
231    */
232   public void registerListener(ZooKeeperListener listener) {
233     listeners.add(listener);
234   }
235 
236   /**
237    * Register the specified listener to receive ZooKeeper events and add it as
238    * the first in the list of current listeners.
239    * @param listener
240    */
241   public void registerListenerFirst(ZooKeeperListener listener) {
242     listeners.add(0, listener);
243   }
244 
245   /**
246    * Get the connection to ZooKeeper.
247    * @return connection reference to zookeeper
248    */
249   public RecoverableZooKeeper getRecoverableZooKeeper() {
250     return recoverableZooKeeper;
251   }
252 
253   public void reconnectAfterExpiration() throws IOException, InterruptedException {
254     recoverableZooKeeper.reconnectAfterExpiration();
255   }
256 
257   /**
258    * Get the quorum address of this instance.
259    * @return quorum string of this zookeeper connection instance
260    */
261   public String getQuorum() {
262     return quorum;
263   }
264 
265   /**
266    * Method called from ZooKeeper for events and connection status.
267    * <p>
268    * Valid events are passed along to listeners.  Connection status changes
269    * are dealt with locally.
270    */
271   @Override
272   public void process(WatchedEvent event) {
273     LOG.debug(prefix("Received ZooKeeper Event, " +
274         "type=" + event.getType() + ", " +
275         "state=" + event.getState() + ", " +
276         "path=" + event.getPath()));
277 
278     switch(event.getType()) {
279 
280       // If event type is NONE, this is a connection status change
281       case None: {
282         connectionEvent(event);
283         break;
284       }
285 
286       // Otherwise pass along to the listeners
287 
288       case NodeCreated: {
289         for(ZooKeeperListener listener : listeners) {
290           listener.nodeCreated(event.getPath());
291         }
292         break;
293       }
294 
295       case NodeDeleted: {
296         for(ZooKeeperListener listener : listeners) {
297           listener.nodeDeleted(event.getPath());
298         }
299         break;
300       }
301 
302       case NodeDataChanged: {
303         for(ZooKeeperListener listener : listeners) {
304           listener.nodeDataChanged(event.getPath());
305         }
306         break;
307       }
308 
309       case NodeChildrenChanged: {
310         for(ZooKeeperListener listener : listeners) {
311           listener.nodeChildrenChanged(event.getPath());
312         }
313         break;
314       }
315     }
316   }
317 
318   // Connection management
319 
320   /**
321    * Called when there is a connection-related event via the Watcher callback.
322    * <p>
323    * If Disconnected or Expired, this should shutdown the cluster. But, since
324    * we send a KeeperException.SessionExpiredException along with the abort
325    * call, it's possible for the Abortable to catch it and try to create a new
326    * session with ZooKeeper. This is what the client does in HCM.
327    * <p>
328    * @param event
329    */
330   private void connectionEvent(WatchedEvent event) {
331     switch(event.getState()) {
332       case SyncConnected:
333         // Now, this callback can be invoked before the this.zookeeper is set.
334         // Wait a little while.
335         long finished = System.currentTimeMillis() +
336           this.conf.getLong("hbase.zookeeper.watcher.sync.connected.wait", 2000);
337         while (System.currentTimeMillis() < finished) {
338           Threads.sleep(1);
339           if (this.recoverableZooKeeper != null) break;
340         }
341         if (this.recoverableZooKeeper == null) {
342           LOG.error("ZK is null on connection event -- see stack trace " +
343             "for the stack trace when constructor was called on this zkw",
344             this.constructorCaller);
345           throw new NullPointerException("ZK is null");
346         }
347         this.identifier = this.identifier + "-0x" +
348           Long.toHexString(this.recoverableZooKeeper.getSessionId());
349         // Update our identifier.  Otherwise ignore.
350         LOG.debug(this.identifier + " connected");
351         break;
352 
353       // Abort the server if Disconnected or Expired
354       case Disconnected:
355         LOG.debug(prefix("Received Disconnected from ZooKeeper, ignoring"));
356         break;
357 
358       case Expired:
359         String msg = prefix(this.identifier + " received expired from " +
360           "ZooKeeper, aborting");
361         // TODO: One thought is to add call to ZooKeeperListener so say,
362         // ZooKeeperNodeTracker can zero out its data values.
363         if (this.abortable != null) this.abortable.abort(msg,
364             new KeeperException.SessionExpiredException());
365         break;
366     }
367   }
368 
369   /**
370    * Forces a synchronization of this ZooKeeper client connection.
371    * <p>
372    * Executing this method before running other methods will ensure that the
373    * subsequent operations are up-to-date and consistent as of the time that
374    * the sync is complete.
375    * <p>
376    * This is used for compareAndSwap type operations where we need to read the
377    * data of an existing node and delete or transition that node, utilizing the
378    * previously read version and data.  We want to ensure that the version read
379    * is up-to-date from when we begin the operation.
380    */
381   public void sync(String path) {
382     this.recoverableZooKeeper.sync(path, null, null);
383   }
384 
385   /**
386    * Handles KeeperExceptions in client calls.
387    * <p>
388    * This may be temporary but for now this gives one place to deal with these.
389    * <p>
390    * TODO: Currently this method rethrows the exception to let the caller handle
391    * <p>
392    * @param ke
393    * @throws KeeperException
394    */
395   public void keeperException(KeeperException ke)
396   throws KeeperException {
397     LOG.error(prefix("Received unexpected KeeperException, re-throwing exception"), ke);
398     throw ke;
399   }
400 
401   /**
402    * Handles InterruptedExceptions in client calls.
403    * <p>
404    * This may be temporary but for now this gives one place to deal with these.
405    * <p>
406    * TODO: Currently, this method does nothing.
407    *       Is this ever expected to happen?  Do we abort or can we let it run?
408    *       Maybe this should be logged as WARN?  It shouldn't happen?
409    * <p>
410    * @param ie
411    */
412   public void interruptedException(InterruptedException ie) {
413     LOG.debug(prefix("Received InterruptedException, doing nothing here"), ie);
414     // At least preserver interrupt.
415     Thread.currentThread().interrupt();
416     // no-op
417   }
418 
419   /**
420    * Close the connection to ZooKeeper.
421    * @throws InterruptedException
422    */
423   public void close() {
424     try {
425       if (recoverableZooKeeper != null) {
426         recoverableZooKeeper.close();
427 //        super.close();
428       }
429     } catch (InterruptedException e) {
430     }
431   }
432 
433   public Configuration getConfiguration() {
434     return conf;
435   }
436 
437   @Override
438   public void abort(String why, Throwable e) {
439     this.abortable.abort(why, e);
440   }
441   
442   @Override
443   public boolean isAborted() {
444     return this.abortable.isAborted();
445   }
446 }