View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.zookeeper;
20  
21  import java.io.BufferedReader;
22  import java.io.IOException;
23  import java.io.InputStreamReader;
24  import java.io.PrintWriter;
25  import java.net.InetSocketAddress;
26  import java.net.Socket;
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.Deque;
30  import java.util.HashMap;
31  import java.util.LinkedList;
32  import java.util.List;
33  import java.util.Map;
34  import java.util.Properties;
35  
36  import javax.security.auth.login.AppConfigurationEntry;
37  import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
38  
39  import org.apache.hadoop.hbase.util.ByteStringer;
40  import org.apache.commons.lang.StringUtils;
41  import org.apache.commons.logging.Log;
42  import org.apache.commons.logging.LogFactory;
43  import org.apache.hadoop.hbase.classification.InterfaceAudience;
44  import org.apache.hadoop.conf.Configuration;
45  import org.apache.hadoop.hbase.HConstants;
46  import org.apache.hadoop.hbase.ServerName;
47  import org.apache.hadoop.hbase.exceptions.DeserializationException;
48  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
49  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
50  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.RegionStoreSequenceIds;
51  import org.apache.hadoop.hbase.util.Bytes;
52  import org.apache.hadoop.hbase.util.Threads;
53  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.CreateAndFailSilent;
54  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.DeleteNodeFailSilent;
55  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.SetData;
56  import org.apache.hadoop.security.SecurityUtil;
57  import org.apache.hadoop.security.authentication.util.KerberosUtil;
58  import org.apache.zookeeper.AsyncCallback;
59  import org.apache.zookeeper.CreateMode;
60  import org.apache.zookeeper.KeeperException;
61  import org.apache.zookeeper.KeeperException.NoNodeException;
62  import org.apache.zookeeper.Op;
63  import org.apache.zookeeper.Watcher;
64  import org.apache.zookeeper.ZooDefs.Ids;
65  import org.apache.zookeeper.ZooDefs.Perms;
66  import org.apache.zookeeper.ZooKeeper;
67  import org.apache.zookeeper.client.ZooKeeperSaslClient;
68  import org.apache.zookeeper.data.ACL;
69  import org.apache.zookeeper.data.Id;
70  import org.apache.zookeeper.data.Stat;
71  import org.apache.zookeeper.proto.CreateRequest;
72  import org.apache.zookeeper.proto.DeleteRequest;
73  import org.apache.zookeeper.proto.SetDataRequest;
74  import org.apache.zookeeper.server.ZooKeeperSaslServer;
75  
76  /**
77   * Internal HBase utility class for ZooKeeper.
78   *
79   * <p>Contains only static methods and constants.
80   *
81   * <p>Methods all throw {@link KeeperException} if there is an unexpected
82   * zookeeper exception, so callers of these methods must handle appropriately.
83   * If ZK is required for the operation, the server will need to be aborted.
84   */
85  @InterfaceAudience.Private
86  public class ZKUtil {
87    private static final Log LOG = LogFactory.getLog(ZKUtil.class);
88  
89    // TODO: Replace this with ZooKeeper constant when ZOOKEEPER-277 is resolved.
90    public static final char ZNODE_PATH_SEPARATOR = '/';
91    private static int zkDumpConnectionTimeOut;
92  
93    /**
94     * Creates a new connection to ZooKeeper, pulling settings and ensemble config
95     * from the specified configuration object using methods from {@link ZKConfig}.
96     *
97     * Sets the connection status monitoring watcher to the specified watcher.
98     *
99     * @param conf configuration to pull ensemble and other settings from
100    * @param watcher watcher to monitor connection changes
101    * @return connection to zookeeper
102    * @throws IOException if unable to connect to zk or config problem
103    */
104   public static RecoverableZooKeeper connect(Configuration conf, Watcher watcher)
105   throws IOException {
106     Properties properties = ZKConfig.makeZKProps(conf);
107     String ensemble = ZKConfig.getZKQuorumServersString(properties);
108     return connect(conf, ensemble, watcher);
109   }
110 
111   public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
112       Watcher watcher)
113   throws IOException {
114     return connect(conf, ensemble, watcher, null);
115   }
116 
117   public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
118       Watcher watcher, final String identifier)
119   throws IOException {
120     if(ensemble == null) {
121       throw new IOException("Unable to determine ZooKeeper ensemble");
122     }
123     int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
124         HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
125     if (LOG.isTraceEnabled()) {
126       LOG.trace(identifier + " opening connection to ZooKeeper ensemble=" + ensemble);
127     }
128     int retry = conf.getInt("zookeeper.recovery.retry", 3);
129     int retryIntervalMillis =
130       conf.getInt("zookeeper.recovery.retry.intervalmill", 1000);
131     zkDumpConnectionTimeOut = conf.getInt("zookeeper.dump.connection.timeout",
132         1000);
133     return new RecoverableZooKeeper(ensemble, timeout, watcher,
134         retry, retryIntervalMillis, identifier);
135   }
136 
137   /**
138    * Log in the current zookeeper server process using the given configuration
139    * keys for the credential file and login principal.
140    *
141    * <p><strong>This is only applicable when running on secure hbase</strong>
142    * On regular HBase (without security features), this will safely be ignored.
143    * </p>
144    *
145    * @param conf The configuration data to use
146    * @param keytabFileKey Property key used to configure the path to the credential file
147    * @param userNameKey Property key used to configure the login principal
148    * @param hostname Current hostname to use in any credentials
149    * @throws IOException underlying exception from SecurityUtil.login() call
150    */
151   public static void loginServer(Configuration conf, String keytabFileKey,
152       String userNameKey, String hostname) throws IOException {
153     login(conf, keytabFileKey, userNameKey, hostname,
154           ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY,
155           JaasConfiguration.SERVER_KEYTAB_KERBEROS_CONFIG_NAME);
156   }
157 
158   /**
159    * Log in the current zookeeper client using the given configuration
160    * keys for the credential file and login principal.
161    *
162    * <p><strong>This is only applicable when running on secure hbase</strong>
163    * On regular HBase (without security features), this will safely be ignored.
164    * </p>
165    *
166    * @param conf The configuration data to use
167    * @param keytabFileKey Property key used to configure the path to the credential file
168    * @param userNameKey Property key used to configure the login principal
169    * @param hostname Current hostname to use in any credentials
170    * @throws IOException underlying exception from SecurityUtil.login() call
171    */
172   public static void loginClient(Configuration conf, String keytabFileKey,
173       String userNameKey, String hostname) throws IOException {
174     login(conf, keytabFileKey, userNameKey, hostname,
175           ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
176           JaasConfiguration.CLIENT_KEYTAB_KERBEROS_CONFIG_NAME);
177   }
178 
179   /**
180    * Log in the current process using the given configuration keys for the
181    * credential file and login principal.
182    *
183    * <p><strong>This is only applicable when running on secure hbase</strong>
184    * On regular HBase (without security features), this will safely be ignored.
185    * </p>
186    *
187    * @param conf The configuration data to use
188    * @param keytabFileKey Property key used to configure the path to the credential file
189    * @param userNameKey Property key used to configure the login principal
190    * @param hostname Current hostname to use in any credentials
191    * @param loginContextProperty property name to expose the entry name
192    * @param loginContextName jaas entry name
193    * @throws IOException underlying exception from SecurityUtil.login() call
194    */
195   private static void login(Configuration conf, String keytabFileKey,
196       String userNameKey, String hostname,
197       String loginContextProperty, String loginContextName)
198       throws IOException {
199     if (!isSecureZooKeeper(conf))
200       return;
201 
202     // User has specified a jaas.conf, keep this one as the good one.
203     // HBASE_OPTS="-Djava.security.auth.login.config=jaas.conf"
204     if (System.getProperty("java.security.auth.login.config") != null)
205       return;
206 
207     // No keytab specified, no auth
208     String keytabFilename = conf.get(keytabFileKey);
209     if (keytabFilename == null) {
210       LOG.warn("no keytab specified for: " + keytabFileKey);
211       return;
212     }
213 
214     String principalConfig = conf.get(userNameKey, System.getProperty("user.name"));
215     String principalName = SecurityUtil.getServerPrincipal(principalConfig, hostname);
216 
217     // Initialize the "jaas.conf" for keyTab/principal,
218     // If keyTab is not specified use the Ticket Cache.
219     // and set the zookeeper login context name.
220     JaasConfiguration jaasConf = new JaasConfiguration(loginContextName,
221         principalName, keytabFilename);
222     javax.security.auth.login.Configuration.setConfiguration(jaasConf);
223     System.setProperty(loginContextProperty, loginContextName);
224   }
225 
226   /**
227    * A JAAS configuration that defines the login modules that we want to use for login.
228    */
229   private static class JaasConfiguration extends javax.security.auth.login.Configuration {
230     private static final String SERVER_KEYTAB_KERBEROS_CONFIG_NAME =
231       "zookeeper-server-keytab-kerberos";
232     private static final String CLIENT_KEYTAB_KERBEROS_CONFIG_NAME =
233       "zookeeper-client-keytab-kerberos";
234 
235     private static final Map<String, String> BASIC_JAAS_OPTIONS =
236       new HashMap<String,String>();
237     static {
238       String jaasEnvVar = System.getenv("HBASE_JAAS_DEBUG");
239       if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) {
240         BASIC_JAAS_OPTIONS.put("debug", "true");
241       }
242     }
243 
244     private static final Map<String,String> KEYTAB_KERBEROS_OPTIONS =
245       new HashMap<String,String>();
246     static {
247       KEYTAB_KERBEROS_OPTIONS.put("doNotPrompt", "true");
248       KEYTAB_KERBEROS_OPTIONS.put("storeKey", "true");
249       KEYTAB_KERBEROS_OPTIONS.put("refreshKrb5Config", "true");
250       KEYTAB_KERBEROS_OPTIONS.putAll(BASIC_JAAS_OPTIONS);
251     }
252 
253     private static final AppConfigurationEntry KEYTAB_KERBEROS_LOGIN =
254       new AppConfigurationEntry(KerberosUtil.getKrb5LoginModuleName(),
255                                 LoginModuleControlFlag.REQUIRED,
256                                 KEYTAB_KERBEROS_OPTIONS);
257 
258     private static final AppConfigurationEntry[] KEYTAB_KERBEROS_CONF =
259       new AppConfigurationEntry[]{KEYTAB_KERBEROS_LOGIN};
260 
261     private javax.security.auth.login.Configuration baseConfig;
262     private final String loginContextName;
263     private final boolean useTicketCache;
264     private final String keytabFile;
265     private final String principal;
266 
267     public JaasConfiguration(String loginContextName, String principal) {
268       this(loginContextName, principal, null, true);
269     }
270 
271     public JaasConfiguration(String loginContextName, String principal, String keytabFile) {
272       this(loginContextName, principal, keytabFile, keytabFile == null || keytabFile.length() == 0);
273     }
274 
275     private JaasConfiguration(String loginContextName, String principal,
276                              String keytabFile, boolean useTicketCache) {
277       try {
278         this.baseConfig = javax.security.auth.login.Configuration.getConfiguration();
279       } catch (SecurityException e) {
280         this.baseConfig = null;
281       }
282       this.loginContextName = loginContextName;
283       this.useTicketCache = useTicketCache;
284       this.keytabFile = keytabFile;
285       this.principal = principal;
286       LOG.info("JaasConfiguration loginContextName=" + loginContextName +
287                " principal=" + principal + " useTicketCache=" + useTicketCache +
288                " keytabFile=" + keytabFile);
289     }
290 
291     @Override
292     public AppConfigurationEntry[] getAppConfigurationEntry(String appName) {
293       if (loginContextName.equals(appName)) {
294         if (!useTicketCache) {
295           KEYTAB_KERBEROS_OPTIONS.put("keyTab", keytabFile);
296           KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true");
297         }
298         KEYTAB_KERBEROS_OPTIONS.put("principal", principal);
299         KEYTAB_KERBEROS_OPTIONS.put("useTicketCache", useTicketCache ? "true" : "false");
300         return KEYTAB_KERBEROS_CONF;
301       }
302       if (baseConfig != null) return baseConfig.getAppConfigurationEntry(appName);
303       return(null);
304     }
305   }
306 
307   //
308   // Helper methods
309   //
310 
311   /**
312    * Join the prefix znode name with the suffix znode name to generate a proper
313    * full znode name.
314    *
315    * Assumes prefix does not end with slash and suffix does not begin with it.
316    *
317    * @param prefix beginning of znode name
318    * @param suffix ending of znode name
319    * @return result of properly joining prefix with suffix
320    */
321   public static String joinZNode(String prefix, String suffix) {
322     return prefix + ZNODE_PATH_SEPARATOR + suffix;
323   }
324 
325   /**
326    * Returns the full path of the immediate parent of the specified node.
327    * @param node path to get parent of
328    * @return parent of path, null if passed the root node or an invalid node
329    */
330   public static String getParent(String node) {
331     int idx = node.lastIndexOf(ZNODE_PATH_SEPARATOR);
332     return idx <= 0 ? null : node.substring(0, idx);
333   }
334 
335   /**
336    * Get the name of the current node from the specified fully-qualified path.
337    * @param path fully-qualified path
338    * @return name of the current node
339    */
340   public static String getNodeName(String path) {
341     return path.substring(path.lastIndexOf("/")+1);
342   }
343 
344   /**
345    * Get the key to the ZK ensemble for this configuration without
346    * adding a name at the end
347    * @param conf Configuration to use to build the key
348    * @return ensemble key without a name
349    */
350   public static String getZooKeeperClusterKey(Configuration conf) {
351     return getZooKeeperClusterKey(conf, null);
352   }
353 
354   /**
355    * Get the key to the ZK ensemble for this configuration and append
356    * a name at the end
357    * @param conf Configuration to use to build the key
358    * @param name Name that should be appended at the end if not empty or null
359    * @return ensemble key with a name (if any)
360    */
361   public static String getZooKeeperClusterKey(Configuration conf, String name) {
362     String ensemble = conf.get(HConstants.ZOOKEEPER_QUORUM.replaceAll(
363         "[\\t\\n\\x0B\\f\\r]", ""));
364     StringBuilder builder = new StringBuilder(ensemble);
365     builder.append(":");
366     builder.append(conf.get(HConstants.ZOOKEEPER_CLIENT_PORT));
367     builder.append(":");
368     builder.append(conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT));
369     if (name != null && !name.isEmpty()) {
370       builder.append(",");
371       builder.append(name);
372     }
373     return builder.toString();
374   }
375 
376   /**
377    * Apply the settings in the given key to the given configuration, this is
378    * used to communicate with distant clusters
379    * @param conf configuration object to configure
380    * @param key string that contains the 3 required configuratins
381    * @throws IOException
382    */
383   public static void applyClusterKeyToConf(Configuration conf, String key)
384       throws IOException{
385     String[] parts = transformClusterKey(key);
386     conf.set(HConstants.ZOOKEEPER_QUORUM, parts[0]);
387     conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, parts[1]);
388     conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parts[2]);
389   }
390 
391   /**
392    * Separate the given key into the three configurations it should contain:
393    * hbase.zookeeper.quorum, hbase.zookeeper.client.port
394    * and zookeeper.znode.parent
395    * @param key
396    * @return the three configuration in the described order
397    * @throws IOException
398    */
399   public static String[] transformClusterKey(String key) throws IOException {
400     String[] parts = key.split(":");
401     if (parts.length != 3) {
402       throw new IOException("Cluster key passed " + key + " is invalid, the format should be:" +
403           HConstants.ZOOKEEPER_QUORUM + ":hbase.zookeeper.client.port:"
404           + HConstants.ZOOKEEPER_ZNODE_PARENT);
405     }
406     return parts;
407   }
408 
409   //
410   // Existence checks and watches
411   //
412 
413   /**
414    * Watch the specified znode for delete/create/change events.  The watcher is
415    * set whether or not the node exists.  If the node already exists, the method
416    * returns true.  If the node does not exist, the method returns false.
417    *
418    * @param zkw zk reference
419    * @param znode path of node to watch
420    * @return true if znode exists, false if does not exist or error
421    * @throws KeeperException if unexpected zookeeper exception
422    */
423   public static boolean watchAndCheckExists(ZooKeeperWatcher zkw, String znode)
424   throws KeeperException {
425     try {
426       Stat s = zkw.getRecoverableZooKeeper().exists(znode, zkw);
427       boolean exists = s != null ? true : false;
428       if (exists) {
429         LOG.debug(zkw.prefix("Set watcher on existing znode=" + znode));
430       } else {
431         LOG.debug(zkw.prefix("Set watcher on znode that does not yet exist, " + znode));
432       }
433       return exists;
434     } catch (KeeperException e) {
435       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
436       zkw.keeperException(e);
437       return false;
438     } catch (InterruptedException e) {
439       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
440       zkw.interruptedException(e);
441       return false;
442     }
443   }
444 
445   /**
446    * Watch the specified znode, but only if exists. Useful when watching
447    * for deletions. Uses .getData() (and handles NoNodeException) instead
448    * of .exists() to accomplish this, as .getData() will only set a watch if
449    * the znode exists.
450    * @param zkw zk reference
451    * @param znode path of node to watch
452    * @return true if the watch is set, false if node does not exists
453    * @throws KeeperException if unexpected zookeeper exception
454    */
455   public static boolean setWatchIfNodeExists(ZooKeeperWatcher zkw, String znode)
456       throws KeeperException {
457     try {
458       zkw.getRecoverableZooKeeper().getData(znode, true, null);
459       return true;
460     } catch (NoNodeException e) {
461       return false;
462     } catch (InterruptedException e) {
463       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
464       zkw.interruptedException(e);
465       return false;
466     }
467   }
468 
469   /**
470    * Check if the specified node exists.  Sets no watches.
471    *
472    * @param zkw zk reference
473    * @param znode path of node to watch
474    * @return version of the node if it exists, -1 if does not exist
475    * @throws KeeperException if unexpected zookeeper exception
476    */
477   public static int checkExists(ZooKeeperWatcher zkw, String znode)
478   throws KeeperException {
479     try {
480       Stat s = zkw.getRecoverableZooKeeper().exists(znode, null);
481       return s != null ? s.getVersion() : -1;
482     } catch (KeeperException e) {
483       LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
484       zkw.keeperException(e);
485       return -1;
486     } catch (InterruptedException e) {
487       LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
488       zkw.interruptedException(e);
489       return -1;
490     }
491   }
492 
493   //
494   // Znode listings
495   //
496 
497   /**
498    * Lists the children znodes of the specified znode.  Also sets a watch on
499    * the specified znode which will capture a NodeDeleted event on the specified
500    * znode as well as NodeChildrenChanged if any children of the specified znode
501    * are created or deleted.
502    *
503    * Returns null if the specified node does not exist.  Otherwise returns a
504    * list of children of the specified node.  If the node exists but it has no
505    * children, an empty list will be returned.
506    *
507    * @param zkw zk reference
508    * @param znode path of node to list and watch children of
509    * @return list of children of the specified node, an empty list if the node
510    *          exists but has no children, and null if the node does not exist
511    * @throws KeeperException if unexpected zookeeper exception
512    */
513   public static List<String> listChildrenAndWatchForNewChildren(
514       ZooKeeperWatcher zkw, String znode)
515   throws KeeperException {
516     try {
517       List<String> children = zkw.getRecoverableZooKeeper().getChildren(znode, zkw);
518       return children;
519     } catch(KeeperException.NoNodeException ke) {
520       LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
521           "because node does not exist (not an error)"));
522       return null;
523     } catch (KeeperException e) {
524       LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
525       zkw.keeperException(e);
526       return null;
527     } catch (InterruptedException e) {
528       LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
529       zkw.interruptedException(e);
530       return null;
531     }
532   }
533 
534   /**
535    * List all the children of the specified znode, setting a watch for children
536    * changes and also setting a watch on every individual child in order to get
537    * the NodeCreated and NodeDeleted events.
538    * @param zkw zookeeper reference
539    * @param znode node to get children of and watch
540    * @return list of znode names, null if the node doesn't exist
541    * @throws KeeperException
542    */
543   public static List<String> listChildrenAndWatchThem(ZooKeeperWatcher zkw,
544       String znode) throws KeeperException {
545     List<String> children = listChildrenAndWatchForNewChildren(zkw, znode);
546     if (children == null) {
547       return null;
548     }
549     for (String child : children) {
550       watchAndCheckExists(zkw, joinZNode(znode, child));
551     }
552     return children;
553   }
554 
555   /**
556    * Lists the children of the specified znode without setting any watches.
557    *
558    * Sets no watches at all, this method is best effort.
559    *
560    * Returns an empty list if the node has no children.  Returns null if the
561    * parent node itself does not exist.
562    *
563    * @param zkw zookeeper reference
564    * @param znode node to get children
565    * @return list of data of children of specified znode, empty if no children,
566    *         null if parent does not exist
567    * @throws KeeperException if unexpected zookeeper exception
568    */
569   public static List<String> listChildrenNoWatch(ZooKeeperWatcher zkw, String znode)
570   throws KeeperException {
571     List<String> children = null;
572     try {
573       // List the children without watching
574       children = zkw.getRecoverableZooKeeper().getChildren(znode, null);
575     } catch(KeeperException.NoNodeException nne) {
576       return null;
577     } catch(InterruptedException ie) {
578       zkw.interruptedException(ie);
579     }
580     return children;
581   }
582 
583   /**
584    * Simple class to hold a node path and node data.
585    * @deprecated Unused
586    */
587   @Deprecated
588   public static class NodeAndData {
589     private String node;
590     private byte [] data;
591     public NodeAndData(String node, byte [] data) {
592       this.node = node;
593       this.data = data;
594     }
595     public String getNode() {
596       return node;
597     }
598     public byte [] getData() {
599       return data;
600     }
601     @Override
602     public String toString() {
603       return node;
604     }
605     public boolean isEmpty() {
606       return (data == null || data.length == 0);
607     }
608   }
609 
610   /**
611    * Checks if the specified znode has any children.  Sets no watches.
612    *
613    * Returns true if the node exists and has children.  Returns false if the
614    * node does not exist or if the node does not have any children.
615    *
616    * Used during master initialization to determine if the master is a
617    * failed-over-to master or the first master during initial cluster startup.
618    * If the directory for regionserver ephemeral nodes is empty then this is
619    * a cluster startup, if not then it is not cluster startup.
620    *
621    * @param zkw zk reference
622    * @param znode path of node to check for children of
623    * @return true if node has children, false if not or node does not exist
624    * @throws KeeperException if unexpected zookeeper exception
625    */
626   public static boolean nodeHasChildren(ZooKeeperWatcher zkw, String znode)
627   throws KeeperException {
628     try {
629       return !zkw.getRecoverableZooKeeper().getChildren(znode, null).isEmpty();
630     } catch(KeeperException.NoNodeException ke) {
631       LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
632       "because node does not exist (not an error)"));
633       return false;
634     } catch (KeeperException e) {
635       LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
636       zkw.keeperException(e);
637       return false;
638     } catch (InterruptedException e) {
639       LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
640       zkw.interruptedException(e);
641       return false;
642     }
643   }
644 
645   /**
646    * Get the number of children of the specified node.
647    *
648    * If the node does not exist or has no children, returns 0.
649    *
650    * Sets no watches at all.
651    *
652    * @param zkw zk reference
653    * @param znode path of node to count children of
654    * @return number of children of specified node, 0 if none or parent does not
655    *         exist
656    * @throws KeeperException if unexpected zookeeper exception
657    */
658   public static int getNumberOfChildren(ZooKeeperWatcher zkw, String znode)
659   throws KeeperException {
660     try {
661       Stat stat = zkw.getRecoverableZooKeeper().exists(znode, null);
662       return stat == null ? 0 : stat.getNumChildren();
663     } catch(KeeperException e) {
664       LOG.warn(zkw.prefix("Unable to get children of node " + znode));
665       zkw.keeperException(e);
666     } catch(InterruptedException e) {
667       zkw.interruptedException(e);
668     }
669     return 0;
670   }
671 
672   //
673   // Data retrieval
674   //
675 
676   /**
677    * Get znode data. Does not set a watcher.
678    * @return ZNode data, null if the node does not exist or if there is an
679    *  error.
680    */
681   public static byte [] getData(ZooKeeperWatcher zkw, String znode)
682   throws KeeperException {
683     try {
684       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, null);
685       logRetrievedMsg(zkw, znode, data, false);
686       return data;
687     } catch (KeeperException.NoNodeException e) {
688       LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
689         "because node does not exist (not an error)"));
690       return null;
691     } catch (KeeperException e) {
692       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
693       zkw.keeperException(e);
694       return null;
695     } catch (InterruptedException e) {
696       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
697       zkw.interruptedException(e);
698       return null;
699     }
700   }
701 
702   /**
703    * Get the data at the specified znode and set a watch.
704    *
705    * Returns the data and sets a watch if the node exists.  Returns null and no
706    * watch is set if the node does not exist or there is an exception.
707    *
708    * @param zkw zk reference
709    * @param znode path of node
710    * @return data of the specified znode, or null
711    * @throws KeeperException if unexpected zookeeper exception
712    */
713   public static byte [] getDataAndWatch(ZooKeeperWatcher zkw, String znode)
714   throws KeeperException {
715     return getDataInternal(zkw, znode, null, true);
716   }
717 
718   /**
719    * Get the data at the specified znode and set a watch.
720    *
721    * Returns the data and sets a watch if the node exists.  Returns null and no
722    * watch is set if the node does not exist or there is an exception.
723    *
724    * @param zkw zk reference
725    * @param znode path of node
726    * @param stat object to populate the version of the znode
727    * @return data of the specified znode, or null
728    * @throws KeeperException if unexpected zookeeper exception
729    */
730   public static byte[] getDataAndWatch(ZooKeeperWatcher zkw, String znode,
731       Stat stat) throws KeeperException {
732     return getDataInternal(zkw, znode, stat, true);
733   }
734 
735   private static byte[] getDataInternal(ZooKeeperWatcher zkw, String znode, Stat stat,
736       boolean watcherSet)
737       throws KeeperException {
738     try {
739       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, zkw, stat);
740       logRetrievedMsg(zkw, znode, data, watcherSet);
741       return data;
742     } catch (KeeperException.NoNodeException e) {
743       // This log can get pretty annoying when we cycle on 100ms waits.
744       // Enable trace if you really want to see it.
745       LOG.trace(zkw.prefix("Unable to get data of znode " + znode + " " +
746         "because node does not exist (not an error)"));
747       return null;
748     } catch (KeeperException e) {
749       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
750       zkw.keeperException(e);
751       return null;
752     } catch (InterruptedException e) {
753       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
754       zkw.interruptedException(e);
755       return null;
756     }
757   }
758 
759   /**
760    * Get the data at the specified znode without setting a watch.
761    *
762    * Returns the data if the node exists.  Returns null if the node does not
763    * exist.
764    *
765    * Sets the stats of the node in the passed Stat object.  Pass a null stat if
766    * not interested.
767    *
768    * @param zkw zk reference
769    * @param znode path of node
770    * @param stat node status to get if node exists
771    * @return data of the specified znode, or null if node does not exist
772    * @throws KeeperException if unexpected zookeeper exception
773    */
774   public static byte [] getDataNoWatch(ZooKeeperWatcher zkw, String znode,
775       Stat stat)
776   throws KeeperException {
777     try {
778       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, stat);
779       logRetrievedMsg(zkw, znode, data, false);
780       return data;
781     } catch (KeeperException.NoNodeException e) {
782       LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
783           "because node does not exist (not necessarily an error)"));
784       return null;
785     } catch (KeeperException e) {
786       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
787       zkw.keeperException(e);
788       return null;
789     } catch (InterruptedException e) {
790       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
791       zkw.interruptedException(e);
792       return null;
793     }
794   }
795 
796   /**
797    * Returns the date of child znodes of the specified znode.  Also sets a watch on
798    * the specified znode which will capture a NodeDeleted event on the specified
799    * znode as well as NodeChildrenChanged if any children of the specified znode
800    * are created or deleted.
801    *
802    * Returns null if the specified node does not exist.  Otherwise returns a
803    * list of children of the specified node.  If the node exists but it has no
804    * children, an empty list will be returned.
805    *
806    * @param zkw zk reference
807    * @param baseNode path of node to list and watch children of
808    * @return list of data of children of the specified node, an empty list if the node
809    *          exists but has no children, and null if the node does not exist
810    * @throws KeeperException if unexpected zookeeper exception
811    * @deprecated Unused
812    */
813   public static List<NodeAndData> getChildDataAndWatchForNewChildren(
814       ZooKeeperWatcher zkw, String baseNode) throws KeeperException {
815     List<String> nodes =
816       ZKUtil.listChildrenAndWatchForNewChildren(zkw, baseNode);
817     List<NodeAndData> newNodes = new ArrayList<NodeAndData>();
818     if (nodes != null) {
819       for (String node : nodes) {
820         String nodePath = ZKUtil.joinZNode(baseNode, node);
821         byte[] data = ZKUtil.getDataAndWatch(zkw, nodePath);
822         newNodes.add(new NodeAndData(nodePath, data));
823       }
824     }
825     return newNodes;
826   }
827 
828   /**
829    * Update the data of an existing node with the expected version to have the
830    * specified data.
831    *
832    * Throws an exception if there is a version mismatch or some other problem.
833    *
834    * Sets no watches under any conditions.
835    *
836    * @param zkw zk reference
837    * @param znode
838    * @param data
839    * @param expectedVersion
840    * @throws KeeperException if unexpected zookeeper exception
841    * @throws KeeperException.BadVersionException if version mismatch
842    * @deprecated Unused
843    */
844   public static void updateExistingNodeData(ZooKeeperWatcher zkw, String znode,
845       byte [] data, int expectedVersion)
846   throws KeeperException {
847     try {
848       zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion);
849     } catch(InterruptedException ie) {
850       zkw.interruptedException(ie);
851     }
852   }
853 
854   //
855   // Data setting
856   //
857 
858   /**
859    * Sets the data of the existing znode to be the specified data.  Ensures that
860    * the current data has the specified expected version.
861    *
862    * <p>If the node does not exist, a {@link NoNodeException} will be thrown.
863    *
864    * <p>If their is a version mismatch, method returns null.
865    *
866    * <p>No watches are set but setting data will trigger other watchers of this
867    * node.
868    *
869    * <p>If there is another problem, a KeeperException will be thrown.
870    *
871    * @param zkw zk reference
872    * @param znode path of node
873    * @param data data to set for node
874    * @param expectedVersion version expected when setting data
875    * @return true if data set, false if version mismatch
876    * @throws KeeperException if unexpected zookeeper exception
877    */
878   public static boolean setData(ZooKeeperWatcher zkw, String znode,
879       byte [] data, int expectedVersion)
880   throws KeeperException, KeeperException.NoNodeException {
881     try {
882       return zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion) != null;
883     } catch (InterruptedException e) {
884       zkw.interruptedException(e);
885       return false;
886     }
887   }
888 
889   /**
890    * Set data into node creating node if it doesn't yet exist.
891    * Does not set watch.
892    *
893    * @param zkw zk reference
894    * @param znode path of node
895    * @param data data to set for node
896    * @throws KeeperException
897    */
898   public static void createSetData(final ZooKeeperWatcher zkw, final String znode,
899       final byte [] data)
900   throws KeeperException {
901     if (checkExists(zkw, znode) == -1) {
902       ZKUtil.createWithParents(zkw, znode, data);
903     } else {
904       ZKUtil.setData(zkw, znode, data);
905     }
906   }
907 
908   /**
909    * Sets the data of the existing znode to be the specified data.  The node
910    * must exist but no checks are done on the existing data or version.
911    *
912    * <p>If the node does not exist, a {@link NoNodeException} will be thrown.
913    *
914    * <p>No watches are set but setting data will trigger other watchers of this
915    * node.
916    *
917    * <p>If there is another problem, a KeeperException will be thrown.
918    *
919    * @param zkw zk reference
920    * @param znode path of node
921    * @param data data to set for node
922    * @throws KeeperException if unexpected zookeeper exception
923    */
924   public static void setData(ZooKeeperWatcher zkw, String znode, byte [] data)
925   throws KeeperException, KeeperException.NoNodeException {
926     setData(zkw, (SetData)ZKUtilOp.setData(znode, data));
927   }
928 
929   private static void setData(ZooKeeperWatcher zkw, SetData setData)
930   throws KeeperException, KeeperException.NoNodeException {
931     SetDataRequest sd = (SetDataRequest)toZooKeeperOp(zkw, setData).toRequestRecord();
932     setData(zkw, sd.getPath(), sd.getData(), sd.getVersion());
933   }
934 
935   /**
936    * Returns whether or not secure authentication is enabled
937    * (whether <code>hbase.security.authentication</code> is set to
938    * <code>kerberos</code>.
939    */
940   public static boolean isSecureZooKeeper(Configuration conf) {
941     // Detection for embedded HBase client with jaas configuration
942     // defined for third party programs.
943     try {
944       javax.security.auth.login.Configuration testConfig =
945           javax.security.auth.login.Configuration.getConfiguration();
946       if (testConfig.getAppConfigurationEntry("Client") == null
947           && testConfig.getAppConfigurationEntry(
948             JaasConfiguration.CLIENT_KEYTAB_KERBEROS_CONFIG_NAME) == null
949           && testConfig.getAppConfigurationEntry(
950               JaasConfiguration.SERVER_KEYTAB_KERBEROS_CONFIG_NAME) == null 
951           && conf.get(HConstants.ZK_CLIENT_KERBEROS_PRINCIPAL) == null
952           && conf.get(HConstants.ZK_SERVER_KERBEROS_PRINCIPAL) == null) {
953         return false;
954       }
955     } catch(Exception e) {
956       // No Jaas configuration defined.
957       return false;
958     }
959 
960     // Master & RSs uses hbase.zookeeper.client.*
961     return "kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication"));
962   }
963 
964   private static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node) {
965     return createACL(zkw, node, isSecureZooKeeper(zkw.getConfiguration()));
966   }
967 
968   public static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node,
969     boolean isSecureZooKeeper) {
970     if (!node.startsWith(zkw.baseZNode)) {
971       return Ids.OPEN_ACL_UNSAFE;
972     }
973     if (isSecureZooKeeper) {
974       String superUser = zkw.getConfiguration().get("hbase.superuser");
975       ArrayList<ACL> acls = new ArrayList<ACL>();
976       // add permission to hbase supper user
977       if (superUser != null) {
978         acls.add(new ACL(Perms.ALL, new Id("auth", superUser)));
979       }
980       // Certain znodes are accessed directly by the client,
981       // so they must be readable by non-authenticated clients
982       if (zkw.isClientReadable(node)) {
983         acls.addAll(Ids.CREATOR_ALL_ACL);
984         acls.addAll(Ids.READ_ACL_UNSAFE);
985       } else {
986         acls.addAll(Ids.CREATOR_ALL_ACL);
987       }
988       return acls;
989     } else {
990       return Ids.OPEN_ACL_UNSAFE;
991     }
992   }
993 
994   //
995   // Node creation
996   //
997 
998   /**
999    *
1000    * Set the specified znode to be an ephemeral node carrying the specified
1001    * data.
1002    *
1003    * If the node is created successfully, a watcher is also set on the node.
1004    *
1005    * If the node is not created successfully because it already exists, this
1006    * method will also set a watcher on the node.
1007    *
1008    * If there is another problem, a KeeperException will be thrown.
1009    *
1010    * @param zkw zk reference
1011    * @param znode path of node
1012    * @param data data of node
1013    * @return true if node created, false if not, watch set in both cases
1014    * @throws KeeperException if unexpected zookeeper exception
1015    */
1016   public static boolean createEphemeralNodeAndWatch(ZooKeeperWatcher zkw,
1017       String znode, byte [] data)
1018   throws KeeperException {
1019     boolean ret = true;
1020     try {
1021       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1022           CreateMode.EPHEMERAL);
1023     } catch (KeeperException.NodeExistsException nee) {
1024       ret = false;
1025     } catch (InterruptedException e) {
1026       LOG.info("Interrupted", e);
1027       Thread.currentThread().interrupt();
1028     }
1029     if(!watchAndCheckExists(zkw, znode)) {
1030       // It did exist but now it doesn't, try again
1031       return createEphemeralNodeAndWatch(zkw, znode, data);
1032     }
1033     return ret;
1034   }
1035 
1036   /**
1037    * Creates the specified znode to be a persistent node carrying the specified
1038    * data.
1039    *
1040    * Returns true if the node was successfully created, false if the node
1041    * already existed.
1042    *
1043    * If the node is created successfully, a watcher is also set on the node.
1044    *
1045    * If the node is not created successfully because it already exists, this
1046    * method will also set a watcher on the node but return false.
1047    *
1048    * If there is another problem, a KeeperException will be thrown.
1049    *
1050    * @param zkw zk reference
1051    * @param znode path of node
1052    * @param data data of node
1053    * @return true if node created, false if not, watch set in both cases
1054    * @throws KeeperException if unexpected zookeeper exception
1055    */
1056   public static boolean createNodeIfNotExistsAndWatch(
1057       ZooKeeperWatcher zkw, String znode, byte [] data)
1058   throws KeeperException {
1059     boolean ret = true;
1060     try {
1061       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1062           CreateMode.PERSISTENT);
1063     } catch (KeeperException.NodeExistsException nee) {
1064       ret = false;
1065     } catch (InterruptedException e) {
1066       zkw.interruptedException(e);
1067       return false;
1068     }
1069     try {
1070       zkw.getRecoverableZooKeeper().exists(znode, zkw);
1071     } catch (InterruptedException e) {
1072       zkw.interruptedException(e);
1073       return false;
1074     }
1075     return ret;
1076   }
1077 
1078   /**
1079    * Creates the specified znode with the specified data but does not watch it.
1080    *
1081    * Returns the znode of the newly created node
1082    *
1083    * If there is another problem, a KeeperException will be thrown.
1084    *
1085    * @param zkw zk reference
1086    * @param znode path of node
1087    * @param data data of node
1088    * @param createMode specifying whether the node to be created is ephemeral and/or sequential
1089    * @return true name of the newly created znode or null
1090    * @throws KeeperException if unexpected zookeeper exception
1091    */
1092   public static String createNodeIfNotExistsNoWatch(ZooKeeperWatcher zkw, String znode,
1093       byte[] data, CreateMode createMode) throws KeeperException {
1094 
1095     String createdZNode = null;
1096     try {
1097       createdZNode = zkw.getRecoverableZooKeeper().create(znode, data,
1098           createACL(zkw, znode), createMode);
1099     } catch (KeeperException.NodeExistsException nee) {
1100       return znode;
1101     } catch (InterruptedException e) {
1102       zkw.interruptedException(e);
1103       return null;
1104     }
1105     return createdZNode;
1106   }
1107 
1108   /**
1109    * Creates the specified node with the specified data and watches it.
1110    *
1111    * <p>Throws an exception if the node already exists.
1112    *
1113    * <p>The node created is persistent and open access.
1114    *
1115    * <p>Returns the version number of the created node if successful.
1116    *
1117    * @param zkw zk reference
1118    * @param znode path of node to create
1119    * @param data data of node to create
1120    * @return version of node created
1121    * @throws KeeperException if unexpected zookeeper exception
1122    * @throws KeeperException.NodeExistsException if node already exists
1123    */
1124   public static int createAndWatch(ZooKeeperWatcher zkw,
1125       String znode, byte [] data)
1126   throws KeeperException, KeeperException.NodeExistsException {
1127     try {
1128       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1129           CreateMode.PERSISTENT);
1130       Stat stat = zkw.getRecoverableZooKeeper().exists(znode, zkw);
1131       if (stat == null){
1132         // Likely a race condition. Someone deleted the znode.
1133         throw KeeperException.create(KeeperException.Code.SYSTEMERROR,
1134             "ZK.exists returned null (i.e.: znode does not exist) for znode=" + znode);
1135       }
1136      return stat.getVersion();
1137     } catch (InterruptedException e) {
1138       zkw.interruptedException(e);
1139       return -1;
1140     }
1141   }
1142 
1143   /**
1144    * Async creates the specified node with the specified data.
1145    *
1146    * <p>Throws an exception if the node already exists.
1147    *
1148    * <p>The node created is persistent and open access.
1149    *
1150    * @param zkw zk reference
1151    * @param znode path of node to create
1152    * @param data data of node to create
1153    * @param cb
1154    * @param ctx
1155    * @throws KeeperException if unexpected zookeeper exception
1156    * @throws KeeperException.NodeExistsException if node already exists
1157    */
1158   public static void asyncCreate(ZooKeeperWatcher zkw,
1159       String znode, byte [] data, final AsyncCallback.StringCallback cb,
1160       final Object ctx) {
1161     zkw.getRecoverableZooKeeper().getZooKeeper().create(znode, data,
1162         createACL(zkw, znode), CreateMode.PERSISTENT, cb, ctx);
1163   }
1164 
1165   /**
1166    * Creates the specified node, iff the node does not exist.  Does not set a
1167    * watch and fails silently if the node already exists.
1168    *
1169    * The node created is persistent and open access.
1170    *
1171    * @param zkw zk reference
1172    * @param znode path of node
1173    * @throws KeeperException if unexpected zookeeper exception
1174    */
1175   public static void createAndFailSilent(ZooKeeperWatcher zkw,
1176       String znode) throws KeeperException {
1177     createAndFailSilent(zkw, znode, new byte[0]);
1178   }
1179 
1180   /**
1181    * Creates the specified node containing specified data, iff the node does not exist.  Does
1182    * not set a watch and fails silently if the node already exists.
1183    *
1184    * The node created is persistent and open access.
1185    *
1186    * @param zkw zk reference
1187    * @param znode path of node
1188    * @param data a byte array data to store in the znode
1189    * @throws KeeperException if unexpected zookeeper exception
1190    */
1191   public static void createAndFailSilent(ZooKeeperWatcher zkw,
1192       String znode, byte[] data)
1193   throws KeeperException {
1194     createAndFailSilent(zkw,
1195         (CreateAndFailSilent)ZKUtilOp.createAndFailSilent(znode, data));
1196   }
1197   
1198   private static void createAndFailSilent(ZooKeeperWatcher zkw, CreateAndFailSilent cafs)
1199   throws KeeperException {
1200     CreateRequest create = (CreateRequest)toZooKeeperOp(zkw, cafs).toRequestRecord();
1201     String znode = create.getPath();
1202     try {
1203       RecoverableZooKeeper zk = zkw.getRecoverableZooKeeper();
1204       if (zk.exists(znode, false) == null) {
1205         zk.create(znode, create.getData(), create.getAcl(), CreateMode.fromFlag(create.getFlags()));
1206       }
1207     } catch(KeeperException.NodeExistsException nee) {
1208     } catch(KeeperException.NoAuthException nee){
1209       try {
1210         if (null == zkw.getRecoverableZooKeeper().exists(znode, false)) {
1211           // If we failed to create the file and it does not already exist.
1212           throw(nee);
1213         }
1214       } catch (InterruptedException ie) {
1215         zkw.interruptedException(ie);
1216       }
1217 
1218     } catch(InterruptedException ie) {
1219       zkw.interruptedException(ie);
1220     }
1221   }
1222 
1223   /**
1224    * Creates the specified node and all parent nodes required for it to exist.
1225    *
1226    * No watches are set and no errors are thrown if the node already exists.
1227    *
1228    * The nodes created are persistent and open access.
1229    *
1230    * @param zkw zk reference
1231    * @param znode path of node
1232    * @throws KeeperException if unexpected zookeeper exception
1233    */
1234   public static void createWithParents(ZooKeeperWatcher zkw, String znode)
1235   throws KeeperException {
1236     createWithParents(zkw, znode, new byte[0]);
1237   }
1238 
1239   /**
1240    * Creates the specified node and all parent nodes required for it to exist.  The creation of
1241    * parent znodes is not atomic with the leafe znode creation but the data is written atomically
1242    * when the leaf node is created.
1243    *
1244    * No watches are set and no errors are thrown if the node already exists.
1245    *
1246    * The nodes created are persistent and open access.
1247    *
1248    * @param zkw zk reference
1249    * @param znode path of node
1250    * @throws KeeperException if unexpected zookeeper exception
1251    */
1252   public static void createWithParents(ZooKeeperWatcher zkw, String znode, byte[] data)
1253   throws KeeperException {
1254     try {
1255       if(znode == null) {
1256         return;
1257       }
1258       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1259           CreateMode.PERSISTENT);
1260     } catch(KeeperException.NodeExistsException nee) {
1261       return;
1262     } catch(KeeperException.NoNodeException nne) {
1263       createWithParents(zkw, getParent(znode));
1264       createWithParents(zkw, znode, data);
1265     } catch(InterruptedException ie) {
1266       zkw.interruptedException(ie);
1267     }
1268   }
1269 
1270   //
1271   // Deletes
1272   //
1273 
1274   /**
1275    * Delete the specified node.  Sets no watches.  Throws all exceptions.
1276    */
1277   public static void deleteNode(ZooKeeperWatcher zkw, String node)
1278   throws KeeperException {
1279     deleteNode(zkw, node, -1);
1280   }
1281 
1282   /**
1283    * Delete the specified node with the specified version.  Sets no watches.
1284    * Throws all exceptions.
1285    */
1286   public static boolean deleteNode(ZooKeeperWatcher zkw, String node,
1287       int version)
1288   throws KeeperException {
1289     try {
1290       zkw.getRecoverableZooKeeper().delete(node, version);
1291       return true;
1292     } catch(KeeperException.BadVersionException bve) {
1293       return false;
1294     } catch(InterruptedException ie) {
1295       zkw.interruptedException(ie);
1296       return false;
1297     }
1298   }
1299 
1300   /**
1301    * Deletes the specified node.  Fails silent if the node does not exist.
1302    * @param zkw
1303    * @param node
1304    * @throws KeeperException
1305    */
1306   public static void deleteNodeFailSilent(ZooKeeperWatcher zkw, String node)
1307   throws KeeperException {
1308     deleteNodeFailSilent(zkw,
1309       (DeleteNodeFailSilent)ZKUtilOp.deleteNodeFailSilent(node));
1310   }
1311 
1312   private static void deleteNodeFailSilent(ZooKeeperWatcher zkw,
1313       DeleteNodeFailSilent dnfs) throws KeeperException {
1314     DeleteRequest delete = (DeleteRequest)toZooKeeperOp(zkw, dnfs).toRequestRecord();
1315     try {
1316       zkw.getRecoverableZooKeeper().delete(delete.getPath(), delete.getVersion());
1317     } catch(KeeperException.NoNodeException nne) {
1318     } catch(InterruptedException ie) {
1319       zkw.interruptedException(ie);
1320     }
1321   }
1322 
1323 
1324   /**
1325    * Delete the specified node and all of it's children.
1326    * <p>
1327    * If the node does not exist, just returns.
1328    * <p>
1329    * Sets no watches. Throws all exceptions besides dealing with deletion of
1330    * children.
1331    */
1332   public static void deleteNodeRecursively(ZooKeeperWatcher zkw, String node)
1333   throws KeeperException {
1334     deleteNodeRecursivelyMultiOrSequential(zkw, true, node);
1335   }
1336 
1337   /**
1338    * Delete all the children of the specified node but not the node itself.
1339    *
1340    * Sets no watches.  Throws all exceptions besides dealing with deletion of
1341    * children.
1342    *
1343    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update functionality.
1344    * Otherwise, run the list of operations sequentially.
1345    *
1346    * @throws KeeperException
1347    */
1348   public static void deleteChildrenRecursively(ZooKeeperWatcher zkw, String node)
1349       throws KeeperException {
1350     deleteChildrenRecursivelyMultiOrSequential(zkw, true, node);
1351   }
1352 
1353   /**
1354    * Delete all the children of the specified node but not the node itself. This
1355    * will first traverse the znode tree for listing the children and then delete
1356    * these znodes using multi-update api or sequential based on the specified
1357    * configurations.
1358    * <p>
1359    * Sets no watches. Throws all exceptions besides dealing with deletion of
1360    * children.
1361    * <p>
1362    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update
1363    * functionality. Otherwise, run the list of operations sequentially.
1364    * <p>
1365    * If all of the following are true:
1366    * <ul>
1367    * <li>runSequentialOnMultiFailure is true
1368    * <li>hbase.zookeeper.useMulti is true
1369    * </ul>
1370    * on calling multi, we get a ZooKeeper exception that can be handled by a
1371    * sequential call(*), we retry the operations one-by-one (sequentially).
1372    *
1373    * @param zkw
1374    *          - zk reference
1375    * @param runSequentialOnMultiFailure
1376    *          - if true when we get a ZooKeeper exception that could retry the
1377    *          operations one-by-one (sequentially)
1378    * @param pathRoots
1379    *          - path of the parent node(s)
1380    * @throws KeeperException.NotEmptyException
1381    *           if node has children while deleting
1382    * @throws KeeperException
1383    *           if unexpected ZooKeeper exception
1384    * @throws IllegalArgumentException
1385    *           if an invalid path is specified
1386    */
1387   public static void deleteChildrenRecursivelyMultiOrSequential(
1388       ZooKeeperWatcher zkw, boolean runSequentialOnMultiFailure,
1389       String... pathRoots) throws KeeperException {
1390     if (pathRoots == null || pathRoots.length <= 0) {
1391       LOG.warn("Given path is not valid!");
1392       return;
1393     }
1394     List<ZKUtilOp> ops = new ArrayList<ZKUtil.ZKUtilOp>();
1395     for (String eachRoot : pathRoots) {
1396       List<String> children = listChildrenBFSNoWatch(zkw, eachRoot);
1397       // Delete the leaves first and eventually get rid of the root
1398       for (int i = children.size() - 1; i >= 0; --i) {
1399         ops.add(ZKUtilOp.deleteNodeFailSilent(children.get(i)));
1400       }
1401     }
1402     // atleast one element should exist
1403     if (ops.size() > 0) {
1404       multiOrSequential(zkw, ops, runSequentialOnMultiFailure);
1405     }
1406   }
1407 
1408   /**
1409    * Delete the specified node and its children. This traverse the
1410    * znode tree for listing the children and then delete
1411    * these znodes including the parent using multi-update api or
1412    * sequential based on the specified configurations.
1413    * <p>
1414    * Sets no watches. Throws all exceptions besides dealing with deletion of
1415    * children.
1416    * <p>
1417    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update
1418    * functionality. Otherwise, run the list of operations sequentially.
1419    * <p>
1420    * If all of the following are true:
1421    * <ul>
1422    * <li>runSequentialOnMultiFailure is true
1423    * <li>hbase.zookeeper.useMulti is true
1424    * </ul>
1425    * on calling multi, we get a ZooKeeper exception that can be handled by a
1426    * sequential call(*), we retry the operations one-by-one (sequentially).
1427    *
1428    * @param zkw
1429    *          - zk reference
1430    * @param runSequentialOnMultiFailure
1431    *          - if true when we get a ZooKeeper exception that could retry the
1432    *          operations one-by-one (sequentially)
1433    * @param pathRoots
1434    *          - path of the parent node(s)
1435    * @throws KeeperException.NotEmptyException
1436    *           if node has children while deleting
1437    * @throws KeeperException
1438    *           if unexpected ZooKeeper exception
1439    * @throws IllegalArgumentException
1440    *           if an invalid path is specified
1441    */
1442   public static void deleteNodeRecursivelyMultiOrSequential(ZooKeeperWatcher zkw,
1443       boolean runSequentialOnMultiFailure, String... pathRoots) throws KeeperException {
1444     if (pathRoots == null || pathRoots.length <= 0) {
1445       LOG.warn("Given path is not valid!");
1446       return;
1447     }
1448     List<ZKUtilOp> ops = new ArrayList<ZKUtil.ZKUtilOp>();
1449     for (String eachRoot : pathRoots) {
1450       // Zookeeper Watches are one time triggers; When children of parent nodes are deleted
1451       // recursively, must set another watch, get notified of delete node
1452       List<String> children = listChildrenBFSAndWatchThem(zkw, eachRoot);
1453       // Delete the leaves first and eventually get rid of the root
1454       for (int i = children.size() - 1; i >= 0; --i) {
1455         ops.add(ZKUtilOp.deleteNodeFailSilent(children.get(i)));
1456       }
1457       try {
1458         if (zkw.getRecoverableZooKeeper().exists(eachRoot, zkw) != null) {
1459           ops.add(ZKUtilOp.deleteNodeFailSilent(eachRoot));
1460         }
1461       } catch (InterruptedException e) {
1462         zkw.interruptedException(e);
1463       }
1464     }
1465     // atleast one element should exist
1466     if (ops.size() > 0) {
1467       multiOrSequential(zkw, ops, runSequentialOnMultiFailure);
1468     }
1469   }
1470 
1471   /**
1472    * BFS Traversal of all the children under path, with the entries in the list,
1473    * in the same order as that of the traversal. Lists all the children without
1474    * setting any watches.
1475    *
1476    * @param zkw
1477    *          - zk reference
1478    * @param znode
1479    *          - path of node
1480    * @return list of children znodes under the path
1481    * @throws KeeperException
1482    *           if unexpected ZooKeeper exception
1483    */
1484   private static List<String> listChildrenBFSNoWatch(ZooKeeperWatcher zkw,
1485       final String znode) throws KeeperException {
1486     Deque<String> queue = new LinkedList<String>();
1487     List<String> tree = new ArrayList<String>();
1488     queue.add(znode);
1489     while (true) {
1490       String node = queue.pollFirst();
1491       if (node == null) {
1492         break;
1493       }
1494       List<String> children = listChildrenNoWatch(zkw, node);
1495       if (children == null) {
1496         continue;
1497       }
1498       for (final String child : children) {
1499         final String childPath = node + "/" + child;
1500         queue.add(childPath);
1501         tree.add(childPath);
1502       }
1503     }
1504     return tree;
1505   }
1506 
1507   /**
1508    * BFS Traversal of all the children under path, with the entries in the list,
1509    * in the same order as that of the traversal.
1510    * Lists all the children and set watches on to them.
1511    *
1512    * @param zkw
1513    *          - zk reference
1514    * @param znode
1515    *          - path of node
1516    * @return list of children znodes under the path
1517    * @throws KeeperException
1518    *           if unexpected ZooKeeper exception
1519    */
1520   private static List<String> listChildrenBFSAndWatchThem(ZooKeeperWatcher zkw, final String znode)
1521       throws KeeperException {
1522     Deque<String> queue = new LinkedList<String>();
1523     List<String> tree = new ArrayList<String>();
1524     queue.add(znode);
1525     while (true) {
1526       String node = queue.pollFirst();
1527       if (node == null) {
1528         break;
1529       }
1530       List<String> children = listChildrenAndWatchThem(zkw, node);
1531       if (children == null) {
1532         continue;
1533       }
1534       for (final String child : children) {
1535         final String childPath = node + "/" + child;
1536         queue.add(childPath);
1537         tree.add(childPath);
1538       }
1539     }
1540     return tree;
1541   }
1542 
1543   /**
1544    * Represents an action taken by ZKUtil, e.g. createAndFailSilent.
1545    * These actions are higher-level than ZKOp actions, which represent
1546    * individual actions in the ZooKeeper API, like create.
1547    */
1548   public abstract static class ZKUtilOp {
1549     private String path;
1550 
1551     private ZKUtilOp(String path) {
1552       this.path = path;
1553     }
1554 
1555     /**
1556      * @return a createAndFailSilent ZKUtilOp
1557      */
1558     public static ZKUtilOp createAndFailSilent(String path, byte[] data) {
1559       return new CreateAndFailSilent(path, data);
1560     }
1561 
1562     /**
1563      * @return a deleteNodeFailSilent ZKUtilOP
1564      */
1565     public static ZKUtilOp deleteNodeFailSilent(String path) {
1566       return new DeleteNodeFailSilent(path);
1567     }
1568 
1569     /**
1570      * @return a setData ZKUtilOp
1571      */
1572     public static ZKUtilOp setData(String path, byte [] data) {
1573       return new SetData(path, data);
1574     }
1575 
1576     /**
1577      * @return path to znode where the ZKOp will occur
1578      */
1579     public String getPath() {
1580       return path;
1581     }
1582 
1583     /**
1584      * ZKUtilOp representing createAndFailSilent in ZooKeeper
1585      * (attempt to create node, ignore error if already exists)
1586      */
1587     public static class CreateAndFailSilent extends ZKUtilOp {
1588       private byte [] data;
1589 
1590       private CreateAndFailSilent(String path, byte [] data) {
1591         super(path);
1592         this.data = data;
1593       }
1594 
1595       public byte[] getData() {
1596         return data;
1597       }
1598 
1599       @Override
1600       public boolean equals(Object o) {
1601         if (this == o) return true;
1602         if (!(o instanceof CreateAndFailSilent)) return false;
1603 
1604         CreateAndFailSilent op = (CreateAndFailSilent) o;
1605         return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1606       }
1607 
1608       @Override
1609       public int hashCode() {
1610         int ret = 17 + getPath().hashCode() * 31;
1611         return ret * 31 + Bytes.hashCode(data);
1612       }
1613     }
1614 
1615     /**
1616      * ZKUtilOp representing deleteNodeFailSilent in ZooKeeper
1617      * (attempt to delete node, ignore error if node doesn't exist)
1618      */
1619     public static class DeleteNodeFailSilent extends ZKUtilOp {
1620       private DeleteNodeFailSilent(String path) {
1621         super(path);
1622       }
1623 
1624       @Override
1625       public boolean equals(Object o) {
1626         if (this == o) return true;
1627         if (!(o instanceof DeleteNodeFailSilent)) return false;
1628 
1629         return super.equals(o);
1630       }
1631 
1632       @Override
1633       public int hashCode() {
1634         return getPath().hashCode();
1635       }
1636     }
1637 
1638     /**
1639      * ZKUtilOp representing setData in ZooKeeper
1640      */
1641     public static class SetData extends ZKUtilOp {
1642       private byte [] data;
1643 
1644       private SetData(String path, byte [] data) {
1645         super(path);
1646         this.data = data;
1647       }
1648 
1649       public byte[] getData() {
1650         return data;
1651       }
1652 
1653       @Override
1654       public boolean equals(Object o) {
1655         if (this == o) return true;
1656         if (!(o instanceof SetData)) return false;
1657 
1658         SetData op = (SetData) o;
1659         return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1660       }
1661 
1662       @Override
1663       public int hashCode() {
1664         int ret = getPath().hashCode();
1665         return ret * 31 + Bytes.hashCode(data);
1666       }
1667     }
1668   }
1669 
1670   /**
1671    * Convert from ZKUtilOp to ZKOp
1672    */
1673   private static Op toZooKeeperOp(ZooKeeperWatcher zkw, ZKUtilOp op)
1674   throws UnsupportedOperationException {
1675     if(op == null) return null;
1676 
1677     if (op instanceof CreateAndFailSilent) {
1678       CreateAndFailSilent cafs = (CreateAndFailSilent)op;
1679       return Op.create(cafs.getPath(), cafs.getData(), createACL(zkw, cafs.getPath()),
1680         CreateMode.PERSISTENT);
1681     } else if (op instanceof DeleteNodeFailSilent) {
1682       DeleteNodeFailSilent dnfs = (DeleteNodeFailSilent)op;
1683       return Op.delete(dnfs.getPath(), -1);
1684     } else if (op instanceof SetData) {
1685       SetData sd = (SetData)op;
1686       return Op.setData(sd.getPath(), sd.getData(), -1);
1687     } else {
1688       throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1689         + op.getClass().getName());
1690     }
1691   }
1692 
1693   /**
1694    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update functionality.
1695    * Otherwise, run the list of operations sequentially.
1696    *
1697    * If all of the following are true:
1698    * - runSequentialOnMultiFailure is true
1699    * - hbase.zookeeper.useMulti is true
1700    * - on calling multi, we get a ZooKeeper exception that can be handled by a sequential call(*)
1701    * Then:
1702    * - we retry the operations one-by-one (sequentially)
1703    *
1704    * Note *: an example is receiving a NodeExistsException from a "create" call.  Without multi,
1705    * a user could call "createAndFailSilent" to ensure that a node exists if they don't care who
1706    * actually created the node (i.e. the NodeExistsException from ZooKeeper is caught).
1707    * This will cause all operations in the multi to fail, however, because
1708    * the NodeExistsException that zk.create throws will fail the multi transaction.
1709    * In this case, if the previous conditions hold, the commands are run sequentially, which should
1710    * result in the correct final state, but means that the operations will not run atomically.
1711    *
1712    * @throws KeeperException
1713    */
1714   public static void multiOrSequential(ZooKeeperWatcher zkw, List<ZKUtilOp> ops,
1715       boolean runSequentialOnMultiFailure) throws KeeperException {
1716     if (ops == null) return;
1717     boolean useMulti = zkw.getConfiguration().getBoolean(HConstants.ZOOKEEPER_USEMULTI, false);
1718 
1719     if (useMulti) {
1720       List<Op> zkOps = new LinkedList<Op>();
1721       for (ZKUtilOp op : ops) {
1722         zkOps.add(toZooKeeperOp(zkw, op));
1723       }
1724       try {
1725         zkw.getRecoverableZooKeeper().multi(zkOps);
1726       } catch (KeeperException ke) {
1727        switch (ke.code()) {
1728          case NODEEXISTS:
1729          case NONODE:
1730          case BADVERSION:
1731          case NOAUTH:
1732            // if we get an exception that could be solved by running sequentially
1733            // (and the client asked us to), then break out and run sequentially
1734            if (runSequentialOnMultiFailure) {
1735              LOG.info("On call to ZK.multi, received exception: " + ke.toString() + "."
1736                + "  Attempting to run operations sequentially because"
1737                + " runSequentialOnMultiFailure is: " + runSequentialOnMultiFailure + ".");
1738              processSequentially(zkw, ops);
1739              break;
1740            }
1741           default:
1742             throw ke;
1743         }
1744       } catch (InterruptedException ie) {
1745         zkw.interruptedException(ie);
1746       }
1747     } else {
1748       // run sequentially
1749       processSequentially(zkw, ops);
1750     }
1751 
1752   }
1753 
1754   private static void processSequentially(ZooKeeperWatcher zkw, List<ZKUtilOp> ops)
1755       throws KeeperException, NoNodeException {
1756     for (ZKUtilOp op : ops) {
1757       if (op instanceof CreateAndFailSilent) {
1758         createAndFailSilent(zkw, (CreateAndFailSilent) op);
1759       } else if (op instanceof DeleteNodeFailSilent) {
1760         deleteNodeFailSilent(zkw, (DeleteNodeFailSilent) op);
1761       } else if (op instanceof SetData) {
1762         setData(zkw, (SetData) op);
1763       } else {
1764         throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1765             + op.getClass().getName());
1766       }
1767     }
1768   }
1769 
1770   //
1771   // ZooKeeper cluster information
1772   //
1773 
1774   /** @return String dump of everything in ZooKeeper. */
1775   public static String dump(ZooKeeperWatcher zkw) {
1776     StringBuilder sb = new StringBuilder();
1777     try {
1778       sb.append("HBase is rooted at ").append(zkw.baseZNode);
1779       sb.append("\nActive master address: ");
1780       try {
1781         sb.append(MasterAddressTracker.getMasterAddress(zkw));
1782       } catch (IOException e) {
1783         sb.append("<<FAILED LOOKUP: " + e.getMessage() + ">>");
1784       }
1785       sb.append("\nBackup master addresses:");
1786       for (String child : listChildrenNoWatch(zkw,
1787                                               zkw.backupMasterAddressesZNode)) {
1788         sb.append("\n ").append(child);
1789       }
1790       sb.append("\nRegion server holding hbase:meta: " + MetaRegionTracker.getMetaRegionLocation(zkw));
1791       sb.append("\nRegion servers:");
1792       for (String child : listChildrenNoWatch(zkw, zkw.rsZNode)) {
1793         sb.append("\n ").append(child);
1794       }
1795       try {
1796         getReplicationZnodesDump(zkw, sb);
1797       } catch (KeeperException ke) {
1798         LOG.warn("Couldn't get the replication znode dump", ke);
1799       }
1800       sb.append("\nQuorum Server Statistics:");
1801       String[] servers = zkw.getQuorum().split(",");
1802       for (String server : servers) {
1803         sb.append("\n ").append(server);
1804         try {
1805           String[] stat = getServerStats(server, ZKUtil.zkDumpConnectionTimeOut);
1806 
1807           if (stat == null) {
1808             sb.append("[Error] invalid quorum server: " + server);
1809             break;
1810           }
1811 
1812           for (String s : stat) {
1813             sb.append("\n  ").append(s);
1814           }
1815         } catch (Exception e) {
1816           sb.append("\n  ERROR: ").append(e.getMessage());
1817         }
1818       }
1819     } catch (KeeperException ke) {
1820       sb.append("\nFATAL ZooKeeper Exception!\n");
1821       sb.append("\n" + ke.getMessage());
1822     }
1823     return sb.toString();
1824   }
1825 
1826   /**
1827    * Appends replication znodes to the passed StringBuilder.
1828    * @param zkw
1829    * @param sb
1830    * @throws KeeperException
1831    */
1832   private static void getReplicationZnodesDump(ZooKeeperWatcher zkw, StringBuilder sb)
1833       throws KeeperException {
1834     String replicationZNodeName = zkw.getConfiguration().get("zookeeper.znode.replication",
1835       "replication");
1836     String replicationZnode = joinZNode(zkw.baseZNode, replicationZNodeName);
1837     if (ZKUtil.checkExists(zkw, replicationZnode) == -1) return;
1838     // do a ls -r on this znode
1839     sb.append("\n").append(replicationZnode).append(": ");
1840     List<String> children = ZKUtil.listChildrenNoWatch(zkw, replicationZnode);
1841     for (String child : children) {
1842       String znode = joinZNode(replicationZnode, child);
1843       if (child.equals(zkw.getConfiguration().get("zookeeper.znode.replication.peers", "peers"))) {
1844         appendPeersZnodes(zkw, znode, sb);
1845       } else if (child.equals(zkw.getConfiguration().
1846           get("zookeeper.znode.replication.rs", "rs"))) {
1847         appendRSZnodes(zkw, znode, sb);
1848       }
1849     }
1850   }
1851 
1852   private static void appendRSZnodes(ZooKeeperWatcher zkw, String znode, StringBuilder sb)
1853       throws KeeperException {
1854     List<String> stack = new LinkedList<String>();
1855     stack.add(znode);
1856     do {
1857       String znodeToProcess = stack.remove(stack.size() - 1);
1858       sb.append("\n").append(znodeToProcess).append(": ");
1859       byte[] data = ZKUtil.getData(zkw, znodeToProcess);
1860       if (data != null && data.length > 0) { // log position
1861         long position = 0;
1862         try {
1863           position = ZKUtil.parseHLogPositionFrom(ZKUtil.getData(zkw, znodeToProcess));
1864           sb.append(position);
1865         } catch (Exception e) {
1866         }
1867       }
1868       for (String zNodeChild : ZKUtil.listChildrenNoWatch(zkw, znodeToProcess)) {
1869         stack.add(ZKUtil.joinZNode(znodeToProcess, zNodeChild));
1870       }
1871     } while (stack.size() > 0);
1872   }
1873 
1874   private static void appendPeersZnodes(ZooKeeperWatcher zkw, String peersZnode,
1875     StringBuilder sb) throws KeeperException {
1876     int pblen = ProtobufUtil.lengthOfPBMagic();
1877     sb.append("\n").append(peersZnode).append(": ");
1878     for (String peerIdZnode : ZKUtil.listChildrenNoWatch(zkw, peersZnode)) {
1879       String znodeToProcess = ZKUtil.joinZNode(peersZnode, peerIdZnode);
1880       byte[] data = ZKUtil.getData(zkw, znodeToProcess);
1881       // parse the data of the above peer znode.
1882       try {
1883         ZooKeeperProtos.ReplicationPeer.Builder builder =
1884           ZooKeeperProtos.ReplicationPeer.newBuilder();
1885         ProtobufUtil.mergeFrom(builder, data, pblen, data.length - pblen);
1886         String clusterKey = builder.getClusterkey();
1887         sb.append("\n").append(znodeToProcess).append(": ").append(clusterKey);
1888         // add the peer-state.
1889         appendPeerState(zkw, znodeToProcess, sb);
1890       } catch (IOException ipbe) {
1891         LOG.warn("Got Exception while parsing peer: " + znodeToProcess, ipbe);
1892       }
1893     }
1894   }
1895 
1896   private static void appendPeerState(ZooKeeperWatcher zkw, String znodeToProcess,
1897       StringBuilder sb) throws KeeperException, IOException {
1898     String peerState = zkw.getConfiguration().get("zookeeper.znode.replication.peers.state",
1899       "peer-state");
1900     int pblen = ProtobufUtil.lengthOfPBMagic();
1901     for (String child : ZKUtil.listChildrenNoWatch(zkw, znodeToProcess)) {
1902       if (!child.equals(peerState)) continue;
1903       String peerStateZnode = ZKUtil.joinZNode(znodeToProcess, child);
1904       sb.append("\n").append(peerStateZnode).append(": ");
1905       byte[] peerStateData = ZKUtil.getData(zkw, peerStateZnode);
1906       ZooKeeperProtos.ReplicationState.Builder builder = 
1907         ZooKeeperProtos.ReplicationState.newBuilder();
1908       ProtobufUtil.mergeFrom(builder, peerStateData, pblen, peerStateData.length - pblen);
1909       sb.append(builder.getState().name());
1910     }
1911   }
1912 
1913   /**
1914    * Gets the statistics from the given server.
1915    *
1916    * @param server  The server to get the statistics from.
1917    * @param timeout  The socket timeout to use.
1918    * @return The array of response strings.
1919    * @throws IOException When the socket communication fails.
1920    */
1921   public static String[] getServerStats(String server, int timeout)
1922   throws IOException {
1923     String[] sp = server.split(":");
1924     if (sp == null || sp.length == 0) {
1925       return null;
1926     }
1927 
1928     String host = sp[0];
1929     int port = sp.length > 1 ? Integer.parseInt(sp[1])
1930         : HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT;
1931 
1932     Socket socket = new Socket();
1933     InetSocketAddress sockAddr = new InetSocketAddress(host, port);
1934     socket.connect(sockAddr, timeout);
1935 
1936     socket.setSoTimeout(timeout);
1937     PrintWriter out = new PrintWriter(socket.getOutputStream(), true);
1938     BufferedReader in = new BufferedReader(new InputStreamReader(
1939       socket.getInputStream()));
1940     out.println("stat");
1941     out.flush();
1942     ArrayList<String> res = new ArrayList<String>();
1943     while (true) {
1944       String line = in.readLine();
1945       if (line != null) {
1946         res.add(line);
1947       } else {
1948         break;
1949       }
1950     }
1951     socket.close();
1952     return res.toArray(new String[res.size()]);
1953   }
1954 
1955   private static void logRetrievedMsg(final ZooKeeperWatcher zkw,
1956       final String znode, final byte [] data, final boolean watcherSet) {
1957     if (!LOG.isTraceEnabled()) return;
1958     LOG.trace(zkw.prefix("Retrieved " + ((data == null)? 0: data.length) +
1959       " byte(s) of data from znode " + znode +
1960       (watcherSet? " and set watcher; ": "; data=") +
1961       (data == null? "null": data.length == 0? "empty": (
1962           znode.startsWith(zkw.assignmentZNode)?
1963             ZKAssign.toString(data): // We should not be doing this reaching into another class
1964           znode.startsWith(zkw.metaServerZNode)?
1965             getServerNameOrEmptyString(data):
1966           znode.startsWith(zkw.backupMasterAddressesZNode)?
1967             getServerNameOrEmptyString(data):
1968           StringUtils.abbreviate(Bytes.toStringBinary(data), 32)))));
1969   }
1970 
1971   private static String getServerNameOrEmptyString(final byte [] data) {
1972     try {
1973       return ServerName.parseFrom(data).toString();
1974     } catch (DeserializationException e) {
1975       return "";
1976     }
1977   }
1978 
1979   /**
1980    * Waits for HBase installation's base (parent) znode to become available.
1981    * @throws IOException on ZK errors
1982    */
1983   public static void waitForBaseZNode(Configuration conf) throws IOException {
1984     LOG.info("Waiting until the base znode is available");
1985     String parentZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
1986         HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
1987     ZooKeeper zk = new ZooKeeper(ZKConfig.getZKQuorumServersString(conf),
1988         conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
1989         HConstants.DEFAULT_ZK_SESSION_TIMEOUT), EmptyWatcher.instance);
1990 
1991     final int maxTimeMs = 10000;
1992     final int maxNumAttempts = maxTimeMs / HConstants.SOCKET_RETRY_WAIT_MS;
1993 
1994     KeeperException keeperEx = null;
1995     try {
1996       try {
1997         for (int attempt = 0; attempt < maxNumAttempts; ++attempt) {
1998           try {
1999             if (zk.exists(parentZNode, false) != null) {
2000               LOG.info("Parent znode exists: " + parentZNode);
2001               keeperEx = null;
2002               break;
2003             }
2004           } catch (KeeperException e) {
2005             keeperEx = e;
2006           }
2007           Threads.sleepWithoutInterrupt(HConstants.SOCKET_RETRY_WAIT_MS);
2008         }
2009       } finally {
2010         zk.close();
2011       }
2012     } catch (InterruptedException ex) {
2013       Thread.currentThread().interrupt();
2014     }
2015 
2016     if (keeperEx != null) {
2017       throw new IOException(keeperEx);
2018     }
2019   }
2020 
2021 
2022   public static byte[] blockUntilAvailable(
2023     final ZooKeeperWatcher zkw, final String znode, final long timeout)
2024     throws InterruptedException {
2025     if (timeout < 0) throw new IllegalArgumentException();
2026     if (zkw == null) throw new IllegalArgumentException();
2027     if (znode == null) throw new IllegalArgumentException();
2028 
2029     byte[] data = null;
2030     boolean finished = false;
2031     final long endTime = System.currentTimeMillis() + timeout;
2032     while (!finished) {
2033       try {
2034         data = ZKUtil.getData(zkw, znode);
2035       } catch(KeeperException e) {
2036         if (e instanceof KeeperException.SessionExpiredException
2037             || e instanceof KeeperException.AuthFailedException) {
2038           // non-recoverable errors so stop here
2039           throw new InterruptedException("interrupted due to " + e);
2040         }
2041         LOG.warn("Unexpected exception handling blockUntilAvailable", e);
2042       }
2043 
2044       if (data == null && (System.currentTimeMillis() +
2045         HConstants.SOCKET_RETRY_WAIT_MS < endTime)) {
2046         Thread.sleep(HConstants.SOCKET_RETRY_WAIT_MS);
2047       } else {
2048         finished = true;
2049       }
2050     }
2051 
2052     return data;
2053   }
2054 
2055 
2056   /**
2057    * Convert a {@link DeserializationException} to a more palatable {@link KeeperException}.
2058    * Used when can't let a {@link DeserializationException} out w/o changing public API.
2059    * @param e Exception to convert
2060    * @return Converted exception
2061    */
2062   public static KeeperException convert(final DeserializationException e) {
2063     KeeperException ke = new KeeperException.DataInconsistencyException();
2064     ke.initCause(e);
2065     return ke;
2066   }
2067 
2068   /**
2069    * Recursively print the current state of ZK (non-transactional)
2070    * @param root name of the root directory in zk to print
2071    * @throws KeeperException
2072    */
2073   public static void logZKTree(ZooKeeperWatcher zkw, String root) {
2074     if (!LOG.isDebugEnabled()) return;
2075     LOG.debug("Current zk system:");
2076     String prefix = "|-";
2077     LOG.debug(prefix + root);
2078     try {
2079       logZKTree(zkw, root, prefix);
2080     } catch (KeeperException e) {
2081       throw new RuntimeException(e);
2082     }
2083   }
2084 
2085   /**
2086    * Helper method to print the current state of the ZK tree.
2087    * @see #logZKTree(ZooKeeperWatcher, String)
2088    * @throws KeeperException if an unexpected exception occurs
2089    */
2090   protected static void logZKTree(ZooKeeperWatcher zkw, String root, String prefix) throws KeeperException {
2091     List<String> children = ZKUtil.listChildrenNoWatch(zkw, root);
2092     if (children == null) return;
2093     for (String child : children) {
2094       LOG.debug(prefix + child);
2095       String node = ZKUtil.joinZNode(root.equals("/") ? "" : root, child);
2096       logZKTree(zkw, node, prefix + "---");
2097     }
2098   }
2099 
2100   /**
2101    * @param position
2102    * @return Serialized protobuf of <code>position</code> with pb magic prefix prepended suitable
2103    *         for use as content of an hlog position in a replication queue.
2104    */
2105   public static byte[] positionToByteArray(final long position) {
2106     byte[] bytes = ZooKeeperProtos.ReplicationHLogPosition.newBuilder().setPosition(position)
2107         .build().toByteArray();
2108     return ProtobufUtil.prependPBMagic(bytes);
2109   }
2110 
2111   /**
2112    * @param bytes - Content of a HLog position znode.
2113    * @return long - The current HLog position.
2114    * @throws DeserializationException
2115    */
2116   public static long parseHLogPositionFrom(final byte[] bytes) throws DeserializationException {
2117     if (bytes == null) {
2118       throw new DeserializationException("Unable to parse null HLog position.");
2119     }
2120     if (ProtobufUtil.isPBMagicPrefix(bytes)) {
2121       int pblen = ProtobufUtil.lengthOfPBMagic();
2122       ZooKeeperProtos.ReplicationHLogPosition.Builder builder =
2123           ZooKeeperProtos.ReplicationHLogPosition.newBuilder();
2124       ZooKeeperProtos.ReplicationHLogPosition position;
2125       try {
2126         ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
2127         position = builder.build();
2128       } catch (IOException e) {
2129         throw new DeserializationException(e);
2130       }
2131       return position.getPosition();
2132     } else {
2133       if (bytes.length > 0) {
2134         return Bytes.toLong(bytes);
2135       }
2136       return 0;
2137     }
2138   }
2139 
2140   /**
2141    * @param regionLastFlushedSequenceId the flushed sequence id of a region which is the min of its
2142    *          store max seq ids
2143    * @param storeSequenceIds column family to sequence Id map
2144    * @return Serialized protobuf of <code>RegionSequenceIds</code> with pb magic prefix prepended
2145    *         suitable for use to filter wal edits in distributedLogReplay mode
2146    */
2147   public static byte[] regionSequenceIdsToByteArray(final Long regionLastFlushedSequenceId,
2148       final Map<byte[], Long> storeSequenceIds) {
2149     ZooKeeperProtos.RegionStoreSequenceIds.Builder regionSequenceIdsBuilder =
2150         ZooKeeperProtos.RegionStoreSequenceIds.newBuilder();
2151     ZooKeeperProtos.StoreSequenceId.Builder storeSequenceIdBuilder =
2152         ZooKeeperProtos.StoreSequenceId.newBuilder();
2153     if (storeSequenceIds != null) {
2154       for (Map.Entry<byte[], Long> e : storeSequenceIds.entrySet()){
2155         byte[] columnFamilyName = e.getKey();
2156         Long curSeqId = e.getValue();
2157         storeSequenceIdBuilder.setFamilyName(ByteStringer.wrap(columnFamilyName));
2158         storeSequenceIdBuilder.setSequenceId(curSeqId);
2159         regionSequenceIdsBuilder.addStoreSequenceId(storeSequenceIdBuilder.build());
2160         storeSequenceIdBuilder.clear();
2161       }
2162     }
2163     regionSequenceIdsBuilder.setLastFlushedSequenceId(regionLastFlushedSequenceId);
2164     byte[] result = regionSequenceIdsBuilder.build().toByteArray();
2165     return ProtobufUtil.prependPBMagic(result);
2166   }
2167 
2168   /**
2169    * @param bytes Content of serialized data of RegionStoreSequenceIds
2170    * @return a RegionStoreSequenceIds object
2171    * @throws DeserializationException
2172    */
2173   public static RegionStoreSequenceIds parseRegionStoreSequenceIds(final byte[] bytes)
2174       throws DeserializationException {
2175     if (bytes == null || !ProtobufUtil.isPBMagicPrefix(bytes)) {
2176       throw new DeserializationException("Unable to parse RegionStoreSequenceIds.");
2177     }
2178     RegionStoreSequenceIds.Builder regionSequenceIdsBuilder =
2179         ZooKeeperProtos.RegionStoreSequenceIds.newBuilder();
2180     int pblen = ProtobufUtil.lengthOfPBMagic();
2181     RegionStoreSequenceIds storeIds = null;
2182     try {
2183       ProtobufUtil.mergeFrom(regionSequenceIdsBuilder, bytes, pblen, bytes.length - pblen);
2184       storeIds = regionSequenceIdsBuilder.build();
2185     } catch (IOException e) {
2186       throw new DeserializationException(e);
2187     }
2188     return storeIds;
2189   }
2190 }