View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.zookeeper;
20  
21  import java.io.BufferedReader;
22  import java.io.IOException;
23  import java.io.InputStreamReader;
24  import java.io.PrintWriter;
25  import java.net.InetSocketAddress;
26  import java.net.Socket;
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.HashMap;
30  import java.util.LinkedList;
31  import java.util.List;
32  import java.util.Map;
33  import java.util.Properties;
34  
35  import javax.security.auth.login.AppConfigurationEntry;
36  import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
37  
38  import org.apache.hadoop.hbase.util.ByteStringer;
39  import org.apache.commons.lang.StringUtils;
40  import org.apache.commons.logging.Log;
41  import org.apache.commons.logging.LogFactory;
42  import org.apache.hadoop.hbase.classification.InterfaceAudience;
43  import org.apache.hadoop.conf.Configuration;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.ServerName;
46  import org.apache.hadoop.hbase.exceptions.DeserializationException;
47  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
48  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
49  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.RegionStoreSequenceIds;
50  import org.apache.hadoop.hbase.util.Bytes;
51  import org.apache.hadoop.hbase.util.Threads;
52  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.CreateAndFailSilent;
53  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.DeleteNodeFailSilent;
54  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.SetData;
55  import org.apache.hadoop.security.SecurityUtil;
56  import org.apache.hadoop.security.authentication.util.KerberosUtil;
57  import org.apache.zookeeper.AsyncCallback;
58  import org.apache.zookeeper.CreateMode;
59  import org.apache.zookeeper.KeeperException;
60  import org.apache.zookeeper.KeeperException.NoNodeException;
61  import org.apache.zookeeper.Op;
62  import org.apache.zookeeper.Watcher;
63  import org.apache.zookeeper.ZooDefs.Ids;
64  import org.apache.zookeeper.ZooDefs.Perms;
65  import org.apache.zookeeper.ZooKeeper;
66  import org.apache.zookeeper.client.ZooKeeperSaslClient;
67  import org.apache.zookeeper.data.ACL;
68  import org.apache.zookeeper.data.Id;
69  import org.apache.zookeeper.data.Stat;
70  import org.apache.zookeeper.proto.CreateRequest;
71  import org.apache.zookeeper.proto.DeleteRequest;
72  import org.apache.zookeeper.proto.SetDataRequest;
73  import org.apache.zookeeper.server.ZooKeeperSaslServer;
74  
75  import com.google.protobuf.InvalidProtocolBufferException;
76  
77  /**
78   * Internal HBase utility class for ZooKeeper.
79   *
80   * <p>Contains only static methods and constants.
81   *
82   * <p>Methods all throw {@link KeeperException} if there is an unexpected
83   * zookeeper exception, so callers of these methods must handle appropriately.
84   * If ZK is required for the operation, the server will need to be aborted.
85   */
86  @InterfaceAudience.Private
87  public class ZKUtil {
88    private static final Log LOG = LogFactory.getLog(ZKUtil.class);
89  
90    // TODO: Replace this with ZooKeeper constant when ZOOKEEPER-277 is resolved.
91    public static final char ZNODE_PATH_SEPARATOR = '/';
92    private static int zkDumpConnectionTimeOut;
93  
94    /**
95     * Creates a new connection to ZooKeeper, pulling settings and ensemble config
96     * from the specified configuration object using methods from {@link ZKConfig}.
97     *
98     * Sets the connection status monitoring watcher to the specified watcher.
99     *
100    * @param conf configuration to pull ensemble and other settings from
101    * @param watcher watcher to monitor connection changes
102    * @return connection to zookeeper
103    * @throws IOException if unable to connect to zk or config problem
104    */
105   public static RecoverableZooKeeper connect(Configuration conf, Watcher watcher)
106   throws IOException {
107     Properties properties = ZKConfig.makeZKProps(conf);
108     String ensemble = ZKConfig.getZKQuorumServersString(properties);
109     return connect(conf, ensemble, watcher);
110   }
111 
112   public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
113       Watcher watcher)
114   throws IOException {
115     return connect(conf, ensemble, watcher, null);
116   }
117 
118   public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
119       Watcher watcher, final String identifier)
120   throws IOException {
121     if(ensemble == null) {
122       throw new IOException("Unable to determine ZooKeeper ensemble");
123     }
124     int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
125         HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
126     if (LOG.isTraceEnabled()) {
127       LOG.trace(identifier + " opening connection to ZooKeeper ensemble=" + ensemble);
128     }
129     int retry = conf.getInt("zookeeper.recovery.retry", 3);
130     int retryIntervalMillis =
131       conf.getInt("zookeeper.recovery.retry.intervalmill", 1000);
132     zkDumpConnectionTimeOut = conf.getInt("zookeeper.dump.connection.timeout",
133         1000);
134     return new RecoverableZooKeeper(ensemble, timeout, watcher,
135         retry, retryIntervalMillis, identifier);
136   }
137 
138   /**
139    * Log in the current zookeeper server process using the given configuration
140    * keys for the credential file and login principal.
141    *
142    * <p><strong>This is only applicable when running on secure hbase</strong>
143    * On regular HBase (without security features), this will safely be ignored.
144    * </p>
145    *
146    * @param conf The configuration data to use
147    * @param keytabFileKey Property key used to configure the path to the credential file
148    * @param userNameKey Property key used to configure the login principal
149    * @param hostname Current hostname to use in any credentials
150    * @throws IOException underlying exception from SecurityUtil.login() call
151    */
152   public static void loginServer(Configuration conf, String keytabFileKey,
153       String userNameKey, String hostname) throws IOException {
154     login(conf, keytabFileKey, userNameKey, hostname,
155           ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY,
156           JaasConfiguration.SERVER_KEYTAB_KERBEROS_CONFIG_NAME);
157   }
158 
159   /**
160    * Log in the current zookeeper client using the given configuration
161    * keys for the credential file and login principal.
162    *
163    * <p><strong>This is only applicable when running on secure hbase</strong>
164    * On regular HBase (without security features), this will safely be ignored.
165    * </p>
166    *
167    * @param conf The configuration data to use
168    * @param keytabFileKey Property key used to configure the path to the credential file
169    * @param userNameKey Property key used to configure the login principal
170    * @param hostname Current hostname to use in any credentials
171    * @throws IOException underlying exception from SecurityUtil.login() call
172    */
173   public static void loginClient(Configuration conf, String keytabFileKey,
174       String userNameKey, String hostname) throws IOException {
175     login(conf, keytabFileKey, userNameKey, hostname,
176           ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
177           JaasConfiguration.CLIENT_KEYTAB_KERBEROS_CONFIG_NAME);
178   }
179 
180   /**
181    * Log in the current process using the given configuration keys for the
182    * credential file and login principal.
183    *
184    * <p><strong>This is only applicable when running on secure hbase</strong>
185    * On regular HBase (without security features), this will safely be ignored.
186    * </p>
187    *
188    * @param conf The configuration data to use
189    * @param keytabFileKey Property key used to configure the path to the credential file
190    * @param userNameKey Property key used to configure the login principal
191    * @param hostname Current hostname to use in any credentials
192    * @param loginContextProperty property name to expose the entry name
193    * @param loginContextName jaas entry name
194    * @throws IOException underlying exception from SecurityUtil.login() call
195    */
196   private static void login(Configuration conf, String keytabFileKey,
197       String userNameKey, String hostname,
198       String loginContextProperty, String loginContextName)
199       throws IOException {
200     if (!isSecureZooKeeper(conf))
201       return;
202 
203     // User has specified a jaas.conf, keep this one as the good one.
204     // HBASE_OPTS="-Djava.security.auth.login.config=jaas.conf"
205     if (System.getProperty("java.security.auth.login.config") != null)
206       return;
207 
208     // No keytab specified, no auth
209     String keytabFilename = conf.get(keytabFileKey);
210     if (keytabFilename == null) {
211       LOG.warn("no keytab specified for: " + keytabFileKey);
212       return;
213     }
214 
215     String principalConfig = conf.get(userNameKey, System.getProperty("user.name"));
216     String principalName = SecurityUtil.getServerPrincipal(principalConfig, hostname);
217 
218     // Initialize the "jaas.conf" for keyTab/principal,
219     // If keyTab is not specified use the Ticket Cache.
220     // and set the zookeeper login context name.
221     JaasConfiguration jaasConf = new JaasConfiguration(loginContextName,
222         principalName, keytabFilename);
223     javax.security.auth.login.Configuration.setConfiguration(jaasConf);
224     System.setProperty(loginContextProperty, loginContextName);
225   }
226 
227   /**
228    * A JAAS configuration that defines the login modules that we want to use for login.
229    */
230   private static class JaasConfiguration extends javax.security.auth.login.Configuration {
231     private static final String SERVER_KEYTAB_KERBEROS_CONFIG_NAME =
232       "zookeeper-server-keytab-kerberos";
233     private static final String CLIENT_KEYTAB_KERBEROS_CONFIG_NAME =
234       "zookeeper-client-keytab-kerberos";
235 
236     private static final Map<String, String> BASIC_JAAS_OPTIONS =
237       new HashMap<String,String>();
238     static {
239       String jaasEnvVar = System.getenv("HBASE_JAAS_DEBUG");
240       if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) {
241         BASIC_JAAS_OPTIONS.put("debug", "true");
242       }
243     }
244 
245     private static final Map<String,String> KEYTAB_KERBEROS_OPTIONS =
246       new HashMap<String,String>();
247     static {
248       KEYTAB_KERBEROS_OPTIONS.put("doNotPrompt", "true");
249       KEYTAB_KERBEROS_OPTIONS.put("storeKey", "true");
250       KEYTAB_KERBEROS_OPTIONS.put("refreshKrb5Config", "true");
251       KEYTAB_KERBEROS_OPTIONS.putAll(BASIC_JAAS_OPTIONS);
252     }
253 
254     private static final AppConfigurationEntry KEYTAB_KERBEROS_LOGIN =
255       new AppConfigurationEntry(KerberosUtil.getKrb5LoginModuleName(),
256                                 LoginModuleControlFlag.REQUIRED,
257                                 KEYTAB_KERBEROS_OPTIONS);
258 
259     private static final AppConfigurationEntry[] KEYTAB_KERBEROS_CONF =
260       new AppConfigurationEntry[]{KEYTAB_KERBEROS_LOGIN};
261 
262     private javax.security.auth.login.Configuration baseConfig;
263     private final String loginContextName;
264     private final boolean useTicketCache;
265     private final String keytabFile;
266     private final String principal;
267 
268     public JaasConfiguration(String loginContextName, String principal) {
269       this(loginContextName, principal, null, true);
270     }
271 
272     public JaasConfiguration(String loginContextName, String principal, String keytabFile) {
273       this(loginContextName, principal, keytabFile, keytabFile == null || keytabFile.length() == 0);
274     }
275 
276     private JaasConfiguration(String loginContextName, String principal,
277                              String keytabFile, boolean useTicketCache) {
278       try {
279         this.baseConfig = javax.security.auth.login.Configuration.getConfiguration();
280       } catch (SecurityException e) {
281         this.baseConfig = null;
282       }
283       this.loginContextName = loginContextName;
284       this.useTicketCache = useTicketCache;
285       this.keytabFile = keytabFile;
286       this.principal = principal;
287       LOG.info("JaasConfiguration loginContextName=" + loginContextName +
288                " principal=" + principal + " useTicketCache=" + useTicketCache +
289                " keytabFile=" + keytabFile);
290     }
291 
292     @Override
293     public AppConfigurationEntry[] getAppConfigurationEntry(String appName) {
294       if (loginContextName.equals(appName)) {
295         if (!useTicketCache) {
296           KEYTAB_KERBEROS_OPTIONS.put("keyTab", keytabFile);
297           KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true");
298         }
299         KEYTAB_KERBEROS_OPTIONS.put("principal", principal);
300         KEYTAB_KERBEROS_OPTIONS.put("useTicketCache", useTicketCache ? "true" : "false");
301         return KEYTAB_KERBEROS_CONF;
302       }
303       if (baseConfig != null) return baseConfig.getAppConfigurationEntry(appName);
304       return(null);
305     }
306   }
307 
308   //
309   // Helper methods
310   //
311 
312   /**
313    * Join the prefix znode name with the suffix znode name to generate a proper
314    * full znode name.
315    *
316    * Assumes prefix does not end with slash and suffix does not begin with it.
317    *
318    * @param prefix beginning of znode name
319    * @param suffix ending of znode name
320    * @return result of properly joining prefix with suffix
321    */
322   public static String joinZNode(String prefix, String suffix) {
323     return prefix + ZNODE_PATH_SEPARATOR + suffix;
324   }
325 
326   /**
327    * Returns the full path of the immediate parent of the specified node.
328    * @param node path to get parent of
329    * @return parent of path, null if passed the root node or an invalid node
330    */
331   public static String getParent(String node) {
332     int idx = node.lastIndexOf(ZNODE_PATH_SEPARATOR);
333     return idx <= 0 ? null : node.substring(0, idx);
334   }
335 
336   /**
337    * Get the name of the current node from the specified fully-qualified path.
338    * @param path fully-qualified path
339    * @return name of the current node
340    */
341   public static String getNodeName(String path) {
342     return path.substring(path.lastIndexOf("/")+1);
343   }
344 
345   /**
346    * Get the key to the ZK ensemble for this configuration without
347    * adding a name at the end
348    * @param conf Configuration to use to build the key
349    * @return ensemble key without a name
350    */
351   public static String getZooKeeperClusterKey(Configuration conf) {
352     return getZooKeeperClusterKey(conf, null);
353   }
354 
355   /**
356    * Get the key to the ZK ensemble for this configuration and append
357    * a name at the end
358    * @param conf Configuration to use to build the key
359    * @param name Name that should be appended at the end if not empty or null
360    * @return ensemble key with a name (if any)
361    */
362   public static String getZooKeeperClusterKey(Configuration conf, String name) {
363     String ensemble = conf.get(HConstants.ZOOKEEPER_QUORUM.replaceAll(
364         "[\\t\\n\\x0B\\f\\r]", ""));
365     StringBuilder builder = new StringBuilder(ensemble);
366     builder.append(":");
367     builder.append(conf.get(HConstants.ZOOKEEPER_CLIENT_PORT));
368     builder.append(":");
369     builder.append(conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT));
370     if (name != null && !name.isEmpty()) {
371       builder.append(",");
372       builder.append(name);
373     }
374     return builder.toString();
375   }
376 
377   /**
378    * Apply the settings in the given key to the given configuration, this is
379    * used to communicate with distant clusters
380    * @param conf configuration object to configure
381    * @param key string that contains the 3 required configuratins
382    * @throws IOException
383    */
384   public static void applyClusterKeyToConf(Configuration conf, String key)
385       throws IOException{
386     String[] parts = transformClusterKey(key);
387     conf.set(HConstants.ZOOKEEPER_QUORUM, parts[0]);
388     conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, parts[1]);
389     conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parts[2]);
390   }
391 
392   /**
393    * Separate the given key into the three configurations it should contain:
394    * hbase.zookeeper.quorum, hbase.zookeeper.client.port
395    * and zookeeper.znode.parent
396    * @param key
397    * @return the three configuration in the described order
398    * @throws IOException
399    */
400   public static String[] transformClusterKey(String key) throws IOException {
401     String[] parts = key.split(":");
402     if (parts.length != 3) {
403       throw new IOException("Cluster key passed " + key + " is invalid, the format should be:" +
404           HConstants.ZOOKEEPER_QUORUM + ":hbase.zookeeper.client.port:"
405           + HConstants.ZOOKEEPER_ZNODE_PARENT);
406     }
407     return parts;
408   }
409 
410   //
411   // Existence checks and watches
412   //
413 
414   /**
415    * Watch the specified znode for delete/create/change events.  The watcher is
416    * set whether or not the node exists.  If the node already exists, the method
417    * returns true.  If the node does not exist, the method returns false.
418    *
419    * @param zkw zk reference
420    * @param znode path of node to watch
421    * @return true if znode exists, false if does not exist or error
422    * @throws KeeperException if unexpected zookeeper exception
423    */
424   public static boolean watchAndCheckExists(ZooKeeperWatcher zkw, String znode)
425   throws KeeperException {
426     try {
427       Stat s = zkw.getRecoverableZooKeeper().exists(znode, zkw);
428       boolean exists = s != null ? true : false;
429       if (exists) {
430         LOG.debug(zkw.prefix("Set watcher on existing znode=" + znode));
431       } else {
432         LOG.debug(zkw.prefix("Set watcher on znode that does not yet exist, " + znode));
433       }
434       return exists;
435     } catch (KeeperException e) {
436       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
437       zkw.keeperException(e);
438       return false;
439     } catch (InterruptedException e) {
440       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
441       zkw.interruptedException(e);
442       return false;
443     }
444   }
445 
446   /**
447    * Watch the specified znode, but only if exists. Useful when watching
448    * for deletions. Uses .getData() (and handles NoNodeException) instead
449    * of .exists() to accomplish this, as .getData() will only set a watch if
450    * the znode exists.
451    * @param zkw zk reference
452    * @param znode path of node to watch
453    * @return true if the watch is set, false if node does not exists
454    * @throws KeeperException if unexpected zookeeper exception
455    */
456   public static boolean setWatchIfNodeExists(ZooKeeperWatcher zkw, String znode)
457       throws KeeperException {
458     try {
459       zkw.getRecoverableZooKeeper().getData(znode, true, null);
460       return true;
461     } catch (NoNodeException e) {
462       return false;
463     } catch (InterruptedException e) {
464       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
465       zkw.interruptedException(e);
466       return false;
467     }
468   }
469 
470   /**
471    * Check if the specified node exists.  Sets no watches.
472    *
473    * @param zkw zk reference
474    * @param znode path of node to watch
475    * @return version of the node if it exists, -1 if does not exist
476    * @throws KeeperException if unexpected zookeeper exception
477    */
478   public static int checkExists(ZooKeeperWatcher zkw, String znode)
479   throws KeeperException {
480     try {
481       Stat s = zkw.getRecoverableZooKeeper().exists(znode, null);
482       return s != null ? s.getVersion() : -1;
483     } catch (KeeperException e) {
484       LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
485       zkw.keeperException(e);
486       return -1;
487     } catch (InterruptedException e) {
488       LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
489       zkw.interruptedException(e);
490       return -1;
491     }
492   }
493 
494   //
495   // Znode listings
496   //
497 
498   /**
499    * Lists the children znodes of the specified znode.  Also sets a watch on
500    * the specified znode which will capture a NodeDeleted event on the specified
501    * znode as well as NodeChildrenChanged if any children of the specified znode
502    * are created or deleted.
503    *
504    * Returns null if the specified node does not exist.  Otherwise returns a
505    * list of children of the specified node.  If the node exists but it has no
506    * children, an empty list will be returned.
507    *
508    * @param zkw zk reference
509    * @param znode path of node to list and watch children of
510    * @return list of children of the specified node, an empty list if the node
511    *          exists but has no children, and null if the node does not exist
512    * @throws KeeperException if unexpected zookeeper exception
513    */
514   public static List<String> listChildrenAndWatchForNewChildren(
515       ZooKeeperWatcher zkw, String znode)
516   throws KeeperException {
517     try {
518       List<String> children = zkw.getRecoverableZooKeeper().getChildren(znode, zkw);
519       return children;
520     } catch(KeeperException.NoNodeException ke) {
521       LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
522           "because node does not exist (not an error)"));
523       return null;
524     } catch (KeeperException e) {
525       LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
526       zkw.keeperException(e);
527       return null;
528     } catch (InterruptedException e) {
529       LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
530       zkw.interruptedException(e);
531       return null;
532     }
533   }
534 
535   /**
536    * List all the children of the specified znode, setting a watch for children
537    * changes and also setting a watch on every individual child in order to get
538    * the NodeCreated and NodeDeleted events.
539    * @param zkw zookeeper reference
540    * @param znode node to get children of and watch
541    * @return list of znode names, null if the node doesn't exist
542    * @throws KeeperException
543    */
544   public static List<String> listChildrenAndWatchThem(ZooKeeperWatcher zkw,
545       String znode) throws KeeperException {
546     List<String> children = listChildrenAndWatchForNewChildren(zkw, znode);
547     if (children == null) {
548       return null;
549     }
550     for (String child : children) {
551       watchAndCheckExists(zkw, joinZNode(znode, child));
552     }
553     return children;
554   }
555 
556   /**
557    * Lists the children of the specified znode without setting any watches.
558    *
559    * Sets no watches at all, this method is best effort.
560    *
561    * Returns an empty list if the node has no children.  Returns null if the
562    * parent node itself does not exist.
563    *
564    * @param zkw zookeeper reference
565    * @param znode node to get children
566    * @return list of data of children of specified znode, empty if no children,
567    *         null if parent does not exist
568    * @throws KeeperException if unexpected zookeeper exception
569    */
570   public static List<String> listChildrenNoWatch(ZooKeeperWatcher zkw, String znode)
571   throws KeeperException {
572     List<String> children = null;
573     try {
574       // List the children without watching
575       children = zkw.getRecoverableZooKeeper().getChildren(znode, null);
576     } catch(KeeperException.NoNodeException nne) {
577       return null;
578     } catch(InterruptedException ie) {
579       zkw.interruptedException(ie);
580     }
581     return children;
582   }
583 
584   /**
585    * Simple class to hold a node path and node data.
586    * @deprecated Unused
587    */
588   @Deprecated
589   public static class NodeAndData {
590     private String node;
591     private byte [] data;
592     public NodeAndData(String node, byte [] data) {
593       this.node = node;
594       this.data = data;
595     }
596     public String getNode() {
597       return node;
598     }
599     public byte [] getData() {
600       return data;
601     }
602     @Override
603     public String toString() {
604       return node;
605     }
606     public boolean isEmpty() {
607       return (data.length == 0);
608     }
609   }
610 
611   /**
612    * Checks if the specified znode has any children.  Sets no watches.
613    *
614    * Returns true if the node exists and has children.  Returns false if the
615    * node does not exist or if the node does not have any children.
616    *
617    * Used during master initialization to determine if the master is a
618    * failed-over-to master or the first master during initial cluster startup.
619    * If the directory for regionserver ephemeral nodes is empty then this is
620    * a cluster startup, if not then it is not cluster startup.
621    *
622    * @param zkw zk reference
623    * @param znode path of node to check for children of
624    * @return true if node has children, false if not or node does not exist
625    * @throws KeeperException if unexpected zookeeper exception
626    */
627   public static boolean nodeHasChildren(ZooKeeperWatcher zkw, String znode)
628   throws KeeperException {
629     try {
630       return !zkw.getRecoverableZooKeeper().getChildren(znode, null).isEmpty();
631     } catch(KeeperException.NoNodeException ke) {
632       LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
633       "because node does not exist (not an error)"));
634       return false;
635     } catch (KeeperException e) {
636       LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
637       zkw.keeperException(e);
638       return false;
639     } catch (InterruptedException e) {
640       LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
641       zkw.interruptedException(e);
642       return false;
643     }
644   }
645 
646   /**
647    * Get the number of children of the specified node.
648    *
649    * If the node does not exist or has no children, returns 0.
650    *
651    * Sets no watches at all.
652    *
653    * @param zkw zk reference
654    * @param znode path of node to count children of
655    * @return number of children of specified node, 0 if none or parent does not
656    *         exist
657    * @throws KeeperException if unexpected zookeeper exception
658    */
659   public static int getNumberOfChildren(ZooKeeperWatcher zkw, String znode)
660   throws KeeperException {
661     try {
662       Stat stat = zkw.getRecoverableZooKeeper().exists(znode, null);
663       return stat == null ? 0 : stat.getNumChildren();
664     } catch(KeeperException e) {
665       LOG.warn(zkw.prefix("Unable to get children of node " + znode));
666       zkw.keeperException(e);
667     } catch(InterruptedException e) {
668       zkw.interruptedException(e);
669     }
670     return 0;
671   }
672 
673   //
674   // Data retrieval
675   //
676 
677   /**
678    * Get znode data. Does not set a watcher.
679    * @return ZNode data, null if the node does not exist or if there is an
680    *  error.
681    */
682   public static byte [] getData(ZooKeeperWatcher zkw, String znode)
683       throws KeeperException, InterruptedException {
684     try {
685       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, null);
686       logRetrievedMsg(zkw, znode, data, false);
687       return data;
688     } catch (KeeperException.NoNodeException e) {
689       LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
690           "because node does not exist (not an error)"));
691       return null;
692     } catch (KeeperException e) {
693       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
694       zkw.keeperException(e);
695       return null;
696     }
697   }
698 
699   /**
700    * Get the data at the specified znode and set a watch.
701    *
702    * Returns the data and sets a watch if the node exists.  Returns null and no
703    * watch is set if the node does not exist or there is an exception.
704    *
705    * @param zkw zk reference
706    * @param znode path of node
707    * @return data of the specified znode, or null
708    * @throws KeeperException if unexpected zookeeper exception
709    */
710   public static byte [] getDataAndWatch(ZooKeeperWatcher zkw, String znode)
711   throws KeeperException {
712     return getDataInternal(zkw, znode, null, true);
713   }
714 
715   /**
716    * Get the data at the specified znode and set a watch.
717    *
718    * Returns the data and sets a watch if the node exists.  Returns null and no
719    * watch is set if the node does not exist or there is an exception.
720    *
721    * @param zkw zk reference
722    * @param znode path of node
723    * @param stat object to populate the version of the znode
724    * @return data of the specified znode, or null
725    * @throws KeeperException if unexpected zookeeper exception
726    */
727   public static byte[] getDataAndWatch(ZooKeeperWatcher zkw, String znode,
728       Stat stat) throws KeeperException {
729     return getDataInternal(zkw, znode, stat, true);
730   }
731 
732   private static byte[] getDataInternal(ZooKeeperWatcher zkw, String znode, Stat stat,
733       boolean watcherSet)
734       throws KeeperException {
735     try {
736       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, zkw, stat);
737       logRetrievedMsg(zkw, znode, data, watcherSet);
738       return data;
739     } catch (KeeperException.NoNodeException e) {
740       // This log can get pretty annoying when we cycle on 100ms waits.
741       // Enable trace if you really want to see it.
742       LOG.trace(zkw.prefix("Unable to get data of znode " + znode + " " +
743         "because node does not exist (not an error)"));
744       return null;
745     } catch (KeeperException e) {
746       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
747       zkw.keeperException(e);
748       return null;
749     } catch (InterruptedException e) {
750       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
751       zkw.interruptedException(e);
752       return null;
753     }
754   }
755 
756   /**
757    * Get the data at the specified znode without setting a watch.
758    *
759    * Returns the data if the node exists.  Returns null if the node does not
760    * exist.
761    *
762    * Sets the stats of the node in the passed Stat object.  Pass a null stat if
763    * not interested.
764    *
765    * @param zkw zk reference
766    * @param znode path of node
767    * @param stat node status to get if node exists
768    * @return data of the specified znode, or null if node does not exist
769    * @throws KeeperException if unexpected zookeeper exception
770    */
771   public static byte [] getDataNoWatch(ZooKeeperWatcher zkw, String znode,
772       Stat stat)
773   throws KeeperException {
774     try {
775       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, stat);
776       logRetrievedMsg(zkw, znode, data, false);
777       return data;
778     } catch (KeeperException.NoNodeException e) {
779       LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
780           "because node does not exist (not necessarily an error)"));
781       return null;
782     } catch (KeeperException e) {
783       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
784       zkw.keeperException(e);
785       return null;
786     } catch (InterruptedException e) {
787       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
788       zkw.interruptedException(e);
789       return null;
790     }
791   }
792 
793   /**
794    * Returns the date of child znodes of the specified znode.  Also sets a watch on
795    * the specified znode which will capture a NodeDeleted event on the specified
796    * znode as well as NodeChildrenChanged if any children of the specified znode
797    * are created or deleted.
798    *
799    * Returns null if the specified node does not exist.  Otherwise returns a
800    * list of children of the specified node.  If the node exists but it has no
801    * children, an empty list will be returned.
802    *
803    * @param zkw zk reference
804    * @param baseNode path of node to list and watch children of
805    * @return list of data of children of the specified node, an empty list if the node
806    *          exists but has no children, and null if the node does not exist
807    * @throws KeeperException if unexpected zookeeper exception
808    * @deprecated Unused
809    */
810   public static List<NodeAndData> getChildDataAndWatchForNewChildren(
811       ZooKeeperWatcher zkw, String baseNode) throws KeeperException {
812     List<String> nodes =
813       ZKUtil.listChildrenAndWatchForNewChildren(zkw, baseNode);
814     if (nodes != null) {
815       List<NodeAndData> newNodes = new ArrayList<NodeAndData>();
816       for (String node : nodes) {
817         String nodePath = ZKUtil.joinZNode(baseNode, node);
818         byte[] data = ZKUtil.getDataAndWatch(zkw, nodePath);
819         newNodes.add(new NodeAndData(nodePath, data));
820       }
821       return newNodes;
822     }
823     return null;
824   }
825 
826   /**
827    * Update the data of an existing node with the expected version to have the
828    * specified data.
829    *
830    * Throws an exception if there is a version mismatch or some other problem.
831    *
832    * Sets no watches under any conditions.
833    *
834    * @param zkw zk reference
835    * @param znode
836    * @param data
837    * @param expectedVersion
838    * @throws KeeperException if unexpected zookeeper exception
839    * @throws KeeperException.BadVersionException if version mismatch
840    * @deprecated Unused
841    */
842   public static void updateExistingNodeData(ZooKeeperWatcher zkw, String znode,
843       byte [] data, int expectedVersion)
844   throws KeeperException {
845     try {
846       zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion);
847     } catch(InterruptedException ie) {
848       zkw.interruptedException(ie);
849     }
850   }
851 
852   //
853   // Data setting
854   //
855 
856   /**
857    * Sets the data of the existing znode to be the specified data.  Ensures that
858    * the current data has the specified expected version.
859    *
860    * <p>If the node does not exist, a {@link NoNodeException} will be thrown.
861    *
862    * <p>If their is a version mismatch, method returns null.
863    *
864    * <p>No watches are set but setting data will trigger other watchers of this
865    * node.
866    *
867    * <p>If there is another problem, a KeeperException will be thrown.
868    *
869    * @param zkw zk reference
870    * @param znode path of node
871    * @param data data to set for node
872    * @param expectedVersion version expected when setting data
873    * @return true if data set, false if version mismatch
874    * @throws KeeperException if unexpected zookeeper exception
875    */
876   public static boolean setData(ZooKeeperWatcher zkw, String znode,
877       byte [] data, int expectedVersion)
878   throws KeeperException, KeeperException.NoNodeException {
879     try {
880       return zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion) != null;
881     } catch (InterruptedException e) {
882       zkw.interruptedException(e);
883       return false;
884     }
885   }
886 
887   /**
888    * Set data into node creating node if it doesn't yet exist.
889    * Does not set watch.
890    *
891    * @param zkw zk reference
892    * @param znode path of node
893    * @param data data to set for node
894    * @throws KeeperException
895    */
896   public static void createSetData(final ZooKeeperWatcher zkw, final String znode,
897       final byte [] data)
898   throws KeeperException {
899     if (checkExists(zkw, znode) == -1) {
900       ZKUtil.createWithParents(zkw, znode, data);
901     } else {
902       ZKUtil.setData(zkw, znode, data);
903     }
904   }
905 
906   /**
907    * Sets the data of the existing znode to be the specified data.  The node
908    * must exist but no checks are done on the existing data or version.
909    *
910    * <p>If the node does not exist, a {@link NoNodeException} will be thrown.
911    *
912    * <p>No watches are set but setting data will trigger other watchers of this
913    * node.
914    *
915    * <p>If there is another problem, a KeeperException will be thrown.
916    *
917    * @param zkw zk reference
918    * @param znode path of node
919    * @param data data to set for node
920    * @throws KeeperException if unexpected zookeeper exception
921    */
922   public static void setData(ZooKeeperWatcher zkw, String znode, byte [] data)
923   throws KeeperException, KeeperException.NoNodeException {
924     setData(zkw, (SetData)ZKUtilOp.setData(znode, data));
925   }
926 
927   private static void setData(ZooKeeperWatcher zkw, SetData setData)
928   throws KeeperException, KeeperException.NoNodeException {
929     SetDataRequest sd = (SetDataRequest)toZooKeeperOp(zkw, setData).toRequestRecord();
930     setData(zkw, sd.getPath(), sd.getData(), sd.getVersion());
931   }
932 
933   /**
934    * Returns whether or not secure authentication is enabled
935    * (whether <code>hbase.security.authentication</code> is set to
936    * <code>kerberos</code>.
937    */
938   public static boolean isSecureZooKeeper(Configuration conf) {
939     // Detection for embedded HBase client with jaas configuration
940     // defined for third party programs.
941     try {
942       javax.security.auth.login.Configuration testConfig = javax.security.auth.login.Configuration.getConfiguration();
943       if(testConfig.getAppConfigurationEntry("Client") == null) {
944         return false;
945       }
946     } catch(Exception e) {
947       // No Jaas configuration defined.
948       return false;
949     }
950 
951     // Master & RSs uses hbase.zookeeper.client.*
952     return("kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication")) &&
953          conf.get("hbase.zookeeper.client.keytab.file") != null);
954   }
955 
956   private static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node) {
957     if (!node.startsWith(zkw.baseZNode)) {
958       return Ids.OPEN_ACL_UNSAFE;
959     }
960     if (isSecureZooKeeper(zkw.getConfiguration())) {
961       String superUser = zkw.getConfiguration().get("hbase.superuser");
962       ArrayList<ACL> acls = new ArrayList<ACL>();
963       // add permission to hbase supper user
964       if (superUser != null) {
965         acls.add(new ACL(Perms.ALL, new Id("auth", superUser)));
966       }
967       // Certain znodes are accessed directly by the client,
968       // so they must be readable by non-authenticated clients
969       if ((node.equals(zkw.baseZNode) == true) ||
970           (node.equals(zkw.metaServerZNode) == true) ||
971           (node.equals(zkw.getMasterAddressZNode()) == true) ||
972           (node.equals(zkw.clusterIdZNode) == true) ||
973           (node.equals(zkw.rsZNode) == true) ||
974           (node.equals(zkw.backupMasterAddressesZNode) == true) ||
975           (node.startsWith(zkw.assignmentZNode) == true) ||
976           (node.startsWith(zkw.tableZNode) == true)) {
977         acls.addAll(Ids.CREATOR_ALL_ACL);
978         acls.addAll(Ids.READ_ACL_UNSAFE);
979       } else {
980         acls.addAll(Ids.CREATOR_ALL_ACL);
981       }
982       return acls;
983     } else {
984       return Ids.OPEN_ACL_UNSAFE;
985     }
986   }
987 
988   //
989   // Node creation
990   //
991 
992   /**
993    *
994    * Set the specified znode to be an ephemeral node carrying the specified
995    * data.
996    *
997    * If the node is created successfully, a watcher is also set on the node.
998    *
999    * If the node is not created successfully because it already exists, this
1000    * method will also set a watcher on the node.
1001    *
1002    * If there is another problem, a KeeperException will be thrown.
1003    *
1004    * @param zkw zk reference
1005    * @param znode path of node
1006    * @param data data of node
1007    * @return true if node created, false if not, watch set in both cases
1008    * @throws KeeperException if unexpected zookeeper exception
1009    */
1010   public static boolean createEphemeralNodeAndWatch(ZooKeeperWatcher zkw,
1011       String znode, byte [] data)
1012   throws KeeperException {
1013     boolean ret = true;
1014     try {
1015       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1016           CreateMode.EPHEMERAL);
1017     } catch (KeeperException.NodeExistsException nee) {
1018       ret = false;
1019     } catch (InterruptedException e) {
1020       LOG.info("Interrupted", e);
1021       Thread.currentThread().interrupt();
1022     }
1023     if(!watchAndCheckExists(zkw, znode)) {
1024       // It did exist but now it doesn't, try again
1025       return createEphemeralNodeAndWatch(zkw, znode, data);
1026     }
1027     return ret;
1028   }
1029 
1030   /**
1031    * Creates the specified znode to be a persistent node carrying the specified
1032    * data.
1033    *
1034    * Returns true if the node was successfully created, false if the node
1035    * already existed.
1036    *
1037    * If the node is created successfully, a watcher is also set on the node.
1038    *
1039    * If the node is not created successfully because it already exists, this
1040    * method will also set a watcher on the node but return false.
1041    *
1042    * If there is another problem, a KeeperException will be thrown.
1043    *
1044    * @param zkw zk reference
1045    * @param znode path of node
1046    * @param data data of node
1047    * @return true if node created, false if not, watch set in both cases
1048    * @throws KeeperException if unexpected zookeeper exception
1049    */
1050   public static boolean createNodeIfNotExistsAndWatch(
1051       ZooKeeperWatcher zkw, String znode, byte [] data)
1052   throws KeeperException {
1053     boolean ret = true;
1054     try {
1055       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1056           CreateMode.PERSISTENT);
1057     } catch (KeeperException.NodeExistsException nee) {
1058       ret = false;
1059     } catch (InterruptedException e) {
1060       zkw.interruptedException(e);
1061       return false;
1062     }
1063     try {
1064       zkw.getRecoverableZooKeeper().exists(znode, zkw);
1065     } catch (InterruptedException e) {
1066       zkw.interruptedException(e);
1067       return false;
1068     }
1069     return ret;
1070   }
1071 
1072   /**
1073    * Creates the specified znode with the specified data but does not watch it.
1074    *
1075    * Returns the znode of the newly created node
1076    *
1077    * If there is another problem, a KeeperException will be thrown.
1078    *
1079    * @param zkw zk reference
1080    * @param znode path of node
1081    * @param data data of node
1082    * @param createMode specifying whether the node to be created is ephemeral and/or sequential
1083    * @return true name of the newly created znode or null
1084    * @throws KeeperException if unexpected zookeeper exception
1085    */
1086   public static String createNodeIfNotExistsNoWatch(ZooKeeperWatcher zkw, String znode,
1087       byte[] data, CreateMode createMode) throws KeeperException {
1088 
1089     String createdZNode = null;
1090     try {
1091       createdZNode = zkw.getRecoverableZooKeeper().create(znode, data,
1092           createACL(zkw, znode), createMode);
1093     } catch (KeeperException.NodeExistsException nee) {
1094       return znode;
1095     } catch (InterruptedException e) {
1096       zkw.interruptedException(e);
1097       return null;
1098     }
1099     return createdZNode;
1100   }
1101 
1102   /**
1103    * Creates the specified node with the specified data and watches it.
1104    *
1105    * <p>Throws an exception if the node already exists.
1106    *
1107    * <p>The node created is persistent and open access.
1108    *
1109    * <p>Returns the version number of the created node if successful.
1110    *
1111    * @param zkw zk reference
1112    * @param znode path of node to create
1113    * @param data data of node to create
1114    * @return version of node created
1115    * @throws KeeperException if unexpected zookeeper exception
1116    * @throws KeeperException.NodeExistsException if node already exists
1117    */
1118   public static int createAndWatch(ZooKeeperWatcher zkw,
1119       String znode, byte [] data)
1120   throws KeeperException, KeeperException.NodeExistsException {
1121     try {
1122       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1123           CreateMode.PERSISTENT);
1124       Stat stat = zkw.getRecoverableZooKeeper().exists(znode, zkw);
1125       if (stat == null){
1126         // Likely a race condition. Someone deleted the znode.
1127         throw KeeperException.create(KeeperException.Code.SYSTEMERROR,
1128             "ZK.exists returned null (i.e.: znode does not exist) for znode=" + znode);
1129       }
1130      return stat.getVersion();
1131     } catch (InterruptedException e) {
1132       zkw.interruptedException(e);
1133       return -1;
1134     }
1135   }
1136 
1137   /**
1138    * Async creates the specified node with the specified data.
1139    *
1140    * <p>Throws an exception if the node already exists.
1141    *
1142    * <p>The node created is persistent and open access.
1143    *
1144    * @param zkw zk reference
1145    * @param znode path of node to create
1146    * @param data data of node to create
1147    * @param cb
1148    * @param ctx
1149    * @throws KeeperException if unexpected zookeeper exception
1150    * @throws KeeperException.NodeExistsException if node already exists
1151    */
1152   public static void asyncCreate(ZooKeeperWatcher zkw,
1153       String znode, byte [] data, final AsyncCallback.StringCallback cb,
1154       final Object ctx) {
1155     zkw.getRecoverableZooKeeper().getZooKeeper().create(znode, data,
1156         createACL(zkw, znode), CreateMode.PERSISTENT, cb, ctx);
1157   }
1158 
1159   /**
1160    * Creates the specified node, iff the node does not exist.  Does not set a
1161    * watch and fails silently if the node already exists.
1162    *
1163    * The node created is persistent and open access.
1164    *
1165    * @param zkw zk reference
1166    * @param znode path of node
1167    * @throws KeeperException if unexpected zookeeper exception
1168    */
1169   public static void createAndFailSilent(ZooKeeperWatcher zkw,
1170       String znode) throws KeeperException {
1171     createAndFailSilent(zkw, znode, new byte[0]);
1172   }
1173 
1174   /**
1175    * Creates the specified node containing specified data, iff the node does not exist.  Does
1176    * not set a watch and fails silently if the node already exists.
1177    *
1178    * The node created is persistent and open access.
1179    *
1180    * @param zkw zk reference
1181    * @param znode path of node
1182    * @param data a byte array data to store in the znode
1183    * @throws KeeperException if unexpected zookeeper exception
1184    */
1185   public static void createAndFailSilent(ZooKeeperWatcher zkw,
1186       String znode, byte[] data)
1187   throws KeeperException {
1188     createAndFailSilent(zkw,
1189         (CreateAndFailSilent)ZKUtilOp.createAndFailSilent(znode, data));
1190   }
1191 
1192   private static void createAndFailSilent(ZooKeeperWatcher zkw, CreateAndFailSilent cafs)
1193   throws KeeperException {
1194     CreateRequest create = (CreateRequest)toZooKeeperOp(zkw, cafs).toRequestRecord();
1195     String znode = create.getPath();
1196     try {
1197       RecoverableZooKeeper zk = zkw.getRecoverableZooKeeper();
1198       if (zk.exists(znode, false) == null) {
1199         zk.create(znode, create.getData(), create.getAcl(), CreateMode.fromFlag(create.getFlags()));
1200       }
1201     } catch(KeeperException.NodeExistsException nee) {
1202     } catch(KeeperException.NoAuthException nee){
1203       try {
1204         if (null == zkw.getRecoverableZooKeeper().exists(znode, false)) {
1205           // If we failed to create the file and it does not already exist.
1206           throw(nee);
1207         }
1208       } catch (InterruptedException ie) {
1209         zkw.interruptedException(ie);
1210       }
1211 
1212     } catch(InterruptedException ie) {
1213       zkw.interruptedException(ie);
1214     }
1215   }
1216 
1217   /**
1218    * Creates the specified node and all parent nodes required for it to exist.
1219    *
1220    * No watches are set and no errors are thrown if the node already exists.
1221    *
1222    * The nodes created are persistent and open access.
1223    *
1224    * @param zkw zk reference
1225    * @param znode path of node
1226    * @throws KeeperException if unexpected zookeeper exception
1227    */
1228   public static void createWithParents(ZooKeeperWatcher zkw, String znode)
1229   throws KeeperException {
1230     createWithParents(zkw, znode, new byte[0]);
1231   }
1232 
1233   /**
1234    * Creates the specified node and all parent nodes required for it to exist.  The creation of
1235    * parent znodes is not atomic with the leafe znode creation but the data is written atomically
1236    * when the leaf node is created.
1237    *
1238    * No watches are set and no errors are thrown if the node already exists.
1239    *
1240    * The nodes created are persistent and open access.
1241    *
1242    * @param zkw zk reference
1243    * @param znode path of node
1244    * @throws KeeperException if unexpected zookeeper exception
1245    */
1246   public static void createWithParents(ZooKeeperWatcher zkw, String znode, byte[] data)
1247   throws KeeperException {
1248     try {
1249       if(znode == null) {
1250         return;
1251       }
1252       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1253           CreateMode.PERSISTENT);
1254     } catch(KeeperException.NodeExistsException nee) {
1255       return;
1256     } catch(KeeperException.NoNodeException nne) {
1257       createWithParents(zkw, getParent(znode));
1258       createWithParents(zkw, znode, data);
1259     } catch(InterruptedException ie) {
1260       zkw.interruptedException(ie);
1261     }
1262   }
1263 
1264   //
1265   // Deletes
1266   //
1267 
1268   /**
1269    * Delete the specified node.  Sets no watches.  Throws all exceptions.
1270    */
1271   public static void deleteNode(ZooKeeperWatcher zkw, String node)
1272   throws KeeperException {
1273     deleteNode(zkw, node, -1);
1274   }
1275 
1276   /**
1277    * Delete the specified node with the specified version.  Sets no watches.
1278    * Throws all exceptions.
1279    */
1280   public static boolean deleteNode(ZooKeeperWatcher zkw, String node,
1281       int version)
1282   throws KeeperException {
1283     try {
1284       zkw.getRecoverableZooKeeper().delete(node, version);
1285       return true;
1286     } catch(KeeperException.BadVersionException bve) {
1287       return false;
1288     } catch(InterruptedException ie) {
1289       zkw.interruptedException(ie);
1290       return false;
1291     }
1292   }
1293 
1294   /**
1295    * Deletes the specified node.  Fails silent if the node does not exist.
1296    * @param zkw
1297    * @param node
1298    * @throws KeeperException
1299    */
1300   public static void deleteNodeFailSilent(ZooKeeperWatcher zkw, String node)
1301   throws KeeperException {
1302     deleteNodeFailSilent(zkw,
1303       (DeleteNodeFailSilent)ZKUtilOp.deleteNodeFailSilent(node));
1304   }
1305 
1306   private static void deleteNodeFailSilent(ZooKeeperWatcher zkw,
1307       DeleteNodeFailSilent dnfs) throws KeeperException {
1308     DeleteRequest delete = (DeleteRequest)toZooKeeperOp(zkw, dnfs).toRequestRecord();
1309     try {
1310       zkw.getRecoverableZooKeeper().delete(delete.getPath(), delete.getVersion());
1311     } catch(KeeperException.NoNodeException nne) {
1312     } catch(InterruptedException ie) {
1313       zkw.interruptedException(ie);
1314     }
1315   }
1316 
1317 
1318   /**
1319    * Delete the specified node and all of it's children.
1320    * <p>
1321    * If the node does not exist, just returns.
1322    * <p>
1323    * Sets no watches. Throws all exceptions besides dealing with deletion of
1324    * children.
1325    */
1326   public static void deleteNodeRecursively(ZooKeeperWatcher zkw, String node)
1327   throws KeeperException {
1328     try {
1329       List<String> children = ZKUtil.listChildrenNoWatch(zkw, node);
1330       // the node is already deleted, so we just finish
1331       if (children == null) return;
1332 
1333       if(!children.isEmpty()) {
1334         for(String child : children) {
1335           deleteNodeRecursively(zkw, joinZNode(node, child));
1336         }
1337       }
1338       //Zookeeper Watches are one time triggers; When children of parent nodes are deleted recursively.
1339       //Must set another watch, get notified of delete node
1340       if (zkw.getRecoverableZooKeeper().exists(node, zkw) != null){
1341         zkw.getRecoverableZooKeeper().delete(node, -1);
1342       }
1343     } catch(InterruptedException ie) {
1344       zkw.interruptedException(ie);
1345     }
1346   }
1347 
1348   /**
1349    * Delete all the children of the specified node but not the node itself.
1350    *
1351    * Sets no watches.  Throws all exceptions besides dealing with deletion of
1352    * children.
1353    */
1354   public static void deleteChildrenRecursively(ZooKeeperWatcher zkw, String node)
1355   throws KeeperException {
1356     List<String> children = ZKUtil.listChildrenNoWatch(zkw, node);
1357     if (children == null || children.isEmpty()) return;
1358     for(String child : children) {
1359       deleteNodeRecursively(zkw, joinZNode(node, child));
1360     }
1361   }
1362 
1363   /**
1364    * Represents an action taken by ZKUtil, e.g. createAndFailSilent.
1365    * These actions are higher-level than ZKOp actions, which represent
1366    * individual actions in the ZooKeeper API, like create.
1367    */
1368   public abstract static class ZKUtilOp {
1369     private String path;
1370 
1371     private ZKUtilOp(String path) {
1372       this.path = path;
1373     }
1374 
1375     /**
1376      * @return a createAndFailSilent ZKUtilOp
1377      */
1378     public static ZKUtilOp createAndFailSilent(String path, byte[] data) {
1379       return new CreateAndFailSilent(path, data);
1380     }
1381 
1382     /**
1383      * @return a deleteNodeFailSilent ZKUtilOP
1384      */
1385     public static ZKUtilOp deleteNodeFailSilent(String path) {
1386       return new DeleteNodeFailSilent(path);
1387     }
1388 
1389     /**
1390      * @return a setData ZKUtilOp
1391      */
1392     public static ZKUtilOp setData(String path, byte [] data) {
1393       return new SetData(path, data);
1394     }
1395 
1396     /**
1397      * @return path to znode where the ZKOp will occur
1398      */
1399     public String getPath() {
1400       return path;
1401     }
1402 
1403     /**
1404      * ZKUtilOp representing createAndFailSilent in ZooKeeper
1405      * (attempt to create node, ignore error if already exists)
1406      */
1407     public static class CreateAndFailSilent extends ZKUtilOp {
1408       private byte [] data;
1409 
1410       private CreateAndFailSilent(String path, byte [] data) {
1411         super(path);
1412         this.data = data;
1413       }
1414 
1415       public byte[] getData() {
1416         return data;
1417       }
1418 
1419       @Override
1420       public boolean equals(Object o) {
1421         if (this == o) return true;
1422         if (!(o instanceof CreateAndFailSilent)) return false;
1423 
1424         CreateAndFailSilent op = (CreateAndFailSilent) o;
1425         return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1426       }
1427 
1428       @Override
1429       public int hashCode() {
1430         int ret = 17 + getPath().hashCode() * 31;
1431         return ret * 31 + Bytes.hashCode(data);
1432       }
1433     }
1434 
1435     /**
1436      * ZKUtilOp representing deleteNodeFailSilent in ZooKeeper
1437      * (attempt to delete node, ignore error if node doesn't exist)
1438      */
1439     public static class DeleteNodeFailSilent extends ZKUtilOp {
1440       private DeleteNodeFailSilent(String path) {
1441         super(path);
1442       }
1443 
1444       @Override
1445       public boolean equals(Object o) {
1446         if (this == o) return true;
1447         if (!(o instanceof DeleteNodeFailSilent)) return false;
1448 
1449         return super.equals(o);
1450       }
1451 
1452       @Override
1453       public int hashCode() {
1454         return getPath().hashCode();
1455       }
1456     }
1457 
1458     /**
1459      * ZKUtilOp representing setData in ZooKeeper
1460      */
1461     public static class SetData extends ZKUtilOp {
1462       private byte [] data;
1463 
1464       private SetData(String path, byte [] data) {
1465         super(path);
1466         this.data = data;
1467       }
1468 
1469       public byte[] getData() {
1470         return data;
1471       }
1472 
1473       @Override
1474       public boolean equals(Object o) {
1475         if (this == o) return true;
1476         if (!(o instanceof SetData)) return false;
1477 
1478         SetData op = (SetData) o;
1479         return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1480       }
1481 
1482       @Override
1483       public int hashCode() {
1484         int ret = getPath().hashCode();
1485         return ret * 31 + Bytes.hashCode(data);
1486       }
1487     }
1488   }
1489 
1490   /**
1491    * Convert from ZKUtilOp to ZKOp
1492    */
1493   private static Op toZooKeeperOp(ZooKeeperWatcher zkw, ZKUtilOp op)
1494   throws UnsupportedOperationException {
1495     if(op == null) return null;
1496 
1497     if (op instanceof CreateAndFailSilent) {
1498       CreateAndFailSilent cafs = (CreateAndFailSilent)op;
1499       return Op.create(cafs.getPath(), cafs.getData(), createACL(zkw, cafs.getPath()),
1500         CreateMode.PERSISTENT);
1501     } else if (op instanceof DeleteNodeFailSilent) {
1502       DeleteNodeFailSilent dnfs = (DeleteNodeFailSilent)op;
1503       return Op.delete(dnfs.getPath(), -1);
1504     } else if (op instanceof SetData) {
1505       SetData sd = (SetData)op;
1506       return Op.setData(sd.getPath(), sd.getData(), -1);
1507     } else {
1508       throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1509         + op.getClass().getName());
1510     }
1511   }
1512 
1513   /**
1514    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update functionality.
1515    * Otherwise, run the list of operations sequentially.
1516    *
1517    * If all of the following are true:
1518    * - runSequentialOnMultiFailure is true
1519    * - hbase.zookeeper.useMulti is true
1520    * - on calling multi, we get a ZooKeeper exception that can be handled by a sequential call(*)
1521    * Then:
1522    * - we retry the operations one-by-one (sequentially)
1523    *
1524    * Note *: an example is receiving a NodeExistsException from a "create" call.  Without multi,
1525    * a user could call "createAndFailSilent" to ensure that a node exists if they don't care who
1526    * actually created the node (i.e. the NodeExistsException from ZooKeeper is caught).
1527    * This will cause all operations in the multi to fail, however, because
1528    * the NodeExistsException that zk.create throws will fail the multi transaction.
1529    * In this case, if the previous conditions hold, the commands are run sequentially, which should
1530    * result in the correct final state, but means that the operations will not run atomically.
1531    *
1532    * @throws KeeperException
1533    */
1534   public static void multiOrSequential(ZooKeeperWatcher zkw, List<ZKUtilOp> ops,
1535       boolean runSequentialOnMultiFailure) throws KeeperException {
1536     if (ops == null) return;
1537     boolean useMulti = zkw.getConfiguration().getBoolean(HConstants.ZOOKEEPER_USEMULTI, false);
1538 
1539     if (useMulti) {
1540       List<Op> zkOps = new LinkedList<Op>();
1541       for (ZKUtilOp op : ops) {
1542         zkOps.add(toZooKeeperOp(zkw, op));
1543       }
1544       try {
1545         zkw.getRecoverableZooKeeper().multi(zkOps);
1546       } catch (KeeperException ke) {
1547        switch (ke.code()) {
1548          case NODEEXISTS:
1549          case NONODE:
1550          case BADVERSION:
1551          case NOAUTH:
1552            // if we get an exception that could be solved by running sequentially
1553            // (and the client asked us to), then break out and run sequentially
1554            if (runSequentialOnMultiFailure) {
1555              LOG.info("On call to ZK.multi, received exception: " + ke.toString() + "."
1556                + "  Attempting to run operations sequentially because"
1557                + " runSequentialOnMultiFailure is: " + runSequentialOnMultiFailure + ".");
1558              processSequentially(zkw, ops);
1559              break;
1560            }
1561           default:
1562             throw ke;
1563         }
1564       } catch (InterruptedException ie) {
1565         zkw.interruptedException(ie);
1566       }
1567     } else {
1568       // run sequentially
1569       processSequentially(zkw, ops);
1570     }
1571 
1572   }
1573 
1574   private static void processSequentially(ZooKeeperWatcher zkw, List<ZKUtilOp> ops)
1575       throws KeeperException, NoNodeException {
1576     for (ZKUtilOp op : ops) {
1577       if (op instanceof CreateAndFailSilent) {
1578         createAndFailSilent(zkw, (CreateAndFailSilent) op);
1579       } else if (op instanceof DeleteNodeFailSilent) {
1580         deleteNodeFailSilent(zkw, (DeleteNodeFailSilent) op);
1581       } else if (op instanceof SetData) {
1582         setData(zkw, (SetData) op);
1583       } else {
1584         throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1585             + op.getClass().getName());
1586       }
1587     }
1588   }
1589 
1590   //
1591   // ZooKeeper cluster information
1592   //
1593 
1594   /** @return String dump of everything in ZooKeeper. */
1595   public static String dump(ZooKeeperWatcher zkw) {
1596     StringBuilder sb = new StringBuilder();
1597     try {
1598       sb.append("HBase is rooted at ").append(zkw.baseZNode);
1599       sb.append("\nActive master address: ");
1600       try {
1601         sb.append(MasterAddressTracker.getMasterAddress(zkw));
1602       } catch (IOException e) {
1603         sb.append("<<FAILED LOOKUP: " + e.getMessage() + ">>");
1604       }
1605       sb.append("\nBackup master addresses:");
1606       for (String child : listChildrenNoWatch(zkw,
1607                                               zkw.backupMasterAddressesZNode)) {
1608         sb.append("\n ").append(child);
1609       }
1610       sb.append("\nRegion server holding hbase:meta: "
1611         + new MetaTableLocator().getMetaRegionLocation(zkw));
1612       sb.append("\nRegion servers:");
1613       for (String child : listChildrenNoWatch(zkw, zkw.rsZNode)) {
1614         sb.append("\n ").append(child);
1615       }
1616       try {
1617         getReplicationZnodesDump(zkw, sb);
1618       } catch (KeeperException ke) {
1619         LOG.warn("Couldn't get the replication znode dump", ke);
1620       }
1621       sb.append("\nQuorum Server Statistics:");
1622       String[] servers = zkw.getQuorum().split(",");
1623       for (String server : servers) {
1624         sb.append("\n ").append(server);
1625         try {
1626           String[] stat = getServerStats(server, ZKUtil.zkDumpConnectionTimeOut);
1627 
1628           if (stat == null) {
1629             sb.append("[Error] invalid quorum server: " + server);
1630             break;
1631           }
1632 
1633           for (String s : stat) {
1634             sb.append("\n  ").append(s);
1635           }
1636         } catch (Exception e) {
1637           sb.append("\n  ERROR: ").append(e.getMessage());
1638         }
1639       }
1640     } catch (KeeperException ke) {
1641       sb.append("\nFATAL ZooKeeper Exception!\n");
1642       sb.append("\n" + ke.getMessage());
1643     }
1644     return sb.toString();
1645   }
1646 
1647   /**
1648    * Appends replication znodes to the passed StringBuilder.
1649    * @param zkw
1650    * @param sb
1651    * @throws KeeperException
1652    */
1653   private static void getReplicationZnodesDump(ZooKeeperWatcher zkw, StringBuilder sb)
1654       throws KeeperException {
1655     String replicationZNodeName = zkw.getConfiguration().get("zookeeper.znode.replication",
1656       "replication");
1657     String replicationZnode = joinZNode(zkw.baseZNode, replicationZNodeName);
1658     if (ZKUtil.checkExists(zkw, replicationZnode) == -1) return;
1659     // do a ls -r on this znode
1660     sb.append("\n").append(replicationZnode).append(": ");
1661     List<String> children = ZKUtil.listChildrenNoWatch(zkw, replicationZnode);
1662     for (String child : children) {
1663       String znode = joinZNode(replicationZnode, child);
1664       if (child.equals(zkw.getConfiguration().get("zookeeper.znode.replication.peers", "peers"))) {
1665         appendPeersZnodes(zkw, znode, sb);
1666       } else if (child.equals(zkw.getConfiguration().
1667           get("zookeeper.znode.replication.rs", "rs"))) {
1668         appendRSZnodes(zkw, znode, sb);
1669       }
1670     }
1671   }
1672 
1673   private static void appendRSZnodes(ZooKeeperWatcher zkw, String znode, StringBuilder sb)
1674       throws KeeperException {
1675     List<String> stack = new LinkedList<String>();
1676     stack.add(znode);
1677     do {
1678       String znodeToProcess = stack.remove(stack.size() - 1);
1679       sb.append("\n").append(znodeToProcess).append(": ");
1680       byte[] data;
1681       try {
1682         data = ZKUtil.getData(zkw, znodeToProcess);
1683       } catch (InterruptedException e) {
1684         zkw.interruptedException(e);
1685         return;
1686       }
1687       if (data != null && data.length > 0) { // log position
1688         long position = 0;
1689         try {
1690           position = ZKUtil.parseWALPositionFrom(ZKUtil.getData(zkw, znodeToProcess));
1691           sb.append(position);
1692         } catch (DeserializationException ignored) {
1693         } catch (InterruptedException e) {
1694           zkw.interruptedException(e);
1695           return;
1696         }
1697       }
1698       for (String zNodeChild : ZKUtil.listChildrenNoWatch(zkw, znodeToProcess)) {
1699         stack.add(ZKUtil.joinZNode(znodeToProcess, zNodeChild));
1700       }
1701     } while (stack.size() > 0);
1702   }
1703 
1704   private static void appendPeersZnodes(ZooKeeperWatcher zkw, String peersZnode,
1705     StringBuilder sb) throws KeeperException {
1706     int pblen = ProtobufUtil.lengthOfPBMagic();
1707     sb.append("\n").append(peersZnode).append(": ");
1708     for (String peerIdZnode : ZKUtil.listChildrenNoWatch(zkw, peersZnode)) {
1709       String znodeToProcess = ZKUtil.joinZNode(peersZnode, peerIdZnode);
1710       byte[] data;
1711       try {
1712         data = ZKUtil.getData(zkw, znodeToProcess);
1713       } catch (InterruptedException e) {
1714         zkw.interruptedException(e);
1715         return;
1716       }
1717       // parse the data of the above peer znode.
1718       try {
1719       String clusterKey = ZooKeeperProtos.ReplicationPeer.newBuilder().
1720         mergeFrom(data, pblen, data.length - pblen).getClusterkey();
1721       sb.append("\n").append(znodeToProcess).append(": ").append(clusterKey);
1722       // add the peer-state.
1723       appendPeerState(zkw, znodeToProcess, sb);
1724       } catch (InvalidProtocolBufferException ipbe) {
1725         LOG.warn("Got Exception while parsing peer: " + znodeToProcess, ipbe);
1726       }
1727     }
1728   }
1729 
1730   private static void appendPeerState(ZooKeeperWatcher zkw, String znodeToProcess,
1731       StringBuilder sb) throws KeeperException, InvalidProtocolBufferException {
1732     String peerState = zkw.getConfiguration().get("zookeeper.znode.replication.peers.state",
1733       "peer-state");
1734     int pblen = ProtobufUtil.lengthOfPBMagic();
1735     for (String child : ZKUtil.listChildrenNoWatch(zkw, znodeToProcess)) {
1736       if (!child.equals(peerState)) continue;
1737       String peerStateZnode = ZKUtil.joinZNode(znodeToProcess, child);
1738       sb.append("\n").append(peerStateZnode).append(": ");
1739       byte[] peerStateData;
1740       try {
1741         peerStateData = ZKUtil.getData(zkw, peerStateZnode);
1742         sb.append(ZooKeeperProtos.ReplicationState.newBuilder()
1743             .mergeFrom(peerStateData, pblen, peerStateData.length - pblen).getState().name());
1744       } catch (InterruptedException e) {
1745         zkw.interruptedException(e);
1746         return;
1747       }
1748     }
1749   }
1750 
1751   /**
1752    * Gets the statistics from the given server.
1753    *
1754    * @param server  The server to get the statistics from.
1755    * @param timeout  The socket timeout to use.
1756    * @return The array of response strings.
1757    * @throws IOException When the socket communication fails.
1758    */
1759   public static String[] getServerStats(String server, int timeout)
1760   throws IOException {
1761     String[] sp = server.split(":");
1762     if (sp == null || sp.length == 0) {
1763       return null;
1764     }
1765 
1766     String host = sp[0];
1767     int port = sp.length > 1 ? Integer.parseInt(sp[1])
1768         : HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT;
1769 
1770     Socket socket = new Socket();
1771     InetSocketAddress sockAddr = new InetSocketAddress(host, port);
1772     socket.connect(sockAddr, timeout);
1773 
1774     socket.setSoTimeout(timeout);
1775     PrintWriter out = new PrintWriter(socket.getOutputStream(), true);
1776     BufferedReader in = new BufferedReader(new InputStreamReader(
1777       socket.getInputStream()));
1778     out.println("stat");
1779     out.flush();
1780     ArrayList<String> res = new ArrayList<String>();
1781     while (true) {
1782       String line = in.readLine();
1783       if (line != null) {
1784         res.add(line);
1785       } else {
1786         break;
1787       }
1788     }
1789     socket.close();
1790     return res.toArray(new String[res.size()]);
1791   }
1792 
1793   private static void logRetrievedMsg(final ZooKeeperWatcher zkw,
1794       final String znode, final byte [] data, final boolean watcherSet) {
1795     if (!LOG.isTraceEnabled()) return;
1796     LOG.trace(zkw.prefix("Retrieved " + ((data == null)? 0: data.length) +
1797       " byte(s) of data from znode " + znode +
1798       (watcherSet? " and set watcher; ": "; data=") +
1799       (data == null? "null": data.length == 0? "empty": (
1800           znode.startsWith(zkw.assignmentZNode)?
1801             ZKAssign.toString(data): // We should not be doing this reaching into another class
1802           znode.startsWith(zkw.metaServerZNode)?
1803             getServerNameOrEmptyString(data):
1804           znode.startsWith(zkw.backupMasterAddressesZNode)?
1805             getServerNameOrEmptyString(data):
1806           StringUtils.abbreviate(Bytes.toStringBinary(data), 32)))));
1807   }
1808 
1809   private static String getServerNameOrEmptyString(final byte [] data) {
1810     try {
1811       return ServerName.parseFrom(data).toString();
1812     } catch (DeserializationException e) {
1813       return "";
1814     }
1815   }
1816 
1817   /**
1818    * Waits for HBase installation's base (parent) znode to become available.
1819    * @throws IOException on ZK errors
1820    */
1821   public static void waitForBaseZNode(Configuration conf) throws IOException {
1822     LOG.info("Waiting until the base znode is available");
1823     String parentZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
1824         HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
1825     ZooKeeper zk = new ZooKeeper(ZKConfig.getZKQuorumServersString(conf),
1826         conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
1827         HConstants.DEFAULT_ZK_SESSION_TIMEOUT), EmptyWatcher.instance);
1828 
1829     final int maxTimeMs = 10000;
1830     final int maxNumAttempts = maxTimeMs / HConstants.SOCKET_RETRY_WAIT_MS;
1831 
1832     KeeperException keeperEx = null;
1833     try {
1834       try {
1835         for (int attempt = 0; attempt < maxNumAttempts; ++attempt) {
1836           try {
1837             if (zk.exists(parentZNode, false) != null) {
1838               LOG.info("Parent znode exists: " + parentZNode);
1839               keeperEx = null;
1840               break;
1841             }
1842           } catch (KeeperException e) {
1843             keeperEx = e;
1844           }
1845           Threads.sleepWithoutInterrupt(HConstants.SOCKET_RETRY_WAIT_MS);
1846         }
1847       } finally {
1848         zk.close();
1849       }
1850     } catch (InterruptedException ex) {
1851       Thread.currentThread().interrupt();
1852     }
1853 
1854     if (keeperEx != null) {
1855       throw new IOException(keeperEx);
1856     }
1857   }
1858 
1859 
1860   public static byte[] blockUntilAvailable(
1861     final ZooKeeperWatcher zkw, final String znode, final long timeout)
1862     throws InterruptedException {
1863     if (timeout < 0) throw new IllegalArgumentException();
1864     if (zkw == null) throw new IllegalArgumentException();
1865     if (znode == null) throw new IllegalArgumentException();
1866 
1867     byte[] data = null;
1868     boolean finished = false;
1869     final long endTime = System.currentTimeMillis() + timeout;
1870     while (!finished) {
1871       try {
1872         data = ZKUtil.getData(zkw, znode);
1873       } catch(KeeperException e) {
1874         if (e instanceof KeeperException.SessionExpiredException
1875             || e instanceof KeeperException.AuthFailedException) {
1876           // non-recoverable errors so stop here
1877           throw new InterruptedException("interrupted due to " + e);
1878         }
1879         LOG.warn("Unexpected exception handling blockUntilAvailable", e);
1880       }
1881 
1882       if (data == null && (System.currentTimeMillis() +
1883         HConstants.SOCKET_RETRY_WAIT_MS < endTime)) {
1884         Thread.sleep(HConstants.SOCKET_RETRY_WAIT_MS);
1885       } else {
1886         finished = true;
1887       }
1888     }
1889 
1890     return data;
1891   }
1892 
1893 
1894   /**
1895    * Convert a {@link DeserializationException} to a more palatable {@link KeeperException}.
1896    * Used when can't let a {@link DeserializationException} out w/o changing public API.
1897    * @param e Exception to convert
1898    * @return Converted exception
1899    */
1900   public static KeeperException convert(final DeserializationException e) {
1901     KeeperException ke = new KeeperException.DataInconsistencyException();
1902     ke.initCause(e);
1903     return ke;
1904   }
1905 
1906   /**
1907    * Recursively print the current state of ZK (non-transactional)
1908    * @param root name of the root directory in zk to print
1909    * @throws KeeperException
1910    */
1911   public static void logZKTree(ZooKeeperWatcher zkw, String root) {
1912     if (!LOG.isDebugEnabled()) return;
1913     LOG.debug("Current zk system:");
1914     String prefix = "|-";
1915     LOG.debug(prefix + root);
1916     try {
1917       logZKTree(zkw, root, prefix);
1918     } catch (KeeperException e) {
1919       throw new RuntimeException(e);
1920     }
1921   }
1922 
1923   /**
1924    * Helper method to print the current state of the ZK tree.
1925    * @see #logZKTree(ZooKeeperWatcher, String)
1926    * @throws KeeperException if an unexpected exception occurs
1927    */
1928   protected static void logZKTree(ZooKeeperWatcher zkw, String root, String prefix) throws KeeperException {
1929     List<String> children = ZKUtil.listChildrenNoWatch(zkw, root);
1930     if (children == null) return;
1931     for (String child : children) {
1932       LOG.debug(prefix + child);
1933       String node = ZKUtil.joinZNode(root.equals("/") ? "" : root, child);
1934       logZKTree(zkw, node, prefix + "---");
1935     }
1936   }
1937 
1938   /**
1939    * @param position
1940    * @return Serialized protobuf of <code>position</code> with pb magic prefix prepended suitable
1941    *         for use as content of an wal position in a replication queue.
1942    */
1943   public static byte[] positionToByteArray(final long position) {
1944     byte[] bytes = ZooKeeperProtos.ReplicationHLogPosition.newBuilder().setPosition(position)
1945         .build().toByteArray();
1946     return ProtobufUtil.prependPBMagic(bytes);
1947   }
1948 
1949   /**
1950    * @param bytes - Content of a WAL position znode.
1951    * @return long - The current WAL position.
1952    * @throws DeserializationException
1953    */
1954   public static long parseWALPositionFrom(final byte[] bytes) throws DeserializationException {
1955     if (bytes == null) {
1956       throw new DeserializationException("Unable to parse null WAL position.");
1957     }
1958     if (ProtobufUtil.isPBMagicPrefix(bytes)) {
1959       int pblen = ProtobufUtil.lengthOfPBMagic();
1960       ZooKeeperProtos.ReplicationHLogPosition.Builder builder =
1961           ZooKeeperProtos.ReplicationHLogPosition.newBuilder();
1962       ZooKeeperProtos.ReplicationHLogPosition position;
1963       try {
1964         position = builder.mergeFrom(bytes, pblen, bytes.length - pblen).build();
1965       } catch (InvalidProtocolBufferException e) {
1966         throw new DeserializationException(e);
1967       }
1968       return position.getPosition();
1969     } else {
1970       if (bytes.length > 0) {
1971         return Bytes.toLong(bytes);
1972       }
1973       return 0;
1974     }
1975   }
1976 
1977   /**
1978    * @param regionLastFlushedSequenceId the flushed sequence id of a region which is the min of its
1979    *          store max seq ids
1980    * @param storeSequenceIds column family to sequence Id map
1981    * @return Serialized protobuf of <code>RegionSequenceIds</code> with pb magic prefix prepended
1982    *         suitable for use to filter wal edits in distributedLogReplay mode
1983    */
1984   public static byte[] regionSequenceIdsToByteArray(final Long regionLastFlushedSequenceId,
1985       final Map<byte[], Long> storeSequenceIds) {
1986     ZooKeeperProtos.RegionStoreSequenceIds.Builder regionSequenceIdsBuilder =
1987         ZooKeeperProtos.RegionStoreSequenceIds.newBuilder();
1988     ZooKeeperProtos.StoreSequenceId.Builder storeSequenceIdBuilder =
1989         ZooKeeperProtos.StoreSequenceId.newBuilder();
1990     if (storeSequenceIds != null) {
1991       for (Map.Entry<byte[], Long> e : storeSequenceIds.entrySet()){
1992         byte[] columnFamilyName = e.getKey();
1993         Long curSeqId = e.getValue();
1994         storeSequenceIdBuilder.setFamilyName(ByteStringer.wrap(columnFamilyName));
1995         storeSequenceIdBuilder.setSequenceId(curSeqId);
1996         regionSequenceIdsBuilder.addStoreSequenceId(storeSequenceIdBuilder.build());
1997         storeSequenceIdBuilder.clear();
1998       }
1999     }
2000     regionSequenceIdsBuilder.setLastFlushedSequenceId(regionLastFlushedSequenceId);
2001     byte[] result = regionSequenceIdsBuilder.build().toByteArray();
2002     return ProtobufUtil.prependPBMagic(result);
2003   }
2004 
2005   /**
2006    * @param bytes Content of serialized data of RegionStoreSequenceIds
2007    * @return a RegionStoreSequenceIds object
2008    * @throws DeserializationException
2009    */
2010   public static RegionStoreSequenceIds parseRegionStoreSequenceIds(final byte[] bytes)
2011       throws DeserializationException {
2012     if (bytes == null || !ProtobufUtil.isPBMagicPrefix(bytes)) {
2013       throw new DeserializationException("Unable to parse RegionStoreSequenceIds.");
2014     }
2015     RegionStoreSequenceIds.Builder regionSequenceIdsBuilder =
2016         ZooKeeperProtos.RegionStoreSequenceIds.newBuilder();
2017     int pblen = ProtobufUtil.lengthOfPBMagic();
2018     RegionStoreSequenceIds storeIds = null;
2019     try {
2020       storeIds = regionSequenceIdsBuilder.mergeFrom(bytes, pblen, bytes.length - pblen).build();
2021     } catch (InvalidProtocolBufferException e) {
2022       throw new DeserializationException(e);
2023     }
2024     return storeIds;
2025   }
2026 }