View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.zookeeper;
20  
21  import org.apache.commons.lang.StringUtils;
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.classification.InterfaceAudience;
25  import org.apache.hadoop.classification.InterfaceStability;
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.hbase.HConstants;
28  import org.apache.hadoop.hbase.ServerName;
29  import org.apache.hadoop.hbase.exceptions.DeserializationException;
30  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
31  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
32  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.RegionStoreSequenceIds;
33  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.StoreSequenceId;
34  import org.apache.hadoop.hbase.util.Bytes;
35  import org.apache.hadoop.hbase.util.Threads;
36  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.CreateAndFailSilent;
37  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.DeleteNodeFailSilent;
38  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.SetData;
39  import org.apache.hadoop.security.SecurityUtil;
40  import org.apache.hadoop.security.authentication.util.KerberosUtil;
41  import org.apache.zookeeper.AsyncCallback;
42  import org.apache.zookeeper.CreateMode;
43  import org.apache.zookeeper.KeeperException;
44  import org.apache.zookeeper.KeeperException.NoNodeException;
45  import org.apache.zookeeper.Op;
46  import org.apache.zookeeper.Watcher;
47  import org.apache.zookeeper.ZooDefs.Ids;
48  import org.apache.zookeeper.ZooKeeper;
49  import org.apache.zookeeper.client.ZooKeeperSaslClient;
50  import org.apache.zookeeper.data.ACL;
51  import org.apache.zookeeper.data.Stat;
52  import org.apache.zookeeper.proto.CreateRequest;
53  import org.apache.zookeeper.proto.DeleteRequest;
54  import org.apache.zookeeper.proto.SetDataRequest;
55  import org.apache.zookeeper.server.ZooKeeperSaslServer;
56  
57  import com.google.protobuf.ByteString;
58  import com.google.protobuf.InvalidProtocolBufferException;
59  
60  import javax.security.auth.login.AppConfigurationEntry;
61  import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
62  import java.io.BufferedReader;
63  import java.io.IOException;
64  import java.io.InputStreamReader;
65  import java.io.PrintWriter;
66  import java.net.InetSocketAddress;
67  import java.net.Socket;
68  import java.util.ArrayList;
69  import java.util.Arrays;
70  import java.util.HashMap;
71  import java.util.LinkedList;
72  import java.util.List;
73  import java.util.Map;
74  import java.util.Properties;
75  import java.util.TreeMap;
76  
77  /**
78   * Internal HBase utility class for ZooKeeper.
79   *
80   * <p>Contains only static methods and constants.
81   *
82   * <p>Methods all throw {@link KeeperException} if there is an unexpected
83   * zookeeper exception, so callers of these methods must handle appropriately.
84   * If ZK is required for the operation, the server will need to be aborted.
85   */
86  @InterfaceAudience.Public
87  @InterfaceStability.Evolving
88  public class ZKUtil {
89    private static final Log LOG = LogFactory.getLog(ZKUtil.class);
90  
91    // TODO: Replace this with ZooKeeper constant when ZOOKEEPER-277 is resolved.
92    public static final char ZNODE_PATH_SEPARATOR = '/';
93    private static int zkDumpConnectionTimeOut;
94  
95    /**
96     * Creates a new connection to ZooKeeper, pulling settings and ensemble config
97     * from the specified configuration object using methods from {@link ZKConfig}.
98     *
99     * Sets the connection status monitoring watcher to the specified watcher.
100    *
101    * @param conf configuration to pull ensemble and other settings from
102    * @param watcher watcher to monitor connection changes
103    * @return connection to zookeeper
104    * @throws IOException if unable to connect to zk or config problem
105    */
106   public static RecoverableZooKeeper connect(Configuration conf, Watcher watcher)
107   throws IOException {
108     Properties properties = ZKConfig.makeZKProps(conf);
109     String ensemble = ZKConfig.getZKQuorumServersString(properties);
110     return connect(conf, ensemble, watcher);
111   }
112 
113   public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
114       Watcher watcher)
115   throws IOException {
116     return connect(conf, ensemble, watcher, null);
117   }
118 
119   public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
120       Watcher watcher, final String identifier)
121   throws IOException {
122     if(ensemble == null) {
123       throw new IOException("Unable to determine ZooKeeper ensemble");
124     }
125     int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
126         HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
127     if (LOG.isTraceEnabled()) {
128       LOG.debug(identifier + " opening connection to ZooKeeper ensemble=" + ensemble);
129     }
130     int retry = conf.getInt("zookeeper.recovery.retry", 3);
131     int retryIntervalMillis =
132       conf.getInt("zookeeper.recovery.retry.intervalmill", 1000);
133     zkDumpConnectionTimeOut = conf.getInt("zookeeper.dump.connection.timeout",
134         1000);
135     return new RecoverableZooKeeper(ensemble, timeout, watcher,
136         retry, retryIntervalMillis, identifier);
137   }
138 
139   /**
140    * Log in the current zookeeper server process using the given configuration
141    * keys for the credential file and login principal.
142    *
143    * <p><strong>This is only applicable when running on secure hbase</strong>
144    * On regular HBase (without security features), this will safely be ignored.
145    * </p>
146    *
147    * @param conf The configuration data to use
148    * @param keytabFileKey Property key used to configure the path to the credential file
149    * @param userNameKey Property key used to configure the login principal
150    * @param hostname Current hostname to use in any credentials
151    * @throws IOException underlying exception from SecurityUtil.login() call
152    */
153   public static void loginServer(Configuration conf, String keytabFileKey,
154       String userNameKey, String hostname) throws IOException {
155     login(conf, keytabFileKey, userNameKey, hostname,
156           ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY,
157           JaasConfiguration.SERVER_KEYTAB_KERBEROS_CONFIG_NAME);
158   }
159 
160   /**
161    * Log in the current zookeeper client using the given configuration
162    * keys for the credential file and login principal.
163    *
164    * <p><strong>This is only applicable when running on secure hbase</strong>
165    * On regular HBase (without security features), this will safely be ignored.
166    * </p>
167    *
168    * @param conf The configuration data to use
169    * @param keytabFileKey Property key used to configure the path to the credential file
170    * @param userNameKey Property key used to configure the login principal
171    * @param hostname Current hostname to use in any credentials
172    * @throws IOException underlying exception from SecurityUtil.login() call
173    */
174   public static void loginClient(Configuration conf, String keytabFileKey,
175       String userNameKey, String hostname) throws IOException {
176     login(conf, keytabFileKey, userNameKey, hostname,
177           ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
178           JaasConfiguration.CLIENT_KEYTAB_KERBEROS_CONFIG_NAME);
179   }
180 
181   /**
182    * Log in the current process using the given configuration keys for the
183    * credential file and login principal.
184    *
185    * <p><strong>This is only applicable when running on secure hbase</strong>
186    * On regular HBase (without security features), this will safely be ignored.
187    * </p>
188    *
189    * @param conf The configuration data to use
190    * @param keytabFileKey Property key used to configure the path to the credential file
191    * @param userNameKey Property key used to configure the login principal
192    * @param hostname Current hostname to use in any credentials
193    * @param loginContextProperty property name to expose the entry name
194    * @param loginContextName jaas entry name
195    * @throws IOException underlying exception from SecurityUtil.login() call
196    */
197   private static void login(Configuration conf, String keytabFileKey,
198       String userNameKey, String hostname,
199       String loginContextProperty, String loginContextName)
200       throws IOException {
201     if (!isSecureZooKeeper(conf))
202       return;
203 
204     // User has specified a jaas.conf, keep this one as the good one.
205     // HBASE_OPTS="-Djava.security.auth.login.config=jaas.conf"
206     if (System.getProperty("java.security.auth.login.config") != null)
207       return;
208 
209     // No keytab specified, no auth
210     String keytabFilename = conf.get(keytabFileKey);
211     if (keytabFilename == null) {
212       LOG.warn("no keytab specified for: " + keytabFileKey);
213       return;
214     }
215 
216     String principalConfig = conf.get(userNameKey, System.getProperty("user.name"));
217     String principalName = SecurityUtil.getServerPrincipal(principalConfig, hostname);
218 
219     // Initialize the "jaas.conf" for keyTab/principal,
220     // If keyTab is not specified use the Ticket Cache.
221     // and set the zookeeper login context name.
222     JaasConfiguration jaasConf = new JaasConfiguration(loginContextName,
223         principalName, keytabFilename);
224     javax.security.auth.login.Configuration.setConfiguration(jaasConf);
225     System.setProperty(loginContextProperty, loginContextName);
226   }
227 
228   /**
229    * A JAAS configuration that defines the login modules that we want to use for login.
230    */
231   private static class JaasConfiguration extends javax.security.auth.login.Configuration {
232     private static final String SERVER_KEYTAB_KERBEROS_CONFIG_NAME =
233       "zookeeper-server-keytab-kerberos";
234     private static final String CLIENT_KEYTAB_KERBEROS_CONFIG_NAME =
235       "zookeeper-client-keytab-kerberos";
236 
237     private static final Map<String, String> BASIC_JAAS_OPTIONS =
238       new HashMap<String,String>();
239     static {
240       String jaasEnvVar = System.getenv("HBASE_JAAS_DEBUG");
241       if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) {
242         BASIC_JAAS_OPTIONS.put("debug", "true");
243       }
244     }
245 
246     private static final Map<String,String> KEYTAB_KERBEROS_OPTIONS =
247       new HashMap<String,String>();
248     static {
249       KEYTAB_KERBEROS_OPTIONS.put("doNotPrompt", "true");
250       KEYTAB_KERBEROS_OPTIONS.put("storeKey", "true");
251       KEYTAB_KERBEROS_OPTIONS.put("refreshKrb5Config", "true");
252       KEYTAB_KERBEROS_OPTIONS.putAll(BASIC_JAAS_OPTIONS);
253     }
254 
255     private static final AppConfigurationEntry KEYTAB_KERBEROS_LOGIN =
256       new AppConfigurationEntry(KerberosUtil.getKrb5LoginModuleName(),
257                                 LoginModuleControlFlag.REQUIRED,
258                                 KEYTAB_KERBEROS_OPTIONS);
259 
260     private static final AppConfigurationEntry[] KEYTAB_KERBEROS_CONF =
261       new AppConfigurationEntry[]{KEYTAB_KERBEROS_LOGIN};
262 
263     private javax.security.auth.login.Configuration baseConfig;
264     private final String loginContextName;
265     private final boolean useTicketCache;
266     private final String keytabFile;
267     private final String principal;
268 
269     public JaasConfiguration(String loginContextName, String principal) {
270       this(loginContextName, principal, null, true);
271     }
272 
273     public JaasConfiguration(String loginContextName, String principal, String keytabFile) {
274       this(loginContextName, principal, keytabFile, keytabFile == null || keytabFile.length() == 0);
275     }
276 
277     private JaasConfiguration(String loginContextName, String principal,
278                              String keytabFile, boolean useTicketCache) {
279       try {
280         this.baseConfig = javax.security.auth.login.Configuration.getConfiguration();
281       } catch (SecurityException e) {
282         this.baseConfig = null;
283       }
284       this.loginContextName = loginContextName;
285       this.useTicketCache = useTicketCache;
286       this.keytabFile = keytabFile;
287       this.principal = principal;
288       LOG.info("JaasConfiguration loginContextName=" + loginContextName +
289                " principal=" + principal + " useTicketCache=" + useTicketCache +
290                " keytabFile=" + keytabFile);
291     }
292 
293     @Override
294     public AppConfigurationEntry[] getAppConfigurationEntry(String appName) {
295       if (loginContextName.equals(appName)) {
296         if (!useTicketCache) {
297           KEYTAB_KERBEROS_OPTIONS.put("keyTab", keytabFile);
298           KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true");
299         }
300         KEYTAB_KERBEROS_OPTIONS.put("principal", principal);
301         KEYTAB_KERBEROS_OPTIONS.put("useTicketCache", useTicketCache ? "true" : "false");
302         return KEYTAB_KERBEROS_CONF;
303       }
304       if (baseConfig != null) return baseConfig.getAppConfigurationEntry(appName);
305       return(null);
306     }
307   }
308 
309   //
310   // Helper methods
311   //
312 
313   /**
314    * Join the prefix znode name with the suffix znode name to generate a proper
315    * full znode name.
316    *
317    * Assumes prefix does not end with slash and suffix does not begin with it.
318    *
319    * @param prefix beginning of znode name
320    * @param suffix ending of znode name
321    * @return result of properly joining prefix with suffix
322    */
323   public static String joinZNode(String prefix, String suffix) {
324     return prefix + ZNODE_PATH_SEPARATOR + suffix;
325   }
326 
327   /**
328    * Returns the full path of the immediate parent of the specified node.
329    * @param node path to get parent of
330    * @return parent of path, null if passed the root node or an invalid node
331    */
332   public static String getParent(String node) {
333     int idx = node.lastIndexOf(ZNODE_PATH_SEPARATOR);
334     return idx <= 0 ? null : node.substring(0, idx);
335   }
336 
337   /**
338    * Get the name of the current node from the specified fully-qualified path.
339    * @param path fully-qualified path
340    * @return name of the current node
341    */
342   public static String getNodeName(String path) {
343     return path.substring(path.lastIndexOf("/")+1);
344   }
345 
346   /**
347    * Get the key to the ZK ensemble for this configuration without
348    * adding a name at the end
349    * @param conf Configuration to use to build the key
350    * @return ensemble key without a name
351    */
352   public static String getZooKeeperClusterKey(Configuration conf) {
353     return getZooKeeperClusterKey(conf, null);
354   }
355 
356   /**
357    * Get the key to the ZK ensemble for this configuration and append
358    * a name at the end
359    * @param conf Configuration to use to build the key
360    * @param name Name that should be appended at the end if not empty or null
361    * @return ensemble key with a name (if any)
362    */
363   public static String getZooKeeperClusterKey(Configuration conf, String name) {
364     String ensemble = conf.get(HConstants.ZOOKEEPER_QUORUM.replaceAll(
365         "[\\t\\n\\x0B\\f\\r]", ""));
366     StringBuilder builder = new StringBuilder(ensemble);
367     builder.append(":");
368     builder.append(conf.get(HConstants.ZOOKEEPER_CLIENT_PORT));
369     builder.append(":");
370     builder.append(conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT));
371     if (name != null && !name.isEmpty()) {
372       builder.append(",");
373       builder.append(name);
374     }
375     return builder.toString();
376   }
377 
378   /**
379    * Apply the settings in the given key to the given configuration, this is
380    * used to communicate with distant clusters
381    * @param conf configuration object to configure
382    * @param key string that contains the 3 required configuratins
383    * @throws IOException
384    */
385   public static void applyClusterKeyToConf(Configuration conf, String key)
386       throws IOException{
387     String[] parts = transformClusterKey(key);
388     conf.set(HConstants.ZOOKEEPER_QUORUM, parts[0]);
389     conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, parts[1]);
390     conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parts[2]);
391   }
392 
393   /**
394    * Separate the given key into the three configurations it should contain:
395    * hbase.zookeeper.quorum, hbase.zookeeper.client.port
396    * and zookeeper.znode.parent
397    * @param key
398    * @return the three configuration in the described order
399    * @throws IOException
400    */
401   public static String[] transformClusterKey(String key) throws IOException {
402     String[] parts = key.split(":");
403     if (parts.length != 3) {
404       throw new IOException("Cluster key invalid, the format should be:" +
405           HConstants.ZOOKEEPER_QUORUM + ":hbase.zookeeper.client.port:"
406           + HConstants.ZOOKEEPER_ZNODE_PARENT);
407     }
408     return parts;
409   }
410 
411   //
412   // Existence checks and watches
413   //
414 
415   /**
416    * Watch the specified znode for delete/create/change events.  The watcher is
417    * set whether or not the node exists.  If the node already exists, the method
418    * returns true.  If the node does not exist, the method returns false.
419    *
420    * @param zkw zk reference
421    * @param znode path of node to watch
422    * @return true if znode exists, false if does not exist or error
423    * @throws KeeperException if unexpected zookeeper exception
424    */
425   public static boolean watchAndCheckExists(ZooKeeperWatcher zkw, String znode)
426   throws KeeperException {
427     try {
428       Stat s = zkw.getRecoverableZooKeeper().exists(znode, zkw);
429       boolean exists = s != null ? true : false;
430       if (exists) {
431         LOG.debug(zkw.prefix("Set watcher on existing znode=" + znode));
432       } else {
433         LOG.debug(zkw.prefix("Set watcher on znode that does not yet exist, " + znode));
434       }
435       return exists;
436     } catch (KeeperException e) {
437       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
438       zkw.keeperException(e);
439       return false;
440     } catch (InterruptedException e) {
441       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
442       zkw.interruptedException(e);
443       return false;
444     }
445   }
446 
447   /**
448    * Watch the specified znode, but only if exists. Useful when watching
449    * for deletions. Uses .getData() (and handles NoNodeException) instead
450    * of .exists() to accomplish this, as .getData() will only set a watch if
451    * the znode exists.
452    * @param zkw zk reference
453    * @param znode path of node to watch
454    * @return true if the watch is set, false if node does not exists
455    * @throws KeeperException if unexpected zookeeper exception
456    */
457   public static boolean setWatchIfNodeExists(ZooKeeperWatcher zkw, String znode)
458       throws KeeperException {
459     try {
460       zkw.getRecoverableZooKeeper().getData(znode, true, null);
461       return true;
462     } catch (NoNodeException e) {
463       return false;
464     } catch (InterruptedException e) {
465       LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
466       zkw.interruptedException(e);
467       return false;
468     }
469   }
470 
471   /**
472    * Check if the specified node exists.  Sets no watches.
473    *
474    * @param zkw zk reference
475    * @param znode path of node to watch
476    * @return version of the node if it exists, -1 if does not exist
477    * @throws KeeperException if unexpected zookeeper exception
478    */
479   public static int checkExists(ZooKeeperWatcher zkw, String znode)
480   throws KeeperException {
481     try {
482       Stat s = zkw.getRecoverableZooKeeper().exists(znode, null);
483       return s != null ? s.getVersion() : -1;
484     } catch (KeeperException e) {
485       LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
486       zkw.keeperException(e);
487       return -1;
488     } catch (InterruptedException e) {
489       LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
490       zkw.interruptedException(e);
491       return -1;
492     }
493   }
494 
495   //
496   // Znode listings
497   //
498 
499   /**
500    * Lists the children znodes of the specified znode.  Also sets a watch on
501    * the specified znode which will capture a NodeDeleted event on the specified
502    * znode as well as NodeChildrenChanged if any children of the specified znode
503    * are created or deleted.
504    *
505    * Returns null if the specified node does not exist.  Otherwise returns a
506    * list of children of the specified node.  If the node exists but it has no
507    * children, an empty list will be returned.
508    *
509    * @param zkw zk reference
510    * @param znode path of node to list and watch children of
511    * @return list of children of the specified node, an empty list if the node
512    *          exists but has no children, and null if the node does not exist
513    * @throws KeeperException if unexpected zookeeper exception
514    */
515   public static List<String> listChildrenAndWatchForNewChildren(
516       ZooKeeperWatcher zkw, String znode)
517   throws KeeperException {
518     try {
519       List<String> children = zkw.getRecoverableZooKeeper().getChildren(znode, zkw);
520       return children;
521     } catch(KeeperException.NoNodeException ke) {
522       LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
523           "because node does not exist (not an error)"));
524       return null;
525     } catch (KeeperException e) {
526       LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
527       zkw.keeperException(e);
528       return null;
529     } catch (InterruptedException e) {
530       LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
531       zkw.interruptedException(e);
532       return null;
533     }
534   }
535 
536   /**
537    * List all the children of the specified znode, setting a watch for children
538    * changes and also setting a watch on every individual child in order to get
539    * the NodeCreated and NodeDeleted events.
540    * @param zkw zookeeper reference
541    * @param znode node to get children of and watch
542    * @return list of znode names, null if the node doesn't exist
543    * @throws KeeperException
544    */
545   public static List<String> listChildrenAndWatchThem(ZooKeeperWatcher zkw,
546       String znode) throws KeeperException {
547     List<String> children = listChildrenAndWatchForNewChildren(zkw, znode);
548     if (children == null) {
549       return null;
550     }
551     for (String child : children) {
552       watchAndCheckExists(zkw, joinZNode(znode, child));
553     }
554     return children;
555   }
556 
557   /**
558    * Lists the children of the specified znode without setting any watches.
559    *
560    * Sets no watches at all, this method is best effort.
561    *
562    * Returns an empty list if the node has no children.  Returns null if the
563    * parent node itself does not exist.
564    *
565    * @param zkw zookeeper reference
566    * @param znode node to get children
567    * @return list of data of children of specified znode, empty if no children,
568    *         null if parent does not exist
569    * @throws KeeperException if unexpected zookeeper exception
570    */
571   public static List<String> listChildrenNoWatch(ZooKeeperWatcher zkw, String znode)
572   throws KeeperException {
573     List<String> children = null;
574     try {
575       // List the children without watching
576       children = zkw.getRecoverableZooKeeper().getChildren(znode, null);
577     } catch(KeeperException.NoNodeException nne) {
578       return null;
579     } catch(InterruptedException ie) {
580       zkw.interruptedException(ie);
581     }
582     return children;
583   }
584 
585   /**
586    * Simple class to hold a node path and node data.
587    * @deprecated Unused
588    */
589   @Deprecated
590   public static class NodeAndData {
591     private String node;
592     private byte [] data;
593     public NodeAndData(String node, byte [] data) {
594       this.node = node;
595       this.data = data;
596     }
597     public String getNode() {
598       return node;
599     }
600     public byte [] getData() {
601       return data;
602     }
603     @Override
604     public String toString() {
605       return node;
606     }
607     public boolean isEmpty() {
608       return (data.length == 0);
609     }
610   }
611 
612   /**
613    * Checks if the specified znode has any children.  Sets no watches.
614    *
615    * Returns true if the node exists and has children.  Returns false if the
616    * node does not exist or if the node does not have any children.
617    *
618    * Used during master initialization to determine if the master is a
619    * failed-over-to master or the first master during initial cluster startup.
620    * If the directory for regionserver ephemeral nodes is empty then this is
621    * a cluster startup, if not then it is not cluster startup.
622    *
623    * @param zkw zk reference
624    * @param znode path of node to check for children of
625    * @return true if node has children, false if not or node does not exist
626    * @throws KeeperException if unexpected zookeeper exception
627    */
628   public static boolean nodeHasChildren(ZooKeeperWatcher zkw, String znode)
629   throws KeeperException {
630     try {
631       return !zkw.getRecoverableZooKeeper().getChildren(znode, null).isEmpty();
632     } catch(KeeperException.NoNodeException ke) {
633       LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
634       "because node does not exist (not an error)"));
635       return false;
636     } catch (KeeperException e) {
637       LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
638       zkw.keeperException(e);
639       return false;
640     } catch (InterruptedException e) {
641       LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
642       zkw.interruptedException(e);
643       return false;
644     }
645   }
646 
647   /**
648    * Get the number of children of the specified node.
649    *
650    * If the node does not exist or has no children, returns 0.
651    *
652    * Sets no watches at all.
653    *
654    * @param zkw zk reference
655    * @param znode path of node to count children of
656    * @return number of children of specified node, 0 if none or parent does not
657    *         exist
658    * @throws KeeperException if unexpected zookeeper exception
659    */
660   public static int getNumberOfChildren(ZooKeeperWatcher zkw, String znode)
661   throws KeeperException {
662     try {
663       Stat stat = zkw.getRecoverableZooKeeper().exists(znode, null);
664       return stat == null ? 0 : stat.getNumChildren();
665     } catch(KeeperException e) {
666       LOG.warn(zkw.prefix("Unable to get children of node " + znode));
667       zkw.keeperException(e);
668     } catch(InterruptedException e) {
669       zkw.interruptedException(e);
670     }
671     return 0;
672   }
673 
674   //
675   // Data retrieval
676   //
677 
678   /**
679    * Get znode data. Does not set a watcher.
680    * @return ZNode data, null if the node does not exist or if there is an
681    *  error.
682    */
683   public static byte [] getData(ZooKeeperWatcher zkw, String znode)
684   throws KeeperException {
685     try {
686       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, null);
687       logRetrievedMsg(zkw, znode, data, false);
688       return data;
689     } catch (KeeperException.NoNodeException e) {
690       LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
691         "because node does not exist (not an error)"));
692       return null;
693     } catch (KeeperException e) {
694       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
695       zkw.keeperException(e);
696       return null;
697     } catch (InterruptedException e) {
698       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
699       zkw.interruptedException(e);
700       return null;
701     }
702   }
703 
704   /**
705    * Get the data at the specified znode and set a watch.
706    *
707    * Returns the data and sets a watch if the node exists.  Returns null and no
708    * watch is set if the node does not exist or there is an exception.
709    *
710    * @param zkw zk reference
711    * @param znode path of node
712    * @return data of the specified znode, or null
713    * @throws KeeperException if unexpected zookeeper exception
714    */
715   public static byte [] getDataAndWatch(ZooKeeperWatcher zkw, String znode)
716   throws KeeperException {
717     return getDataInternal(zkw, znode, null, true);
718   }
719 
720   /**
721    * Get the data at the specified znode and set a watch.
722    *
723    * Returns the data and sets a watch if the node exists.  Returns null and no
724    * watch is set if the node does not exist or there is an exception.
725    *
726    * @param zkw zk reference
727    * @param znode path of node
728    * @param stat object to populate the version of the znode
729    * @return data of the specified znode, or null
730    * @throws KeeperException if unexpected zookeeper exception
731    */
732   public static byte[] getDataAndWatch(ZooKeeperWatcher zkw, String znode,
733       Stat stat) throws KeeperException {
734     return getDataInternal(zkw, znode, stat, true);
735   }
736 
737   private static byte[] getDataInternal(ZooKeeperWatcher zkw, String znode, Stat stat,
738       boolean watcherSet)
739       throws KeeperException {
740     try {
741       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, zkw, stat);
742       logRetrievedMsg(zkw, znode, data, watcherSet);
743       return data;
744     } catch (KeeperException.NoNodeException e) {
745       // This log can get pretty annoying when we cycle on 100ms waits.
746       // Enable trace if you really want to see it.
747       LOG.trace(zkw.prefix("Unable to get data of znode " + znode + " " +
748         "because node does not exist (not an error)"));
749       return null;
750     } catch (KeeperException e) {
751       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
752       zkw.keeperException(e);
753       return null;
754     } catch (InterruptedException e) {
755       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
756       zkw.interruptedException(e);
757       return null;
758     }
759   }
760 
761   /**
762    * Get the data at the specified znode without setting a watch.
763    *
764    * Returns the data if the node exists.  Returns null if the node does not
765    * exist.
766    *
767    * Sets the stats of the node in the passed Stat object.  Pass a null stat if
768    * not interested.
769    *
770    * @param zkw zk reference
771    * @param znode path of node
772    * @param stat node status to get if node exists
773    * @return data of the specified znode, or null if node does not exist
774    * @throws KeeperException if unexpected zookeeper exception
775    */
776   public static byte [] getDataNoWatch(ZooKeeperWatcher zkw, String znode,
777       Stat stat)
778   throws KeeperException {
779     try {
780       byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, stat);
781       logRetrievedMsg(zkw, znode, data, false);
782       return data;
783     } catch (KeeperException.NoNodeException e) {
784       LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
785           "because node does not exist (not necessarily an error)"));
786       return null;
787     } catch (KeeperException e) {
788       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
789       zkw.keeperException(e);
790       return null;
791     } catch (InterruptedException e) {
792       LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
793       zkw.interruptedException(e);
794       return null;
795     }
796   }
797 
798   /**
799    * Returns the date of child znodes of the specified znode.  Also sets a watch on
800    * the specified znode which will capture a NodeDeleted event on the specified
801    * znode as well as NodeChildrenChanged if any children of the specified znode
802    * are created or deleted.
803    *
804    * Returns null if the specified node does not exist.  Otherwise returns a
805    * list of children of the specified node.  If the node exists but it has no
806    * children, an empty list will be returned.
807    *
808    * @param zkw zk reference
809    * @param baseNode path of node to list and watch children of
810    * @return list of data of children of the specified node, an empty list if the node
811    *          exists but has no children, and null if the node does not exist
812    * @throws KeeperException if unexpected zookeeper exception
813    * @deprecated Unused
814    */
815   public static List<NodeAndData> getChildDataAndWatchForNewChildren(
816       ZooKeeperWatcher zkw, String baseNode) throws KeeperException {
817     List<String> nodes =
818       ZKUtil.listChildrenAndWatchForNewChildren(zkw, baseNode);
819     List<NodeAndData> newNodes = new ArrayList<NodeAndData>();
820     if (nodes != null) {
821       for (String node : nodes) {
822         String nodePath = ZKUtil.joinZNode(baseNode, node);
823         byte[] data = ZKUtil.getDataAndWatch(zkw, nodePath);
824         newNodes.add(new NodeAndData(nodePath, data));
825       }
826     }
827     return newNodes;
828   }
829 
830   /**
831    * Update the data of an existing node with the expected version to have the
832    * specified data.
833    *
834    * Throws an exception if there is a version mismatch or some other problem.
835    *
836    * Sets no watches under any conditions.
837    *
838    * @param zkw zk reference
839    * @param znode
840    * @param data
841    * @param expectedVersion
842    * @throws KeeperException if unexpected zookeeper exception
843    * @throws KeeperException.BadVersionException if version mismatch
844    * @deprecated Unused
845    */
846   public static void updateExistingNodeData(ZooKeeperWatcher zkw, String znode,
847       byte [] data, int expectedVersion)
848   throws KeeperException {
849     try {
850       zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion);
851     } catch(InterruptedException ie) {
852       zkw.interruptedException(ie);
853     }
854   }
855 
856   //
857   // Data setting
858   //
859 
860   /**
861    * Sets the data of the existing znode to be the specified data.  Ensures that
862    * the current data has the specified expected version.
863    *
864    * <p>If the node does not exist, a {@link NoNodeException} will be thrown.
865    *
866    * <p>If their is a version mismatch, method returns null.
867    *
868    * <p>No watches are set but setting data will trigger other watchers of this
869    * node.
870    *
871    * <p>If there is another problem, a KeeperException will be thrown.
872    *
873    * @param zkw zk reference
874    * @param znode path of node
875    * @param data data to set for node
876    * @param expectedVersion version expected when setting data
877    * @return true if data set, false if version mismatch
878    * @throws KeeperException if unexpected zookeeper exception
879    */
880   public static boolean setData(ZooKeeperWatcher zkw, String znode,
881       byte [] data, int expectedVersion)
882   throws KeeperException, KeeperException.NoNodeException {
883     try {
884       return zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion) != null;
885     } catch (InterruptedException e) {
886       zkw.interruptedException(e);
887       return false;
888     }
889   }
890 
891   /**
892    * Set data into node creating node if it doesn't yet exist.
893    * Does not set watch.
894    *
895    * @param zkw zk reference
896    * @param znode path of node
897    * @param data data to set for node
898    * @throws KeeperException
899    */
900   public static void createSetData(final ZooKeeperWatcher zkw, final String znode,
901       final byte [] data)
902   throws KeeperException {
903     if (checkExists(zkw, znode) == -1) {
904       ZKUtil.createWithParents(zkw, znode, data);
905     } else {
906       ZKUtil.setData(zkw, znode, data);
907     }
908   }
909 
910   /**
911    * Sets the data of the existing znode to be the specified data.  The node
912    * must exist but no checks are done on the existing data or version.
913    *
914    * <p>If the node does not exist, a {@link NoNodeException} will be thrown.
915    *
916    * <p>No watches are set but setting data will trigger other watchers of this
917    * node.
918    *
919    * <p>If there is another problem, a KeeperException will be thrown.
920    *
921    * @param zkw zk reference
922    * @param znode path of node
923    * @param data data to set for node
924    * @throws KeeperException if unexpected zookeeper exception
925    */
926   public static void setData(ZooKeeperWatcher zkw, String znode, byte [] data)
927   throws KeeperException, KeeperException.NoNodeException {
928     setData(zkw, (SetData)ZKUtilOp.setData(znode, data));
929   }
930 
931   private static void setData(ZooKeeperWatcher zkw, SetData setData)
932   throws KeeperException, KeeperException.NoNodeException {
933     SetDataRequest sd = (SetDataRequest)toZooKeeperOp(zkw, setData).toRequestRecord();
934     setData(zkw, sd.getPath(), sd.getData(), sd.getVersion());
935   }
936 
937   /**
938    * Returns whether or not secure authentication is enabled
939    * (whether <code>hbase.security.authentication</code> is set to
940    * <code>kerberos</code>.
941    */
942   public static boolean isSecureZooKeeper(Configuration conf) {
943     // hbase shell need to use:
944     //    -Djava.security.auth.login.config=user-jaas.conf
945     // since each user has a different jaas.conf
946     if (System.getProperty("java.security.auth.login.config") != null)
947       return true;
948 
949     // Master & RSs uses hbase.zookeeper.client.*
950     return("kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication")) &&
951          conf.get("hbase.zookeeper.client.keytab.file") != null);
952   }
953 
954   private static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node) {
955     if (isSecureZooKeeper(zkw.getConfiguration())) {
956       // Certain znodes are accessed directly by the client,
957       // so they must be readable by non-authenticated clients
958       if ((node.equals(zkw.baseZNode) == true) ||
959           (node.equals(zkw.metaServerZNode) == true) ||
960           (node.equals(zkw.getMasterAddressZNode()) == true) ||
961           (node.equals(zkw.clusterIdZNode) == true) ||
962           (node.equals(zkw.rsZNode) == true) ||
963           (node.equals(zkw.backupMasterAddressesZNode) == true) ||
964           (node.startsWith(zkw.assignmentZNode) == true) ||
965           (node.startsWith(zkw.tableZNode) == true)) {
966         return ZooKeeperWatcher.CREATOR_ALL_AND_WORLD_READABLE;
967       }
968       return Ids.CREATOR_ALL_ACL;
969     } else {
970       return Ids.OPEN_ACL_UNSAFE;
971     }
972   }
973 
974   //
975   // Node creation
976   //
977 
978   /**
979    *
980    * Set the specified znode to be an ephemeral node carrying the specified
981    * data.
982    *
983    * If the node is created successfully, a watcher is also set on the node.
984    *
985    * If the node is not created successfully because it already exists, this
986    * method will also set a watcher on the node.
987    *
988    * If there is another problem, a KeeperException will be thrown.
989    *
990    * @param zkw zk reference
991    * @param znode path of node
992    * @param data data of node
993    * @return true if node created, false if not, watch set in both cases
994    * @throws KeeperException if unexpected zookeeper exception
995    */
996   public static boolean createEphemeralNodeAndWatch(ZooKeeperWatcher zkw,
997       String znode, byte [] data)
998   throws KeeperException {
999     try {
1000       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1001           CreateMode.EPHEMERAL);
1002     } catch (KeeperException.NodeExistsException nee) {
1003       if(!watchAndCheckExists(zkw, znode)) {
1004         // It did exist but now it doesn't, try again
1005         return createEphemeralNodeAndWatch(zkw, znode, data);
1006       }
1007       return false;
1008     } catch (InterruptedException e) {
1009       LOG.info("Interrupted", e);
1010       Thread.currentThread().interrupt();
1011     }
1012     return true;
1013   }
1014 
1015   /**
1016    * Creates the specified znode to be a persistent node carrying the specified
1017    * data.
1018    *
1019    * Returns true if the node was successfully created, false if the node
1020    * already existed.
1021    *
1022    * If the node is created successfully, a watcher is also set on the node.
1023    *
1024    * If the node is not created successfully because it already exists, this
1025    * method will also set a watcher on the node but return false.
1026    *
1027    * If there is another problem, a KeeperException will be thrown.
1028    *
1029    * @param zkw zk reference
1030    * @param znode path of node
1031    * @param data data of node
1032    * @return true if node created, false if not, watch set in both cases
1033    * @throws KeeperException if unexpected zookeeper exception
1034    */
1035   public static boolean createNodeIfNotExistsAndWatch(
1036       ZooKeeperWatcher zkw, String znode, byte [] data)
1037   throws KeeperException {
1038     try {
1039       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1040           CreateMode.PERSISTENT);
1041     } catch (KeeperException.NodeExistsException nee) {
1042       try {
1043         zkw.getRecoverableZooKeeper().exists(znode, zkw);
1044       } catch (InterruptedException e) {
1045         zkw.interruptedException(e);
1046         return false;
1047       }
1048       return false;
1049     } catch (InterruptedException e) {
1050       zkw.interruptedException(e);
1051       return false;
1052     }
1053     return true;
1054   }
1055 
1056   /**
1057    * Creates the specified znode with the specified data but does not watch it.
1058    *
1059    * Returns the znode of the newly created node
1060    *
1061    * If there is another problem, a KeeperException will be thrown.
1062    *
1063    * @param zkw zk reference
1064    * @param znode path of node
1065    * @param data data of node
1066    * @param createMode specifying whether the node to be created is ephemeral and/or sequential
1067    * @return true name of the newly created znode or null
1068    * @throws KeeperException if unexpected zookeeper exception
1069    */
1070   public static String createNodeIfNotExistsNoWatch(ZooKeeperWatcher zkw, String znode,
1071       byte[] data, CreateMode createMode) throws KeeperException {
1072 
1073     String createdZNode = null;
1074     try {
1075       createdZNode = zkw.getRecoverableZooKeeper().create(znode, data,
1076           createACL(zkw, znode), createMode);
1077     } catch (KeeperException.NodeExistsException nee) {
1078       return znode;
1079     } catch (InterruptedException e) {
1080       zkw.interruptedException(e);
1081       return null;
1082     }
1083     return createdZNode;
1084   }
1085 
1086   /**
1087    * Creates the specified node with the specified data and watches it.
1088    *
1089    * <p>Throws an exception if the node already exists.
1090    *
1091    * <p>The node created is persistent and open access.
1092    *
1093    * <p>Returns the version number of the created node if successful.
1094    *
1095    * @param zkw zk reference
1096    * @param znode path of node to create
1097    * @param data data of node to create
1098    * @return version of node created
1099    * @throws KeeperException if unexpected zookeeper exception
1100    * @throws KeeperException.NodeExistsException if node already exists
1101    */
1102   public static int createAndWatch(ZooKeeperWatcher zkw,
1103       String znode, byte [] data)
1104   throws KeeperException, KeeperException.NodeExistsException {
1105     try {
1106       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1107           CreateMode.PERSISTENT);
1108       Stat stat = zkw.getRecoverableZooKeeper().exists(znode, zkw);
1109       if (stat == null){
1110         // Likely a race condition. Someone deleted the znode.
1111         throw KeeperException.create(KeeperException.Code.SYSTEMERROR,
1112             "ZK.exists returned null (i.e.: znode does not exist) for znode=" + znode);
1113       }
1114      return stat.getVersion();
1115     } catch (InterruptedException e) {
1116       zkw.interruptedException(e);
1117       return -1;
1118     }
1119   }
1120 
1121   /**
1122    * Async creates the specified node with the specified data.
1123    *
1124    * <p>Throws an exception if the node already exists.
1125    *
1126    * <p>The node created is persistent and open access.
1127    *
1128    * @param zkw zk reference
1129    * @param znode path of node to create
1130    * @param data data of node to create
1131    * @param cb
1132    * @param ctx
1133    * @throws KeeperException if unexpected zookeeper exception
1134    * @throws KeeperException.NodeExistsException if node already exists
1135    */
1136   public static void asyncCreate(ZooKeeperWatcher zkw,
1137       String znode, byte [] data, final AsyncCallback.StringCallback cb,
1138       final Object ctx) {
1139     zkw.getRecoverableZooKeeper().getZooKeeper().create(znode, data,
1140         createACL(zkw, znode), CreateMode.PERSISTENT, cb, ctx);
1141   }
1142 
1143   /**
1144    * Creates the specified node, iff the node does not exist.  Does not set a
1145    * watch and fails silently if the node already exists.
1146    *
1147    * The node created is persistent and open access.
1148    *
1149    * @param zkw zk reference
1150    * @param znode path of node
1151    * @throws KeeperException if unexpected zookeeper exception
1152    */
1153   public static void createAndFailSilent(ZooKeeperWatcher zkw,
1154       String znode) throws KeeperException {
1155     createAndFailSilent(zkw, znode, new byte[0]);
1156   }
1157 
1158   /**
1159    * Creates the specified node containing specified data, iff the node does not exist.  Does
1160    * not set a watch and fails silently if the node already exists.
1161    *
1162    * The node created is persistent and open access.
1163    *
1164    * @param zkw zk reference
1165    * @param znode path of node
1166    * @param data a byte array data to store in the znode
1167    * @throws KeeperException if unexpected zookeeper exception
1168    */
1169   public static void createAndFailSilent(ZooKeeperWatcher zkw,
1170       String znode, byte[] data)
1171   throws KeeperException {
1172     createAndFailSilent(zkw,
1173         (CreateAndFailSilent)ZKUtilOp.createAndFailSilent(znode, data));
1174   }
1175   
1176   private static void createAndFailSilent(ZooKeeperWatcher zkw, CreateAndFailSilent cafs)
1177   throws KeeperException {
1178     CreateRequest create = (CreateRequest)toZooKeeperOp(zkw, cafs).toRequestRecord();
1179     String znode = create.getPath();
1180     try {
1181       RecoverableZooKeeper zk = zkw.getRecoverableZooKeeper();
1182       if (zk.exists(znode, false) == null) {
1183         zk.create(znode, create.getData(), create.getAcl(), CreateMode.fromFlag(create.getFlags()));
1184       }
1185     } catch(KeeperException.NodeExistsException nee) {
1186     } catch(KeeperException.NoAuthException nee){
1187       try {
1188         if (null == zkw.getRecoverableZooKeeper().exists(znode, false)) {
1189           // If we failed to create the file and it does not already exist.
1190           throw(nee);
1191         }
1192       } catch (InterruptedException ie) {
1193         zkw.interruptedException(ie);
1194       }
1195 
1196     } catch(InterruptedException ie) {
1197       zkw.interruptedException(ie);
1198     }
1199   }
1200 
1201   /**
1202    * Creates the specified node and all parent nodes required for it to exist.
1203    *
1204    * No watches are set and no errors are thrown if the node already exists.
1205    *
1206    * The nodes created are persistent and open access.
1207    *
1208    * @param zkw zk reference
1209    * @param znode path of node
1210    * @throws KeeperException if unexpected zookeeper exception
1211    */
1212   public static void createWithParents(ZooKeeperWatcher zkw, String znode)
1213   throws KeeperException {
1214     createWithParents(zkw, znode, new byte[0]);
1215   }
1216 
1217   /**
1218    * Creates the specified node and all parent nodes required for it to exist.  The creation of
1219    * parent znodes is not atomic with the leafe znode creation but the data is written atomically
1220    * when the leaf node is created.
1221    *
1222    * No watches are set and no errors are thrown if the node already exists.
1223    *
1224    * The nodes created are persistent and open access.
1225    *
1226    * @param zkw zk reference
1227    * @param znode path of node
1228    * @throws KeeperException if unexpected zookeeper exception
1229    */
1230   public static void createWithParents(ZooKeeperWatcher zkw, String znode, byte[] data)
1231   throws KeeperException {
1232     try {
1233       if(znode == null) {
1234         return;
1235       }
1236       zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1237           CreateMode.PERSISTENT);
1238     } catch(KeeperException.NodeExistsException nee) {
1239       return;
1240     } catch(KeeperException.NoNodeException nne) {
1241       createWithParents(zkw, getParent(znode));
1242       createWithParents(zkw, znode, data);
1243     } catch(InterruptedException ie) {
1244       zkw.interruptedException(ie);
1245     }
1246   }
1247 
1248   //
1249   // Deletes
1250   //
1251 
1252   /**
1253    * Delete the specified node.  Sets no watches.  Throws all exceptions.
1254    */
1255   public static void deleteNode(ZooKeeperWatcher zkw, String node)
1256   throws KeeperException {
1257     deleteNode(zkw, node, -1);
1258   }
1259 
1260   /**
1261    * Delete the specified node with the specified version.  Sets no watches.
1262    * Throws all exceptions.
1263    */
1264   public static boolean deleteNode(ZooKeeperWatcher zkw, String node,
1265       int version)
1266   throws KeeperException {
1267     try {
1268       zkw.getRecoverableZooKeeper().delete(node, version);
1269       return true;
1270     } catch(KeeperException.BadVersionException bve) {
1271       return false;
1272     } catch(InterruptedException ie) {
1273       zkw.interruptedException(ie);
1274       return false;
1275     }
1276   }
1277 
1278   /**
1279    * Deletes the specified node.  Fails silent if the node does not exist.
1280    * @param zkw
1281    * @param node
1282    * @throws KeeperException
1283    */
1284   public static void deleteNodeFailSilent(ZooKeeperWatcher zkw, String node)
1285   throws KeeperException {
1286     deleteNodeFailSilent(zkw,
1287       (DeleteNodeFailSilent)ZKUtilOp.deleteNodeFailSilent(node));
1288   }
1289 
1290   private static void deleteNodeFailSilent(ZooKeeperWatcher zkw,
1291       DeleteNodeFailSilent dnfs) throws KeeperException {
1292     DeleteRequest delete = (DeleteRequest)toZooKeeperOp(zkw, dnfs).toRequestRecord();
1293     try {
1294       zkw.getRecoverableZooKeeper().delete(delete.getPath(), delete.getVersion());
1295     } catch(KeeperException.NoNodeException nne) {
1296     } catch(InterruptedException ie) {
1297       zkw.interruptedException(ie);
1298     }
1299   }
1300 
1301 
1302   /**
1303    * Delete the specified node and all of it's children.
1304    * <p>
1305    * If the node does not exist, just returns.
1306    * <p>
1307    * Sets no watches. Throws all exceptions besides dealing with deletion of
1308    * children.
1309    */
1310   public static void deleteNodeRecursively(ZooKeeperWatcher zkw, String node)
1311   throws KeeperException {
1312     try {
1313       List<String> children = ZKUtil.listChildrenNoWatch(zkw, node);
1314       // the node is already deleted, so we just finish
1315       if (children == null) return;
1316 
1317       if(!children.isEmpty()) {
1318         for(String child : children) {
1319           deleteNodeRecursively(zkw, joinZNode(node, child));
1320         }
1321       }
1322       zkw.getRecoverableZooKeeper().delete(node, -1);
1323     } catch(InterruptedException ie) {
1324       zkw.interruptedException(ie);
1325     }
1326   }
1327 
1328   /**
1329    * Delete all the children of the specified node but not the node itself.
1330    *
1331    * Sets no watches.  Throws all exceptions besides dealing with deletion of
1332    * children.
1333    */
1334   public static void deleteChildrenRecursively(ZooKeeperWatcher zkw, String node)
1335   throws KeeperException {
1336     List<String> children = ZKUtil.listChildrenNoWatch(zkw, node);
1337     if (children == null || children.isEmpty()) return;
1338     for(String child : children) {
1339       deleteNodeRecursively(zkw, joinZNode(node, child));
1340     }
1341   }
1342 
1343   /**
1344    * Represents an action taken by ZKUtil, e.g. createAndFailSilent.
1345    * These actions are higher-level than ZKOp actions, which represent
1346    * individual actions in the ZooKeeper API, like create.
1347    */
1348   public abstract static class ZKUtilOp {
1349     private String path;
1350 
1351     private ZKUtilOp(String path) {
1352       this.path = path;
1353     }
1354 
1355     /**
1356      * @return a createAndFailSilent ZKUtilOp
1357      */
1358     public static ZKUtilOp createAndFailSilent(String path, byte[] data) {
1359       return new CreateAndFailSilent(path, data);
1360     }
1361 
1362     /**
1363      * @return a deleteNodeFailSilent ZKUtilOP
1364      */
1365     public static ZKUtilOp deleteNodeFailSilent(String path) {
1366       return new DeleteNodeFailSilent(path);
1367     }
1368 
1369     /**
1370      * @return a setData ZKUtilOp
1371      */
1372     public static ZKUtilOp setData(String path, byte [] data) {
1373       return new SetData(path, data);
1374     }
1375 
1376     /**
1377      * @return path to znode where the ZKOp will occur
1378      */
1379     public String getPath() {
1380       return path;
1381     }
1382 
1383     /**
1384      * ZKUtilOp representing createAndFailSilent in ZooKeeper
1385      * (attempt to create node, ignore error if already exists)
1386      */
1387     public static class CreateAndFailSilent extends ZKUtilOp {
1388       private byte [] data;
1389 
1390       private CreateAndFailSilent(String path, byte [] data) {
1391         super(path);
1392         this.data = data;
1393       }
1394 
1395       public byte[] getData() {
1396         return data;
1397       }
1398 
1399       @Override
1400       public boolean equals(Object o) {
1401         if (this == o) return true;
1402         if (!(o instanceof CreateAndFailSilent)) return false;
1403 
1404         CreateAndFailSilent op = (CreateAndFailSilent) o;
1405         return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1406       }
1407 
1408       @Override
1409       public int hashCode() {
1410         int ret = 17 + getPath().hashCode() * 31;
1411         return ret * 31 + Bytes.hashCode(data);
1412       }
1413     }
1414 
1415     /**
1416      * ZKUtilOp representing deleteNodeFailSilent in ZooKeeper
1417      * (attempt to delete node, ignore error if node doesn't exist)
1418      */
1419     public static class DeleteNodeFailSilent extends ZKUtilOp {
1420       private DeleteNodeFailSilent(String path) {
1421         super(path);
1422       }
1423 
1424       @Override
1425       public boolean equals(Object o) {
1426         if (this == o) return true;
1427         if (!(o instanceof DeleteNodeFailSilent)) return false;
1428 
1429         return super.equals(o);
1430       }
1431 
1432       @Override
1433       public int hashCode() {
1434         return getPath().hashCode();
1435       }
1436     }
1437 
1438     /**
1439      * ZKUtilOp representing setData in ZooKeeper
1440      */
1441     public static class SetData extends ZKUtilOp {
1442       private byte [] data;
1443 
1444       private SetData(String path, byte [] data) {
1445         super(path);
1446         this.data = data;
1447       }
1448 
1449       public byte[] getData() {
1450         return data;
1451       }
1452 
1453       @Override
1454       public boolean equals(Object o) {
1455         if (this == o) return true;
1456         if (!(o instanceof SetData)) return false;
1457 
1458         SetData op = (SetData) o;
1459         return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1460       }
1461 
1462       @Override
1463       public int hashCode() {
1464         int ret = getPath().hashCode();
1465         return ret * 31 + Bytes.hashCode(data);
1466       }
1467     }
1468   }
1469 
1470   /**
1471    * Convert from ZKUtilOp to ZKOp
1472    */
1473   private static Op toZooKeeperOp(ZooKeeperWatcher zkw, ZKUtilOp op)
1474   throws UnsupportedOperationException {
1475     if(op == null) return null;
1476 
1477     if (op instanceof CreateAndFailSilent) {
1478       CreateAndFailSilent cafs = (CreateAndFailSilent)op;
1479       return Op.create(cafs.getPath(), cafs.getData(), createACL(zkw, cafs.getPath()),
1480         CreateMode.PERSISTENT);
1481     } else if (op instanceof DeleteNodeFailSilent) {
1482       DeleteNodeFailSilent dnfs = (DeleteNodeFailSilent)op;
1483       return Op.delete(dnfs.getPath(), -1);
1484     } else if (op instanceof SetData) {
1485       SetData sd = (SetData)op;
1486       return Op.setData(sd.getPath(), sd.getData(), -1);
1487     } else {
1488       throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1489         + op.getClass().getName());
1490     }
1491   }
1492 
1493   /**
1494    * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update functionality.
1495    * Otherwise, run the list of operations sequentially.
1496    *
1497    * If all of the following are true:
1498    * - runSequentialOnMultiFailure is true
1499    * - hbase.zookeeper.useMulti is true
1500    * - on calling multi, we get a ZooKeeper exception that can be handled by a sequential call(*)
1501    * Then:
1502    * - we retry the operations one-by-one (sequentially)
1503    *
1504    * Note *: an example is receiving a NodeExistsException from a "create" call.  Without multi,
1505    * a user could call "createAndFailSilent" to ensure that a node exists if they don't care who
1506    * actually created the node (i.e. the NodeExistsException from ZooKeeper is caught).
1507    * This will cause all operations in the multi to fail, however, because
1508    * the NodeExistsException that zk.create throws will fail the multi transaction.
1509    * In this case, if the previous conditions hold, the commands are run sequentially, which should
1510    * result in the correct final state, but means that the operations will not run atomically.
1511    *
1512    * @throws KeeperException
1513    */
1514   public static void multiOrSequential(ZooKeeperWatcher zkw, List<ZKUtilOp> ops,
1515       boolean runSequentialOnMultiFailure) throws KeeperException {
1516     if (ops == null) return;
1517     boolean useMulti = zkw.getConfiguration().getBoolean(HConstants.ZOOKEEPER_USEMULTI, false);
1518 
1519     if (useMulti) {
1520       List<Op> zkOps = new LinkedList<Op>();
1521       for (ZKUtilOp op : ops) {
1522         zkOps.add(toZooKeeperOp(zkw, op));
1523       }
1524       try {
1525         zkw.getRecoverableZooKeeper().multi(zkOps);
1526       } catch (KeeperException ke) {
1527        switch (ke.code()) {
1528          case NODEEXISTS:
1529          case NONODE:
1530          case BADVERSION:
1531          case NOAUTH:
1532            // if we get an exception that could be solved by running sequentially
1533            // (and the client asked us to), then break out and run sequentially
1534            if (runSequentialOnMultiFailure) {
1535              LOG.info("On call to ZK.multi, received exception: " + ke.toString() + "."
1536                + "  Attempting to run operations sequentially because"
1537                + " runSequentialOnMultiFailure is: " + runSequentialOnMultiFailure + ".");
1538              processSequentially(zkw, ops);
1539              break;
1540            }
1541           default:
1542             throw ke;
1543         }
1544       } catch (InterruptedException ie) {
1545         zkw.interruptedException(ie);
1546       }
1547     } else {
1548       // run sequentially
1549       processSequentially(zkw, ops);
1550     }
1551 
1552   }
1553 
1554   private static void processSequentially(ZooKeeperWatcher zkw, List<ZKUtilOp> ops)
1555       throws KeeperException, NoNodeException {
1556     for (ZKUtilOp op : ops) {
1557       if (op instanceof CreateAndFailSilent) {
1558         createAndFailSilent(zkw, (CreateAndFailSilent) op);
1559       } else if (op instanceof DeleteNodeFailSilent) {
1560         deleteNodeFailSilent(zkw, (DeleteNodeFailSilent) op);
1561       } else if (op instanceof SetData) {
1562         setData(zkw, (SetData) op);
1563       } else {
1564         throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1565             + op.getClass().getName());
1566       }
1567     }
1568   }
1569 
1570   //
1571   // ZooKeeper cluster information
1572   //
1573 
1574   /** @return String dump of everything in ZooKeeper. */
1575   public static String dump(ZooKeeperWatcher zkw) {
1576     StringBuilder sb = new StringBuilder();
1577     try {
1578       sb.append("HBase is rooted at ").append(zkw.baseZNode);
1579       sb.append("\nActive master address: ");
1580       try {
1581         sb.append(MasterAddressTracker.getMasterAddress(zkw));
1582       } catch (IOException e) {
1583         sb.append("<<FAILED LOOKUP: " + e.getMessage() + ">>");
1584       }
1585       sb.append("\nBackup master addresses:");
1586       for (String child : listChildrenNoWatch(zkw,
1587                                               zkw.backupMasterAddressesZNode)) {
1588         sb.append("\n ").append(child);
1589       }
1590       sb.append("\nRegion server holding .META.: " + MetaRegionTracker.getMetaRegionLocation(zkw));
1591       sb.append("\nRegion servers:");
1592       for (String child : listChildrenNoWatch(zkw, zkw.rsZNode)) {
1593         sb.append("\n ").append(child);
1594       }
1595       try {
1596         getReplicationZnodesDump(zkw, sb);
1597       } catch (KeeperException ke) {
1598         LOG.warn("Couldn't get the replication znode dump", ke);
1599       }
1600       sb.append("\nQuorum Server Statistics:");
1601       String[] servers = zkw.getQuorum().split(",");
1602       for (String server : servers) {
1603         sb.append("\n ").append(server);
1604         try {
1605           String[] stat = getServerStats(server, ZKUtil.zkDumpConnectionTimeOut);
1606 
1607           if (stat == null) {
1608             sb.append("[Error] invalid quorum server: " + server);
1609             break;
1610           }
1611 
1612           for (String s : stat) {
1613             sb.append("\n  ").append(s);
1614           }
1615         } catch (Exception e) {
1616           sb.append("\n  ERROR: ").append(e.getMessage());
1617         }
1618       }
1619     } catch (KeeperException ke) {
1620       sb.append("\nFATAL ZooKeeper Exception!\n");
1621       sb.append("\n" + ke.getMessage());
1622     }
1623     return sb.toString();
1624   }
1625 
1626   private static void getReplicationZnodesDump(ZooKeeperWatcher zkw, StringBuilder sb)
1627       throws KeeperException {
1628     String replicationZNodeName = zkw.getConfiguration().get("zookeeper.znode.replication",
1629       "replication");
1630     String replicationZnode = joinZNode(zkw.baseZNode, replicationZNodeName);
1631     if (ZKUtil.checkExists(zkw, replicationZnode) == -1) return;
1632     // do a ls -r on this znode
1633     List<String> stack = new LinkedList<String>();
1634     stack.add(replicationZnode);
1635     do {
1636       String znodeToProcess = stack.remove(stack.size() - 1);
1637       sb.append("\n").append(znodeToProcess).append(": ")
1638           .append(Bytes.toString(ZKUtil.getData(zkw, znodeToProcess)));
1639       for (String zNodeChild : ZKUtil.listChildrenNoWatch(zkw, znodeToProcess)) {
1640         stack.add(ZKUtil.joinZNode(znodeToProcess, zNodeChild));
1641       }
1642     } while (stack.size() > 0);
1643   }
1644 
1645   /**
1646    * Gets the statistics from the given server.
1647    *
1648    * @param server  The server to get the statistics from.
1649    * @param timeout  The socket timeout to use.
1650    * @return The array of response strings.
1651    * @throws IOException When the socket communication fails.
1652    */
1653   public static String[] getServerStats(String server, int timeout)
1654   throws IOException {
1655     String[] sp = server.split(":");
1656     if (sp == null || sp.length == 0) {
1657       return null;
1658     }
1659 
1660     String host = sp[0];
1661     int port = sp.length > 1 ? Integer.parseInt(sp[1])
1662         : HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT;
1663 
1664     Socket socket = new Socket();
1665     InetSocketAddress sockAddr = new InetSocketAddress(host, port);
1666     socket.connect(sockAddr, timeout);
1667 
1668     socket.setSoTimeout(timeout);
1669     PrintWriter out = new PrintWriter(socket.getOutputStream(), true);
1670     BufferedReader in = new BufferedReader(new InputStreamReader(
1671       socket.getInputStream()));
1672     out.println("stat");
1673     out.flush();
1674     ArrayList<String> res = new ArrayList<String>();
1675     while (true) {
1676       String line = in.readLine();
1677       if (line != null) {
1678         res.add(line);
1679       } else {
1680         break;
1681       }
1682     }
1683     socket.close();
1684     return res.toArray(new String[res.size()]);
1685   }
1686 
1687   private static void logRetrievedMsg(final ZooKeeperWatcher zkw,
1688       final String znode, final byte [] data, final boolean watcherSet) {
1689     if (!LOG.isTraceEnabled()) return;
1690     LOG.trace(zkw.prefix("Retrieved " + ((data == null)? 0: data.length) +
1691       " byte(s) of data from znode " + znode +
1692       (watcherSet? " and set watcher; ": "; data=") +
1693       (data == null? "null": data.length == 0? "empty": (
1694           znode.startsWith(zkw.assignmentZNode)?
1695             ZKAssign.toString(data): // We should not be doing this reaching into another class
1696           znode.startsWith(zkw.metaServerZNode)?
1697             getServerNameOrEmptyString(data):
1698           znode.startsWith(zkw.backupMasterAddressesZNode)?
1699             getServerNameOrEmptyString(data):
1700           StringUtils.abbreviate(Bytes.toStringBinary(data), 32)))));
1701   }
1702 
1703   private static String getServerNameOrEmptyString(final byte [] data) {
1704     try {
1705       return ServerName.parseFrom(data).toString();
1706     } catch (DeserializationException e) {
1707       return "";
1708     }
1709   }
1710 
1711   /**
1712    * Waits for HBase installation's base (parent) znode to become available.
1713    * @throws IOException on ZK errors
1714    */
1715   public static void waitForBaseZNode(Configuration conf) throws IOException {
1716     LOG.info("Waiting until the base znode is available");
1717     String parentZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
1718         HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
1719     ZooKeeper zk = new ZooKeeper(ZKConfig.getZKQuorumServersString(conf),
1720         conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
1721         HConstants.DEFAULT_ZK_SESSION_TIMEOUT), EmptyWatcher.instance);
1722 
1723     final int maxTimeMs = 10000;
1724     final int maxNumAttempts = maxTimeMs / HConstants.SOCKET_RETRY_WAIT_MS;
1725 
1726     KeeperException keeperEx = null;
1727     try {
1728       try {
1729         for (int attempt = 0; attempt < maxNumAttempts; ++attempt) {
1730           try {
1731             if (zk.exists(parentZNode, false) != null) {
1732               LOG.info("Parent znode exists: " + parentZNode);
1733               keeperEx = null;
1734               break;
1735             }
1736           } catch (KeeperException e) {
1737             keeperEx = e;
1738           }
1739           Threads.sleepWithoutInterrupt(HConstants.SOCKET_RETRY_WAIT_MS);
1740         }
1741       } finally {
1742         zk.close();
1743       }
1744     } catch (InterruptedException ex) {
1745       Thread.currentThread().interrupt();
1746     }
1747 
1748     if (keeperEx != null) {
1749       throw new IOException(keeperEx);
1750     }
1751   }
1752 
1753 
1754   public static byte[] blockUntilAvailable(
1755     final ZooKeeperWatcher zkw, final String znode, final long timeout)
1756     throws InterruptedException {
1757     if (timeout < 0) throw new IllegalArgumentException();
1758     if (zkw == null) throw new IllegalArgumentException();
1759     if (znode == null) throw new IllegalArgumentException();
1760 
1761     byte[] data = null;
1762     boolean finished = false;
1763     final long endTime = System.currentTimeMillis() + timeout;
1764     while (!finished) {
1765       try {
1766         data = ZKUtil.getData(zkw, znode);
1767       } catch(KeeperException e) {
1768         LOG.warn("Unexpected exception handling blockUntilAvailable", e);
1769       }
1770 
1771       if (data == null && (System.currentTimeMillis() +
1772         HConstants.SOCKET_RETRY_WAIT_MS < endTime)) {
1773         Thread.sleep(HConstants.SOCKET_RETRY_WAIT_MS);
1774       } else {
1775         finished = true;
1776       }
1777     }
1778 
1779     return data;
1780   }
1781 
1782 
1783   /**
1784    * Convert a {@link DeserializationException} to a more palatable {@link KeeperException}.
1785    * Used when can't let a {@link DeserializationException} out w/o changing public API.
1786    * @param e Exception to convert
1787    * @return Converted exception
1788    */
1789   public static KeeperException convert(final DeserializationException e) {
1790     KeeperException ke = new KeeperException.DataInconsistencyException();
1791     ke.initCause(e);
1792     return ke;
1793   }
1794 
1795   /**
1796    * Recursively print the current state of ZK (non-transactional)
1797    * @param root name of the root directory in zk to print
1798    * @throws KeeperException
1799    */
1800   public static void logZKTree(ZooKeeperWatcher zkw, String root) {
1801     if (!LOG.isDebugEnabled()) return;
1802     LOG.debug("Current zk system:");
1803     String prefix = "|-";
1804     LOG.debug(prefix + root);
1805     try {
1806       logZKTree(zkw, root, prefix);
1807     } catch (KeeperException e) {
1808       throw new RuntimeException(e);
1809     }
1810   }
1811 
1812   /**
1813    * Helper method to print the current state of the ZK tree.
1814    * @see #logZKTree(ZooKeeperWatcher, String)
1815    * @throws KeeperException if an unexpected exception occurs
1816    */
1817   protected static void logZKTree(ZooKeeperWatcher zkw, String root, String prefix) throws KeeperException {
1818     List<String> children = ZKUtil.listChildrenNoWatch(zkw, root);
1819     if (children == null) return;
1820     for (String child : children) {
1821       LOG.debug(prefix + child);
1822       String node = ZKUtil.joinZNode(root.equals("/") ? "" : root, child);
1823       logZKTree(zkw, node, prefix + "---");
1824     }
1825   }
1826 
1827   /**
1828    * @param position
1829    * @return Serialized protobuf of <code>position</code> with pb magic prefix prepended suitable
1830    *         for use as content of an hlog position in a replication queue.
1831    */
1832   public static byte[] positionToByteArray(final long position) {
1833     byte[] bytes = ZooKeeperProtos.ReplicationHLogPosition.newBuilder().setPosition(position)
1834         .build().toByteArray();
1835     return ProtobufUtil.prependPBMagic(bytes);
1836   }
1837 
1838   /**
1839    * @param bytes - Content of a HLog position znode.
1840    * @return long - The current HLog position.
1841    * @throws DeserializationException
1842    */
1843   public static long parseHLogPositionFrom(final byte[] bytes) throws DeserializationException {
1844     if (bytes == null) {
1845       throw new DeserializationException("Unable to parse null HLog position.");
1846     }
1847     if (ProtobufUtil.isPBMagicPrefix(bytes)) {
1848       int pblen = ProtobufUtil.lengthOfPBMagic();
1849       ZooKeeperProtos.ReplicationHLogPosition.Builder builder =
1850           ZooKeeperProtos.ReplicationHLogPosition.newBuilder();
1851       ZooKeeperProtos.ReplicationHLogPosition position;
1852       try {
1853         position = builder.mergeFrom(bytes, pblen, bytes.length - pblen).build();
1854       } catch (InvalidProtocolBufferException e) {
1855         throw new DeserializationException(e);
1856       }
1857       return position.getPosition();
1858     } else {
1859       if (bytes.length > 0) {
1860         return Bytes.toLong(bytes);
1861       }
1862       return 0;
1863     }
1864   }
1865 
1866   /**
1867    * @param regionLastFlushedSequenceId the flushed sequence id of a region which is the min of its
1868    *          store max seq ids
1869    * @param storeSequenceIds column family to sequence Id map
1870    * @return Serialized protobuf of <code>RegionSequenceIds</code> with pb magic prefix prepended
1871    *         suitable for use to filter wal edits in distributedLogReplay mode
1872    */
1873   public static byte[] regionSequenceIdsToByteArray(final Long regionLastFlushedSequenceId,
1874       final Map<byte[], Long> storeSequenceIds) {
1875     ZooKeeperProtos.RegionStoreSequenceIds.Builder regionSequenceIdsBuilder =
1876         ZooKeeperProtos.RegionStoreSequenceIds.newBuilder();
1877     ZooKeeperProtos.StoreSequenceId.Builder storeSequenceIdBuilder =
1878         ZooKeeperProtos.StoreSequenceId.newBuilder();
1879     if (storeSequenceIds != null) {
1880       for (byte[] columnFamilyName : storeSequenceIds.keySet()) {
1881         Long curSeqId = storeSequenceIds.get(columnFamilyName);
1882         storeSequenceIdBuilder.setFamilyName(ByteString.copyFrom(columnFamilyName));
1883         storeSequenceIdBuilder.setSequenceId(curSeqId);
1884         regionSequenceIdsBuilder.addStoreSequenceId(storeSequenceIdBuilder.build());
1885         storeSequenceIdBuilder.clear();
1886       }
1887     }
1888     regionSequenceIdsBuilder.setLastFlushedSequenceId(regionLastFlushedSequenceId);
1889     byte[] result = regionSequenceIdsBuilder.build().toByteArray();
1890     return ProtobufUtil.prependPBMagic(result);
1891   }
1892 
1893   /**
1894    * @param bytes Content of serialized data of RegionStoreSequenceIds
1895    * @return a RegionStoreSequenceIds object
1896    * @throws DeserializationException
1897    */
1898   public static RegionStoreSequenceIds parseRegionStoreSequenceIds(final byte[] bytes)
1899       throws DeserializationException {
1900     if (bytes == null || !ProtobufUtil.isPBMagicPrefix(bytes)) {
1901       throw new DeserializationException("Unable to parse RegionStoreSequenceIds.");
1902     }
1903     RegionStoreSequenceIds.Builder regionSequenceIdsBuilder =
1904         ZooKeeperProtos.RegionStoreSequenceIds.newBuilder();
1905     int pblen = ProtobufUtil.lengthOfPBMagic();
1906     RegionStoreSequenceIds storeIds = null;
1907     try {
1908       storeIds = regionSequenceIdsBuilder.mergeFrom(bytes, pblen, bytes.length - pblen).build();
1909     } catch (InvalidProtocolBufferException e) {
1910       throw new DeserializationException(e);
1911     }
1912     return storeIds;
1913   }
1914 }