1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.zookeeper;
21
22 import java.io.BufferedReader;
23 import java.io.IOException;
24 import java.io.InputStreamReader;
25 import java.io.PrintWriter;
26 import java.net.InetSocketAddress;
27 import java.net.InetAddress;
28 import java.net.Socket;
29 import java.util.ArrayList;
30 import java.util.Arrays;
31 import java.util.LinkedList;
32 import java.util.List;
33 import java.util.Properties;
34 import java.util.HashMap;
35 import java.util.Map;
36
37 import javax.security.auth.login.LoginException;
38 import javax.security.auth.login.AppConfigurationEntry;
39 import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
40 import org.apache.hadoop.security.SecurityUtil;
41 import org.apache.hadoop.security.authentication.util.KerberosUtil;
42
43 import org.apache.commons.lang.StringUtils;
44 import org.apache.commons.logging.Log;
45 import org.apache.commons.logging.LogFactory;
46 import org.apache.hadoop.conf.Configuration;
47 import org.apache.hadoop.hbase.EmptyWatcher;
48 import org.apache.hadoop.hbase.HConstants;
49 import org.apache.hadoop.hbase.ServerName;
50 import org.apache.hadoop.hbase.executor.RegionTransitionData;
51 import org.apache.hadoop.hbase.util.Bytes;
52 import org.apache.hadoop.hbase.util.Threads;
53 import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.CreateAndFailSilent;
54 import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.DeleteNodeFailSilent;
55 import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.SetData;
56 import org.apache.zookeeper.AsyncCallback;
57 import org.apache.zookeeper.CreateMode;
58 import org.apache.zookeeper.KeeperException;
59 import org.apache.zookeeper.KeeperException.NoNodeException;
60 import org.apache.zookeeper.Op;
61 import org.apache.zookeeper.Watcher;
62 import org.apache.zookeeper.ZooDefs.Ids;
63 import org.apache.zookeeper.ZooKeeper;
64 import org.apache.zookeeper.client.ZooKeeperSaslClient;
65 import org.apache.zookeeper.data.ACL;
66 import org.apache.zookeeper.data.Stat;
67 import org.apache.zookeeper.server.ZooKeeperSaslServer;
68 import org.apache.zookeeper.proto.CreateRequest;
69 import org.apache.zookeeper.proto.DeleteRequest;
70 import org.apache.zookeeper.proto.SetDataRequest;
71
72
73
74
75
76
77
78
79
80
81 public class ZKUtil {
82 private static final Log LOG = LogFactory.getLog(ZKUtil.class);
83
84
85 public static final char ZNODE_PATH_SEPARATOR = '/';
86 private static int zkDumpConnectionTimeOut;
87
88
89
90
91
92
93
94
95
96
97
98
99 public static RecoverableZooKeeper connect(Configuration conf, Watcher watcher)
100 throws IOException {
101 Properties properties = ZKConfig.makeZKProps(conf);
102 String ensemble = ZKConfig.getZKQuorumServersString(properties);
103 return connect(conf, ensemble, watcher);
104 }
105
106 public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
107 Watcher watcher)
108 throws IOException {
109 return connect(conf, ensemble, watcher, "");
110 }
111
112 public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
113 Watcher watcher, final String descriptor)
114 throws IOException {
115 if(ensemble == null) {
116 throw new IOException("Unable to determine ZooKeeper ensemble");
117 }
118 int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
119 HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
120 LOG.debug(descriptor + " opening connection to ZooKeeper with ensemble (" +
121 ensemble + ")");
122 int retry = conf.getInt("zookeeper.recovery.retry", 3);
123 int retryIntervalMillis =
124 conf.getInt("zookeeper.recovery.retry.intervalmill", 1000);
125 zkDumpConnectionTimeOut = conf.getInt("zookeeper.dump.connection.timeout",
126 1000);
127 return new RecoverableZooKeeper(ensemble, timeout, watcher,
128 retry, retryIntervalMillis);
129 }
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145 public static void loginServer(Configuration conf, String keytabFileKey,
146 String userNameKey, String hostname) throws IOException {
147 login(conf, keytabFileKey, userNameKey, hostname,
148 ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY,
149 JaasConfiguration.SERVER_KEYTAB_KERBEROS_CONFIG_NAME);
150 }
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166 public static void loginClient(Configuration conf, String keytabFileKey,
167 String userNameKey, String hostname) throws IOException {
168 login(conf, keytabFileKey, userNameKey, hostname,
169 ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
170 JaasConfiguration.CLIENT_KEYTAB_KERBEROS_CONFIG_NAME);
171 }
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189 private static void login(Configuration conf, String keytabFileKey,
190 String userNameKey, String hostname,
191 String loginContextProperty, String loginContextName)
192 throws IOException {
193 if (!isSecureZooKeeper(conf))
194 return;
195
196
197
198 if (System.getProperty("java.security.auth.login.config") != null)
199 return;
200
201
202 String keytabFilename = conf.get(keytabFileKey);
203 if (keytabFilename == null) {
204 LOG.warn("no keytab specified for: " + keytabFileKey);
205 return;
206 }
207
208 String principalConfig = conf.get(userNameKey, System.getProperty("user.name"));
209 String principalName = SecurityUtil.getServerPrincipal(principalConfig, hostname);
210
211
212
213
214 JaasConfiguration jaasConf = new JaasConfiguration(loginContextName,
215 principalName, keytabFilename);
216 javax.security.auth.login.Configuration.setConfiguration(jaasConf);
217 System.setProperty(loginContextProperty, loginContextName);
218 }
219
220
221
222
223 private static class JaasConfiguration extends javax.security.auth.login.Configuration {
224 private static final String SERVER_KEYTAB_KERBEROS_CONFIG_NAME =
225 "zookeeper-server-keytab-kerberos";
226 private static final String CLIENT_KEYTAB_KERBEROS_CONFIG_NAME =
227 "zookeeper-client-keytab-kerberos";
228
229 private static final Map<String, String> BASIC_JAAS_OPTIONS =
230 new HashMap<String,String>();
231 static {
232 String jaasEnvVar = System.getenv("HBASE_JAAS_DEBUG");
233 if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) {
234 BASIC_JAAS_OPTIONS.put("debug", "true");
235 }
236 }
237
238 private static final Map<String,String> KEYTAB_KERBEROS_OPTIONS =
239 new HashMap<String,String>();
240 static {
241 KEYTAB_KERBEROS_OPTIONS.put("doNotPrompt", "true");
242 KEYTAB_KERBEROS_OPTIONS.put("storeKey", "true");
243 KEYTAB_KERBEROS_OPTIONS.put("refreshKrb5Config", "true");
244 KEYTAB_KERBEROS_OPTIONS.putAll(BASIC_JAAS_OPTIONS);
245 }
246
247 private static final AppConfigurationEntry KEYTAB_KERBEROS_LOGIN =
248 new AppConfigurationEntry(KerberosUtil.getKrb5LoginModuleName(),
249 LoginModuleControlFlag.REQUIRED,
250 KEYTAB_KERBEROS_OPTIONS);
251
252 private static final AppConfigurationEntry[] KEYTAB_KERBEROS_CONF =
253 new AppConfigurationEntry[]{KEYTAB_KERBEROS_LOGIN};
254
255 private javax.security.auth.login.Configuration baseConfig;
256 private final String loginContextName;
257 private final boolean useTicketCache;
258 private final String keytabFile;
259 private final String principal;
260
261 public JaasConfiguration(String loginContextName, String principal) {
262 this(loginContextName, principal, null, true);
263 }
264
265 public JaasConfiguration(String loginContextName, String principal, String keytabFile) {
266 this(loginContextName, principal, keytabFile, keytabFile == null || keytabFile.length() == 0);
267 }
268
269 private JaasConfiguration(String loginContextName, String principal,
270 String keytabFile, boolean useTicketCache) {
271 try {
272 this.baseConfig = javax.security.auth.login.Configuration.getConfiguration();
273 } catch (SecurityException e) {
274 this.baseConfig = null;
275 }
276 this.loginContextName = loginContextName;
277 this.useTicketCache = useTicketCache;
278 this.keytabFile = keytabFile;
279 this.principal = principal;
280 LOG.info("JaasConfiguration loginContextName=" + loginContextName +
281 " principal=" + principal + " useTicketCache=" + useTicketCache +
282 " keytabFile=" + keytabFile);
283 }
284
285 @Override
286 public AppConfigurationEntry[] getAppConfigurationEntry(String appName) {
287 if (loginContextName.equals(appName)) {
288 if (!useTicketCache) {
289 KEYTAB_KERBEROS_OPTIONS.put("keyTab", keytabFile);
290 KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true");
291 }
292 KEYTAB_KERBEROS_OPTIONS.put("principal", principal);
293 KEYTAB_KERBEROS_OPTIONS.put("useTicketCache", useTicketCache ? "true" : "false");
294 return KEYTAB_KERBEROS_CONF;
295 }
296 if (baseConfig != null) return baseConfig.getAppConfigurationEntry(appName);
297 return(null);
298 }
299 }
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315 public static String joinZNode(String prefix, String suffix) {
316 return prefix + ZNODE_PATH_SEPARATOR + suffix;
317 }
318
319
320
321
322
323
324 public static String getParent(String node) {
325 int idx = node.lastIndexOf(ZNODE_PATH_SEPARATOR);
326 return idx <= 0 ? null : node.substring(0, idx);
327 }
328
329
330
331
332
333
334 public static String getNodeName(String path) {
335 return path.substring(path.lastIndexOf("/")+1);
336 }
337
338
339
340
341
342
343
344 public static String getZooKeeperClusterKey(Configuration conf) {
345 return getZooKeeperClusterKey(conf, null);
346 }
347
348
349
350
351
352
353
354
355 public static String getZooKeeperClusterKey(Configuration conf, String name) {
356 String ensemble = conf.get(HConstants.ZOOKEEPER_QUORUM.replaceAll(
357 "[\\t\\n\\x0B\\f\\r]", ""));
358 StringBuilder builder = new StringBuilder(ensemble);
359 builder.append(":");
360 builder.append(conf.get(HConstants.ZOOKEEPER_CLIENT_PORT));
361 builder.append(":");
362 builder.append(conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT));
363 if (name != null && !name.isEmpty()) {
364 builder.append(",");
365 builder.append(name);
366 }
367 return builder.toString();
368 }
369
370
371
372
373
374
375
376
377 public static void applyClusterKeyToConf(Configuration conf, String key)
378 throws IOException{
379 String[] parts = transformClusterKey(key);
380 conf.set(HConstants.ZOOKEEPER_QUORUM, parts[0]);
381 conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, parts[1]);
382 conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parts[2]);
383 }
384
385
386
387
388
389
390
391
392
393 public static String[] transformClusterKey(String key) throws IOException {
394 String[] parts = key.split(":");
395 if (parts.length != 3) {
396 throw new IOException("Cluster key passed " + key + " is invalid, the format should be:" +
397 HConstants.ZOOKEEPER_QUORUM + ":hbase.zookeeper.client.port:"
398 + HConstants.ZOOKEEPER_ZNODE_PARENT);
399 }
400 return parts;
401 }
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417 public static boolean watchAndCheckExists(ZooKeeperWatcher zkw, String znode)
418 throws KeeperException {
419 try {
420 Stat s = zkw.getRecoverableZooKeeper().exists(znode, zkw);
421 boolean exists = s != null ? true : false;
422 if (exists) {
423 LOG.debug(zkw.prefix("Set watcher on existing znode " + znode));
424 } else {
425 LOG.debug(zkw.prefix(znode+" does not exist. Watcher is set."));
426 }
427 return exists;
428 } catch (KeeperException e) {
429 LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
430 zkw.keeperException(e);
431 return false;
432 } catch (InterruptedException e) {
433 LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
434 zkw.interruptedException(e);
435 return false;
436 }
437 }
438
439
440
441
442
443
444
445
446
447 public static int checkExists(ZooKeeperWatcher zkw, String znode)
448 throws KeeperException {
449 try {
450 Stat s = zkw.getRecoverableZooKeeper().exists(znode, null);
451 return s != null ? s.getVersion() : -1;
452 } catch (KeeperException e) {
453 LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
454 zkw.keeperException(e);
455 return -1;
456 } catch (InterruptedException e) {
457 LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
458 zkw.interruptedException(e);
459 return -1;
460 }
461 }
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483 public static List<String> listChildrenAndWatchForNewChildren(
484 ZooKeeperWatcher zkw, String znode)
485 throws KeeperException {
486 try {
487 List<String> children = zkw.getRecoverableZooKeeper().getChildren(znode, zkw);
488 return children;
489 } catch(KeeperException.NoNodeException ke) {
490 LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
491 "because node does not exist (not an error)"));
492 return null;
493 } catch (KeeperException e) {
494 LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
495 zkw.keeperException(e);
496 return null;
497 } catch (InterruptedException e) {
498 LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
499 zkw.interruptedException(e);
500 return null;
501 }
502 }
503
504
505
506
507
508
509
510
511
512
513 public static List<String> listChildrenAndWatchThem(ZooKeeperWatcher zkw,
514 String znode) throws KeeperException {
515 List<String> children = listChildrenAndWatchForNewChildren(zkw, znode);
516 if (children == null) {
517 return null;
518 }
519 for (String child : children) {
520 watchAndCheckExists(zkw, joinZNode(znode, child));
521 }
522 return children;
523 }
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541 public static List<String> listChildrenNoWatch(
542 ZooKeeperWatcher zkw, String znode)
543 throws KeeperException {
544 List<String> children = null;
545 try {
546
547 children = zkw.getRecoverableZooKeeper().getChildren(znode, null);
548 } catch(KeeperException.NoNodeException nne) {
549 return null;
550 } catch(InterruptedException ie) {
551 zkw.interruptedException(ie);
552 }
553 return children;
554 }
555
556
557
558
559 public static class NodeAndData {
560 private String node;
561 private byte [] data;
562 public NodeAndData(String node, byte [] data) {
563 this.node = node;
564 this.data = data;
565 }
566 public String getNode() {
567 return node;
568 }
569 public byte [] getData() {
570 return data;
571 }
572 @Override
573 public String toString() {
574 return node + " (" + RegionTransitionData.fromBytes(data) + ")";
575 }
576 public boolean isEmpty() {
577 return (data.length == 0);
578 }
579 }
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597 public static boolean nodeHasChildren(ZooKeeperWatcher zkw, String znode)
598 throws KeeperException {
599 try {
600 return !zkw.getRecoverableZooKeeper().getChildren(znode, null).isEmpty();
601 } catch(KeeperException.NoNodeException ke) {
602 LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
603 "because node does not exist (not an error)"));
604 return false;
605 } catch (KeeperException e) {
606 LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
607 zkw.keeperException(e);
608 return false;
609 } catch (InterruptedException e) {
610 LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
611 zkw.interruptedException(e);
612 return false;
613 }
614 }
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629 public static int getNumberOfChildren(ZooKeeperWatcher zkw, String znode)
630 throws KeeperException {
631 try {
632 Stat stat = zkw.getRecoverableZooKeeper().exists(znode, null);
633 return stat == null ? 0 : stat.getNumChildren();
634 } catch(KeeperException e) {
635 LOG.warn(zkw.prefix("Unable to get children of node " + znode));
636 zkw.keeperException(e);
637 } catch(InterruptedException e) {
638 zkw.interruptedException(e);
639 }
640 return 0;
641 }
642
643
644
645
646
647
648
649
650
651 public static byte [] getData(ZooKeeperWatcher zkw, String znode)
652 throws KeeperException {
653 try {
654 byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, null);
655 logRetrievedMsg(zkw, znode, data, false);
656 return data;
657 } catch (KeeperException.NoNodeException e) {
658 LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
659 "because node does not exist (not an error)"));
660 return null;
661 } catch (KeeperException e) {
662 LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
663 zkw.keeperException(e);
664 return null;
665 } catch (InterruptedException e) {
666 LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
667 zkw.interruptedException(e);
668 return null;
669 }
670 }
671
672
673
674
675
676
677
678
679
680
681
682
683 public static byte [] getDataAndWatch(ZooKeeperWatcher zkw, String znode)
684 throws KeeperException {
685 return getDataInternal(zkw, znode, null, true);
686 }
687
688
689
690
691
692
693
694
695
696
697
698
699
700 public static byte[] getDataAndWatch(ZooKeeperWatcher zkw, String znode,
701 Stat stat) throws KeeperException {
702 return getDataInternal(zkw, znode, stat, true);
703 }
704
705 private static byte[] getDataInternal(ZooKeeperWatcher zkw, String znode, Stat stat,
706 boolean watcherSet)
707 throws KeeperException {
708 try {
709 byte [] data = zkw.getRecoverableZooKeeper().getData(znode, zkw, stat);
710 logRetrievedMsg(zkw, znode, data, watcherSet);
711 return data;
712 } catch (KeeperException.NoNodeException e) {
713 LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
714 "because node does not exist (not an error)"));
715 return null;
716 } catch (KeeperException e) {
717 LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
718 zkw.keeperException(e);
719 return null;
720 } catch (InterruptedException e) {
721 LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
722 zkw.interruptedException(e);
723 return null;
724 }
725 }
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742 public static byte [] getDataNoWatch(ZooKeeperWatcher zkw, String znode,
743 Stat stat)
744 throws KeeperException {
745 try {
746 byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, stat);
747 logRetrievedMsg(zkw, znode, data, false);
748 return data;
749 } catch (KeeperException.NoNodeException e) {
750 LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
751 "because node does not exist (not necessarily an error)"));
752 return null;
753 } catch (KeeperException e) {
754 LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
755 zkw.keeperException(e);
756 return null;
757 } catch (InterruptedException e) {
758 LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
759 zkw.interruptedException(e);
760 return null;
761 }
762 }
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780 public static List<NodeAndData> getChildDataAndWatchForNewChildren(
781 ZooKeeperWatcher zkw, String baseNode) throws KeeperException {
782 List<String> nodes =
783 ZKUtil.listChildrenAndWatchForNewChildren(zkw, baseNode);
784 List<NodeAndData> newNodes = new ArrayList<NodeAndData>();
785 if (nodes != null) {
786 for (String node : nodes) {
787 String nodePath = ZKUtil.joinZNode(baseNode, node);
788 byte[] data = ZKUtil.getDataAndWatch(zkw, nodePath);
789 newNodes.add(new NodeAndData(nodePath, data));
790 }
791 }
792 return newNodes;
793 }
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810 public static void updateExistingNodeData(ZooKeeperWatcher zkw, String znode,
811 byte [] data, int expectedVersion)
812 throws KeeperException {
813 try {
814 zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion);
815 } catch(InterruptedException ie) {
816 zkw.interruptedException(ie);
817 }
818 }
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844 public static boolean setData(ZooKeeperWatcher zkw, String znode,
845 byte [] data, int expectedVersion)
846 throws KeeperException, KeeperException.NoNodeException {
847 try {
848 return zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion) != null;
849 } catch (InterruptedException e) {
850 zkw.interruptedException(e);
851 return false;
852 }
853 }
854
855
856
857
858
859
860
861
862
863
864 public static void createSetData(final ZooKeeperWatcher zkw, final String znode,
865 final byte [] data)
866 throws KeeperException {
867 if (checkExists(zkw, znode) == -1) {
868 ZKUtil.createWithParents(zkw, znode, data);
869 } else {
870 ZKUtil.setData(zkw, znode, data);
871 }
872 }
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890 public static void setData(ZooKeeperWatcher zkw, String znode, byte [] data)
891 throws KeeperException, KeeperException.NoNodeException {
892 setData(zkw, (SetData)ZKUtilOp.setData(znode, data));
893 }
894
895 private static void setData(ZooKeeperWatcher zkw, SetData setData)
896 throws KeeperException, KeeperException.NoNodeException {
897 SetDataRequest sd = (SetDataRequest)toZooKeeperOp(zkw, setData).toRequestRecord();
898 setData(zkw, sd.getPath(), sd.getData(), sd.getVersion());
899 }
900
901
902
903
904
905
906 public static boolean isSecureZooKeeper(Configuration conf) {
907
908
909
910 if (System.getProperty("java.security.auth.login.config") != null)
911 return true;
912
913
914 return("kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication")) &&
915 conf.get("hbase.zookeeper.client.keytab.file") != null);
916 }
917
918 private static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node) {
919 if (isSecureZooKeeper(zkw.getConfiguration())) {
920
921
922 if ((node.equals(zkw.baseZNode) == true) ||
923 (node.equals(zkw.rootServerZNode) == true) ||
924 (node.equals(zkw.masterAddressZNode) == true) ||
925 (node.equals(zkw.clusterIdZNode) == true) ||
926 (node.equals(zkw.rsZNode) == true) ||
927 (node.equals(zkw.backupMasterAddressesZNode) == true) ||
928 (node.startsWith(zkw.masterTableZNode) == true) ||
929 (node.startsWith(zkw.masterTableZNode92) == true)) {
930 return ZooKeeperWatcher.CREATOR_ALL_AND_WORLD_READABLE;
931 }
932 return Ids.CREATOR_ALL_ACL;
933 } else {
934 return Ids.OPEN_ACL_UNSAFE;
935 }
936 }
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960 public static boolean createEphemeralNodeAndWatch(ZooKeeperWatcher zkw,
961 String znode, byte [] data)
962 throws KeeperException {
963 boolean ret = true;
964 try {
965 zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
966 CreateMode.EPHEMERAL);
967 } catch (KeeperException.NodeExistsException nee) {
968 ret = false;
969 } catch (InterruptedException e) {
970 LOG.info("Interrupted", e);
971 Thread.currentThread().interrupt();
972 }
973 if(!watchAndCheckExists(zkw, znode)) {
974
975 return createEphemeralNodeAndWatch(zkw, znode, data);
976 }
977 return ret;
978 }
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000 public static boolean createNodeIfNotExistsAndWatch(
1001 ZooKeeperWatcher zkw, String znode, byte [] data)
1002 throws KeeperException {
1003 boolean ret = true;
1004 try {
1005 zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1006 CreateMode.PERSISTENT);
1007 } catch (KeeperException.NodeExistsException nee) {
1008 ret = false;
1009 } catch (InterruptedException e) {
1010 zkw.interruptedException(e);
1011 return false;
1012 }
1013 try {
1014 zkw.getRecoverableZooKeeper().exists(znode, zkw);
1015 } catch (InterruptedException e) {
1016 zkw.interruptedException(e);
1017 return false;
1018 }
1019 return ret;
1020 }
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038 public static int createAndWatch(ZooKeeperWatcher zkw,
1039 String znode, byte [] data)
1040 throws KeeperException, KeeperException.NodeExistsException {
1041 try {
1042 zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1043 CreateMode.PERSISTENT);
1044 return zkw.getRecoverableZooKeeper().exists(znode, zkw).getVersion();
1045 } catch (InterruptedException e) {
1046 zkw.interruptedException(e);
1047 return -1;
1048 }
1049 }
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066 public static void asyncCreate(ZooKeeperWatcher zkw,
1067 String znode, byte [] data, final AsyncCallback.StringCallback cb,
1068 final Object ctx) {
1069 zkw.getRecoverableZooKeeper().getZooKeeper().create(znode, data,
1070 createACL(zkw, znode), CreateMode.PERSISTENT, cb, ctx);
1071 }
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083 public static void createAndFailSilent(ZooKeeperWatcher zkw,
1084 String znode) throws KeeperException {
1085 createAndFailSilent(zkw, znode, new byte[0]);
1086 }
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099 public static void createAndFailSilent(ZooKeeperWatcher zkw,
1100 String znode, byte[] data) throws KeeperException {
1101 createAndFailSilent(zkw,
1102 (CreateAndFailSilent)ZKUtilOp.createAndFailSilent(znode, data));
1103 }
1104
1105 private static void createAndFailSilent(ZooKeeperWatcher zkw, CreateAndFailSilent cafs)
1106 throws KeeperException {
1107 CreateRequest create = (CreateRequest)toZooKeeperOp(zkw, cafs).toRequestRecord();
1108 String znode = create.getPath();
1109 try {
1110 RecoverableZooKeeper zk = zkw.getRecoverableZooKeeper();
1111 if (zk.exists(znode, false) == null) {
1112 zk.create(znode, create.getData(), create.getAcl(), CreateMode.fromFlag(create.getFlags()));
1113 }
1114 } catch(KeeperException.NodeExistsException nee) {
1115 } catch(KeeperException.NoAuthException nee){
1116 try {
1117 if (null == zkw.getRecoverableZooKeeper().exists(znode, false)) {
1118
1119 throw(nee);
1120 }
1121 } catch (InterruptedException ie) {
1122 zkw.interruptedException(ie);
1123 }
1124
1125 } catch(InterruptedException ie) {
1126 zkw.interruptedException(ie);
1127 }
1128 }
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141 public static void createWithParents(ZooKeeperWatcher zkw, String znode)
1142 throws KeeperException {
1143 createWithParents(zkw, znode, new byte[0]);
1144 }
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159 public static void createWithParents(ZooKeeperWatcher zkw, String znode, byte[] data)
1160 throws KeeperException {
1161 try {
1162 if(znode == null) {
1163 return;
1164 }
1165 zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1166 CreateMode.PERSISTENT);
1167 } catch(KeeperException.NodeExistsException nee) {
1168 return;
1169 } catch(KeeperException.NoNodeException nne) {
1170 createWithParents(zkw, getParent(znode));
1171 createWithParents(zkw, znode, data);
1172 } catch(InterruptedException ie) {
1173 zkw.interruptedException(ie);
1174 }
1175 }
1176
1177
1178
1179
1180
1181
1182
1183
1184 public static void deleteNode(ZooKeeperWatcher zkw, String node)
1185 throws KeeperException {
1186 deleteNode(zkw, node, -1);
1187 }
1188
1189
1190
1191
1192
1193 public static boolean deleteNode(ZooKeeperWatcher zkw, String node,
1194 int version)
1195 throws KeeperException {
1196 try {
1197 zkw.getRecoverableZooKeeper().delete(node, version);
1198 return true;
1199 } catch(KeeperException.BadVersionException bve) {
1200 return false;
1201 } catch(InterruptedException ie) {
1202 zkw.interruptedException(ie);
1203 return false;
1204 }
1205 }
1206
1207
1208
1209
1210
1211
1212
1213 public static void deleteNodeFailSilent(ZooKeeperWatcher zkw, String node)
1214 throws KeeperException {
1215 deleteNodeFailSilent(zkw,
1216 (DeleteNodeFailSilent)ZKUtilOp.deleteNodeFailSilent(node));
1217 }
1218
1219 private static void deleteNodeFailSilent(ZooKeeperWatcher zkw,
1220 DeleteNodeFailSilent dnfs) throws KeeperException {
1221 DeleteRequest delete = (DeleteRequest)toZooKeeperOp(zkw, dnfs).toRequestRecord();
1222 try {
1223 zkw.getRecoverableZooKeeper().delete(delete.getPath(), delete.getVersion());
1224 } catch(KeeperException.NoNodeException nne) {
1225 } catch(InterruptedException ie) {
1226 zkw.interruptedException(ie);
1227 }
1228 }
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238 public static void deleteNodeRecursively(ZooKeeperWatcher zkw, String node)
1239 throws KeeperException {
1240 try {
1241 List<String> children = ZKUtil.listChildrenNoWatch(zkw, node);
1242
1243 if (children == null) return;
1244
1245 if(!children.isEmpty()) {
1246 for(String child : children) {
1247 deleteNodeRecursively(zkw, joinZNode(node, child));
1248 }
1249 }
1250 zkw.getRecoverableZooKeeper().delete(node, -1);
1251 } catch(InterruptedException ie) {
1252 zkw.interruptedException(ie);
1253 }
1254 }
1255
1256
1257
1258
1259
1260
1261
1262 public static void deleteChildrenRecursively(ZooKeeperWatcher zkw, String node)
1263 throws KeeperException {
1264 List<String> children = ZKUtil.listChildrenNoWatch(zkw, node);
1265 if (children == null || children.isEmpty()) return;
1266 for(String child : children) {
1267 deleteNodeRecursively(zkw, joinZNode(node, child));
1268 }
1269 }
1270
1271
1272
1273
1274
1275
1276 public abstract static class ZKUtilOp {
1277 private String path;
1278
1279 private ZKUtilOp(String path) {
1280 this.path = path;
1281 }
1282
1283
1284
1285
1286 public static ZKUtilOp createAndFailSilent(String path, byte[] data) {
1287 return new CreateAndFailSilent(path, data);
1288 }
1289
1290
1291
1292
1293 public static ZKUtilOp deleteNodeFailSilent(String path) {
1294 return new DeleteNodeFailSilent(path);
1295 }
1296
1297
1298
1299
1300 public static ZKUtilOp setData(String path, byte [] data) {
1301 return new SetData(path, data);
1302 }
1303
1304
1305
1306
1307 public String getPath() {
1308 return path;
1309 }
1310
1311
1312
1313
1314
1315 public static class CreateAndFailSilent extends ZKUtilOp {
1316 private byte [] data;
1317
1318 private CreateAndFailSilent(String path, byte [] data) {
1319 super(path);
1320 this.data = data;
1321 }
1322
1323 public byte[] getData() {
1324 return data;
1325 }
1326
1327 @Override
1328 public boolean equals(Object o) {
1329 if (this == o) return true;
1330 if (!(o instanceof CreateAndFailSilent)) return false;
1331
1332 CreateAndFailSilent op = (CreateAndFailSilent) o;
1333 return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1334 }
1335 }
1336
1337
1338
1339
1340
1341 public static class DeleteNodeFailSilent extends ZKUtilOp {
1342 private DeleteNodeFailSilent(String path) {
1343 super(path);
1344 }
1345
1346 @Override
1347 public boolean equals(Object o) {
1348 if (this == o) return true;
1349 if (!(o instanceof DeleteNodeFailSilent)) return false;
1350
1351 return super.equals(o);
1352 }
1353 }
1354
1355
1356
1357
1358 public static class SetData extends ZKUtilOp {
1359 private byte [] data;
1360
1361 private SetData(String path, byte [] data) {
1362 super(path);
1363 this.data = data;
1364 }
1365
1366 public byte[] getData() {
1367 return data;
1368 }
1369
1370 @Override
1371 public boolean equals(Object o) {
1372 if (this == o) return true;
1373 if (!(o instanceof SetData)) return false;
1374
1375 SetData op = (SetData) o;
1376 return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1377 }
1378 }
1379 }
1380
1381
1382
1383
1384 private static Op toZooKeeperOp(ZooKeeperWatcher zkw, ZKUtilOp op)
1385 throws UnsupportedOperationException {
1386 if(op == null) return null;
1387
1388 if (op instanceof CreateAndFailSilent) {
1389 CreateAndFailSilent cafs = (CreateAndFailSilent)op;
1390 return Op.create(cafs.getPath(), cafs.getData(), createACL(zkw, cafs.getPath()),
1391 CreateMode.PERSISTENT);
1392 } else if (op instanceof DeleteNodeFailSilent) {
1393 DeleteNodeFailSilent dnfs = (DeleteNodeFailSilent)op;
1394 return Op.delete(dnfs.getPath(), -1);
1395 } else if (op instanceof SetData) {
1396 SetData sd = (SetData)op;
1397 return Op.setData(sd.getPath(), sd.getData(), -1);
1398 } else {
1399 throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1400 + op.getClass().getName());
1401 }
1402 }
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425 public static void multiOrSequential(ZooKeeperWatcher zkw, List<ZKUtilOp> ops,
1426 boolean runSequentialOnMultiFailure) throws KeeperException {
1427 if (ops == null) return;
1428 boolean useMulti = zkw.getConfiguration().getBoolean(HConstants.ZOOKEEPER_USEMULTI, false);
1429
1430 if (useMulti) {
1431 List<Op> zkOps = new LinkedList<Op>();
1432 for (ZKUtilOp op : ops) {
1433 zkOps.add(toZooKeeperOp(zkw, op));
1434 }
1435 try {
1436 zkw.getRecoverableZooKeeper().multi(zkOps);
1437 } catch (KeeperException ke) {
1438 switch (ke.code()) {
1439 case NODEEXISTS:
1440 case NONODE:
1441 case BADVERSION:
1442 case NOAUTH:
1443
1444
1445 if (runSequentialOnMultiFailure) {
1446 LOG.info("On call to ZK.multi, received exception: " + ke.toString() + "."
1447 + " Attempting to run operations sequentially because"
1448 + " runSequentialOnMultiFailure is: " + runSequentialOnMultiFailure + ".");
1449 processSequentially(zkw, ops);
1450 break;
1451 }
1452 default:
1453 throw ke;
1454 }
1455 } catch (InterruptedException ie) {
1456 zkw.interruptedException(ie);
1457 }
1458 } else {
1459
1460 processSequentially(zkw, ops);
1461 }
1462 }
1463
1464 private static void processSequentially(ZooKeeperWatcher zkw, List<ZKUtilOp> ops)
1465 throws KeeperException, NoNodeException {
1466 for (ZKUtilOp op : ops) {
1467 if (op instanceof CreateAndFailSilent) {
1468 createAndFailSilent(zkw, (CreateAndFailSilent) op);
1469 } else if (op instanceof DeleteNodeFailSilent) {
1470 deleteNodeFailSilent(zkw, (DeleteNodeFailSilent) op);
1471 } else if (op instanceof SetData) {
1472 setData(zkw, (SetData) op);
1473 } else {
1474 throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1475 + op.getClass().getName());
1476 }
1477 }
1478 }
1479
1480
1481
1482
1483
1484
1485 public static String dump(ZooKeeperWatcher zkw) {
1486 StringBuilder sb = new StringBuilder();
1487 try {
1488 sb.append("HBase is rooted at ").append(zkw.baseZNode);
1489 sb.append("\nActive master address: ").append(
1490 ServerName.parseVersionedServerName(getData(zkw, zkw.masterAddressZNode)));
1491 sb.append("\nBackup master addresses:");
1492 for (String child : listChildrenNoWatch(zkw,
1493 zkw.backupMasterAddressesZNode)) {
1494 sb.append("\n ").append(child);
1495 }
1496 sb.append("\nRegion server holding ROOT: ").append(
1497 Bytes.toStringBinary(getData(zkw, zkw.rootServerZNode)));
1498 sb.append("\nRegion servers:");
1499 for (String child : listChildrenNoWatch(zkw, zkw.rsZNode)) {
1500 sb.append("\n ").append(child);
1501 }
1502 try {
1503 getReplicationZnodesDump(zkw, sb);
1504 } catch (KeeperException ke) {
1505 LOG.warn("Couldn't get the replication znode dump." + ke.getStackTrace());
1506 }
1507 sb.append("\nQuorum Server Statistics:");
1508 String[] servers = zkw.getQuorum().split(",");
1509 for (String server : servers) {
1510 sb.append("\n ").append(server);
1511 try {
1512 String[] stat = getServerStats(server, ZKUtil.zkDumpConnectionTimeOut);
1513
1514 if (stat == null) {
1515 sb.append("[Error] invalid quorum server: " + server);
1516 break;
1517 }
1518
1519 for (String s : stat) {
1520 sb.append("\n ").append(s);
1521 }
1522 } catch (Exception e) {
1523 sb.append("\n ERROR: ").append(e.getMessage());
1524 }
1525 }
1526 } catch (KeeperException ke) {
1527 sb.append("\nFATAL ZooKeeper Exception!\n");
1528 sb.append("\n" + ke.getMessage());
1529 }
1530 return sb.toString();
1531 }
1532
1533 private static void getReplicationZnodesDump(ZooKeeperWatcher zkw, StringBuilder sb)
1534 throws KeeperException {
1535 String replicationZNodeName = zkw.getConfiguration().get("zookeeper.znode.replication",
1536 "replication");
1537 String replicationZnode = joinZNode(zkw.baseZNode, replicationZNodeName);
1538 if (ZKUtil.checkExists(zkw, replicationZnode) == -1)
1539 return;
1540
1541 List<String> stack = new LinkedList<String>();
1542 stack.add(replicationZnode);
1543 do {
1544 String znodeToProcess = stack.remove(stack.size() - 1);
1545 sb.append("\n").append(znodeToProcess).append(": ")
1546 .append(Bytes.toString(ZKUtil.getData(zkw, znodeToProcess)));
1547 for (String zNodeChild : ZKUtil.listChildrenNoWatch(zkw, znodeToProcess)) {
1548 stack.add(ZKUtil.joinZNode(znodeToProcess, zNodeChild));
1549 }
1550 } while (stack.size() > 0);
1551 }
1552
1553
1554
1555
1556
1557
1558
1559
1560 public static String[] getServerStats(String server, int timeout)
1561 throws IOException {
1562 String[] sp = server.split(":");
1563 if (sp == null || sp.length == 0) {
1564 return null;
1565 }
1566
1567 String host = sp[0];
1568 int port = sp.length > 1 ? Integer.parseInt(sp[1])
1569 : HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT;
1570
1571 Socket socket = new Socket();
1572 InetSocketAddress sockAddr = new InetSocketAddress(host, port);
1573 socket.connect(sockAddr, timeout);
1574
1575 socket.setSoTimeout(timeout);
1576 PrintWriter out = new PrintWriter(socket.getOutputStream(), true);
1577 BufferedReader in = new BufferedReader(new InputStreamReader(
1578 socket.getInputStream()));
1579 out.println("stat");
1580 out.flush();
1581 ArrayList<String> res = new ArrayList<String>();
1582 while (true) {
1583 String line = in.readLine();
1584 if (line != null) {
1585 res.add(line);
1586 } else {
1587 break;
1588 }
1589 }
1590 socket.close();
1591 return res.toArray(new String[res.size()]);
1592 }
1593
1594 private static void logRetrievedMsg(final ZooKeeperWatcher zkw,
1595 final String znode, final byte [] data, final boolean watcherSet) {
1596 if (!LOG.isDebugEnabled()) return;
1597 LOG.debug(zkw.prefix("Retrieved " + ((data == null)? 0: data.length) +
1598 " byte(s) of data from znode " + znode +
1599 (watcherSet? " and set watcher; ": "; data=") +
1600 (data == null? "null": data.length == 0? "empty": (
1601 znode.startsWith(zkw.assignmentZNode) ?
1602 RegionTransitionData.fromBytes(data).toString()
1603 : StringUtils.abbreviate(Bytes.toStringBinary(data), 32)))));
1604 }
1605
1606
1607
1608
1609
1610 public static void waitForBaseZNode(Configuration conf) throws IOException {
1611 LOG.info("Waiting until the base znode is available");
1612 String parentZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
1613 HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
1614 ZooKeeper zk = new ZooKeeper(ZKConfig.getZKQuorumServersString(conf),
1615 conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
1616 HConstants.DEFAULT_ZK_SESSION_TIMEOUT), EmptyWatcher.instance);
1617
1618 final int maxTimeMs = 10000;
1619 final int maxNumAttempts = maxTimeMs / HConstants.SOCKET_RETRY_WAIT_MS;
1620
1621 KeeperException keeperEx = null;
1622 try {
1623 try {
1624 for (int attempt = 0; attempt < maxNumAttempts; ++attempt) {
1625 try {
1626 if (zk.exists(parentZNode, false) != null) {
1627 LOG.info("Parent znode exists: " + parentZNode);
1628 keeperEx = null;
1629 break;
1630 }
1631 } catch (KeeperException e) {
1632 keeperEx = e;
1633 }
1634 Threads.sleepWithoutInterrupt(HConstants.SOCKET_RETRY_WAIT_MS);
1635 }
1636 } finally {
1637 zk.close();
1638 }
1639 } catch (InterruptedException ex) {
1640 Thread.currentThread().interrupt();
1641 }
1642
1643 if (keeperEx != null) {
1644 throw new IOException(keeperEx);
1645 }
1646 }
1647
1648
1649
1650
1651
1652
1653 public static void logZKTree(ZooKeeperWatcher zkw, String root) {
1654 if (!LOG.isDebugEnabled()) return;
1655 LOG.debug("Current zk system:");
1656 String prefix = "|-";
1657 LOG.debug(prefix + root);
1658 try {
1659 logZKTree(zkw, root, prefix);
1660 } catch (KeeperException e) {
1661 throw new RuntimeException(e);
1662 }
1663 }
1664
1665
1666
1667
1668
1669
1670 protected static void logZKTree(ZooKeeperWatcher zkw, String root, String prefix) throws KeeperException {
1671 List<String> children = ZKUtil.listChildrenNoWatch(zkw, root);
1672 if (children == null) return;
1673 for (String child : children) {
1674 LOG.debug(prefix + child);
1675 String node = ZKUtil.joinZNode(root.equals("/") ? "" : root, child);
1676 logZKTree(zkw, node, prefix + "---");
1677 }
1678 }
1679
1680 }