1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.zookeeper;
21
22 import java.io.BufferedReader;
23 import java.io.IOException;
24 import java.io.InputStreamReader;
25 import java.io.PrintWriter;
26 import java.net.InetSocketAddress;
27 import java.net.InetAddress;
28 import java.net.Socket;
29 import java.util.ArrayList;
30 import java.util.Arrays;
31 import java.util.LinkedList;
32 import java.util.List;
33 import java.util.Properties;
34 import java.util.HashMap;
35 import java.util.Map;
36
37 import javax.security.auth.login.LoginException;
38 import javax.security.auth.login.AppConfigurationEntry;
39 import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
40 import org.apache.hadoop.security.SecurityUtil;
41 import org.apache.hadoop.security.authentication.util.KerberosUtil;
42
43 import org.apache.commons.lang.StringUtils;
44 import org.apache.commons.logging.Log;
45 import org.apache.commons.logging.LogFactory;
46 import org.apache.hadoop.conf.Configuration;
47 import org.apache.hadoop.hbase.EmptyWatcher;
48 import org.apache.hadoop.hbase.HConstants;
49 import org.apache.hadoop.hbase.ServerName;
50 import org.apache.hadoop.hbase.executor.RegionTransitionData;
51 import org.apache.hadoop.hbase.util.Bytes;
52 import org.apache.hadoop.hbase.util.Threads;
53 import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.CreateAndFailSilent;
54 import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.DeleteNodeFailSilent;
55 import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.SetData;
56 import org.apache.zookeeper.AsyncCallback;
57 import org.apache.zookeeper.CreateMode;
58 import org.apache.zookeeper.KeeperException;
59 import org.apache.zookeeper.KeeperException.NoNodeException;
60 import org.apache.zookeeper.Op;
61 import org.apache.zookeeper.Watcher;
62 import org.apache.zookeeper.ZooDefs.Ids;
63 import org.apache.zookeeper.ZooKeeper;
64 import org.apache.zookeeper.client.ZooKeeperSaslClient;
65 import org.apache.zookeeper.data.ACL;
66 import org.apache.zookeeper.data.Stat;
67 import org.apache.zookeeper.server.ZooKeeperSaslServer;
68 import org.apache.zookeeper.proto.CreateRequest;
69 import org.apache.zookeeper.proto.DeleteRequest;
70 import org.apache.zookeeper.proto.SetDataRequest;
71
72
73
74
75
76
77
78
79
80
81 public class ZKUtil {
82 private static final Log LOG = LogFactory.getLog(ZKUtil.class);
83
84
85 private static final char ZNODE_PATH_SEPARATOR = '/';
86 private static int zkDumpConnectionTimeOut;
87
88
89
90
91
92
93
94
95
96
97
98
99 public static RecoverableZooKeeper connect(Configuration conf, Watcher watcher)
100 throws IOException {
101 Properties properties = ZKConfig.makeZKProps(conf);
102 String ensemble = ZKConfig.getZKQuorumServersString(properties);
103 return connect(conf, ensemble, watcher);
104 }
105
106 public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
107 Watcher watcher)
108 throws IOException {
109 return connect(conf, ensemble, watcher, "");
110 }
111
112 public static RecoverableZooKeeper connect(Configuration conf, String ensemble,
113 Watcher watcher, final String descriptor)
114 throws IOException {
115 if(ensemble == null) {
116 throw new IOException("Unable to determine ZooKeeper ensemble");
117 }
118 int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
119 HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
120 LOG.debug(descriptor + " opening connection to ZooKeeper with ensemble (" +
121 ensemble + ")");
122 int retry = conf.getInt("zookeeper.recovery.retry", 3);
123 int retryIntervalMillis =
124 conf.getInt("zookeeper.recovery.retry.intervalmill", 1000);
125 zkDumpConnectionTimeOut = conf.getInt("zookeeper.dump.connection.timeout",
126 1000);
127 return new RecoverableZooKeeper(ensemble, timeout, watcher,
128 retry, retryIntervalMillis);
129 }
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145 public static void loginServer(Configuration conf, String keytabFileKey,
146 String userNameKey, String hostname) throws IOException {
147 login(conf, keytabFileKey, userNameKey, hostname,
148 ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY,
149 JaasConfiguration.SERVER_KEYTAB_KERBEROS_CONFIG_NAME);
150 }
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166 public static void loginClient(Configuration conf, String keytabFileKey,
167 String userNameKey, String hostname) throws IOException {
168 login(conf, keytabFileKey, userNameKey, hostname,
169 ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
170 JaasConfiguration.CLIENT_KEYTAB_KERBEROS_CONFIG_NAME);
171 }
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189 private static void login(Configuration conf, String keytabFileKey,
190 String userNameKey, String hostname,
191 String loginContextProperty, String loginContextName)
192 throws IOException {
193 if (!isSecureZooKeeper(conf))
194 return;
195
196
197
198 if (System.getProperty("java.security.auth.login.config") != null)
199 return;
200
201
202 String keytabFilename = conf.get(keytabFileKey);
203 if (keytabFilename == null) {
204 LOG.warn("no keytab specified for: " + keytabFileKey);
205 return;
206 }
207
208 String principalConfig = conf.get(userNameKey, System.getProperty("user.name"));
209 String principalName = SecurityUtil.getServerPrincipal(principalConfig, hostname);
210
211
212
213
214 JaasConfiguration jaasConf = new JaasConfiguration(loginContextName,
215 principalName, keytabFilename);
216 javax.security.auth.login.Configuration.setConfiguration(jaasConf);
217 System.setProperty(loginContextProperty, loginContextName);
218 }
219
220
221
222
223 private static class JaasConfiguration extends javax.security.auth.login.Configuration {
224 private static final String SERVER_KEYTAB_KERBEROS_CONFIG_NAME =
225 "zookeeper-server-keytab-kerberos";
226 private static final String CLIENT_KEYTAB_KERBEROS_CONFIG_NAME =
227 "zookeeper-client-keytab-kerberos";
228
229 private static final Map<String, String> BASIC_JAAS_OPTIONS =
230 new HashMap<String,String>();
231 static {
232 String jaasEnvVar = System.getenv("HBASE_JAAS_DEBUG");
233 if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) {
234 BASIC_JAAS_OPTIONS.put("debug", "true");
235 }
236 }
237
238 private static final Map<String,String> KEYTAB_KERBEROS_OPTIONS =
239 new HashMap<String,String>();
240 static {
241 KEYTAB_KERBEROS_OPTIONS.put("doNotPrompt", "true");
242 KEYTAB_KERBEROS_OPTIONS.put("storeKey", "true");
243 KEYTAB_KERBEROS_OPTIONS.put("refreshKrb5Config", "true");
244 KEYTAB_KERBEROS_OPTIONS.putAll(BASIC_JAAS_OPTIONS);
245 }
246
247 private static final AppConfigurationEntry KEYTAB_KERBEROS_LOGIN =
248 new AppConfigurationEntry(KerberosUtil.getKrb5LoginModuleName(),
249 LoginModuleControlFlag.REQUIRED,
250 KEYTAB_KERBEROS_OPTIONS);
251
252 private static final AppConfigurationEntry[] KEYTAB_KERBEROS_CONF =
253 new AppConfigurationEntry[]{KEYTAB_KERBEROS_LOGIN};
254
255 private javax.security.auth.login.Configuration baseConfig;
256 private final String loginContextName;
257 private final boolean useTicketCache;
258 private final String keytabFile;
259 private final String principal;
260
261 public JaasConfiguration(String loginContextName, String principal) {
262 this(loginContextName, principal, null, true);
263 }
264
265 public JaasConfiguration(String loginContextName, String principal, String keytabFile) {
266 this(loginContextName, principal, keytabFile, keytabFile == null || keytabFile.length() == 0);
267 }
268
269 private JaasConfiguration(String loginContextName, String principal,
270 String keytabFile, boolean useTicketCache) {
271 try {
272 this.baseConfig = javax.security.auth.login.Configuration.getConfiguration();
273 } catch (SecurityException e) {
274 this.baseConfig = null;
275 }
276 this.loginContextName = loginContextName;
277 this.useTicketCache = useTicketCache;
278 this.keytabFile = keytabFile;
279 this.principal = principal;
280 LOG.info("JaasConfiguration loginContextName=" + loginContextName +
281 " principal=" + principal + " useTicketCache=" + useTicketCache +
282 " keytabFile=" + keytabFile);
283 }
284
285 @Override
286 public AppConfigurationEntry[] getAppConfigurationEntry(String appName) {
287 if (loginContextName.equals(appName)) {
288 if (!useTicketCache) {
289 KEYTAB_KERBEROS_OPTIONS.put("keyTab", keytabFile);
290 KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true");
291 }
292 KEYTAB_KERBEROS_OPTIONS.put("principal", principal);
293 KEYTAB_KERBEROS_OPTIONS.put("useTicketCache", useTicketCache ? "true" : "false");
294 return KEYTAB_KERBEROS_CONF;
295 }
296 if (baseConfig != null) return baseConfig.getAppConfigurationEntry(appName);
297 return(null);
298 }
299 }
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315 public static String joinZNode(String prefix, String suffix) {
316 return prefix + ZNODE_PATH_SEPARATOR + suffix;
317 }
318
319
320
321
322
323
324 public static String getParent(String node) {
325 int idx = node.lastIndexOf(ZNODE_PATH_SEPARATOR);
326 return idx <= 0 ? null : node.substring(0, idx);
327 }
328
329
330
331
332
333
334 public static String getNodeName(String path) {
335 return path.substring(path.lastIndexOf("/")+1);
336 }
337
338
339
340
341
342
343
344 public static String getZooKeeperClusterKey(Configuration conf) {
345 return getZooKeeperClusterKey(conf, null);
346 }
347
348
349
350
351
352
353
354
355 public static String getZooKeeperClusterKey(Configuration conf, String name) {
356 String ensemble = conf.get(HConstants.ZOOKEEPER_QUORUM.replaceAll(
357 "[\\t\\n\\x0B\\f\\r]", ""));
358 StringBuilder builder = new StringBuilder(ensemble);
359 builder.append(":");
360 builder.append(conf.get(HConstants.ZOOKEEPER_CLIENT_PORT));
361 builder.append(":");
362 builder.append(conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT));
363 if (name != null && !name.isEmpty()) {
364 builder.append(",");
365 builder.append(name);
366 }
367 return builder.toString();
368 }
369
370
371
372
373
374
375
376
377 public static void applyClusterKeyToConf(Configuration conf, String key)
378 throws IOException{
379 String[] parts = transformClusterKey(key);
380 conf.set(HConstants.ZOOKEEPER_QUORUM, parts[0]);
381 conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, parts[1]);
382 conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parts[2]);
383 }
384
385
386
387
388
389
390
391
392
393 public static String[] transformClusterKey(String key) throws IOException {
394 String[] parts = key.split(":");
395 if (parts.length != 3) {
396 throw new IOException("Cluster key invalid, the format should be:" +
397 HConstants.ZOOKEEPER_QUORUM + ":hbase.zookeeper.client.port:"
398 + HConstants.ZOOKEEPER_ZNODE_PARENT);
399 }
400 return parts;
401 }
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417 public static boolean watchAndCheckExists(ZooKeeperWatcher zkw, String znode)
418 throws KeeperException {
419 try {
420 Stat s = zkw.getRecoverableZooKeeper().exists(znode, zkw);
421 boolean exists = s != null ? true : false;
422 if (exists) {
423 LOG.debug(zkw.prefix("Set watcher on existing znode " + znode));
424 } else {
425 LOG.debug(zkw.prefix(znode+" does not exist. Watcher is set."));
426 }
427 return exists;
428 } catch (KeeperException e) {
429 LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
430 zkw.keeperException(e);
431 return false;
432 } catch (InterruptedException e) {
433 LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e);
434 zkw.interruptedException(e);
435 return false;
436 }
437 }
438
439
440
441
442
443
444
445
446
447 public static int checkExists(ZooKeeperWatcher zkw, String znode)
448 throws KeeperException {
449 try {
450 Stat s = zkw.getRecoverableZooKeeper().exists(znode, null);
451 return s != null ? s.getVersion() : -1;
452 } catch (KeeperException e) {
453 LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
454 zkw.keeperException(e);
455 return -1;
456 } catch (InterruptedException e) {
457 LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e);
458 zkw.interruptedException(e);
459 return -1;
460 }
461 }
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483 public static List<String> listChildrenAndWatchForNewChildren(
484 ZooKeeperWatcher zkw, String znode)
485 throws KeeperException {
486 try {
487 List<String> children = zkw.getRecoverableZooKeeper().getChildren(znode, zkw);
488 return children;
489 } catch(KeeperException.NoNodeException ke) {
490 LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
491 "because node does not exist (not an error)"));
492 return null;
493 } catch (KeeperException e) {
494 LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
495 zkw.keeperException(e);
496 return null;
497 } catch (InterruptedException e) {
498 LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e);
499 zkw.interruptedException(e);
500 return null;
501 }
502 }
503
504
505
506
507
508
509
510
511
512
513 public static List<String> listChildrenAndWatchThem(ZooKeeperWatcher zkw,
514 String znode) throws KeeperException {
515 List<String> children = listChildrenAndWatchForNewChildren(zkw, znode);
516 if (children == null) {
517 return null;
518 }
519 for (String child : children) {
520 watchAndCheckExists(zkw, joinZNode(znode, child));
521 }
522 return children;
523 }
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541 public static List<String> listChildrenNoWatch(
542 ZooKeeperWatcher zkw, String znode)
543 throws KeeperException {
544 List<String> children = null;
545 try {
546
547 children = zkw.getRecoverableZooKeeper().getChildren(znode, null);
548 } catch(KeeperException.NoNodeException nne) {
549 return null;
550 } catch(InterruptedException ie) {
551 zkw.interruptedException(ie);
552 }
553 return children;
554 }
555
556
557
558
559 public static class NodeAndData {
560 private String node;
561 private byte [] data;
562 public NodeAndData(String node, byte [] data) {
563 this.node = node;
564 this.data = data;
565 }
566 public String getNode() {
567 return node;
568 }
569 public byte [] getData() {
570 return data;
571 }
572 @Override
573 public String toString() {
574 return node + " (" + RegionTransitionData.fromBytes(data) + ")";
575 }
576 public boolean isEmpty() {
577 return (data.length == 0);
578 }
579 }
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597 public static boolean nodeHasChildren(ZooKeeperWatcher zkw, String znode)
598 throws KeeperException {
599 try {
600 return !zkw.getRecoverableZooKeeper().getChildren(znode, null).isEmpty();
601 } catch(KeeperException.NoNodeException ke) {
602 LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " +
603 "because node does not exist (not an error)"));
604 return false;
605 } catch (KeeperException e) {
606 LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
607 zkw.keeperException(e);
608 return false;
609 } catch (InterruptedException e) {
610 LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e);
611 zkw.interruptedException(e);
612 return false;
613 }
614 }
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629 public static int getNumberOfChildren(ZooKeeperWatcher zkw, String znode)
630 throws KeeperException {
631 try {
632 Stat stat = zkw.getRecoverableZooKeeper().exists(znode, null);
633 return stat == null ? 0 : stat.getNumChildren();
634 } catch(KeeperException e) {
635 LOG.warn(zkw.prefix("Unable to get children of node " + znode));
636 zkw.keeperException(e);
637 } catch(InterruptedException e) {
638 zkw.interruptedException(e);
639 }
640 return 0;
641 }
642
643
644
645
646
647
648
649
650
651 public static byte [] getData(ZooKeeperWatcher zkw, String znode)
652 throws KeeperException {
653 try {
654 byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, null);
655 logRetrievedMsg(zkw, znode, data, false);
656 return data;
657 } catch (KeeperException.NoNodeException e) {
658 LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
659 "because node does not exist (not an error)"));
660 return null;
661 } catch (KeeperException e) {
662 LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
663 zkw.keeperException(e);
664 return null;
665 } catch (InterruptedException e) {
666 LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
667 zkw.interruptedException(e);
668 return null;
669 }
670 }
671
672
673
674
675
676
677
678
679
680
681
682
683 public static byte [] getDataAndWatch(ZooKeeperWatcher zkw, String znode)
684 throws KeeperException {
685 return getDataInternal(zkw, znode, null, true);
686 }
687
688
689
690
691
692
693
694
695
696
697
698
699
700 public static byte[] getDataAndWatch(ZooKeeperWatcher zkw, String znode,
701 Stat stat) throws KeeperException {
702 return getDataInternal(zkw, znode, stat, true);
703 }
704
705 private static byte[] getDataInternal(ZooKeeperWatcher zkw, String znode, Stat stat,
706 boolean watcherSet)
707 throws KeeperException {
708 try {
709 byte [] data = zkw.getRecoverableZooKeeper().getData(znode, zkw, stat);
710 logRetrievedMsg(zkw, znode, data, watcherSet);
711 return data;
712 } catch (KeeperException.NoNodeException e) {
713 LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
714 "because node does not exist (not an error)"));
715 return null;
716 } catch (KeeperException e) {
717 LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
718 zkw.keeperException(e);
719 return null;
720 } catch (InterruptedException e) {
721 LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
722 zkw.interruptedException(e);
723 return null;
724 }
725 }
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742 public static byte [] getDataNoWatch(ZooKeeperWatcher zkw, String znode,
743 Stat stat)
744 throws KeeperException {
745 try {
746 byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, stat);
747 logRetrievedMsg(zkw, znode, data, false);
748 return data;
749 } catch (KeeperException.NoNodeException e) {
750 LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " +
751 "because node does not exist (not necessarily an error)"));
752 return null;
753 } catch (KeeperException e) {
754 LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
755 zkw.keeperException(e);
756 return null;
757 } catch (InterruptedException e) {
758 LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e);
759 zkw.interruptedException(e);
760 return null;
761 }
762 }
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780 public static List<NodeAndData> getChildDataAndWatchForNewChildren(
781 ZooKeeperWatcher zkw, String baseNode) throws KeeperException {
782 List<String> nodes =
783 ZKUtil.listChildrenAndWatchForNewChildren(zkw, baseNode);
784 List<NodeAndData> newNodes = new ArrayList<NodeAndData>();
785 if (nodes != null) {
786 for (String node : nodes) {
787 String nodePath = ZKUtil.joinZNode(baseNode, node);
788 byte[] data = ZKUtil.getDataAndWatch(zkw, nodePath);
789 newNodes.add(new NodeAndData(nodePath, data));
790 }
791 }
792 return newNodes;
793 }
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810 public static void updateExistingNodeData(ZooKeeperWatcher zkw, String znode,
811 byte [] data, int expectedVersion)
812 throws KeeperException {
813 try {
814 zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion);
815 } catch(InterruptedException ie) {
816 zkw.interruptedException(ie);
817 }
818 }
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844 public static boolean setData(ZooKeeperWatcher zkw, String znode,
845 byte [] data, int expectedVersion)
846 throws KeeperException, KeeperException.NoNodeException {
847 try {
848 return zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion) != null;
849 } catch (InterruptedException e) {
850 zkw.interruptedException(e);
851 return false;
852 }
853 }
854
855
856
857
858
859
860
861
862
863
864 public static void createSetData(final ZooKeeperWatcher zkw, final String znode,
865 final byte [] data)
866 throws KeeperException {
867 if (checkExists(zkw, znode) == -1) {
868 ZKUtil.createWithParents(zkw, znode, data);
869 } else {
870 ZKUtil.setData(zkw, znode, data);
871 }
872 }
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890 public static void setData(ZooKeeperWatcher zkw, String znode, byte [] data)
891 throws KeeperException, KeeperException.NoNodeException {
892 setData(zkw, (SetData)ZKUtilOp.setData(znode, data));
893 }
894
895 private static void setData(ZooKeeperWatcher zkw, SetData setData)
896 throws KeeperException, KeeperException.NoNodeException {
897 SetDataRequest sd = (SetDataRequest)toZooKeeperOp(zkw, setData).toRequestRecord();
898 setData(zkw, sd.getPath(), sd.getData(), sd.getVersion());
899 }
900
901
902
903
904
905
906 public static boolean isSecureZooKeeper(Configuration conf) {
907
908
909
910 if (System.getProperty("java.security.auth.login.config") != null)
911 return true;
912
913
914 return("kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication")) &&
915 conf.get("hbase.zookeeper.client.keytab.file") != null);
916 }
917
918 private static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node) {
919 if (isSecureZooKeeper(zkw.getConfiguration())) {
920
921
922 if ((node.equals(zkw.baseZNode) == true) ||
923 (node.equals(zkw.rootServerZNode) == true) ||
924 (node.equals(zkw.masterAddressZNode) == true) ||
925 (node.equals(zkw.clusterIdZNode) == true) ||
926 (node.equals(zkw.rsZNode) == true) ||
927 (node.equals(zkw.backupMasterAddressesZNode) == true) ||
928 (node.startsWith(zkw.masterTableZNode) == true) ||
929 (node.startsWith(zkw.masterTableZNode92) == true)) {
930 return ZooKeeperWatcher.CREATOR_ALL_AND_WORLD_READABLE;
931 }
932 return Ids.CREATOR_ALL_ACL;
933 } else {
934 return Ids.OPEN_ACL_UNSAFE;
935 }
936 }
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960 public static boolean createEphemeralNodeAndWatch(ZooKeeperWatcher zkw,
961 String znode, byte [] data)
962 throws KeeperException {
963 try {
964 zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
965 CreateMode.EPHEMERAL);
966 } catch (KeeperException.NodeExistsException nee) {
967 if(!watchAndCheckExists(zkw, znode)) {
968
969 return createEphemeralNodeAndWatch(zkw, znode, data);
970 }
971 return false;
972 } catch (InterruptedException e) {
973 LOG.info("Interrupted", e);
974 Thread.currentThread().interrupt();
975 }
976 return true;
977 }
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999 public static boolean createNodeIfNotExistsAndWatch(
1000 ZooKeeperWatcher zkw, String znode, byte [] data)
1001 throws KeeperException {
1002 try {
1003 zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1004 CreateMode.PERSISTENT);
1005 } catch (KeeperException.NodeExistsException nee) {
1006 try {
1007 zkw.getRecoverableZooKeeper().exists(znode, zkw);
1008 } catch (InterruptedException e) {
1009 zkw.interruptedException(e);
1010 return false;
1011 }
1012 return false;
1013 } catch (InterruptedException e) {
1014 zkw.interruptedException(e);
1015 return false;
1016 }
1017 return true;
1018 }
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036 public static int createAndWatch(ZooKeeperWatcher zkw,
1037 String znode, byte [] data)
1038 throws KeeperException, KeeperException.NodeExistsException {
1039 try {
1040 zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1041 CreateMode.PERSISTENT);
1042 return zkw.getRecoverableZooKeeper().exists(znode, zkw).getVersion();
1043 } catch (InterruptedException e) {
1044 zkw.interruptedException(e);
1045 return -1;
1046 }
1047 }
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064 public static void asyncCreate(ZooKeeperWatcher zkw,
1065 String znode, byte [] data, final AsyncCallback.StringCallback cb,
1066 final Object ctx) {
1067 zkw.getRecoverableZooKeeper().getZooKeeper().create(znode, data,
1068 createACL(zkw, znode), CreateMode.PERSISTENT, cb, ctx);
1069 }
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081 public static void createAndFailSilent(ZooKeeperWatcher zkw,
1082 String znode) throws KeeperException {
1083 createAndFailSilent(zkw, znode, new byte[0]);
1084 }
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097 public static void createAndFailSilent(ZooKeeperWatcher zkw,
1098 String znode, byte[] data) throws KeeperException {
1099 createAndFailSilent(zkw,
1100 (CreateAndFailSilent)ZKUtilOp.createAndFailSilent(znode, data));
1101 }
1102
1103 private static void createAndFailSilent(ZooKeeperWatcher zkw, CreateAndFailSilent cafs)
1104 throws KeeperException {
1105 CreateRequest create = (CreateRequest)toZooKeeperOp(zkw, cafs).toRequestRecord();
1106 String znode = create.getPath();
1107 try {
1108 RecoverableZooKeeper zk = zkw.getRecoverableZooKeeper();
1109 if (zk.exists(znode, false) == null) {
1110 zk.create(znode, create.getData(), create.getAcl(), CreateMode.fromFlag(create.getFlags()));
1111 }
1112 } catch(KeeperException.NodeExistsException nee) {
1113 } catch(KeeperException.NoAuthException nee){
1114 try {
1115 if (null == zkw.getRecoverableZooKeeper().exists(znode, false)) {
1116
1117 throw(nee);
1118 }
1119 } catch (InterruptedException ie) {
1120 zkw.interruptedException(ie);
1121 }
1122
1123 } catch(InterruptedException ie) {
1124 zkw.interruptedException(ie);
1125 }
1126 }
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139 public static void createWithParents(ZooKeeperWatcher zkw, String znode)
1140 throws KeeperException {
1141 createWithParents(zkw, znode, new byte[0]);
1142 }
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157 public static void createWithParents(ZooKeeperWatcher zkw, String znode, byte[] data)
1158 throws KeeperException {
1159 try {
1160 if(znode == null) {
1161 return;
1162 }
1163 zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode),
1164 CreateMode.PERSISTENT);
1165 } catch(KeeperException.NodeExistsException nee) {
1166 return;
1167 } catch(KeeperException.NoNodeException nne) {
1168 createWithParents(zkw, getParent(znode));
1169 createWithParents(zkw, znode);
1170 } catch(InterruptedException ie) {
1171 zkw.interruptedException(ie);
1172 }
1173 }
1174
1175
1176
1177
1178
1179
1180
1181
1182 public static void deleteNode(ZooKeeperWatcher zkw, String node)
1183 throws KeeperException {
1184 deleteNode(zkw, node, -1);
1185 }
1186
1187
1188
1189
1190
1191 public static boolean deleteNode(ZooKeeperWatcher zkw, String node,
1192 int version)
1193 throws KeeperException {
1194 try {
1195 zkw.getRecoverableZooKeeper().delete(node, version);
1196 return true;
1197 } catch(KeeperException.BadVersionException bve) {
1198 return false;
1199 } catch(InterruptedException ie) {
1200 zkw.interruptedException(ie);
1201 return false;
1202 }
1203 }
1204
1205
1206
1207
1208
1209
1210
1211 public static void deleteNodeFailSilent(ZooKeeperWatcher zkw, String node)
1212 throws KeeperException {
1213 deleteNodeFailSilent(zkw,
1214 (DeleteNodeFailSilent)ZKUtilOp.deleteNodeFailSilent(node));
1215 }
1216
1217 private static void deleteNodeFailSilent(ZooKeeperWatcher zkw,
1218 DeleteNodeFailSilent dnfs) throws KeeperException {
1219 DeleteRequest delete = (DeleteRequest)toZooKeeperOp(zkw, dnfs).toRequestRecord();
1220 try {
1221 zkw.getRecoverableZooKeeper().delete(delete.getPath(), delete.getVersion());
1222 } catch(KeeperException.NoNodeException nne) {
1223 } catch(InterruptedException ie) {
1224 zkw.interruptedException(ie);
1225 }
1226 }
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236 public static void deleteNodeRecursively(ZooKeeperWatcher zkw, String node)
1237 throws KeeperException {
1238 try {
1239 List<String> children = ZKUtil.listChildrenNoWatch(zkw, node);
1240
1241 if (children == null) return;
1242
1243 if(!children.isEmpty()) {
1244 for(String child : children) {
1245 deleteNodeRecursively(zkw, joinZNode(node, child));
1246 }
1247 }
1248 zkw.getRecoverableZooKeeper().delete(node, -1);
1249 } catch(InterruptedException ie) {
1250 zkw.interruptedException(ie);
1251 }
1252 }
1253
1254
1255
1256
1257
1258
1259
1260 public static void deleteChildrenRecursively(ZooKeeperWatcher zkw, String node)
1261 throws KeeperException {
1262 List<String> children = ZKUtil.listChildrenNoWatch(zkw, node);
1263 if (children == null || children.isEmpty()) return;
1264 for(String child : children) {
1265 deleteNodeRecursively(zkw, joinZNode(node, child));
1266 }
1267 }
1268
1269
1270
1271
1272
1273
1274 public abstract static class ZKUtilOp {
1275 private String path;
1276
1277 private ZKUtilOp(String path) {
1278 this.path = path;
1279 }
1280
1281
1282
1283
1284 public static ZKUtilOp createAndFailSilent(String path, byte[] data) {
1285 return new CreateAndFailSilent(path, data);
1286 }
1287
1288
1289
1290
1291 public static ZKUtilOp deleteNodeFailSilent(String path) {
1292 return new DeleteNodeFailSilent(path);
1293 }
1294
1295
1296
1297
1298 public static ZKUtilOp setData(String path, byte [] data) {
1299 return new SetData(path, data);
1300 }
1301
1302
1303
1304
1305 public String getPath() {
1306 return path;
1307 }
1308
1309
1310
1311
1312
1313 public static class CreateAndFailSilent extends ZKUtilOp {
1314 private byte [] data;
1315
1316 private CreateAndFailSilent(String path, byte [] data) {
1317 super(path);
1318 this.data = data;
1319 }
1320
1321 public byte[] getData() {
1322 return data;
1323 }
1324
1325 @Override
1326 public boolean equals(Object o) {
1327 if (this == o) return true;
1328 if (!(o instanceof CreateAndFailSilent)) return false;
1329
1330 CreateAndFailSilent op = (CreateAndFailSilent) o;
1331 return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1332 }
1333 }
1334
1335
1336
1337
1338
1339 public static class DeleteNodeFailSilent extends ZKUtilOp {
1340 private DeleteNodeFailSilent(String path) {
1341 super(path);
1342 }
1343
1344 @Override
1345 public boolean equals(Object o) {
1346 if (this == o) return true;
1347 if (!(o instanceof DeleteNodeFailSilent)) return false;
1348
1349 return super.equals(o);
1350 }
1351 }
1352
1353
1354
1355
1356 public static class SetData extends ZKUtilOp {
1357 private byte [] data;
1358
1359 private SetData(String path, byte [] data) {
1360 super(path);
1361 this.data = data;
1362 }
1363
1364 public byte[] getData() {
1365 return data;
1366 }
1367
1368 @Override
1369 public boolean equals(Object o) {
1370 if (this == o) return true;
1371 if (!(o instanceof SetData)) return false;
1372
1373 SetData op = (SetData) o;
1374 return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
1375 }
1376 }
1377 }
1378
1379
1380
1381
1382 private static Op toZooKeeperOp(ZooKeeperWatcher zkw, ZKUtilOp op)
1383 throws UnsupportedOperationException {
1384 if(op == null) return null;
1385
1386 if (op instanceof CreateAndFailSilent) {
1387 CreateAndFailSilent cafs = (CreateAndFailSilent)op;
1388 return Op.create(cafs.getPath(), cafs.getData(), createACL(zkw, cafs.getPath()),
1389 CreateMode.PERSISTENT);
1390 } else if (op instanceof DeleteNodeFailSilent) {
1391 DeleteNodeFailSilent dnfs = (DeleteNodeFailSilent)op;
1392 return Op.delete(dnfs.getPath(), -1);
1393 } else if (op instanceof SetData) {
1394 SetData sd = (SetData)op;
1395 return Op.setData(sd.getPath(), sd.getData(), -1);
1396 } else {
1397 throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1398 + op.getClass().getName());
1399 }
1400 }
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423 public static void multiOrSequential(ZooKeeperWatcher zkw, List<ZKUtilOp> ops,
1424 boolean runSequentialOnMultiFailure) throws KeeperException {
1425 if (ops == null) return;
1426 boolean useMulti = zkw.getConfiguration().getBoolean(HConstants.ZOOKEEPER_USEMULTI, false);
1427
1428 if (useMulti) {
1429 List<Op> zkOps = new LinkedList<Op>();
1430 for (ZKUtilOp op : ops) {
1431 zkOps.add(toZooKeeperOp(zkw, op));
1432 }
1433 try {
1434 zkw.getRecoverableZooKeeper().multi(zkOps);
1435 } catch (KeeperException ke) {
1436 switch (ke.code()) {
1437 case NODEEXISTS:
1438 case NONODE:
1439 case BADVERSION:
1440 case NOAUTH:
1441
1442
1443 if (runSequentialOnMultiFailure) {
1444 LOG.info("On call to ZK.multi, received exception: " + ke.toString() + "."
1445 + " Attempting to run operations sequentially because"
1446 + " runSequentialOnMultiFailure is: " + runSequentialOnMultiFailure + ".");
1447 processSequentially(zkw, ops);
1448 break;
1449 }
1450 default:
1451 throw ke;
1452 }
1453 } catch (InterruptedException ie) {
1454 zkw.interruptedException(ie);
1455 }
1456 } else {
1457
1458 processSequentially(zkw, ops);
1459 }
1460 }
1461
1462 private static void processSequentially(ZooKeeperWatcher zkw, List<ZKUtilOp> ops)
1463 throws KeeperException, NoNodeException {
1464 for (ZKUtilOp op : ops) {
1465 if (op instanceof CreateAndFailSilent) {
1466 createAndFailSilent(zkw, (CreateAndFailSilent) op);
1467 } else if (op instanceof DeleteNodeFailSilent) {
1468 deleteNodeFailSilent(zkw, (DeleteNodeFailSilent) op);
1469 } else if (op instanceof SetData) {
1470 setData(zkw, (SetData) op);
1471 } else {
1472 throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
1473 + op.getClass().getName());
1474 }
1475 }
1476 }
1477
1478
1479
1480
1481
1482
1483 public static String dump(ZooKeeperWatcher zkw) {
1484 StringBuilder sb = new StringBuilder();
1485 try {
1486 sb.append("HBase is rooted at ").append(zkw.baseZNode);
1487 sb.append("\nActive master address: ").append(
1488 ServerName.parseVersionedServerName(getData(zkw, zkw.masterAddressZNode)));
1489 sb.append("\nBackup master addresses:");
1490 for (String child : listChildrenNoWatch(zkw,
1491 zkw.backupMasterAddressesZNode)) {
1492 sb.append("\n ").append(child);
1493 }
1494 sb.append("\nRegion server holding ROOT: ").append(
1495 Bytes.toStringBinary(getData(zkw, zkw.rootServerZNode)));
1496 sb.append("\nRegion servers:");
1497 for (String child : listChildrenNoWatch(zkw, zkw.rsZNode)) {
1498 sb.append("\n ").append(child);
1499 }
1500 try {
1501 getReplicationZnodesDump(zkw, sb);
1502 } catch (KeeperException ke) {
1503 LOG.warn("Couldn't get the replication znode dump." + ke.getStackTrace());
1504 }
1505 sb.append("\nQuorum Server Statistics:");
1506 String[] servers = zkw.getQuorum().split(",");
1507 for (String server : servers) {
1508 sb.append("\n ").append(server);
1509 try {
1510 String[] stat = getServerStats(server, ZKUtil.zkDumpConnectionTimeOut);
1511
1512 if (stat == null) {
1513 sb.append("[Error] invalid quorum server: " + server);
1514 break;
1515 }
1516
1517 for (String s : stat) {
1518 sb.append("\n ").append(s);
1519 }
1520 } catch (Exception e) {
1521 sb.append("\n ERROR: ").append(e.getMessage());
1522 }
1523 }
1524 } catch (KeeperException ke) {
1525 sb.append("\nFATAL ZooKeeper Exception!\n");
1526 sb.append("\n" + ke.getMessage());
1527 }
1528 return sb.toString();
1529 }
1530
1531 private static void getReplicationZnodesDump(ZooKeeperWatcher zkw, StringBuilder sb)
1532 throws KeeperException {
1533 String replicationZNodeName = zkw.getConfiguration().get("zookeeper.znode.replication",
1534 "replication");
1535 String replicationZnode = joinZNode(zkw.baseZNode, replicationZNodeName);
1536 if (ZKUtil.checkExists(zkw, replicationZnode) == -1)
1537 return;
1538
1539 List<String> stack = new LinkedList<String>();
1540 stack.add(replicationZnode);
1541 do {
1542 String znodeToProcess = stack.remove(stack.size() - 1);
1543 sb.append("\n").append(znodeToProcess).append(": ")
1544 .append(Bytes.toString(ZKUtil.getData(zkw, znodeToProcess)));
1545 for (String zNodeChild : ZKUtil.listChildrenNoWatch(zkw, znodeToProcess)) {
1546 stack.add(ZKUtil.joinZNode(znodeToProcess, zNodeChild));
1547 }
1548 } while (stack.size() > 0);
1549 }
1550
1551
1552
1553
1554
1555
1556
1557
1558 public static String[] getServerStats(String server, int timeout)
1559 throws IOException {
1560 String[] sp = server.split(":");
1561 if (sp == null || sp.length == 0) {
1562 return null;
1563 }
1564
1565 String host = sp[0];
1566 int port = sp.length > 1 ? Integer.parseInt(sp[1])
1567 : HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT;
1568
1569 Socket socket = new Socket();
1570 InetSocketAddress sockAddr = new InetSocketAddress(host, port);
1571 socket.connect(sockAddr, timeout);
1572
1573 socket.setSoTimeout(timeout);
1574 PrintWriter out = new PrintWriter(socket.getOutputStream(), true);
1575 BufferedReader in = new BufferedReader(new InputStreamReader(
1576 socket.getInputStream()));
1577 out.println("stat");
1578 out.flush();
1579 ArrayList<String> res = new ArrayList<String>();
1580 while (true) {
1581 String line = in.readLine();
1582 if (line != null) {
1583 res.add(line);
1584 } else {
1585 break;
1586 }
1587 }
1588 socket.close();
1589 return res.toArray(new String[res.size()]);
1590 }
1591
1592 private static void logRetrievedMsg(final ZooKeeperWatcher zkw,
1593 final String znode, final byte [] data, final boolean watcherSet) {
1594 if (!LOG.isDebugEnabled()) return;
1595 LOG.debug(zkw.prefix("Retrieved " + ((data == null)? 0: data.length) +
1596 " byte(s) of data from znode " + znode +
1597 (watcherSet? " and set watcher; ": "; data=") +
1598 (data == null? "null": data.length == 0? "empty": (
1599 znode.startsWith(zkw.assignmentZNode) ?
1600 RegionTransitionData.fromBytes(data).toString()
1601 : StringUtils.abbreviate(Bytes.toStringBinary(data), 32)))));
1602 }
1603
1604
1605
1606
1607
1608 public static void waitForBaseZNode(Configuration conf) throws IOException {
1609 LOG.info("Waiting until the base znode is available");
1610 String parentZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
1611 HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
1612 ZooKeeper zk = new ZooKeeper(ZKConfig.getZKQuorumServersString(conf),
1613 conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
1614 HConstants.DEFAULT_ZK_SESSION_TIMEOUT), EmptyWatcher.instance);
1615
1616 final int maxTimeMs = 10000;
1617 final int maxNumAttempts = maxTimeMs / HConstants.SOCKET_RETRY_WAIT_MS;
1618
1619 KeeperException keeperEx = null;
1620 try {
1621 try {
1622 for (int attempt = 0; attempt < maxNumAttempts; ++attempt) {
1623 try {
1624 if (zk.exists(parentZNode, false) != null) {
1625 LOG.info("Parent znode exists: " + parentZNode);
1626 keeperEx = null;
1627 break;
1628 }
1629 } catch (KeeperException e) {
1630 keeperEx = e;
1631 }
1632 Threads.sleepWithoutInterrupt(HConstants.SOCKET_RETRY_WAIT_MS);
1633 }
1634 } finally {
1635 zk.close();
1636 }
1637 } catch (InterruptedException ex) {
1638 Thread.currentThread().interrupt();
1639 }
1640
1641 if (keeperEx != null) {
1642 throw new IOException(keeperEx);
1643 }
1644 }
1645
1646
1647
1648
1649
1650
1651 public static void logZKTree(ZooKeeperWatcher zkw, String root) {
1652 if (!LOG.isDebugEnabled()) return;
1653 LOG.debug("Current zk system:");
1654 String prefix = "|-";
1655 LOG.debug(prefix + root);
1656 try {
1657 logZKTree(zkw, root, prefix);
1658 } catch (KeeperException e) {
1659 throw new RuntimeException(e);
1660 }
1661 }
1662
1663
1664
1665
1666
1667
1668 protected static void logZKTree(ZooKeeperWatcher zkw, String root, String prefix) throws KeeperException {
1669 List<String> children = ZKUtil.listChildrenNoWatch(zkw, root);
1670 if (children == null) return;
1671 for (String child : children) {
1672 LOG.debug(prefix + child);
1673 String node = ZKUtil.joinZNode(root.equals("/") ? "" : root, child);
1674 logZKTree(zkw, node, prefix + "---");
1675 }
1676 }
1677
1678 }