1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.zookeeper;
20
21 import java.io.Closeable;
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.List;
25 import java.util.concurrent.CopyOnWriteArrayList;
26 import java.util.concurrent.CountDownLatch;
27 import java.util.regex.Matcher;
28 import java.util.regex.Pattern;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.hbase.classification.InterfaceAudience;
33 import org.apache.hadoop.hbase.util.Threads;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.hbase.Abortable;
36 import org.apache.hadoop.hbase.AuthUtil;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
39 import org.apache.hadoop.hbase.classification.InterfaceAudience;
40 import org.apache.hadoop.hbase.security.Superusers;
41 import org.apache.hadoop.security.UserGroupInformation;
42 import org.apache.zookeeper.KeeperException;
43 import org.apache.zookeeper.WatchedEvent;
44 import org.apache.zookeeper.Watcher;
45 import org.apache.zookeeper.ZooDefs;
46 import org.apache.zookeeper.ZooDefs.Ids;
47 import org.apache.zookeeper.ZooDefs.Perms;
48 import org.apache.zookeeper.data.ACL;
49 import org.apache.zookeeper.data.Id;
50 import org.apache.zookeeper.data.Stat;
51
52
53
54
55
56
57
58
59
60
61
62
63 @InterfaceAudience.Private
64 public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
65 private static final Log LOG = LogFactory.getLog(ZooKeeperWatcher.class);
66
67
68
69 private String prefix;
70 private String identifier;
71
72
73 private String quorum;
74
75
76 private RecoverableZooKeeper recoverableZooKeeper;
77
78
79 protected Abortable abortable;
80
81 private boolean aborted = false;
82
83
84 private final List<ZooKeeperListener> listeners =
85 new CopyOnWriteArrayList<ZooKeeperListener>();
86
87
88
89 public CountDownLatch saslLatch = new CountDownLatch(1);
90
91
92
93
94 public String baseZNode;
95
96 public String metaServerZNode;
97
98 public String rsZNode;
99
100 public String drainingZNode;
101
102 private String masterAddressZNode;
103
104 public String backupMasterAddressesZNode;
105
106 public String clusterStateZNode;
107
108 public String assignmentZNode;
109
110 public String tableZNode;
111
112 public String clusterIdZNode;
113
114 public String splitLogZNode;
115
116 public String balancerZNode;
117
118 public String tableLockZNode;
119
120 public String recoveringRegionsZNode;
121
122 public static String namespaceZNode = "namespace";
123
124
125 public static final ArrayList<ACL> CREATOR_ALL_AND_WORLD_READABLE =
126 new ArrayList<ACL>() { {
127 add(new ACL(ZooDefs.Perms.READ,ZooDefs.Ids.ANYONE_ID_UNSAFE));
128 add(new ACL(ZooDefs.Perms.ALL,ZooDefs.Ids.AUTH_IDS));
129 }};
130
131 private final Configuration conf;
132
133 private final Exception constructorCaller;
134
135
136 private static final Pattern NAME_PATTERN = Pattern.compile("([^/@]*)(/([^/@]*))?@([^/@]*)");
137
138
139
140
141
142
143
144
145 public ZooKeeperWatcher(Configuration conf, String identifier,
146 Abortable abortable) throws ZooKeeperConnectionException, IOException {
147 this(conf, identifier, abortable, false);
148 }
149
150
151
152
153
154
155
156
157
158
159
160
161 public ZooKeeperWatcher(Configuration conf, String identifier,
162 Abortable abortable, boolean canCreateBaseZNode)
163 throws IOException, ZooKeeperConnectionException {
164 this.conf = conf;
165
166
167 try {
168 throw new Exception("ZKW CONSTRUCTOR STACK TRACE FOR DEBUGGING");
169 } catch (Exception e) {
170 this.constructorCaller = e;
171 }
172 this.quorum = ZKConfig.getZKQuorumServersString(conf);
173 this.prefix = identifier;
174
175
176 this.identifier = identifier + "0x0";
177 this.abortable = abortable;
178 setNodeNames(conf);
179 this.recoverableZooKeeper = ZKUtil.connect(conf, quorum, this, identifier);
180 if (canCreateBaseZNode) {
181 createBaseZNodes();
182 }
183 }
184
185 private void createBaseZNodes() throws ZooKeeperConnectionException {
186 try {
187
188 ZKUtil.createWithParents(this, baseZNode);
189 if (conf.getBoolean("hbase.assignment.usezk", true)) {
190 ZKUtil.createAndFailSilent(this, assignmentZNode);
191 }
192 ZKUtil.createAndFailSilent(this, rsZNode);
193 ZKUtil.createAndFailSilent(this, drainingZNode);
194 ZKUtil.createAndFailSilent(this, tableZNode);
195 ZKUtil.createAndFailSilent(this, splitLogZNode);
196 ZKUtil.createAndFailSilent(this, backupMasterAddressesZNode);
197 ZKUtil.createAndFailSilent(this, tableLockZNode);
198 ZKUtil.createAndFailSilent(this, recoveringRegionsZNode);
199 } catch (KeeperException e) {
200 throw new ZooKeeperConnectionException(
201 prefix("Unexpected KeeperException creating base node"), e);
202 }
203 }
204
205
206
207 public boolean isClientReadable(String node) {
208
209
210
211 return
212 node.equals(baseZNode) ||
213 node.equals(metaServerZNode) ||
214 node.equals(getMasterAddressZNode()) ||
215 node.equals(clusterIdZNode)||
216 node.equals(rsZNode) ||
217
218 node.equals(tableZNode) ||
219 node.startsWith(tableZNode + "/");
220 }
221
222
223
224
225
226
227
228 public void checkAndSetZNodeAcls() {
229 if (!ZKUtil.isSecureZooKeeper(getConfiguration())) {
230 LOG.info("not a secure deployment, proceeding");
231 return;
232 }
233
234
235
236 try {
237 List<ACL> actualAcls = recoverableZooKeeper.getAcl(baseZNode, new Stat());
238
239 if (!isBaseZnodeAclSetup(actualAcls)) {
240 LOG.info("setting znode ACLs");
241 setZnodeAclsRecursive(baseZNode);
242 }
243 } catch(KeeperException.NoNodeException nne) {
244 return;
245 } catch(InterruptedException ie) {
246 interruptedException(ie);
247 } catch (IOException e) {
248 LOG.warn("Received exception while checking and setting zookeeper ACLs", e);
249 } catch (KeeperException e) {
250 LOG.warn("Received exception while checking and setting zookeeper ACLs", e);
251 }
252 }
253
254
255
256
257
258
259 private void setZnodeAclsRecursive(String znode) throws KeeperException, InterruptedException {
260 List<String> children = recoverableZooKeeper.getChildren(znode, false);
261
262 for (String child : children) {
263 setZnodeAclsRecursive(ZKUtil.joinZNode(znode, child));
264 }
265 List<ACL> acls = ZKUtil.createACL(this, znode, true);
266 LOG.info("Setting ACLs for znode:" + znode + " , acl:" + acls);
267 recoverableZooKeeper.setAcl(znode, acls, -1);
268 }
269
270
271
272
273
274
275
276 private boolean isBaseZnodeAclSetup(List<ACL> acls) throws IOException {
277 if (LOG.isDebugEnabled()) {
278 LOG.debug("Checking znode ACLs");
279 }
280 String[] superUsers = conf.getStrings(Superusers.SUPERUSER_CONF_KEY);
281
282 if (superUsers != null && !checkACLForSuperUsers(superUsers, acls)) {
283 return false;
284 }
285
286
287
288 String hbaseUser = UserGroupInformation.getCurrentUser().getShortUserName();
289
290 if (acls.isEmpty()) {
291 if (LOG.isDebugEnabled()) {
292 LOG.debug("ACL is empty");
293 }
294 return false;
295 }
296
297 for (ACL acl : acls) {
298 int perms = acl.getPerms();
299 Id id = acl.getId();
300
301
302 if (Ids.ANYONE_ID_UNSAFE.equals(id)) {
303 if (perms != Perms.READ) {
304 if (LOG.isDebugEnabled()) {
305 LOG.debug(String.format("permissions for '%s' are not correct: have 0x%x, want 0x%x",
306 id, perms, Perms.READ));
307 }
308 return false;
309 }
310 } else if (superUsers != null && isSuperUserId(superUsers, id)) {
311 if (perms != Perms.ALL) {
312 if (LOG.isDebugEnabled()) {
313 LOG.debug(String.format("permissions for '%s' are not correct: have 0x%x, want 0x%x",
314 id, perms, Perms.ALL));
315 }
316 return false;
317 }
318 } else if ("sasl".equals(id.getScheme())) {
319 String name = id.getId();
320
321 Matcher match = NAME_PATTERN.matcher(name);
322 if (match.matches()) {
323 name = match.group(1);
324 }
325 if (name.equals(hbaseUser)) {
326 if (perms != Perms.ALL) {
327 if (LOG.isDebugEnabled()) {
328 LOG.debug(String.format("permissions for '%s' are not correct: have 0x%x, want 0x%x",
329 id, perms, Perms.ALL));
330 }
331 return false;
332 }
333 } else {
334 if (LOG.isDebugEnabled()) {
335 LOG.debug("Unexpected shortname in SASL ACL: " + id);
336 }
337 return false;
338 }
339 } else {
340 if (LOG.isDebugEnabled()) {
341 LOG.debug("unexpected ACL id '" + id + "'");
342 }
343 return false;
344 }
345 }
346 return true;
347 }
348
349
350
351
352 private boolean checkACLForSuperUsers(String[] superUsers, List<ACL> acls) {
353 for (String user : superUsers) {
354 boolean hasAccess = false;
355
356 if (!user.startsWith(AuthUtil.GROUP_PREFIX)) {
357 for (ACL acl : acls) {
358 if (user.equals(acl.getId().getId())) {
359 if (acl.getPerms() == Perms.ALL) {
360 hasAccess = true;
361 } else {
362 if (LOG.isDebugEnabled()) {
363 LOG.debug(String.format(
364 "superuser '%s' does not have correct permissions: have 0x%x, want 0x%x",
365 acl.getId().getId(), acl.getPerms(), Perms.ALL));
366 }
367 }
368 break;
369 }
370 }
371 if (!hasAccess) {
372 return false;
373 }
374 }
375 }
376 return true;
377 }
378
379
380
381
382 public static boolean isSuperUserId(String[] superUsers, Id id) {
383 for (String user : superUsers) {
384
385 if (!user.startsWith(AuthUtil.GROUP_PREFIX) && new Id("sasl", user).equals(id)) {
386 return true;
387 }
388 }
389 return false;
390 }
391
392 @Override
393 public String toString() {
394 return this.identifier + ", quorum=" + quorum + ", baseZNode=" + baseZNode;
395 }
396
397
398
399
400
401
402
403 public String prefix(final String str) {
404 return this.toString() + " " + str;
405 }
406
407
408
409
410 private void setNodeNames(Configuration conf) {
411 baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
412 HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
413 metaServerZNode = ZKUtil.joinZNode(baseZNode,
414 conf.get("zookeeper.znode.metaserver", "meta-region-server"));
415 rsZNode = ZKUtil.joinZNode(baseZNode,
416 conf.get("zookeeper.znode.rs", "rs"));
417 drainingZNode = ZKUtil.joinZNode(baseZNode,
418 conf.get("zookeeper.znode.draining.rs", "draining"));
419 masterAddressZNode = ZKUtil.joinZNode(baseZNode,
420 conf.get("zookeeper.znode.master", "master"));
421 backupMasterAddressesZNode = ZKUtil.joinZNode(baseZNode,
422 conf.get("zookeeper.znode.backup.masters", "backup-masters"));
423 clusterStateZNode = ZKUtil.joinZNode(baseZNode,
424 conf.get("zookeeper.znode.state", "running"));
425 assignmentZNode = ZKUtil.joinZNode(baseZNode,
426 conf.get("zookeeper.znode.unassigned", "region-in-transition"));
427 tableZNode = ZKUtil.joinZNode(baseZNode,
428 conf.get("zookeeper.znode.tableEnableDisable", "table"));
429 clusterIdZNode = ZKUtil.joinZNode(baseZNode,
430 conf.get("zookeeper.znode.clusterId", "hbaseid"));
431 splitLogZNode = ZKUtil.joinZNode(baseZNode,
432 conf.get("zookeeper.znode.splitlog", HConstants.SPLIT_LOGDIR_NAME));
433 balancerZNode = ZKUtil.joinZNode(baseZNode,
434 conf.get("zookeeper.znode.balancer", "balancer"));
435 tableLockZNode = ZKUtil.joinZNode(baseZNode,
436 conf.get("zookeeper.znode.tableLock", "table-lock"));
437 recoveringRegionsZNode = ZKUtil.joinZNode(baseZNode,
438 conf.get("zookeeper.znode.recovering.regions", "recovering-regions"));
439 namespaceZNode = ZKUtil.joinZNode(baseZNode,
440 conf.get("zookeeper.znode.namespace", "namespace"));
441 }
442
443
444
445
446
447 public void registerListener(ZooKeeperListener listener) {
448 listeners.add(listener);
449 }
450
451
452
453
454
455
456 public void registerListenerFirst(ZooKeeperListener listener) {
457 listeners.add(0, listener);
458 }
459
460 public void unregisterListener(ZooKeeperListener listener) {
461 listeners.remove(listener);
462 }
463
464
465
466
467 public void unregisterAllListeners() {
468 listeners.clear();
469 }
470
471
472
473
474 public List<ZooKeeperListener> getListeners() {
475 return new ArrayList<ZooKeeperListener>(listeners);
476 }
477
478
479
480
481 public int getNumberOfListeners() {
482 return listeners.size();
483 }
484
485
486
487
488
489 public RecoverableZooKeeper getRecoverableZooKeeper() {
490 return recoverableZooKeeper;
491 }
492
493 public void reconnectAfterExpiration() throws IOException, KeeperException, InterruptedException {
494 recoverableZooKeeper.reconnectAfterExpiration();
495 }
496
497
498
499
500
501 public String getQuorum() {
502 return quorum;
503 }
504
505
506
507
508 public String getBaseZNode() {
509 return baseZNode;
510 }
511
512
513
514
515
516
517
518 @Override
519 public void process(WatchedEvent event) {
520 LOG.debug(prefix("Received ZooKeeper Event, " +
521 "type=" + event.getType() + ", " +
522 "state=" + event.getState() + ", " +
523 "path=" + event.getPath()));
524
525 switch(event.getType()) {
526
527
528 case None: {
529 connectionEvent(event);
530 break;
531 }
532
533
534
535 case NodeCreated: {
536 for(ZooKeeperListener listener : listeners) {
537 listener.nodeCreated(event.getPath());
538 }
539 break;
540 }
541
542 case NodeDeleted: {
543 for(ZooKeeperListener listener : listeners) {
544 listener.nodeDeleted(event.getPath());
545 }
546 break;
547 }
548
549 case NodeDataChanged: {
550 for(ZooKeeperListener listener : listeners) {
551 listener.nodeDataChanged(event.getPath());
552 }
553 break;
554 }
555
556 case NodeChildrenChanged: {
557 for(ZooKeeperListener listener : listeners) {
558 listener.nodeChildrenChanged(event.getPath());
559 }
560 break;
561 }
562 }
563 }
564
565
566
567
568
569
570
571
572
573
574
575
576
577 private void connectionEvent(WatchedEvent event) {
578 switch(event.getState()) {
579 case SyncConnected:
580
581
582 long finished = System.currentTimeMillis() +
583 this.conf.getLong("hbase.zookeeper.watcher.sync.connected.wait", 2000);
584 while (System.currentTimeMillis() < finished) {
585 Threads.sleep(1);
586 if (this.recoverableZooKeeper != null) break;
587 }
588 if (this.recoverableZooKeeper == null) {
589 LOG.error("ZK is null on connection event -- see stack trace " +
590 "for the stack trace when constructor was called on this zkw",
591 this.constructorCaller);
592 throw new NullPointerException("ZK is null");
593 }
594 this.identifier = this.prefix + "-0x" +
595 Long.toHexString(this.recoverableZooKeeper.getSessionId());
596
597 LOG.debug(this.identifier + " connected");
598 break;
599
600
601 case Disconnected:
602 LOG.debug(prefix("Received Disconnected from ZooKeeper, ignoring"));
603 break;
604
605 case Expired:
606 String msg = prefix(this.identifier + " received expired from " +
607 "ZooKeeper, aborting");
608
609
610 if (this.abortable != null) {
611 this.abortable.abort(msg, new KeeperException.SessionExpiredException());
612 }
613 break;
614
615 case ConnectedReadOnly:
616 case SaslAuthenticated:
617 case AuthFailed:
618 break;
619
620 default:
621 throw new IllegalStateException("Received event is not valid: " + event.getState());
622 }
623 }
624
625
626
627
628
629
630
631
632
633
634
635
636
637 public void sync(String path) throws KeeperException {
638 this.recoverableZooKeeper.sync(path, null, null);
639 }
640
641
642
643
644
645
646
647
648
649
650
651 public void keeperException(KeeperException ke)
652 throws KeeperException {
653 LOG.error(prefix("Received unexpected KeeperException, re-throwing exception"), ke);
654 throw ke;
655 }
656
657
658
659
660
661
662
663
664
665
666
667
668 public void interruptedException(InterruptedException ie) {
669 LOG.debug(prefix("Received InterruptedException, doing nothing here"), ie);
670
671 Thread.currentThread().interrupt();
672
673 }
674
675
676
677
678
679
680 @Override
681 public void close() {
682 try {
683 if (recoverableZooKeeper != null) {
684 recoverableZooKeeper.close();
685 }
686 } catch (InterruptedException e) {
687 Thread.currentThread().interrupt();
688 }
689 }
690
691 public Configuration getConfiguration() {
692 return conf;
693 }
694
695 @Override
696 public void abort(String why, Throwable e) {
697 if (this.abortable != null) this.abortable.abort(why, e);
698 else this.aborted = true;
699 }
700
701 @Override
702 public boolean isAborted() {
703 return this.abortable == null? this.aborted: this.abortable.isAborted();
704 }
705
706
707
708
709 public String getMasterAddressZNode() {
710 return this.masterAddressZNode;
711 }
712
713 }