1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import java.io.IOException;
22 import java.net.InetAddress;
23 import java.util.ArrayList;
24 import java.util.Collections;
25 import java.util.HashMap;
26 import java.util.HashSet;
27 import java.util.Iterator;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.Map.Entry;
31 import java.util.Set;
32 import java.util.SortedMap;
33 import java.util.concurrent.ConcurrentHashMap;
34 import java.util.concurrent.ConcurrentSkipListMap;
35
36 import org.apache.commons.logging.Log;
37 import org.apache.commons.logging.LogFactory;
38 import org.apache.hadoop.classification.InterfaceAudience;
39 import org.apache.hadoop.conf.Configuration;
40 import org.apache.hadoop.hbase.ClockOutOfSyncException;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.RegionLoad;
43 import org.apache.hadoop.hbase.Server;
44 import org.apache.hadoop.hbase.ServerLoad;
45 import org.apache.hadoop.hbase.ServerName;
46 import org.apache.hadoop.hbase.YouAreDeadException;
47 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
48 import org.apache.hadoop.hbase.client.HConnection;
49 import org.apache.hadoop.hbase.client.HConnectionManager;
50 import org.apache.hadoop.hbase.client.RetriesExhaustedException;
51 import org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler;
52 import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
53 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
54 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
55 import org.apache.hadoop.hbase.protobuf.RequestConverter;
56 import org.apache.hadoop.hbase.protobuf.ResponseConverter;
57 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
58 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
59 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
60 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
61 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
62 import org.apache.hadoop.hbase.util.Bytes;
63 import org.apache.hadoop.hbase.util.Triple;
64
65 import com.google.protobuf.ServiceException;
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89 @InterfaceAudience.Private
90 public class ServerManager {
91 public static final String WAIT_ON_REGIONSERVERS_MAXTOSTART =
92 "hbase.master.wait.on.regionservers.maxtostart";
93
94 public static final String WAIT_ON_REGIONSERVERS_MINTOSTART =
95 "hbase.master.wait.on.regionservers.mintostart";
96
97 public static final String WAIT_ON_REGIONSERVERS_TIMEOUT =
98 "hbase.master.wait.on.regionservers.timeout";
99
100 public static final String WAIT_ON_REGIONSERVERS_INTERVAL =
101 "hbase.master.wait.on.regionservers.interval";
102
103 private static final Log LOG = LogFactory.getLog(ServerManager.class);
104
105
106 private volatile boolean clusterShutdown = false;
107
108 private final SortedMap<byte[], Long> flushedSequenceIdByRegion =
109 new ConcurrentSkipListMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
110
111
112 private final Map<ServerName, ServerLoad> onlineServers =
113 new ConcurrentHashMap<ServerName, ServerLoad>();
114
115
116
117
118
119 private final Map<ServerName, AdminService.BlockingInterface> rsAdmins =
120 new HashMap<ServerName, AdminService.BlockingInterface>();
121
122
123
124
125
126 private final ArrayList<ServerName> drainingServers =
127 new ArrayList<ServerName>();
128
129 private final Server master;
130 private final MasterServices services;
131 private final HConnection connection;
132
133 private final DeadServer deadservers = new DeadServer();
134
135 private final long maxSkew;
136 private final long warningSkew;
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154 private Set<ServerName> queuedDeadServers = new HashSet<ServerName>();
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171 private Map<ServerName, Boolean> requeuedDeadServers = new HashMap<ServerName, Boolean>();
172
173
174
175
176
177
178
179 public ServerManager(final Server master, final MasterServices services)
180 throws IOException {
181 this(master, services, true);
182 }
183
184 @SuppressWarnings("deprecation")
185 ServerManager(final Server master, final MasterServices services,
186 final boolean connect) throws IOException {
187 this.master = master;
188 this.services = services;
189 Configuration c = master.getConfiguration();
190 maxSkew = c.getLong("hbase.master.maxclockskew", 30000);
191 warningSkew = c.getLong("hbase.master.warningclockskew", 10000);
192 this.connection = connect ? HConnectionManager.getConnection(c) : null;
193 }
194
195
196
197
198
199
200
201
202
203
204 ServerName regionServerStartup(final InetAddress ia, final int port,
205 final long serverStartcode, long serverCurrentTime)
206 throws IOException {
207
208
209
210
211
212
213
214 ServerName sn = ServerName.valueOf(ia.getHostName(), port, serverStartcode);
215 checkClockSkew(sn, serverCurrentTime);
216 checkIsDead(sn, "STARTUP");
217 if (!checkAlreadySameHostPortAndRecordNewServer(
218 sn, ServerLoad.EMPTY_SERVERLOAD)) {
219 LOG.warn("THIS SHOULD NOT HAPPEN, RegionServerStartup"
220 + " could not record the server: " + sn);
221 }
222 return sn;
223 }
224
225
226
227
228
229
230 private void updateLastFlushedSequenceIds(ServerName sn, ServerLoad hsl) {
231 Map<byte[], RegionLoad> regionsLoad = hsl.getRegionsLoad();
232 for (Entry<byte[], RegionLoad> entry : regionsLoad.entrySet()) {
233 Long existingValue = flushedSequenceIdByRegion.get(entry.getKey());
234 long l = entry.getValue().getCompleteSequenceId();
235 if (existingValue != null) {
236 if (l != -1 && l < existingValue) {
237 if (LOG.isDebugEnabled()) {
238 LOG.debug("RegionServer " + sn +
239 " indicates a last flushed sequence id (" + entry.getValue() +
240 ") that is less than the previous last flushed sequence id (" +
241 existingValue + ") for region " +
242 Bytes.toString(entry.getKey()) + " Ignoring.");
243 }
244 continue;
245
246 }
247 }
248 flushedSequenceIdByRegion.put(entry.getKey(), l);
249 }
250 }
251
252 void regionServerReport(ServerName sn,
253 ServerLoad sl) throws YouAreDeadException {
254 checkIsDead(sn, "REPORT");
255 if (!this.onlineServers.containsKey(sn)) {
256
257
258
259
260
261
262 if (!checkAlreadySameHostPortAndRecordNewServer(sn, sl)) {
263 LOG.info("RegionServerReport ignored, could not record the sever: " + sn);
264 return;
265 }
266 } else {
267 this.onlineServers.put(sn, sl);
268 }
269 updateLastFlushedSequenceIds(sn, sl);
270 }
271
272
273
274
275
276
277
278
279
280 boolean checkAlreadySameHostPortAndRecordNewServer(
281 final ServerName serverName, final ServerLoad sl) {
282 ServerName existingServer = findServerWithSameHostnamePort(serverName);
283 if (existingServer != null) {
284 if (existingServer.getStartcode() > serverName.getStartcode()) {
285 LOG.info("Server serverName=" + serverName +
286 " rejected; we already have " + existingServer.toString() +
287 " registered with same hostname and port");
288 return false;
289 }
290 LOG.info("Triggering server recovery; existingServer " +
291 existingServer + " looks stale, new server:" + serverName);
292 expireServer(existingServer);
293 }
294 recordNewServer(serverName, sl);
295 return true;
296 }
297
298
299
300
301
302
303
304
305
306 private void checkClockSkew(final ServerName serverName, final long serverCurrentTime)
307 throws ClockOutOfSyncException {
308 long skew = Math.abs(System.currentTimeMillis() - serverCurrentTime);
309 if (skew > maxSkew) {
310 String message = "Server " + serverName + " has been " +
311 "rejected; Reported time is too far out of sync with master. " +
312 "Time difference of " + skew + "ms > max allowed of " + maxSkew + "ms";
313 LOG.warn(message);
314 throw new ClockOutOfSyncException(message);
315 } else if (skew > warningSkew){
316 String message = "Reported time for server " + serverName + " is out of sync with master " +
317 "by " + skew + "ms. (Warning threshold is " + warningSkew + "ms; " +
318 "error threshold is " + maxSkew + "ms)";
319 LOG.warn(message);
320 }
321 }
322
323
324
325
326
327
328
329
330
331 private void checkIsDead(final ServerName serverName, final String what)
332 throws YouAreDeadException {
333 if (this.deadservers.isDeadServer(serverName)) {
334
335
336 String message = "Server " + what + " rejected; currently processing " +
337 serverName + " as dead server";
338 LOG.debug(message);
339 throw new YouAreDeadException(message);
340 }
341
342
343 if ((this.services == null || ((HMaster) this.services).isInitialized())
344 && this.deadservers.cleanPreviousInstance(serverName)) {
345
346
347 LOG.debug(what + ":" + " Server " + serverName + " came back up," +
348 " removed it from the dead servers list");
349 }
350 }
351
352
353
354
355 private ServerName findServerWithSameHostnamePort(
356 final ServerName serverName) {
357 for (ServerName sn: getOnlineServersList()) {
358 if (ServerName.isSameHostnameAndPort(serverName, sn)) return sn;
359 }
360 return null;
361 }
362
363
364
365
366
367
368 void recordNewServer(final ServerName serverName, final ServerLoad sl) {
369 LOG.info("Registering server=" + serverName);
370 this.onlineServers.put(serverName, sl);
371 this.rsAdmins.remove(serverName);
372 }
373
374 public long getLastFlushedSequenceId(byte[] regionName) {
375 long seqId = -1;
376 if (flushedSequenceIdByRegion.containsKey(regionName)) {
377 seqId = flushedSequenceIdByRegion.get(regionName);
378 }
379 return seqId;
380 }
381
382
383
384
385
386 public ServerLoad getLoad(final ServerName serverName) {
387 return this.onlineServers.get(serverName);
388 }
389
390
391
392
393
394
395
396 public double getAverageLoad() {
397 int totalLoad = 0;
398 int numServers = 0;
399 double averageLoad;
400 for (ServerLoad sl: this.onlineServers.values()) {
401 numServers++;
402 totalLoad += sl.getNumberOfRegions();
403 }
404 averageLoad = (double)totalLoad / (double)numServers;
405 return averageLoad;
406 }
407
408
409 int countOfRegionServers() {
410
411 return this.onlineServers.size();
412 }
413
414
415
416
417 public Map<ServerName, ServerLoad> getOnlineServers() {
418
419 synchronized (this.onlineServers) {
420 return Collections.unmodifiableMap(this.onlineServers);
421 }
422 }
423
424
425 public DeadServer getDeadServers() {
426 return this.deadservers;
427 }
428
429
430
431
432
433 public boolean areDeadServersInProgress() {
434 return this.deadservers.areDeadServersInProgress();
435 }
436
437 void letRegionServersShutdown() {
438 long previousLogTime = 0;
439 while (!onlineServers.isEmpty()) {
440
441 if (System.currentTimeMillis() > (previousLogTime + 1000)) {
442 StringBuilder sb = new StringBuilder();
443 for (ServerName key : this.onlineServers.keySet()) {
444 if (sb.length() > 0) {
445 sb.append(", ");
446 }
447 sb.append(key);
448 }
449 LOG.info("Waiting on regionserver(s) to go down " + sb.toString());
450 previousLogTime = System.currentTimeMillis();
451 }
452
453 synchronized (onlineServers) {
454 try {
455 onlineServers.wait(100);
456 } catch (InterruptedException ignored) {
457
458 }
459 }
460 }
461 }
462
463
464
465
466
467 public synchronized void expireServer(final ServerName serverName) {
468 if (!services.isServerShutdownHandlerEnabled()) {
469 LOG.info("Master doesn't enable ServerShutdownHandler during initialization, "
470 + "delay expiring server " + serverName);
471 this.queuedDeadServers.add(serverName);
472 return;
473 }
474 if (!this.onlineServers.containsKey(serverName)) {
475 LOG.warn("Expiration of " + serverName +
476 " but server not online");
477 }
478 if (this.deadservers.isDeadServer(serverName)) {
479
480 LOG.warn("Expiration of " + serverName +
481 " but server shutdown already in progress");
482 return;
483 }
484
485
486
487 this.deadservers.add(serverName);
488 this.onlineServers.remove(serverName);
489 synchronized (onlineServers) {
490 onlineServers.notifyAll();
491 }
492 this.rsAdmins.remove(serverName);
493
494
495 if (this.clusterShutdown) {
496 LOG.info("Cluster shutdown set; " + serverName +
497 " expired; onlineServers=" + this.onlineServers.size());
498 if (this.onlineServers.isEmpty()) {
499 master.stop("Cluster shutdown set; onlineServer=0");
500 }
501 return;
502 }
503
504 boolean carryingMeta = services.getAssignmentManager().isCarryingMeta(serverName);
505 if (carryingMeta) {
506 this.services.getExecutorService().submit(new MetaServerShutdownHandler(this.master,
507 this.services, this.deadservers, serverName));
508 } else {
509 this.services.getExecutorService().submit(new ServerShutdownHandler(this.master,
510 this.services, this.deadservers, serverName, true));
511 }
512 LOG.debug("Added=" + serverName +
513 " to dead servers, submitted shutdown handler to be executed meta=" + carryingMeta);
514 }
515
516 public synchronized void processDeadServer(final ServerName serverName) {
517 this.processDeadServer(serverName, false);
518 }
519
520 public synchronized void processDeadServer(final ServerName serverName, boolean shouldSplitHlog) {
521
522
523
524
525
526
527
528
529 if (!services.getAssignmentManager().isFailoverCleanupDone()) {
530 requeuedDeadServers.put(serverName, shouldSplitHlog);
531 return;
532 }
533
534 this.deadservers.add(serverName);
535 this.services.getExecutorService().submit(
536 new ServerShutdownHandler(this.master, this.services, this.deadservers, serverName,
537 shouldSplitHlog));
538 }
539
540
541
542
543
544 synchronized void processQueuedDeadServers() {
545 if (!services.isServerShutdownHandlerEnabled()) {
546 LOG.info("Master hasn't enabled ServerShutdownHandler");
547 }
548 Iterator<ServerName> serverIterator = queuedDeadServers.iterator();
549 while (serverIterator.hasNext()) {
550 ServerName tmpServerName = serverIterator.next();
551 expireServer(tmpServerName);
552 serverIterator.remove();
553 requeuedDeadServers.remove(tmpServerName);
554 }
555
556 if (!services.getAssignmentManager().isFailoverCleanupDone()) {
557 LOG.info("AssignmentManager hasn't finished failover cleanup; waiting");
558 }
559
560 for(ServerName tmpServerName : requeuedDeadServers.keySet()){
561 processDeadServer(tmpServerName, requeuedDeadServers.get(tmpServerName));
562 }
563 requeuedDeadServers.clear();
564 }
565
566
567
568
569 public boolean removeServerFromDrainList(final ServerName sn) {
570
571
572
573 if (!this.isServerOnline(sn)) {
574 LOG.warn("Server " + sn + " is not currently online. " +
575 "Removing from draining list anyway, as requested.");
576 }
577
578 return this.drainingServers.remove(sn);
579 }
580
581
582
583
584 public boolean addServerToDrainList(final ServerName sn) {
585
586
587
588 if (!this.isServerOnline(sn)) {
589 LOG.warn("Server " + sn + " is not currently online. " +
590 "Ignoring request to add it to draining list.");
591 return false;
592 }
593
594
595 if (this.drainingServers.contains(sn)) {
596 LOG.warn("Server " + sn + " is already in the draining server list." +
597 "Ignoring request to add it again.");
598 return false;
599 }
600 return this.drainingServers.add(sn);
601 }
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616 public RegionOpeningState sendRegionOpen(final ServerName server,
617 HRegionInfo region, int versionOfOfflineNode, List<ServerName> favoredNodes)
618 throws IOException {
619 AdminService.BlockingInterface admin = getRsAdmin(server);
620 if (admin == null) {
621 LOG.warn("Attempting to send OPEN RPC to server " + server.toString() +
622 " failed because no RPC connection found to this server");
623 return RegionOpeningState.FAILED_OPENING;
624 }
625 OpenRegionRequest request =
626 RequestConverter.buildOpenRegionRequest(server, region, versionOfOfflineNode, favoredNodes);
627 try {
628 OpenRegionResponse response = admin.openRegion(null, request);
629 return ResponseConverter.getRegionOpeningState(response);
630 } catch (ServiceException se) {
631 throw ProtobufUtil.getRemoteException(se);
632 }
633 }
634
635
636
637
638
639
640
641
642
643
644 public List<RegionOpeningState> sendRegionOpen(ServerName server,
645 List<Triple<HRegionInfo, Integer, List<ServerName>>> regionOpenInfos)
646 throws IOException {
647 AdminService.BlockingInterface admin = getRsAdmin(server);
648 if (admin == null) {
649 LOG.warn("Attempting to send OPEN RPC to server " + server.toString() +
650 " failed because no RPC connection found to this server");
651 return null;
652 }
653
654 OpenRegionRequest request =
655 RequestConverter.buildOpenRegionRequest(regionOpenInfos);
656 try {
657 OpenRegionResponse response = admin.openRegion(null, request);
658 return ResponseConverter.getRegionOpeningStateList(response);
659 } catch (ServiceException se) {
660 throw ProtobufUtil.getRemoteException(se);
661 }
662 }
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678 public boolean sendRegionClose(ServerName server, HRegionInfo region,
679 int versionOfClosingNode, ServerName dest, boolean transitionInZK) throws IOException {
680 if (server == null) throw new NullPointerException("Passed server is null");
681 AdminService.BlockingInterface admin = getRsAdmin(server);
682 if (admin == null) {
683 throw new IOException("Attempting to send CLOSE RPC to server " +
684 server.toString() + " for region " +
685 region.getRegionNameAsString() +
686 " failed because no RPC connection found to this server");
687 }
688 return ProtobufUtil.closeRegion(admin, server, region.getRegionName(),
689 versionOfClosingNode, dest, transitionInZK);
690 }
691
692 public boolean sendRegionClose(ServerName server,
693 HRegionInfo region, int versionOfClosingNode) throws IOException {
694 return sendRegionClose(server, region, versionOfClosingNode, null, true);
695 }
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710 public void sendRegionsMerge(ServerName server, HRegionInfo region_a,
711 HRegionInfo region_b, boolean forcible) throws IOException {
712 if (server == null)
713 throw new NullPointerException("Passed server is null");
714 if (region_a == null || region_b == null)
715 throw new NullPointerException("Passed region is null");
716 AdminService.BlockingInterface admin = getRsAdmin(server);
717 if (admin == null) {
718 throw new IOException("Attempting to send MERGE REGIONS RPC to server "
719 + server.toString() + " for region "
720 + region_a.getRegionNameAsString() + ","
721 + region_b.getRegionNameAsString()
722 + " failed because no RPC connection found to this server");
723 }
724 ProtobufUtil.mergeRegions(admin, region_a, region_b, forcible);
725 }
726
727
728
729
730 public boolean isServerReachable(ServerName server) {
731 if (server == null) throw new NullPointerException("Passed server is null");
732 int maximumAttempts = Math.max(1, master.getConfiguration().getInt(
733 "hbase.master.maximum.ping.server.attempts", 10));
734 for (int i = 0; i < maximumAttempts; i++) {
735 try {
736 AdminService.BlockingInterface admin = getRsAdmin(server);
737 if (admin != null) {
738 ServerInfo info = ProtobufUtil.getServerInfo(admin);
739 return info != null && info.hasServerName()
740 && server.getStartcode() == info.getServerName().getStartCode();
741 }
742 } catch (IOException ioe) {
743 LOG.debug("Couldn't reach " + server + ", try=" + i
744 + " of " + maximumAttempts, ioe);
745 }
746 }
747 return false;
748 }
749
750
751
752
753
754
755
756 private AdminService.BlockingInterface getRsAdmin(final ServerName sn)
757 throws IOException {
758 AdminService.BlockingInterface admin = this.rsAdmins.get(sn);
759 if (admin == null) {
760 LOG.debug("New admin connection to " + sn.toString());
761 admin = this.connection.getAdmin(sn);
762 this.rsAdmins.put(sn, admin);
763 }
764 return admin;
765 }
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780 public void waitForRegionServers(MonitoredTask status)
781 throws InterruptedException {
782 final long interval = this.master.getConfiguration().
783 getLong(WAIT_ON_REGIONSERVERS_INTERVAL, 1500);
784 final long timeout = this.master.getConfiguration().
785 getLong(WAIT_ON_REGIONSERVERS_TIMEOUT, 4500);
786 int minToStart = this.master.getConfiguration().
787 getInt(WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
788 if (minToStart < 1) {
789 LOG.warn(String.format(
790 "The value of '%s' (%d) can not be less than 1, ignoring.",
791 WAIT_ON_REGIONSERVERS_MINTOSTART, minToStart));
792 minToStart = 1;
793 }
794 int maxToStart = this.master.getConfiguration().
795 getInt(WAIT_ON_REGIONSERVERS_MAXTOSTART, Integer.MAX_VALUE);
796 if (maxToStart < minToStart) {
797 LOG.warn(String.format(
798 "The value of '%s' (%d) is set less than '%s' (%d), ignoring.",
799 WAIT_ON_REGIONSERVERS_MAXTOSTART, maxToStart,
800 WAIT_ON_REGIONSERVERS_MINTOSTART, minToStart));
801 maxToStart = Integer.MAX_VALUE;
802 }
803
804 long now = System.currentTimeMillis();
805 final long startTime = now;
806 long slept = 0;
807 long lastLogTime = 0;
808 long lastCountChange = startTime;
809 int count = countOfRegionServers();
810 int oldCount = 0;
811 while (
812 !this.master.isStopped() &&
813 count < maxToStart &&
814 (lastCountChange+interval > now || timeout > slept || count < minToStart)
815 ){
816
817
818 if (oldCount != count || lastLogTime+interval < now){
819 lastLogTime = now;
820 String msg =
821 "Waiting for region servers count to settle; currently"+
822 " checked in " + count + ", slept for " + slept + " ms," +
823 " expecting minimum of " + minToStart + ", maximum of "+ maxToStart+
824 ", timeout of "+timeout+" ms, interval of "+interval+" ms.";
825 LOG.info(msg);
826 status.setStatus(msg);
827 }
828
829
830 final long sleepTime = 50;
831 Thread.sleep(sleepTime);
832 now = System.currentTimeMillis();
833 slept = now - startTime;
834
835 oldCount = count;
836 count = countOfRegionServers();
837 if (count != oldCount) {
838 lastCountChange = now;
839 }
840 }
841
842 LOG.info("Finished waiting for region servers count to settle;" +
843 " checked in " + count + ", slept for " + slept + " ms," +
844 " expecting minimum of " + minToStart + ", maximum of "+ maxToStart+","+
845 " master is "+ (this.master.isStopped() ? "stopped.": "running.")
846 );
847 }
848
849
850
851
852 public List<ServerName> getOnlineServersList() {
853
854
855 return new ArrayList<ServerName>(this.onlineServers.keySet());
856 }
857
858
859
860
861 public List<ServerName> getDrainingServersList() {
862 return new ArrayList<ServerName>(this.drainingServers);
863 }
864
865
866
867
868 Set<ServerName> getDeadNotExpiredServers() {
869 return new HashSet<ServerName>(this.queuedDeadServers);
870 }
871
872
873
874
875
876 Map<ServerName, Boolean> getRequeuedDeadServers() {
877 return Collections.unmodifiableMap(this.requeuedDeadServers);
878 }
879
880 public boolean isServerOnline(ServerName serverName) {
881 return serverName != null && onlineServers.containsKey(serverName);
882 }
883
884
885
886
887
888
889
890 public synchronized boolean isServerDead(ServerName serverName) {
891 return serverName == null || deadservers.isDeadServer(serverName)
892 || queuedDeadServers.contains(serverName)
893 || requeuedDeadServers.containsKey(serverName);
894 }
895
896 public void shutdownCluster() {
897 this.clusterShutdown = true;
898 this.master.stop("Cluster shutdown requested");
899 }
900
901 public boolean isClusterShutdown() {
902 return this.clusterShutdown;
903 }
904
905
906
907
908 public void stop() {
909 if (connection != null) {
910 try {
911 connection.close();
912 } catch (IOException e) {
913 LOG.error("Attempt to close connection to master failed", e);
914 }
915 }
916 }
917
918
919
920
921
922
923 public List<ServerName> createDestinationServersList(final ServerName serverToExclude){
924 final List<ServerName> destServers = getOnlineServersList();
925
926 if (serverToExclude != null){
927 destServers.remove(serverToExclude);
928 }
929
930
931 final List<ServerName> drainingServersCopy = getDrainingServersList();
932 if (!drainingServersCopy.isEmpty()) {
933 for (final ServerName server: drainingServersCopy) {
934 destServers.remove(server);
935 }
936 }
937
938
939 removeDeadNotExpiredServers(destServers);
940
941 return destServers;
942 }
943
944
945
946
947 public List<ServerName> createDestinationServersList(){
948 return createDestinationServersList(null);
949 }
950
951
952
953
954
955
956
957 void removeDeadNotExpiredServers(List<ServerName> servers) {
958 Set<ServerName> deadNotExpiredServersCopy = this.getDeadNotExpiredServers();
959 if (!deadNotExpiredServersCopy.isEmpty()) {
960 for (ServerName server : deadNotExpiredServersCopy) {
961 LOG.debug("Removing dead but not expired server: " + server
962 + " from eligible server pool.");
963 servers.remove(server);
964 }
965 }
966 }
967
968
969
970
971 void clearDeadServersWithSameHostNameAndPortOfOnlineServer() {
972 for (ServerName serverName : getOnlineServersList()) {
973 deadservers.cleanAllPreviousInstances(serverName);
974 }
975 }
976 }