View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.IOException;
22  import java.net.InetAddress;
23  import java.util.ArrayList;
24  import java.util.Collections;
25  import java.util.HashMap;
26  import java.util.HashSet;
27  import java.util.Iterator;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Map.Entry;
31  import java.util.Set;
32  import java.util.SortedMap;
33  import java.util.concurrent.ConcurrentHashMap;
34  import java.util.concurrent.ConcurrentSkipListMap;
35  import java.util.concurrent.CopyOnWriteArrayList;
36  
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.hadoop.classification.InterfaceAudience;
40  import org.apache.hadoop.conf.Configuration;
41  import org.apache.hadoop.hbase.ClockOutOfSyncException;
42  import org.apache.hadoop.hbase.HRegionInfo;
43  import org.apache.hadoop.hbase.RegionLoad;
44  import org.apache.hadoop.hbase.Server;
45  import org.apache.hadoop.hbase.ServerLoad;
46  import org.apache.hadoop.hbase.ServerName;
47  import org.apache.hadoop.hbase.YouAreDeadException;
48  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
49  import org.apache.hadoop.hbase.client.HConnection;
50  import org.apache.hadoop.hbase.client.HConnectionManager;
51  import org.apache.hadoop.hbase.client.RetriesExhaustedException;
52  import org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler;
53  import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
54  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
55  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
56  import org.apache.hadoop.hbase.protobuf.RequestConverter;
57  import org.apache.hadoop.hbase.protobuf.ResponseConverter;
58  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
59  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
60  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
61  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
62  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
63  import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
64  import org.apache.hadoop.hbase.util.Bytes;
65  import org.apache.hadoop.hbase.util.Triple;
66  
67  import com.google.common.annotations.VisibleForTesting;
68  import com.google.protobuf.ServiceException;
69  
70  /**
71   * The ServerManager class manages info about region servers.
72   * <p>
73   * Maintains lists of online and dead servers.  Processes the startups,
74   * shutdowns, and deaths of region servers.
75   * <p>
76   * Servers are distinguished in two different ways.  A given server has a
77   * location, specified by hostname and port, and of which there can only be one
78   * online at any given time.  A server instance is specified by the location
79   * (hostname and port) as well as the startcode (timestamp from when the server
80   * was started).  This is used to differentiate a restarted instance of a given
81   * server from the original instance.
82   * <p>
83   * If a sever is known not to be running any more, it is called dead. The dead
84   * server needs to be handled by a ServerShutdownHandler.  If the handler is not
85   * enabled yet, the server can't be handled right away so it is queued up.
86   * After the handler is enabled, the server will be submitted to a handler to handle.
87   * However, the handler may be just partially enabled.  If so,
88   * the server cannot be fully processed, and be queued up for further processing.
89   * A server is fully processed only after the handler is fully enabled
90   * and has completed the handling.
91   */
92  @InterfaceAudience.Private
93  public class ServerManager {
94    public static final String WAIT_ON_REGIONSERVERS_MAXTOSTART =
95        "hbase.master.wait.on.regionservers.maxtostart";
96  
97    public static final String WAIT_ON_REGIONSERVERS_MINTOSTART =
98        "hbase.master.wait.on.regionservers.mintostart";
99  
100   public static final String WAIT_ON_REGIONSERVERS_TIMEOUT =
101       "hbase.master.wait.on.regionservers.timeout";
102 
103   public static final String WAIT_ON_REGIONSERVERS_INTERVAL =
104       "hbase.master.wait.on.regionservers.interval";
105 
106   private static final Log LOG = LogFactory.getLog(ServerManager.class);
107 
108   // Set if we are to shutdown the cluster.
109   private volatile boolean clusterShutdown = false;
110 
111   private final SortedMap<byte[], Long> flushedSequenceIdByRegion =
112     new ConcurrentSkipListMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
113 
114   /** Map of registered servers to their current load */
115   private final ConcurrentHashMap<ServerName, ServerLoad> onlineServers =
116     new ConcurrentHashMap<ServerName, ServerLoad>();
117 
118   /**
119    * Map of admin interfaces per registered regionserver; these interfaces we use to control
120    * regionservers out on the cluster
121    */
122   private final Map<ServerName, AdminService.BlockingInterface> rsAdmins =
123     new HashMap<ServerName, AdminService.BlockingInterface>();
124 
125   /**
126    * List of region servers <ServerName> that should not get any more new
127    * regions.
128    */
129   private final ArrayList<ServerName> drainingServers =
130     new ArrayList<ServerName>();
131 
132   private final Server master;
133   private final MasterServices services;
134   private final HConnection connection;
135 
136   private final DeadServer deadservers = new DeadServer();
137 
138   private final long maxSkew;
139   private final long warningSkew;
140 
141   /**
142    * Set of region servers which are dead but not processed immediately. If one
143    * server died before master enables ServerShutdownHandler, the server will be
144    * added to this set and will be processed through calling
145    * {@link ServerManager#processQueuedDeadServers()} by master.
146    * <p>
147    * A dead server is a server instance known to be dead, not listed in the /hbase/rs
148    * znode any more. It may have not been submitted to ServerShutdownHandler yet
149    * because the handler is not enabled.
150    * <p>
151    * A dead server, which has been submitted to ServerShutdownHandler while the
152    * handler is not enabled, is queued up.
153    * <p>
154    * So this is a set of region servers known to be dead but not submitted to
155    * ServerShutdownHander for processing yet.
156    */
157   private Set<ServerName> queuedDeadServers = new HashSet<ServerName>();
158 
159   /**
160    * Set of region servers which are dead and submitted to ServerShutdownHandler to process but not
161    * fully processed immediately.
162    * <p>
163    * If one server died before assignment manager finished the failover cleanup, the server will be
164    * added to this set and will be processed through calling
165    * {@link ServerManager#processQueuedDeadServers()} by assignment manager.
166    * <p>
167    * The Boolean value indicates whether log split is needed inside ServerShutdownHandler
168    * <p>
169    * ServerShutdownHandler processes a dead server submitted to the handler after the handler is
170    * enabled. It may not be able to complete the processing because meta is not yet online or master
171    * is currently in startup mode. In this case, the dead server will be parked in this set
172    * temporarily.
173    */
174   private Map<ServerName, Boolean> requeuedDeadServers
175     = new ConcurrentHashMap<ServerName, Boolean>();
176 
177   /** Listeners that are called on server events. */
178   private List<ServerListener> listeners = new CopyOnWriteArrayList<ServerListener>();
179 
180   /**
181    * Constructor.
182    * @param master
183    * @param services
184    * @throws ZooKeeperConnectionException
185    */
186   public ServerManager(final Server master, final MasterServices services)
187       throws IOException {
188     this(master, services, true);
189   }
190 
191   @SuppressWarnings("deprecation")
192   ServerManager(final Server master, final MasterServices services,
193       final boolean connect) throws IOException {
194     this.master = master;
195     this.services = services;
196     Configuration c = master.getConfiguration();
197     maxSkew = c.getLong("hbase.master.maxclockskew", 30000);
198     warningSkew = c.getLong("hbase.master.warningclockskew", 10000);
199     this.connection = connect ? HConnectionManager.getConnection(c) : null;
200   }
201 
202   /**
203    * Add the listener to the notification list.
204    * @param listener The ServerListener to register
205    */
206   public void registerListener(final ServerListener listener) {
207     this.listeners.add(listener);
208   }
209 
210   /**
211    * Remove the listener from the notification list.
212    * @param listener The ServerListener to unregister
213    */
214   public boolean unregisterListener(final ServerListener listener) {
215     return this.listeners.remove(listener);
216   }
217 
218   /**
219    * Let the server manager know a new regionserver has come online
220    * @param ia The remote address
221    * @param port The remote port
222    * @param serverStartcode
223    * @param serverCurrentTime The current time of the region server in ms
224    * @return The ServerName we know this server as.
225    * @throws IOException
226    */
227   ServerName regionServerStartup(final InetAddress ia, final int port,
228     final long serverStartcode, long serverCurrentTime)
229   throws IOException {
230     // Test for case where we get a region startup message from a regionserver
231     // that has been quickly restarted but whose znode expiration handler has
232     // not yet run, or from a server whose fail we are currently processing.
233     // Test its host+port combo is present in serverAddresstoServerInfo.  If it
234     // is, reject the server and trigger its expiration. The next time it comes
235     // in, it should have been removed from serverAddressToServerInfo and queued
236     // for processing by ProcessServerShutdown.
237     ServerName sn = ServerName.valueOf(ia.getHostName(), port, serverStartcode);
238     checkClockSkew(sn, serverCurrentTime);
239     checkIsDead(sn, "STARTUP");
240     if (!checkAndRecordNewServer(sn, ServerLoad.EMPTY_SERVERLOAD)) {
241       LOG.warn("THIS SHOULD NOT HAPPEN, RegionServerStartup"
242         + " could not record the server: " + sn);
243     }
244     return sn;
245   }
246 
247   /**
248    * Updates last flushed sequence Ids for the regions on server sn
249    * @param sn
250    * @param hsl
251    */
252   private void updateLastFlushedSequenceIds(ServerName sn, ServerLoad hsl) {
253     Map<byte[], RegionLoad> regionsLoad = hsl.getRegionsLoad();
254     for (Entry<byte[], RegionLoad> entry : regionsLoad.entrySet()) {
255       Long existingValue = flushedSequenceIdByRegion.get(entry.getKey());
256       long l = entry.getValue().getCompleteSequenceId();
257       if (existingValue != null) {
258         if (l != -1 && l < existingValue) {
259           LOG.warn("RegionServer " + sn +
260               " indicates a last flushed sequence id (" + entry.getValue() +
261               ") that is less than the previous last flushed sequence id (" +
262               existingValue + ") for region " +
263               Bytes.toString(entry.getKey()) + " Ignoring.");
264 
265           continue; // Don't let smaller sequence ids override greater
266           // sequence ids.
267         }
268       }
269       flushedSequenceIdByRegion.put(entry.getKey(), l);
270     }
271   }
272 
273   void regionServerReport(ServerName sn,
274       ServerLoad sl) throws YouAreDeadException {
275     checkIsDead(sn, "REPORT");
276     if (null == this.onlineServers.replace(sn, sl)) {
277       // Already have this host+port combo and its just different start code?
278       // Just let the server in. Presume master joining a running cluster.
279       // recordNewServer is what happens at the end of reportServerStartup.
280       // The only thing we are skipping is passing back to the regionserver
281       // the ServerName to use. Here we presume a master has already done
282       // that so we'll press on with whatever it gave us for ServerName.
283       if (!checkAndRecordNewServer(sn, sl)) {
284         LOG.info("RegionServerReport ignored, could not record the server: " + sn);
285         return; // Not recorded, so no need to move on
286       }
287     }
288     updateLastFlushedSequenceIds(sn, sl);
289   }
290 
291   /**
292    * Check is a server of same host and port already exists,
293    * if not, or the existed one got a smaller start code, record it.
294    *
295    * @param sn the server to check and record
296    * @param sl the server load on the server
297    * @return true if the server is recorded, otherwise, false
298    */
299   boolean checkAndRecordNewServer(
300       final ServerName serverName, final ServerLoad sl) {
301     ServerName existingServer = null;
302     synchronized (this.onlineServers) {
303       existingServer = findServerWithSameHostnamePortWithLock(serverName);
304       if (existingServer != null && (existingServer.getStartcode() > serverName.getStartcode())) {
305         LOG.info("Server serverName=" + serverName + " rejected; we already have "
306             + existingServer.toString() + " registered with same hostname and port");
307         return false;
308       }
309       recordNewServerWithLock(serverName, sl);
310     }
311 
312     // Tell our listeners that a server was added
313     if (!this.listeners.isEmpty()) {
314       for (ServerListener listener : this.listeners) {
315         listener.serverAdded(serverName);
316       }
317     }
318 
319     // Note that we assume that same ts means same server, and don't expire in that case.
320     //  TODO: ts can theoretically collide due to clock shifts, so this is a bit hacky.
321     if (existingServer != null && (existingServer.getStartcode() < serverName.getStartcode())) {
322       LOG.info("Triggering server recovery; existingServer " +
323           existingServer + " looks stale, new server:" + serverName);
324       expireServer(existingServer);
325     }
326     return true;
327   }
328 
329   /**
330    * Checks if the clock skew between the server and the master. If the clock skew exceeds the
331    * configured max, it will throw an exception; if it exceeds the configured warning threshold,
332    * it will log a warning but start normally.
333    * @param serverName Incoming servers's name
334    * @param serverCurrentTime
335    * @throws ClockOutOfSyncException if the skew exceeds the configured max value
336    */
337   private void checkClockSkew(final ServerName serverName, final long serverCurrentTime)
338   throws ClockOutOfSyncException {
339     long skew = Math.abs(System.currentTimeMillis() - serverCurrentTime);
340     if (skew > maxSkew) {
341       String message = "Server " + serverName + " has been " +
342         "rejected; Reported time is too far out of sync with master.  " +
343         "Time difference of " + skew + "ms > max allowed of " + maxSkew + "ms";
344       LOG.warn(message);
345       throw new ClockOutOfSyncException(message);
346     } else if (skew > warningSkew){
347       String message = "Reported time for server " + serverName + " is out of sync with master " +
348         "by " + skew + "ms. (Warning threshold is " + warningSkew + "ms; " +
349         "error threshold is " + maxSkew + "ms)";
350       LOG.warn(message);
351     }
352   }
353 
354   /**
355    * If this server is on the dead list, reject it with a YouAreDeadException.
356    * If it was dead but came back with a new start code, remove the old entry
357    * from the dead list.
358    * @param serverName
359    * @param what START or REPORT
360    * @throws org.apache.hadoop.hbase.YouAreDeadException
361    */
362   private void checkIsDead(final ServerName serverName, final String what)
363       throws YouAreDeadException {
364     if (this.deadservers.isDeadServer(serverName)) {
365       // host name, port and start code all match with existing one of the
366       // dead servers. So, this server must be dead.
367       String message = "Server " + what + " rejected; currently processing " +
368           serverName + " as dead server";
369       LOG.debug(message);
370       throw new YouAreDeadException(message);
371     }
372     // remove dead server with same hostname and port of newly checking in rs after master
373     // initialization.See HBASE-5916 for more information.
374     if ((this.services == null || ((HMaster) this.services).isInitialized())
375         && this.deadservers.cleanPreviousInstance(serverName)) {
376       // This server has now become alive after we marked it as dead.
377       // We removed it's previous entry from the dead list to reflect it.
378       LOG.debug(what + ":" + " Server " + serverName + " came back up," +
379           " removed it from the dead servers list");
380     }
381   }
382 
383   /**
384    * Assumes onlineServers is locked.
385    * @return ServerName with matching hostname and port.
386    */
387   private ServerName findServerWithSameHostnamePortWithLock(
388       final ServerName serverName) {
389     for (ServerName sn: this.onlineServers.keySet()) {
390       if (ServerName.isSameHostnameAndPort(serverName, sn)) return sn;
391     }
392     return null;
393   }
394 
395   /**
396    * Adds the onlineServers list. onlineServers should be locked.
397    * @param serverName The remote servers name.
398    * @param sl
399    * @return Server load from the removed server, if any.
400    */
401   @VisibleForTesting
402   void recordNewServerWithLock(final ServerName serverName, final ServerLoad sl) {
403     LOG.info("Registering server=" + serverName);
404     this.onlineServers.put(serverName, sl);
405     this.rsAdmins.remove(serverName);
406   }
407 
408   public long getLastFlushedSequenceId(byte[] regionName) {
409     long seqId = -1;
410     if (flushedSequenceIdByRegion.containsKey(regionName)) {
411       seqId = flushedSequenceIdByRegion.get(regionName);
412     }
413     return seqId;
414   }
415 
416   /**
417    * @param serverName
418    * @return ServerLoad if serverName is known else null
419    */
420   public ServerLoad getLoad(final ServerName serverName) {
421     return this.onlineServers.get(serverName);
422   }
423 
424   /**
425    * Compute the average load across all region servers.
426    * Currently, this uses a very naive computation - just uses the number of
427    * regions being served, ignoring stats about number of requests.
428    * @return the average load
429    */
430   public double getAverageLoad() {
431     int totalLoad = 0;
432     int numServers = 0;
433     double averageLoad;
434     for (ServerLoad sl: this.onlineServers.values()) {
435         numServers++;
436         totalLoad += sl.getNumberOfRegions();
437     }
438     averageLoad = (double)totalLoad / (double)numServers;
439     return averageLoad;
440   }
441 
442   /** @return the count of active regionservers */
443   int countOfRegionServers() {
444     // Presumes onlineServers is a concurrent map
445     return this.onlineServers.size();
446   }
447 
448   /**
449    * @return Read-only map of servers to serverinfo
450    */
451   public Map<ServerName, ServerLoad> getOnlineServers() {
452     // Presumption is that iterating the returned Map is OK.
453     synchronized (this.onlineServers) {
454       return Collections.unmodifiableMap(this.onlineServers);
455     }
456   }
457 
458 
459   public DeadServer getDeadServers() {
460     return this.deadservers;
461   }
462 
463   /**
464    * Checks if any dead servers are currently in progress.
465    * @return true if any RS are being processed as dead, false if not
466    */
467   public boolean areDeadServersInProgress() {
468     return this.deadservers.areDeadServersInProgress();
469   }
470 
471   void letRegionServersShutdown() {
472     long previousLogTime = 0;
473     int onlineServersCt;
474     while ((onlineServersCt = onlineServers.size()) > 0) {
475 
476       if (System.currentTimeMillis() > (previousLogTime + 1000)) {
477         StringBuilder sb = new StringBuilder();
478         // It's ok here to not sync on onlineServers - merely logging
479         for (ServerName key : this.onlineServers.keySet()) {
480           if (sb.length() > 0) {
481             sb.append(", ");
482           }
483           sb.append(key);
484         }
485         LOG.info("Waiting on regionserver(s) to go down " + sb.toString());
486         previousLogTime = System.currentTimeMillis();
487       }
488 
489       synchronized (onlineServers) {
490         try {
491           if (onlineServersCt == onlineServers.size()) onlineServers.wait(100);
492         } catch (InterruptedException ignored) {
493           // continue
494         }
495       }
496     }
497   }
498 
499   /*
500    * Expire the passed server.  Add it to list of dead servers and queue a
501    * shutdown processing.
502    */
503   public synchronized void expireServer(final ServerName serverName) {
504     if (!services.isServerShutdownHandlerEnabled()) {
505       LOG.info("Master doesn't enable ServerShutdownHandler during initialization, "
506           + "delay expiring server " + serverName);
507       this.queuedDeadServers.add(serverName);
508       return;
509     }
510     if (this.deadservers.isDeadServer(serverName)) {
511       // TODO: Can this happen?  It shouldn't be online in this case?
512       LOG.warn("Expiration of " + serverName +
513           " but server shutdown already in progress");
514       return;
515     }
516     synchronized (onlineServers) {
517       if (!this.onlineServers.containsKey(serverName)) {
518         LOG.warn("Expiration of " + serverName + " but server not online");
519       }
520       // Remove the server from the known servers lists and update load info BUT
521       // add to deadservers first; do this so it'll show in dead servers list if
522       // not in online servers list.
523       this.deadservers.add(serverName);
524       this.onlineServers.remove(serverName);
525       onlineServers.notifyAll();
526     }
527     this.rsAdmins.remove(serverName);
528     // If cluster is going down, yes, servers are going to be expiring; don't
529     // process as a dead server
530     if (this.clusterShutdown) {
531       LOG.info("Cluster shutdown set; " + serverName +
532         " expired; onlineServers=" + this.onlineServers.size());
533       if (this.onlineServers.isEmpty()) {
534         master.stop("Cluster shutdown set; onlineServer=0");
535       }
536       return;
537     }
538 
539     boolean carryingMeta = services.getAssignmentManager().isCarryingMeta(serverName);
540     if (carryingMeta) {
541       this.services.getExecutorService().submit(new MetaServerShutdownHandler(this.master,
542         this.services, this.deadservers, serverName));
543     } else {
544       this.services.getExecutorService().submit(new ServerShutdownHandler(this.master,
545         this.services, this.deadservers, serverName, true));
546     }
547     LOG.debug("Added=" + serverName +
548       " to dead servers, submitted shutdown handler to be executed meta=" + carryingMeta);
549 
550     // Tell our listeners that a server was removed
551     if (!this.listeners.isEmpty()) {
552       for (ServerListener listener : this.listeners) {
553         listener.serverRemoved(serverName);
554       }
555     }
556   }
557 
558   public synchronized void processDeadServer(final ServerName serverName) {
559     this.processDeadServer(serverName, false);
560   }
561 
562   public synchronized void processDeadServer(final ServerName serverName, boolean shouldSplitHlog) {
563     // When assignment manager is cleaning up the zookeeper nodes and rebuilding the
564     // in-memory region states, region servers could be down. Meta table can and
565     // should be re-assigned, log splitting can be done too. However, it is better to
566     // wait till the cleanup is done before re-assigning user regions.
567     //
568     // We should not wait in the server shutdown handler thread since it can clog
569     // the handler threads and meta table could not be re-assigned in case
570     // the corresponding server is down. So we queue them up here instead.
571     if (!services.getAssignmentManager().isFailoverCleanupDone()) {
572       requeuedDeadServers.put(serverName, shouldSplitHlog);
573       return;
574     }
575 
576     this.deadservers.add(serverName);
577     this.services.getExecutorService().submit(
578       new ServerShutdownHandler(this.master, this.services, this.deadservers, serverName,
579           shouldSplitHlog));
580   }
581 
582   /**
583    * Process the servers which died during master's initialization. It will be
584    * called after HMaster#assignMeta and AssignmentManager#joinCluster.
585    * */
586   synchronized void processQueuedDeadServers() {
587     if (!services.isServerShutdownHandlerEnabled()) {
588       LOG.info("Master hasn't enabled ServerShutdownHandler");
589     }
590     Iterator<ServerName> serverIterator = queuedDeadServers.iterator();
591     while (serverIterator.hasNext()) {
592       ServerName tmpServerName = serverIterator.next();
593       expireServer(tmpServerName);
594       serverIterator.remove();
595       requeuedDeadServers.remove(tmpServerName);
596     }
597 
598     if (!services.getAssignmentManager().isFailoverCleanupDone()) {
599       LOG.info("AssignmentManager hasn't finished failover cleanup; waiting");
600     }
601 
602     for(ServerName tmpServerName : requeuedDeadServers.keySet()){
603       processDeadServer(tmpServerName, requeuedDeadServers.get(tmpServerName));
604     }
605     requeuedDeadServers.clear();
606   }
607 
608   /*
609    * Remove the server from the drain list.
610    */
611   public boolean removeServerFromDrainList(final ServerName sn) {
612     // Warn if the server (sn) is not online.  ServerName is of the form:
613     // <hostname> , <port> , <startcode>
614 
615     if (!this.isServerOnline(sn)) {
616       LOG.warn("Server " + sn + " is not currently online. " +
617                "Removing from draining list anyway, as requested.");
618     }
619     // Remove the server from the draining servers lists.
620     return this.drainingServers.remove(sn);
621   }
622 
623   /*
624    * Add the server to the drain list.
625    */
626   public boolean addServerToDrainList(final ServerName sn) {
627     // Warn if the server (sn) is not online.  ServerName is of the form:
628     // <hostname> , <port> , <startcode>
629 
630     if (!this.isServerOnline(sn)) {
631       LOG.warn("Server " + sn + " is not currently online. " +
632                "Ignoring request to add it to draining list.");
633       return false;
634     }
635     // Add the server to the draining servers lists, if it's not already in
636     // it.
637     if (this.drainingServers.contains(sn)) {
638       LOG.warn("Server " + sn + " is already in the draining server list." +
639                "Ignoring request to add it again.");
640       return false;
641     }
642     return this.drainingServers.add(sn);
643   }
644 
645   // RPC methods to region servers
646 
647   /**
648    * Sends an OPEN RPC to the specified server to open the specified region.
649    * <p>
650    * Open should not fail but can if server just crashed.
651    * <p>
652    * @param server server to open a region
653    * @param region region to open
654    * @param versionOfOfflineNode that needs to be present in the offline node
655    * when RS tries to change the state from OFFLINE to other states.
656    * @param favoredNodes
657    */
658   public RegionOpeningState sendRegionOpen(final ServerName server,
659       HRegionInfo region, int versionOfOfflineNode, List<ServerName> favoredNodes)
660   throws IOException {
661     AdminService.BlockingInterface admin = getRsAdmin(server);
662     if (admin == null) {
663       LOG.warn("Attempting to send OPEN RPC to server " + server.toString() +
664         " failed because no RPC connection found to this server");
665       return RegionOpeningState.FAILED_OPENING;
666     }
667     OpenRegionRequest request = RequestConverter.buildOpenRegionRequest(server, 
668       region, versionOfOfflineNode, favoredNodes, 
669       (RecoveryMode.LOG_REPLAY == this.services.getMasterFileSystem().getLogRecoveryMode()));
670     try {
671       OpenRegionResponse response = admin.openRegion(null, request);
672       return ResponseConverter.getRegionOpeningState(response);
673     } catch (ServiceException se) {
674       throw ProtobufUtil.getRemoteException(se);
675     }
676   }
677 
678   /**
679    * Sends an OPEN RPC to the specified server to open the specified region.
680    * <p>
681    * Open should not fail but can if server just crashed.
682    * <p>
683    * @param server server to open a region
684    * @param regionOpenInfos info of a list of regions to open
685    * @return a list of region opening states
686    */
687   public List<RegionOpeningState> sendRegionOpen(ServerName server,
688       List<Triple<HRegionInfo, Integer, List<ServerName>>> regionOpenInfos)
689   throws IOException {
690     AdminService.BlockingInterface admin = getRsAdmin(server);
691     if (admin == null) {
692       LOG.warn("Attempting to send OPEN RPC to server " + server.toString() +
693         " failed because no RPC connection found to this server");
694       return null;
695     }
696 
697     OpenRegionRequest request = RequestConverter.buildOpenRegionRequest(regionOpenInfos, 
698       (RecoveryMode.LOG_REPLAY == this.services.getMasterFileSystem().getLogRecoveryMode()));
699     try {
700       OpenRegionResponse response = admin.openRegion(null, request);
701       return ResponseConverter.getRegionOpeningStateList(response);
702     } catch (ServiceException se) {
703       throw ProtobufUtil.getRemoteException(se);
704     }
705   }
706 
707   /**
708    * Sends an CLOSE RPC to the specified server to close the specified region.
709    * <p>
710    * A region server could reject the close request because it either does not
711    * have the specified region or the region is being split.
712    * @param server server to open a region
713    * @param region region to open
714    * @param versionOfClosingNode
715    *   the version of znode to compare when RS transitions the znode from
716    *   CLOSING state.
717    * @param dest - if the region is moved to another server, the destination server. null otherwise.
718    * @return true if server acknowledged close, false if not
719    * @throws IOException
720    */
721   public boolean sendRegionClose(ServerName server, HRegionInfo region,
722     int versionOfClosingNode, ServerName dest, boolean transitionInZK) throws IOException {
723     if (server == null) throw new NullPointerException("Passed server is null");
724     AdminService.BlockingInterface admin = getRsAdmin(server);
725     if (admin == null) {
726       throw new IOException("Attempting to send CLOSE RPC to server " +
727         server.toString() + " for region " +
728         region.getRegionNameAsString() +
729         " failed because no RPC connection found to this server");
730     }
731     return ProtobufUtil.closeRegion(admin, server, region.getRegionName(),
732       versionOfClosingNode, dest, transitionInZK);
733   }
734 
735   public boolean sendRegionClose(ServerName server,
736       HRegionInfo region, int versionOfClosingNode) throws IOException {
737     return sendRegionClose(server, region, versionOfClosingNode, null, true);
738   }
739 
740   /**
741    * Sends an MERGE REGIONS RPC to the specified server to merge the specified
742    * regions.
743    * <p>
744    * A region server could reject the close request because it either does not
745    * have the specified region.
746    * @param server server to merge regions
747    * @param region_a region to merge
748    * @param region_b region to merge
749    * @param forcible true if do a compulsory merge, otherwise we will only merge
750    *          two adjacent regions
751    * @throws IOException
752    */
753   public void sendRegionsMerge(ServerName server, HRegionInfo region_a,
754       HRegionInfo region_b, boolean forcible) throws IOException {
755     if (server == null)
756       throw new NullPointerException("Passed server is null");
757     if (region_a == null || region_b == null)
758       throw new NullPointerException("Passed region is null");
759     AdminService.BlockingInterface admin = getRsAdmin(server);
760     if (admin == null) {
761       throw new IOException("Attempting to send MERGE REGIONS RPC to server "
762           + server.toString() + " for region "
763           + region_a.getRegionNameAsString() + ","
764           + region_b.getRegionNameAsString()
765           + " failed because no RPC connection found to this server");
766     }
767     ProtobufUtil.mergeRegions(admin, region_a, region_b, forcible);
768   }
769 
770   /**
771    * Check if a region server is reachable and has the expected start code
772    */
773   public boolean isServerReachable(ServerName server) {
774     if (server == null) throw new NullPointerException("Passed server is null");
775     int maximumAttempts = Math.max(1, master.getConfiguration().getInt(
776       "hbase.master.maximum.ping.server.attempts", 10));
777     for (int i = 0; i < maximumAttempts; i++) {
778       try {
779         AdminService.BlockingInterface admin = getRsAdmin(server);
780         if (admin != null) {
781           ServerInfo info = ProtobufUtil.getServerInfo(admin);
782           return info != null && info.hasServerName()
783             && server.getStartcode() == info.getServerName().getStartCode();
784         }
785       } catch (IOException ioe) {
786         LOG.debug("Couldn't reach " + server + ", try=" + i
787           + " of " + maximumAttempts, ioe);
788       }
789     }
790     return false;
791   }
792 
793     /**
794     * @param sn
795     * @return Admin interface for the remote regionserver named <code>sn</code>
796     * @throws IOException
797     * @throws RetriesExhaustedException wrapping a ConnectException if failed
798     */
799   private AdminService.BlockingInterface getRsAdmin(final ServerName sn)
800   throws IOException {
801     AdminService.BlockingInterface admin = this.rsAdmins.get(sn);
802     if (admin == null) {
803       LOG.debug("New admin connection to " + sn.toString());
804       admin = this.connection.getAdmin(sn);
805       this.rsAdmins.put(sn, admin);
806     }
807     return admin;
808   }
809 
810   /**
811    * Wait for the region servers to report in.
812    * We will wait until one of this condition is met:
813    *  - the master is stopped
814    *  - the 'hbase.master.wait.on.regionservers.maxtostart' number of
815    *    region servers is reached
816    *  - the 'hbase.master.wait.on.regionservers.mintostart' is reached AND
817    *   there have been no new region server in for
818    *      'hbase.master.wait.on.regionservers.interval' time AND
819    *   the 'hbase.master.wait.on.regionservers.timeout' is reached
820    *
821    * @throws InterruptedException
822    */
823   public void waitForRegionServers(MonitoredTask status)
824   throws InterruptedException {
825     final long interval = this.master.getConfiguration().
826       getLong(WAIT_ON_REGIONSERVERS_INTERVAL, 1500);
827     final long timeout = this.master.getConfiguration().
828       getLong(WAIT_ON_REGIONSERVERS_TIMEOUT, 4500);
829     int minToStart = this.master.getConfiguration().
830       getInt(WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
831     if (minToStart < 1) {
832       LOG.warn(String.format(
833         "The value of '%s' (%d) can not be less than 1, ignoring.",
834         WAIT_ON_REGIONSERVERS_MINTOSTART, minToStart));
835       minToStart = 1;
836     }
837     int maxToStart = this.master.getConfiguration().
838       getInt(WAIT_ON_REGIONSERVERS_MAXTOSTART, Integer.MAX_VALUE);
839     if (maxToStart < minToStart) {
840         LOG.warn(String.format(
841             "The value of '%s' (%d) is set less than '%s' (%d), ignoring.",
842             WAIT_ON_REGIONSERVERS_MAXTOSTART, maxToStart,
843             WAIT_ON_REGIONSERVERS_MINTOSTART, minToStart));
844         maxToStart = Integer.MAX_VALUE;
845     }
846 
847     long now =  System.currentTimeMillis();
848     final long startTime = now;
849     long slept = 0;
850     long lastLogTime = 0;
851     long lastCountChange = startTime;
852     int count = countOfRegionServers();
853     int oldCount = 0;
854     while (
855       !this.master.isStopped() &&
856         count < maxToStart &&
857         (lastCountChange+interval > now || timeout > slept || count < minToStart)
858       ){
859 
860       // Log some info at every interval time or if there is a change
861       if (oldCount != count || lastLogTime+interval < now){
862         lastLogTime = now;
863         String msg =
864           "Waiting for region servers count to settle; currently"+
865             " checked in " + count + ", slept for " + slept + " ms," +
866             " expecting minimum of " + minToStart + ", maximum of "+ maxToStart+
867             ", timeout of "+timeout+" ms, interval of "+interval+" ms.";
868         LOG.info(msg);
869         status.setStatus(msg);
870       }
871 
872       // We sleep for some time
873       final long sleepTime = 50;
874       Thread.sleep(sleepTime);
875       now =  System.currentTimeMillis();
876       slept = now - startTime;
877 
878       oldCount = count;
879       count = countOfRegionServers();
880       if (count != oldCount) {
881         lastCountChange = now;
882       }
883     }
884 
885     LOG.info("Finished waiting for region servers count to settle;" +
886       " checked in " + count + ", slept for " + slept + " ms," +
887       " expecting minimum of " + minToStart + ", maximum of "+ maxToStart+","+
888       " master is "+ (this.master.isStopped() ? "stopped.": "running.")
889     );
890   }
891 
892   /**
893    * @return A copy of the internal list of online servers.
894    */
895   public List<ServerName> getOnlineServersList() {
896     // TODO: optimize the load balancer call so we don't need to make a new list
897     // TODO: FIX. THIS IS POPULAR CALL.
898     return new ArrayList<ServerName>(this.onlineServers.keySet());
899   }
900 
901   /**
902    * @return A copy of the internal list of draining servers.
903    */
904   public List<ServerName> getDrainingServersList() {
905     return new ArrayList<ServerName>(this.drainingServers);
906   }
907 
908   /**
909    * @return A copy of the internal set of deadNotExpired servers.
910    */
911   Set<ServerName> getDeadNotExpiredServers() {
912     return new HashSet<ServerName>(this.queuedDeadServers);
913   }
914 
915   /**
916    * During startup, if we figure it is not a failover, i.e. there is
917    * no more HLog files to split, we won't try to recover these dead servers.
918    * So we just remove them from the queue. Use caution in calling this.
919    */
920   void removeRequeuedDeadServers() {
921     requeuedDeadServers.clear();
922   }
923 
924   /**
925    * @return A copy of the internal map of requeuedDeadServers servers and their corresponding
926    *         splitlog need flag.
927    */
928   Map<ServerName, Boolean> getRequeuedDeadServers() {
929     return Collections.unmodifiableMap(this.requeuedDeadServers);
930   }
931 
932   public boolean isServerOnline(ServerName serverName) {
933     return serverName != null && onlineServers.containsKey(serverName);
934   }
935 
936   /**
937    * Check if a server is known to be dead.  A server can be online,
938    * or known to be dead, or unknown to this manager (i.e, not online,
939    * not known to be dead either. it is simply not tracked by the
940    * master any more, for example, a very old previous instance).
941    */
942   public synchronized boolean isServerDead(ServerName serverName) {
943     return serverName == null || deadservers.isDeadServer(serverName)
944       || queuedDeadServers.contains(serverName)
945       || requeuedDeadServers.containsKey(serverName);
946   }
947 
948   public void shutdownCluster() {
949     this.clusterShutdown = true;
950     this.master.stop("Cluster shutdown requested");
951   }
952 
953   public boolean isClusterShutdown() {
954     return this.clusterShutdown;
955   }
956 
957   /**
958    * Stop the ServerManager.  Currently closes the connection to the master.
959    */
960   public void stop() {
961     if (connection != null) {
962       try {
963         connection.close();
964       } catch (IOException e) {
965         LOG.error("Attempt to close connection to master failed", e);
966       }
967     }
968   }
969 
970   /**
971    * Creates a list of possible destinations for a region. It contains the online servers, but not
972    *  the draining or dying servers.
973    *  @param serverToExclude can be null if there is no server to exclude
974    */
975   public List<ServerName> createDestinationServersList(final ServerName serverToExclude){
976     final List<ServerName> destServers = getOnlineServersList();
977 
978     if (serverToExclude != null){
979       destServers.remove(serverToExclude);
980     }
981 
982     // Loop through the draining server list and remove them from the server list
983     final List<ServerName> drainingServersCopy = getDrainingServersList();
984     if (!drainingServersCopy.isEmpty()) {
985       for (final ServerName server: drainingServersCopy) {
986         destServers.remove(server);
987       }
988     }
989 
990     // Remove the deadNotExpired servers from the server list.
991     removeDeadNotExpiredServers(destServers);
992 
993     return destServers;
994   }
995 
996   /**
997    * Calls {@link #createDestinationServersList} without server to exclude.
998    */
999   public List<ServerName> createDestinationServersList(){
1000     return createDestinationServersList(null);
1001   }
1002 
1003     /**
1004     * Loop through the deadNotExpired server list and remove them from the
1005     * servers.
1006     * This function should be used carefully outside of this class. You should use a high level
1007     *  method such as {@link #createDestinationServersList()} instead of managing you own list.
1008     */
1009   void removeDeadNotExpiredServers(List<ServerName> servers) {
1010     Set<ServerName> deadNotExpiredServersCopy = this.getDeadNotExpiredServers();
1011     if (!deadNotExpiredServersCopy.isEmpty()) {
1012       for (ServerName server : deadNotExpiredServersCopy) {
1013         LOG.debug("Removing dead but not expired server: " + server
1014           + " from eligible server pool.");
1015         servers.remove(server);
1016       }
1017     }
1018   }
1019 
1020   /**
1021    * To clear any dead server with same host name and port of any online server
1022    */
1023   void clearDeadServersWithSameHostNameAndPortOfOnlineServer() {
1024     for (ServerName serverName : getOnlineServersList()) {
1025       deadservers.cleanAllPreviousInstances(serverName);
1026     }
1027   }
1028 }