View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.IOException;
22  import java.net.InetAddress;
23  import java.util.ArrayList;
24  import java.util.Collections;
25  import java.util.HashMap;
26  import java.util.HashSet;
27  import java.util.Iterator;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Map.Entry;
31  import java.util.Set;
32  import java.util.SortedMap;
33  import java.util.concurrent.ConcurrentHashMap;
34  import java.util.concurrent.ConcurrentSkipListMap;
35  
36  import org.apache.commons.logging.Log;
37  import org.apache.commons.logging.LogFactory;
38  import org.apache.hadoop.classification.InterfaceAudience;
39  import org.apache.hadoop.conf.Configuration;
40  import org.apache.hadoop.hbase.ClockOutOfSyncException;
41  import org.apache.hadoop.hbase.HRegionInfo;
42  import org.apache.hadoop.hbase.RegionLoad;
43  import org.apache.hadoop.hbase.Server;
44  import org.apache.hadoop.hbase.ServerLoad;
45  import org.apache.hadoop.hbase.ServerName;
46  import org.apache.hadoop.hbase.YouAreDeadException;
47  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
48  import org.apache.hadoop.hbase.client.HConnection;
49  import org.apache.hadoop.hbase.client.HConnectionManager;
50  import org.apache.hadoop.hbase.client.RetriesExhaustedException;
51  import org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler;
52  import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
53  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
54  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
55  import org.apache.hadoop.hbase.protobuf.RequestConverter;
56  import org.apache.hadoop.hbase.protobuf.ResponseConverter;
57  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
58  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
59  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
60  import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
61  import org.apache.hadoop.hbase.util.Bytes;
62  import org.apache.hadoop.hbase.util.Triple;
63  
64  import com.google.protobuf.ServiceException;
65  
66  /**
67   * The ServerManager class manages info about region servers.
68   * <p>
69   * Maintains lists of online and dead servers.  Processes the startups,
70   * shutdowns, and deaths of region servers.
71   * <p>
72   * Servers are distinguished in two different ways.  A given server has a
73   * location, specified by hostname and port, and of which there can only be one
74   * online at any given time.  A server instance is specified by the location
75   * (hostname and port) as well as the startcode (timestamp from when the server
76   * was started).  This is used to differentiate a restarted instance of a given
77   * server from the original instance.
78   * <p>
79   * If a sever is known not to be running any more, it is called dead. The dead
80   * server needs to be handled by a ServerShutdownHandler.  If the handler is not
81   * enabled yet, the server can't be handled right away so it is queued up.
82   * After the handler is enabled, the server will be submitted to a handler to handle.
83   * However, the handler may be just partially enabled.  If so,
84   * the server cannot be fully processed, and be queued up for further processing.
85   * A server is fully processed only after the handler is fully enabled
86   * and has completed the handling.
87   */
88  @InterfaceAudience.Private
89  public class ServerManager {
90    public static final String WAIT_ON_REGIONSERVERS_MAXTOSTART =
91        "hbase.master.wait.on.regionservers.maxtostart";
92  
93    public static final String WAIT_ON_REGIONSERVERS_MINTOSTART =
94        "hbase.master.wait.on.regionservers.mintostart";
95  
96    public static final String WAIT_ON_REGIONSERVERS_TIMEOUT =
97        "hbase.master.wait.on.regionservers.timeout";
98  
99    public static final String WAIT_ON_REGIONSERVERS_INTERVAL =
100       "hbase.master.wait.on.regionservers.interval";
101 
102   private static final Log LOG = LogFactory.getLog(ServerManager.class);
103 
104   // Set if we are to shutdown the cluster.
105   private volatile boolean clusterShutdown = false;
106 
107   private final SortedMap<byte[], Long> flushedSequenceIdByRegion =
108     new ConcurrentSkipListMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
109 
110   /** Map of registered servers to their current load */
111   private final Map<ServerName, ServerLoad> onlineServers =
112     new ConcurrentHashMap<ServerName, ServerLoad>();
113 
114   /**
115    * Map of admin interfaces per registered regionserver; these interfaces we use to control
116    * regionservers out on the cluster
117    */
118   private final Map<ServerName, AdminService.BlockingInterface> rsAdmins =
119     new HashMap<ServerName, AdminService.BlockingInterface>();
120 
121   /**
122    * List of region servers <ServerName> that should not get any more new
123    * regions.
124    */
125   private final ArrayList<ServerName> drainingServers =
126     new ArrayList<ServerName>();
127 
128   private final Server master;
129   private final MasterServices services;
130   private final HConnection connection;
131 
132   private final DeadServer deadservers = new DeadServer();
133 
134   private final long maxSkew;
135   private final long warningSkew;
136 
137   /**
138    * Set of region servers which are dead but not processed immediately. If one
139    * server died before master enables ServerShutdownHandler, the server will be
140    * added to this set and will be processed through calling
141    * {@link ServerManager#processQueuedDeadServers()} by master.
142    * <p>
143    * A dead server is a server instance known to be dead, not listed in the /hbase/rs
144    * znode any more. It may have not been submitted to ServerShutdownHandler yet
145    * because the handler is not enabled.
146    * <p>
147    * A dead server, which has been submitted to ServerShutdownHandler while the
148    * handler is not enabled, is queued up.
149    * <p>
150    * So this is a set of region servers known to be dead but not submitted to
151    * ServerShutdownHander for processing yet.
152    */
153   private Set<ServerName> queuedDeadServers = new HashSet<ServerName>();
154 
155   /**
156    * Set of region servers which are dead and submitted to ServerShutdownHandler to process but not
157    * fully processed immediately.
158    * <p>
159    * If one server died before assignment manager finished the failover cleanup, the server will be
160    * added to this set and will be processed through calling
161    * {@link ServerManager#processQueuedDeadServers()} by assignment manager.
162    * <p>
163    * The Boolean value indicates whether log split is needed inside ServerShutdownHandler
164    * <p>
165    * ServerShutdownHandler processes a dead server submitted to the handler after the handler is
166    * enabled. It may not be able to complete the processing because meta is not yet online or master
167    * is currently in startup mode. In this case, the dead server will be parked in this set
168    * temporarily.
169    */
170   private Map<ServerName, Boolean> requeuedDeadServers = new HashMap<ServerName, Boolean>();
171 
172   /**
173    * Constructor.
174    * @param master
175    * @param services
176    * @throws ZooKeeperConnectionException
177    */
178   public ServerManager(final Server master, final MasterServices services)
179       throws IOException {
180     this(master, services, true);
181   }
182 
183   ServerManager(final Server master, final MasterServices services,
184       final boolean connect) throws IOException {
185     this.master = master;
186     this.services = services;
187     Configuration c = master.getConfiguration();
188     maxSkew = c.getLong("hbase.master.maxclockskew", 30000);
189     warningSkew = c.getLong("hbase.master.warningclockskew", 10000);
190     this.connection = connect ? HConnectionManager.getConnection(c) : null;
191   }
192 
193   /**
194    * Let the server manager know a new regionserver has come online
195    * @param ia The remote address
196    * @param port The remote port
197    * @param serverStartcode
198    * @param serverCurrentTime The current time of the region server in ms
199    * @return The ServerName we know this server as.
200    * @throws IOException
201    */
202   ServerName regionServerStartup(final InetAddress ia, final int port,
203     final long serverStartcode, long serverCurrentTime)
204   throws IOException {
205     // Test for case where we get a region startup message from a regionserver
206     // that has been quickly restarted but whose znode expiration handler has
207     // not yet run, or from a server whose fail we are currently processing.
208     // Test its host+port combo is present in serverAddresstoServerInfo.  If it
209     // is, reject the server and trigger its expiration. The next time it comes
210     // in, it should have been removed from serverAddressToServerInfo and queued
211     // for processing by ProcessServerShutdown.
212     ServerName sn = new ServerName(ia.getHostName(), port, serverStartcode);
213     checkClockSkew(sn, serverCurrentTime);
214     checkIsDead(sn, "STARTUP");
215     if (!checkAlreadySameHostPortAndRecordNewServer(
216         sn, ServerLoad.EMPTY_SERVERLOAD)) {
217       LOG.warn("THIS SHOULD NOT HAPPEN, RegionServerStartup"
218         + " could not record the server: " + sn);
219     }
220     return sn;
221   }
222 
223   /**
224    * Updates last flushed sequence Ids for the regions on server sn
225    * @param sn
226    * @param hsl
227    */
228   private void updateLastFlushedSequenceIds(ServerName sn, ServerLoad hsl) {
229     Map<byte[], RegionLoad> regionsLoad = hsl.getRegionsLoad();
230     for (Entry<byte[], RegionLoad> entry : regionsLoad.entrySet()) {
231       Long existingValue = flushedSequenceIdByRegion.get(entry.getKey());
232       long l = entry.getValue().getCompleteSequenceId();
233       if (existingValue != null) {
234         if (l != -1 && l < existingValue) {
235           if (LOG.isDebugEnabled()) {
236             LOG.debug("RegionServer " + sn +
237                 " indicates a last flushed sequence id (" + entry.getValue() +
238                 ") that is less than the previous last flushed sequence id (" +
239                 existingValue + ") for region " +
240                 Bytes.toString(entry.getKey()) + " Ignoring.");
241           }
242           continue; // Don't let smaller sequence ids override greater
243           // sequence ids.
244         }
245       }
246       flushedSequenceIdByRegion.put(entry.getKey(), l);
247     }
248   }
249 
250   void regionServerReport(ServerName sn,
251       ServerLoad sl) throws YouAreDeadException {
252     checkIsDead(sn, "REPORT");
253     if (!this.onlineServers.containsKey(sn)) {
254       // Already have this host+port combo and its just different start code?
255       // Just let the server in. Presume master joining a running cluster.
256       // recordNewServer is what happens at the end of reportServerStartup.
257       // The only thing we are skipping is passing back to the regionserver
258       // the ServerName to use. Here we presume a master has already done
259       // that so we'll press on with whatever it gave us for ServerName.
260       if (!checkAlreadySameHostPortAndRecordNewServer(sn, sl)) {
261         LOG.info("RegionServerReport ignored, could not record the sever: " + sn);
262         return; // Not recorded, so no need to move on
263       }
264     } else {
265       this.onlineServers.put(sn, sl);
266     }
267     updateLastFlushedSequenceIds(sn, sl);
268   }
269 
270   /**
271    * Check is a server of same host and port already exists,
272    * if not, or the existed one got a smaller start code, record it.
273    *
274    * @param sn the server to check and record
275    * @param sl the server load on the server
276    * @return true if the server is recorded, otherwise, false
277    */
278   boolean checkAlreadySameHostPortAndRecordNewServer(
279       final ServerName serverName, final ServerLoad sl) {
280     ServerName existingServer = findServerWithSameHostnamePort(serverName);
281     if (existingServer != null) {
282       if (existingServer.getStartcode() > serverName.getStartcode()) {
283         LOG.info("Server serverName=" + serverName +
284           " rejected; we already have " + existingServer.toString() +
285           " registered with same hostname and port");
286         return false;
287       }
288       LOG.info("Triggering server recovery; existingServer " +
289         existingServer + " looks stale, new server:" + serverName);
290       expireServer(existingServer);
291     }
292     recordNewServer(serverName, sl);
293     return true;
294   }
295 
296   /**
297    * Checks if the clock skew between the server and the master. If the clock skew exceeds the
298    * configured max, it will throw an exception; if it exceeds the configured warning threshold,
299    * it will log a warning but start normally.
300    * @param serverName Incoming servers's name
301    * @param serverCurrentTime
302    * @throws ClockOutOfSyncException if the skew exceeds the configured max value
303    */
304   private void checkClockSkew(final ServerName serverName, final long serverCurrentTime)
305   throws ClockOutOfSyncException {
306     long skew = System.currentTimeMillis() - serverCurrentTime;
307     if (skew > maxSkew) {
308       String message = "Server " + serverName + " has been " +
309         "rejected; Reported time is too far out of sync with master.  " +
310         "Time difference of " + skew + "ms > max allowed of " + maxSkew + "ms";
311       LOG.warn(message);
312       throw new ClockOutOfSyncException(message);
313     } else if (skew > warningSkew){
314       String message = "Reported time for server " + serverName + " is out of sync with master " +
315         "by " + skew + "ms. (Warning threshold is " + warningSkew + "ms; " +
316         "error threshold is " + maxSkew + "ms)";
317       LOG.warn(message);
318     }
319   }
320 
321   /**
322    * If this server is on the dead list, reject it with a YouAreDeadException.
323    * If it was dead but came back with a new start code, remove the old entry
324    * from the dead list.
325    * @param serverName
326    * @param what START or REPORT
327    * @throws org.apache.hadoop.hbase.YouAreDeadException
328    */
329   private void checkIsDead(final ServerName serverName, final String what)
330       throws YouAreDeadException {
331     if (this.deadservers.isDeadServer(serverName)) {
332       // host name, port and start code all match with existing one of the
333       // dead servers. So, this server must be dead.
334       String message = "Server " + what + " rejected; currently processing " +
335           serverName + " as dead server";
336       LOG.debug(message);
337       throw new YouAreDeadException(message);
338     }
339     // remove dead server with same hostname and port of newly checking in rs after master
340     // initialization.See HBASE-5916 for more information.
341     if ((this.services == null || ((HMaster) this.services).isInitialized())
342         && this.deadservers.cleanPreviousInstance(serverName)) {
343       // This server has now become alive after we marked it as dead.
344       // We removed it's previous entry from the dead list to reflect it.
345       LOG.debug(what + ":" + " Server " + serverName + " came back up," +
346           " removed it from the dead servers list");
347     }
348   }
349 
350   /**
351    * @return ServerName with matching hostname and port.
352    */
353   private ServerName findServerWithSameHostnamePort(
354       final ServerName serverName) {
355     for (ServerName sn: getOnlineServersList()) {
356       if (ServerName.isSameHostnameAndPort(serverName, sn)) return sn;
357     }
358     return null;
359   }
360 
361   /**
362    * Adds the onlineServers list.
363    * @param serverName The remote servers name.
364    * @param sl
365    */
366   void recordNewServer(final ServerName serverName, final ServerLoad sl) {
367     LOG.info("Registering server=" + serverName);
368     this.onlineServers.put(serverName, sl);
369     this.rsAdmins.remove(serverName);
370   }
371 
372   public long getLastFlushedSequenceId(byte[] regionName) {
373     long seqId = -1;
374     if (flushedSequenceIdByRegion.containsKey(regionName)) {
375       seqId = flushedSequenceIdByRegion.get(regionName);
376     }
377     return seqId;
378   }
379 
380   /**
381    * @param serverName
382    * @return ServerLoad if serverName is known else null
383    */
384   public ServerLoad getLoad(final ServerName serverName) {
385     return this.onlineServers.get(serverName);
386   }
387 
388   /**
389    * Compute the average load across all region servers.
390    * Currently, this uses a very naive computation - just uses the number of
391    * regions being served, ignoring stats about number of requests.
392    * @return the average load
393    */
394   public double getAverageLoad() {
395     int totalLoad = 0;
396     int numServers = 0;
397     double averageLoad;
398     for (ServerLoad sl: this.onlineServers.values()) {
399         numServers++;
400         totalLoad += sl.getNumberOfRegions();
401     }
402     averageLoad = (double)totalLoad / (double)numServers;
403     return averageLoad;
404   }
405 
406   /** @return the count of active regionservers */
407   int countOfRegionServers() {
408     // Presumes onlineServers is a concurrent map
409     return this.onlineServers.size();
410   }
411 
412   /**
413    * @return Read-only map of servers to serverinfo
414    */
415   public Map<ServerName, ServerLoad> getOnlineServers() {
416     // Presumption is that iterating the returned Map is OK.
417     synchronized (this.onlineServers) {
418       return Collections.unmodifiableMap(this.onlineServers);
419     }
420   }
421 
422 
423   public DeadServer getDeadServers() {
424     return this.deadservers;
425   }
426 
427   /**
428    * Checks if any dead servers are currently in progress.
429    * @return true if any RS are being processed as dead, false if not
430    */
431   public boolean areDeadServersInProgress() {
432     return this.deadservers.areDeadServersInProgress();
433   }
434 
435   void letRegionServersShutdown() {
436     long previousLogTime = 0;
437     while (!onlineServers.isEmpty()) {
438 
439       if (System.currentTimeMillis() > (previousLogTime + 1000)) {
440         StringBuilder sb = new StringBuilder();
441         for (ServerName key : this.onlineServers.keySet()) {
442           if (sb.length() > 0) {
443             sb.append(", ");
444           }
445           sb.append(key);
446         }
447         LOG.info("Waiting on regionserver(s) to go down " + sb.toString());
448         previousLogTime = System.currentTimeMillis();
449       }
450 
451       synchronized (onlineServers) {
452         try {
453           onlineServers.wait(100);
454         } catch (InterruptedException ignored) {
455           // continue
456         }
457       }
458     }
459   }
460 
461   /*
462    * Expire the passed server.  Add it to list of dead servers and queue a
463    * shutdown processing.
464    */
465   public synchronized void expireServer(final ServerName serverName) {
466     if (!services.isServerShutdownHandlerEnabled()) {
467       LOG.info("Master doesn't enable ServerShutdownHandler during initialization, "
468           + "delay expiring server " + serverName);
469       this.queuedDeadServers.add(serverName);
470       return;
471     }
472     if (!this.onlineServers.containsKey(serverName)) {
473       LOG.warn("Received expiration of " + serverName +
474         " but server is not currently online");
475     }
476     if (this.deadservers.isDeadServer(serverName)) {
477       // TODO: Can this happen?  It shouldn't be online in this case?
478       LOG.warn("Received expiration of " + serverName +
479           " but server shutdown is already in progress");
480       return;
481     }
482     // Remove the server from the known servers lists and update load info BUT
483     // add to deadservers first; do this so it'll show in dead servers list if
484     // not in online servers list.
485     this.deadservers.add(serverName);
486     this.onlineServers.remove(serverName);
487     synchronized (onlineServers) {
488       onlineServers.notifyAll();
489     }
490     this.rsAdmins.remove(serverName);
491     // If cluster is going down, yes, servers are going to be expiring; don't
492     // process as a dead server
493     if (this.clusterShutdown) {
494       LOG.info("Cluster shutdown set; " + serverName +
495         " expired; onlineServers=" + this.onlineServers.size());
496       if (this.onlineServers.isEmpty()) {
497         master.stop("Cluster shutdown set; onlineServer=0");
498       }
499       return;
500     }
501 
502     boolean carryingMeta = services.getAssignmentManager().isCarryingMeta(serverName);
503     if (carryingMeta) {
504       this.services.getExecutorService().submit(new MetaServerShutdownHandler(this.master,
505         this.services, this.deadservers, serverName));
506     } else {
507       this.services.getExecutorService().submit(new ServerShutdownHandler(this.master,
508         this.services, this.deadservers, serverName, true));
509     }
510     LOG.debug("Added=" + serverName +
511       " to dead servers, submitted shutdown handler to be executed meta=" + carryingMeta);
512   }
513 
514   public synchronized void processDeadServer(final ServerName serverName) {
515     this.processDeadServer(serverName, false);
516   }
517 
518   public synchronized void processDeadServer(final ServerName serverName, boolean shouldSplitHlog) {
519     // When assignment manager is cleaning up the zookeeper nodes and rebuilding the
520     // in-memory region states, region servers could be down. Meta table can and
521     // should be re-assigned, log splitting can be done too. However, it is better to
522     // wait till the cleanup is done before re-assigning user regions.
523     //
524     // We should not wait in the server shutdown handler thread since it can clog
525     // the handler threads and meta table could not be re-assigned in case
526     // the corresponding server is down. So we queue them up here instead.
527     if (!services.getAssignmentManager().isFailoverCleanupDone()) {
528       requeuedDeadServers.put(serverName, shouldSplitHlog);
529       return;
530     }
531 
532     this.deadservers.add(serverName);
533     this.services.getExecutorService().submit(
534       new ServerShutdownHandler(this.master, this.services, this.deadservers, serverName,
535           shouldSplitHlog));
536   }
537 
538   /**
539    * Process the servers which died during master's initialization. It will be
540    * called after HMaster#assignMeta and AssignmentManager#joinCluster.
541    * */
542   synchronized void processQueuedDeadServers() {
543     if (!services.isServerShutdownHandlerEnabled()) {
544       LOG.info("Master hasn't enabled ServerShutdownHandler");
545     }
546     Iterator<ServerName> serverIterator = queuedDeadServers.iterator();
547     while (serverIterator.hasNext()) {
548       ServerName tmpServerName = serverIterator.next();
549       expireServer(tmpServerName);
550       serverIterator.remove();
551       requeuedDeadServers.remove(tmpServerName);
552     }
553 
554     if (!services.getAssignmentManager().isFailoverCleanupDone()) {
555       LOG.info("AssignmentManager hasn't finished failover cleanup");
556     }
557 
558     for(ServerName tmpServerName : requeuedDeadServers.keySet()){
559       processDeadServer(tmpServerName, requeuedDeadServers.get(tmpServerName));
560     }
561     requeuedDeadServers.clear();
562   }
563 
564   /*
565    * Remove the server from the drain list.
566    */
567   public boolean removeServerFromDrainList(final ServerName sn) {
568     // Warn if the server (sn) is not online.  ServerName is of the form:
569     // <hostname> , <port> , <startcode>
570 
571     if (!this.isServerOnline(sn)) {
572       LOG.warn("Server " + sn + " is not currently online. " +
573                "Removing from draining list anyway, as requested.");
574     }
575     // Remove the server from the draining servers lists.
576     return this.drainingServers.remove(sn);
577   }
578 
579   /*
580    * Add the server to the drain list.
581    */
582   public boolean addServerToDrainList(final ServerName sn) {
583     // Warn if the server (sn) is not online.  ServerName is of the form:
584     // <hostname> , <port> , <startcode>
585 
586     if (!this.isServerOnline(sn)) {
587       LOG.warn("Server " + sn + " is not currently online. " +
588                "Ignoring request to add it to draining list.");
589       return false;
590     }
591     // Add the server to the draining servers lists, if it's not already in
592     // it.
593     if (this.drainingServers.contains(sn)) {
594       LOG.warn("Server " + sn + " is already in the draining server list." +
595                "Ignoring request to add it again.");
596       return false;
597     }
598     return this.drainingServers.add(sn);
599   }
600 
601   // RPC methods to region servers
602 
603   /**
604    * Sends an OPEN RPC to the specified server to open the specified region.
605    * <p>
606    * Open should not fail but can if server just crashed.
607    * <p>
608    * @param server server to open a region
609    * @param region region to open
610    * @param versionOfOfflineNode that needs to be present in the offline node
611    * when RS tries to change the state from OFFLINE to other states.
612    * @param favoredNodes
613    */
614   public RegionOpeningState sendRegionOpen(final ServerName server,
615       HRegionInfo region, int versionOfOfflineNode, List<ServerName> favoredNodes)
616   throws IOException {
617     AdminService.BlockingInterface admin = getRsAdmin(server);
618     if (admin == null) {
619       LOG.warn("Attempting to send OPEN RPC to server " + server.toString() +
620         " failed because no RPC connection found to this server");
621       return RegionOpeningState.FAILED_OPENING;
622     }
623     OpenRegionRequest request =
624       RequestConverter.buildOpenRegionRequest(region, versionOfOfflineNode, favoredNodes);
625     try {
626       OpenRegionResponse response = admin.openRegion(null, request);
627       return ResponseConverter.getRegionOpeningState(response);
628     } catch (ServiceException se) {
629       throw ProtobufUtil.getRemoteException(se);
630     }
631   }
632 
633   /**
634    * Sends an OPEN RPC to the specified server to open the specified region.
635    * <p>
636    * Open should not fail but can if server just crashed.
637    * <p>
638    * @param server server to open a region
639    * @param regionOpenInfos info of a list of regions to open
640    * @return a list of region opening states
641    */
642   public List<RegionOpeningState> sendRegionOpen(ServerName server,
643       List<Triple<HRegionInfo, Integer, List<ServerName>>> regionOpenInfos)
644   throws IOException {
645     AdminService.BlockingInterface admin = getRsAdmin(server);
646     if (admin == null) {
647       LOG.warn("Attempting to send OPEN RPC to server " + server.toString() +
648         " failed because no RPC connection found to this server");
649       return null;
650     }
651 
652     OpenRegionRequest request =
653       RequestConverter.buildOpenRegionRequest(regionOpenInfos);
654     try {
655       OpenRegionResponse response = admin.openRegion(null, request);
656       return ResponseConverter.getRegionOpeningStateList(response);
657     } catch (ServiceException se) {
658       throw ProtobufUtil.getRemoteException(se);
659     }
660   }
661 
662   /**
663    * Sends an CLOSE RPC to the specified server to close the specified region.
664    * <p>
665    * A region server could reject the close request because it either does not
666    * have the specified region or the region is being split.
667    * @param server server to open a region
668    * @param region region to open
669    * @param versionOfClosingNode
670    *   the version of znode to compare when RS transitions the znode from
671    *   CLOSING state.
672    * @param dest - if the region is moved to another server, the destination server. null otherwise.
673    * @return true if server acknowledged close, false if not
674    * @throws IOException
675    */
676   public boolean sendRegionClose(ServerName server, HRegionInfo region,
677     int versionOfClosingNode, ServerName dest, boolean transitionInZK) throws IOException {
678     if (server == null) throw new NullPointerException("Passed server is null");
679     AdminService.BlockingInterface admin = getRsAdmin(server);
680     if (admin == null) {
681       throw new IOException("Attempting to send CLOSE RPC to server " +
682         server.toString() + " for region " +
683         region.getRegionNameAsString() +
684         " failed because no RPC connection found to this server");
685     }
686     return ProtobufUtil.closeRegion(admin, region.getRegionName(),
687       versionOfClosingNode, dest, transitionInZK);
688   }
689 
690   public boolean sendRegionClose(ServerName server,
691       HRegionInfo region, int versionOfClosingNode) throws IOException {
692     return sendRegionClose(server, region, versionOfClosingNode, null, true);
693   }
694 
695   /**
696    * Sends an MERGE REGIONS RPC to the specified server to merge the specified
697    * regions.
698    * <p>
699    * A region server could reject the close request because it either does not
700    * have the specified region.
701    * @param server server to merge regions
702    * @param region_a region to merge
703    * @param region_b region to merge
704    * @param forcible true if do a compulsory merge, otherwise we will only merge
705    *          two adjacent regions
706    * @throws IOException
707    */
708   public void sendRegionsMerge(ServerName server, HRegionInfo region_a,
709       HRegionInfo region_b, boolean forcible) throws IOException {
710     if (server == null)
711       throw new NullPointerException("Passed server is null");
712     if (region_a == null || region_b == null)
713       throw new NullPointerException("Passed region is null");
714     AdminService.BlockingInterface admin = getRsAdmin(server);
715     if (admin == null) {
716       throw new IOException("Attempting to send MERGE REGIONS RPC to server "
717           + server.toString() + " for region "
718           + region_a.getRegionNameAsString() + ","
719           + region_b.getRegionNameAsString()
720           + " failed because no RPC connection found to this server");
721     }
722     ProtobufUtil.mergeRegions(admin, region_a, region_b, forcible);
723   }
724 
725     /**
726     * @param sn
727     * @return Admin interface for the remote regionserver named <code>sn</code>
728     * @throws IOException
729     * @throws RetriesExhaustedException wrapping a ConnectException if failed
730     */
731   private AdminService.BlockingInterface getRsAdmin(final ServerName sn)
732   throws IOException {
733     AdminService.BlockingInterface admin = this.rsAdmins.get(sn);
734     if (admin == null) {
735       LOG.debug("New admin connection to " + sn.toString());
736       admin = this.connection.getAdmin(sn);
737       this.rsAdmins.put(sn, admin);
738     }
739     return admin;
740   }
741 
742   /**
743    * Wait for the region servers to report in.
744    * We will wait until one of this condition is met:
745    *  - the master is stopped
746    *  - the 'hbase.master.wait.on.regionservers.maxtostart' number of
747    *    region servers is reached
748    *  - the 'hbase.master.wait.on.regionservers.mintostart' is reached AND
749    *   there have been no new region server in for
750    *      'hbase.master.wait.on.regionservers.interval' time AND
751    *   the 'hbase.master.wait.on.regionservers.timeout' is reached
752    *
753    * @throws InterruptedException
754    */
755   public void waitForRegionServers(MonitoredTask status)
756   throws InterruptedException {
757     final long interval = this.master.getConfiguration().
758       getLong(WAIT_ON_REGIONSERVERS_INTERVAL, 1500);
759     final long timeout = this.master.getConfiguration().
760       getLong(WAIT_ON_REGIONSERVERS_TIMEOUT, 4500);
761     int minToStart = this.master.getConfiguration().
762       getInt(WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
763     if (minToStart < 1) {
764       LOG.warn(String.format(
765         "The value of '%s' (%d) can not be less than 1, ignoring.",
766         WAIT_ON_REGIONSERVERS_MINTOSTART, minToStart));
767       minToStart = 1;
768     }
769     int maxToStart = this.master.getConfiguration().
770       getInt(WAIT_ON_REGIONSERVERS_MAXTOSTART, Integer.MAX_VALUE);
771     if (maxToStart < minToStart) {
772         LOG.warn(String.format(
773             "The value of '%s' (%d) is set less than '%s' (%d), ignoring.",
774             WAIT_ON_REGIONSERVERS_MAXTOSTART, maxToStart,
775             WAIT_ON_REGIONSERVERS_MINTOSTART, minToStart));
776         maxToStart = Integer.MAX_VALUE;
777     }
778 
779     long now =  System.currentTimeMillis();
780     final long startTime = now;
781     long slept = 0;
782     long lastLogTime = 0;
783     long lastCountChange = startTime;
784     int count = countOfRegionServers();
785     int oldCount = 0;
786     while (
787       !this.master.isStopped() &&
788         count < maxToStart &&
789         (lastCountChange+interval > now || timeout > slept || count < minToStart)
790       ){
791 
792       // Log some info at every interval time or if there is a change
793       if (oldCount != count || lastLogTime+interval < now){
794         lastLogTime = now;
795         String msg =
796           "Waiting for region servers count to settle; currently"+
797             " checked in " + count + ", slept for " + slept + " ms," +
798             " expecting minimum of " + minToStart + ", maximum of "+ maxToStart+
799             ", timeout of "+timeout+" ms, interval of "+interval+" ms.";
800         LOG.info(msg);
801         status.setStatus(msg);
802       }
803 
804       // We sleep for some time
805       final long sleepTime = 50;
806       Thread.sleep(sleepTime);
807       now =  System.currentTimeMillis();
808       slept = now - startTime;
809 
810       oldCount = count;
811       count = countOfRegionServers();
812       if (count != oldCount) {
813         lastCountChange = now;
814       }
815     }
816 
817     LOG.info("Finished waiting for region servers count to settle;" +
818       " checked in " + count + ", slept for " + slept + " ms," +
819       " expecting minimum of " + minToStart + ", maximum of "+ maxToStart+","+
820       " master is "+ (this.master.isStopped() ? "stopped.": "running.")
821     );
822   }
823 
824   /**
825    * @return A copy of the internal list of online servers.
826    */
827   public List<ServerName> getOnlineServersList() {
828     // TODO: optimize the load balancer call so we don't need to make a new list
829     // TODO: FIX. THIS IS POPULAR CALL.
830     return new ArrayList<ServerName>(this.onlineServers.keySet());
831   }
832 
833   /**
834    * @return A copy of the internal list of draining servers.
835    */
836   public List<ServerName> getDrainingServersList() {
837     return new ArrayList<ServerName>(this.drainingServers);
838   }
839 
840   /**
841    * @return A copy of the internal set of deadNotExpired servers.
842    */
843   Set<ServerName> getDeadNotExpiredServers() {
844     return new HashSet<ServerName>(this.queuedDeadServers);
845   }
846 
847   /**
848    * @return A copy of the internal map of requeuedDeadServers servers and their corresponding
849    *         splitlog need flag.
850    */
851   Map<ServerName, Boolean> getRequeuedDeadServers() {
852     return Collections.unmodifiableMap(this.requeuedDeadServers);
853   }
854   
855   public boolean isServerOnline(ServerName serverName) {
856     return serverName != null && onlineServers.containsKey(serverName);
857   }
858 
859   /**
860    * Check if a server is known to be dead.  A server can be online,
861    * or known to be dead, or unknown to this manager (i.e, not online,
862    * not known to be dead either. it is simply not tracked by the
863    * master any more, for example, a very old previous instance).
864    */
865   public synchronized boolean isServerDead(ServerName serverName) {
866     return serverName == null || deadservers.isDeadServer(serverName)
867       || queuedDeadServers.contains(serverName)
868       || requeuedDeadServers.containsKey(serverName);
869   }
870 
871   public void shutdownCluster() {
872     this.clusterShutdown = true;
873     this.master.stop("Cluster shutdown requested");
874   }
875 
876   public boolean isClusterShutdown() {
877     return this.clusterShutdown;
878   }
879 
880   /**
881    * Stop the ServerManager.  Currently closes the connection to the master.
882    */
883   public void stop() {
884     if (connection != null) {
885       try {
886         connection.close();
887       } catch (IOException e) {
888         LOG.error("Attempt to close connection to master failed", e);
889       }
890     }
891   }
892 
893   /**
894    * Creates a list of possible destinations for a region. It contains the online servers, but not
895    *  the draining or dying servers.
896    *  @param serverToExclude can be null if there is no server to exclude
897    */
898   public List<ServerName> createDestinationServersList(final ServerName serverToExclude){
899     final List<ServerName> destServers = getOnlineServersList();
900 
901     if (serverToExclude != null){
902       destServers.remove(serverToExclude);
903     }
904 
905     // Loop through the draining server list and remove them from the server list
906     final List<ServerName> drainingServersCopy = getDrainingServersList();
907     if (!drainingServersCopy.isEmpty()) {
908       for (final ServerName server: drainingServersCopy) {
909         destServers.remove(server);
910       }
911     }
912 
913     // Remove the deadNotExpired servers from the server list.
914     removeDeadNotExpiredServers(destServers);
915 
916     return destServers;
917   }
918 
919   /**
920    * Calls {@link #createDestinationServersList} without server to exclude.
921    */
922   public List<ServerName> createDestinationServersList(){
923     return createDestinationServersList(null);
924   }
925 
926     /**
927     * Loop through the deadNotExpired server list and remove them from the
928     * servers.
929     * This function should be used carefully outside of this class. You should use a high level
930     *  method such as {@link #createDestinationServersList()} instead of managing you own list.
931     */
932   void removeDeadNotExpiredServers(List<ServerName> servers) {
933     Set<ServerName> deadNotExpiredServersCopy = this.getDeadNotExpiredServers();
934     if (!deadNotExpiredServersCopy.isEmpty()) {
935       for (ServerName server : deadNotExpiredServersCopy) {
936         LOG.debug("Removing dead but not expired server: " + server
937           + " from eligible server pool.");
938         servers.remove(server);
939       }
940     }
941   }
942   
943   /**
944    * To clear any dead server with same host name and port of any online server
945    */
946   void clearDeadServersWithSameHostNameAndPortOfOnlineServer() {
947     for (ServerName serverName : getOnlineServersList()) {
948       deadservers.cleanAllPreviousInstances(serverName);
949     }
950   }
951 }