View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.IOException;
22  import java.net.InetAddress;
23  import java.util.ArrayList;
24  import java.util.Collections;
25  import java.util.HashMap;
26  import java.util.HashSet;
27  import java.util.Iterator;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Map.Entry;
31  import java.util.Set;
32  import java.util.SortedMap;
33  import java.util.concurrent.ConcurrentHashMap;
34  import java.util.concurrent.ConcurrentSkipListMap;
35  
36  import org.apache.commons.logging.Log;
37  import org.apache.commons.logging.LogFactory;
38  import org.apache.hadoop.classification.InterfaceAudience;
39  import org.apache.hadoop.conf.Configuration;
40  import org.apache.hadoop.hbase.ClockOutOfSyncException;
41  import org.apache.hadoop.hbase.HRegionInfo;
42  import org.apache.hadoop.hbase.RegionLoad;
43  import org.apache.hadoop.hbase.Server;
44  import org.apache.hadoop.hbase.ServerLoad;
45  import org.apache.hadoop.hbase.ServerName;
46  import org.apache.hadoop.hbase.YouAreDeadException;
47  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
48  import org.apache.hadoop.hbase.client.HConnection;
49  import org.apache.hadoop.hbase.client.HConnectionManager;
50  import org.apache.hadoop.hbase.client.RetriesExhaustedException;
51  import org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler;
52  import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
53  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
54  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
55  import org.apache.hadoop.hbase.protobuf.RequestConverter;
56  import org.apache.hadoop.hbase.protobuf.ResponseConverter;
57  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
58  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
59  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
60  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
61  import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
62  import org.apache.hadoop.hbase.util.Bytes;
63  import org.apache.hadoop.hbase.util.Triple;
64  
65  import com.google.protobuf.ServiceException;
66  
67  /**
68   * The ServerManager class manages info about region servers.
69   * <p>
70   * Maintains lists of online and dead servers.  Processes the startups,
71   * shutdowns, and deaths of region servers.
72   * <p>
73   * Servers are distinguished in two different ways.  A given server has a
74   * location, specified by hostname and port, and of which there can only be one
75   * online at any given time.  A server instance is specified by the location
76   * (hostname and port) as well as the startcode (timestamp from when the server
77   * was started).  This is used to differentiate a restarted instance of a given
78   * server from the original instance.
79   * <p>
80   * If a sever is known not to be running any more, it is called dead. The dead
81   * server needs to be handled by a ServerShutdownHandler.  If the handler is not
82   * enabled yet, the server can't be handled right away so it is queued up.
83   * After the handler is enabled, the server will be submitted to a handler to handle.
84   * However, the handler may be just partially enabled.  If so,
85   * the server cannot be fully processed, and be queued up for further processing.
86   * A server is fully processed only after the handler is fully enabled
87   * and has completed the handling.
88   */
89  @InterfaceAudience.Private
90  public class ServerManager {
91    public static final String WAIT_ON_REGIONSERVERS_MAXTOSTART =
92        "hbase.master.wait.on.regionservers.maxtostart";
93  
94    public static final String WAIT_ON_REGIONSERVERS_MINTOSTART =
95        "hbase.master.wait.on.regionservers.mintostart";
96  
97    public static final String WAIT_ON_REGIONSERVERS_TIMEOUT =
98        "hbase.master.wait.on.regionservers.timeout";
99  
100   public static final String WAIT_ON_REGIONSERVERS_INTERVAL =
101       "hbase.master.wait.on.regionservers.interval";
102 
103   private static final Log LOG = LogFactory.getLog(ServerManager.class);
104 
105   // Set if we are to shutdown the cluster.
106   private volatile boolean clusterShutdown = false;
107 
108   private final SortedMap<byte[], Long> flushedSequenceIdByRegion =
109     new ConcurrentSkipListMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
110 
111   /** Map of registered servers to their current load */
112   private final Map<ServerName, ServerLoad> onlineServers =
113     new ConcurrentHashMap<ServerName, ServerLoad>();
114 
115   /**
116    * Map of admin interfaces per registered regionserver; these interfaces we use to control
117    * regionservers out on the cluster
118    */
119   private final Map<ServerName, AdminService.BlockingInterface> rsAdmins =
120     new HashMap<ServerName, AdminService.BlockingInterface>();
121 
122   /**
123    * List of region servers <ServerName> that should not get any more new
124    * regions.
125    */
126   private final ArrayList<ServerName> drainingServers =
127     new ArrayList<ServerName>();
128 
129   private final Server master;
130   private final MasterServices services;
131   private final HConnection connection;
132 
133   private final DeadServer deadservers = new DeadServer();
134 
135   private final long maxSkew;
136   private final long warningSkew;
137 
138   /**
139    * Set of region servers which are dead but not processed immediately. If one
140    * server died before master enables ServerShutdownHandler, the server will be
141    * added to this set and will be processed through calling
142    * {@link ServerManager#processQueuedDeadServers()} by master.
143    * <p>
144    * A dead server is a server instance known to be dead, not listed in the /hbase/rs
145    * znode any more. It may have not been submitted to ServerShutdownHandler yet
146    * because the handler is not enabled.
147    * <p>
148    * A dead server, which has been submitted to ServerShutdownHandler while the
149    * handler is not enabled, is queued up.
150    * <p>
151    * So this is a set of region servers known to be dead but not submitted to
152    * ServerShutdownHander for processing yet.
153    */
154   private Set<ServerName> queuedDeadServers = new HashSet<ServerName>();
155 
156   /**
157    * Set of region servers which are dead and submitted to ServerShutdownHandler to process but not
158    * fully processed immediately.
159    * <p>
160    * If one server died before assignment manager finished the failover cleanup, the server will be
161    * added to this set and will be processed through calling
162    * {@link ServerManager#processQueuedDeadServers()} by assignment manager.
163    * <p>
164    * The Boolean value indicates whether log split is needed inside ServerShutdownHandler
165    * <p>
166    * ServerShutdownHandler processes a dead server submitted to the handler after the handler is
167    * enabled. It may not be able to complete the processing because meta is not yet online or master
168    * is currently in startup mode. In this case, the dead server will be parked in this set
169    * temporarily.
170    */
171   private Map<ServerName, Boolean> requeuedDeadServers = new HashMap<ServerName, Boolean>();
172 
173   /**
174    * Constructor.
175    * @param master
176    * @param services
177    * @throws ZooKeeperConnectionException
178    */
179   public ServerManager(final Server master, final MasterServices services)
180       throws IOException {
181     this(master, services, true);
182   }
183 
184   @SuppressWarnings("deprecation")
185   ServerManager(final Server master, final MasterServices services,
186       final boolean connect) throws IOException {
187     this.master = master;
188     this.services = services;
189     Configuration c = master.getConfiguration();
190     maxSkew = c.getLong("hbase.master.maxclockskew", 30000);
191     warningSkew = c.getLong("hbase.master.warningclockskew", 10000);
192     this.connection = connect ? HConnectionManager.getConnection(c) : null;
193   }
194 
195   /**
196    * Let the server manager know a new regionserver has come online
197    * @param ia The remote address
198    * @param port The remote port
199    * @param serverStartcode
200    * @param serverCurrentTime The current time of the region server in ms
201    * @return The ServerName we know this server as.
202    * @throws IOException
203    */
204   ServerName regionServerStartup(final InetAddress ia, final int port,
205     final long serverStartcode, long serverCurrentTime)
206   throws IOException {
207     // Test for case where we get a region startup message from a regionserver
208     // that has been quickly restarted but whose znode expiration handler has
209     // not yet run, or from a server whose fail we are currently processing.
210     // Test its host+port combo is present in serverAddresstoServerInfo.  If it
211     // is, reject the server and trigger its expiration. The next time it comes
212     // in, it should have been removed from serverAddressToServerInfo and queued
213     // for processing by ProcessServerShutdown.
214     ServerName sn = ServerName.valueOf(ia.getHostName(), port, serverStartcode);
215     checkClockSkew(sn, serverCurrentTime);
216     checkIsDead(sn, "STARTUP");
217     if (!checkAlreadySameHostPortAndRecordNewServer(
218         sn, ServerLoad.EMPTY_SERVERLOAD)) {
219       LOG.warn("THIS SHOULD NOT HAPPEN, RegionServerStartup"
220         + " could not record the server: " + sn);
221     }
222     return sn;
223   }
224 
225   /**
226    * Updates last flushed sequence Ids for the regions on server sn
227    * @param sn
228    * @param hsl
229    */
230   private void updateLastFlushedSequenceIds(ServerName sn, ServerLoad hsl) {
231     Map<byte[], RegionLoad> regionsLoad = hsl.getRegionsLoad();
232     for (Entry<byte[], RegionLoad> entry : regionsLoad.entrySet()) {
233       Long existingValue = flushedSequenceIdByRegion.get(entry.getKey());
234       long l = entry.getValue().getCompleteSequenceId();
235       if (existingValue != null) {
236         if (l != -1 && l < existingValue) {
237           if (LOG.isDebugEnabled()) {
238             LOG.debug("RegionServer " + sn +
239                 " indicates a last flushed sequence id (" + entry.getValue() +
240                 ") that is less than the previous last flushed sequence id (" +
241                 existingValue + ") for region " +
242                 Bytes.toString(entry.getKey()) + " Ignoring.");
243           }
244           continue; // Don't let smaller sequence ids override greater
245           // sequence ids.
246         }
247       }
248       flushedSequenceIdByRegion.put(entry.getKey(), l);
249     }
250   }
251 
252   void regionServerReport(ServerName sn,
253       ServerLoad sl) throws YouAreDeadException {
254     checkIsDead(sn, "REPORT");
255     if (!this.onlineServers.containsKey(sn)) {
256       // Already have this host+port combo and its just different start code?
257       // Just let the server in. Presume master joining a running cluster.
258       // recordNewServer is what happens at the end of reportServerStartup.
259       // The only thing we are skipping is passing back to the regionserver
260       // the ServerName to use. Here we presume a master has already done
261       // that so we'll press on with whatever it gave us for ServerName.
262       if (!checkAlreadySameHostPortAndRecordNewServer(sn, sl)) {
263         LOG.info("RegionServerReport ignored, could not record the sever: " + sn);
264         return; // Not recorded, so no need to move on
265       }
266     } else {
267       this.onlineServers.put(sn, sl);
268     }
269     updateLastFlushedSequenceIds(sn, sl);
270   }
271 
272   /**
273    * Check is a server of same host and port already exists,
274    * if not, or the existed one got a smaller start code, record it.
275    *
276    * @param sn the server to check and record
277    * @param sl the server load on the server
278    * @return true if the server is recorded, otherwise, false
279    */
280   boolean checkAlreadySameHostPortAndRecordNewServer(
281       final ServerName serverName, final ServerLoad sl) {
282     ServerName existingServer = findServerWithSameHostnamePort(serverName);
283     if (existingServer != null) {
284       if (existingServer.getStartcode() > serverName.getStartcode()) {
285         LOG.info("Server serverName=" + serverName +
286           " rejected; we already have " + existingServer.toString() +
287           " registered with same hostname and port");
288         return false;
289       }
290       LOG.info("Triggering server recovery; existingServer " +
291         existingServer + " looks stale, new server:" + serverName);
292       expireServer(existingServer);
293     }
294     recordNewServer(serverName, sl);
295     return true;
296   }
297 
298   /**
299    * Checks if the clock skew between the server and the master. If the clock skew exceeds the
300    * configured max, it will throw an exception; if it exceeds the configured warning threshold,
301    * it will log a warning but start normally.
302    * @param serverName Incoming servers's name
303    * @param serverCurrentTime
304    * @throws ClockOutOfSyncException if the skew exceeds the configured max value
305    */
306   private void checkClockSkew(final ServerName serverName, final long serverCurrentTime)
307   throws ClockOutOfSyncException {
308     long skew = Math.abs(System.currentTimeMillis() - serverCurrentTime);
309     if (skew > maxSkew) {
310       String message = "Server " + serverName + " has been " +
311         "rejected; Reported time is too far out of sync with master.  " +
312         "Time difference of " + skew + "ms > max allowed of " + maxSkew + "ms";
313       LOG.warn(message);
314       throw new ClockOutOfSyncException(message);
315     } else if (skew > warningSkew){
316       String message = "Reported time for server " + serverName + " is out of sync with master " +
317         "by " + skew + "ms. (Warning threshold is " + warningSkew + "ms; " +
318         "error threshold is " + maxSkew + "ms)";
319       LOG.warn(message);
320     }
321   }
322 
323   /**
324    * If this server is on the dead list, reject it with a YouAreDeadException.
325    * If it was dead but came back with a new start code, remove the old entry
326    * from the dead list.
327    * @param serverName
328    * @param what START or REPORT
329    * @throws org.apache.hadoop.hbase.YouAreDeadException
330    */
331   private void checkIsDead(final ServerName serverName, final String what)
332       throws YouAreDeadException {
333     if (this.deadservers.isDeadServer(serverName)) {
334       // host name, port and start code all match with existing one of the
335       // dead servers. So, this server must be dead.
336       String message = "Server " + what + " rejected; currently processing " +
337           serverName + " as dead server";
338       LOG.debug(message);
339       throw new YouAreDeadException(message);
340     }
341     // remove dead server with same hostname and port of newly checking in rs after master
342     // initialization.See HBASE-5916 for more information.
343     if ((this.services == null || ((HMaster) this.services).isInitialized())
344         && this.deadservers.cleanPreviousInstance(serverName)) {
345       // This server has now become alive after we marked it as dead.
346       // We removed it's previous entry from the dead list to reflect it.
347       LOG.debug(what + ":" + " Server " + serverName + " came back up," +
348           " removed it from the dead servers list");
349     }
350   }
351 
352   /**
353    * @return ServerName with matching hostname and port.
354    */
355   private ServerName findServerWithSameHostnamePort(
356       final ServerName serverName) {
357     for (ServerName sn: getOnlineServersList()) {
358       if (ServerName.isSameHostnameAndPort(serverName, sn)) return sn;
359     }
360     return null;
361   }
362 
363   /**
364    * Adds the onlineServers list.
365    * @param serverName The remote servers name.
366    * @param sl
367    */
368   void recordNewServer(final ServerName serverName, final ServerLoad sl) {
369     LOG.info("Registering server=" + serverName);
370     this.onlineServers.put(serverName, sl);
371     this.rsAdmins.remove(serverName);
372   }
373 
374   public long getLastFlushedSequenceId(byte[] regionName) {
375     long seqId = -1;
376     if (flushedSequenceIdByRegion.containsKey(regionName)) {
377       seqId = flushedSequenceIdByRegion.get(regionName);
378     }
379     return seqId;
380   }
381 
382   /**
383    * @param serverName
384    * @return ServerLoad if serverName is known else null
385    */
386   public ServerLoad getLoad(final ServerName serverName) {
387     return this.onlineServers.get(serverName);
388   }
389 
390   /**
391    * Compute the average load across all region servers.
392    * Currently, this uses a very naive computation - just uses the number of
393    * regions being served, ignoring stats about number of requests.
394    * @return the average load
395    */
396   public double getAverageLoad() {
397     int totalLoad = 0;
398     int numServers = 0;
399     double averageLoad;
400     for (ServerLoad sl: this.onlineServers.values()) {
401         numServers++;
402         totalLoad += sl.getNumberOfRegions();
403     }
404     averageLoad = (double)totalLoad / (double)numServers;
405     return averageLoad;
406   }
407 
408   /** @return the count of active regionservers */
409   int countOfRegionServers() {
410     // Presumes onlineServers is a concurrent map
411     return this.onlineServers.size();
412   }
413 
414   /**
415    * @return Read-only map of servers to serverinfo
416    */
417   public Map<ServerName, ServerLoad> getOnlineServers() {
418     // Presumption is that iterating the returned Map is OK.
419     synchronized (this.onlineServers) {
420       return Collections.unmodifiableMap(this.onlineServers);
421     }
422   }
423 
424 
425   public DeadServer getDeadServers() {
426     return this.deadservers;
427   }
428 
429   /**
430    * Checks if any dead servers are currently in progress.
431    * @return true if any RS are being processed as dead, false if not
432    */
433   public boolean areDeadServersInProgress() {
434     return this.deadservers.areDeadServersInProgress();
435   }
436 
437   void letRegionServersShutdown() {
438     long previousLogTime = 0;
439     while (!onlineServers.isEmpty()) {
440 
441       if (System.currentTimeMillis() > (previousLogTime + 1000)) {
442         StringBuilder sb = new StringBuilder();
443         for (ServerName key : this.onlineServers.keySet()) {
444           if (sb.length() > 0) {
445             sb.append(", ");
446           }
447           sb.append(key);
448         }
449         LOG.info("Waiting on regionserver(s) to go down " + sb.toString());
450         previousLogTime = System.currentTimeMillis();
451       }
452 
453       synchronized (onlineServers) {
454         try {
455           onlineServers.wait(100);
456         } catch (InterruptedException ignored) {
457           // continue
458         }
459       }
460     }
461   }
462 
463   /*
464    * Expire the passed server.  Add it to list of dead servers and queue a
465    * shutdown processing.
466    */
467   public synchronized void expireServer(final ServerName serverName) {
468     if (!services.isServerShutdownHandlerEnabled()) {
469       LOG.info("Master doesn't enable ServerShutdownHandler during initialization, "
470           + "delay expiring server " + serverName);
471       this.queuedDeadServers.add(serverName);
472       return;
473     }
474     if (!this.onlineServers.containsKey(serverName)) {
475       LOG.warn("Expiration of " + serverName +
476         " but server not online");
477     }
478     if (this.deadservers.isDeadServer(serverName)) {
479       // TODO: Can this happen?  It shouldn't be online in this case?
480       LOG.warn("Expiration of " + serverName +
481           " but server shutdown already in progress");
482       return;
483     }
484     // Remove the server from the known servers lists and update load info BUT
485     // add to deadservers first; do this so it'll show in dead servers list if
486     // not in online servers list.
487     this.deadservers.add(serverName);
488     this.onlineServers.remove(serverName);
489     synchronized (onlineServers) {
490       onlineServers.notifyAll();
491     }
492     this.rsAdmins.remove(serverName);
493     // If cluster is going down, yes, servers are going to be expiring; don't
494     // process as a dead server
495     if (this.clusterShutdown) {
496       LOG.info("Cluster shutdown set; " + serverName +
497         " expired; onlineServers=" + this.onlineServers.size());
498       if (this.onlineServers.isEmpty()) {
499         master.stop("Cluster shutdown set; onlineServer=0");
500       }
501       return;
502     }
503 
504     boolean carryingMeta = services.getAssignmentManager().isCarryingMeta(serverName);
505     if (carryingMeta) {
506       this.services.getExecutorService().submit(new MetaServerShutdownHandler(this.master,
507         this.services, this.deadservers, serverName));
508     } else {
509       this.services.getExecutorService().submit(new ServerShutdownHandler(this.master,
510         this.services, this.deadservers, serverName, true));
511     }
512     LOG.debug("Added=" + serverName +
513       " to dead servers, submitted shutdown handler to be executed meta=" + carryingMeta);
514   }
515 
516   public synchronized void processDeadServer(final ServerName serverName) {
517     this.processDeadServer(serverName, false);
518   }
519 
520   public synchronized void processDeadServer(final ServerName serverName, boolean shouldSplitHlog) {
521     // When assignment manager is cleaning up the zookeeper nodes and rebuilding the
522     // in-memory region states, region servers could be down. Meta table can and
523     // should be re-assigned, log splitting can be done too. However, it is better to
524     // wait till the cleanup is done before re-assigning user regions.
525     //
526     // We should not wait in the server shutdown handler thread since it can clog
527     // the handler threads and meta table could not be re-assigned in case
528     // the corresponding server is down. So we queue them up here instead.
529     if (!services.getAssignmentManager().isFailoverCleanupDone()) {
530       requeuedDeadServers.put(serverName, shouldSplitHlog);
531       return;
532     }
533 
534     this.deadservers.add(serverName);
535     this.services.getExecutorService().submit(
536       new ServerShutdownHandler(this.master, this.services, this.deadservers, serverName,
537           shouldSplitHlog));
538   }
539 
540   /**
541    * Process the servers which died during master's initialization. It will be
542    * called after HMaster#assignMeta and AssignmentManager#joinCluster.
543    * */
544   synchronized void processQueuedDeadServers() {
545     if (!services.isServerShutdownHandlerEnabled()) {
546       LOG.info("Master hasn't enabled ServerShutdownHandler");
547     }
548     Iterator<ServerName> serverIterator = queuedDeadServers.iterator();
549     while (serverIterator.hasNext()) {
550       ServerName tmpServerName = serverIterator.next();
551       expireServer(tmpServerName);
552       serverIterator.remove();
553       requeuedDeadServers.remove(tmpServerName);
554     }
555 
556     if (!services.getAssignmentManager().isFailoverCleanupDone()) {
557       LOG.info("AssignmentManager hasn't finished failover cleanup; waiting");
558     }
559 
560     for(ServerName tmpServerName : requeuedDeadServers.keySet()){
561       processDeadServer(tmpServerName, requeuedDeadServers.get(tmpServerName));
562     }
563     requeuedDeadServers.clear();
564   }
565 
566   /*
567    * Remove the server from the drain list.
568    */
569   public boolean removeServerFromDrainList(final ServerName sn) {
570     // Warn if the server (sn) is not online.  ServerName is of the form:
571     // <hostname> , <port> , <startcode>
572 
573     if (!this.isServerOnline(sn)) {
574       LOG.warn("Server " + sn + " is not currently online. " +
575                "Removing from draining list anyway, as requested.");
576     }
577     // Remove the server from the draining servers lists.
578     return this.drainingServers.remove(sn);
579   }
580 
581   /*
582    * Add the server to the drain list.
583    */
584   public boolean addServerToDrainList(final ServerName sn) {
585     // Warn if the server (sn) is not online.  ServerName is of the form:
586     // <hostname> , <port> , <startcode>
587 
588     if (!this.isServerOnline(sn)) {
589       LOG.warn("Server " + sn + " is not currently online. " +
590                "Ignoring request to add it to draining list.");
591       return false;
592     }
593     // Add the server to the draining servers lists, if it's not already in
594     // it.
595     if (this.drainingServers.contains(sn)) {
596       LOG.warn("Server " + sn + " is already in the draining server list." +
597                "Ignoring request to add it again.");
598       return false;
599     }
600     return this.drainingServers.add(sn);
601   }
602 
603   // RPC methods to region servers
604 
605   /**
606    * Sends an OPEN RPC to the specified server to open the specified region.
607    * <p>
608    * Open should not fail but can if server just crashed.
609    * <p>
610    * @param server server to open a region
611    * @param region region to open
612    * @param versionOfOfflineNode that needs to be present in the offline node
613    * when RS tries to change the state from OFFLINE to other states.
614    * @param favoredNodes
615    */
616   public RegionOpeningState sendRegionOpen(final ServerName server,
617       HRegionInfo region, int versionOfOfflineNode, List<ServerName> favoredNodes)
618   throws IOException {
619     AdminService.BlockingInterface admin = getRsAdmin(server);
620     if (admin == null) {
621       LOG.warn("Attempting to send OPEN RPC to server " + server.toString() +
622         " failed because no RPC connection found to this server");
623       return RegionOpeningState.FAILED_OPENING;
624     }
625     OpenRegionRequest request =
626       RequestConverter.buildOpenRegionRequest(region, versionOfOfflineNode, favoredNodes);
627     try {
628       OpenRegionResponse response = admin.openRegion(null, request);
629       return ResponseConverter.getRegionOpeningState(response);
630     } catch (ServiceException se) {
631       throw ProtobufUtil.getRemoteException(se);
632     }
633   }
634 
635   /**
636    * Sends an OPEN RPC to the specified server to open the specified region.
637    * <p>
638    * Open should not fail but can if server just crashed.
639    * <p>
640    * @param server server to open a region
641    * @param regionOpenInfos info of a list of regions to open
642    * @return a list of region opening states
643    */
644   public List<RegionOpeningState> sendRegionOpen(ServerName server,
645       List<Triple<HRegionInfo, Integer, List<ServerName>>> regionOpenInfos)
646   throws IOException {
647     AdminService.BlockingInterface admin = getRsAdmin(server);
648     if (admin == null) {
649       LOG.warn("Attempting to send OPEN RPC to server " + server.toString() +
650         " failed because no RPC connection found to this server");
651       return null;
652     }
653 
654     OpenRegionRequest request =
655       RequestConverter.buildOpenRegionRequest(regionOpenInfos);
656     try {
657       OpenRegionResponse response = admin.openRegion(null, request);
658       return ResponseConverter.getRegionOpeningStateList(response);
659     } catch (ServiceException se) {
660       throw ProtobufUtil.getRemoteException(se);
661     }
662   }
663 
664   /**
665    * Sends an CLOSE RPC to the specified server to close the specified region.
666    * <p>
667    * A region server could reject the close request because it either does not
668    * have the specified region or the region is being split.
669    * @param server server to open a region
670    * @param region region to open
671    * @param versionOfClosingNode
672    *   the version of znode to compare when RS transitions the znode from
673    *   CLOSING state.
674    * @param dest - if the region is moved to another server, the destination server. null otherwise.
675    * @return true if server acknowledged close, false if not
676    * @throws IOException
677    */
678   public boolean sendRegionClose(ServerName server, HRegionInfo region,
679     int versionOfClosingNode, ServerName dest, boolean transitionInZK) throws IOException {
680     if (server == null) throw new NullPointerException("Passed server is null");
681     AdminService.BlockingInterface admin = getRsAdmin(server);
682     if (admin == null) {
683       throw new IOException("Attempting to send CLOSE RPC to server " +
684         server.toString() + " for region " +
685         region.getRegionNameAsString() +
686         " failed because no RPC connection found to this server");
687     }
688     return ProtobufUtil.closeRegion(admin, region.getRegionName(),
689       versionOfClosingNode, dest, transitionInZK);
690   }
691 
692   public boolean sendRegionClose(ServerName server,
693       HRegionInfo region, int versionOfClosingNode) throws IOException {
694     return sendRegionClose(server, region, versionOfClosingNode, null, true);
695   }
696 
697   /**
698    * Sends an MERGE REGIONS RPC to the specified server to merge the specified
699    * regions.
700    * <p>
701    * A region server could reject the close request because it either does not
702    * have the specified region.
703    * @param server server to merge regions
704    * @param region_a region to merge
705    * @param region_b region to merge
706    * @param forcible true if do a compulsory merge, otherwise we will only merge
707    *          two adjacent regions
708    * @throws IOException
709    */
710   public void sendRegionsMerge(ServerName server, HRegionInfo region_a,
711       HRegionInfo region_b, boolean forcible) throws IOException {
712     if (server == null)
713       throw new NullPointerException("Passed server is null");
714     if (region_a == null || region_b == null)
715       throw new NullPointerException("Passed region is null");
716     AdminService.BlockingInterface admin = getRsAdmin(server);
717     if (admin == null) {
718       throw new IOException("Attempting to send MERGE REGIONS RPC to server "
719           + server.toString() + " for region "
720           + region_a.getRegionNameAsString() + ","
721           + region_b.getRegionNameAsString()
722           + " failed because no RPC connection found to this server");
723     }
724     ProtobufUtil.mergeRegions(admin, region_a, region_b, forcible);
725   }
726 
727   /**
728    * Check if a region server is reachable and has the expected start code
729    */
730   public boolean isServerReachable(ServerName server) {
731     if (server == null) throw new NullPointerException("Passed server is null");
732     int maximumAttempts = Math.max(1, master.getConfiguration().getInt(
733       "hbase.master.maximum.ping.server.attempts", 10));
734     for (int i = 0; i < maximumAttempts; i++) {
735       try {
736         AdminService.BlockingInterface admin = getRsAdmin(server);
737         if (admin != null) {
738           ServerInfo info = ProtobufUtil.getServerInfo(admin);
739           return info != null && info.hasServerName()
740             && server.getStartcode() == info.getServerName().getStartCode();
741         }
742       } catch (IOException ioe) {
743         LOG.debug("Couldn't reach " + server + ", try=" + i
744           + " of " + maximumAttempts, ioe);
745       }
746     }
747     return false;
748   }
749 
750     /**
751     * @param sn
752     * @return Admin interface for the remote regionserver named <code>sn</code>
753     * @throws IOException
754     * @throws RetriesExhaustedException wrapping a ConnectException if failed
755     */
756   private AdminService.BlockingInterface getRsAdmin(final ServerName sn)
757   throws IOException {
758     AdminService.BlockingInterface admin = this.rsAdmins.get(sn);
759     if (admin == null) {
760       LOG.debug("New admin connection to " + sn.toString());
761       admin = this.connection.getAdmin(sn);
762       this.rsAdmins.put(sn, admin);
763     }
764     return admin;
765   }
766 
767   /**
768    * Wait for the region servers to report in.
769    * We will wait until one of this condition is met:
770    *  - the master is stopped
771    *  - the 'hbase.master.wait.on.regionservers.maxtostart' number of
772    *    region servers is reached
773    *  - the 'hbase.master.wait.on.regionservers.mintostart' is reached AND
774    *   there have been no new region server in for
775    *      'hbase.master.wait.on.regionservers.interval' time AND
776    *   the 'hbase.master.wait.on.regionservers.timeout' is reached
777    *
778    * @throws InterruptedException
779    */
780   public void waitForRegionServers(MonitoredTask status)
781   throws InterruptedException {
782     final long interval = this.master.getConfiguration().
783       getLong(WAIT_ON_REGIONSERVERS_INTERVAL, 1500);
784     final long timeout = this.master.getConfiguration().
785       getLong(WAIT_ON_REGIONSERVERS_TIMEOUT, 4500);
786     int minToStart = this.master.getConfiguration().
787       getInt(WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
788     if (minToStart < 1) {
789       LOG.warn(String.format(
790         "The value of '%s' (%d) can not be less than 1, ignoring.",
791         WAIT_ON_REGIONSERVERS_MINTOSTART, minToStart));
792       minToStart = 1;
793     }
794     int maxToStart = this.master.getConfiguration().
795       getInt(WAIT_ON_REGIONSERVERS_MAXTOSTART, Integer.MAX_VALUE);
796     if (maxToStart < minToStart) {
797         LOG.warn(String.format(
798             "The value of '%s' (%d) is set less than '%s' (%d), ignoring.",
799             WAIT_ON_REGIONSERVERS_MAXTOSTART, maxToStart,
800             WAIT_ON_REGIONSERVERS_MINTOSTART, minToStart));
801         maxToStart = Integer.MAX_VALUE;
802     }
803 
804     long now =  System.currentTimeMillis();
805     final long startTime = now;
806     long slept = 0;
807     long lastLogTime = 0;
808     long lastCountChange = startTime;
809     int count = countOfRegionServers();
810     int oldCount = 0;
811     while (
812       !this.master.isStopped() &&
813         count < maxToStart &&
814         (lastCountChange+interval > now || timeout > slept || count < minToStart)
815       ){
816 
817       // Log some info at every interval time or if there is a change
818       if (oldCount != count || lastLogTime+interval < now){
819         lastLogTime = now;
820         String msg =
821           "Waiting for region servers count to settle; currently"+
822             " checked in " + count + ", slept for " + slept + " ms," +
823             " expecting minimum of " + minToStart + ", maximum of "+ maxToStart+
824             ", timeout of "+timeout+" ms, interval of "+interval+" ms.";
825         LOG.info(msg);
826         status.setStatus(msg);
827       }
828 
829       // We sleep for some time
830       final long sleepTime = 50;
831       Thread.sleep(sleepTime);
832       now =  System.currentTimeMillis();
833       slept = now - startTime;
834 
835       oldCount = count;
836       count = countOfRegionServers();
837       if (count != oldCount) {
838         lastCountChange = now;
839       }
840     }
841 
842     LOG.info("Finished waiting for region servers count to settle;" +
843       " checked in " + count + ", slept for " + slept + " ms," +
844       " expecting minimum of " + minToStart + ", maximum of "+ maxToStart+","+
845       " master is "+ (this.master.isStopped() ? "stopped.": "running.")
846     );
847   }
848 
849   /**
850    * @return A copy of the internal list of online servers.
851    */
852   public List<ServerName> getOnlineServersList() {
853     // TODO: optimize the load balancer call so we don't need to make a new list
854     // TODO: FIX. THIS IS POPULAR CALL.
855     return new ArrayList<ServerName>(this.onlineServers.keySet());
856   }
857 
858   /**
859    * @return A copy of the internal list of draining servers.
860    */
861   public List<ServerName> getDrainingServersList() {
862     return new ArrayList<ServerName>(this.drainingServers);
863   }
864 
865   /**
866    * @return A copy of the internal set of deadNotExpired servers.
867    */
868   Set<ServerName> getDeadNotExpiredServers() {
869     return new HashSet<ServerName>(this.queuedDeadServers);
870   }
871 
872   /**
873    * @return A copy of the internal map of requeuedDeadServers servers and their corresponding
874    *         splitlog need flag.
875    */
876   Map<ServerName, Boolean> getRequeuedDeadServers() {
877     return Collections.unmodifiableMap(this.requeuedDeadServers);
878   }
879   
880   public boolean isServerOnline(ServerName serverName) {
881     return serverName != null && onlineServers.containsKey(serverName);
882   }
883 
884   /**
885    * Check if a server is known to be dead.  A server can be online,
886    * or known to be dead, or unknown to this manager (i.e, not online,
887    * not known to be dead either. it is simply not tracked by the
888    * master any more, for example, a very old previous instance).
889    */
890   public synchronized boolean isServerDead(ServerName serverName) {
891     return serverName == null || deadservers.isDeadServer(serverName)
892       || queuedDeadServers.contains(serverName)
893       || requeuedDeadServers.containsKey(serverName);
894   }
895 
896   public void shutdownCluster() {
897     this.clusterShutdown = true;
898     this.master.stop("Cluster shutdown requested");
899   }
900 
901   public boolean isClusterShutdown() {
902     return this.clusterShutdown;
903   }
904 
905   /**
906    * Stop the ServerManager.  Currently closes the connection to the master.
907    */
908   public void stop() {
909     if (connection != null) {
910       try {
911         connection.close();
912       } catch (IOException e) {
913         LOG.error("Attempt to close connection to master failed", e);
914       }
915     }
916   }
917 
918   /**
919    * Creates a list of possible destinations for a region. It contains the online servers, but not
920    *  the draining or dying servers.
921    *  @param serverToExclude can be null if there is no server to exclude
922    */
923   public List<ServerName> createDestinationServersList(final ServerName serverToExclude){
924     final List<ServerName> destServers = getOnlineServersList();
925 
926     if (serverToExclude != null){
927       destServers.remove(serverToExclude);
928     }
929 
930     // Loop through the draining server list and remove them from the server list
931     final List<ServerName> drainingServersCopy = getDrainingServersList();
932     if (!drainingServersCopy.isEmpty()) {
933       for (final ServerName server: drainingServersCopy) {
934         destServers.remove(server);
935       }
936     }
937 
938     // Remove the deadNotExpired servers from the server list.
939     removeDeadNotExpiredServers(destServers);
940 
941     return destServers;
942   }
943 
944   /**
945    * Calls {@link #createDestinationServersList} without server to exclude.
946    */
947   public List<ServerName> createDestinationServersList(){
948     return createDestinationServersList(null);
949   }
950 
951     /**
952     * Loop through the deadNotExpired server list and remove them from the
953     * servers.
954     * This function should be used carefully outside of this class. You should use a high level
955     *  method such as {@link #createDestinationServersList()} instead of managing you own list.
956     */
957   void removeDeadNotExpiredServers(List<ServerName> servers) {
958     Set<ServerName> deadNotExpiredServersCopy = this.getDeadNotExpiredServers();
959     if (!deadNotExpiredServersCopy.isEmpty()) {
960       for (ServerName server : deadNotExpiredServersCopy) {
961         LOG.debug("Removing dead but not expired server: " + server
962           + " from eligible server pool.");
963         servers.remove(server);
964       }
965     }
966   }
967   
968   /**
969    * To clear any dead server with same host name and port of any online server
970    */
971   void clearDeadServersWithSameHostNameAndPortOfOnlineServer() {
972     for (ServerName serverName : getOnlineServersList()) {
973       deadservers.cleanAllPreviousInstances(serverName);
974     }
975   }
976 }