View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.IOException;
22  import java.io.InterruptedIOException;
23  import java.lang.reflect.Constructor;
24  import java.lang.reflect.InvocationTargetException;
25  import java.net.InetAddress;
26  import java.net.InetSocketAddress;
27  import java.net.UnknownHostException;
28  import java.util.ArrayList;
29  import java.util.Arrays;
30  import java.util.Collection;
31  import java.util.Collections;
32  import java.util.Comparator;
33  import java.util.HashSet;
34  import java.util.Iterator;
35  import java.util.List;
36  import java.util.Map;
37  import java.util.Set;
38  import java.util.concurrent.TimeUnit;
39  import java.util.concurrent.atomic.AtomicReference;
40  import java.util.regex.Pattern;
41  
42  import javax.servlet.ServletException;
43  import javax.servlet.http.HttpServlet;
44  import javax.servlet.http.HttpServletRequest;
45  import javax.servlet.http.HttpServletResponse;
46  
47  import org.apache.commons.logging.Log;
48  import org.apache.commons.logging.LogFactory;
49  import org.apache.hadoop.conf.Configuration;
50  import org.apache.hadoop.fs.Path;
51  import org.apache.hadoop.hbase.ClusterStatus;
52  import org.apache.hadoop.hbase.CoordinatedStateException;
53  import org.apache.hadoop.hbase.CoordinatedStateManager;
54  import org.apache.hadoop.hbase.DoNotRetryIOException;
55  import org.apache.hadoop.hbase.HBaseIOException;
56  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
57  import org.apache.hadoop.hbase.HColumnDescriptor;
58  import org.apache.hadoop.hbase.HConstants;
59  import org.apache.hadoop.hbase.HRegionInfo;
60  import org.apache.hadoop.hbase.HTableDescriptor;
61  import org.apache.hadoop.hbase.MasterNotRunningException;
62  import org.apache.hadoop.hbase.MetaMigrationConvertingToPB;
63  import org.apache.hadoop.hbase.MetaTableAccessor;
64  import org.apache.hadoop.hbase.NamespaceDescriptor;
65  import org.apache.hadoop.hbase.NamespaceNotFoundException;
66  import org.apache.hadoop.hbase.PleaseHoldException;
67  import org.apache.hadoop.hbase.Server;
68  import org.apache.hadoop.hbase.ServerLoad;
69  import org.apache.hadoop.hbase.ServerName;
70  import org.apache.hadoop.hbase.TableDescriptors;
71  import org.apache.hadoop.hbase.TableName;
72  import org.apache.hadoop.hbase.TableNotDisabledException;
73  import org.apache.hadoop.hbase.TableNotFoundException;
74  import org.apache.hadoop.hbase.UnknownRegionException;
75  import org.apache.hadoop.hbase.classification.InterfaceAudience;
76  import org.apache.hadoop.hbase.client.MetaScanner;
77  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
78  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
79  import org.apache.hadoop.hbase.client.Result;
80  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
81  import org.apache.hadoop.hbase.exceptions.DeserializationException;
82  import org.apache.hadoop.hbase.executor.ExecutorType;
83  import org.apache.hadoop.hbase.ipc.RequestContext;
84  import org.apache.hadoop.hbase.ipc.RpcServer;
85  import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
86  import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode;
87  import org.apache.hadoop.hbase.master.RegionState.State;
88  import org.apache.hadoop.hbase.master.balancer.BalancerChore;
89  import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
90  import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore;
91  import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
92  import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
93  import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
94  import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
95  import org.apache.hadoop.hbase.master.handler.DeleteTableHandler;
96  import org.apache.hadoop.hbase.master.handler.DisableTableHandler;
97  import org.apache.hadoop.hbase.master.handler.DispatchMergingRegionHandler;
98  import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
99  import org.apache.hadoop.hbase.master.handler.ModifyTableHandler;
100 import org.apache.hadoop.hbase.master.handler.TableAddFamilyHandler;
101 import org.apache.hadoop.hbase.master.handler.TableDeleteFamilyHandler;
102 import org.apache.hadoop.hbase.master.handler.TableModifyFamilyHandler;
103 import org.apache.hadoop.hbase.master.handler.TruncateTableHandler;
104 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
105 import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
106 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
107 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
108 import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
109 import org.apache.hadoop.hbase.procedure.flush.MasterFlushTableProcedureManager;
110 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo;
111 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
112 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
113 import org.apache.hadoop.hbase.regionserver.HRegionServer;
114 import org.apache.hadoop.hbase.regionserver.RSRpcServices;
115 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
116 import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy;
117 import org.apache.hadoop.hbase.replication.regionserver.Replication;
118 import org.apache.hadoop.hbase.security.UserProvider;
119 import org.apache.hadoop.hbase.util.Addressing;
120 import org.apache.hadoop.hbase.util.Bytes;
121 import org.apache.hadoop.hbase.util.CompressionTest;
122 import org.apache.hadoop.hbase.util.ConfigUtil;
123 import org.apache.hadoop.hbase.util.EncryptionTest;
124 import org.apache.hadoop.hbase.util.FSUtils;
125 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
126 import org.apache.hadoop.hbase.util.HasThread;
127 import org.apache.hadoop.hbase.util.Pair;
128 import org.apache.hadoop.hbase.util.Threads;
129 import org.apache.hadoop.hbase.util.VersionInfo;
130 import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker;
131 import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
132 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
133 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
134 import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
135 import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
136 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
137 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
138 import org.apache.zookeeper.KeeperException;
139 import org.mortbay.jetty.Connector;
140 import org.mortbay.jetty.nio.SelectChannelConnector;
141 import org.mortbay.jetty.servlet.Context;
142 
143 import com.google.common.annotations.VisibleForTesting;
144 import com.google.common.collect.Maps;
145 import com.google.protobuf.Descriptors;
146 import com.google.protobuf.Service;
147 
148 /**
149  * HMaster is the "master server" for HBase. An HBase cluster has one active
150  * master.  If many masters are started, all compete.  Whichever wins goes on to
151  * run the cluster.  All others park themselves in their constructor until
152  * master or cluster shutdown or until the active master loses its lease in
153  * zookeeper.  Thereafter, all running master jostle to take over master role.
154  *
155  * <p>The Master can be asked shutdown the cluster. See {@link #shutdown()}.  In
156  * this case it will tell all regionservers to go down and then wait on them
157  * all reporting in that they are down.  This master will then shut itself down.
158  *
159  * <p>You can also shutdown just this master.  Call {@link #stopMaster()}.
160  *
161  * @see org.apache.zookeeper.Watcher
162  */
163 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
164 @SuppressWarnings("deprecation")
165 public class HMaster extends HRegionServer implements MasterServices, Server {
166   private static final Log LOG = LogFactory.getLog(HMaster.class.getName());
167 
168   /**
169    * Protection against zombie master. Started once Master accepts active responsibility and
170    * starts taking over responsibilities. Allows a finite time window before giving up ownership.
171    */
172   private static class InitializationMonitor extends HasThread {
173     /** The amount of time in milliseconds to sleep before checking initialization status. */
174     public static final String TIMEOUT_KEY = "hbase.master.initializationmonitor.timeout";
175     public static final long TIMEOUT_DEFAULT = TimeUnit.MILLISECONDS.convert(15, TimeUnit.MINUTES);
176 
177     /**
178      * When timeout expired and initialization has not complete, call {@link System#exit(int)} when
179      * true, do nothing otherwise.
180      */
181     public static final String HALT_KEY = "hbase.master.initializationmonitor.haltontimeout";
182     public static final boolean HALT_DEFAULT = false;
183 
184     private final HMaster master;
185     private final long timeout;
186     private final boolean haltOnTimeout;
187 
188     /** Creates a Thread that monitors the {@link #isInitialized()} state. */
189     InitializationMonitor(HMaster master) {
190       super("MasterInitializationMonitor");
191       this.master = master;
192       this.timeout = master.getConfiguration().getLong(TIMEOUT_KEY, TIMEOUT_DEFAULT);
193       this.haltOnTimeout = master.getConfiguration().getBoolean(HALT_KEY, HALT_DEFAULT);
194       this.setDaemon(true);
195     }
196 
197     @Override
198     public void run() {
199       try {
200         while (!master.isStopped() && master.isActiveMaster()) {
201           Thread.sleep(timeout);
202           if (master.isInitialized()) {
203             LOG.debug("Initialization completed within allotted tolerance. Monitor exiting.");
204           } else {
205             LOG.error("Master failed to complete initialization after " + timeout + "ms. Please"
206                 + " consider submitting a bug report including a thread dump of this process.");
207             if (haltOnTimeout) {
208               LOG.error("Zombie Master exiting. Thread dump to stdout");
209               Threads.printThreadInfo(System.out, "Zombie HMaster");
210               System.exit(-1);
211             }
212           }
213         }
214       } catch (InterruptedException ie) {
215         LOG.trace("InitMonitor thread interrupted. Existing.");
216       }
217     }
218   }
219 
220   // MASTER is name of the webapp and the attribute name used stuffing this
221   //instance into web context.
222   public static final String MASTER = "master";
223 
224   // Manager and zk listener for master election
225   private final ActiveMasterManager activeMasterManager;
226   // Region server tracker
227   RegionServerTracker regionServerTracker;
228   // Draining region server tracker
229   private DrainingServerTracker drainingServerTracker;
230   // Tracker for load balancer state
231   LoadBalancerTracker loadBalancerTracker;
232 
233   /** Namespace stuff */
234   private TableNamespaceManager tableNamespaceManager;
235 
236   // Metrics for the HMaster
237   final MetricsMaster metricsMaster;
238   // file system manager for the master FS operations
239   private MasterFileSystem fileSystemManager;
240 
241   // server manager to deal with region server info
242   volatile ServerManager serverManager;
243 
244   // manager of assignment nodes in zookeeper
245   AssignmentManager assignmentManager;
246 
247   // buffer for "fatal error" notices from region servers
248   // in the cluster. This is only used for assisting
249   // operations/debugging.
250   MemoryBoundedLogMessageBuffer rsFatals;
251 
252   // flag set after we become the active master (used for testing)
253   private volatile boolean isActiveMaster = false;
254 
255   // flag set after we complete initialization once active,
256   // it is not private since it's used in unit tests
257   volatile boolean initialized = false;
258 
259   // flag set after master services are started,
260   // initialization may have not completed yet.
261   volatile boolean serviceStarted = false;
262 
263   // flag set after we complete assignMeta.
264   private volatile boolean serverShutdownHandlerEnabled = false;
265 
266   LoadBalancer balancer;
267   private BalancerChore balancerChore;
268   private ClusterStatusChore clusterStatusChore;
269   private ClusterStatusPublisher clusterStatusPublisherChore = null;
270 
271   CatalogJanitor catalogJanitorChore;
272   private LogCleaner logCleaner;
273   private HFileCleaner hfileCleaner;
274 
275   MasterCoprocessorHost cpHost;
276 
277   private final boolean preLoadTableDescriptors;
278 
279   // Time stamps for when a hmaster became active
280   private long masterActiveTime;
281 
282   //should we check the compression codec type at master side, default true, HBASE-6370
283   private final boolean masterCheckCompression;
284 
285   //should we check encryption settings at master side, default true
286   private final boolean masterCheckEncryption;
287 
288   Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap();
289 
290   // monitor for snapshot of hbase tables
291   SnapshotManager snapshotManager;
292   // monitor for distributed procedures
293   MasterProcedureManagerHost mpmHost;
294 
295   /** flag used in test cases in order to simulate RS failures during master initialization */
296   private volatile boolean initializationBeforeMetaAssignment = false;
297 
298   /** jetty server for master to redirect requests to regionserver infoServer */
299   private org.mortbay.jetty.Server masterJettyServer;
300 
301   public static class RedirectServlet extends HttpServlet {
302     private static final long serialVersionUID = 2894774810058302472L;
303     private static int regionServerInfoPort;
304 
305     @Override
306     public void doGet(HttpServletRequest request,
307         HttpServletResponse response) throws ServletException, IOException {
308       String redirectUrl = request.getScheme() + "://"
309         + request.getServerName() + ":" + regionServerInfoPort
310         + request.getRequestURI();
311       response.sendRedirect(redirectUrl);
312     }
313   }
314 
315   /**
316    * Initializes the HMaster. The steps are as follows:
317    * <p>
318    * <ol>
319    * <li>Initialize the local HRegionServer
320    * <li>Start the ActiveMasterManager.
321    * </ol>
322    * <p>
323    * Remaining steps of initialization occur in
324    * #finishActiveMasterInitialization(MonitoredTask) after
325    * the master becomes the active one.
326    *
327    * @throws InterruptedException
328    * @throws KeeperException
329    * @throws IOException
330    */
331   public HMaster(final Configuration conf, CoordinatedStateManager csm)
332       throws IOException, KeeperException, InterruptedException {
333     super(conf, csm);
334     this.rsFatals = new MemoryBoundedLogMessageBuffer(
335       conf.getLong("hbase.master.buffer.for.rs.fatals", 1*1024*1024));
336 
337     LOG.info("hbase.rootdir=" + FSUtils.getRootDir(this.conf) +
338         ", hbase.cluster.distributed=" + this.conf.getBoolean(HConstants.CLUSTER_DISTRIBUTED, false));
339 
340     Replication.decorateMasterConfiguration(this.conf);
341 
342     // Hack! Maps DFSClient => Master for logs.  HDFS made this
343     // config param for task trackers, but we can piggyback off of it.
344     if (this.conf.get("mapreduce.task.attempt.id") == null) {
345       this.conf.set("mapreduce.task.attempt.id", "hb_m_" + this.serverName.toString());
346     }
347 
348     // should we check the compression codec type at master side, default true, HBASE-6370
349     this.masterCheckCompression = conf.getBoolean("hbase.master.check.compression", true);
350 
351     // should we check encryption settings at master side, default true
352     this.masterCheckEncryption = conf.getBoolean("hbase.master.check.encryption", true);
353 
354     this.metricsMaster = new MetricsMaster( new MetricsMasterWrapperImpl(this));
355 
356     // preload table descriptor at startup
357     this.preLoadTableDescriptors = conf.getBoolean("hbase.master.preload.tabledescriptors", true);
358 
359     // Do we publish the status?
360 
361     boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED,
362         HConstants.STATUS_PUBLISHED_DEFAULT);
363     Class<? extends ClusterStatusPublisher.Publisher> publisherClass =
364         conf.getClass(ClusterStatusPublisher.STATUS_PUBLISHER_CLASS,
365             ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS,
366             ClusterStatusPublisher.Publisher.class);
367 
368     if (shouldPublish) {
369       if (publisherClass == null) {
370         LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " +
371             ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS +
372             " is not set - not publishing status");
373       } else {
374         clusterStatusPublisherChore = new ClusterStatusPublisher(this, conf, publisherClass);
375         Threads.setDaemonThreadRunning(clusterStatusPublisherChore.getThread());
376       }
377     }
378     activeMasterManager = new ActiveMasterManager(zooKeeper, this.serverName, this);
379     int infoPort = putUpJettyServer();
380     startActiveMasterManager(infoPort);
381   }
382 
383   // return the actual infoPort, -1 means disable info server.
384   private int putUpJettyServer() throws IOException {
385     if (!conf.getBoolean("hbase.master.infoserver.redirect", true)) {
386       return -1;
387     }
388     int infoPort = conf.getInt("hbase.master.info.port.orig",
389       HConstants.DEFAULT_MASTER_INFOPORT);
390     // -1 is for disabling info server, so no redirecting
391     if (infoPort < 0 || infoServer == null) {
392       return -1;
393     }
394     String addr = conf.get("hbase.master.info.bindAddress", "0.0.0.0");
395     if (!Addressing.isLocalAddress(InetAddress.getByName(addr))) {
396       String msg =
397           "Failed to start redirecting jetty server. Address " + addr
398               + " does not belong to this host. Correct configuration parameter: "
399               + "hbase.master.info.bindAddress";
400       LOG.error(msg);
401       throw new IOException(msg);
402     }
403 
404     RedirectServlet.regionServerInfoPort = infoServer.getPort();
405     masterJettyServer = new org.mortbay.jetty.Server();
406     Connector connector = new SelectChannelConnector();
407     connector.setHost(addr);
408     connector.setPort(infoPort);
409     masterJettyServer.addConnector(connector);
410     masterJettyServer.setStopAtShutdown(true);
411     Context context = new Context(masterJettyServer, "/", Context.NO_SESSIONS);
412     context.addServlet(RedirectServlet.class, "/*");
413     try {
414       masterJettyServer.start();
415     } catch (Exception e) {
416       throw new IOException("Failed to start redirecting jetty server", e);
417     }
418     return connector.getLocalPort();
419   }
420 
421   /**
422    * For compatibility, if failed with regionserver credentials, try the master one
423    */
424   @Override
425   protected void login(UserProvider user, String host) throws IOException {
426     try {
427       super.login(user, host);
428     } catch (IOException ie) {
429       user.login("hbase.master.keytab.file",
430         "hbase.master.kerberos.principal", host);
431     }
432   }
433 
434   /**
435    * If configured to put regions on active master,
436    * wait till a backup master becomes active.
437    * Otherwise, loop till the server is stopped or aborted.
438    */
439   @Override
440   protected void waitForMasterActive(){
441     boolean tablesOnMaster = BaseLoadBalancer.tablesOnMaster(conf);
442     while (!(tablesOnMaster && isActiveMaster)
443         && !isStopped() && !isAborted()) {
444       sleeper.sleep();
445     }
446   }
447 
448   @VisibleForTesting
449   public MasterRpcServices getMasterRpcServices() {
450     return (MasterRpcServices)rpcServices;
451   }
452 
453   public boolean balanceSwitch(final boolean b) throws IOException {
454     return getMasterRpcServices().switchBalancer(b, BalanceSwitchMode.ASYNC);
455   }
456 
457   @Override
458   protected String getProcessName() {
459     return MASTER;
460   }
461 
462   @Override
463   protected boolean canCreateBaseZNode() {
464     return true;
465   }
466 
467   @Override
468   protected boolean canUpdateTableDescriptor() {
469     return true;
470   }
471 
472   @Override
473   protected RSRpcServices createRpcServices() throws IOException {
474     return new MasterRpcServices(this);
475   }
476 
477   @Override
478   protected void configureInfoServer() {
479     infoServer.addServlet("master-status", "/master-status", MasterStatusServlet.class);
480     infoServer.setAttribute(MASTER, this);
481     if (BaseLoadBalancer.tablesOnMaster(conf)) {
482       super.configureInfoServer();
483     }
484   }
485 
486   @Override
487   protected Class<? extends HttpServlet> getDumpServlet() {
488     return MasterDumpServlet.class;
489   }
490 
491   /**
492    * Emit the HMaster metrics, such as region in transition metrics.
493    * Surrounding in a try block just to be sure metrics doesn't abort HMaster.
494    */
495   @Override
496   protected void doMetrics() {
497     try {
498       if (assignmentManager != null) {
499         assignmentManager.updateRegionsInTransitionMetrics();
500       }
501     } catch (Throwable e) {
502       LOG.error("Couldn't update metrics: " + e.getMessage());
503     }
504   }
505 
506   MetricsMaster getMasterMetrics() {
507     return metricsMaster;
508   }
509 
510   /**
511    * Initialize all ZK based system trackers.
512    * @throws IOException
513    * @throws InterruptedException
514    * @throws KeeperException
515    * @throws CoordinatedStateException
516    */
517   void initializeZKBasedSystemTrackers() throws IOException,
518       InterruptedException, KeeperException, CoordinatedStateException {
519     this.balancer = LoadBalancerFactory.getLoadBalancer(conf);
520     this.loadBalancerTracker = new LoadBalancerTracker(zooKeeper, this);
521     this.loadBalancerTracker.start();
522     this.assignmentManager = new AssignmentManager(this, serverManager,
523       this.balancer, this.service, this.metricsMaster,
524       this.tableLockManager);
525     zooKeeper.registerListenerFirst(assignmentManager);
526 
527     this.regionServerTracker = new RegionServerTracker(zooKeeper, this,
528         this.serverManager);
529     this.regionServerTracker.start();
530 
531     this.drainingServerTracker = new DrainingServerTracker(zooKeeper, this,
532       this.serverManager);
533     this.drainingServerTracker.start();
534 
535     // Set the cluster as up.  If new RSs, they'll be waiting on this before
536     // going ahead with their startup.
537     boolean wasUp = this.clusterStatusTracker.isClusterUp();
538     if (!wasUp) this.clusterStatusTracker.setClusterUp();
539 
540     LOG.info("Server active/primary master=" + this.serverName +
541         ", sessionid=0x" +
542         Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()) +
543         ", setting cluster-up flag (Was=" + wasUp + ")");
544 
545     // create/initialize the snapshot manager and other procedure managers
546     this.snapshotManager = new SnapshotManager();
547     this.mpmHost = new MasterProcedureManagerHost();
548     this.mpmHost.register(this.snapshotManager);
549     this.mpmHost.register(new MasterFlushTableProcedureManager());
550     this.mpmHost.loadProcedures(conf);
551     this.mpmHost.initialize(this, this.metricsMaster);
552   }
553 
554   /**
555    * Finish initialization of HMaster after becoming the primary master.
556    *
557    * <ol>
558    * <li>Initialize master components - file system manager, server manager,
559    *     assignment manager, region server tracker, etc</li>
560    * <li>Start necessary service threads - balancer, catalog janior,
561    *     executor services, etc</li>
562    * <li>Set cluster as UP in ZooKeeper</li>
563    * <li>Wait for RegionServers to check-in</li>
564    * <li>Split logs and perform data recovery, if necessary</li>
565    * <li>Ensure assignment of meta/namespace regions<li>
566    * <li>Handle either fresh cluster start or master failover</li>
567    * </ol>
568    *
569    * @throws IOException
570    * @throws InterruptedException
571    * @throws KeeperException
572    * @throws CoordinatedStateException
573    */
574   private void finishActiveMasterInitialization(MonitoredTask status)
575       throws IOException, InterruptedException, KeeperException, CoordinatedStateException {
576 
577     isActiveMaster = true;
578     Thread zombieDetector = new Thread(new InitializationMonitor(this));
579     zombieDetector.start();
580 
581     /*
582      * We are active master now... go initialize components we need to run.
583      * Note, there may be dross in zk from previous runs; it'll get addressed
584      * below after we determine if cluster startup or failover.
585      */
586 
587     status.setStatus("Initializing Master file system");
588 
589     this.masterActiveTime = System.currentTimeMillis();
590     // TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
591     this.fileSystemManager = new MasterFileSystem(this, this);
592 
593     // enable table descriptors cache
594     this.tableDescriptors.setCacheOn();
595 
596     // warm-up HTDs cache on master initialization
597     if (preLoadTableDescriptors) {
598       status.setStatus("Pre-loading table descriptors");
599       this.tableDescriptors.getAll();
600     }
601 
602     // publish cluster ID
603     status.setStatus("Publishing Cluster ID in ZooKeeper");
604     ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());
605     this.serverManager = createServerManager(this, this);
606 
607     setupClusterConnection();
608 
609     // Invalidate all write locks held previously
610     this.tableLockManager.reapWriteLocks();
611 
612     status.setStatus("Initializing ZK system trackers");
613     initializeZKBasedSystemTrackers();
614 
615     // initialize master side coprocessors before we start handling requests
616     status.setStatus("Initializing master coprocessors");
617     this.cpHost = new MasterCoprocessorHost(this, this.conf);
618 
619     // start up all service threads.
620     status.setStatus("Initializing master service threads");
621     startServiceThreads();
622 
623     // Wake up this server to check in
624     sleeper.skipSleepCycle();
625 
626     // Wait for region servers to report in
627     this.serverManager.waitForRegionServers(status);
628     // Check zk for region servers that are up but didn't register
629     for (ServerName sn: this.regionServerTracker.getOnlineServers()) {
630       // The isServerOnline check is opportunistic, correctness is handled inside
631       if (!this.serverManager.isServerOnline(sn)
632           && serverManager.checkAndRecordNewServer(sn, ServerLoad.EMPTY_SERVERLOAD)) {
633         LOG.info("Registered server found up in zk but who has not yet reported in: " + sn);
634       }
635     }
636 
637     // get a list for previously failed RS which need log splitting work
638     // we recover hbase:meta region servers inside master initialization and
639     // handle other failed servers in SSH in order to start up master node ASAP
640     Set<ServerName> previouslyFailedServers = this.fileSystemManager
641         .getFailedServersFromLogFolders();
642 
643     // remove stale recovering regions from previous run
644     this.fileSystemManager.removeStaleRecoveringRegionsFromZK(previouslyFailedServers);
645 
646     // log splitting for hbase:meta server
647     ServerName oldMetaServerLocation = metaTableLocator.getMetaRegionLocation(this.getZooKeeper());
648     if (oldMetaServerLocation != null && previouslyFailedServers.contains(oldMetaServerLocation)) {
649       splitMetaLogBeforeAssignment(oldMetaServerLocation);
650       // Note: we can't remove oldMetaServerLocation from previousFailedServers list because it
651       // may also host user regions
652     }
653     Set<ServerName> previouslyFailedMetaRSs = getPreviouselyFailedMetaServersFromZK();
654     // need to use union of previouslyFailedMetaRSs recorded in ZK and previouslyFailedServers
655     // instead of previouslyFailedMetaRSs alone to address the following two situations:
656     // 1) the chained failure situation(recovery failed multiple times in a row).
657     // 2) master get killed right before it could delete the recovering hbase:meta from ZK while the
658     // same server still has non-meta wals to be replayed so that
659     // removeStaleRecoveringRegionsFromZK can't delete the stale hbase:meta region
660     // Passing more servers into splitMetaLog is all right. If a server doesn't have hbase:meta wal,
661     // there is no op for the server.
662     previouslyFailedMetaRSs.addAll(previouslyFailedServers);
663 
664     this.initializationBeforeMetaAssignment = true;
665 
666     // Wait for regionserver to finish initialization.
667     if (BaseLoadBalancer.tablesOnMaster(conf)) {
668       waitForServerOnline();
669     }
670 
671     //initialize load balancer
672     this.balancer.setClusterStatus(getClusterStatus());
673     this.balancer.setMasterServices(this);
674     this.balancer.initialize();
675 
676     // Check if master is shutting down because of some issue
677     // in initializing the regionserver or the balancer.
678     if(isStopped()) return;
679 
680     // Make sure meta assigned before proceeding.
681     status.setStatus("Assigning Meta Region");
682     assignMeta(status, previouslyFailedMetaRSs);
683     // check if master is shutting down because above assignMeta could return even hbase:meta isn't
684     // assigned when master is shutting down
685     if(isStopped()) return;
686 
687     status.setStatus("Submitting log splitting work for previously failed region servers");
688     // Master has recovered hbase:meta region server and we put
689     // other failed region servers in a queue to be handled later by SSH
690     for (ServerName tmpServer : previouslyFailedServers) {
691       this.serverManager.processDeadServer(tmpServer, true);
692     }
693 
694     // Update meta with new PB serialization if required. i.e migrate all HRI to PB serialization
695     // in meta. This must happen before we assign all user regions or else the assignment will
696     // fail.
697     if (this.conf.getBoolean("hbase.MetaMigrationConvertingToPB", true)) {
698       MetaMigrationConvertingToPB.updateMetaIfNecessary(this);
699     }
700 
701     // Fix up assignment manager status
702     status.setStatus("Starting assignment manager");
703     this.assignmentManager.joinCluster();
704 
705     //set cluster status again after user regions are assigned
706     this.balancer.setClusterStatus(getClusterStatus());
707 
708     // Start balancer and meta catalog janitor after meta and regions have
709     // been assigned.
710     status.setStatus("Starting balancer and catalog janitor");
711     this.clusterStatusChore = new ClusterStatusChore(this, balancer);
712     Threads.setDaemonThreadRunning(clusterStatusChore.getThread());
713     this.balancerChore = new BalancerChore(this);
714     Threads.setDaemonThreadRunning(balancerChore.getThread());
715     this.catalogJanitorChore = new CatalogJanitor(this, this);
716     Threads.setDaemonThreadRunning(catalogJanitorChore.getThread());
717 
718     status.setStatus("Starting namespace manager");
719     initNamespace();
720 
721     if (this.cpHost != null) {
722       try {
723         this.cpHost.preMasterInitialization();
724       } catch (IOException e) {
725         LOG.error("Coprocessor preMasterInitialization() hook failed", e);
726       }
727     }
728 
729     status.markComplete("Initialization successful");
730     LOG.info("Master has completed initialization");
731     configurationManager.registerObserver(this.balancer);
732     initialized = true;
733     // clear the dead servers with same host name and port of online server because we are not
734     // removing dead server with same hostname and port of rs which is trying to check in before
735     // master initialization. See HBASE-5916.
736     this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();
737 
738     if (this.cpHost != null) {
739       // don't let cp initialization errors kill the master
740       try {
741         this.cpHost.postStartMaster();
742       } catch (IOException ioe) {
743         LOG.error("Coprocessor postStartMaster() hook failed", ioe);
744       }
745     }
746 
747     zombieDetector.interrupt();
748   }
749 
750   /**
751    * Create a {@link ServerManager} instance.
752    * @param master
753    * @param services
754    * @return An instance of {@link ServerManager}
755    * @throws org.apache.hadoop.hbase.ZooKeeperConnectionException
756    * @throws IOException
757    */
758   ServerManager createServerManager(final Server master,
759       final MasterServices services)
760   throws IOException {
761     // We put this out here in a method so can do a Mockito.spy and stub it out
762     // w/ a mocked up ServerManager.
763     return new ServerManager(master, services);
764   }
765 
766   /**
767    * Check <code>hbase:meta</code> is assigned. If not, assign it.
768    * @param status MonitoredTask
769    * @param previouslyFailedMetaRSs
770    * @throws InterruptedException
771    * @throws IOException
772    * @throws KeeperException
773    */
774   void assignMeta(MonitoredTask status, Set<ServerName> previouslyFailedMetaRSs)
775       throws InterruptedException, IOException, KeeperException {
776     // Work on meta region
777     int assigned = 0;
778     long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);
779     status.setStatus("Assigning hbase:meta region");
780     // Get current meta state from zk.
781     RegionStates regionStates = assignmentManager.getRegionStates();
782     RegionState metaState = MetaTableLocator.getMetaRegionState(getZooKeeper());
783     ServerName currentMetaServer = metaState.getServerName();
784     if (!ConfigUtil.useZKForAssignment(conf)) {
785       regionStates.createRegionState(HRegionInfo.FIRST_META_REGIONINFO, metaState.getState(),
786         currentMetaServer, null);
787     } else {
788       regionStates.createRegionState(HRegionInfo.FIRST_META_REGIONINFO);
789     }
790     boolean rit = this.assignmentManager.
791       processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO);
792     boolean metaRegionLocation = metaTableLocator.verifyMetaRegionLocation(
793       this.getConnection(), this.getZooKeeper(), timeout);
794     if (!metaRegionLocation || !metaState.isOpened()) {
795       // Meta location is not verified. It should be in transition, or offline.
796       // We will wait for it to be assigned in enableSSHandWaitForMeta below.
797       assigned++;
798       if (!ConfigUtil.useZKForAssignment(conf)) {
799         assignMetaZkLess(regionStates, metaState, timeout, previouslyFailedMetaRSs);
800       } else if (!rit) {
801         // Assign meta since not already in transition
802         if (currentMetaServer != null) {
803           // If the meta server is not known to be dead or online,
804           // just split the meta log, and don't expire it since this
805           // could be a full cluster restart. Otherwise, we will think
806           // this is a failover and lose previous region locations.
807           // If it is really a failover case, AM will find out in rebuilding
808           // user regions. Otherwise, we are good since all logs are split
809           // or known to be replayed before user regions are assigned.
810           if (serverManager.isServerOnline(currentMetaServer)) {
811             LOG.info("Forcing expire of " + currentMetaServer);
812             serverManager.expireServer(currentMetaServer);
813           }
814           splitMetaLogBeforeAssignment(currentMetaServer);
815           previouslyFailedMetaRSs.add(currentMetaServer);
816         }
817         assignmentManager.assignMeta();
818       }
819     } else {
820       // Region already assigned. We didn't assign it. Add to in-memory state.
821       regionStates.updateRegionState(
822         HRegionInfo.FIRST_META_REGIONINFO, State.OPEN, currentMetaServer);
823       this.assignmentManager.regionOnline(
824         HRegionInfo.FIRST_META_REGIONINFO, currentMetaServer);
825     }
826 
827     enableMeta(TableName.META_TABLE_NAME);
828 
829     if ((RecoveryMode.LOG_REPLAY == this.getMasterFileSystem().getLogRecoveryMode())
830         && (!previouslyFailedMetaRSs.isEmpty())) {
831       // replay WAL edits mode need new hbase:meta RS is assigned firstly
832       status.setStatus("replaying log for Meta Region");
833       this.fileSystemManager.splitMetaLog(previouslyFailedMetaRSs);
834     }
835 
836     // Make sure a hbase:meta location is set. We need to enable SSH here since
837     // if the meta region server is died at this time, we need it to be re-assigned
838     // by SSH so that system tables can be assigned.
839     // No need to wait for meta is assigned = 0 when meta is just verified.
840     enableServerShutdownHandler(assigned != 0);
841 
842     LOG.info("hbase:meta assigned=" + assigned + ", rit=" + rit +
843       ", location=" + metaTableLocator.getMetaRegionLocation(this.getZooKeeper()));
844     status.setStatus("META assigned.");
845   }
846 
847   private void assignMetaZkLess(RegionStates regionStates, RegionState regionState, long timeout,
848       Set<ServerName> previouslyFailedRs) throws IOException, KeeperException {
849     ServerName currentServer = regionState.getServerName();
850     if (serverManager.isServerOnline(currentServer)) {
851       LOG.info("Meta was in transition on " + currentServer);
852       assignmentManager.processRegionInTransitionZkLess();
853     } else {
854       if (currentServer != null) {
855         splitMetaLogBeforeAssignment(currentServer);
856         regionStates.logSplit(HRegionInfo.FIRST_META_REGIONINFO);
857         previouslyFailedRs.add(currentServer);
858       }
859       LOG.info("Re-assigning hbase:meta, it was on " + currentServer);
860       regionStates.updateRegionState(HRegionInfo.FIRST_META_REGIONINFO, State.OFFLINE);
861       assignmentManager.assignMeta();
862     }
863   }
864 
865   void initNamespace() throws IOException {
866     //create namespace manager
867     tableNamespaceManager = new TableNamespaceManager(this);
868     tableNamespaceManager.start();
869   }
870 
871   boolean isCatalogJanitorEnabled() {
872     return catalogJanitorChore != null ?
873       catalogJanitorChore.getEnabled() : false;
874   }
875 
876   private void splitMetaLogBeforeAssignment(ServerName currentMetaServer) throws IOException {
877     if (RecoveryMode.LOG_REPLAY == this.getMasterFileSystem().getLogRecoveryMode()) {
878       // In log replay mode, we mark hbase:meta region as recovering in ZK
879       Set<HRegionInfo> regions = new HashSet<HRegionInfo>();
880       regions.add(HRegionInfo.FIRST_META_REGIONINFO);
881       this.fileSystemManager.prepareLogReplay(currentMetaServer, regions);
882     } else {
883       // In recovered.edits mode: create recovered edits file for hbase:meta server
884       this.fileSystemManager.splitMetaLog(currentMetaServer);
885     }
886   }
887 
888   private void enableServerShutdownHandler(
889       final boolean waitForMeta) throws IOException, InterruptedException {
890     // If ServerShutdownHandler is disabled, we enable it and expire those dead
891     // but not expired servers. This is required so that if meta is assigning to
892     // a server which dies after assignMeta starts assignment,
893     // SSH can re-assign it. Otherwise, we will be
894     // stuck here waiting forever if waitForMeta is specified.
895     if (!serverShutdownHandlerEnabled) {
896       serverShutdownHandlerEnabled = true;
897       this.serverManager.processQueuedDeadServers();
898     }
899 
900     if (waitForMeta) {
901       metaTableLocator.waitMetaRegionLocation(this.getZooKeeper());
902       // Above check waits for general meta availability but this does not
903       // guarantee that the transition has completed
904       this.assignmentManager.waitForAssignment(HRegionInfo.FIRST_META_REGIONINFO);
905     }
906   }
907 
908   private void enableMeta(TableName metaTableName) {
909     if (!this.assignmentManager.getTableStateManager().isTableState(metaTableName,
910         ZooKeeperProtos.Table.State.ENABLED)) {
911       this.assignmentManager.setEnabledTable(metaTableName);
912     }
913   }
914 
915   /**
916    * This function returns a set of region server names under hbase:meta recovering region ZK node
917    * @return Set of meta server names which were recorded in ZK
918    * @throws KeeperException
919    */
920   private Set<ServerName> getPreviouselyFailedMetaServersFromZK() throws KeeperException {
921     Set<ServerName> result = new HashSet<ServerName>();
922     String metaRecoveringZNode = ZKUtil.joinZNode(zooKeeper.recoveringRegionsZNode,
923       HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
924     List<String> regionFailedServers = ZKUtil.listChildrenNoWatch(zooKeeper, metaRecoveringZNode);
925     if (regionFailedServers == null) return result;
926 
927     for(String failedServer : regionFailedServers) {
928       ServerName server = ServerName.parseServerName(failedServer);
929       result.add(server);
930     }
931     return result;
932   }
933 
934   @Override
935   public TableDescriptors getTableDescriptors() {
936     return this.tableDescriptors;
937   }
938 
939   @Override
940   public ServerManager getServerManager() {
941     return this.serverManager;
942   }
943 
944   @Override
945   public MasterFileSystem getMasterFileSystem() {
946     return this.fileSystemManager;
947   }
948 
949   /*
950    * Start up all services. If any of these threads gets an unhandled exception
951    * then they just die with a logged message.  This should be fine because
952    * in general, we do not expect the master to get such unhandled exceptions
953    *  as OOMEs; it should be lightly loaded. See what HRegionServer does if
954    *  need to install an unexpected exception handler.
955    */
956   private void startServiceThreads() throws IOException{
957    // Start the executor service pools
958    this.service.startExecutorService(ExecutorType.MASTER_OPEN_REGION,
959       conf.getInt("hbase.master.executor.openregion.threads", 5));
960    this.service.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,
961       conf.getInt("hbase.master.executor.closeregion.threads", 5));
962    this.service.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,
963       conf.getInt("hbase.master.executor.serverops.threads", 5));
964    this.service.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,
965       conf.getInt("hbase.master.executor.serverops.threads", 5));
966    this.service.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS,
967       conf.getInt("hbase.master.executor.logreplayops.threads", 10));
968 
969    // We depend on there being only one instance of this executor running
970    // at a time.  To do concurrency, would need fencing of enable/disable of
971    // tables.
972    // Any time changing this maxThreads to > 1, pls see the comment at
973    // AccessController#postCreateTableHandler
974    this.service.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);
975 
976    // Start log cleaner thread
977    int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 * 1000);
978    this.logCleaner =
979       new LogCleaner(cleanerInterval,
980          this, conf, getMasterFileSystem().getFileSystem(),
981          getMasterFileSystem().getOldLogDir());
982          Threads.setDaemonThreadRunning(logCleaner.getThread(),
983            getServerName().toShortString() + ".oldLogCleaner");
984 
985    //start the hfile archive cleaner thread
986     Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
987     this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()
988         .getFileSystem(), archiveDir);
989     Threads.setDaemonThreadRunning(hfileCleaner.getThread(),
990       getServerName().toShortString() + ".archivedHFileCleaner");
991 
992     serviceStarted = true;
993     if (LOG.isTraceEnabled()) {
994       LOG.trace("Started service threads");
995     }
996   }
997 
998   @Override
999   protected void stopServiceThreads() {
1000     if (masterJettyServer != null) {
1001       LOG.info("Stopping master jetty server");
1002       try {
1003         masterJettyServer.stop();
1004       } catch (Exception e) {
1005         LOG.error("Failed to stop master jetty server", e);
1006       }
1007     }
1008     super.stopServiceThreads();
1009     stopChores();
1010     // Wait for all the remaining region servers to report in IFF we were
1011     // running a cluster shutdown AND we were NOT aborting.
1012     if (!isAborted() && this.serverManager != null &&
1013         this.serverManager.isClusterShutdown()) {
1014       this.serverManager.letRegionServersShutdown();
1015     }
1016     if (LOG.isDebugEnabled()) {
1017       LOG.debug("Stopping service threads");
1018     }
1019     // Clean up and close up shop
1020     if (this.logCleaner!= null) this.logCleaner.interrupt();
1021     if (this.hfileCleaner != null) this.hfileCleaner.interrupt();
1022     if (this.activeMasterManager != null) this.activeMasterManager.stop();
1023     if (this.serverManager != null) this.serverManager.stop();
1024     if (this.assignmentManager != null) this.assignmentManager.stop();
1025     if (this.fileSystemManager != null) this.fileSystemManager.stop();
1026     if (this.mpmHost != null) this.mpmHost.stop("server shutting down.");
1027   }
1028 
1029   private void stopChores() {
1030     if (this.balancerChore != null) {
1031       this.balancerChore.interrupt();
1032     }
1033     if (this.clusterStatusChore != null) {
1034       this.clusterStatusChore.interrupt();
1035     }
1036     if (this.catalogJanitorChore != null) {
1037       this.catalogJanitorChore.interrupt();
1038     }
1039     if (this.clusterStatusPublisherChore != null){
1040       clusterStatusPublisherChore.interrupt();
1041     }
1042   }
1043 
1044   /**
1045    * @return Get remote side's InetAddress
1046    * @throws UnknownHostException
1047    */
1048   InetAddress getRemoteInetAddress(final int port,
1049       final long serverStartCode) throws UnknownHostException {
1050     // Do it out here in its own little method so can fake an address when
1051     // mocking up in tests.
1052     InetAddress ia = RpcServer.getRemoteIp();
1053 
1054     // The call could be from the local regionserver,
1055     // in which case, there is no remote address.
1056     if (ia == null && serverStartCode == startcode) {
1057       InetSocketAddress isa = rpcServices.getSocketAddress();
1058       if (isa != null && isa.getPort() == port) {
1059         ia = isa.getAddress();
1060       }
1061     }
1062     return ia;
1063   }
1064 
1065   /**
1066    * @return Maximum time we should run balancer for
1067    */
1068   private int getBalancerCutoffTime() {
1069     int balancerCutoffTime =
1070       getConfiguration().getInt("hbase.balancer.max.balancing", -1);
1071     if (balancerCutoffTime == -1) {
1072       // No time period set so create one
1073       int balancerPeriod =
1074         getConfiguration().getInt("hbase.balancer.period", 300000);
1075       balancerCutoffTime = balancerPeriod;
1076       // If nonsense period, set it to balancerPeriod
1077       if (balancerCutoffTime <= 0) balancerCutoffTime = balancerPeriod;
1078     }
1079     return balancerCutoffTime;
1080   }
1081 
1082   public boolean balance() throws IOException {
1083     // if master not initialized, don't run balancer.
1084     if (!this.initialized) {
1085       LOG.debug("Master has not been initialized, don't run balancer.");
1086       return false;
1087     }
1088     // Do this call outside of synchronized block.
1089     int maximumBalanceTime = getBalancerCutoffTime();
1090     synchronized (this.balancer) {
1091       // If balance not true, don't run balancer.
1092       if (!this.loadBalancerTracker.isBalancerOn()) return false;
1093       // Only allow one balance run at at time.
1094       if (this.assignmentManager.getRegionStates().isRegionsInTransition()) {
1095         Map<String, RegionState> regionsInTransition =
1096           this.assignmentManager.getRegionStates().getRegionsInTransition();
1097         LOG.debug("Not running balancer because " + regionsInTransition.size() +
1098           " region(s) in transition: " + org.apache.commons.lang.StringUtils.
1099             abbreviate(regionsInTransition.toString(), 256));
1100         return false;
1101       }
1102       if (this.serverManager.areDeadServersInProgress()) {
1103         LOG.debug("Not running balancer because processing dead regionserver(s): " +
1104           this.serverManager.getDeadServers());
1105         return false;
1106       }
1107 
1108       if (this.cpHost != null) {
1109         try {
1110           if (this.cpHost.preBalance()) {
1111             LOG.debug("Coprocessor bypassing balancer request");
1112             return false;
1113           }
1114         } catch (IOException ioe) {
1115           LOG.error("Error invoking master coprocessor preBalance()", ioe);
1116           return false;
1117         }
1118       }
1119 
1120       Map<TableName, Map<ServerName, List<HRegionInfo>>> assignmentsByTable =
1121         this.assignmentManager.getRegionStates().getAssignmentsByTable();
1122 
1123       List<RegionPlan> plans = new ArrayList<RegionPlan>();
1124       //Give the balancer the current cluster state.
1125       this.balancer.setClusterStatus(getClusterStatus());
1126       for (Map<ServerName, List<HRegionInfo>> assignments : assignmentsByTable.values()) {
1127         List<RegionPlan> partialPlans = this.balancer.balanceCluster(assignments);
1128         if (partialPlans != null) plans.addAll(partialPlans);
1129       }
1130       long cutoffTime = System.currentTimeMillis() + maximumBalanceTime;
1131       int rpCount = 0;  // number of RegionPlans balanced so far
1132       long totalRegPlanExecTime = 0;
1133       if (plans != null && !plans.isEmpty()) {
1134         for (RegionPlan plan: plans) {
1135           LOG.info("balance " + plan);
1136           long balStartTime = System.currentTimeMillis();
1137           //TODO: bulk assign
1138           this.assignmentManager.balance(plan);
1139           totalRegPlanExecTime += System.currentTimeMillis()-balStartTime;
1140           rpCount++;
1141           if (rpCount < plans.size() &&
1142               // if performing next balance exceeds cutoff time, exit the loop
1143               (System.currentTimeMillis() + (totalRegPlanExecTime / rpCount)) > cutoffTime) {
1144             //TODO: After balance, there should not be a cutoff time (keeping it as a security net for now)
1145             LOG.debug("No more balancing till next balance run; maximumBalanceTime=" +
1146               maximumBalanceTime);
1147             break;
1148           }
1149         }
1150       }
1151       if (this.cpHost != null) {
1152         try {
1153           this.cpHost.postBalance(rpCount < plans.size() ? plans.subList(0, rpCount) : plans);
1154         } catch (IOException ioe) {
1155           // balancing already succeeded so don't change the result
1156           LOG.error("Error invoking master coprocessor postBalance()", ioe);
1157         }
1158       }
1159     }
1160     // If LoadBalancer did not generate any plans, it means the cluster is already balanced.
1161     // Return true indicating a success.
1162     return true;
1163   }
1164 
1165   /**
1166    * @return Client info for use as prefix on an audit log string; who did an action
1167    */
1168   String getClientIdAuditPrefix() {
1169     return "Client=" + RequestContext.getRequestUserName() + "/" +
1170       RequestContext.get().getRemoteAddress();
1171   }
1172 
1173   /**
1174    * Switch for the background CatalogJanitor thread.
1175    * Used for testing.  The thread will continue to run.  It will just be a noop
1176    * if disabled.
1177    * @param b If false, the catalog janitor won't do anything.
1178    */
1179   public void setCatalogJanitorEnabled(final boolean b) {
1180     this.catalogJanitorChore.setEnabled(b);
1181   }
1182 
1183   @Override
1184   public void dispatchMergingRegions(final HRegionInfo region_a,
1185       final HRegionInfo region_b, final boolean forcible) throws IOException {
1186     checkInitialized();
1187     this.service.submit(new DispatchMergingRegionHandler(this,
1188         this.catalogJanitorChore, region_a, region_b, forcible));
1189   }
1190 
1191   void move(final byte[] encodedRegionName,
1192       final byte[] destServerName) throws HBaseIOException {
1193     RegionState regionState = assignmentManager.getRegionStates().
1194       getRegionState(Bytes.toString(encodedRegionName));
1195     if (regionState == null) {
1196       throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));
1197     }
1198 
1199     HRegionInfo hri = regionState.getRegion();
1200     ServerName dest;
1201     if (destServerName == null || destServerName.length == 0) {
1202       LOG.info("Passed destination servername is null/empty so " +
1203         "choosing a server at random");
1204       final List<ServerName> destServers = this.serverManager.createDestinationServersList(
1205         regionState.getServerName());
1206       dest = balancer.randomAssignment(hri, destServers);
1207       if (dest == null) {
1208         LOG.debug("Unable to determine a plan to assign " + hri);
1209         return;
1210       }
1211     } else {
1212       dest = ServerName.valueOf(Bytes.toString(destServerName));
1213       if (dest.equals(serverName) && balancer instanceof BaseLoadBalancer
1214           && !((BaseLoadBalancer)balancer).shouldBeOnMaster(hri)) {
1215         // To avoid unnecessary region moving later by balancer. Don't put user
1216         // regions on master. Regions on master could be put on other region
1217         // server intentionally by test however.
1218         LOG.debug("Skipping move of region " + hri.getRegionNameAsString()
1219           + " to avoid unnecessary region moving later by load balancer,"
1220           + " because it should not be on master");
1221         return;
1222       }
1223     }
1224 
1225     if (dest.equals(regionState.getServerName())) {
1226       LOG.debug("Skipping move of region " + hri.getRegionNameAsString()
1227         + " because region already assigned to the same server " + dest + ".");
1228       return;
1229     }
1230 
1231     // Now we can do the move
1232     RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), dest);
1233 
1234     try {
1235       checkInitialized();
1236       if (this.cpHost != null) {
1237         if (this.cpHost.preMove(hri, rp.getSource(), rp.getDestination())) {
1238           return;
1239         }
1240       }
1241       LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");
1242       this.assignmentManager.balance(rp);
1243       if (this.cpHost != null) {
1244         this.cpHost.postMove(hri, rp.getSource(), rp.getDestination());
1245       }
1246     } catch (IOException ioe) {
1247       if (ioe instanceof HBaseIOException) {
1248         throw (HBaseIOException)ioe;
1249       }
1250       throw new HBaseIOException(ioe);
1251     }
1252   }
1253 
1254   @Override
1255   public void createTable(HTableDescriptor hTableDescriptor,
1256       byte [][] splitKeys) throws IOException {
1257     if (isStopped()) {
1258       throw new MasterNotRunningException();
1259     }
1260 
1261     String namespace = hTableDescriptor.getTableName().getNamespaceAsString();
1262     ensureNamespaceExists(namespace);
1263 
1264     HRegionInfo[] newRegions = getHRegionInfos(hTableDescriptor, splitKeys);
1265     checkInitialized();
1266     sanityCheckTableDescriptor(hTableDescriptor);
1267     if (cpHost != null) {
1268       cpHost.preCreateTable(hTableDescriptor, newRegions);
1269     }
1270     LOG.info(getClientIdAuditPrefix() + " create " + hTableDescriptor);
1271     this.service.submit(new CreateTableHandler(this,
1272       this.fileSystemManager, hTableDescriptor, conf,
1273       newRegions, this).prepare());
1274     if (cpHost != null) {
1275       cpHost.postCreateTable(hTableDescriptor, newRegions);
1276     }
1277 
1278   }
1279 
1280   /**
1281    * Checks whether the table conforms to some sane limits, and configured
1282    * values (compression, etc) work. Throws an exception if something is wrong.
1283    * @throws IOException
1284    */
1285   private void sanityCheckTableDescriptor(final HTableDescriptor htd) throws IOException {
1286     final String CONF_KEY = "hbase.table.sanity.checks";
1287     if (!conf.getBoolean(CONF_KEY, true)) {
1288       return;
1289     }
1290     String tableVal = htd.getConfigurationValue(CONF_KEY);
1291     if (tableVal != null && !Boolean.valueOf(tableVal)) {
1292       return;
1293     }
1294 
1295     // check max file size
1296     long maxFileSizeLowerLimit = 2 * 1024 * 1024L; // 2M is the default lower limit
1297     long maxFileSize = htd.getMaxFileSize();
1298     if (maxFileSize < 0) {
1299       maxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, maxFileSizeLowerLimit);
1300     }
1301     if (maxFileSize < conf.getLong("hbase.hregion.max.filesize.limit", maxFileSizeLowerLimit)) {
1302       throw new DoNotRetryIOException("MAX_FILESIZE for table descriptor or "
1303         + "\"hbase.hregion.max.filesize\" (" + maxFileSize
1304         + ") is too small, which might cause over splitting into unmanageable "
1305         + "number of regions. Set " + CONF_KEY + " to false at conf or table descriptor "
1306           + "if you want to bypass sanity checks");
1307     }
1308 
1309     // check flush size
1310     long flushSizeLowerLimit = 1024 * 1024L; // 1M is the default lower limit
1311     long flushSize = htd.getMemStoreFlushSize();
1312     if (flushSize < 0) {
1313       flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSizeLowerLimit);
1314     }
1315     if (flushSize < conf.getLong("hbase.hregion.memstore.flush.size.limit", flushSizeLowerLimit)) {
1316       throw new DoNotRetryIOException("MEMSTORE_FLUSHSIZE for table descriptor or "
1317           + "\"hbase.hregion.memstore.flush.size\" ("+flushSize+") is too small, which might cause"
1318           + " very frequent flushing. Set " + CONF_KEY + " to false at conf or table descriptor "
1319           + "if you want to bypass sanity checks");
1320     }
1321 
1322     // check that coprocessors and other specified plugin classes can be loaded
1323     try {
1324       checkClassLoading(conf, htd);
1325     } catch (Exception ex) {
1326       throw new DoNotRetryIOException(ex);
1327     }
1328 
1329     // check compression can be loaded
1330     try {
1331       checkCompression(htd);
1332     } catch (IOException e) {
1333       throw new DoNotRetryIOException(e.getMessage(), e);
1334     }
1335 
1336     // check encryption can be loaded
1337     try {
1338       checkEncryption(conf, htd);
1339     } catch (IOException e) {
1340       throw new DoNotRetryIOException(e.getMessage(), e);
1341     }
1342 
1343     // check that we have at least 1 CF
1344     if (htd.getColumnFamilies().length == 0) {
1345       throw new DoNotRetryIOException("Table should have at least one column family "
1346           + "Set "+CONF_KEY+" at conf or table descriptor if you want to bypass sanity checks");
1347     }
1348 
1349     for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
1350       if (hcd.getTimeToLive() <= 0) {
1351         throw new DoNotRetryIOException("TTL for column family " + hcd.getNameAsString()
1352           + "  must be positive. Set " + CONF_KEY + " to false at conf or table descriptor "
1353           + "if you want to bypass sanity checks");
1354       }
1355 
1356       // check blockSize
1357       if (hcd.getBlocksize() < 1024 || hcd.getBlocksize() > 16 * 1024 * 1024) {
1358         throw new DoNotRetryIOException("Block size for column family " + hcd.getNameAsString()
1359           + "  must be between 1K and 16MB Set "+CONF_KEY+" to false at conf or table descriptor "
1360           + "if you want to bypass sanity checks");
1361       }
1362 
1363       // check versions
1364       if (hcd.getMinVersions() < 0) {
1365         throw new DoNotRetryIOException("Min versions for column family " + hcd.getNameAsString()
1366           + "  must be positive. Set " + CONF_KEY + " to false at conf or table descriptor "
1367           + "if you want to bypass sanity checks");
1368       }
1369       // max versions already being checked
1370 
1371       // check replication scope
1372       if (hcd.getScope() < 0) {
1373         throw new DoNotRetryIOException("Replication scope for column family "
1374           + hcd.getNameAsString() + "  must be positive. Set " + CONF_KEY + " to false at conf "
1375           + "or table descriptor if you want to bypass sanity checks");
1376       }
1377 
1378       // TODO: should we check coprocessors and encryption ?
1379     }
1380   }
1381 
1382   private void startActiveMasterManager(int infoPort) throws KeeperException {
1383     String backupZNode = ZKUtil.joinZNode(
1384       zooKeeper.backupMasterAddressesZNode, serverName.toString());
1385     /*
1386     * Add a ZNode for ourselves in the backup master directory since we
1387     * may not become the active master. If so, we want the actual active
1388     * master to know we are backup masters, so that it won't assign
1389     * regions to us if so configured.
1390     *
1391     * If we become the active master later, ActiveMasterManager will delete
1392     * this node explicitly.  If we crash before then, ZooKeeper will delete
1393     * this node for us since it is ephemeral.
1394     */
1395     LOG.info("Adding backup master ZNode " + backupZNode);
1396     if (!MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode,
1397         serverName, infoPort)) {
1398       LOG.warn("Failed create of " + backupZNode + " by " + serverName);
1399     }
1400 
1401     activeMasterManager.setInfoPort(infoPort);
1402     // Start a thread to try to become the active master, so we won't block here
1403     Threads.setDaemonThreadRunning(new Thread(new Runnable() {
1404       @Override
1405       public void run() {
1406         int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
1407           HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
1408         // If we're a backup master, stall until a primary to writes his address
1409         if (conf.getBoolean(HConstants.MASTER_TYPE_BACKUP,
1410             HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {
1411           LOG.debug("HMaster started in backup mode. "
1412             + "Stalling until master znode is written.");
1413           // This will only be a minute or so while the cluster starts up,
1414           // so don't worry about setting watches on the parent znode
1415           while (!activeMasterManager.hasActiveMaster()) {
1416             LOG.debug("Waiting for master address ZNode to be written "
1417               + "(Also watching cluster state node)");
1418             Threads.sleep(timeout);
1419           }
1420         }
1421         MonitoredTask status = TaskMonitor.get().createStatus("Master startup");
1422         status.setDescription("Master startup");
1423         try {
1424           if (activeMasterManager.blockUntilBecomingActiveMaster(timeout, status)) {
1425             finishActiveMasterInitialization(status);
1426           }
1427         } catch (Throwable t) {
1428           status.setStatus("Failed to become active: " + t.getMessage());
1429           LOG.fatal("Failed to become active master", t);
1430           // HBASE-5680: Likely hadoop23 vs hadoop 20.x/1.x incompatibility
1431           if (t instanceof NoClassDefFoundError &&
1432               t.getMessage().contains("org/apache/hadoop/hdfs/protocol/FSConstants$SafeModeAction")) {
1433             // improved error message for this special case
1434             abort("HBase is having a problem with its Hadoop jars.  You may need to "
1435               + "recompile HBase against Hadoop version "
1436               +  org.apache.hadoop.util.VersionInfo.getVersion()
1437               + " or change your hadoop jars to start properly", t);
1438           } else {
1439             abort("Unhandled exception. Starting shutdown.", t);
1440           }
1441         } finally {
1442           status.cleanup();
1443         }
1444       }
1445     }, getServerName().toShortString() + ".activeMasterManager"));
1446   }
1447 
1448   private void checkCompression(final HTableDescriptor htd)
1449   throws IOException {
1450     if (!this.masterCheckCompression) return;
1451     for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
1452       checkCompression(hcd);
1453     }
1454   }
1455 
1456   private void checkCompression(final HColumnDescriptor hcd)
1457   throws IOException {
1458     if (!this.masterCheckCompression) return;
1459     CompressionTest.testCompression(hcd.getCompression());
1460     CompressionTest.testCompression(hcd.getCompactionCompression());
1461   }
1462 
1463   private void checkEncryption(final Configuration conf, final HTableDescriptor htd)
1464   throws IOException {
1465     if (!this.masterCheckEncryption) return;
1466     for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
1467       checkEncryption(conf, hcd);
1468     }
1469   }
1470 
1471   private void checkEncryption(final Configuration conf, final HColumnDescriptor hcd)
1472   throws IOException {
1473     if (!this.masterCheckEncryption) return;
1474     EncryptionTest.testEncryption(conf, hcd.getEncryptionType(), hcd.getEncryptionKey());
1475   }
1476 
1477   private void checkClassLoading(final Configuration conf, final HTableDescriptor htd)
1478   throws IOException {
1479     RegionSplitPolicy.getSplitPolicyClass(htd, conf);
1480     RegionCoprocessorHost.testTableCoprocessorAttrs(conf, htd);
1481   }
1482 
1483   private HRegionInfo[] getHRegionInfos(HTableDescriptor hTableDescriptor,
1484     byte[][] splitKeys) {
1485     long regionId = System.currentTimeMillis();
1486     HRegionInfo[] hRegionInfos = null;
1487     if (splitKeys == null || splitKeys.length == 0) {
1488       hRegionInfos = new HRegionInfo[]{new HRegionInfo(hTableDescriptor.getTableName(), null, null,
1489                 false, regionId)};
1490     } else {
1491       int numRegions = splitKeys.length + 1;
1492       hRegionInfos = new HRegionInfo[numRegions];
1493       byte[] startKey = null;
1494       byte[] endKey = null;
1495       for (int i = 0; i < numRegions; i++) {
1496         endKey = (i == splitKeys.length) ? null : splitKeys[i];
1497         hRegionInfos[i] =
1498              new HRegionInfo(hTableDescriptor.getTableName(), startKey, endKey,
1499                  false, regionId);
1500         startKey = endKey;
1501       }
1502     }
1503     return hRegionInfos;
1504   }
1505 
1506   private static boolean isCatalogTable(final TableName tableName) {
1507     return tableName.equals(TableName.META_TABLE_NAME);
1508   }
1509 
1510   @Override
1511   public void deleteTable(final TableName tableName) throws IOException {
1512     checkInitialized();
1513     if (cpHost != null) {
1514       cpHost.preDeleteTable(tableName);
1515     }
1516     LOG.info(getClientIdAuditPrefix() + " delete " + tableName);
1517     this.service.submit(new DeleteTableHandler(tableName, this, this).prepare());
1518     if (cpHost != null) {
1519       cpHost.postDeleteTable(tableName);
1520     }
1521   }
1522 
1523   @Override
1524   public void truncateTable(TableName tableName, boolean preserveSplits) throws IOException {
1525     checkInitialized();
1526     if (cpHost != null) {
1527       cpHost.preTruncateTable(tableName);
1528     }
1529     LOG.info(getClientIdAuditPrefix() + " truncate " + tableName);
1530     TruncateTableHandler handler = new TruncateTableHandler(tableName, this, this, preserveSplits);
1531     handler.prepare();
1532     handler.process();
1533     if (cpHost != null) {
1534       cpHost.postTruncateTable(tableName);
1535     }
1536   }
1537 
1538   @Override
1539   public void addColumn(final TableName tableName, final HColumnDescriptor columnDescriptor)
1540       throws IOException {
1541     checkInitialized();
1542     checkCompression(columnDescriptor);
1543     checkEncryption(conf, columnDescriptor);
1544     if (cpHost != null) {
1545       if (cpHost.preAddColumn(tableName, columnDescriptor)) {
1546         return;
1547       }
1548     }
1549     //TODO: we should process this (and some others) in an executor
1550     new TableAddFamilyHandler(tableName, columnDescriptor, this, this).prepare().process();
1551     if (cpHost != null) {
1552       cpHost.postAddColumn(tableName, columnDescriptor);
1553     }
1554   }
1555 
1556   @Override
1557   public void modifyColumn(TableName tableName, HColumnDescriptor descriptor)
1558       throws IOException {
1559     checkInitialized();
1560     checkCompression(descriptor);
1561     checkEncryption(conf, descriptor);
1562     if (cpHost != null) {
1563       if (cpHost.preModifyColumn(tableName, descriptor)) {
1564         return;
1565       }
1566     }
1567     LOG.info(getClientIdAuditPrefix() + " modify " + descriptor);
1568     new TableModifyFamilyHandler(tableName, descriptor, this, this)
1569       .prepare().process();
1570     if (cpHost != null) {
1571       cpHost.postModifyColumn(tableName, descriptor);
1572     }
1573   }
1574 
1575   @Override
1576   public void deleteColumn(final TableName tableName, final byte[] columnName)
1577       throws IOException {
1578     checkInitialized();
1579     if (cpHost != null) {
1580       if (cpHost.preDeleteColumn(tableName, columnName)) {
1581         return;
1582       }
1583     }
1584     LOG.info(getClientIdAuditPrefix() + " delete " + Bytes.toString(columnName));
1585     new TableDeleteFamilyHandler(tableName, columnName, this, this).prepare().process();
1586     if (cpHost != null) {
1587       cpHost.postDeleteColumn(tableName, columnName);
1588     }
1589   }
1590 
1591   @Override
1592   public void enableTable(final TableName tableName) throws IOException {
1593     checkInitialized();
1594     if (cpHost != null) {
1595       cpHost.preEnableTable(tableName);
1596     }
1597     LOG.info(getClientIdAuditPrefix() + " enable " + tableName);
1598     this.service.submit(new EnableTableHandler(this, tableName,
1599       assignmentManager, tableLockManager, false).prepare());
1600     if (cpHost != null) {
1601       cpHost.postEnableTable(tableName);
1602    }
1603   }
1604 
1605   @Override
1606   public void disableTable(final TableName tableName) throws IOException {
1607     checkInitialized();
1608     if (cpHost != null) {
1609       cpHost.preDisableTable(tableName);
1610     }
1611     LOG.info(getClientIdAuditPrefix() + " disable " + tableName);
1612     this.service.submit(new DisableTableHandler(this, tableName,
1613       assignmentManager, tableLockManager, false).prepare());
1614     if (cpHost != null) {
1615       cpHost.postDisableTable(tableName);
1616     }
1617   }
1618 
1619   /**
1620    * Return the region and current deployment for the region containing
1621    * the given row. If the region cannot be found, returns null. If it
1622    * is found, but not currently deployed, the second element of the pair
1623    * may be null.
1624    */
1625   @VisibleForTesting // Used by TestMaster.
1626   Pair<HRegionInfo, ServerName> getTableRegionForRow(
1627       final TableName tableName, final byte [] rowKey)
1628   throws IOException {
1629     final AtomicReference<Pair<HRegionInfo, ServerName>> result =
1630       new AtomicReference<Pair<HRegionInfo, ServerName>>(null);
1631 
1632     MetaScannerVisitor visitor =
1633       new MetaScannerVisitorBase() {
1634         @Override
1635         public boolean processRow(Result data) throws IOException {
1636           if (data == null || data.size() <= 0) {
1637             return true;
1638           }
1639           Pair<HRegionInfo, ServerName> pair = HRegionInfo.getHRegionInfoAndServerName(data);
1640           if (pair == null) {
1641             return false;
1642           }
1643           if (!pair.getFirst().getTable().equals(tableName)) {
1644             return false;
1645           }
1646           result.set(pair);
1647           return true;
1648         }
1649     };
1650 
1651     MetaScanner.metaScan(clusterConnection, visitor, tableName, rowKey, 1);
1652     return result.get();
1653   }
1654 
1655   @Override
1656   public void modifyTable(final TableName tableName, final HTableDescriptor descriptor)
1657       throws IOException {
1658     checkInitialized();
1659     sanityCheckTableDescriptor(descriptor);
1660     if (cpHost != null) {
1661       cpHost.preModifyTable(tableName, descriptor);
1662     }
1663     LOG.info(getClientIdAuditPrefix() + " modify " + tableName);
1664     new ModifyTableHandler(tableName, descriptor, this, this).prepare().process();
1665     if (cpHost != null) {
1666       cpHost.postModifyTable(tableName, descriptor);
1667     }
1668   }
1669 
1670   @Override
1671   public void checkTableModifiable(final TableName tableName)
1672       throws IOException, TableNotFoundException, TableNotDisabledException {
1673     if (isCatalogTable(tableName)) {
1674       throw new IOException("Can't modify catalog tables");
1675     }
1676     if (!MetaTableAccessor.tableExists(getConnection(), tableName)) {
1677       throw new TableNotFoundException(tableName);
1678     }
1679     if (!getAssignmentManager().getTableStateManager().
1680         isTableState(tableName, ZooKeeperProtos.Table.State.DISABLED)) {
1681       throw new TableNotDisabledException(tableName);
1682     }
1683   }
1684 
1685   /**
1686    * @return cluster status
1687    */
1688   public ClusterStatus getClusterStatus() throws InterruptedIOException {
1689     // Build Set of backup masters from ZK nodes
1690     List<String> backupMasterStrings;
1691     try {
1692       backupMasterStrings = ZKUtil.listChildrenNoWatch(this.zooKeeper,
1693         this.zooKeeper.backupMasterAddressesZNode);
1694     } catch (KeeperException e) {
1695       LOG.warn(this.zooKeeper.prefix("Unable to list backup servers"), e);
1696       backupMasterStrings = new ArrayList<String>(0);
1697     }
1698     List<ServerName> backupMasters = new ArrayList<ServerName>(
1699                                           backupMasterStrings.size());
1700     for (String s: backupMasterStrings) {
1701       try {
1702         byte [] bytes;
1703         try {
1704           bytes = ZKUtil.getData(this.zooKeeper, ZKUtil.joinZNode(
1705               this.zooKeeper.backupMasterAddressesZNode, s));
1706         } catch (InterruptedException e) {
1707           throw new InterruptedIOException();
1708         }
1709         if (bytes != null) {
1710           ServerName sn;
1711           try {
1712             sn = ServerName.parseFrom(bytes);
1713           } catch (DeserializationException e) {
1714             LOG.warn("Failed parse, skipping registering backup server", e);
1715             continue;
1716           }
1717           backupMasters.add(sn);
1718         }
1719       } catch (KeeperException e) {
1720         LOG.warn(this.zooKeeper.prefix("Unable to get information about " +
1721                  "backup servers"), e);
1722       }
1723     }
1724     Collections.sort(backupMasters, new Comparator<ServerName>() {
1725       @Override
1726       public int compare(ServerName s1, ServerName s2) {
1727         return s1.getServerName().compareTo(s2.getServerName());
1728       }});
1729 
1730     String clusterId = fileSystemManager != null ?
1731       fileSystemManager.getClusterId().toString() : null;
1732     Map<String, RegionState> regionsInTransition = assignmentManager != null ?
1733       assignmentManager.getRegionStates().getRegionsInTransition() : null;
1734     String[] coprocessors = cpHost != null ? getMasterCoprocessors() : null;
1735     boolean balancerOn = loadBalancerTracker != null ?
1736       loadBalancerTracker.isBalancerOn() : false;
1737     Map<ServerName, ServerLoad> onlineServers = null;
1738     Set<ServerName> deadServers = null;
1739     if (serverManager != null) {
1740       deadServers = serverManager.getDeadServers().copyServerNames();
1741       onlineServers = serverManager.getOnlineServers();
1742     }
1743     return new ClusterStatus(VersionInfo.getVersion(), clusterId,
1744       onlineServers, deadServers, serverName, backupMasters,
1745       regionsInTransition, coprocessors, balancerOn);
1746   }
1747 
1748   /**
1749    * The set of loaded coprocessors is stored in a static set. Since it's
1750    * statically allocated, it does not require that HMaster's cpHost be
1751    * initialized prior to accessing it.
1752    * @return a String representation of the set of names of the loaded
1753    * coprocessors.
1754    */
1755   public static String getLoadedCoprocessors() {
1756     return CoprocessorHost.getLoadedCoprocessors().toString();
1757   }
1758 
1759   /**
1760    * @return timestamp in millis when HMaster was started.
1761    */
1762   public long getMasterStartTime() {
1763     return startcode;
1764   }
1765 
1766   /**
1767    * @return timestamp in millis when HMaster became the active master.
1768    */
1769   public long getMasterActiveTime() {
1770     return masterActiveTime;
1771   }
1772 
1773   public int getRegionServerInfoPort(final ServerName sn) {
1774     RegionServerInfo info = this.regionServerTracker.getRegionServerInfo(sn);
1775     if (info == null || info.getInfoPort() == 0) {
1776       return conf.getInt(HConstants.REGIONSERVER_INFO_PORT,
1777         HConstants.DEFAULT_REGIONSERVER_INFOPORT);
1778     }
1779     return info.getInfoPort();
1780   }
1781 
1782   /**
1783    * @return array of coprocessor SimpleNames.
1784    */
1785   public String[] getMasterCoprocessors() {
1786     Set<String> masterCoprocessors = getMasterCoprocessorHost().getCoprocessors();
1787     return masterCoprocessors.toArray(new String[masterCoprocessors.size()]);
1788   }
1789 
1790   @Override
1791   public void abort(final String msg, final Throwable t) {
1792     if (isAborted() || isStopped()) {
1793       return;
1794     }
1795     if (cpHost != null) {
1796       // HBASE-4014: dump a list of loaded coprocessors.
1797       LOG.fatal("Master server abort: loaded coprocessors are: " +
1798           getLoadedCoprocessors());
1799     }
1800     if (t != null) LOG.fatal(msg, t);
1801     stop(msg);
1802   }
1803 
1804   @Override
1805   public ZooKeeperWatcher getZooKeeper() {
1806     return zooKeeper;
1807   }
1808 
1809   @Override
1810   public MasterCoprocessorHost getMasterCoprocessorHost() {
1811     return cpHost;
1812   }
1813 
1814   @Override
1815   public ServerName getServerName() {
1816     return this.serverName;
1817   }
1818 
1819   @Override
1820   public AssignmentManager getAssignmentManager() {
1821     return this.assignmentManager;
1822   }
1823 
1824   public MemoryBoundedLogMessageBuffer getRegionServerFatalLogBuffer() {
1825     return rsFatals;
1826   }
1827 
1828   public void shutdown() {
1829     if (cpHost != null) {
1830       try {
1831         cpHost.preShutdown();
1832       } catch (IOException ioe) {
1833         LOG.error("Error call master coprocessor preShutdown()", ioe);
1834       }
1835     }
1836 
1837     if (this.serverManager != null) {
1838       this.serverManager.shutdownCluster();
1839     }
1840     if (this.clusterStatusTracker != null){
1841       try {
1842         this.clusterStatusTracker.setClusterDown();
1843       } catch (KeeperException e) {
1844         LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
1845       }
1846     }
1847   }
1848 
1849   public void stopMaster() {
1850     if (cpHost != null) {
1851       try {
1852         cpHost.preStopMaster();
1853       } catch (IOException ioe) {
1854         LOG.error("Error call master coprocessor preStopMaster()", ioe);
1855       }
1856     }
1857     stop("Stopped by " + Thread.currentThread().getName());
1858   }
1859 
1860   void checkServiceStarted() throws ServerNotRunningYetException {
1861     if (!serviceStarted) {
1862       throw new ServerNotRunningYetException("Server is not running yet");
1863     }
1864   }
1865 
1866   void checkInitialized() throws PleaseHoldException, ServerNotRunningYetException {
1867     checkServiceStarted();
1868     if (!this.initialized) {
1869       throw new PleaseHoldException("Master is initializing");
1870     }
1871   }
1872 
1873   void checkNamespaceManagerReady() throws IOException {
1874     checkInitialized();
1875     if (tableNamespaceManager == null ||
1876         !tableNamespaceManager.isTableAvailableAndInitialized()) {
1877       throw new IOException("Table Namespace Manager not ready yet, try again later");
1878     }
1879   }
1880   /**
1881    * Report whether this master is currently the active master or not.
1882    * If not active master, we are parked on ZK waiting to become active.
1883    *
1884    * This method is used for testing.
1885    *
1886    * @return true if active master, false if not.
1887    */
1888   public boolean isActiveMaster() {
1889     return isActiveMaster;
1890   }
1891 
1892   /**
1893    * Report whether this master has completed with its initialization and is
1894    * ready.  If ready, the master is also the active master.  A standby master
1895    * is never ready.
1896    *
1897    * This method is used for testing.
1898    *
1899    * @return true if master is ready to go, false if not.
1900    */
1901   @Override
1902   public boolean isInitialized() {
1903     return initialized;
1904   }
1905 
1906   /**
1907    * ServerShutdownHandlerEnabled is set false before completing
1908    * assignMeta to prevent processing of ServerShutdownHandler.
1909    * @return true if assignMeta has completed;
1910    */
1911   @Override
1912   public boolean isServerShutdownHandlerEnabled() {
1913     return this.serverShutdownHandlerEnabled;
1914   }
1915 
1916   /**
1917    * Report whether this master has started initialization and is about to do meta region assignment
1918    * @return true if master is in initialization & about to assign hbase:meta regions
1919    */
1920   public boolean isInitializationStartsMetaRegionAssignment() {
1921     return this.initializationBeforeMetaAssignment;
1922   }
1923 
1924   public void assignRegion(HRegionInfo hri) {
1925     assignmentManager.assign(hri, true);
1926   }
1927 
1928   /**
1929    * Compute the average load across all region servers.
1930    * Currently, this uses a very naive computation - just uses the number of
1931    * regions being served, ignoring stats about number of requests.
1932    * @return the average load
1933    */
1934   public double getAverageLoad() {
1935     if (this.assignmentManager == null) {
1936       return 0;
1937     }
1938 
1939     RegionStates regionStates = this.assignmentManager.getRegionStates();
1940     if (regionStates == null) {
1941       return 0;
1942     }
1943     return regionStates.getAverageLoad();
1944   }
1945 
1946   @Override
1947   public boolean registerService(Service instance) {
1948     /*
1949      * No stacking of instances is allowed for a single service name
1950      */
1951     Descriptors.ServiceDescriptor serviceDesc = instance.getDescriptorForType();
1952     if (coprocessorServiceHandlers.containsKey(serviceDesc.getFullName())) {
1953       LOG.error("Coprocessor service "+serviceDesc.getFullName()+
1954           " already registered, rejecting request from "+instance
1955       );
1956       return false;
1957     }
1958 
1959     coprocessorServiceHandlers.put(serviceDesc.getFullName(), instance);
1960     if (LOG.isDebugEnabled()) {
1961       LOG.debug("Registered master coprocessor service: service="+serviceDesc.getFullName());
1962     }
1963     return true;
1964   }
1965 
1966   /**
1967    * Utility for constructing an instance of the passed HMaster class.
1968    * @param masterClass
1969    * @param conf
1970    * @return HMaster instance.
1971    */
1972   public static HMaster constructMaster(Class<? extends HMaster> masterClass,
1973       final Configuration conf, final CoordinatedStateManager cp)  {
1974     try {
1975       Constructor<? extends HMaster> c =
1976         masterClass.getConstructor(Configuration.class, CoordinatedStateManager.class);
1977       return c.newInstance(conf, cp);
1978     } catch (InvocationTargetException ite) {
1979       Throwable target = ite.getTargetException() != null?
1980         ite.getTargetException(): ite;
1981       if (target.getCause() != null) target = target.getCause();
1982       throw new RuntimeException("Failed construction of Master: " +
1983         masterClass.toString(), target);
1984     } catch (Exception e) {
1985       throw new RuntimeException("Failed construction of Master: " +
1986         masterClass.toString() + ((e.getCause() != null)?
1987           e.getCause().getMessage(): ""), e);
1988     }
1989   }
1990 
1991   /**
1992    * @see org.apache.hadoop.hbase.master.HMasterCommandLine
1993    */
1994   public static void main(String [] args) {
1995     VersionInfo.logVersion();
1996     new HMasterCommandLine(HMaster.class).doMain(args);
1997   }
1998 
1999   public HFileCleaner getHFileCleaner() {
2000     return this.hfileCleaner;
2001   }
2002 
2003   /**
2004    * Exposed for TESTING!
2005    * @return the underlying snapshot manager
2006    */
2007   public SnapshotManager getSnapshotManagerForTesting() {
2008     return this.snapshotManager;
2009   }
2010 
2011   @Override
2012   public void createNamespace(NamespaceDescriptor descriptor) throws IOException {
2013     TableName.isLegalNamespaceName(Bytes.toBytes(descriptor.getName()));
2014     checkNamespaceManagerReady();
2015     if (cpHost != null) {
2016       if (cpHost.preCreateNamespace(descriptor)) {
2017         return;
2018       }
2019     }
2020     LOG.info(getClientIdAuditPrefix() + " creating " + descriptor);
2021     tableNamespaceManager.create(descriptor);
2022     if (cpHost != null) {
2023       cpHost.postCreateNamespace(descriptor);
2024     }
2025   }
2026 
2027   @Override
2028   public void modifyNamespace(NamespaceDescriptor descriptor) throws IOException {
2029     TableName.isLegalNamespaceName(Bytes.toBytes(descriptor.getName()));
2030     checkNamespaceManagerReady();
2031     if (cpHost != null) {
2032       if (cpHost.preModifyNamespace(descriptor)) {
2033         return;
2034       }
2035     }
2036     LOG.info(getClientIdAuditPrefix() + " modify " + descriptor);
2037     tableNamespaceManager.update(descriptor);
2038     if (cpHost != null) {
2039       cpHost.postModifyNamespace(descriptor);
2040     }
2041   }
2042 
2043   @Override
2044   public void deleteNamespace(String name) throws IOException {
2045     checkNamespaceManagerReady();
2046     if (cpHost != null) {
2047       if (cpHost.preDeleteNamespace(name)) {
2048         return;
2049       }
2050     }
2051     LOG.info(getClientIdAuditPrefix() + " delete " + name);
2052     tableNamespaceManager.remove(name);
2053     if (cpHost != null) {
2054       cpHost.postDeleteNamespace(name);
2055     }
2056   }
2057 
2058   /**
2059    * Ensure that the specified namespace exists, otherwise throws a NamespaceNotFoundException
2060    *
2061    * @param name the namespace to check
2062    * @throws IOException if the namespace manager is not ready yet.
2063    * @throws NamespaceNotFoundException if the namespace does not exists
2064    */
2065   private void ensureNamespaceExists(final String name)
2066       throws IOException, NamespaceNotFoundException {
2067     checkNamespaceManagerReady();
2068     NamespaceDescriptor nsd = tableNamespaceManager.get(name);
2069     if (nsd == null) {
2070       throw new NamespaceNotFoundException(name);
2071     }
2072   }
2073 
2074   @Override
2075   public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException {
2076     checkNamespaceManagerReady();
2077 
2078     if (cpHost != null) {
2079       cpHost.preGetNamespaceDescriptor(name);
2080     }
2081 
2082     NamespaceDescriptor nsd = tableNamespaceManager.get(name);
2083     if (nsd == null) {
2084       throw new NamespaceNotFoundException(name);
2085     }
2086 
2087     if (cpHost != null) {
2088       cpHost.postGetNamespaceDescriptor(nsd);
2089     }
2090 
2091     return nsd;
2092   }
2093 
2094   @Override
2095   public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException {
2096     checkNamespaceManagerReady();
2097 
2098     final List<NamespaceDescriptor> descriptors = new ArrayList<NamespaceDescriptor>();
2099     boolean bypass = false;
2100     if (cpHost != null) {
2101       bypass = cpHost.preListNamespaceDescriptors(descriptors);
2102     }
2103 
2104     if (!bypass) {
2105       descriptors.addAll(tableNamespaceManager.list());
2106 
2107       if (cpHost != null) {
2108         cpHost.postListNamespaceDescriptors(descriptors);
2109       }
2110     }
2111     return descriptors;
2112   }
2113 
2114   @Override
2115   public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
2116     ensureNamespaceExists(name);
2117     return listTableDescriptors(name, null, null, true);
2118   }
2119 
2120   @Override
2121   public List<TableName> listTableNamesByNamespace(String name) throws IOException {
2122     ensureNamespaceExists(name);
2123     return listTableNames(name, null, true);
2124   }
2125 
2126   /**
2127    * Returns the list of table descriptors that match the specified request
2128    *
2129    * @param namespace the namespace to query, or null if querying for all
2130    * @param regex The regular expression to match against, or null if querying for all
2131    * @param tableNameList the list of table names, or null if querying for all
2132    * @param includeSysTables False to match only against userspace tables
2133    * @return the list of table descriptors
2134    */
2135   public List<HTableDescriptor> listTableDescriptors(final String namespace, final String regex,
2136       final List<TableName> tableNameList, final boolean includeSysTables)
2137       throws IOException {
2138     final List<HTableDescriptor> descriptors = new ArrayList<HTableDescriptor>();
2139 
2140     boolean bypass = false;
2141     if (cpHost != null) {
2142       bypass = cpHost.preGetTableDescriptors(tableNameList, descriptors);
2143       // method required for AccessController.
2144       bypass |= cpHost.preGetTableDescriptors(tableNameList, descriptors, regex);
2145     }
2146 
2147     if (!bypass) {
2148       if (tableNameList == null || tableNameList.size() == 0) {
2149         // request for all TableDescriptors
2150         Collection<HTableDescriptor> htds;
2151         if (namespace != null && namespace.length() > 0) {
2152           htds = tableDescriptors.getByNamespace(namespace).values();
2153         } else {
2154           htds = tableDescriptors.getAll().values();
2155         }
2156 
2157         for (HTableDescriptor desc: htds) {
2158           if (includeSysTables || !desc.getTableName().isSystemTable()) {
2159             descriptors.add(desc);
2160           }
2161         }
2162       } else {
2163         for (TableName s: tableNameList) {
2164           HTableDescriptor desc = tableDescriptors.get(s);
2165           if (desc != null) {
2166             descriptors.add(desc);
2167           }
2168         }
2169       }
2170 
2171       // Retains only those matched by regular expression.
2172       if (regex != null) {
2173         filterTablesByRegex(descriptors, Pattern.compile(regex));
2174       }
2175 
2176       if (cpHost != null) {
2177         cpHost.postGetTableDescriptors(descriptors);
2178         // method required for AccessController.
2179         cpHost.postGetTableDescriptors(tableNameList, descriptors, regex);
2180       }
2181     }
2182     return descriptors;
2183   }
2184 
2185   /**
2186    * Returns the list of table names that match the specified request
2187    * @param regex The regular expression to match against, or null if querying for all
2188    * @param namespace the namespace to query, or null if querying for all
2189    * @param includeSysTables False to match only against userspace tables
2190    * @return the list of table names
2191    */
2192   public List<TableName> listTableNames(final String namespace, final String regex,
2193       final boolean includeSysTables) throws IOException {
2194     final List<HTableDescriptor> descriptors = new ArrayList<HTableDescriptor>();
2195 
2196     boolean bypass = false;
2197     if (cpHost != null) {
2198       bypass = cpHost.preGetTableNames(descriptors, regex);
2199     }
2200 
2201     if (!bypass) {
2202       // get all descriptors
2203       Collection<HTableDescriptor> htds;
2204       if (namespace != null && namespace.length() > 0) {
2205         htds = tableDescriptors.getByNamespace(namespace).values();
2206       } else {
2207         htds = tableDescriptors.getAll().values();
2208       }
2209 
2210       for (HTableDescriptor htd: htds) {
2211         if (includeSysTables || !htd.getTableName().isSystemTable()) {
2212           descriptors.add(htd);
2213         }
2214       }
2215 
2216       // Retains only those matched by regular expression.
2217       if (regex != null) {
2218         filterTablesByRegex(descriptors, Pattern.compile(regex));
2219       }
2220 
2221       if (cpHost != null) {
2222         cpHost.postGetTableNames(descriptors, regex);
2223       }
2224     }
2225 
2226     List<TableName> result = new ArrayList<TableName>(descriptors.size());
2227     for (HTableDescriptor htd: descriptors) {
2228       result.add(htd.getTableName());
2229     }
2230     return result;
2231   }
2232 
2233 
2234   /**
2235    * Removes the table descriptors that don't match the pattern.
2236    * @param descriptors list of table descriptors to filter
2237    * @param pattern the regex to use
2238    */
2239   private static void filterTablesByRegex(final Collection<HTableDescriptor> descriptors,
2240       final Pattern pattern) {
2241     final String defaultNS = NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR;
2242     Iterator<HTableDescriptor> itr = descriptors.iterator();
2243     while (itr.hasNext()) {
2244       HTableDescriptor htd = itr.next();
2245       String tableName = htd.getTableName().getNameAsString();
2246       boolean matched = pattern.matcher(tableName).matches();
2247       if (!matched && htd.getTableName().getNamespaceAsString().equals(defaultNS)) {
2248         matched = pattern.matcher(defaultNS + TableName.NAMESPACE_DELIM + tableName).matches();
2249       }
2250       if (!matched) {
2251         itr.remove();
2252       }
2253     }
2254   }
2255 }