1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import java.io.IOException;
22 import java.io.InterruptedIOException;
23 import java.lang.reflect.Constructor;
24 import java.lang.reflect.InvocationTargetException;
25 import java.net.InetAddress;
26 import java.net.InetSocketAddress;
27 import java.net.UnknownHostException;
28 import java.util.ArrayList;
29 import java.util.Arrays;
30 import java.util.Collection;
31 import java.util.Collections;
32 import java.util.Comparator;
33 import java.util.HashSet;
34 import java.util.Iterator;
35 import java.util.List;
36 import java.util.Map;
37 import java.util.Set;
38 import java.util.concurrent.TimeUnit;
39 import java.util.concurrent.atomic.AtomicReference;
40 import java.util.regex.Pattern;
41
42 import javax.servlet.ServletException;
43 import javax.servlet.http.HttpServlet;
44 import javax.servlet.http.HttpServletRequest;
45 import javax.servlet.http.HttpServletResponse;
46
47 import org.apache.commons.logging.Log;
48 import org.apache.commons.logging.LogFactory;
49 import org.apache.hadoop.conf.Configuration;
50 import org.apache.hadoop.fs.Path;
51 import org.apache.hadoop.hbase.ClusterStatus;
52 import org.apache.hadoop.hbase.CoordinatedStateException;
53 import org.apache.hadoop.hbase.CoordinatedStateManager;
54 import org.apache.hadoop.hbase.DoNotRetryIOException;
55 import org.apache.hadoop.hbase.HBaseIOException;
56 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
57 import org.apache.hadoop.hbase.HColumnDescriptor;
58 import org.apache.hadoop.hbase.HConstants;
59 import org.apache.hadoop.hbase.HRegionInfo;
60 import org.apache.hadoop.hbase.HTableDescriptor;
61 import org.apache.hadoop.hbase.MasterNotRunningException;
62 import org.apache.hadoop.hbase.MetaMigrationConvertingToPB;
63 import org.apache.hadoop.hbase.MetaTableAccessor;
64 import org.apache.hadoop.hbase.NamespaceDescriptor;
65 import org.apache.hadoop.hbase.NamespaceNotFoundException;
66 import org.apache.hadoop.hbase.PleaseHoldException;
67 import org.apache.hadoop.hbase.Server;
68 import org.apache.hadoop.hbase.ServerLoad;
69 import org.apache.hadoop.hbase.ServerName;
70 import org.apache.hadoop.hbase.TableDescriptors;
71 import org.apache.hadoop.hbase.TableName;
72 import org.apache.hadoop.hbase.TableNotDisabledException;
73 import org.apache.hadoop.hbase.TableNotFoundException;
74 import org.apache.hadoop.hbase.UnknownRegionException;
75 import org.apache.hadoop.hbase.classification.InterfaceAudience;
76 import org.apache.hadoop.hbase.client.MetaScanner;
77 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
78 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
79 import org.apache.hadoop.hbase.client.Result;
80 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
81 import org.apache.hadoop.hbase.exceptions.DeserializationException;
82 import org.apache.hadoop.hbase.executor.ExecutorType;
83 import org.apache.hadoop.hbase.ipc.RequestContext;
84 import org.apache.hadoop.hbase.ipc.RpcServer;
85 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
86 import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode;
87 import org.apache.hadoop.hbase.master.RegionState.State;
88 import org.apache.hadoop.hbase.master.balancer.BalancerChore;
89 import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
90 import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore;
91 import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
92 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
93 import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
94 import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
95 import org.apache.hadoop.hbase.master.handler.DeleteTableHandler;
96 import org.apache.hadoop.hbase.master.handler.DisableTableHandler;
97 import org.apache.hadoop.hbase.master.handler.DispatchMergingRegionHandler;
98 import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
99 import org.apache.hadoop.hbase.master.handler.ModifyTableHandler;
100 import org.apache.hadoop.hbase.master.handler.TableAddFamilyHandler;
101 import org.apache.hadoop.hbase.master.handler.TableDeleteFamilyHandler;
102 import org.apache.hadoop.hbase.master.handler.TableModifyFamilyHandler;
103 import org.apache.hadoop.hbase.master.handler.TruncateTableHandler;
104 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
105 import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
106 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
107 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
108 import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
109 import org.apache.hadoop.hbase.procedure.flush.MasterFlushTableProcedureManager;
110 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo;
111 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
112 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
113 import org.apache.hadoop.hbase.regionserver.HRegionServer;
114 import org.apache.hadoop.hbase.regionserver.RSRpcServices;
115 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
116 import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy;
117 import org.apache.hadoop.hbase.replication.regionserver.Replication;
118 import org.apache.hadoop.hbase.security.UserProvider;
119 import org.apache.hadoop.hbase.util.Addressing;
120 import org.apache.hadoop.hbase.util.Bytes;
121 import org.apache.hadoop.hbase.util.CompressionTest;
122 import org.apache.hadoop.hbase.util.ConfigUtil;
123 import org.apache.hadoop.hbase.util.EncryptionTest;
124 import org.apache.hadoop.hbase.util.FSUtils;
125 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
126 import org.apache.hadoop.hbase.util.HasThread;
127 import org.apache.hadoop.hbase.util.Pair;
128 import org.apache.hadoop.hbase.util.Threads;
129 import org.apache.hadoop.hbase.util.VersionInfo;
130 import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker;
131 import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
132 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
133 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
134 import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
135 import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
136 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
137 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
138 import org.apache.zookeeper.KeeperException;
139 import org.mortbay.jetty.Connector;
140 import org.mortbay.jetty.nio.SelectChannelConnector;
141 import org.mortbay.jetty.servlet.Context;
142
143 import com.google.common.annotations.VisibleForTesting;
144 import com.google.common.collect.Maps;
145 import com.google.protobuf.Descriptors;
146 import com.google.protobuf.Service;
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
164 @SuppressWarnings("deprecation")
165 public class HMaster extends HRegionServer implements MasterServices, Server {
166 private static final Log LOG = LogFactory.getLog(HMaster.class.getName());
167
168
169
170
171
172 private static class InitializationMonitor extends HasThread {
173
174 public static final String TIMEOUT_KEY = "hbase.master.initializationmonitor.timeout";
175 public static final long TIMEOUT_DEFAULT = TimeUnit.MILLISECONDS.convert(15, TimeUnit.MINUTES);
176
177
178
179
180
181 public static final String HALT_KEY = "hbase.master.initializationmonitor.haltontimeout";
182 public static final boolean HALT_DEFAULT = false;
183
184 private final HMaster master;
185 private final long timeout;
186 private final boolean haltOnTimeout;
187
188
189 InitializationMonitor(HMaster master) {
190 super("MasterInitializationMonitor");
191 this.master = master;
192 this.timeout = master.getConfiguration().getLong(TIMEOUT_KEY, TIMEOUT_DEFAULT);
193 this.haltOnTimeout = master.getConfiguration().getBoolean(HALT_KEY, HALT_DEFAULT);
194 this.setDaemon(true);
195 }
196
197 @Override
198 public void run() {
199 try {
200 while (!master.isStopped() && master.isActiveMaster()) {
201 Thread.sleep(timeout);
202 if (master.isInitialized()) {
203 LOG.debug("Initialization completed within allotted tolerance. Monitor exiting.");
204 } else {
205 LOG.error("Master failed to complete initialization after " + timeout + "ms. Please"
206 + " consider submitting a bug report including a thread dump of this process.");
207 if (haltOnTimeout) {
208 LOG.error("Zombie Master exiting. Thread dump to stdout");
209 Threads.printThreadInfo(System.out, "Zombie HMaster");
210 System.exit(-1);
211 }
212 }
213 }
214 } catch (InterruptedException ie) {
215 LOG.trace("InitMonitor thread interrupted. Existing.");
216 }
217 }
218 }
219
220
221
222 public static final String MASTER = "master";
223
224
225 private final ActiveMasterManager activeMasterManager;
226
227 RegionServerTracker regionServerTracker;
228
229 private DrainingServerTracker drainingServerTracker;
230
231 LoadBalancerTracker loadBalancerTracker;
232
233
234 private TableNamespaceManager tableNamespaceManager;
235
236
237 final MetricsMaster metricsMaster;
238
239 private MasterFileSystem fileSystemManager;
240
241
242 volatile ServerManager serverManager;
243
244
245 AssignmentManager assignmentManager;
246
247
248
249
250 MemoryBoundedLogMessageBuffer rsFatals;
251
252
253 private volatile boolean isActiveMaster = false;
254
255
256
257 volatile boolean initialized = false;
258
259
260
261 volatile boolean serviceStarted = false;
262
263
264 private volatile boolean serverShutdownHandlerEnabled = false;
265
266 LoadBalancer balancer;
267 private BalancerChore balancerChore;
268 private ClusterStatusChore clusterStatusChore;
269 private ClusterStatusPublisher clusterStatusPublisherChore = null;
270
271 CatalogJanitor catalogJanitorChore;
272 private LogCleaner logCleaner;
273 private HFileCleaner hfileCleaner;
274
275 MasterCoprocessorHost cpHost;
276
277 private final boolean preLoadTableDescriptors;
278
279
280 private long masterActiveTime;
281
282
283 private final boolean masterCheckCompression;
284
285
286 private final boolean masterCheckEncryption;
287
288 Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap();
289
290
291 SnapshotManager snapshotManager;
292
293 MasterProcedureManagerHost mpmHost;
294
295
296 private volatile boolean initializationBeforeMetaAssignment = false;
297
298
299 private org.mortbay.jetty.Server masterJettyServer;
300
301 public static class RedirectServlet extends HttpServlet {
302 private static final long serialVersionUID = 2894774810058302472L;
303 private static int regionServerInfoPort;
304
305 @Override
306 public void doGet(HttpServletRequest request,
307 HttpServletResponse response) throws ServletException, IOException {
308 String redirectUrl = request.getScheme() + "://"
309 + request.getServerName() + ":" + regionServerInfoPort
310 + request.getRequestURI();
311 response.sendRedirect(redirectUrl);
312 }
313 }
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331 public HMaster(final Configuration conf, CoordinatedStateManager csm)
332 throws IOException, KeeperException, InterruptedException {
333 super(conf, csm);
334 this.rsFatals = new MemoryBoundedLogMessageBuffer(
335 conf.getLong("hbase.master.buffer.for.rs.fatals", 1*1024*1024));
336
337 LOG.info("hbase.rootdir=" + FSUtils.getRootDir(this.conf) +
338 ", hbase.cluster.distributed=" + this.conf.getBoolean(HConstants.CLUSTER_DISTRIBUTED, false));
339
340 Replication.decorateMasterConfiguration(this.conf);
341
342
343
344 if (this.conf.get("mapreduce.task.attempt.id") == null) {
345 this.conf.set("mapreduce.task.attempt.id", "hb_m_" + this.serverName.toString());
346 }
347
348
349 this.masterCheckCompression = conf.getBoolean("hbase.master.check.compression", true);
350
351
352 this.masterCheckEncryption = conf.getBoolean("hbase.master.check.encryption", true);
353
354 this.metricsMaster = new MetricsMaster( new MetricsMasterWrapperImpl(this));
355
356
357 this.preLoadTableDescriptors = conf.getBoolean("hbase.master.preload.tabledescriptors", true);
358
359
360
361 boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED,
362 HConstants.STATUS_PUBLISHED_DEFAULT);
363 Class<? extends ClusterStatusPublisher.Publisher> publisherClass =
364 conf.getClass(ClusterStatusPublisher.STATUS_PUBLISHER_CLASS,
365 ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS,
366 ClusterStatusPublisher.Publisher.class);
367
368 if (shouldPublish) {
369 if (publisherClass == null) {
370 LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " +
371 ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS +
372 " is not set - not publishing status");
373 } else {
374 clusterStatusPublisherChore = new ClusterStatusPublisher(this, conf, publisherClass);
375 Threads.setDaemonThreadRunning(clusterStatusPublisherChore.getThread());
376 }
377 }
378 activeMasterManager = new ActiveMasterManager(zooKeeper, this.serverName, this);
379 int infoPort = putUpJettyServer();
380 startActiveMasterManager(infoPort);
381 }
382
383
384 private int putUpJettyServer() throws IOException {
385 if (!conf.getBoolean("hbase.master.infoserver.redirect", true)) {
386 return -1;
387 }
388 int infoPort = conf.getInt("hbase.master.info.port.orig",
389 HConstants.DEFAULT_MASTER_INFOPORT);
390
391 if (infoPort < 0 || infoServer == null) {
392 return -1;
393 }
394 String addr = conf.get("hbase.master.info.bindAddress", "0.0.0.0");
395 if (!Addressing.isLocalAddress(InetAddress.getByName(addr))) {
396 String msg =
397 "Failed to start redirecting jetty server. Address " + addr
398 + " does not belong to this host. Correct configuration parameter: "
399 + "hbase.master.info.bindAddress";
400 LOG.error(msg);
401 throw new IOException(msg);
402 }
403
404 RedirectServlet.regionServerInfoPort = infoServer.getPort();
405 masterJettyServer = new org.mortbay.jetty.Server();
406 Connector connector = new SelectChannelConnector();
407 connector.setHost(addr);
408 connector.setPort(infoPort);
409 masterJettyServer.addConnector(connector);
410 masterJettyServer.setStopAtShutdown(true);
411 Context context = new Context(masterJettyServer, "/", Context.NO_SESSIONS);
412 context.addServlet(RedirectServlet.class, "/*");
413 try {
414 masterJettyServer.start();
415 } catch (Exception e) {
416 throw new IOException("Failed to start redirecting jetty server", e);
417 }
418 return connector.getLocalPort();
419 }
420
421
422
423
424 @Override
425 protected void login(UserProvider user, String host) throws IOException {
426 try {
427 super.login(user, host);
428 } catch (IOException ie) {
429 user.login("hbase.master.keytab.file",
430 "hbase.master.kerberos.principal", host);
431 }
432 }
433
434
435
436
437
438
439 @Override
440 protected void waitForMasterActive(){
441 boolean tablesOnMaster = BaseLoadBalancer.tablesOnMaster(conf);
442 while (!(tablesOnMaster && isActiveMaster)
443 && !isStopped() && !isAborted()) {
444 sleeper.sleep();
445 }
446 }
447
448 @VisibleForTesting
449 public MasterRpcServices getMasterRpcServices() {
450 return (MasterRpcServices)rpcServices;
451 }
452
453 public boolean balanceSwitch(final boolean b) throws IOException {
454 return getMasterRpcServices().switchBalancer(b, BalanceSwitchMode.ASYNC);
455 }
456
457 @Override
458 protected String getProcessName() {
459 return MASTER;
460 }
461
462 @Override
463 protected boolean canCreateBaseZNode() {
464 return true;
465 }
466
467 @Override
468 protected boolean canUpdateTableDescriptor() {
469 return true;
470 }
471
472 @Override
473 protected RSRpcServices createRpcServices() throws IOException {
474 return new MasterRpcServices(this);
475 }
476
477 @Override
478 protected void configureInfoServer() {
479 infoServer.addServlet("master-status", "/master-status", MasterStatusServlet.class);
480 infoServer.setAttribute(MASTER, this);
481 if (BaseLoadBalancer.tablesOnMaster(conf)) {
482 super.configureInfoServer();
483 }
484 }
485
486 @Override
487 protected Class<? extends HttpServlet> getDumpServlet() {
488 return MasterDumpServlet.class;
489 }
490
491
492
493
494
495 @Override
496 protected void doMetrics() {
497 try {
498 if (assignmentManager != null) {
499 assignmentManager.updateRegionsInTransitionMetrics();
500 }
501 } catch (Throwable e) {
502 LOG.error("Couldn't update metrics: " + e.getMessage());
503 }
504 }
505
506 MetricsMaster getMasterMetrics() {
507 return metricsMaster;
508 }
509
510
511
512
513
514
515
516
517 void initializeZKBasedSystemTrackers() throws IOException,
518 InterruptedException, KeeperException, CoordinatedStateException {
519 this.balancer = LoadBalancerFactory.getLoadBalancer(conf);
520 this.loadBalancerTracker = new LoadBalancerTracker(zooKeeper, this);
521 this.loadBalancerTracker.start();
522 this.assignmentManager = new AssignmentManager(this, serverManager,
523 this.balancer, this.service, this.metricsMaster,
524 this.tableLockManager);
525 zooKeeper.registerListenerFirst(assignmentManager);
526
527 this.regionServerTracker = new RegionServerTracker(zooKeeper, this,
528 this.serverManager);
529 this.regionServerTracker.start();
530
531 this.drainingServerTracker = new DrainingServerTracker(zooKeeper, this,
532 this.serverManager);
533 this.drainingServerTracker.start();
534
535
536
537 boolean wasUp = this.clusterStatusTracker.isClusterUp();
538 if (!wasUp) this.clusterStatusTracker.setClusterUp();
539
540 LOG.info("Server active/primary master=" + this.serverName +
541 ", sessionid=0x" +
542 Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()) +
543 ", setting cluster-up flag (Was=" + wasUp + ")");
544
545
546 this.snapshotManager = new SnapshotManager();
547 this.mpmHost = new MasterProcedureManagerHost();
548 this.mpmHost.register(this.snapshotManager);
549 this.mpmHost.register(new MasterFlushTableProcedureManager());
550 this.mpmHost.loadProcedures(conf);
551 this.mpmHost.initialize(this, this.metricsMaster);
552 }
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574 private void finishActiveMasterInitialization(MonitoredTask status)
575 throws IOException, InterruptedException, KeeperException, CoordinatedStateException {
576
577 isActiveMaster = true;
578 Thread zombieDetector = new Thread(new InitializationMonitor(this));
579 zombieDetector.start();
580
581
582
583
584
585
586
587 status.setStatus("Initializing Master file system");
588
589 this.masterActiveTime = System.currentTimeMillis();
590
591 this.fileSystemManager = new MasterFileSystem(this, this);
592
593
594 this.tableDescriptors.setCacheOn();
595
596
597 if (preLoadTableDescriptors) {
598 status.setStatus("Pre-loading table descriptors");
599 this.tableDescriptors.getAll();
600 }
601
602
603 status.setStatus("Publishing Cluster ID in ZooKeeper");
604 ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());
605 this.serverManager = createServerManager(this, this);
606
607 setupClusterConnection();
608
609
610 this.tableLockManager.reapWriteLocks();
611
612 status.setStatus("Initializing ZK system trackers");
613 initializeZKBasedSystemTrackers();
614
615
616 status.setStatus("Initializing master coprocessors");
617 this.cpHost = new MasterCoprocessorHost(this, this.conf);
618
619
620 status.setStatus("Initializing master service threads");
621 startServiceThreads();
622
623
624 sleeper.skipSleepCycle();
625
626
627 this.serverManager.waitForRegionServers(status);
628
629 for (ServerName sn: this.regionServerTracker.getOnlineServers()) {
630
631 if (!this.serverManager.isServerOnline(sn)
632 && serverManager.checkAndRecordNewServer(sn, ServerLoad.EMPTY_SERVERLOAD)) {
633 LOG.info("Registered server found up in zk but who has not yet reported in: " + sn);
634 }
635 }
636
637
638
639
640 Set<ServerName> previouslyFailedServers = this.fileSystemManager
641 .getFailedServersFromLogFolders();
642
643
644 this.fileSystemManager.removeStaleRecoveringRegionsFromZK(previouslyFailedServers);
645
646
647 ServerName oldMetaServerLocation = metaTableLocator.getMetaRegionLocation(this.getZooKeeper());
648 if (oldMetaServerLocation != null && previouslyFailedServers.contains(oldMetaServerLocation)) {
649 splitMetaLogBeforeAssignment(oldMetaServerLocation);
650
651
652 }
653 Set<ServerName> previouslyFailedMetaRSs = getPreviouselyFailedMetaServersFromZK();
654
655
656
657
658
659
660
661
662 previouslyFailedMetaRSs.addAll(previouslyFailedServers);
663
664 this.initializationBeforeMetaAssignment = true;
665
666
667 if (BaseLoadBalancer.tablesOnMaster(conf)) {
668 waitForServerOnline();
669 }
670
671
672 this.balancer.setClusterStatus(getClusterStatus());
673 this.balancer.setMasterServices(this);
674 this.balancer.initialize();
675
676
677
678 if(isStopped()) return;
679
680
681 status.setStatus("Assigning Meta Region");
682 assignMeta(status, previouslyFailedMetaRSs);
683
684
685 if(isStopped()) return;
686
687 status.setStatus("Submitting log splitting work for previously failed region servers");
688
689
690 for (ServerName tmpServer : previouslyFailedServers) {
691 this.serverManager.processDeadServer(tmpServer, true);
692 }
693
694
695
696
697 if (this.conf.getBoolean("hbase.MetaMigrationConvertingToPB", true)) {
698 MetaMigrationConvertingToPB.updateMetaIfNecessary(this);
699 }
700
701
702 status.setStatus("Starting assignment manager");
703 this.assignmentManager.joinCluster();
704
705
706 this.balancer.setClusterStatus(getClusterStatus());
707
708
709
710 status.setStatus("Starting balancer and catalog janitor");
711 this.clusterStatusChore = new ClusterStatusChore(this, balancer);
712 Threads.setDaemonThreadRunning(clusterStatusChore.getThread());
713 this.balancerChore = new BalancerChore(this);
714 Threads.setDaemonThreadRunning(balancerChore.getThread());
715 this.catalogJanitorChore = new CatalogJanitor(this, this);
716 Threads.setDaemonThreadRunning(catalogJanitorChore.getThread());
717
718 status.setStatus("Starting namespace manager");
719 initNamespace();
720
721 if (this.cpHost != null) {
722 try {
723 this.cpHost.preMasterInitialization();
724 } catch (IOException e) {
725 LOG.error("Coprocessor preMasterInitialization() hook failed", e);
726 }
727 }
728
729 status.markComplete("Initialization successful");
730 LOG.info("Master has completed initialization");
731 configurationManager.registerObserver(this.balancer);
732 initialized = true;
733
734
735
736 this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();
737
738 if (this.cpHost != null) {
739
740 try {
741 this.cpHost.postStartMaster();
742 } catch (IOException ioe) {
743 LOG.error("Coprocessor postStartMaster() hook failed", ioe);
744 }
745 }
746
747 zombieDetector.interrupt();
748 }
749
750
751
752
753
754
755
756
757
758 ServerManager createServerManager(final Server master,
759 final MasterServices services)
760 throws IOException {
761
762
763 return new ServerManager(master, services);
764 }
765
766
767
768
769
770
771
772
773
774 void assignMeta(MonitoredTask status, Set<ServerName> previouslyFailedMetaRSs)
775 throws InterruptedException, IOException, KeeperException {
776
777 int assigned = 0;
778 long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);
779 status.setStatus("Assigning hbase:meta region");
780
781 RegionStates regionStates = assignmentManager.getRegionStates();
782 RegionState metaState = MetaTableLocator.getMetaRegionState(getZooKeeper());
783 ServerName currentMetaServer = metaState.getServerName();
784 if (!ConfigUtil.useZKForAssignment(conf)) {
785 regionStates.createRegionState(HRegionInfo.FIRST_META_REGIONINFO, metaState.getState(),
786 currentMetaServer, null);
787 } else {
788 regionStates.createRegionState(HRegionInfo.FIRST_META_REGIONINFO);
789 }
790 boolean rit = this.assignmentManager.
791 processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO);
792 boolean metaRegionLocation = metaTableLocator.verifyMetaRegionLocation(
793 this.getConnection(), this.getZooKeeper(), timeout);
794 if (!metaRegionLocation || !metaState.isOpened()) {
795
796
797 assigned++;
798 if (!ConfigUtil.useZKForAssignment(conf)) {
799 assignMetaZkLess(regionStates, metaState, timeout, previouslyFailedMetaRSs);
800 } else if (!rit) {
801
802 if (currentMetaServer != null) {
803
804
805
806
807
808
809
810 if (serverManager.isServerOnline(currentMetaServer)) {
811 LOG.info("Forcing expire of " + currentMetaServer);
812 serverManager.expireServer(currentMetaServer);
813 }
814 splitMetaLogBeforeAssignment(currentMetaServer);
815 previouslyFailedMetaRSs.add(currentMetaServer);
816 }
817 assignmentManager.assignMeta();
818 }
819 } else {
820
821 regionStates.updateRegionState(
822 HRegionInfo.FIRST_META_REGIONINFO, State.OPEN, currentMetaServer);
823 this.assignmentManager.regionOnline(
824 HRegionInfo.FIRST_META_REGIONINFO, currentMetaServer);
825 }
826
827 enableMeta(TableName.META_TABLE_NAME);
828
829 if ((RecoveryMode.LOG_REPLAY == this.getMasterFileSystem().getLogRecoveryMode())
830 && (!previouslyFailedMetaRSs.isEmpty())) {
831
832 status.setStatus("replaying log for Meta Region");
833 this.fileSystemManager.splitMetaLog(previouslyFailedMetaRSs);
834 }
835
836
837
838
839
840 enableServerShutdownHandler(assigned != 0);
841
842 LOG.info("hbase:meta assigned=" + assigned + ", rit=" + rit +
843 ", location=" + metaTableLocator.getMetaRegionLocation(this.getZooKeeper()));
844 status.setStatus("META assigned.");
845 }
846
847 private void assignMetaZkLess(RegionStates regionStates, RegionState regionState, long timeout,
848 Set<ServerName> previouslyFailedRs) throws IOException, KeeperException {
849 ServerName currentServer = regionState.getServerName();
850 if (serverManager.isServerOnline(currentServer)) {
851 LOG.info("Meta was in transition on " + currentServer);
852 assignmentManager.processRegionInTransitionZkLess();
853 } else {
854 if (currentServer != null) {
855 splitMetaLogBeforeAssignment(currentServer);
856 regionStates.logSplit(HRegionInfo.FIRST_META_REGIONINFO);
857 previouslyFailedRs.add(currentServer);
858 }
859 LOG.info("Re-assigning hbase:meta, it was on " + currentServer);
860 regionStates.updateRegionState(HRegionInfo.FIRST_META_REGIONINFO, State.OFFLINE);
861 assignmentManager.assignMeta();
862 }
863 }
864
865 void initNamespace() throws IOException {
866
867 tableNamespaceManager = new TableNamespaceManager(this);
868 tableNamespaceManager.start();
869 }
870
871 boolean isCatalogJanitorEnabled() {
872 return catalogJanitorChore != null ?
873 catalogJanitorChore.getEnabled() : false;
874 }
875
876 private void splitMetaLogBeforeAssignment(ServerName currentMetaServer) throws IOException {
877 if (RecoveryMode.LOG_REPLAY == this.getMasterFileSystem().getLogRecoveryMode()) {
878
879 Set<HRegionInfo> regions = new HashSet<HRegionInfo>();
880 regions.add(HRegionInfo.FIRST_META_REGIONINFO);
881 this.fileSystemManager.prepareLogReplay(currentMetaServer, regions);
882 } else {
883
884 this.fileSystemManager.splitMetaLog(currentMetaServer);
885 }
886 }
887
888 private void enableServerShutdownHandler(
889 final boolean waitForMeta) throws IOException, InterruptedException {
890
891
892
893
894
895 if (!serverShutdownHandlerEnabled) {
896 serverShutdownHandlerEnabled = true;
897 this.serverManager.processQueuedDeadServers();
898 }
899
900 if (waitForMeta) {
901 metaTableLocator.waitMetaRegionLocation(this.getZooKeeper());
902
903
904 this.assignmentManager.waitForAssignment(HRegionInfo.FIRST_META_REGIONINFO);
905 }
906 }
907
908 private void enableMeta(TableName metaTableName) {
909 if (!this.assignmentManager.getTableStateManager().isTableState(metaTableName,
910 ZooKeeperProtos.Table.State.ENABLED)) {
911 this.assignmentManager.setEnabledTable(metaTableName);
912 }
913 }
914
915
916
917
918
919
920 private Set<ServerName> getPreviouselyFailedMetaServersFromZK() throws KeeperException {
921 Set<ServerName> result = new HashSet<ServerName>();
922 String metaRecoveringZNode = ZKUtil.joinZNode(zooKeeper.recoveringRegionsZNode,
923 HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
924 List<String> regionFailedServers = ZKUtil.listChildrenNoWatch(zooKeeper, metaRecoveringZNode);
925 if (regionFailedServers == null) return result;
926
927 for(String failedServer : regionFailedServers) {
928 ServerName server = ServerName.parseServerName(failedServer);
929 result.add(server);
930 }
931 return result;
932 }
933
934 @Override
935 public TableDescriptors getTableDescriptors() {
936 return this.tableDescriptors;
937 }
938
939 @Override
940 public ServerManager getServerManager() {
941 return this.serverManager;
942 }
943
944 @Override
945 public MasterFileSystem getMasterFileSystem() {
946 return this.fileSystemManager;
947 }
948
949
950
951
952
953
954
955
956 private void startServiceThreads() throws IOException{
957
958 this.service.startExecutorService(ExecutorType.MASTER_OPEN_REGION,
959 conf.getInt("hbase.master.executor.openregion.threads", 5));
960 this.service.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,
961 conf.getInt("hbase.master.executor.closeregion.threads", 5));
962 this.service.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,
963 conf.getInt("hbase.master.executor.serverops.threads", 5));
964 this.service.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,
965 conf.getInt("hbase.master.executor.serverops.threads", 5));
966 this.service.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS,
967 conf.getInt("hbase.master.executor.logreplayops.threads", 10));
968
969
970
971
972
973
974 this.service.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);
975
976
977 int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 * 1000);
978 this.logCleaner =
979 new LogCleaner(cleanerInterval,
980 this, conf, getMasterFileSystem().getFileSystem(),
981 getMasterFileSystem().getOldLogDir());
982 Threads.setDaemonThreadRunning(logCleaner.getThread(),
983 getServerName().toShortString() + ".oldLogCleaner");
984
985
986 Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
987 this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()
988 .getFileSystem(), archiveDir);
989 Threads.setDaemonThreadRunning(hfileCleaner.getThread(),
990 getServerName().toShortString() + ".archivedHFileCleaner");
991
992 serviceStarted = true;
993 if (LOG.isTraceEnabled()) {
994 LOG.trace("Started service threads");
995 }
996 }
997
998 @Override
999 protected void stopServiceThreads() {
1000 if (masterJettyServer != null) {
1001 LOG.info("Stopping master jetty server");
1002 try {
1003 masterJettyServer.stop();
1004 } catch (Exception e) {
1005 LOG.error("Failed to stop master jetty server", e);
1006 }
1007 }
1008 super.stopServiceThreads();
1009 stopChores();
1010
1011
1012 if (!isAborted() && this.serverManager != null &&
1013 this.serverManager.isClusterShutdown()) {
1014 this.serverManager.letRegionServersShutdown();
1015 }
1016 if (LOG.isDebugEnabled()) {
1017 LOG.debug("Stopping service threads");
1018 }
1019
1020 if (this.logCleaner!= null) this.logCleaner.interrupt();
1021 if (this.hfileCleaner != null) this.hfileCleaner.interrupt();
1022 if (this.activeMasterManager != null) this.activeMasterManager.stop();
1023 if (this.serverManager != null) this.serverManager.stop();
1024 if (this.assignmentManager != null) this.assignmentManager.stop();
1025 if (this.fileSystemManager != null) this.fileSystemManager.stop();
1026 if (this.mpmHost != null) this.mpmHost.stop("server shutting down.");
1027 }
1028
1029 private void stopChores() {
1030 if (this.balancerChore != null) {
1031 this.balancerChore.interrupt();
1032 }
1033 if (this.clusterStatusChore != null) {
1034 this.clusterStatusChore.interrupt();
1035 }
1036 if (this.catalogJanitorChore != null) {
1037 this.catalogJanitorChore.interrupt();
1038 }
1039 if (this.clusterStatusPublisherChore != null){
1040 clusterStatusPublisherChore.interrupt();
1041 }
1042 }
1043
1044
1045
1046
1047
1048 InetAddress getRemoteInetAddress(final int port,
1049 final long serverStartCode) throws UnknownHostException {
1050
1051
1052 InetAddress ia = RpcServer.getRemoteIp();
1053
1054
1055
1056 if (ia == null && serverStartCode == startcode) {
1057 InetSocketAddress isa = rpcServices.getSocketAddress();
1058 if (isa != null && isa.getPort() == port) {
1059 ia = isa.getAddress();
1060 }
1061 }
1062 return ia;
1063 }
1064
1065
1066
1067
1068 private int getBalancerCutoffTime() {
1069 int balancerCutoffTime =
1070 getConfiguration().getInt("hbase.balancer.max.balancing", -1);
1071 if (balancerCutoffTime == -1) {
1072
1073 int balancerPeriod =
1074 getConfiguration().getInt("hbase.balancer.period", 300000);
1075 balancerCutoffTime = balancerPeriod;
1076
1077 if (balancerCutoffTime <= 0) balancerCutoffTime = balancerPeriod;
1078 }
1079 return balancerCutoffTime;
1080 }
1081
1082 public boolean balance() throws IOException {
1083
1084 if (!this.initialized) {
1085 LOG.debug("Master has not been initialized, don't run balancer.");
1086 return false;
1087 }
1088
1089 int maximumBalanceTime = getBalancerCutoffTime();
1090 synchronized (this.balancer) {
1091
1092 if (!this.loadBalancerTracker.isBalancerOn()) return false;
1093
1094 if (this.assignmentManager.getRegionStates().isRegionsInTransition()) {
1095 Map<String, RegionState> regionsInTransition =
1096 this.assignmentManager.getRegionStates().getRegionsInTransition();
1097 LOG.debug("Not running balancer because " + regionsInTransition.size() +
1098 " region(s) in transition: " + org.apache.commons.lang.StringUtils.
1099 abbreviate(regionsInTransition.toString(), 256));
1100 return false;
1101 }
1102 if (this.serverManager.areDeadServersInProgress()) {
1103 LOG.debug("Not running balancer because processing dead regionserver(s): " +
1104 this.serverManager.getDeadServers());
1105 return false;
1106 }
1107
1108 if (this.cpHost != null) {
1109 try {
1110 if (this.cpHost.preBalance()) {
1111 LOG.debug("Coprocessor bypassing balancer request");
1112 return false;
1113 }
1114 } catch (IOException ioe) {
1115 LOG.error("Error invoking master coprocessor preBalance()", ioe);
1116 return false;
1117 }
1118 }
1119
1120 Map<TableName, Map<ServerName, List<HRegionInfo>>> assignmentsByTable =
1121 this.assignmentManager.getRegionStates().getAssignmentsByTable();
1122
1123 List<RegionPlan> plans = new ArrayList<RegionPlan>();
1124
1125 this.balancer.setClusterStatus(getClusterStatus());
1126 for (Map<ServerName, List<HRegionInfo>> assignments : assignmentsByTable.values()) {
1127 List<RegionPlan> partialPlans = this.balancer.balanceCluster(assignments);
1128 if (partialPlans != null) plans.addAll(partialPlans);
1129 }
1130 long cutoffTime = System.currentTimeMillis() + maximumBalanceTime;
1131 int rpCount = 0;
1132 long totalRegPlanExecTime = 0;
1133 if (plans != null && !plans.isEmpty()) {
1134 for (RegionPlan plan: plans) {
1135 LOG.info("balance " + plan);
1136 long balStartTime = System.currentTimeMillis();
1137
1138 this.assignmentManager.balance(plan);
1139 totalRegPlanExecTime += System.currentTimeMillis()-balStartTime;
1140 rpCount++;
1141 if (rpCount < plans.size() &&
1142
1143 (System.currentTimeMillis() + (totalRegPlanExecTime / rpCount)) > cutoffTime) {
1144
1145 LOG.debug("No more balancing till next balance run; maximumBalanceTime=" +
1146 maximumBalanceTime);
1147 break;
1148 }
1149 }
1150 }
1151 if (this.cpHost != null) {
1152 try {
1153 this.cpHost.postBalance(rpCount < plans.size() ? plans.subList(0, rpCount) : plans);
1154 } catch (IOException ioe) {
1155
1156 LOG.error("Error invoking master coprocessor postBalance()", ioe);
1157 }
1158 }
1159 }
1160
1161
1162 return true;
1163 }
1164
1165
1166
1167
1168 String getClientIdAuditPrefix() {
1169 return "Client=" + RequestContext.getRequestUserName() + "/" +
1170 RequestContext.get().getRemoteAddress();
1171 }
1172
1173
1174
1175
1176
1177
1178
1179 public void setCatalogJanitorEnabled(final boolean b) {
1180 this.catalogJanitorChore.setEnabled(b);
1181 }
1182
1183 @Override
1184 public void dispatchMergingRegions(final HRegionInfo region_a,
1185 final HRegionInfo region_b, final boolean forcible) throws IOException {
1186 checkInitialized();
1187 this.service.submit(new DispatchMergingRegionHandler(this,
1188 this.catalogJanitorChore, region_a, region_b, forcible));
1189 }
1190
1191 void move(final byte[] encodedRegionName,
1192 final byte[] destServerName) throws HBaseIOException {
1193 RegionState regionState = assignmentManager.getRegionStates().
1194 getRegionState(Bytes.toString(encodedRegionName));
1195 if (regionState == null) {
1196 throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));
1197 }
1198
1199 HRegionInfo hri = regionState.getRegion();
1200 ServerName dest;
1201 if (destServerName == null || destServerName.length == 0) {
1202 LOG.info("Passed destination servername is null/empty so " +
1203 "choosing a server at random");
1204 final List<ServerName> destServers = this.serverManager.createDestinationServersList(
1205 regionState.getServerName());
1206 dest = balancer.randomAssignment(hri, destServers);
1207 if (dest == null) {
1208 LOG.debug("Unable to determine a plan to assign " + hri);
1209 return;
1210 }
1211 } else {
1212 dest = ServerName.valueOf(Bytes.toString(destServerName));
1213 if (dest.equals(serverName) && balancer instanceof BaseLoadBalancer
1214 && !((BaseLoadBalancer)balancer).shouldBeOnMaster(hri)) {
1215
1216
1217
1218 LOG.debug("Skipping move of region " + hri.getRegionNameAsString()
1219 + " to avoid unnecessary region moving later by load balancer,"
1220 + " because it should not be on master");
1221 return;
1222 }
1223 }
1224
1225 if (dest.equals(regionState.getServerName())) {
1226 LOG.debug("Skipping move of region " + hri.getRegionNameAsString()
1227 + " because region already assigned to the same server " + dest + ".");
1228 return;
1229 }
1230
1231
1232 RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), dest);
1233
1234 try {
1235 checkInitialized();
1236 if (this.cpHost != null) {
1237 if (this.cpHost.preMove(hri, rp.getSource(), rp.getDestination())) {
1238 return;
1239 }
1240 }
1241 LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");
1242 this.assignmentManager.balance(rp);
1243 if (this.cpHost != null) {
1244 this.cpHost.postMove(hri, rp.getSource(), rp.getDestination());
1245 }
1246 } catch (IOException ioe) {
1247 if (ioe instanceof HBaseIOException) {
1248 throw (HBaseIOException)ioe;
1249 }
1250 throw new HBaseIOException(ioe);
1251 }
1252 }
1253
1254 @Override
1255 public void createTable(HTableDescriptor hTableDescriptor,
1256 byte [][] splitKeys) throws IOException {
1257 if (isStopped()) {
1258 throw new MasterNotRunningException();
1259 }
1260
1261 String namespace = hTableDescriptor.getTableName().getNamespaceAsString();
1262 ensureNamespaceExists(namespace);
1263
1264 HRegionInfo[] newRegions = getHRegionInfos(hTableDescriptor, splitKeys);
1265 checkInitialized();
1266 sanityCheckTableDescriptor(hTableDescriptor);
1267 if (cpHost != null) {
1268 cpHost.preCreateTable(hTableDescriptor, newRegions);
1269 }
1270 LOG.info(getClientIdAuditPrefix() + " create " + hTableDescriptor);
1271 this.service.submit(new CreateTableHandler(this,
1272 this.fileSystemManager, hTableDescriptor, conf,
1273 newRegions, this).prepare());
1274 if (cpHost != null) {
1275 cpHost.postCreateTable(hTableDescriptor, newRegions);
1276 }
1277
1278 }
1279
1280
1281
1282
1283
1284
1285 private void sanityCheckTableDescriptor(final HTableDescriptor htd) throws IOException {
1286 final String CONF_KEY = "hbase.table.sanity.checks";
1287 if (!conf.getBoolean(CONF_KEY, true)) {
1288 return;
1289 }
1290 String tableVal = htd.getConfigurationValue(CONF_KEY);
1291 if (tableVal != null && !Boolean.valueOf(tableVal)) {
1292 return;
1293 }
1294
1295
1296 long maxFileSizeLowerLimit = 2 * 1024 * 1024L;
1297 long maxFileSize = htd.getMaxFileSize();
1298 if (maxFileSize < 0) {
1299 maxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, maxFileSizeLowerLimit);
1300 }
1301 if (maxFileSize < conf.getLong("hbase.hregion.max.filesize.limit", maxFileSizeLowerLimit)) {
1302 throw new DoNotRetryIOException("MAX_FILESIZE for table descriptor or "
1303 + "\"hbase.hregion.max.filesize\" (" + maxFileSize
1304 + ") is too small, which might cause over splitting into unmanageable "
1305 + "number of regions. Set " + CONF_KEY + " to false at conf or table descriptor "
1306 + "if you want to bypass sanity checks");
1307 }
1308
1309
1310 long flushSizeLowerLimit = 1024 * 1024L;
1311 long flushSize = htd.getMemStoreFlushSize();
1312 if (flushSize < 0) {
1313 flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSizeLowerLimit);
1314 }
1315 if (flushSize < conf.getLong("hbase.hregion.memstore.flush.size.limit", flushSizeLowerLimit)) {
1316 throw new DoNotRetryIOException("MEMSTORE_FLUSHSIZE for table descriptor or "
1317 + "\"hbase.hregion.memstore.flush.size\" ("+flushSize+") is too small, which might cause"
1318 + " very frequent flushing. Set " + CONF_KEY + " to false at conf or table descriptor "
1319 + "if you want to bypass sanity checks");
1320 }
1321
1322
1323 try {
1324 checkClassLoading(conf, htd);
1325 } catch (Exception ex) {
1326 throw new DoNotRetryIOException(ex);
1327 }
1328
1329
1330 try {
1331 checkCompression(htd);
1332 } catch (IOException e) {
1333 throw new DoNotRetryIOException(e.getMessage(), e);
1334 }
1335
1336
1337 try {
1338 checkEncryption(conf, htd);
1339 } catch (IOException e) {
1340 throw new DoNotRetryIOException(e.getMessage(), e);
1341 }
1342
1343
1344 if (htd.getColumnFamilies().length == 0) {
1345 throw new DoNotRetryIOException("Table should have at least one column family "
1346 + "Set "+CONF_KEY+" at conf or table descriptor if you want to bypass sanity checks");
1347 }
1348
1349 for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
1350 if (hcd.getTimeToLive() <= 0) {
1351 throw new DoNotRetryIOException("TTL for column family " + hcd.getNameAsString()
1352 + " must be positive. Set " + CONF_KEY + " to false at conf or table descriptor "
1353 + "if you want to bypass sanity checks");
1354 }
1355
1356
1357 if (hcd.getBlocksize() < 1024 || hcd.getBlocksize() > 16 * 1024 * 1024) {
1358 throw new DoNotRetryIOException("Block size for column family " + hcd.getNameAsString()
1359 + " must be between 1K and 16MB Set "+CONF_KEY+" to false at conf or table descriptor "
1360 + "if you want to bypass sanity checks");
1361 }
1362
1363
1364 if (hcd.getMinVersions() < 0) {
1365 throw new DoNotRetryIOException("Min versions for column family " + hcd.getNameAsString()
1366 + " must be positive. Set " + CONF_KEY + " to false at conf or table descriptor "
1367 + "if you want to bypass sanity checks");
1368 }
1369
1370
1371
1372 if (hcd.getScope() < 0) {
1373 throw new DoNotRetryIOException("Replication scope for column family "
1374 + hcd.getNameAsString() + " must be positive. Set " + CONF_KEY + " to false at conf "
1375 + "or table descriptor if you want to bypass sanity checks");
1376 }
1377
1378
1379 }
1380 }
1381
1382 private void startActiveMasterManager(int infoPort) throws KeeperException {
1383 String backupZNode = ZKUtil.joinZNode(
1384 zooKeeper.backupMasterAddressesZNode, serverName.toString());
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395 LOG.info("Adding backup master ZNode " + backupZNode);
1396 if (!MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode,
1397 serverName, infoPort)) {
1398 LOG.warn("Failed create of " + backupZNode + " by " + serverName);
1399 }
1400
1401 activeMasterManager.setInfoPort(infoPort);
1402
1403 Threads.setDaemonThreadRunning(new Thread(new Runnable() {
1404 @Override
1405 public void run() {
1406 int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT,
1407 HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
1408
1409 if (conf.getBoolean(HConstants.MASTER_TYPE_BACKUP,
1410 HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {
1411 LOG.debug("HMaster started in backup mode. "
1412 + "Stalling until master znode is written.");
1413
1414
1415 while (!activeMasterManager.hasActiveMaster()) {
1416 LOG.debug("Waiting for master address ZNode to be written "
1417 + "(Also watching cluster state node)");
1418 Threads.sleep(timeout);
1419 }
1420 }
1421 MonitoredTask status = TaskMonitor.get().createStatus("Master startup");
1422 status.setDescription("Master startup");
1423 try {
1424 if (activeMasterManager.blockUntilBecomingActiveMaster(timeout, status)) {
1425 finishActiveMasterInitialization(status);
1426 }
1427 } catch (Throwable t) {
1428 status.setStatus("Failed to become active: " + t.getMessage());
1429 LOG.fatal("Failed to become active master", t);
1430
1431 if (t instanceof NoClassDefFoundError &&
1432 t.getMessage().contains("org/apache/hadoop/hdfs/protocol/FSConstants$SafeModeAction")) {
1433
1434 abort("HBase is having a problem with its Hadoop jars. You may need to "
1435 + "recompile HBase against Hadoop version "
1436 + org.apache.hadoop.util.VersionInfo.getVersion()
1437 + " or change your hadoop jars to start properly", t);
1438 } else {
1439 abort("Unhandled exception. Starting shutdown.", t);
1440 }
1441 } finally {
1442 status.cleanup();
1443 }
1444 }
1445 }, getServerName().toShortString() + ".activeMasterManager"));
1446 }
1447
1448 private void checkCompression(final HTableDescriptor htd)
1449 throws IOException {
1450 if (!this.masterCheckCompression) return;
1451 for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
1452 checkCompression(hcd);
1453 }
1454 }
1455
1456 private void checkCompression(final HColumnDescriptor hcd)
1457 throws IOException {
1458 if (!this.masterCheckCompression) return;
1459 CompressionTest.testCompression(hcd.getCompression());
1460 CompressionTest.testCompression(hcd.getCompactionCompression());
1461 }
1462
1463 private void checkEncryption(final Configuration conf, final HTableDescriptor htd)
1464 throws IOException {
1465 if (!this.masterCheckEncryption) return;
1466 for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
1467 checkEncryption(conf, hcd);
1468 }
1469 }
1470
1471 private void checkEncryption(final Configuration conf, final HColumnDescriptor hcd)
1472 throws IOException {
1473 if (!this.masterCheckEncryption) return;
1474 EncryptionTest.testEncryption(conf, hcd.getEncryptionType(), hcd.getEncryptionKey());
1475 }
1476
1477 private void checkClassLoading(final Configuration conf, final HTableDescriptor htd)
1478 throws IOException {
1479 RegionSplitPolicy.getSplitPolicyClass(htd, conf);
1480 RegionCoprocessorHost.testTableCoprocessorAttrs(conf, htd);
1481 }
1482
1483 private HRegionInfo[] getHRegionInfos(HTableDescriptor hTableDescriptor,
1484 byte[][] splitKeys) {
1485 long regionId = System.currentTimeMillis();
1486 HRegionInfo[] hRegionInfos = null;
1487 if (splitKeys == null || splitKeys.length == 0) {
1488 hRegionInfos = new HRegionInfo[]{new HRegionInfo(hTableDescriptor.getTableName(), null, null,
1489 false, regionId)};
1490 } else {
1491 int numRegions = splitKeys.length + 1;
1492 hRegionInfos = new HRegionInfo[numRegions];
1493 byte[] startKey = null;
1494 byte[] endKey = null;
1495 for (int i = 0; i < numRegions; i++) {
1496 endKey = (i == splitKeys.length) ? null : splitKeys[i];
1497 hRegionInfos[i] =
1498 new HRegionInfo(hTableDescriptor.getTableName(), startKey, endKey,
1499 false, regionId);
1500 startKey = endKey;
1501 }
1502 }
1503 return hRegionInfos;
1504 }
1505
1506 private static boolean isCatalogTable(final TableName tableName) {
1507 return tableName.equals(TableName.META_TABLE_NAME);
1508 }
1509
1510 @Override
1511 public void deleteTable(final TableName tableName) throws IOException {
1512 checkInitialized();
1513 if (cpHost != null) {
1514 cpHost.preDeleteTable(tableName);
1515 }
1516 LOG.info(getClientIdAuditPrefix() + " delete " + tableName);
1517 this.service.submit(new DeleteTableHandler(tableName, this, this).prepare());
1518 if (cpHost != null) {
1519 cpHost.postDeleteTable(tableName);
1520 }
1521 }
1522
1523 @Override
1524 public void truncateTable(TableName tableName, boolean preserveSplits) throws IOException {
1525 checkInitialized();
1526 if (cpHost != null) {
1527 cpHost.preTruncateTable(tableName);
1528 }
1529 LOG.info(getClientIdAuditPrefix() + " truncate " + tableName);
1530 TruncateTableHandler handler = new TruncateTableHandler(tableName, this, this, preserveSplits);
1531 handler.prepare();
1532 handler.process();
1533 if (cpHost != null) {
1534 cpHost.postTruncateTable(tableName);
1535 }
1536 }
1537
1538 @Override
1539 public void addColumn(final TableName tableName, final HColumnDescriptor columnDescriptor)
1540 throws IOException {
1541 checkInitialized();
1542 checkCompression(columnDescriptor);
1543 checkEncryption(conf, columnDescriptor);
1544 if (cpHost != null) {
1545 if (cpHost.preAddColumn(tableName, columnDescriptor)) {
1546 return;
1547 }
1548 }
1549
1550 new TableAddFamilyHandler(tableName, columnDescriptor, this, this).prepare().process();
1551 if (cpHost != null) {
1552 cpHost.postAddColumn(tableName, columnDescriptor);
1553 }
1554 }
1555
1556 @Override
1557 public void modifyColumn(TableName tableName, HColumnDescriptor descriptor)
1558 throws IOException {
1559 checkInitialized();
1560 checkCompression(descriptor);
1561 checkEncryption(conf, descriptor);
1562 if (cpHost != null) {
1563 if (cpHost.preModifyColumn(tableName, descriptor)) {
1564 return;
1565 }
1566 }
1567 LOG.info(getClientIdAuditPrefix() + " modify " + descriptor);
1568 new TableModifyFamilyHandler(tableName, descriptor, this, this)
1569 .prepare().process();
1570 if (cpHost != null) {
1571 cpHost.postModifyColumn(tableName, descriptor);
1572 }
1573 }
1574
1575 @Override
1576 public void deleteColumn(final TableName tableName, final byte[] columnName)
1577 throws IOException {
1578 checkInitialized();
1579 if (cpHost != null) {
1580 if (cpHost.preDeleteColumn(tableName, columnName)) {
1581 return;
1582 }
1583 }
1584 LOG.info(getClientIdAuditPrefix() + " delete " + Bytes.toString(columnName));
1585 new TableDeleteFamilyHandler(tableName, columnName, this, this).prepare().process();
1586 if (cpHost != null) {
1587 cpHost.postDeleteColumn(tableName, columnName);
1588 }
1589 }
1590
1591 @Override
1592 public void enableTable(final TableName tableName) throws IOException {
1593 checkInitialized();
1594 if (cpHost != null) {
1595 cpHost.preEnableTable(tableName);
1596 }
1597 LOG.info(getClientIdAuditPrefix() + " enable " + tableName);
1598 this.service.submit(new EnableTableHandler(this, tableName,
1599 assignmentManager, tableLockManager, false).prepare());
1600 if (cpHost != null) {
1601 cpHost.postEnableTable(tableName);
1602 }
1603 }
1604
1605 @Override
1606 public void disableTable(final TableName tableName) throws IOException {
1607 checkInitialized();
1608 if (cpHost != null) {
1609 cpHost.preDisableTable(tableName);
1610 }
1611 LOG.info(getClientIdAuditPrefix() + " disable " + tableName);
1612 this.service.submit(new DisableTableHandler(this, tableName,
1613 assignmentManager, tableLockManager, false).prepare());
1614 if (cpHost != null) {
1615 cpHost.postDisableTable(tableName);
1616 }
1617 }
1618
1619
1620
1621
1622
1623
1624
1625 @VisibleForTesting
1626 Pair<HRegionInfo, ServerName> getTableRegionForRow(
1627 final TableName tableName, final byte [] rowKey)
1628 throws IOException {
1629 final AtomicReference<Pair<HRegionInfo, ServerName>> result =
1630 new AtomicReference<Pair<HRegionInfo, ServerName>>(null);
1631
1632 MetaScannerVisitor visitor =
1633 new MetaScannerVisitorBase() {
1634 @Override
1635 public boolean processRow(Result data) throws IOException {
1636 if (data == null || data.size() <= 0) {
1637 return true;
1638 }
1639 Pair<HRegionInfo, ServerName> pair = HRegionInfo.getHRegionInfoAndServerName(data);
1640 if (pair == null) {
1641 return false;
1642 }
1643 if (!pair.getFirst().getTable().equals(tableName)) {
1644 return false;
1645 }
1646 result.set(pair);
1647 return true;
1648 }
1649 };
1650
1651 MetaScanner.metaScan(clusterConnection, visitor, tableName, rowKey, 1);
1652 return result.get();
1653 }
1654
1655 @Override
1656 public void modifyTable(final TableName tableName, final HTableDescriptor descriptor)
1657 throws IOException {
1658 checkInitialized();
1659 sanityCheckTableDescriptor(descriptor);
1660 if (cpHost != null) {
1661 cpHost.preModifyTable(tableName, descriptor);
1662 }
1663 LOG.info(getClientIdAuditPrefix() + " modify " + tableName);
1664 new ModifyTableHandler(tableName, descriptor, this, this).prepare().process();
1665 if (cpHost != null) {
1666 cpHost.postModifyTable(tableName, descriptor);
1667 }
1668 }
1669
1670 @Override
1671 public void checkTableModifiable(final TableName tableName)
1672 throws IOException, TableNotFoundException, TableNotDisabledException {
1673 if (isCatalogTable(tableName)) {
1674 throw new IOException("Can't modify catalog tables");
1675 }
1676 if (!MetaTableAccessor.tableExists(getConnection(), tableName)) {
1677 throw new TableNotFoundException(tableName);
1678 }
1679 if (!getAssignmentManager().getTableStateManager().
1680 isTableState(tableName, ZooKeeperProtos.Table.State.DISABLED)) {
1681 throw new TableNotDisabledException(tableName);
1682 }
1683 }
1684
1685
1686
1687
1688 public ClusterStatus getClusterStatus() throws InterruptedIOException {
1689
1690 List<String> backupMasterStrings;
1691 try {
1692 backupMasterStrings = ZKUtil.listChildrenNoWatch(this.zooKeeper,
1693 this.zooKeeper.backupMasterAddressesZNode);
1694 } catch (KeeperException e) {
1695 LOG.warn(this.zooKeeper.prefix("Unable to list backup servers"), e);
1696 backupMasterStrings = new ArrayList<String>(0);
1697 }
1698 List<ServerName> backupMasters = new ArrayList<ServerName>(
1699 backupMasterStrings.size());
1700 for (String s: backupMasterStrings) {
1701 try {
1702 byte [] bytes;
1703 try {
1704 bytes = ZKUtil.getData(this.zooKeeper, ZKUtil.joinZNode(
1705 this.zooKeeper.backupMasterAddressesZNode, s));
1706 } catch (InterruptedException e) {
1707 throw new InterruptedIOException();
1708 }
1709 if (bytes != null) {
1710 ServerName sn;
1711 try {
1712 sn = ServerName.parseFrom(bytes);
1713 } catch (DeserializationException e) {
1714 LOG.warn("Failed parse, skipping registering backup server", e);
1715 continue;
1716 }
1717 backupMasters.add(sn);
1718 }
1719 } catch (KeeperException e) {
1720 LOG.warn(this.zooKeeper.prefix("Unable to get information about " +
1721 "backup servers"), e);
1722 }
1723 }
1724 Collections.sort(backupMasters, new Comparator<ServerName>() {
1725 @Override
1726 public int compare(ServerName s1, ServerName s2) {
1727 return s1.getServerName().compareTo(s2.getServerName());
1728 }});
1729
1730 String clusterId = fileSystemManager != null ?
1731 fileSystemManager.getClusterId().toString() : null;
1732 Map<String, RegionState> regionsInTransition = assignmentManager != null ?
1733 assignmentManager.getRegionStates().getRegionsInTransition() : null;
1734 String[] coprocessors = cpHost != null ? getMasterCoprocessors() : null;
1735 boolean balancerOn = loadBalancerTracker != null ?
1736 loadBalancerTracker.isBalancerOn() : false;
1737 Map<ServerName, ServerLoad> onlineServers = null;
1738 Set<ServerName> deadServers = null;
1739 if (serverManager != null) {
1740 deadServers = serverManager.getDeadServers().copyServerNames();
1741 onlineServers = serverManager.getOnlineServers();
1742 }
1743 return new ClusterStatus(VersionInfo.getVersion(), clusterId,
1744 onlineServers, deadServers, serverName, backupMasters,
1745 regionsInTransition, coprocessors, balancerOn);
1746 }
1747
1748
1749
1750
1751
1752
1753
1754
1755 public static String getLoadedCoprocessors() {
1756 return CoprocessorHost.getLoadedCoprocessors().toString();
1757 }
1758
1759
1760
1761
1762 public long getMasterStartTime() {
1763 return startcode;
1764 }
1765
1766
1767
1768
1769 public long getMasterActiveTime() {
1770 return masterActiveTime;
1771 }
1772
1773 public int getRegionServerInfoPort(final ServerName sn) {
1774 RegionServerInfo info = this.regionServerTracker.getRegionServerInfo(sn);
1775 if (info == null || info.getInfoPort() == 0) {
1776 return conf.getInt(HConstants.REGIONSERVER_INFO_PORT,
1777 HConstants.DEFAULT_REGIONSERVER_INFOPORT);
1778 }
1779 return info.getInfoPort();
1780 }
1781
1782
1783
1784
1785 public String[] getMasterCoprocessors() {
1786 Set<String> masterCoprocessors = getMasterCoprocessorHost().getCoprocessors();
1787 return masterCoprocessors.toArray(new String[masterCoprocessors.size()]);
1788 }
1789
1790 @Override
1791 public void abort(final String msg, final Throwable t) {
1792 if (isAborted() || isStopped()) {
1793 return;
1794 }
1795 if (cpHost != null) {
1796
1797 LOG.fatal("Master server abort: loaded coprocessors are: " +
1798 getLoadedCoprocessors());
1799 }
1800 if (t != null) LOG.fatal(msg, t);
1801 stop(msg);
1802 }
1803
1804 @Override
1805 public ZooKeeperWatcher getZooKeeper() {
1806 return zooKeeper;
1807 }
1808
1809 @Override
1810 public MasterCoprocessorHost getMasterCoprocessorHost() {
1811 return cpHost;
1812 }
1813
1814 @Override
1815 public ServerName getServerName() {
1816 return this.serverName;
1817 }
1818
1819 @Override
1820 public AssignmentManager getAssignmentManager() {
1821 return this.assignmentManager;
1822 }
1823
1824 public MemoryBoundedLogMessageBuffer getRegionServerFatalLogBuffer() {
1825 return rsFatals;
1826 }
1827
1828 public void shutdown() {
1829 if (cpHost != null) {
1830 try {
1831 cpHost.preShutdown();
1832 } catch (IOException ioe) {
1833 LOG.error("Error call master coprocessor preShutdown()", ioe);
1834 }
1835 }
1836
1837 if (this.serverManager != null) {
1838 this.serverManager.shutdownCluster();
1839 }
1840 if (this.clusterStatusTracker != null){
1841 try {
1842 this.clusterStatusTracker.setClusterDown();
1843 } catch (KeeperException e) {
1844 LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
1845 }
1846 }
1847 }
1848
1849 public void stopMaster() {
1850 if (cpHost != null) {
1851 try {
1852 cpHost.preStopMaster();
1853 } catch (IOException ioe) {
1854 LOG.error("Error call master coprocessor preStopMaster()", ioe);
1855 }
1856 }
1857 stop("Stopped by " + Thread.currentThread().getName());
1858 }
1859
1860 void checkServiceStarted() throws ServerNotRunningYetException {
1861 if (!serviceStarted) {
1862 throw new ServerNotRunningYetException("Server is not running yet");
1863 }
1864 }
1865
1866 void checkInitialized() throws PleaseHoldException, ServerNotRunningYetException {
1867 checkServiceStarted();
1868 if (!this.initialized) {
1869 throw new PleaseHoldException("Master is initializing");
1870 }
1871 }
1872
1873 void checkNamespaceManagerReady() throws IOException {
1874 checkInitialized();
1875 if (tableNamespaceManager == null ||
1876 !tableNamespaceManager.isTableAvailableAndInitialized()) {
1877 throw new IOException("Table Namespace Manager not ready yet, try again later");
1878 }
1879 }
1880
1881
1882
1883
1884
1885
1886
1887
1888 public boolean isActiveMaster() {
1889 return isActiveMaster;
1890 }
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901 @Override
1902 public boolean isInitialized() {
1903 return initialized;
1904 }
1905
1906
1907
1908
1909
1910
1911 @Override
1912 public boolean isServerShutdownHandlerEnabled() {
1913 return this.serverShutdownHandlerEnabled;
1914 }
1915
1916
1917
1918
1919
1920 public boolean isInitializationStartsMetaRegionAssignment() {
1921 return this.initializationBeforeMetaAssignment;
1922 }
1923
1924 public void assignRegion(HRegionInfo hri) {
1925 assignmentManager.assign(hri, true);
1926 }
1927
1928
1929
1930
1931
1932
1933
1934 public double getAverageLoad() {
1935 if (this.assignmentManager == null) {
1936 return 0;
1937 }
1938
1939 RegionStates regionStates = this.assignmentManager.getRegionStates();
1940 if (regionStates == null) {
1941 return 0;
1942 }
1943 return regionStates.getAverageLoad();
1944 }
1945
1946 @Override
1947 public boolean registerService(Service instance) {
1948
1949
1950
1951 Descriptors.ServiceDescriptor serviceDesc = instance.getDescriptorForType();
1952 if (coprocessorServiceHandlers.containsKey(serviceDesc.getFullName())) {
1953 LOG.error("Coprocessor service "+serviceDesc.getFullName()+
1954 " already registered, rejecting request from "+instance
1955 );
1956 return false;
1957 }
1958
1959 coprocessorServiceHandlers.put(serviceDesc.getFullName(), instance);
1960 if (LOG.isDebugEnabled()) {
1961 LOG.debug("Registered master coprocessor service: service="+serviceDesc.getFullName());
1962 }
1963 return true;
1964 }
1965
1966
1967
1968
1969
1970
1971
1972 public static HMaster constructMaster(Class<? extends HMaster> masterClass,
1973 final Configuration conf, final CoordinatedStateManager cp) {
1974 try {
1975 Constructor<? extends HMaster> c =
1976 masterClass.getConstructor(Configuration.class, CoordinatedStateManager.class);
1977 return c.newInstance(conf, cp);
1978 } catch (InvocationTargetException ite) {
1979 Throwable target = ite.getTargetException() != null?
1980 ite.getTargetException(): ite;
1981 if (target.getCause() != null) target = target.getCause();
1982 throw new RuntimeException("Failed construction of Master: " +
1983 masterClass.toString(), target);
1984 } catch (Exception e) {
1985 throw new RuntimeException("Failed construction of Master: " +
1986 masterClass.toString() + ((e.getCause() != null)?
1987 e.getCause().getMessage(): ""), e);
1988 }
1989 }
1990
1991
1992
1993
1994 public static void main(String [] args) {
1995 VersionInfo.logVersion();
1996 new HMasterCommandLine(HMaster.class).doMain(args);
1997 }
1998
1999 public HFileCleaner getHFileCleaner() {
2000 return this.hfileCleaner;
2001 }
2002
2003
2004
2005
2006
2007 public SnapshotManager getSnapshotManagerForTesting() {
2008 return this.snapshotManager;
2009 }
2010
2011 @Override
2012 public void createNamespace(NamespaceDescriptor descriptor) throws IOException {
2013 TableName.isLegalNamespaceName(Bytes.toBytes(descriptor.getName()));
2014 checkNamespaceManagerReady();
2015 if (cpHost != null) {
2016 if (cpHost.preCreateNamespace(descriptor)) {
2017 return;
2018 }
2019 }
2020 LOG.info(getClientIdAuditPrefix() + " creating " + descriptor);
2021 tableNamespaceManager.create(descriptor);
2022 if (cpHost != null) {
2023 cpHost.postCreateNamespace(descriptor);
2024 }
2025 }
2026
2027 @Override
2028 public void modifyNamespace(NamespaceDescriptor descriptor) throws IOException {
2029 TableName.isLegalNamespaceName(Bytes.toBytes(descriptor.getName()));
2030 checkNamespaceManagerReady();
2031 if (cpHost != null) {
2032 if (cpHost.preModifyNamespace(descriptor)) {
2033 return;
2034 }
2035 }
2036 LOG.info(getClientIdAuditPrefix() + " modify " + descriptor);
2037 tableNamespaceManager.update(descriptor);
2038 if (cpHost != null) {
2039 cpHost.postModifyNamespace(descriptor);
2040 }
2041 }
2042
2043 @Override
2044 public void deleteNamespace(String name) throws IOException {
2045 checkNamespaceManagerReady();
2046 if (cpHost != null) {
2047 if (cpHost.preDeleteNamespace(name)) {
2048 return;
2049 }
2050 }
2051 LOG.info(getClientIdAuditPrefix() + " delete " + name);
2052 tableNamespaceManager.remove(name);
2053 if (cpHost != null) {
2054 cpHost.postDeleteNamespace(name);
2055 }
2056 }
2057
2058
2059
2060
2061
2062
2063
2064
2065 private void ensureNamespaceExists(final String name)
2066 throws IOException, NamespaceNotFoundException {
2067 checkNamespaceManagerReady();
2068 NamespaceDescriptor nsd = tableNamespaceManager.get(name);
2069 if (nsd == null) {
2070 throw new NamespaceNotFoundException(name);
2071 }
2072 }
2073
2074 @Override
2075 public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException {
2076 checkNamespaceManagerReady();
2077
2078 if (cpHost != null) {
2079 cpHost.preGetNamespaceDescriptor(name);
2080 }
2081
2082 NamespaceDescriptor nsd = tableNamespaceManager.get(name);
2083 if (nsd == null) {
2084 throw new NamespaceNotFoundException(name);
2085 }
2086
2087 if (cpHost != null) {
2088 cpHost.postGetNamespaceDescriptor(nsd);
2089 }
2090
2091 return nsd;
2092 }
2093
2094 @Override
2095 public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException {
2096 checkNamespaceManagerReady();
2097
2098 final List<NamespaceDescriptor> descriptors = new ArrayList<NamespaceDescriptor>();
2099 boolean bypass = false;
2100 if (cpHost != null) {
2101 bypass = cpHost.preListNamespaceDescriptors(descriptors);
2102 }
2103
2104 if (!bypass) {
2105 descriptors.addAll(tableNamespaceManager.list());
2106
2107 if (cpHost != null) {
2108 cpHost.postListNamespaceDescriptors(descriptors);
2109 }
2110 }
2111 return descriptors;
2112 }
2113
2114 @Override
2115 public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
2116 ensureNamespaceExists(name);
2117 return listTableDescriptors(name, null, null, true);
2118 }
2119
2120 @Override
2121 public List<TableName> listTableNamesByNamespace(String name) throws IOException {
2122 ensureNamespaceExists(name);
2123 return listTableNames(name, null, true);
2124 }
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135 public List<HTableDescriptor> listTableDescriptors(final String namespace, final String regex,
2136 final List<TableName> tableNameList, final boolean includeSysTables)
2137 throws IOException {
2138 final List<HTableDescriptor> descriptors = new ArrayList<HTableDescriptor>();
2139
2140 boolean bypass = false;
2141 if (cpHost != null) {
2142 bypass = cpHost.preGetTableDescriptors(tableNameList, descriptors);
2143
2144 bypass |= cpHost.preGetTableDescriptors(tableNameList, descriptors, regex);
2145 }
2146
2147 if (!bypass) {
2148 if (tableNameList == null || tableNameList.size() == 0) {
2149
2150 Collection<HTableDescriptor> htds;
2151 if (namespace != null && namespace.length() > 0) {
2152 htds = tableDescriptors.getByNamespace(namespace).values();
2153 } else {
2154 htds = tableDescriptors.getAll().values();
2155 }
2156
2157 for (HTableDescriptor desc: htds) {
2158 if (includeSysTables || !desc.getTableName().isSystemTable()) {
2159 descriptors.add(desc);
2160 }
2161 }
2162 } else {
2163 for (TableName s: tableNameList) {
2164 HTableDescriptor desc = tableDescriptors.get(s);
2165 if (desc != null) {
2166 descriptors.add(desc);
2167 }
2168 }
2169 }
2170
2171
2172 if (regex != null) {
2173 filterTablesByRegex(descriptors, Pattern.compile(regex));
2174 }
2175
2176 if (cpHost != null) {
2177 cpHost.postGetTableDescriptors(descriptors);
2178
2179 cpHost.postGetTableDescriptors(tableNameList, descriptors, regex);
2180 }
2181 }
2182 return descriptors;
2183 }
2184
2185
2186
2187
2188
2189
2190
2191
2192 public List<TableName> listTableNames(final String namespace, final String regex,
2193 final boolean includeSysTables) throws IOException {
2194 final List<HTableDescriptor> descriptors = new ArrayList<HTableDescriptor>();
2195
2196 boolean bypass = false;
2197 if (cpHost != null) {
2198 bypass = cpHost.preGetTableNames(descriptors, regex);
2199 }
2200
2201 if (!bypass) {
2202
2203 Collection<HTableDescriptor> htds;
2204 if (namespace != null && namespace.length() > 0) {
2205 htds = tableDescriptors.getByNamespace(namespace).values();
2206 } else {
2207 htds = tableDescriptors.getAll().values();
2208 }
2209
2210 for (HTableDescriptor htd: htds) {
2211 if (includeSysTables || !htd.getTableName().isSystemTable()) {
2212 descriptors.add(htd);
2213 }
2214 }
2215
2216
2217 if (regex != null) {
2218 filterTablesByRegex(descriptors, Pattern.compile(regex));
2219 }
2220
2221 if (cpHost != null) {
2222 cpHost.postGetTableNames(descriptors, regex);
2223 }
2224 }
2225
2226 List<TableName> result = new ArrayList<TableName>(descriptors.size());
2227 for (HTableDescriptor htd: descriptors) {
2228 result.add(htd.getTableName());
2229 }
2230 return result;
2231 }
2232
2233
2234
2235
2236
2237
2238
2239 private static void filterTablesByRegex(final Collection<HTableDescriptor> descriptors,
2240 final Pattern pattern) {
2241 final String defaultNS = NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR;
2242 Iterator<HTableDescriptor> itr = descriptors.iterator();
2243 while (itr.hasNext()) {
2244 HTableDescriptor htd = itr.next();
2245 String tableName = htd.getTableName().getNameAsString();
2246 boolean matched = pattern.matcher(tableName).matches();
2247 if (!matched && htd.getTableName().getNamespaceAsString().equals(defaultNS)) {
2248 matched = pattern.matcher(defaultNS + TableName.NAMESPACE_DELIM + tableName).matches();
2249 }
2250 if (!matched) {
2251 itr.remove();
2252 }
2253 }
2254 }
2255 }