View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.HashSet;
24  import java.util.List;
25  import java.util.NavigableMap;
26  import java.util.Set;
27  import java.util.concurrent.locks.Lock;
28  import java.util.concurrent.locks.ReentrantLock;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.classification.InterfaceAudience;
33  import org.apache.hadoop.conf.Configuration;
34  import org.apache.hadoop.fs.FileStatus;
35  import org.apache.hadoop.fs.FileSystem;
36  import org.apache.hadoop.fs.Path;
37  import org.apache.hadoop.fs.PathFilter;
38  import org.apache.hadoop.hbase.ClusterId;
39  import org.apache.hadoop.hbase.TableName;
40  import org.apache.hadoop.hbase.HColumnDescriptor;
41  import org.apache.hadoop.hbase.HConstants;
42  import org.apache.hadoop.hbase.HRegionInfo;
43  import org.apache.hadoop.hbase.HTableDescriptor;
44  import org.apache.hadoop.hbase.InvalidFamilyOperationException;
45  import org.apache.hadoop.hbase.RemoteExceptionHandler;
46  import org.apache.hadoop.hbase.Server;
47  import org.apache.hadoop.hbase.ServerName;
48  import org.apache.hadoop.hbase.backup.HFileArchiver;
49  import org.apache.hadoop.hbase.catalog.MetaReader;
50  import org.apache.hadoop.hbase.client.Result;
51  import org.apache.hadoop.hbase.exceptions.DeserializationException;
52  import org.apache.hadoop.hbase.fs.HFileSystem;
53  import org.apache.hadoop.hbase.regionserver.HRegion;
54  import org.apache.hadoop.hbase.regionserver.wal.HLog;
55  import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
56  import org.apache.hadoop.hbase.util.Bytes;
57  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
58  import org.apache.hadoop.hbase.util.FSTableDescriptors;
59  import org.apache.hadoop.hbase.util.FSUtils;
60  import org.apache.zookeeper.KeeperException;
61  
62  /**
63   * This class abstracts a bunch of operations the HMaster needs to interact with
64   * the underlying file system, including splitting log files, checking file
65   * system status, etc.
66   */
67  @InterfaceAudience.Private
68  public class MasterFileSystem {
69    private static final Log LOG = LogFactory.getLog(MasterFileSystem.class.getName());
70    // HBase configuration
71    Configuration conf;
72    // master status
73    Server master;
74    // metrics for master
75    private final MetricsMasterFileSystem metricsMasterFilesystem = new MetricsMasterFileSystem();
76    // Persisted unique cluster ID
77    private ClusterId clusterId;
78    // Keep around for convenience.
79    private final FileSystem fs;
80    // Is the fileystem ok?
81    private volatile boolean fsOk = true;
82    // The Path to the old logs dir
83    private final Path oldLogDir;
84    // root hbase directory on the FS
85    private final Path rootdir;
86    // hbase temp directory used for table construction and deletion
87    private final Path tempdir;
88    // create the split log lock
89    final Lock splitLogLock = new ReentrantLock();
90    final boolean distributedLogReplay;
91    final SplitLogManager splitLogManager;
92    private final MasterServices services;
93  
94    final static PathFilter META_FILTER = new PathFilter() {
95      public boolean accept(Path p) {
96        return HLogUtil.isMetaFile(p);
97      }
98    };
99  
100   final static PathFilter NON_META_FILTER = new PathFilter() {
101     public boolean accept(Path p) {
102       return !HLogUtil.isMetaFile(p);
103     }
104   };
105 
106   public MasterFileSystem(Server master, MasterServices services, boolean masterRecovery)
107   throws IOException {
108     this.conf = master.getConfiguration();
109     this.master = master;
110     this.services = services;
111     // Set filesystem to be that of this.rootdir else we get complaints about
112     // mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is
113     // default localfs.  Presumption is that rootdir is fully-qualified before
114     // we get to here with appropriate fs scheme.
115     this.rootdir = FSUtils.getRootDir(conf);
116     this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
117     // Cover both bases, the old way of setting default fs and the new.
118     // We're supposed to run on 0.20 and 0.21 anyways.
119     this.fs = this.rootdir.getFileSystem(conf);
120     FSUtils.setFsDefault(conf, new Path(this.fs.getUri()));
121     // make sure the fs has the same conf
122     fs.setConf(conf);
123     this.distributedLogReplay = this.conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY,
124       HConstants.DEFAULT_DISTRIBUTED_LOG_REPLAY_CONFIG);
125     // setup the filesystem variable
126     // set up the archived logs path
127     this.oldLogDir = createInitialFileSystemLayout();
128     HFileSystem.addLocationsOrderInterceptor(conf);
129     this.splitLogManager = new SplitLogManager(master.getZooKeeper(),
130       master.getConfiguration(), master, services,
131       master.getServerName(), masterRecovery);
132   }
133 
134   /**
135    * Create initial layout in filesystem.
136    * <ol>
137    * <li>Check if the meta region exists and is readable, if not create it.
138    * Create hbase.version and the hbase:meta directory if not one.
139    * </li>
140    * <li>Create a log archive directory for RS to put archived logs</li>
141    * </ol>
142    * Idempotent.
143    */
144   private Path createInitialFileSystemLayout() throws IOException {
145     // check if the root directory exists
146     checkRootDir(this.rootdir, conf, this.fs);
147 
148     // check if temp directory exists and clean it
149     checkTempDir(this.tempdir, conf, this.fs);
150 
151     Path oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
152 
153     // Make sure the region servers can archive their old logs
154     if(!this.fs.exists(oldLogDir)) {
155       this.fs.mkdirs(oldLogDir);
156     }
157 
158     return oldLogDir;
159   }
160 
161   public FileSystem getFileSystem() {
162     return this.fs;
163   }
164 
165   /**
166    * Get the directory where old logs go
167    * @return the dir
168    */
169   public Path getOldLogDir() {
170     return this.oldLogDir;
171   }
172 
173   /**
174    * Checks to see if the file system is still accessible.
175    * If not, sets closed
176    * @return false if file system is not available
177    */
178   public boolean checkFileSystem() {
179     if (this.fsOk) {
180       try {
181         FSUtils.checkFileSystemAvailable(this.fs);
182         FSUtils.checkDfsSafeMode(this.conf);
183       } catch (IOException e) {
184         master.abort("Shutting down HBase cluster: file system not available", e);
185         this.fsOk = false;
186       }
187     }
188     return this.fsOk;
189   }
190 
191   /**
192    * @return HBase root dir.
193    */
194   public Path getRootDir() {
195     return this.rootdir;
196   }
197 
198   /**
199    * @return HBase temp dir.
200    */
201   public Path getTempDir() {
202     return this.tempdir;
203   }
204 
205   /**
206    * @return The unique identifier generated for this cluster
207    */
208   public ClusterId getClusterId() {
209     return clusterId;
210   }
211 
212   /**
213    * Inspect the log directory to find dead servers which need recovery work
214    * @return A set of ServerNames which aren't running but still have WAL files left in file system
215    */
216   Set<ServerName> getFailedServersFromLogFolders() {
217     boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
218       HLog.SPLIT_SKIP_ERRORS_DEFAULT);
219 
220     Set<ServerName> serverNames = new HashSet<ServerName>();
221     Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
222 
223     do {
224       if (master.isStopped()) {
225         LOG.warn("Master stopped while trying to get failed servers.");
226         break;
227       }
228       try {
229         if (!this.fs.exists(logsDirPath)) return serverNames;
230         FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
231         // Get online servers after getting log folders to avoid log folder deletion of newly
232         // checked in region servers . see HBASE-5916
233         Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
234             .keySet();
235 
236         if (logFolders == null || logFolders.length == 0) {
237           LOG.debug("No log files to split, proceeding...");
238           return serverNames;
239         }
240         for (FileStatus status : logFolders) {
241           String sn = status.getPath().getName();
242           // truncate splitting suffix if present (for ServerName parsing)
243           if (sn.endsWith(HLog.SPLITTING_EXT)) {
244             sn = sn.substring(0, sn.length() - HLog.SPLITTING_EXT.length());
245           }
246           ServerName serverName = ServerName.parseServerName(sn);
247           if (!onlineServers.contains(serverName)) {
248             LOG.info("Log folder " + status.getPath() + " doesn't belong "
249                 + "to a known region server, splitting");
250             serverNames.add(serverName);
251           } else {
252             LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");
253           }
254         }
255         retrySplitting = false;
256       } catch (IOException ioe) {
257         LOG.warn("Failed getting failed servers to be recovered.", ioe);
258         if (!checkFileSystem()) {
259           LOG.warn("Bad Filesystem, exiting");
260           Runtime.getRuntime().halt(1);
261         }
262         try {
263           if (retrySplitting) {
264             Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));
265           }
266         } catch (InterruptedException e) {
267           LOG.warn("Interrupted, aborting since cannot return w/o splitting");
268           Thread.currentThread().interrupt();
269           retrySplitting = false;
270           Runtime.getRuntime().halt(1);
271         }
272       }
273     } while (retrySplitting);
274 
275     return serverNames;
276   }
277 
278   public void splitLog(final ServerName serverName) throws IOException {
279     Set<ServerName> serverNames = new HashSet<ServerName>();
280     serverNames.add(serverName);
281     splitLog(serverNames);
282   }
283 
284   /**
285    * Specialized method to handle the splitting for meta HLog
286    * @param serverName
287    * @throws IOException
288    */
289   public void splitMetaLog(final ServerName serverName) throws IOException {
290     Set<ServerName> serverNames = new HashSet<ServerName>();
291     serverNames.add(serverName);
292     splitMetaLog(serverNames);
293   }
294 
295   /**
296    * Specialized method to handle the splitting for meta HLog
297    * @param serverNames
298    * @throws IOException
299    */
300   public void splitMetaLog(final Set<ServerName> serverNames) throws IOException {
301     splitLog(serverNames, META_FILTER);
302   }
303 
304   private List<Path> getLogDirs(final Set<ServerName> serverNames) throws IOException {
305     List<Path> logDirs = new ArrayList<Path>();
306     boolean needReleaseLock = false;
307     if (!this.services.isInitialized()) {
308       // during master initialization, we could have multiple places splitting a same wal
309       this.splitLogLock.lock();
310       needReleaseLock = true;
311     }
312     try {
313       for (ServerName serverName : serverNames) {
314         Path logDir = new Path(this.rootdir, HLogUtil.getHLogDirectoryName(serverName.toString()));
315         Path splitDir = logDir.suffix(HLog.SPLITTING_EXT);
316         // Rename the directory so a rogue RS doesn't create more HLogs
317         if (fs.exists(logDir)) {
318           if (!this.fs.rename(logDir, splitDir)) {
319             throw new IOException("Failed fs.rename for log split: " + logDir);
320           }
321           logDir = splitDir;
322           LOG.debug("Renamed region directory: " + splitDir);
323         } else if (!fs.exists(splitDir)) {
324           LOG.info("Log dir for server " + serverName + " does not exist");
325           continue;
326         }
327         logDirs.add(splitDir);
328       }
329     } finally {
330       if (needReleaseLock) {
331         this.splitLogLock.unlock();
332       }
333     }
334     return logDirs;
335   }
336 
337   /**
338    * Mark regions in recovering state when distributedLogReplay are set true
339    * @param serverNames Set of ServerNames to be replayed wals in order to recover changes contained
340    *          in them
341    * @throws IOException
342    */
343   public void prepareLogReplay(Set<ServerName> serverNames) throws IOException {
344     if (!this.distributedLogReplay) {
345       return;
346     }
347     // mark regions in recovering state
348     for (ServerName serverName : serverNames) {
349       NavigableMap<HRegionInfo, Result> regions = this.getServerUserRegions(serverName);
350       if (regions == null) {
351         continue;
352       }
353       try {
354         this.splitLogManager.markRegionsRecoveringInZK(serverName, regions.keySet());
355       } catch (KeeperException e) {
356         throw new IOException(e);
357       }
358     }
359   }
360 
361   /**
362    * Mark regions in recovering state when distributedLogReplay are set true
363    * @param serverName Failed region server whose wals to be replayed
364    * @param regions Set of regions to be recovered
365    * @throws IOException
366    */
367   public void prepareLogReplay(ServerName serverName, Set<HRegionInfo> regions) throws IOException {
368     if (!this.distributedLogReplay) {
369       return;
370     }
371     // mark regions in recovering state
372     if (regions == null || regions.isEmpty()) {
373       return;
374     }
375     try {
376       this.splitLogManager.markRegionsRecoveringInZK(serverName, regions);
377     } catch (KeeperException e) {
378       throw new IOException(e);
379     }
380   }
381 
382   public void splitLog(final Set<ServerName> serverNames) throws IOException {
383     splitLog(serverNames, NON_META_FILTER);
384   }
385 
386   /**
387    * Wrapper function on {@link SplitLogManager#removeStaleRecoveringRegionsFromZK(Set)}
388    * @param failedServers
389    * @throws KeeperException
390    */
391   void removeStaleRecoveringRegionsFromZK(final Set<ServerName> failedServers)
392       throws KeeperException {
393     this.splitLogManager.removeStaleRecoveringRegionsFromZK(failedServers);
394   }
395 
396   /**
397    * This method is the base split method that splits HLog files matching a filter. Callers should
398    * pass the appropriate filter for meta and non-meta HLogs.
399    * @param serverNames
400    * @param filter
401    * @throws IOException
402    */
403   public void splitLog(final Set<ServerName> serverNames, PathFilter filter) throws IOException {
404     long splitTime = 0, splitLogSize = 0;
405     List<Path> logDirs = getLogDirs(serverNames);
406 
407     splitLogManager.handleDeadWorkers(serverNames);
408     splitTime = EnvironmentEdgeManager.currentTimeMillis();
409     splitLogSize = splitLogManager.splitLogDistributed(serverNames, logDirs, filter);
410     splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
411 
412     if (this.metricsMasterFilesystem != null) {
413       if (filter == META_FILTER) {
414         this.metricsMasterFilesystem.addMetaWALSplit(splitTime, splitLogSize);
415       } else {
416         this.metricsMasterFilesystem.addSplit(splitTime, splitLogSize);
417       }
418     }
419   }
420 
421   /**
422    * Get the rootdir.  Make sure its wholesome and exists before returning.
423    * @param rd
424    * @param c
425    * @param fs
426    * @return hbase.rootdir (after checks for existence and bootstrapping if
427    * needed populating the directory with necessary bootup files).
428    * @throws IOException
429    */
430   @SuppressWarnings("deprecation")
431   private Path checkRootDir(final Path rd, final Configuration c,
432     final FileSystem fs)
433   throws IOException {
434     // If FS is in safe mode wait till out of it.
435     FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
436     // Filesystem is good. Go ahead and check for hbase.rootdir.
437     try {
438       if (!fs.exists(rd)) {
439         fs.mkdirs(rd);
440         // DFS leaves safe mode with 0 DNs when there are 0 blocks.
441         // We used to handle this by checking the current DN count and waiting until
442         // it is nonzero. With security, the check for datanode count doesn't work --
443         // it is a privileged op. So instead we adopt the strategy of the jobtracker
444         // and simply retry file creation during bootstrap indefinitely. As soon as
445         // there is one datanode it will succeed. Permission problems should have
446         // already been caught by mkdirs above.
447         FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
448           10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
449             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
450       } else {
451         if (!fs.isDirectory(rd)) {
452           throw new IllegalArgumentException(rd.toString() + " is not a directory");
453         }
454         // as above
455         FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
456           10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
457             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
458       }
459     } catch (DeserializationException de) {
460       LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
461       IOException ioe = new IOException();
462       ioe.initCause(de);
463       throw ioe;
464     } catch (IllegalArgumentException iae) {
465       LOG.fatal("Please fix invalid configuration for "
466         + HConstants.HBASE_DIR + " " + rd.toString(), iae);
467       throw iae;
468     }
469     // Make sure cluster ID exists
470     if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
471         HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
472       FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
473     }
474     clusterId = FSUtils.getClusterId(fs, rd);
475 
476     // Make sure the meta region directory exists!
477     if (!FSUtils.metaRegionExists(fs, rd)) {
478       bootstrap(rd, c);
479     } else {
480       // Migrate table descriptor files if necessary
481       org.apache.hadoop.hbase.util.FSTableDescriptorMigrationToSubdir
482         .migrateFSTableDescriptorsIfNecessary(fs, rd);
483     }
484       
485     // Create tableinfo-s for hbase:meta if not already there.
486     new FSTableDescriptors(fs, rd).createTableDescriptor(HTableDescriptor.META_TABLEDESC);
487 
488     return rd;
489   }
490 
491   /**
492    * Make sure the hbase temp directory exists and is empty.
493    * NOTE that this method is only executed once just after the master becomes the active one.
494    */
495   private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
496       throws IOException {
497     // If the temp directory exists, clear the content (left over, from the previous run)
498     if (fs.exists(tmpdir)) {
499       // Archive table in temp, maybe left over from failed deletion,
500       // if not the cleaner will take care of them.
501       for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
502         for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
503           HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
504         }
505       }
506       if (!fs.delete(tmpdir, true)) {
507         throw new IOException("Unable to clean the temp directory: " + tmpdir);
508       }
509     }
510 
511     // Create the temp directory
512     if (!fs.mkdirs(tmpdir)) {
513       throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
514     }
515   }
516 
517   private static void bootstrap(final Path rd, final Configuration c)
518   throws IOException {
519     LOG.info("BOOTSTRAP: creating hbase:meta region");
520     try {
521       // Bootstrapping, make sure blockcache is off.  Else, one will be
522       // created here in bootstrap and it'll need to be cleaned up.  Better to
523       // not make it in first place.  Turn off block caching for bootstrap.
524       // Enable after.
525       HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
526       setInfoFamilyCachingForMeta(false);
527       HRegion meta = HRegion.createHRegion(metaHRI, rd, c,
528           HTableDescriptor.META_TABLEDESC);
529       setInfoFamilyCachingForMeta(true);
530       HRegion.closeHRegion(meta);
531     } catch (IOException e) {
532       e = RemoteExceptionHandler.checkIOException(e);
533       LOG.error("bootstrap", e);
534       throw e;
535     }
536   }
537 
538   /**
539    * Enable in memory caching for hbase:meta
540    */
541   public static void setInfoFamilyCachingForMeta(final boolean b) {
542     for (HColumnDescriptor hcd:
543         HTableDescriptor.META_TABLEDESC.getColumnFamilies()) {
544       if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
545         hcd.setBlockCacheEnabled(b);
546         hcd.setInMemory(b);
547       }
548     }
549   }
550 
551 
552   public void deleteRegion(HRegionInfo region) throws IOException {
553     HFileArchiver.archiveRegion(conf, fs, region);
554   }
555 
556   public void deleteTable(TableName tableName) throws IOException {
557     fs.delete(FSUtils.getTableDir(rootdir, tableName), true);
558   }
559 
560   /**
561    * Move the specified table to the hbase temp directory
562    * @param tableName Table name to move
563    * @return The temp location of the table moved
564    * @throws IOException in case of file-system failure
565    */
566   public Path moveTableToTemp(TableName tableName) throws IOException {
567     Path srcPath = FSUtils.getTableDir(rootdir, tableName);
568     Path tempPath = FSUtils.getTableDir(this.tempdir, tableName);
569 
570     // Ensure temp exists
571     if (!fs.exists(tempPath.getParent()) && !fs.mkdirs(tempPath.getParent())) {
572       throw new IOException("HBase temp directory '" + tempPath.getParent() + "' creation failure.");
573     }
574 
575     if (!fs.rename(srcPath, tempPath)) {
576       throw new IOException("Unable to move '" + srcPath + "' to temp '" + tempPath + "'");
577     }
578 
579     return tempPath;
580   }
581 
582   public void updateRegionInfo(HRegionInfo region) {
583     // TODO implement this.  i think this is currently broken in trunk i don't
584     //      see this getting updated.
585     //      @see HRegion.checkRegioninfoOnFilesystem()
586   }
587 
588   public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
589       throws IOException {
590     // archive family store files
591     Path tableDir = FSUtils.getTableDir(rootdir, region.getTable());
592     HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
593 
594     // delete the family folder
595     Path familyDir = new Path(tableDir,
596       new Path(region.getEncodedName(), Bytes.toString(familyName)));
597     if (fs.delete(familyDir, true) == false) {
598       throw new IOException("Could not delete family "
599           + Bytes.toString(familyName) + " from FileSystem for region "
600           + region.getRegionNameAsString() + "(" + region.getEncodedName()
601           + ")");
602     }
603   }
604 
605   public void stop() {
606     if (splitLogManager != null) {
607       this.splitLogManager.stop();
608     }
609   }
610 
611   /**
612    * Delete column of a table
613    * @param tableName
614    * @param familyName
615    * @return Modified HTableDescriptor with requested column deleted.
616    * @throws IOException
617    */
618   public HTableDescriptor deleteColumn(TableName tableName, byte[] familyName)
619       throws IOException {
620     LOG.info("DeleteColumn. Table = " + tableName
621         + " family = " + Bytes.toString(familyName));
622     HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
623     htd.removeFamily(familyName);
624     this.services.getTableDescriptors().add(htd);
625     return htd;
626   }
627 
628   /**
629    * Modify Column of a table
630    * @param tableName
631    * @param hcd HColumnDesciptor
632    * @return Modified HTableDescriptor with the column modified.
633    * @throws IOException
634    */
635   public HTableDescriptor modifyColumn(TableName tableName, HColumnDescriptor hcd)
636       throws IOException {
637     LOG.info("AddModifyColumn. Table = " + tableName
638         + " HCD = " + hcd.toString());
639 
640     HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
641     byte [] familyName = hcd.getName();
642     if(!htd.hasFamily(familyName)) {
643       throw new InvalidFamilyOperationException("Family '" +
644         Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
645     }
646     htd.addFamily(hcd);
647     this.services.getTableDescriptors().add(htd);
648     return htd;
649   }
650 
651   /**
652    * Add column to a table
653    * @param tableName
654    * @param hcd
655    * @return Modified HTableDescriptor with new column added.
656    * @throws IOException
657    */
658   public HTableDescriptor addColumn(TableName tableName, HColumnDescriptor hcd)
659       throws IOException {
660     LOG.info("AddColumn. Table = " + tableName + " HCD = " +
661       hcd.toString());
662     HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
663     if (htd == null) {
664       throw new InvalidFamilyOperationException("Family '" +
665         hcd.getNameAsString() + "' cannot be modified as HTD is null");
666     }
667     htd.addFamily(hcd);
668     this.services.getTableDescriptors().add(htd);
669     return htd;
670   }
671 
672   private NavigableMap<HRegionInfo, Result> getServerUserRegions(ServerName serverName)
673       throws IOException {
674     if (!this.master.isStopped()) {
675       try {
676         this.master.getCatalogTracker().waitForMeta();
677         return MetaReader.getServerUserRegions(this.master.getCatalogTracker(), serverName);
678       } catch (InterruptedException e) {
679         Thread.currentThread().interrupt();
680         throw new IOException("Interrupted", e);
681       }
682     }
683     return null;
684   }
685 }