View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.List;
24  import java.util.Set;
25  import java.util.concurrent.locks.Lock;
26  import java.util.concurrent.locks.ReentrantLock;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.classification.InterfaceAudience;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.fs.PathFilter;
36  import org.apache.hadoop.hbase.ClusterId;
37  import org.apache.hadoop.hbase.exceptions.DeserializationException;
38  import org.apache.hadoop.hbase.HColumnDescriptor;
39  import org.apache.hadoop.hbase.HConstants;
40  import org.apache.hadoop.hbase.HRegionInfo;
41  import org.apache.hadoop.hbase.HTableDescriptor;
42  import org.apache.hadoop.hbase.exceptions.InvalidFamilyOperationException;
43  import org.apache.hadoop.hbase.RemoteExceptionHandler;
44  import org.apache.hadoop.hbase.Server;
45  import org.apache.hadoop.hbase.ServerName;
46  import org.apache.hadoop.hbase.backup.HFileArchiver;
47  import org.apache.hadoop.hbase.fs.HFileSystem;
48  import org.apache.hadoop.hbase.regionserver.HRegion;
49  import org.apache.hadoop.hbase.regionserver.wal.HLog;
50  import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter;
51  import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
52  import org.apache.hadoop.hbase.exceptions.OrphanHLogAfterSplitException;
53  import org.apache.hadoop.hbase.util.Bytes;
54  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
55  import org.apache.hadoop.hbase.util.FSTableDescriptors;
56  import org.apache.hadoop.hbase.util.FSUtils;
57  
58  /**
59   * This class abstracts a bunch of operations the HMaster needs to interact with
60   * the underlying file system, including splitting log files, checking file
61   * system status, etc.
62   */
63  @InterfaceAudience.Private
64  public class MasterFileSystem {
65    private static final Log LOG = LogFactory.getLog(MasterFileSystem.class.getName());
66    // HBase configuration
67    Configuration conf;
68    // master status
69    Server master;
70    // metrics for master
71    MetricsMaster metricsMaster;
72    // Persisted unique cluster ID
73    private ClusterId clusterId;
74    // Keep around for convenience.
75    private final FileSystem fs;
76    // Is the fileystem ok?
77    private volatile boolean fsOk = true;
78    // The Path to the old logs dir
79    private final Path oldLogDir;
80    // root hbase directory on the FS
81    private final Path rootdir;
82    // hbase temp directory used for table construction and deletion
83    private final Path tempdir;
84    // create the split log lock
85    final Lock splitLogLock = new ReentrantLock();
86    final boolean distributedLogSplitting;
87    final SplitLogManager splitLogManager;
88    private final MasterServices services;
89  
90    private final static PathFilter META_FILTER = new PathFilter() {
91      public boolean accept(Path p) {
92        return HLogUtil.isMetaFile(p);
93      }
94    };
95  
96    private final static PathFilter NON_META_FILTER = new PathFilter() {
97      public boolean accept(Path p) {
98        return !HLogUtil.isMetaFile(p);
99      }
100   };
101 
102   public MasterFileSystem(Server master, MasterServices services,
103       MetricsMaster metricsMaster, boolean masterRecovery)
104   throws IOException {
105     this.conf = master.getConfiguration();
106     this.master = master;
107     this.services = services;
108     this.metricsMaster = metricsMaster;
109     // Set filesystem to be that of this.rootdir else we get complaints about
110     // mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is
111     // default localfs.  Presumption is that rootdir is fully-qualified before
112     // we get to here with appropriate fs scheme.
113     this.rootdir = FSUtils.getRootDir(conf);
114     this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
115     // Cover both bases, the old way of setting default fs and the new.
116     // We're supposed to run on 0.20 and 0.21 anyways.
117     this.fs = this.rootdir.getFileSystem(conf);
118     FSUtils.setFsDefault(conf, new Path(this.fs.getUri()));
119     // make sure the fs has the same conf
120     fs.setConf(conf);
121     this.distributedLogSplitting =
122       conf.getBoolean(HConstants.DISTRIBUTED_LOG_SPLITTING_KEY, true);
123     if (this.distributedLogSplitting) {
124       this.splitLogManager = new SplitLogManager(master.getZooKeeper(),
125           master.getConfiguration(), master, services, master.getServerName());
126       this.splitLogManager.finishInitialization(masterRecovery);
127     } else {
128       this.splitLogManager = null;
129     }
130     // setup the filesystem variable
131     // set up the archived logs path
132     this.oldLogDir = createInitialFileSystemLayout();
133     HFileSystem.addLocationsOrderInterceptor(conf);
134   }
135 
136   /**
137    * Create initial layout in filesystem.
138    * <ol>
139    * <li>Check if the meta region exists and is readable, if not create it.
140    * Create hbase.version and the .META. directory if not one.
141    * </li>
142    * <li>Create a log archive directory for RS to put archived logs</li>
143    * </ol>
144    * Idempotent.
145    */
146   private Path createInitialFileSystemLayout() throws IOException {
147     // check if the root directory exists
148     checkRootDir(this.rootdir, conf, this.fs);
149 
150     // check if temp directory exists and clean it
151     checkTempDir(this.tempdir, conf, this.fs);
152 
153     Path oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
154 
155     // Make sure the region servers can archive their old logs
156     if(!this.fs.exists(oldLogDir)) {
157       this.fs.mkdirs(oldLogDir);
158     }
159 
160     return oldLogDir;
161   }
162 
163   public FileSystem getFileSystem() {
164     return this.fs;
165   }
166 
167   /**
168    * Get the directory where old logs go
169    * @return the dir
170    */
171   public Path getOldLogDir() {
172     return this.oldLogDir;
173   }
174 
175   /**
176    * Checks to see if the file system is still accessible.
177    * If not, sets closed
178    * @return false if file system is not available
179    */
180   public boolean checkFileSystem() {
181     if (this.fsOk) {
182       try {
183         FSUtils.checkFileSystemAvailable(this.fs);
184         FSUtils.checkDfsSafeMode(this.conf);
185       } catch (IOException e) {
186         master.abort("Shutting down HBase cluster: file system not available", e);
187         this.fsOk = false;
188       }
189     }
190     return this.fsOk;
191   }
192 
193   /**
194    * @return HBase root dir.
195    */
196   public Path getRootDir() {
197     return this.rootdir;
198   }
199 
200   /**
201    * @return HBase temp dir.
202    */
203   public Path getTempDir() {
204     return this.tempdir;
205   }
206 
207   /**
208    * @return The unique identifier generated for this cluster
209    */
210   public ClusterId getClusterId() {
211     return clusterId;
212   }
213 
214   /**
215    * Inspect the log directory to recover any log file without
216    * an active region server.
217    */
218   void splitLogAfterStartup() {
219     boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
220         HLog.SPLIT_SKIP_ERRORS_DEFAULT);
221     Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
222     do {
223       if (master.isStopped()) {
224         LOG.warn("Master stopped while splitting logs");
225         break;
226       }
227       List<ServerName> serverNames = new ArrayList<ServerName>();
228       try {
229         if (!this.fs.exists(logsDirPath)) return;
230         FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
231         // Get online servers after getting log folders to avoid log folder deletion of newly
232         // checked in region servers . see HBASE-5916
233         Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
234             .keySet();
235 
236         if (logFolders == null || logFolders.length == 0) {
237           LOG.debug("No log files to split, proceeding...");
238           return;
239         }
240         for (FileStatus status : logFolders) {
241           String sn = status.getPath().getName();
242           // truncate splitting suffix if present (for ServerName parsing)
243           if (sn.endsWith(HLog.SPLITTING_EXT)) {
244             sn = sn.substring(0, sn.length() - HLog.SPLITTING_EXT.length());
245           }
246           ServerName serverName = ServerName.parseServerName(sn);
247           if (!onlineServers.contains(serverName)) {
248             LOG.info("Log folder " + status.getPath() + " doesn't belong "
249                 + "to a known region server, splitting");
250             serverNames.add(serverName);
251           } else {
252             LOG.info("Log folder " + status.getPath()
253                 + " belongs to an existing region server");
254           }
255         }
256         splitLog(serverNames, META_FILTER);
257         splitLog(serverNames, NON_META_FILTER);
258         retrySplitting = false;
259       } catch (IOException ioe) {
260         LOG.warn("Failed splitting of " + serverNames, ioe);
261         if (!checkFileSystem()) {
262           LOG.warn("Bad Filesystem, exiting");
263           Runtime.getRuntime().halt(1);
264         }
265         try {
266           if (retrySplitting) {
267             Thread.sleep(conf.getInt(
268               "hbase.hlog.split.failure.retry.interval", 30 * 1000));
269           }
270         } catch (InterruptedException e) {
271           LOG.warn("Interrupted, aborting since cannot return w/o splitting");
272           Thread.currentThread().interrupt();
273           retrySplitting = false;
274           Runtime.getRuntime().halt(1);
275         }
276       }
277     } while (retrySplitting);
278   }
279 
280   public void splitLog(final ServerName serverName) throws IOException {
281     List<ServerName> serverNames = new ArrayList<ServerName>();
282     serverNames.add(serverName);
283     splitLog(serverNames);
284   }
285 
286   /**
287    * Specialized method to handle the splitting for meta HLog
288    * @param serverName
289    * @throws IOException
290    */
291   public void splitMetaLog(final ServerName serverName) throws IOException {
292     long splitTime = 0, splitLogSize = 0;
293     List<ServerName> serverNames = new ArrayList<ServerName>();
294     serverNames.add(serverName);
295     List<Path> logDirs = getLogDirs(serverNames);
296     if (logDirs.isEmpty()) {
297       LOG.info("No meta logs to split");
298       return;
299     }
300     splitLogManager.handleDeadWorkers(serverNames);
301     splitTime = EnvironmentEdgeManager.currentTimeMillis();
302     splitLogSize = splitLogManager.splitLogDistributed(logDirs, META_FILTER);
303     splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
304     if (this.metricsMaster != null) {
305       this.metricsMaster.addSplit(splitTime, splitLogSize);
306     }
307   }
308 
309   private List<Path> getLogDirs(final List<ServerName> serverNames) throws IOException {
310     List<Path> logDirs = new ArrayList<Path>();
311     for (ServerName serverName: serverNames) {
312       Path logDir = new Path(this.rootdir, HLogUtil.getHLogDirectoryName(serverName.toString()));
313       Path splitDir = logDir.suffix(HLog.SPLITTING_EXT);
314       // Rename the directory so a rogue RS doesn't create more HLogs
315       if (fs.exists(logDir)) {
316         if (!this.fs.rename(logDir, splitDir)) {
317           throw new IOException("Failed fs.rename for log split: " + logDir);
318         }
319         logDir = splitDir;
320         LOG.debug("Renamed region directory: " + splitDir);
321       } else if (!fs.exists(splitDir)) {
322         LOG.info("Log dir for server " + serverName + " does not exist");
323         continue;
324       }
325       logDirs.add(splitDir);
326     }
327     return logDirs;
328   }
329 
330   public void splitLog(final List<ServerName> serverNames) throws IOException {
331     splitLog(serverNames, NON_META_FILTER);
332   }
333 
334   /**
335    * This method is the base split method that splits HLog files matching a filter.
336    * Callers should pass the appropriate filter for meta and non-meta HLogs.
337    * @param serverNames
338    * @param filter
339    * @throws IOException
340    */
341   public void splitLog(final List<ServerName> serverNames, PathFilter filter) throws IOException {
342     long splitTime = 0, splitLogSize = 0;
343     List<Path> logDirs = getLogDirs(serverNames);
344 
345     if (logDirs.isEmpty()) {
346       LOG.info("No logs to split");
347       return;
348     }
349 
350     if (distributedLogSplitting) {
351       splitLogManager.handleDeadWorkers(serverNames);
352       splitTime = EnvironmentEdgeManager.currentTimeMillis();
353       splitLogSize = splitLogManager.splitLogDistributed(logDirs,filter);
354       splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
355     } else {
356       for(Path logDir: logDirs){
357         // splitLogLock ensures that dead region servers' logs are processed
358         // one at a time
359         this.splitLogLock.lock();
360         try {
361           HLogSplitter splitter = HLogSplitter.createLogSplitter(
362             conf, rootdir, logDir, oldLogDir, this.fs);
363           try {
364             // If FS is in safe mode, just wait till out of it.
365             FSUtils.waitOnSafeMode(conf, conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 1000));
366             splitter.splitLog();
367           } catch (OrphanHLogAfterSplitException e) {
368             LOG.warn("Retrying splitting because of:", e);
369             //An HLogSplitter instance can only be used once.  Get new instance.
370             splitter = HLogSplitter.createLogSplitter(conf, rootdir, logDir,
371               oldLogDir, this.fs);
372             splitter.splitLog();
373           }
374           splitTime = splitter.getTime();
375           splitLogSize = splitter.getSize();
376         } finally {
377           this.splitLogLock.unlock();
378         }
379       }
380     }
381 
382     if (this.metricsMaster != null) {
383       this.metricsMaster.addSplit(splitTime, splitLogSize);
384     }
385   }
386 
387   /**
388    * Get the rootdir.  Make sure its wholesome and exists before returning.
389    * @param rd
390    * @param c
391    * @param fs
392    * @return hbase.rootdir (after checks for existence and bootstrapping if
393    * needed populating the directory with necessary bootup files).
394    * @throws IOException
395    */
396   private Path checkRootDir(final Path rd, final Configuration c,
397     final FileSystem fs)
398   throws IOException {
399     // If FS is in safe mode wait till out of it.
400     FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
401     // Filesystem is good. Go ahead and check for hbase.rootdir.
402     try {
403       if (!fs.exists(rd)) {
404         fs.mkdirs(rd);
405         // DFS leaves safe mode with 0 DNs when there are 0 blocks.
406         // We used to handle this by checking the current DN count and waiting until
407         // it is nonzero. With security, the check for datanode count doesn't work --
408         // it is a privileged op. So instead we adopt the strategy of the jobtracker
409         // and simply retry file creation during bootstrap indefinitely. As soon as
410         // there is one datanode it will succeed. Permission problems should have
411         // already been caught by mkdirs above.
412         FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
413           10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
414             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
415       } else {
416         if (!fs.isDirectory(rd)) {
417           throw new IllegalArgumentException(rd.toString() + " is not a directory");
418         }
419         // as above
420         FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
421           10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
422             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
423       }
424     } catch (DeserializationException de) {
425       LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
426       IOException ioe = new IOException();
427       ioe.initCause(de);
428       throw ioe;
429     } catch (IllegalArgumentException iae) {
430       LOG.fatal("Please fix invalid configuration for "
431         + HConstants.HBASE_DIR + " " + rd.toString(), iae);
432       throw iae;
433     }
434     // Make sure cluster ID exists
435     if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
436         HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
437       FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
438     }
439     clusterId = FSUtils.getClusterId(fs, rd);
440 
441     // Make sure the meta region directory exists!
442     if (!FSUtils.metaRegionExists(fs, rd)) {
443       bootstrap(rd, c);
444     }
445 
446     // Create tableinfo-s for META if not already there.
447     FSTableDescriptors.createTableDescriptor(fs, rd, HTableDescriptor.META_TABLEDESC, false);
448 
449     return rd;
450   }
451 
452   /**
453    * Make sure the hbase temp directory exists and is empty.
454    * NOTE that this method is only executed once just after the master becomes the active one.
455    */
456   private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
457       throws IOException {
458     // If the temp directory exists, clear the content (left over, from the previous run)
459     if (fs.exists(tmpdir)) {
460       // Archive table in temp, maybe left over from failed deletion,
461       // if not the cleaner will take care of them.
462       for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
463         for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
464           HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
465         }
466       }
467       if (!fs.delete(tmpdir, true)) {
468         throw new IOException("Unable to clean the temp directory: " + tmpdir);
469       }
470     }
471 
472     // Create the temp directory
473     if (!fs.mkdirs(tmpdir)) {
474       throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
475     }
476   }
477 
478   private static void bootstrap(final Path rd, final Configuration c)
479   throws IOException {
480     LOG.info("BOOTSTRAP: creating first META region");
481     try {
482       // Bootstrapping, make sure blockcache is off.  Else, one will be
483       // created here in bootstap and it'll need to be cleaned up.  Better to
484       // not make it in first place.  Turn off block caching for bootstrap.
485       // Enable after.
486       HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
487       setInfoFamilyCachingForMeta(false);
488       HRegion meta = HRegion.createHRegion(metaHRI, rd, c,
489           HTableDescriptor.META_TABLEDESC);
490       setInfoFamilyCachingForMeta(true);
491       HRegion.closeHRegion(meta);
492     } catch (IOException e) {
493       e = RemoteExceptionHandler.checkIOException(e);
494       LOG.error("bootstrap", e);
495       throw e;
496     }
497   }
498 
499   /**
500    * Enable in memory caching for .META.
501    */
502   public static void setInfoFamilyCachingForMeta(final boolean b) {
503     for (HColumnDescriptor hcd:
504         HTableDescriptor.META_TABLEDESC.getColumnFamilies()) {
505       if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
506         hcd.setBlockCacheEnabled(b);
507         hcd.setInMemory(b);
508       }
509     }
510   }
511 
512 
513   public void deleteRegion(HRegionInfo region) throws IOException {
514     HFileArchiver.archiveRegion(conf, fs, region);
515   }
516 
517   public void deleteTable(byte[] tableName) throws IOException {
518     fs.delete(new Path(rootdir, Bytes.toString(tableName)), true);
519   }
520 
521   /**
522    * Move the specified file/directory to the hbase temp directory.
523    * @param path The path of the file/directory to move
524    * @return The temp location of the file/directory moved
525    * @throws IOException in case of file-system failure
526    */
527   public Path moveToTemp(final Path path) throws IOException {
528     Path tempPath = new Path(this.tempdir, path.getName());
529 
530     // Ensure temp exists
531     if (!fs.exists(tempdir) && !fs.mkdirs(tempdir)) {
532       throw new IOException("HBase temp directory '" + tempdir + "' creation failure.");
533     }
534 
535     if (!fs.rename(path, tempPath)) {
536       throw new IOException("Unable to move '" + path + "' to temp '" + tempPath + "'");
537     }
538 
539     return tempPath;
540   }
541 
542   /**
543    * Move the specified table to the hbase temp directory
544    * @param tableName Table name to move
545    * @return The temp location of the table moved
546    * @throws IOException in case of file-system failure
547    */
548   public Path moveTableToTemp(byte[] tableName) throws IOException {
549     return moveToTemp(HTableDescriptor.getTableDir(this.rootdir, tableName));
550   }
551 
552   public void updateRegionInfo(HRegionInfo region) {
553     // TODO implement this.  i think this is currently broken in trunk i don't
554     //      see this getting updated.
555     //      @see HRegion.checkRegioninfoOnFilesystem()
556   }
557 
558   public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
559       throws IOException {
560     // archive family store files
561     Path tableDir = new Path(rootdir, region.getTableNameAsString());
562     HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
563 
564     // delete the family folder
565     Path familyDir = new Path(tableDir,
566       new Path(region.getEncodedName(), Bytes.toString(familyName)));
567     if (fs.delete(familyDir, true) == false) {
568       throw new IOException("Could not delete family "
569           + Bytes.toString(familyName) + " from FileSystem for region "
570           + region.getRegionNameAsString() + "(" + region.getEncodedName()
571           + ")");
572     }
573   }
574 
575   public void stop() {
576     if (splitLogManager != null) {
577       this.splitLogManager.stop();
578     }
579   }
580 
581   /**
582    * Create new HTableDescriptor in HDFS.
583    *
584    * @param htableDescriptor
585    */
586   public void createTableDescriptor(HTableDescriptor htableDescriptor)
587       throws IOException {
588     FSTableDescriptors.createTableDescriptor(htableDescriptor, conf);
589   }
590 
591   /**
592    * Delete column of a table
593    * @param tableName
594    * @param familyName
595    * @return Modified HTableDescriptor with requested column deleted.
596    * @throws IOException
597    */
598   public HTableDescriptor deleteColumn(byte[] tableName, byte[] familyName)
599       throws IOException {
600     LOG.info("DeleteColumn. Table = " + Bytes.toString(tableName)
601         + " family = " + Bytes.toString(familyName));
602     HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
603     htd.removeFamily(familyName);
604     this.services.getTableDescriptors().add(htd);
605     return htd;
606   }
607 
608   /**
609    * Modify Column of a table
610    * @param tableName
611    * @param hcd HColumnDesciptor
612    * @return Modified HTableDescriptor with the column modified.
613    * @throws IOException
614    */
615   public HTableDescriptor modifyColumn(byte[] tableName, HColumnDescriptor hcd)
616       throws IOException {
617     LOG.info("AddModifyColumn. Table = " + Bytes.toString(tableName)
618         + " HCD = " + hcd.toString());
619 
620     HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
621     byte [] familyName = hcd.getName();
622     if(!htd.hasFamily(familyName)) {
623       throw new InvalidFamilyOperationException("Family '" +
624         Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
625     }
626     htd.addFamily(hcd);
627     this.services.getTableDescriptors().add(htd);
628     return htd;
629   }
630 
631   /**
632    * Add column to a table
633    * @param tableName
634    * @param hcd
635    * @return Modified HTableDescriptor with new column added.
636    * @throws IOException
637    */
638   public HTableDescriptor addColumn(byte[] tableName, HColumnDescriptor hcd)
639       throws IOException {
640     LOG.info("AddColumn. Table = " + Bytes.toString(tableName) + " HCD = " +
641       hcd.toString());
642     HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
643     if (htd == null) {
644       throw new InvalidFamilyOperationException("Family '" +
645         hcd.getNameAsString() + "' cannot be modified as HTD is null");
646     }
647     htd.addFamily(hcd);
648     this.services.getTableDescriptors().add(htd);
649     return htd;
650   }
651 }