View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.IOException;
22  import java.io.InterruptedIOException;
23  import java.util.ArrayList;
24  import java.util.HashSet;
25  import java.util.List;
26  import java.util.Set;
27  import java.util.concurrent.locks.Lock;
28  import java.util.concurrent.locks.ReentrantLock;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.hbase.classification.InterfaceAudience;
33  import org.apache.hadoop.conf.Configuration;
34  import org.apache.hadoop.fs.FileStatus;
35  import org.apache.hadoop.fs.FileSystem;
36  import org.apache.hadoop.fs.Path;
37  import org.apache.hadoop.fs.PathFilter;
38  import org.apache.hadoop.hbase.ClusterId;
39  import org.apache.hadoop.hbase.TableName;
40  import org.apache.hadoop.hbase.HColumnDescriptor;
41  import org.apache.hadoop.hbase.HConstants;
42  import org.apache.hadoop.hbase.HRegionInfo;
43  import org.apache.hadoop.hbase.HTableDescriptor;
44  import org.apache.hadoop.hbase.InvalidFamilyOperationException;
45  import org.apache.hadoop.hbase.RemoteExceptionHandler;
46  import org.apache.hadoop.hbase.Server;
47  import org.apache.hadoop.hbase.ServerName;
48  import org.apache.hadoop.hbase.backup.HFileArchiver;
49  import org.apache.hadoop.hbase.exceptions.DeserializationException;
50  import org.apache.hadoop.hbase.fs.HFileSystem;
51  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
52  import org.apache.hadoop.hbase.regionserver.HRegion;
53  import org.apache.hadoop.hbase.wal.DefaultWALProvider;
54  import org.apache.hadoop.hbase.wal.WALSplitter;
55  import org.apache.hadoop.hbase.util.Bytes;
56  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
57  import org.apache.hadoop.hbase.util.FSTableDescriptors;
58  import org.apache.hadoop.hbase.util.FSUtils;
59  
60  /**
61   * This class abstracts a bunch of operations the HMaster needs to interact with
62   * the underlying file system, including splitting log files, checking file
63   * system status, etc.
64   */
65  @InterfaceAudience.Private
66  public class MasterFileSystem {
67    private static final Log LOG = LogFactory.getLog(MasterFileSystem.class.getName());
68    // HBase configuration
69    Configuration conf;
70    // master status
71    Server master;
72    // metrics for master
73    private final MetricsMasterFileSystem metricsMasterFilesystem = new MetricsMasterFileSystem();
74    // Persisted unique cluster ID
75    private ClusterId clusterId;
76    // Keep around for convenience.
77    private final FileSystem fs;
78    // Is the fileystem ok?
79    private volatile boolean fsOk = true;
80    // The Path to the old logs dir
81    private final Path oldLogDir;
82    // root hbase directory on the FS
83    private final Path rootdir;
84    // hbase temp directory used for table construction and deletion
85    private final Path tempdir;
86    // create the split log lock
87    final Lock splitLogLock = new ReentrantLock();
88    final boolean distributedLogReplay;
89    final SplitLogManager splitLogManager;
90    private final MasterServices services;
91  
92    final static PathFilter META_FILTER = new PathFilter() {
93      @Override
94      public boolean accept(Path p) {
95        return DefaultWALProvider.isMetaFile(p);
96      }
97    };
98  
99    final static PathFilter NON_META_FILTER = new PathFilter() {
100     @Override
101     public boolean accept(Path p) {
102       return !DefaultWALProvider.isMetaFile(p);
103     }
104   };
105 
106   public MasterFileSystem(Server master, MasterServices services)
107   throws IOException {
108     this.conf = master.getConfiguration();
109     this.master = master;
110     this.services = services;
111     // Set filesystem to be that of this.rootdir else we get complaints about
112     // mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is
113     // default localfs.  Presumption is that rootdir is fully-qualified before
114     // we get to here with appropriate fs scheme.
115     this.rootdir = FSUtils.getRootDir(conf);
116     this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
117     // Cover both bases, the old way of setting default fs and the new.
118     // We're supposed to run on 0.20 and 0.21 anyways.
119     this.fs = this.rootdir.getFileSystem(conf);
120     FSUtils.setFsDefault(conf, new Path(this.fs.getUri()));
121     // make sure the fs has the same conf
122     fs.setConf(conf);
123     // setup the filesystem variable
124     // set up the archived logs path
125     this.oldLogDir = createInitialFileSystemLayout();
126     HFileSystem.addLocationsOrderInterceptor(conf);
127     this.splitLogManager =
128         new SplitLogManager(master, master.getConfiguration(), master, services,
129             master.getServerName());
130     this.distributedLogReplay = this.splitLogManager.isLogReplaying();
131   }
132 
133   /**
134    * Create initial layout in filesystem.
135    * <ol>
136    * <li>Check if the meta region exists and is readable, if not create it.
137    * Create hbase.version and the hbase:meta directory if not one.
138    * </li>
139    * <li>Create a log archive directory for RS to put archived logs</li>
140    * </ol>
141    * Idempotent.
142    */
143   private Path createInitialFileSystemLayout() throws IOException {
144     // check if the root directory exists
145     checkRootDir(this.rootdir, conf, this.fs);
146 
147     // check if temp directory exists and clean it
148     checkTempDir(this.tempdir, conf, this.fs);
149 
150     Path oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
151 
152     // Make sure the region servers can archive their old logs
153     if(!this.fs.exists(oldLogDir)) {
154       this.fs.mkdirs(oldLogDir);
155     }
156 
157     return oldLogDir;
158   }
159 
160   public FileSystem getFileSystem() {
161     return this.fs;
162   }
163 
164   /**
165    * Get the directory where old logs go
166    * @return the dir
167    */
168   public Path getOldLogDir() {
169     return this.oldLogDir;
170   }
171 
172   /**
173    * Checks to see if the file system is still accessible.
174    * If not, sets closed
175    * @return false if file system is not available
176    */
177   public boolean checkFileSystem() {
178     if (this.fsOk) {
179       try {
180         FSUtils.checkFileSystemAvailable(this.fs);
181         FSUtils.checkDfsSafeMode(this.conf);
182       } catch (IOException e) {
183         master.abort("Shutting down HBase cluster: file system not available", e);
184         this.fsOk = false;
185       }
186     }
187     return this.fsOk;
188   }
189 
190   /**
191    * @return HBase root dir.
192    */
193   public Path getRootDir() {
194     return this.rootdir;
195   }
196 
197   /**
198    * @return HBase temp dir.
199    */
200   public Path getTempDir() {
201     return this.tempdir;
202   }
203 
204   /**
205    * @return The unique identifier generated for this cluster
206    */
207   public ClusterId getClusterId() {
208     return clusterId;
209   }
210 
211   /**
212    * Inspect the log directory to find dead servers which need recovery work
213    * @return A set of ServerNames which aren't running but still have WAL files left in file system
214    */
215   Set<ServerName> getFailedServersFromLogFolders() {
216     boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
217         WALSplitter.SPLIT_SKIP_ERRORS_DEFAULT);
218 
219     Set<ServerName> serverNames = new HashSet<ServerName>();
220     Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
221 
222     do {
223       if (master.isStopped()) {
224         LOG.warn("Master stopped while trying to get failed servers.");
225         break;
226       }
227       try {
228         if (!this.fs.exists(logsDirPath)) return serverNames;
229         FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
230         // Get online servers after getting log folders to avoid log folder deletion of newly
231         // checked in region servers . see HBASE-5916
232         Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
233             .keySet();
234 
235         if (logFolders == null || logFolders.length == 0) {
236           LOG.debug("No log files to split, proceeding...");
237           return serverNames;
238         }
239         for (FileStatus status : logFolders) {
240           FileStatus[] curLogFiles = FSUtils.listStatus(this.fs, status.getPath(), null);
241           if (curLogFiles == null || curLogFiles.length == 0) {
242             // Empty log folder. No recovery needed
243             continue;
244           }
245           final ServerName serverName = DefaultWALProvider.getServerNameFromWALDirectoryName(
246               status.getPath());
247           if (null == serverName) {
248             LOG.warn("Log folder " + status.getPath() + " doesn't look like its name includes a " +
249                 "region server name; leaving in place. If you see later errors about missing " +
250                 "write ahead logs they may be saved in this location.");
251           } else if (!onlineServers.contains(serverName)) {
252             LOG.info("Log folder " + status.getPath() + " doesn't belong "
253                 + "to a known region server, splitting");
254             serverNames.add(serverName);
255           } else {
256             LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");
257           }
258         }
259         retrySplitting = false;
260       } catch (IOException ioe) {
261         LOG.warn("Failed getting failed servers to be recovered.", ioe);
262         if (!checkFileSystem()) {
263           LOG.warn("Bad Filesystem, exiting");
264           Runtime.getRuntime().halt(1);
265         }
266         try {
267           if (retrySplitting) {
268             Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));
269           }
270         } catch (InterruptedException e) {
271           LOG.warn("Interrupted, aborting since cannot return w/o splitting");
272           Thread.currentThread().interrupt();
273           retrySplitting = false;
274           Runtime.getRuntime().halt(1);
275         }
276       }
277     } while (retrySplitting);
278 
279     return serverNames;
280   }
281 
282   public void splitLog(final ServerName serverName) throws IOException {
283     Set<ServerName> serverNames = new HashSet<ServerName>();
284     serverNames.add(serverName);
285     splitLog(serverNames);
286   }
287 
288   /**
289    * Specialized method to handle the splitting for meta WAL
290    * @param serverName
291    * @throws IOException
292    */
293   public void splitMetaLog(final ServerName serverName) throws IOException {
294     Set<ServerName> serverNames = new HashSet<ServerName>();
295     serverNames.add(serverName);
296     splitMetaLog(serverNames);
297   }
298 
299   /**
300    * Specialized method to handle the splitting for meta WAL
301    * @param serverNames
302    * @throws IOException
303    */
304   public void splitMetaLog(final Set<ServerName> serverNames) throws IOException {
305     splitLog(serverNames, META_FILTER);
306   }
307 
308   @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="UL_UNRELEASED_LOCK", justification=
309       "We only release this lock when we set it. Updates to code that uses it should verify use " +
310       "of the guard boolean.")
311   private List<Path> getLogDirs(final Set<ServerName> serverNames) throws IOException {
312     List<Path> logDirs = new ArrayList<Path>();
313     boolean needReleaseLock = false;
314     if (!this.services.isInitialized()) {
315       // during master initialization, we could have multiple places splitting a same wal
316       this.splitLogLock.lock();
317       needReleaseLock = true;
318     }
319     try {
320       for (ServerName serverName : serverNames) {
321         Path logDir = new Path(this.rootdir,
322             DefaultWALProvider.getWALDirectoryName(serverName.toString()));
323         Path splitDir = logDir.suffix(DefaultWALProvider.SPLITTING_EXT);
324         // Rename the directory so a rogue RS doesn't create more WALs
325         if (fs.exists(logDir)) {
326           if (!this.fs.rename(logDir, splitDir)) {
327             throw new IOException("Failed fs.rename for log split: " + logDir);
328           }
329           logDir = splitDir;
330           LOG.debug("Renamed region directory: " + splitDir);
331         } else if (!fs.exists(splitDir)) {
332           LOG.info("Log dir for server " + serverName + " does not exist");
333           continue;
334         }
335         logDirs.add(splitDir);
336       }
337     } finally {
338       if (needReleaseLock) {
339         this.splitLogLock.unlock();
340       }
341     }
342     return logDirs;
343   }
344 
345   /**
346    * Mark regions in recovering state when distributedLogReplay are set true
347    * @param serverName Failed region server whose wals to be replayed
348    * @param regions Set of regions to be recovered
349    * @throws IOException
350    */
351   public void prepareLogReplay(ServerName serverName, Set<HRegionInfo> regions) throws IOException {
352     if (!this.distributedLogReplay) {
353       return;
354     }
355     // mark regions in recovering state
356     if (regions == null || regions.isEmpty()) {
357       return;
358     }
359     this.splitLogManager.markRegionsRecovering(serverName, regions);
360   }
361 
362   public void splitLog(final Set<ServerName> serverNames) throws IOException {
363     splitLog(serverNames, NON_META_FILTER);
364   }
365 
366   /**
367    * Wrapper function on {@link SplitLogManager#removeStaleRecoveringRegions(Set)}
368    * @param failedServers
369    * @throws IOException
370    */
371   void removeStaleRecoveringRegionsFromZK(final Set<ServerName> failedServers)
372       throws IOException, InterruptedIOException {
373     this.splitLogManager.removeStaleRecoveringRegions(failedServers);
374   }
375 
376   /**
377    * This method is the base split method that splits WAL files matching a filter. Callers should
378    * pass the appropriate filter for meta and non-meta WALs.
379    * @param serverNames logs belonging to these servers will be split; this will rename the log
380    *                    directory out from under a soft-failed server
381    * @param filter
382    * @throws IOException
383    */
384   public void splitLog(final Set<ServerName> serverNames, PathFilter filter) throws IOException {
385     long splitTime = 0, splitLogSize = 0;
386     List<Path> logDirs = getLogDirs(serverNames);
387 
388     splitLogManager.handleDeadWorkers(serverNames);
389     splitTime = EnvironmentEdgeManager.currentTime();
390     splitLogSize = splitLogManager.splitLogDistributed(serverNames, logDirs, filter);
391     splitTime = EnvironmentEdgeManager.currentTime() - splitTime;
392 
393     if (this.metricsMasterFilesystem != null) {
394       if (filter == META_FILTER) {
395         this.metricsMasterFilesystem.addMetaWALSplit(splitTime, splitLogSize);
396       } else {
397         this.metricsMasterFilesystem.addSplit(splitTime, splitLogSize);
398       }
399     }
400   }
401 
402   /**
403    * Get the rootdir.  Make sure its wholesome and exists before returning.
404    * @param rd
405    * @param c
406    * @param fs
407    * @return hbase.rootdir (after checks for existence and bootstrapping if
408    * needed populating the directory with necessary bootup files).
409    * @throws IOException
410    */
411   @SuppressWarnings("deprecation")
412   private Path checkRootDir(final Path rd, final Configuration c,
413     final FileSystem fs)
414   throws IOException {
415     // If FS is in safe mode wait till out of it.
416     FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
417     // Filesystem is good. Go ahead and check for hbase.rootdir.
418     try {
419       if (!fs.exists(rd)) {
420         fs.mkdirs(rd);
421         // DFS leaves safe mode with 0 DNs when there are 0 blocks.
422         // We used to handle this by checking the current DN count and waiting until
423         // it is nonzero. With security, the check for datanode count doesn't work --
424         // it is a privileged op. So instead we adopt the strategy of the jobtracker
425         // and simply retry file creation during bootstrap indefinitely. As soon as
426         // there is one datanode it will succeed. Permission problems should have
427         // already been caught by mkdirs above.
428         FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
429           10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
430             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
431       } else {
432         if (!fs.isDirectory(rd)) {
433           throw new IllegalArgumentException(rd.toString() + " is not a directory");
434         }
435         // as above
436         FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
437           10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
438             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
439       }
440     } catch (DeserializationException de) {
441       LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
442       IOException ioe = new IOException();
443       ioe.initCause(de);
444       throw ioe;
445     } catch (IllegalArgumentException iae) {
446       LOG.fatal("Please fix invalid configuration for "
447         + HConstants.HBASE_DIR + " " + rd.toString(), iae);
448       throw iae;
449     }
450     // Make sure cluster ID exists
451     if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
452         HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
453       FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
454     }
455     clusterId = FSUtils.getClusterId(fs, rd);
456 
457     // Make sure the meta region directory exists!
458     if (!FSUtils.metaRegionExists(fs, rd)) {
459       bootstrap(rd, c);
460     } else {
461       // Migrate table descriptor files if necessary
462       org.apache.hadoop.hbase.util.FSTableDescriptorMigrationToSubdir
463         .migrateFSTableDescriptorsIfNecessary(fs, rd);
464     }
465 
466     // Create tableinfo-s for hbase:meta if not already there.
467 
468     // meta table is a system table, so descriptors are predefined,
469     // we should get them from registry.
470     FSTableDescriptors fsd = new FSTableDescriptors(c, fs, rd);
471     fsd.createTableDescriptor(
472       new HTableDescriptor(fsd.get(TableName.META_TABLE_NAME)));
473 
474     return rd;
475   }
476 
477   /**
478    * Make sure the hbase temp directory exists and is empty.
479    * NOTE that this method is only executed once just after the master becomes the active one.
480    */
481   private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
482       throws IOException {
483     // If the temp directory exists, clear the content (left over, from the previous run)
484     if (fs.exists(tmpdir)) {
485       // Archive table in temp, maybe left over from failed deletion,
486       // if not the cleaner will take care of them.
487       for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
488         for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
489           HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
490         }
491       }
492       if (!fs.delete(tmpdir, true)) {
493         throw new IOException("Unable to clean the temp directory: " + tmpdir);
494       }
495     }
496 
497     // Create the temp directory
498     if (!fs.mkdirs(tmpdir)) {
499       throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
500     }
501   }
502 
503   private static void bootstrap(final Path rd, final Configuration c)
504   throws IOException {
505     LOG.info("BOOTSTRAP: creating hbase:meta region");
506     try {
507       // Bootstrapping, make sure blockcache is off.  Else, one will be
508       // created here in bootstrap and it'll need to be cleaned up.  Better to
509       // not make it in first place.  Turn off block caching for bootstrap.
510       // Enable after.
511       HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
512       HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
513       setInfoFamilyCachingForMeta(metaDescriptor, false);
514       HRegion meta = HRegion.createHRegion(metaHRI, rd, c, metaDescriptor);
515       setInfoFamilyCachingForMeta(metaDescriptor, true);
516       HRegion.closeHRegion(meta);
517     } catch (IOException e) {
518       e = RemoteExceptionHandler.checkIOException(e);
519       LOG.error("bootstrap", e);
520       throw e;
521     }
522   }
523 
524   /**
525    * Enable in memory caching for hbase:meta
526    */
527   public static void setInfoFamilyCachingForMeta(final HTableDescriptor metaDescriptor,
528       final boolean b) {
529     for (HColumnDescriptor hcd: metaDescriptor.getColumnFamilies()) {
530       if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
531         hcd.setBlockCacheEnabled(b);
532         hcd.setInMemory(b);
533       }
534     }
535   }
536 
537   public void deleteRegion(HRegionInfo region) throws IOException {
538     HFileArchiver.archiveRegion(conf, fs, region);
539   }
540 
541   public void deleteTable(TableName tableName) throws IOException {
542     fs.delete(FSUtils.getTableDir(rootdir, tableName), true);
543   }
544 
545   /**
546    * Move the specified table to the hbase temp directory
547    * @param tableName Table name to move
548    * @return The temp location of the table moved
549    * @throws IOException in case of file-system failure
550    */
551   public Path moveTableToTemp(TableName tableName) throws IOException {
552     Path srcPath = FSUtils.getTableDir(rootdir, tableName);
553     Path tempPath = FSUtils.getTableDir(this.tempdir, tableName);
554 
555     // Ensure temp exists
556     if (!fs.exists(tempPath.getParent()) && !fs.mkdirs(tempPath.getParent())) {
557       throw new IOException("HBase temp directory '" + tempPath.getParent() + "' creation failure.");
558     }
559 
560     if (!fs.rename(srcPath, tempPath)) {
561       throw new IOException("Unable to move '" + srcPath + "' to temp '" + tempPath + "'");
562     }
563 
564     return tempPath;
565   }
566 
567   public void updateRegionInfo(HRegionInfo region) {
568     // TODO implement this.  i think this is currently broken in trunk i don't
569     //      see this getting updated.
570     //      @see HRegion.checkRegioninfoOnFilesystem()
571   }
572 
573   public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
574       throws IOException {
575     // archive family store files
576     Path tableDir = FSUtils.getTableDir(rootdir, region.getTable());
577     HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
578 
579     // delete the family folder
580     Path familyDir = new Path(tableDir,
581       new Path(region.getEncodedName(), Bytes.toString(familyName)));
582     if (fs.delete(familyDir, true) == false) {
583       throw new IOException("Could not delete family "
584           + Bytes.toString(familyName) + " from FileSystem for region "
585           + region.getRegionNameAsString() + "(" + region.getEncodedName()
586           + ")");
587     }
588   }
589 
590   public void stop() {
591     if (splitLogManager != null) {
592       this.splitLogManager.stop();
593     }
594   }
595 
596   /**
597    * Delete column of a table
598    * @param tableName
599    * @param familyName
600    * @return Modified HTableDescriptor with requested column deleted.
601    * @throws IOException
602    */
603   public HTableDescriptor deleteColumn(TableName tableName, byte[] familyName)
604       throws IOException {
605     LOG.info("DeleteColumn. Table = " + tableName
606         + " family = " + Bytes.toString(familyName));
607     HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
608     htd.removeFamily(familyName);
609     this.services.getTableDescriptors().add(htd);
610     return htd;
611   }
612 
613   /**
614    * Modify Column of a table
615    * @param tableName
616    * @param hcd HColumnDesciptor
617    * @return Modified HTableDescriptor with the column modified.
618    * @throws IOException
619    */
620   public HTableDescriptor modifyColumn(TableName tableName, HColumnDescriptor hcd)
621       throws IOException {
622     LOG.info("AddModifyColumn. Table = " + tableName
623         + " HCD = " + hcd.toString());
624 
625     HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
626     byte [] familyName = hcd.getName();
627     if(!htd.hasFamily(familyName)) {
628       throw new InvalidFamilyOperationException("Family '" +
629         Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
630     }
631     htd.modifyFamily(hcd);
632     this.services.getTableDescriptors().add(htd);
633     return htd;
634   }
635 
636   /**
637    * Add column to a table
638    * @param tableName
639    * @param hcd
640    * @return Modified HTableDescriptor with new column added.
641    * @throws IOException
642    */
643   public HTableDescriptor addColumn(TableName tableName, HColumnDescriptor hcd)
644       throws IOException {
645     LOG.info("AddColumn. Table = " + tableName + " HCD = " +
646       hcd.toString());
647     HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
648     if (htd == null) {
649       throw new InvalidFamilyOperationException("Family '" +
650         hcd.getNameAsString() + "' cannot be modified as HTD is null");
651     }
652     htd.addFamily(hcd);
653     this.services.getTableDescriptors().add(htd);
654     return htd;
655   }
656 
657   /**
658    * The function is used in SSH to set recovery mode based on configuration after all outstanding
659    * log split tasks drained.
660    * @throws IOException
661    */
662   public void setLogRecoveryMode() throws IOException {
663       this.splitLogManager.setRecoveryMode(false);
664   }
665 
666   public RecoveryMode getLogRecoveryMode() {
667     return this.splitLogManager.getRecoveryMode();
668   }
669 }