View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.IOException;
22  import java.io.InterruptedIOException;
23  import java.util.ArrayList;
24  import java.util.HashSet;
25  import java.util.List;
26  import java.util.NavigableMap;
27  import java.util.Set;
28  import java.util.concurrent.locks.Lock;
29  import java.util.concurrent.locks.ReentrantLock;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.hbase.catalog.MetaReader;
34  import org.apache.hadoop.hbase.classification.InterfaceAudience;
35  import org.apache.hadoop.hbase.client.Result;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FileStatus;
38  import org.apache.hadoop.fs.FileSystem;
39  import org.apache.hadoop.fs.Path;
40  import org.apache.hadoop.fs.PathFilter;
41  import org.apache.hadoop.fs.permission.FsPermission;
42  import org.apache.hadoop.hbase.ClusterId;
43  import org.apache.hadoop.hbase.TableName;
44  import org.apache.hadoop.hbase.HColumnDescriptor;
45  import org.apache.hadoop.hbase.HConstants;
46  import org.apache.hadoop.hbase.HRegionInfo;
47  import org.apache.hadoop.hbase.HTableDescriptor;
48  import org.apache.hadoop.hbase.InvalidFamilyOperationException;
49  import org.apache.hadoop.hbase.RemoteExceptionHandler;
50  import org.apache.hadoop.hbase.Server;
51  import org.apache.hadoop.hbase.ServerName;
52  import org.apache.hadoop.hbase.backup.HFileArchiver;
53  import org.apache.hadoop.hbase.exceptions.DeserializationException;
54  import org.apache.hadoop.hbase.fs.HFileSystem;
55  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
56  import org.apache.hadoop.hbase.regionserver.HRegion;
57  import org.apache.hadoop.hbase.regionserver.wal.HLog;
58  import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
59  import org.apache.hadoop.hbase.util.Bytes;
60  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
61  import org.apache.hadoop.hbase.util.FSTableDescriptors;
62  import org.apache.hadoop.hbase.util.FSUtils;
63  import org.apache.zookeeper.KeeperException;
64  
65  /**
66   * This class abstracts a bunch of operations the HMaster needs to interact with
67   * the underlying file system, including splitting log files, checking file
68   * system status, etc.
69   */
70  @InterfaceAudience.Private
71  public class MasterFileSystem {
72    private static final Log LOG = LogFactory.getLog(MasterFileSystem.class.getName());
73    // HBase configuration
74    Configuration conf;
75    // master status
76    Server master;
77    // metrics for master
78    private final MetricsMasterFileSystem metricsMasterFilesystem = new MetricsMasterFileSystem();
79    // Persisted unique cluster ID
80    private ClusterId clusterId;
81    // Keep around for convenience.
82    private final FileSystem fs;
83    // Is the fileystem ok?
84    private volatile boolean fsOk = true;
85    // The Path to the old logs dir
86    private final Path oldLogDir;
87    // root hbase directory on the FS
88    private final Path rootdir;
89    // hbase temp directory used for table construction and deletion
90    private final Path tempdir;
91    // create the split log lock
92    final Lock splitLogLock = new ReentrantLock();
93    final boolean distributedLogReplay;
94    final SplitLogManager splitLogManager;
95    private final MasterServices services;
96  
97    final static PathFilter META_FILTER = new PathFilter() {
98      @Override
99      public boolean accept(Path p) {
100       return HLogUtil.isMetaFile(p);
101     }
102   };
103 
104   final static PathFilter NON_META_FILTER = new PathFilter() {
105     @Override
106     public boolean accept(Path p) {
107       return !HLogUtil.isMetaFile(p);
108     }
109   };
110 
111   public MasterFileSystem(Server master, MasterServices services, boolean masterRecovery)
112   throws IOException {
113     this.conf = master.getConfiguration();
114     this.master = master;
115     this.services = services;
116     // Set filesystem to be that of this.rootdir else we get complaints about
117     // mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is
118     // default localfs.  Presumption is that rootdir is fully-qualified before
119     // we get to here with appropriate fs scheme.
120     this.rootdir = FSUtils.getRootDir(conf);
121     this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
122     // Cover both bases, the old way of setting default fs and the new.
123     // We're supposed to run on 0.20 and 0.21 anyways.
124     this.fs = this.rootdir.getFileSystem(conf);
125     FSUtils.setFsDefault(conf, new Path(this.fs.getUri()));
126     // make sure the fs has the same conf
127     fs.setConf(conf);
128     // setup the filesystem variable
129     // set up the archived logs path
130     this.oldLogDir = createInitialFileSystemLayout();
131     HFileSystem.addLocationsOrderInterceptor(conf);
132     try {
133       this.splitLogManager = new SplitLogManager(master.getZooKeeper(), master.getConfiguration(),
134  master, services,
135               master.getServerName(), masterRecovery);
136     } catch (KeeperException e) {
137       throw new IOException(e);
138     }
139     this.distributedLogReplay = (this.splitLogManager.getRecoveryMode() == RecoveryMode.LOG_REPLAY);
140   }
141 
142   /**
143    * Create initial layout in filesystem.
144    * <ol>
145    * <li>Check if the meta region exists and is readable, if not create it.
146    * Create hbase.version and the hbase:meta directory if not one.
147    * </li>
148    * <li>Create a log archive directory for RS to put archived logs</li>
149    * </ol>
150    * Idempotent.
151    */
152   private Path createInitialFileSystemLayout() throws IOException {
153     // check if the root directory exists
154     checkRootDir(this.rootdir, conf, this.fs);
155 
156     // check if temp directory exists and clean it
157     checkTempDir(this.tempdir, conf, this.fs);
158 
159     Path oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
160 
161     // Make sure the region servers can archive their old logs
162     if(!this.fs.exists(oldLogDir)) {
163       this.fs.mkdirs(oldLogDir);
164     }
165 
166     return oldLogDir;
167   }
168 
169   public FileSystem getFileSystem() {
170     return this.fs;
171   }
172 
173   /**
174    * Get the directory where old logs go
175    * @return the dir
176    */
177   public Path getOldLogDir() {
178     return this.oldLogDir;
179   }
180 
181   /**
182    * Checks to see if the file system is still accessible.
183    * If not, sets closed
184    * @return false if file system is not available
185    */
186   public boolean checkFileSystem() {
187     if (this.fsOk) {
188       try {
189         FSUtils.checkFileSystemAvailable(this.fs);
190         FSUtils.checkDfsSafeMode(this.conf);
191       } catch (IOException e) {
192         master.abort("Shutting down HBase cluster: file system not available", e);
193         this.fsOk = false;
194       }
195     }
196     return this.fsOk;
197   }
198 
199   /**
200    * @return HBase root dir.
201    */
202   public Path getRootDir() {
203     return this.rootdir;
204   }
205 
206   /**
207    * @return HBase temp dir.
208    */
209   public Path getTempDir() {
210     return this.tempdir;
211   }
212 
213   /**
214    * @return The unique identifier generated for this cluster
215    */
216   public ClusterId getClusterId() {
217     return clusterId;
218   }
219 
220   /**
221    * Inspect the log directory to find dead servers which need recovery work
222    * @return A set of ServerNames which aren't running but still have WAL files left in file system
223    */
224   Set<ServerName> getFailedServersFromLogFolders() {
225     boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
226       HLog.SPLIT_SKIP_ERRORS_DEFAULT);
227 
228     Set<ServerName> serverNames = new HashSet<ServerName>();
229     Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
230 
231     do {
232       if (master.isStopped()) {
233         LOG.warn("Master stopped while trying to get failed servers.");
234         break;
235       }
236       try {
237         if (!this.fs.exists(logsDirPath)) return serverNames;
238         FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
239         // Get online servers after getting log folders to avoid log folder deletion of newly
240         // checked in region servers . see HBASE-5916
241         Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
242             .keySet();
243 
244         if (logFolders == null || logFolders.length == 0) {
245           LOG.debug("No log files to split, proceeding...");
246           return serverNames;
247         }
248         for (FileStatus status : logFolders) {
249           String sn = status.getPath().getName();
250           // truncate splitting suffix if present (for ServerName parsing)
251           if (sn.endsWith(HLog.SPLITTING_EXT)) {
252             sn = sn.substring(0, sn.length() - HLog.SPLITTING_EXT.length());
253           }
254           ServerName serverName = ServerName.parseServerName(sn);
255           if (!onlineServers.contains(serverName)) {
256             LOG.info("Log folder " + status.getPath() + " doesn't belong "
257                 + "to a known region server, splitting");
258             serverNames.add(serverName);
259           } else {
260             LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");
261           }
262         }
263         retrySplitting = false;
264       } catch (IOException ioe) {
265         LOG.warn("Failed getting failed servers to be recovered.", ioe);
266         if (!checkFileSystem()) {
267           LOG.warn("Bad Filesystem, exiting");
268           Runtime.getRuntime().halt(1);
269         }
270         try {
271           if (retrySplitting) {
272             Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));
273           }
274         } catch (InterruptedException e) {
275           LOG.warn("Interrupted, aborting since cannot return w/o splitting");
276           Thread.currentThread().interrupt();
277           retrySplitting = false;
278           Runtime.getRuntime().halt(1);
279         }
280       }
281     } while (retrySplitting);
282 
283     return serverNames;
284   }
285 
286   public void splitLog(final ServerName serverName) throws IOException {
287     Set<ServerName> serverNames = new HashSet<ServerName>();
288     serverNames.add(serverName);
289     splitLog(serverNames);
290   }
291 
292   /**
293    * Specialized method to handle the splitting for meta HLog
294    * @param serverName
295    * @throws IOException
296    */
297   public void splitMetaLog(final ServerName serverName) throws IOException {
298     Set<ServerName> serverNames = new HashSet<ServerName>();
299     serverNames.add(serverName);
300     splitMetaLog(serverNames);
301   }
302 
303   /**
304    * Specialized method to handle the splitting for meta HLog
305    * @param serverNames
306    * @throws IOException
307    */
308   public void splitMetaLog(final Set<ServerName> serverNames) throws IOException {
309     splitLog(serverNames, META_FILTER);
310   }
311 
312   private List<Path> getLogDirs(final Set<ServerName> serverNames) throws IOException {
313     List<Path> logDirs = new ArrayList<Path>();
314     boolean needReleaseLock = false;
315     if (!this.services.isInitialized()) {
316       // during master initialization, we could have multiple places splitting a same wal
317       this.splitLogLock.lock();
318       needReleaseLock = true;
319     }
320     try {
321       for (ServerName serverName : serverNames) {
322         Path logDir = new Path(this.rootdir, HLogUtil.getHLogDirectoryName(serverName.toString()));
323         Path splitDir = logDir.suffix(HLog.SPLITTING_EXT);
324         // Rename the directory so a rogue RS doesn't create more HLogs
325         if (fs.exists(logDir)) {
326           if (!this.fs.rename(logDir, splitDir)) {
327             throw new IOException("Failed fs.rename for log split: " + logDir);
328           }
329           logDir = splitDir;
330           LOG.debug("Renamed region directory: " + splitDir);
331         } else if (!fs.exists(splitDir)) {
332           LOG.info("Log dir for server " + serverName + " does not exist");
333           continue;
334         }
335         logDirs.add(splitDir);
336       }
337     } finally {
338       if (needReleaseLock) {
339         this.splitLogLock.unlock();
340       }
341     }
342     return logDirs;
343   }
344 
345   /**
346    * Mark regions in recovering state when distributedLogReplay are set true
347    * @param serverNames Set of ServerNames to be replayed wals in order to recover changes contained
348    *          in them
349    * @throws IOException
350    */
351   public void prepareLogReplay(Set<ServerName> serverNames) throws IOException {
352     if (!this.distributedLogReplay) {
353       return;
354     }
355     // mark regions in recovering state
356     for (ServerName serverName : serverNames) {
357       NavigableMap<HRegionInfo, Result> regions = this.getServerUserRegions(serverName);
358       if (regions == null) {
359         continue;
360       }
361       try {
362         this.splitLogManager.markRegionsRecoveringInZK(serverName, regions.keySet());
363       } catch (KeeperException e) {
364         throw new IOException(e);
365       }
366     }
367   }
368 
369   /**
370    * Mark regions in recovering state when distributedLogReplay are set true
371    * @param serverName Failed region server whose wals to be replayed
372    * @param regions Set of regions to be recovered
373    * @throws IOException
374    */
375   public void prepareLogReplay(ServerName serverName, Set<HRegionInfo> regions) throws IOException {
376     if (!this.distributedLogReplay) {
377       return;
378     }
379     // mark regions in recovering state
380     if (regions == null || regions.isEmpty()) {
381       return;
382     }
383     try {
384       this.splitLogManager.markRegionsRecoveringInZK(serverName, regions);
385     } catch (KeeperException e) {
386       throw new IOException(e);
387     }
388   }
389 
390   public void splitLog(final Set<ServerName> serverNames) throws IOException {
391     splitLog(serverNames, NON_META_FILTER);
392   }
393 
394   /**
395    * Wrapper function on {@link SplitLogManager#removeStaleRecoveringRegionsFromZK(Set)}
396    * @param failedServers
397    * @throws KeeperException
398    */
399   void removeStaleRecoveringRegionsFromZK(final Set<ServerName> failedServers)
400       throws KeeperException {
401     this.splitLogManager.removeStaleRecoveringRegionsFromZK(failedServers);
402   }
403 
404   /**
405    * This method is the base split method that splits HLog files matching a filter. Callers should
406    * pass the appropriate filter for meta and non-meta HLogs.
407    * @param serverNames
408    * @param filter
409    * @throws IOException
410    */
411   public void splitLog(final Set<ServerName> serverNames, PathFilter filter) throws IOException {
412     long splitTime = 0, splitLogSize = 0;
413     List<Path> logDirs = getLogDirs(serverNames);
414 
415     splitLogManager.handleDeadWorkers(serverNames);
416     splitTime = EnvironmentEdgeManager.currentTimeMillis();
417     splitLogSize = splitLogManager.splitLogDistributed(serverNames, logDirs, filter);
418     splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
419 
420     if (this.metricsMasterFilesystem != null) {
421       if (filter == META_FILTER) {
422         this.metricsMasterFilesystem.addMetaWALSplit(splitTime, splitLogSize);
423       } else {
424         this.metricsMasterFilesystem.addSplit(splitTime, splitLogSize);
425       }
426     }
427   }
428 
429   /**
430    * Get the rootdir.  Make sure its wholesome and exists before returning.
431    * @param rd
432    * @param c
433    * @param fs
434    * @return hbase.rootdir (after checks for existence and bootstrapping if
435    * needed populating the directory with necessary bootup files).
436    * @throws IOException
437    */
438   @SuppressWarnings("deprecation")
439   private Path checkRootDir(final Path rd, final Configuration c,
440     final FileSystem fs)
441   throws IOException {
442     // If FS is in safe mode wait till out of it.
443     FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
444 
445     boolean isSecurityEnabled = "kerberos".equalsIgnoreCase(c.get("hbase.security.authentication"));
446     FsPermission rootDirPerms = new FsPermission(c.get("hbase.rootdir.perms", "700"));
447 
448     // Filesystem is good. Go ahead and check for hbase.rootdir.
449     try {
450       if (!fs.exists(rd)) {
451         if (isSecurityEnabled) {
452           fs.mkdirs(rd, rootDirPerms);
453         } else {
454           fs.mkdirs(rd);
455         }
456         // DFS leaves safe mode with 0 DNs when there are 0 blocks.
457         // We used to handle this by checking the current DN count and waiting until
458         // it is nonzero. With security, the check for datanode count doesn't work --
459         // it is a privileged op. So instead we adopt the strategy of the jobtracker
460         // and simply retry file creation during bootstrap indefinitely. As soon as
461         // there is one datanode it will succeed. Permission problems should have
462         // already been caught by mkdirs above.
463         FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
464           10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
465             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
466       } else {
467         if (!fs.isDirectory(rd)) {
468           throw new IllegalArgumentException(rd.toString() + " is not a directory");
469         }
470         if (isSecurityEnabled && !rootDirPerms.equals(fs.getFileStatus(rd).getPermission())) {
471           // check whether the permission match
472           LOG.warn("Found rootdir permissions NOT matching expected \"hbase.rootdir.perms\" for "
473               + "rootdir=" + rd.toString() + " permissions=" + fs.getFileStatus(rd).getPermission()
474               + " and  \"hbase.rootdir.perms\" configured as "
475               + c.get("hbase.rootdir.perms", "700") + ". Automatically setting the permissions. You"
476               + " can change the permissions by setting \"hbase.rootdir.perms\" in hbase-site.xml "
477               + "and restarting the master");
478           fs.setPermission(rd, rootDirPerms);
479         }
480         // as above
481         FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
482           10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
483             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
484       }
485     } catch (DeserializationException de) {
486       LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
487       IOException ioe = new IOException();
488       ioe.initCause(de);
489       throw ioe;
490     } catch (IllegalArgumentException iae) {
491       LOG.fatal("Please fix invalid configuration for "
492         + HConstants.HBASE_DIR + " " + rd.toString(), iae);
493       throw iae;
494     }
495     // Make sure cluster ID exists
496     if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
497         HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
498       FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
499     }
500     clusterId = FSUtils.getClusterId(fs, rd);
501 
502     // Make sure the meta region directory exists!
503     if (!FSUtils.metaRegionExists(fs, rd)) {
504       bootstrap(rd, c);
505     } else {
506       // Migrate table descriptor files if necessary
507       org.apache.hadoop.hbase.util.FSTableDescriptorMigrationToSubdir
508         .migrateFSTableDescriptorsIfNecessary(fs, rd);
509     }
510 
511     // Create tableinfo-s for hbase:meta if not already there.
512 
513     // meta table is a system table, so descriptors are predefined,
514     // we should get them from registry.
515     FSTableDescriptors fsd = new FSTableDescriptors(c, fs, rd);
516     fsd.createTableDescriptor(
517       new HTableDescriptor(fsd.get(TableName.META_TABLE_NAME)));
518 
519     return rd;
520   }
521 
522   /**
523    * Make sure the hbase temp directory exists and is empty.
524    * NOTE that this method is only executed once just after the master becomes the active one.
525    */
526   private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
527       throws IOException {
528     // If the temp directory exists, clear the content (left over, from the previous run)
529     if (fs.exists(tmpdir)) {
530       // Archive table in temp, maybe left over from failed deletion,
531       // if not the cleaner will take care of them.
532       for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
533         for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
534           HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
535         }
536       }
537       if (!fs.delete(tmpdir, true)) {
538         throw new IOException("Unable to clean the temp directory: " + tmpdir);
539       }
540     }
541 
542     // Create the temp directory
543     if (!fs.mkdirs(tmpdir)) {
544       throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
545     }
546   }
547 
548   private static void bootstrap(final Path rd, final Configuration c)
549   throws IOException {
550     LOG.info("BOOTSTRAP: creating hbase:meta region");
551     try {
552       // Bootstrapping, make sure blockcache is off.  Else, one will be
553       // created here in bootstrap and it'll need to be cleaned up.  Better to
554       // not make it in first place.  Turn off block caching for bootstrap.
555       // Enable after.
556       HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
557       HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
558       setInfoFamilyCachingForMeta(metaDescriptor, false);
559       HRegion meta = HRegion.createHRegion(metaHRI, rd, c, metaDescriptor);
560       setInfoFamilyCachingForMeta(metaDescriptor, true);
561       HRegion.closeHRegion(meta);
562     } catch (IOException e) {
563       e = RemoteExceptionHandler.checkIOException(e);
564       LOG.error("bootstrap", e);
565       throw e;
566     }
567   }
568 
569   /**
570    * Enable in memory caching for hbase:meta
571    */
572   public static void setInfoFamilyCachingForMeta(final HTableDescriptor metaDescriptor,
573       final boolean b) {
574     for (HColumnDescriptor hcd: metaDescriptor.getColumnFamilies()) {
575       if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
576         hcd.setBlockCacheEnabled(b);
577         hcd.setInMemory(b);
578       }
579     }
580   }
581 
582   public void deleteRegion(HRegionInfo region) throws IOException {
583     HFileArchiver.archiveRegion(conf, fs, region);
584   }
585 
586   public void deleteTable(TableName tableName) throws IOException {
587     fs.delete(FSUtils.getTableDir(rootdir, tableName), true);
588   }
589 
590   /**
591    * Move the specified table to the hbase temp directory
592    * @param tableName Table name to move
593    * @return The temp location of the table moved
594    * @throws IOException in case of file-system failure
595    */
596   public Path moveTableToTemp(TableName tableName) throws IOException {
597     Path srcPath = FSUtils.getTableDir(rootdir, tableName);
598     Path tempPath = FSUtils.getTableDir(this.tempdir, tableName);
599 
600     // Ensure temp exists
601     if (!fs.exists(tempPath.getParent()) && !fs.mkdirs(tempPath.getParent())) {
602       throw new IOException("HBase temp directory '" + tempPath.getParent() + "' creation failure.");
603     }
604 
605     if (!fs.rename(srcPath, tempPath)) {
606       throw new IOException("Unable to move '" + srcPath + "' to temp '" + tempPath + "'");
607     }
608 
609     return tempPath;
610   }
611 
612   public void updateRegionInfo(HRegionInfo region) {
613     // TODO implement this.  i think this is currently broken in trunk i don't
614     //      see this getting updated.
615     //      @see HRegion.checkRegioninfoOnFilesystem()
616   }
617 
618   public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
619       throws IOException {
620     // archive family store files
621     Path tableDir = FSUtils.getTableDir(rootdir, region.getTable());
622     HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
623 
624     // delete the family folder
625     Path familyDir = new Path(tableDir,
626       new Path(region.getEncodedName(), Bytes.toString(familyName)));
627     if (fs.delete(familyDir, true) == false) {
628       throw new IOException("Could not delete family "
629           + Bytes.toString(familyName) + " from FileSystem for region "
630           + region.getRegionNameAsString() + "(" + region.getEncodedName()
631           + ")");
632     }
633   }
634 
635   public void stop() {
636     if (splitLogManager != null) {
637       this.splitLogManager.stop();
638     }
639   }
640 
641   /**
642    * Delete column of a table
643    * @param tableName
644    * @param familyName
645    * @return Modified HTableDescriptor with requested column deleted.
646    * @throws IOException
647    */
648   public HTableDescriptor deleteColumn(TableName tableName, byte[] familyName)
649       throws IOException {
650     LOG.info("DeleteColumn. Table = " + tableName
651         + " family = " + Bytes.toString(familyName));
652     HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
653     htd.removeFamily(familyName);
654     this.services.getTableDescriptors().add(htd);
655     return htd;
656   }
657 
658   /**
659    * Modify Column of a table
660    * @param tableName
661    * @param hcd HColumnDesciptor
662    * @return Modified HTableDescriptor with the column modified.
663    * @throws IOException
664    */
665   public HTableDescriptor modifyColumn(TableName tableName, HColumnDescriptor hcd)
666       throws IOException {
667     LOG.info("AddModifyColumn. Table = " + tableName
668         + " HCD = " + hcd.toString());
669 
670     HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
671     byte [] familyName = hcd.getName();
672     if(!htd.hasFamily(familyName)) {
673       throw new InvalidFamilyOperationException("Family '" +
674         Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
675     }
676     htd.addFamily(hcd);
677     this.services.getTableDescriptors().add(htd);
678     return htd;
679   }
680 
681   /**
682    * Add column to a table
683    * @param tableName
684    * @param hcd
685    * @return Modified HTableDescriptor with new column added.
686    * @throws IOException
687    */
688   public HTableDescriptor addColumn(TableName tableName, HColumnDescriptor hcd)
689       throws IOException {
690     LOG.info("AddColumn. Table = " + tableName + " HCD = " +
691       hcd.toString());
692     HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
693     if (htd == null) {
694       throw new InvalidFamilyOperationException("Family '" +
695         hcd.getNameAsString() + "' cannot be modified as HTD is null");
696     }
697     htd.addFamily(hcd);
698     this.services.getTableDescriptors().add(htd);
699     return htd;
700   }
701 
702   private NavigableMap<HRegionInfo, Result> getServerUserRegions(ServerName serverName)
703       throws IOException {
704     if (!this.master.isStopped()) {
705       try {
706         this.master.getCatalogTracker().waitForMeta();
707         return MetaReader.getServerUserRegions(this.master.getCatalogTracker(), serverName);
708       } catch (InterruptedException e) {
709         throw (InterruptedIOException)new InterruptedIOException().initCause(e);
710       }
711     }
712     return null;
713   }
714 
715   /**
716    * The function is used in SSH to set recovery mode based on configuration after all outstanding
717    * log split tasks drained.
718    * @throws KeeperException
719    * @throws InterruptedIOException
720    */
721   public void setLogRecoveryMode() throws IOException {
722     try {
723       this.splitLogManager.setRecoveryMode(false);
724     } catch (KeeperException e) {
725       throw new IOException(e);
726     }
727   }
728 
729   public RecoveryMode getLogRecoveryMode() {
730     return this.splitLogManager.getRecoveryMode();
731   }
732 }