View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.util;
20  
21  import java.io.ByteArrayInputStream;
22  import java.io.DataInputStream;
23  import java.io.EOFException;
24  import java.io.FileNotFoundException;
25  import java.io.IOException;
26  import java.io.InputStream;
27  import java.lang.reflect.Method;
28  import java.net.URI;
29  import java.net.URISyntaxException;
30  import java.util.ArrayList;
31  import java.util.HashMap;
32  import java.util.List;
33  import java.util.Map;
34  import java.util.regex.Pattern;
35  
36  import org.apache.commons.logging.Log;
37  import org.apache.commons.logging.LogFactory;
38  import org.apache.hadoop.classification.InterfaceAudience;
39  import org.apache.hadoop.classification.InterfaceStability;
40  import org.apache.hadoop.conf.Configuration;
41  import org.apache.hadoop.fs.BlockLocation;
42  import org.apache.hadoop.fs.FSDataInputStream;
43  import org.apache.hadoop.fs.FSDataOutputStream;
44  import org.apache.hadoop.fs.FileStatus;
45  import org.apache.hadoop.fs.FileSystem;
46  import org.apache.hadoop.fs.Path;
47  import org.apache.hadoop.fs.PathFilter;
48  import org.apache.hadoop.fs.permission.FsAction;
49  import org.apache.hadoop.fs.permission.FsPermission;
50  import org.apache.hadoop.hbase.ClusterId;
51  import org.apache.hadoop.hbase.exceptions.DeserializationException;
52  import org.apache.hadoop.hbase.HColumnDescriptor;
53  import org.apache.hadoop.hbase.HConstants;
54  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
55  import org.apache.hadoop.hbase.HRegionInfo;
56  import org.apache.hadoop.hbase.RemoteExceptionHandler;
57  import org.apache.hadoop.hbase.exceptions.FileSystemVersionException;
58  import org.apache.hadoop.hbase.master.HMaster;
59  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
60  import org.apache.hadoop.hbase.protobuf.generated.FSProtos;
61  import org.apache.hadoop.hbase.regionserver.HRegion;
62  import org.apache.hadoop.hdfs.DistributedFileSystem;
63  import org.apache.hadoop.io.IOUtils;
64  import org.apache.hadoop.io.SequenceFile;
65  import org.apache.hadoop.security.AccessControlException;
66  import org.apache.hadoop.security.UserGroupInformation;
67  import org.apache.hadoop.util.ReflectionUtils;
68  import org.apache.hadoop.util.StringUtils;
69  
70  import com.google.common.primitives.Ints;
71  import com.google.protobuf.InvalidProtocolBufferException;
72  
73  /**
74   * Utility methods for interacting with the underlying file system.
75   */
76  @InterfaceAudience.Public
77  @InterfaceStability.Evolving
78  public abstract class FSUtils {
79    private static final Log LOG = LogFactory.getLog(FSUtils.class);
80  
81    /** Full access permissions (starting point for a umask) */
82    private static final String FULL_RWX_PERMISSIONS = "777";
83  
84    protected FSUtils() {
85      super();
86    }
87  
88    /**
89     * Compare of path component. Does not consider schema; i.e. if schemas different but <code>path
90     * <code> starts with <code>rootPath<code>, then the function returns true
91     * @param rootPath
92     * @param path 
93     * @return True if <code>path</code> starts with <code>rootPath</code>
94     */
95    public static boolean isStartingWithPath(final Path rootPath, final String path) {
96      String uriRootPath = rootPath.toUri().getPath();
97      String tailUriPath = (new Path(path)).toUri().getPath();
98      return tailUriPath.startsWith(uriRootPath);
99    }
100 
101   /**
102    * Compare path component of the Path URI; e.g. if hdfs://a/b/c and /a/b/c, it will compare the
103    * '/a/b/c' part. Does not consider schema; i.e. if schemas different but path or subpath matches,
104    * the two will equate.
105    * @param pathToSearch Path we will be trying to match.
106    * @param pathTail
107    * @return True if <code>pathTail</code> is tail on the path of <code>pathToSearch</code>
108    */
109   public static boolean isMatchingTail(final Path pathToSearch, String pathTail) {
110     return isMatchingTail(pathToSearch, new Path(pathTail));
111   }
112 
113   /**
114    * Compare path component of the Path URI; e.g. if hdfs://a/b/c and /a/b/c, it will compare the
115    * '/a/b/c' part. If you passed in 'hdfs://a/b/c and b/c, it would return true.  Does not consider
116    * schema; i.e. if schemas different but path or subpath matches, the two will equate.
117    * @param pathToSearch Path we will be trying to match.
118    * @param pathTail
119    * @return True if <code>pathTail</code> is tail on the path of <code>pathToSearch</code>
120    */
121   public static boolean isMatchingTail(final Path pathToSearch, final Path pathTail) {
122     if (pathToSearch.depth() != pathTail.depth()) return false;
123     Path tailPath = pathTail;
124     String tailName;
125     Path toSearch = pathToSearch;
126     String toSearchName;
127     boolean result = false;
128     do {
129       tailName = tailPath.getName();
130       if (tailName == null || tailName.length() <= 0) {
131         result = true;
132         break;
133       }
134       toSearchName = toSearch.getName();
135       if (toSearchName == null || toSearchName.length() <= 0) break;
136       // Move up a parent on each path for next go around.  Path doesn't let us go off the end.
137       tailPath = tailPath.getParent();
138       toSearch = toSearch.getParent();
139     } while(tailName.equals(toSearchName));
140     return result;
141   }
142 
143   public static FSUtils getInstance(FileSystem fs, Configuration conf) {
144     String scheme = fs.getUri().getScheme();
145     if (scheme == null) {
146       LOG.warn("Could not find scheme for uri " +
147           fs.getUri() + ", default to hdfs");
148       scheme = "hdfs";
149     }
150     Class<?> fsUtilsClass = conf.getClass("hbase.fsutil." +
151         scheme + ".impl", FSHDFSUtils.class); // Default to HDFS impl
152     FSUtils fsUtils = (FSUtils)ReflectionUtils.newInstance(fsUtilsClass, conf);
153     return fsUtils;
154   }
155 
156   /**
157    * Delete if exists.
158    * @param fs filesystem object
159    * @param dir directory to delete
160    * @return True if deleted <code>dir</code>
161    * @throws IOException e
162    */
163   public static boolean deleteDirectory(final FileSystem fs, final Path dir)
164   throws IOException {
165     return fs.exists(dir) && fs.delete(dir, true);
166   }
167 
168 
169   /**
170    * Create the specified file on the filesystem. By default, this will:
171    * <ol>
172    * <li>overwrite the file if it exists</li>
173    * <li>apply the umask in the configuration (if it is enabled)</li>
174    * <li>use the fs configured buffer size (or 4096 if not set)</li>
175    * <li>use the default replication</li>
176    * <li>use the default block size</li>
177    * <li>not track progress</li>
178    * </ol>
179    *
180    * @param fs {@link FileSystem} on which to write the file
181    * @param path {@link Path} to the file to write
182    * @return output stream to the created file
183    * @throws IOException if the file cannot be created
184    */
185   public static FSDataOutputStream create(FileSystem fs, Path path,
186       FsPermission perm) throws IOException {
187     return create(fs, path, perm, true);
188   }
189 
190   /**
191    * Create the specified file on the filesystem. By default, this will:
192    * <ol>
193    * <li>apply the umask in the configuration (if it is enabled)</li>
194    * <li>use the fs configured buffer size (or 4096 if not set)</li>
195    * <li>use the default replication</li>
196    * <li>use the default block size</li>
197    * <li>not track progress</li>
198    * </ol>
199    *
200    * @param fs {@link FileSystem} on which to write the file
201    * @param path {@link Path} to the file to write
202    * @param perm
203    * @param overwrite Whether or not the created file should be overwritten.
204    * @return output stream to the created file
205    * @throws IOException if the file cannot be created
206    */
207   public static FSDataOutputStream create(FileSystem fs, Path path,
208       FsPermission perm, boolean overwrite) throws IOException {
209     LOG.debug("Creating file=" + path + " with permission=" + perm);
210 
211     return fs.create(path, perm, overwrite,
212         fs.getConf().getInt("io.file.buffer.size", 4096),
213         fs.getDefaultReplication(), fs.getDefaultBlockSize(), null);
214   }
215 
216   /**
217    * Get the file permissions specified in the configuration, if they are
218    * enabled.
219    *
220    * @param fs filesystem that the file will be created on.
221    * @param conf configuration to read for determining if permissions are
222    *          enabled and which to use
223    * @param permssionConfKey property key in the configuration to use when
224    *          finding the permission
225    * @return the permission to use when creating a new file on the fs. If
226    *         special permissions are not specified in the configuration, then
227    *         the default permissions on the the fs will be returned.
228    */
229   public static FsPermission getFilePermissions(final FileSystem fs,
230       final Configuration conf, final String permssionConfKey) {
231     boolean enablePermissions = conf.getBoolean(
232         HConstants.ENABLE_DATA_FILE_UMASK, false);
233 
234     if (enablePermissions) {
235       try {
236         FsPermission perm = new FsPermission(FULL_RWX_PERMISSIONS);
237         // make sure that we have a mask, if not, go default.
238         String mask = conf.get(permssionConfKey);
239         if (mask == null)
240           return FsPermission.getDefault();
241         // appy the umask
242         FsPermission umask = new FsPermission(mask);
243         return perm.applyUMask(umask);
244       } catch (IllegalArgumentException e) {
245         LOG.warn(
246             "Incorrect umask attempted to be created: "
247                 + conf.get(permssionConfKey)
248                 + ", using default file permissions.", e);
249         return FsPermission.getDefault();
250       }
251     }
252     return FsPermission.getDefault();
253   }
254 
255   /**
256    * Checks to see if the specified file system is available
257    *
258    * @param fs filesystem
259    * @throws IOException e
260    */
261   public static void checkFileSystemAvailable(final FileSystem fs)
262   throws IOException {
263     if (!(fs instanceof DistributedFileSystem)) {
264       return;
265     }
266     IOException exception = null;
267     DistributedFileSystem dfs = (DistributedFileSystem) fs;
268     try {
269       if (dfs.exists(new Path("/"))) {
270         return;
271       }
272     } catch (IOException e) {
273       exception = RemoteExceptionHandler.checkIOException(e);
274     }
275     try {
276       fs.close();
277     } catch (Exception e) {
278       LOG.error("file system close failed: ", e);
279     }
280     IOException io = new IOException("File system is not available");
281     io.initCause(exception);
282     throw io;
283   }
284 
285   /**
286    * We use reflection because {@link DistributedFileSystem#setSafeMode(
287    * FSConstants.SafeModeAction action, boolean isChecked)} is not in hadoop 1.1
288    * 
289    * @param dfs
290    * @return whether we're in safe mode
291    * @throws IOException
292    */
293   private static boolean isInSafeMode(DistributedFileSystem dfs) throws IOException {
294     boolean inSafeMode = false;
295     try {
296       Method m = DistributedFileSystem.class.getMethod("setSafeMode", new Class<?> []{
297           org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.class, boolean.class});
298       inSafeMode = (Boolean) m.invoke(dfs,
299         org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.SAFEMODE_GET, true);
300     } catch (Exception e) {
301       if (e instanceof IOException) throw (IOException) e;
302       
303       // Check whether dfs is on safemode.
304       inSafeMode = dfs.setSafeMode(
305         org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.SAFEMODE_GET);      
306     }
307     return inSafeMode;    
308   }
309   
310   /**
311    * Check whether dfs is in safemode.
312    * @param conf
313    * @throws IOException
314    */
315   public static void checkDfsSafeMode(final Configuration conf)
316   throws IOException {
317     boolean isInSafeMode = false;
318     FileSystem fs = FileSystem.get(conf);
319     if (fs instanceof DistributedFileSystem) {
320       DistributedFileSystem dfs = (DistributedFileSystem)fs;
321       isInSafeMode = isInSafeMode(dfs);
322     }
323     if (isInSafeMode) {
324       throw new IOException("File system is in safemode, it can't be written now");
325     }
326   }
327 
328   /**
329    * Verifies current version of file system
330    *
331    * @param fs filesystem object
332    * @param rootdir root hbase directory
333    * @return null if no version file exists, version string otherwise.
334    * @throws IOException e
335    * @throws org.apache.hadoop.hbase.exceptions.DeserializationException
336    */
337   public static String getVersion(FileSystem fs, Path rootdir)
338   throws IOException, DeserializationException {
339     Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
340     FileStatus[] status = null;
341     try {
342       // hadoop 2.0 throws FNFE if directory does not exist.  
343       // hadoop 1.0 returns null if directory does not exist.
344       status = fs.listStatus(versionFile);
345     } catch (FileNotFoundException fnfe) {
346       return null;
347     }
348     if (status == null || status.length == 0) return null;
349     String version = null;
350     byte [] content = new byte [(int)status[0].getLen()];
351     FSDataInputStream s = fs.open(versionFile);
352     try {
353       IOUtils.readFully(s, content, 0, content.length);
354       if (ProtobufUtil.isPBMagicPrefix(content)) {
355         version = parseVersionFrom(content);
356       } else {
357         // Presume it pre-pb format.
358         InputStream is = new ByteArrayInputStream(content);
359         DataInputStream dis = new DataInputStream(is);
360         try {
361           version = dis.readUTF();
362         } finally {
363           dis.close();
364         }
365         // Update the format
366         LOG.info("Updating the hbase.version file format with version=" + version);
367         setVersion(fs, rootdir, version, 0, HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);
368       }
369     } catch (EOFException eof) {
370       LOG.warn("Version file was empty, odd, will try to set it.");
371     } finally {
372       s.close();
373     }
374     return version;
375   }
376 
377   /**
378    * Parse the content of the ${HBASE_ROOTDIR}/hbase.version file.
379    * @param bytes The byte content of the hbase.version file.
380    * @return The version found in the file as a String.
381    * @throws DeserializationException
382    */
383   static String parseVersionFrom(final byte [] bytes)
384   throws DeserializationException {
385     ProtobufUtil.expectPBMagicPrefix(bytes);
386     int pblen = ProtobufUtil.lengthOfPBMagic();
387     FSProtos.HBaseVersionFileContent.Builder builder =
388       FSProtos.HBaseVersionFileContent.newBuilder();
389     FSProtos.HBaseVersionFileContent fileContent;
390     try {
391       fileContent = builder.mergeFrom(bytes, pblen, bytes.length - pblen).build();
392       return fileContent.getVersion();
393     } catch (InvalidProtocolBufferException e) {
394       // Convert
395       throw new DeserializationException(e);
396     }
397   }
398 
399   /**
400    * Create the content to write into the ${HBASE_ROOTDIR}/hbase.version file.
401    * @param version Version to persist
402    * @return Serialized protobuf with <code>version</code> content and a bit of pb magic for a prefix.
403    */
404   static byte [] toVersionByteArray(final String version) {
405     FSProtos.HBaseVersionFileContent.Builder builder =
406       FSProtos.HBaseVersionFileContent.newBuilder();
407     return ProtobufUtil.prependPBMagic(builder.setVersion(version).build().toByteArray());
408   }
409 
410   /**
411    * Verifies current version of file system
412    *
413    * @param fs file system
414    * @param rootdir root directory of HBase installation
415    * @param message if true, issues a message on System.out
416    *
417    * @throws IOException e
418    * @throws DeserializationException
419    */
420   public static void checkVersion(FileSystem fs, Path rootdir, boolean message)
421   throws IOException, DeserializationException {
422     checkVersion(fs, rootdir, message, 0, HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);
423   }
424 
425   /**
426    * Verifies current version of file system
427    *
428    * @param fs file system
429    * @param rootdir root directory of HBase installation
430    * @param message if true, issues a message on System.out
431    * @param wait wait interval
432    * @param retries number of times to retry
433    *
434    * @throws IOException e
435    * @throws DeserializationException
436    */
437   public static void checkVersion(FileSystem fs, Path rootdir,
438       boolean message, int wait, int retries)
439   throws IOException, DeserializationException {
440     String version = getVersion(fs, rootdir);
441     if (version == null) {
442       if (!metaRegionExists(fs, rootdir)) {
443         // rootDir is empty (no version file and no root region)
444         // just create new version file (HBASE-1195)
445         setVersion(fs, rootdir, wait, retries);
446         return;
447       }
448     } else if (version.compareTo(HConstants.FILE_SYSTEM_VERSION) == 0) return;
449 
450     // version is deprecated require migration
451     // Output on stdout so user sees it in terminal.
452     String msg = "File system needs to be upgraded."
453       + "  You have version " + version
454       + " and I want version " + HConstants.FILE_SYSTEM_VERSION
455       + ".  Run the '${HBASE_HOME}/bin/hbase migrate' script.";
456     if (message) {
457       System.out.println("WARNING! " + msg);
458     }
459     throw new FileSystemVersionException(msg);
460   }
461 
462   /**
463    * Sets version of file system
464    *
465    * @param fs filesystem object
466    * @param rootdir hbase root
467    * @throws IOException e
468    */
469   public static void setVersion(FileSystem fs, Path rootdir)
470   throws IOException {
471     setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, 0,
472       HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);
473   }
474 
475   /**
476    * Sets version of file system
477    *
478    * @param fs filesystem object
479    * @param rootdir hbase root
480    * @param wait time to wait for retry
481    * @param retries number of times to retry before failing
482    * @throws IOException e
483    */
484   public static void setVersion(FileSystem fs, Path rootdir, int wait, int retries)
485   throws IOException {
486     setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, wait, retries);
487   }
488 
489 
490   /**
491    * Sets version of file system
492    *
493    * @param fs filesystem object
494    * @param rootdir hbase root directory
495    * @param version version to set
496    * @param wait time to wait for retry
497    * @param retries number of times to retry before throwing an IOException
498    * @throws IOException e
499    */
500   public static void setVersion(FileSystem fs, Path rootdir, String version,
501       int wait, int retries) throws IOException {
502     Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
503     while (true) {
504       try {
505         FSDataOutputStream s = fs.create(versionFile);
506         s.write(toVersionByteArray(version));
507         s.close();
508         LOG.debug("Created version file at " + rootdir.toString() + " with version=" + version);
509         return;
510       } catch (IOException e) {
511         if (retries > 0) {
512           LOG.warn("Unable to create version file at " + rootdir.toString() + ", retrying", e);
513           fs.delete(versionFile, false);
514           try {
515             if (wait > 0) {
516               Thread.sleep(wait);
517             }
518           } catch (InterruptedException ex) {
519             // ignore
520           }
521           retries--;
522         } else {
523           throw e;
524         }
525       }
526     }
527   }
528 
529   /**
530    * Checks that a cluster ID file exists in the HBase root directory
531    * @param fs the root directory FileSystem
532    * @param rootdir the HBase root directory in HDFS
533    * @param wait how long to wait between retries
534    * @return <code>true</code> if the file exists, otherwise <code>false</code>
535    * @throws IOException if checking the FileSystem fails
536    */
537   public static boolean checkClusterIdExists(FileSystem fs, Path rootdir,
538       int wait) throws IOException {
539     while (true) {
540       try {
541         Path filePath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
542         return fs.exists(filePath);
543       } catch (IOException ioe) {
544         if (wait > 0) {
545           LOG.warn("Unable to check cluster ID file in " + rootdir.toString() +
546               ", retrying in "+wait+"msec: "+StringUtils.stringifyException(ioe));
547           try {
548             Thread.sleep(wait);
549           } catch (InterruptedException ie) {
550             Thread.interrupted();
551             break;
552           }
553         } else {
554           throw ioe;
555         }
556       }
557     }
558     return false;
559   }
560 
561   /**
562    * Returns the value of the unique cluster ID stored for this HBase instance.
563    * @param fs the root directory FileSystem
564    * @param rootdir the path to the HBase root directory
565    * @return the unique cluster identifier
566    * @throws IOException if reading the cluster ID file fails
567    */
568   public static ClusterId getClusterId(FileSystem fs, Path rootdir)
569   throws IOException {
570     Path idPath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
571     ClusterId clusterId = null;
572     FileStatus status = fs.exists(idPath)? fs.getFileStatus(idPath):  null;
573     if (status != null) {
574       int len = Ints.checkedCast(status.getLen());
575       byte [] content = new byte[len];
576       FSDataInputStream in = fs.open(idPath);
577       try {
578         in.readFully(content);
579       } catch (EOFException eof) {
580         LOG.warn("Cluster ID file " + idPath.toString() + " was empty");
581       } finally{
582         in.close();
583       }
584       try {
585         clusterId = ClusterId.parseFrom(content);
586       } catch (DeserializationException e) {
587         throw new IOException("content=" + Bytes.toString(content), e);
588       }
589       // If not pb'd, make it so.
590       if (!ProtobufUtil.isPBMagicPrefix(content)) rewriteAsPb(fs, rootdir, idPath, clusterId);
591       return clusterId;
592     } else {
593       LOG.warn("Cluster ID file does not exist at " + idPath.toString());
594     }
595     return clusterId;
596   }
597 
598   /**
599    * @param cid
600    * @throws IOException
601    */
602   private static void rewriteAsPb(final FileSystem fs, final Path rootdir, final Path p,
603       final ClusterId cid)
604   throws IOException {
605     // Rewrite the file as pb.  Move aside the old one first, write new
606     // then delete the moved-aside file.
607     Path movedAsideName = new Path(p + "." + System.currentTimeMillis());
608     if (!fs.rename(p, movedAsideName)) throw new IOException("Failed rename of " + p);
609     setClusterId(fs, rootdir, cid, 100);
610     if (!fs.delete(movedAsideName, false)) {
611       throw new IOException("Failed delete of " + movedAsideName);
612     }
613     LOG.debug("Rewrote the hbase.id file as pb");
614   }
615 
616   /**
617    * Writes a new unique identifier for this cluster to the "hbase.id" file
618    * in the HBase root directory
619    * @param fs the root directory FileSystem
620    * @param rootdir the path to the HBase root directory
621    * @param clusterId the unique identifier to store
622    * @param wait how long (in milliseconds) to wait between retries
623    * @throws IOException if writing to the FileSystem fails and no wait value
624    */
625   public static void setClusterId(FileSystem fs, Path rootdir, ClusterId clusterId,
626       int wait) throws IOException {
627     while (true) {
628       try {
629         Path filePath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
630         FSDataOutputStream s = fs.create(filePath);
631         try {
632           s.write(clusterId.toByteArray());
633         } finally {
634           s.close();
635         }
636         if (LOG.isDebugEnabled()) {
637           LOG.debug("Created cluster ID file at " + filePath.toString() + " with ID: " + clusterId);
638         }
639         return;
640       } catch (IOException ioe) {
641         if (wait > 0) {
642           LOG.warn("Unable to create cluster ID file in " + rootdir.toString() +
643               ", retrying in " + wait + "msec: " + StringUtils.stringifyException(ioe));
644           try {
645             Thread.sleep(wait);
646           } catch (InterruptedException ie) {
647             Thread.interrupted();
648             break;
649           }
650         } else {
651           throw ioe;
652         }
653       }
654     }
655   }
656 
657   /**
658    * Verifies root directory path is a valid URI with a scheme
659    *
660    * @param root root directory path
661    * @return Passed <code>root</code> argument.
662    * @throws IOException if not a valid URI with a scheme
663    */
664   public static Path validateRootPath(Path root) throws IOException {
665     try {
666       URI rootURI = new URI(root.toString());
667       String scheme = rootURI.getScheme();
668       if (scheme == null) {
669         throw new IOException("Root directory does not have a scheme");
670       }
671       return root;
672     } catch (URISyntaxException e) {
673       IOException io = new IOException("Root directory path is not a valid " +
674         "URI -- check your " + HConstants.HBASE_DIR + " configuration");
675       io.initCause(e);
676       throw io;
677     }
678   }
679 
680   /**
681    * Checks for the presence of the root path (using the provided conf object) in the given path. If
682    * it exists, this method removes it and returns the String representation of remaining relative path.
683    * @param path
684    * @param conf
685    * @return String representation of the remaining relative path
686    * @throws IOException
687    */
688   public static String removeRootPath(Path path, final Configuration conf) throws IOException {
689     Path root = FSUtils.getRootDir(conf);
690     String pathStr = path.toString();
691     // check that the path is absolute... it has the root path in it.
692     if (!pathStr.startsWith(root.toString())) return pathStr;
693     // if not, return as it is.
694     return pathStr.substring(root.toString().length() + 1);// remove the "/" too.
695   }
696 
697   /**
698    * If DFS, check safe mode and if so, wait until we clear it.
699    * @param conf configuration
700    * @param wait Sleep between retries
701    * @throws IOException e
702    */
703   public static void waitOnSafeMode(final Configuration conf,
704     final long wait)
705   throws IOException {
706     FileSystem fs = FileSystem.get(conf);
707     if (!(fs instanceof DistributedFileSystem)) return;
708     DistributedFileSystem dfs = (DistributedFileSystem)fs;
709     // Make sure dfs is not in safe mode
710     while (isInSafeMode(dfs)) {
711       LOG.info("Waiting for dfs to exit safe mode...");
712       try {
713         Thread.sleep(wait);
714       } catch (InterruptedException e) {
715         //continue
716       }
717     }
718   }
719 
720   /**
721    * Return the 'path' component of a Path.  In Hadoop, Path is an URI.  This
722    * method returns the 'path' component of a Path's URI: e.g. If a Path is
723    * <code>hdfs://example.org:9000/hbase_trunk/TestTable/compaction.dir</code>,
724    * this method returns <code>/hbase_trunk/TestTable/compaction.dir</code>.
725    * This method is useful if you want to print out a Path without qualifying
726    * Filesystem instance.
727    * @param p Filesystem Path whose 'path' component we are to return.
728    * @return Path portion of the Filesystem
729    */
730   public static String getPath(Path p) {
731     return p.toUri().getPath();
732   }
733 
734   /**
735    * @param c configuration
736    * @return Path to hbase root directory: i.e. <code>hbase.rootdir</code> from
737    * configuration as a qualified Path.
738    * @throws IOException e
739    */
740   public static Path getRootDir(final Configuration c) throws IOException {
741     Path p = new Path(c.get(HConstants.HBASE_DIR));
742     FileSystem fs = p.getFileSystem(c);
743     return p.makeQualified(fs);
744   }
745 
746   public static void setRootDir(final Configuration c, final Path root) throws IOException {
747     c.set(HConstants.HBASE_DIR, root.toString());
748   }
749 
750   public static void setFsDefault(final Configuration c, final Path root) throws IOException {
751     c.set("fs.defaultFS", root.toString());    // for hadoop 0.21+
752     c.set("fs.default.name", root.toString()); // for hadoop 0.20
753   }
754 
755   /**
756    * Checks if root region exists
757    *
758    * @param fs file system
759    * @param rootdir root directory of HBase installation
760    * @return true if exists
761    * @throws IOException e
762    */
763   public static boolean metaRegionExists(FileSystem fs, Path rootdir)
764   throws IOException {
765     Path rootRegionDir =
766       HRegion.getRegionDir(rootdir, HRegionInfo.FIRST_META_REGIONINFO);
767     return fs.exists(rootRegionDir);
768   }
769 
770   /**
771    * Compute HDFS blocks distribution of a given file, or a portion of the file
772    * @param fs file system
773    * @param status file status of the file
774    * @param start start position of the portion
775    * @param length length of the portion
776    * @return The HDFS blocks distribution
777    */
778   static public HDFSBlocksDistribution computeHDFSBlocksDistribution(
779     final FileSystem fs, FileStatus status, long start, long length)
780     throws IOException {
781     HDFSBlocksDistribution blocksDistribution = new HDFSBlocksDistribution();
782     BlockLocation [] blockLocations =
783       fs.getFileBlockLocations(status, start, length);
784     for(BlockLocation bl : blockLocations) {
785       String [] hosts = bl.getHosts();
786       long len = bl.getLength();
787       blocksDistribution.addHostsAndBlockWeight(hosts, len);
788     }
789 
790     return blocksDistribution;
791   }
792 
793 
794 
795   /**
796    * Runs through the hbase rootdir and checks all stores have only
797    * one file in them -- that is, they've been major compacted.  Looks
798    * at root and meta tables too.
799    * @param fs filesystem
800    * @param hbaseRootDir hbase root directory
801    * @return True if this hbase install is major compacted.
802    * @throws IOException e
803    */
804   public static boolean isMajorCompacted(final FileSystem fs,
805       final Path hbaseRootDir)
806   throws IOException {
807     // Presumes any directory under hbase.rootdir is a table.
808     FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, new DirFilter(fs));
809     for (FileStatus tableDir : tableDirs) {
810       // Skip the .log directory.  All others should be tables.  Inside a table,
811       // there are compaction.dir directories to skip.  Otherwise, all else
812       // should be regions.  Then in each region, should only be family
813       // directories.  Under each of these, should be one file only.
814       Path d = tableDir.getPath();
815       if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
816         continue;
817       }
818       FileStatus[] regionDirs = fs.listStatus(d, new DirFilter(fs));
819       for (FileStatus regionDir : regionDirs) {
820         Path dd = regionDir.getPath();
821         if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
822           continue;
823         }
824         // Else its a region name.  Now look in region for families.
825         FileStatus[] familyDirs = fs.listStatus(dd, new DirFilter(fs));
826         for (FileStatus familyDir : familyDirs) {
827           Path family = familyDir.getPath();
828           // Now in family make sure only one file.
829           FileStatus[] familyStatus = fs.listStatus(family);
830           if (familyStatus.length > 1) {
831             LOG.debug(family.toString() + " has " + familyStatus.length +
832                 " files.");
833             return false;
834           }
835         }
836       }
837     }
838     return true;
839   }
840 
841   // TODO move this method OUT of FSUtils. No dependencies to HMaster
842   /**
843    * Returns the total overall fragmentation percentage. Includes .META. and
844    * -ROOT- as well.
845    *
846    * @param master  The master defining the HBase root and file system.
847    * @return A map for each table and its percentage.
848    * @throws IOException When scanning the directory fails.
849    */
850   public static int getTotalTableFragmentation(final HMaster master)
851   throws IOException {
852     Map<String, Integer> map = getTableFragmentation(master);
853     return map != null && map.size() > 0 ? map.get("-TOTAL-") : -1;
854   }
855 
856   /**
857    * Runs through the HBase rootdir and checks how many stores for each table
858    * have more than one file in them. Checks -ROOT- and .META. too. The total
859    * percentage across all tables is stored under the special key "-TOTAL-".
860    *
861    * @param master  The master defining the HBase root and file system.
862    * @return A map for each table and its percentage.
863    *
864    * @throws IOException When scanning the directory fails.
865    */
866   public static Map<String, Integer> getTableFragmentation(
867     final HMaster master)
868   throws IOException {
869     Path path = getRootDir(master.getConfiguration());
870     // since HMaster.getFileSystem() is package private
871     FileSystem fs = path.getFileSystem(master.getConfiguration());
872     return getTableFragmentation(fs, path);
873   }
874 
875   /**
876    * Runs through the HBase rootdir and checks how many stores for each table
877    * have more than one file in them. Checks -ROOT- and .META. too. The total
878    * percentage across all tables is stored under the special key "-TOTAL-".
879    *
880    * @param fs  The file system to use.
881    * @param hbaseRootDir  The root directory to scan.
882    * @return A map for each table and its percentage.
883    * @throws IOException When scanning the directory fails.
884    */
885   public static Map<String, Integer> getTableFragmentation(
886     final FileSystem fs, final Path hbaseRootDir)
887   throws IOException {
888     Map<String, Integer> frags = new HashMap<String, Integer>();
889     int cfCountTotal = 0;
890     int cfFragTotal = 0;
891     DirFilter df = new DirFilter(fs);
892     // presumes any directory under hbase.rootdir is a table
893     FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, df);
894     for (FileStatus tableDir : tableDirs) {
895       // Skip the .log directory.  All others should be tables.  Inside a table,
896       // there are compaction.dir directories to skip.  Otherwise, all else
897       // should be regions.  Then in each region, should only be family
898       // directories.  Under each of these, should be one file only.
899       Path d = tableDir.getPath();
900       if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
901         continue;
902       }
903       int cfCount = 0;
904       int cfFrag = 0;
905       FileStatus[] regionDirs = fs.listStatus(d, df);
906       for (FileStatus regionDir : regionDirs) {
907         Path dd = regionDir.getPath();
908         if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
909           continue;
910         }
911         // else its a region name, now look in region for families
912         FileStatus[] familyDirs = fs.listStatus(dd, df);
913         for (FileStatus familyDir : familyDirs) {
914           cfCount++;
915           cfCountTotal++;
916           Path family = familyDir.getPath();
917           // now in family make sure only one file
918           FileStatus[] familyStatus = fs.listStatus(family);
919           if (familyStatus.length > 1) {
920             cfFrag++;
921             cfFragTotal++;
922           }
923         }
924       }
925       // compute percentage per table and store in result list
926       frags.put(d.getName(), Math.round((float) cfFrag / cfCount * 100));
927     }
928     // set overall percentage for all tables
929     frags.put("-TOTAL-", Math.round((float) cfFragTotal / cfCountTotal * 100));
930     return frags;
931   }
932 
933   /**
934    * Expects to find -ROOT- directory.
935    * @param fs filesystem
936    * @param hbaseRootDir hbase root directory
937    * @return True if this a pre020 layout.
938    * @throws IOException e
939    */
940   public static boolean isPre020FileLayout(final FileSystem fs,
941     final Path hbaseRootDir)
942   throws IOException {
943     Path mapfiles = new Path(new Path(new Path(new Path(hbaseRootDir, "-ROOT-"),
944       "70236052"), "info"), "mapfiles");
945     return fs.exists(mapfiles);
946   }
947 
948   /**
949    * Runs through the hbase rootdir and checks all stores have only
950    * one file in them -- that is, they've been major compacted.  Looks
951    * at root and meta tables too.  This version differs from
952    * {@link #isMajorCompacted(FileSystem, Path)} in that it expects a
953    * pre-0.20.0 hbase layout on the filesystem.  Used migrating.
954    * @param fs filesystem
955    * @param hbaseRootDir hbase root directory
956    * @return True if this hbase install is major compacted.
957    * @throws IOException e
958    */
959   public static boolean isMajorCompactedPre020(final FileSystem fs,
960       final Path hbaseRootDir)
961   throws IOException {
962     // Presumes any directory under hbase.rootdir is a table.
963     FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, new DirFilter(fs));
964     for (FileStatus tableDir : tableDirs) {
965       // Inside a table, there are compaction.dir directories to skip.
966       // Otherwise, all else should be regions.  Then in each region, should
967       // only be family directories.  Under each of these, should be a mapfile
968       // and info directory and in these only one file.
969       Path d = tableDir.getPath();
970       if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
971         continue;
972       }
973       FileStatus[] regionDirs = fs.listStatus(d, new DirFilter(fs));
974       for (FileStatus regionDir : regionDirs) {
975         Path dd = regionDir.getPath();
976         if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
977           continue;
978         }
979         // Else its a region name.  Now look in region for families.
980         FileStatus[] familyDirs = fs.listStatus(dd, new DirFilter(fs));
981         for (FileStatus familyDir : familyDirs) {
982           Path family = familyDir.getPath();
983           FileStatus[] infoAndMapfile = fs.listStatus(family);
984           // Assert that only info and mapfile in family dir.
985           if (infoAndMapfile.length != 0 && infoAndMapfile.length != 2) {
986             LOG.debug(family.toString() +
987                 " has more than just info and mapfile: " + infoAndMapfile.length);
988             return false;
989           }
990           // Make sure directory named info or mapfile.
991           for (int ll = 0; ll < 2; ll++) {
992             if (infoAndMapfile[ll].getPath().getName().equals("info") ||
993                 infoAndMapfile[ll].getPath().getName().equals("mapfiles"))
994               continue;
995             LOG.debug("Unexpected directory name: " +
996                 infoAndMapfile[ll].getPath());
997             return false;
998           }
999           // Now in family, there are 'mapfile' and 'info' subdirs.  Just
1000           // look in the 'mapfile' subdir.
1001           FileStatus[] familyStatus =
1002               fs.listStatus(new Path(family, "mapfiles"));
1003           if (familyStatus.length > 1) {
1004             LOG.debug(family.toString() + " has " + familyStatus.length +
1005                 " files.");
1006             return false;
1007           }
1008         }
1009       }
1010     }
1011     return true;
1012   }
1013 
1014   /**
1015    * A {@link PathFilter} that returns only regular files.
1016    */
1017   static class FileFilter implements PathFilter {
1018     private final FileSystem fs;
1019 
1020     public FileFilter(final FileSystem fs) {
1021       this.fs = fs;
1022     }
1023 
1024     @Override
1025     public boolean accept(Path p) {
1026       try {
1027         return fs.isFile(p);
1028       } catch (IOException e) {
1029         LOG.debug("unable to verify if path=" + p + " is a regular file", e);
1030         return false;
1031       }
1032     }
1033   }
1034 
1035   /**
1036    * A {@link PathFilter} that returns directories.
1037    */
1038   public static class DirFilter implements PathFilter {
1039     private final FileSystem fs;
1040 
1041     public DirFilter(final FileSystem fs) {
1042       this.fs = fs;
1043     }
1044 
1045     @Override
1046     public boolean accept(Path p) {
1047       boolean isValid = false;
1048       try {
1049         if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(p.toString())) {
1050           isValid = false;
1051         } else {
1052           isValid = fs.getFileStatus(p).isDir();
1053         }
1054       } catch (IOException e) {
1055         LOG.warn("An error occurred while verifying if [" + p.toString() + 
1056                  "] is a valid directory. Returning 'not valid' and continuing.", e);
1057       }
1058       return isValid;
1059     }
1060   }
1061 
1062   /**
1063    * Heuristic to determine whether is safe or not to open a file for append
1064    * Looks both for dfs.support.append and use reflection to search
1065    * for SequenceFile.Writer.syncFs() or FSDataOutputStream.hflush()
1066    * @param conf
1067    * @return True if append support
1068    */
1069   public static boolean isAppendSupported(final Configuration conf) {
1070     boolean append = conf.getBoolean("dfs.support.append", false);
1071     if (append) {
1072       try {
1073         // TODO: The implementation that comes back when we do a createWriter
1074         // may not be using SequenceFile so the below is not a definitive test.
1075         // Will do for now (hdfs-200).
1076         SequenceFile.Writer.class.getMethod("syncFs", new Class<?> []{});
1077         append = true;
1078       } catch (SecurityException e) {
1079       } catch (NoSuchMethodException e) {
1080         append = false;
1081       }
1082     }
1083     if (!append) {
1084       // Look for the 0.21, 0.22, new-style append evidence.
1085       try {
1086         FSDataOutputStream.class.getMethod("hflush", new Class<?> []{});
1087         append = true;
1088       } catch (NoSuchMethodException e) {
1089         append = false;
1090       }
1091     }
1092     return append;
1093   }
1094 
1095   /**
1096    * @param conf
1097    * @return True if this filesystem whose scheme is 'hdfs'.
1098    * @throws IOException
1099    */
1100   public static boolean isHDFS(final Configuration conf) throws IOException {
1101     FileSystem fs = FileSystem.get(conf);
1102     String scheme = fs.getUri().getScheme();
1103     return scheme.equalsIgnoreCase("hdfs");
1104   }
1105 
1106   /**
1107    * Recover file lease. Used when a file might be suspect
1108    * to be had been left open by another process.
1109    * @param fs FileSystem handle
1110    * @param p Path of file to recover lease
1111    * @param conf Configuration handle
1112    * @throws IOException
1113    */
1114   public abstract void recoverFileLease(final FileSystem fs, final Path p,
1115       Configuration conf) throws IOException;
1116 
1117   /**
1118    * @param fs
1119    * @param rootdir
1120    * @return All the table directories under <code>rootdir</code>. Ignore non table hbase folders such as
1121    * .logs, .oldlogs, .corrupt, .META., and -ROOT- folders.
1122    * @throws IOException
1123    */
1124   public static List<Path> getTableDirs(final FileSystem fs, final Path rootdir)
1125   throws IOException {
1126     // presumes any directory under hbase.rootdir is a table
1127     FileStatus [] dirs = fs.listStatus(rootdir, new DirFilter(fs));
1128     List<Path> tabledirs = new ArrayList<Path>(dirs.length);
1129     for (FileStatus dir: dirs) {
1130       Path p = dir.getPath();
1131       String tableName = p.getName();
1132       if (!HConstants.HBASE_NON_USER_TABLE_DIRS.contains(tableName)) {
1133         tabledirs.add(p);
1134       }
1135     }
1136     return tabledirs;
1137   }
1138 
1139   public static Path getTablePath(Path rootdir, byte [] tableName) {
1140     return getTablePath(rootdir, Bytes.toString(tableName));
1141   }
1142 
1143   public static Path getTablePath(Path rootdir, final String tableName) {
1144     return new Path(rootdir, tableName);
1145   }
1146 
1147   /**
1148    * Filter for all dirs that don't start with '.'
1149    */
1150   public static class RegionDirFilter implements PathFilter {
1151     // This pattern will accept 0.90+ style hex region dirs and older numeric region dir names.
1152     final public static Pattern regionDirPattern = Pattern.compile("^[0-9a-f]*$");
1153     final FileSystem fs;
1154 
1155     public RegionDirFilter(FileSystem fs) {
1156       this.fs = fs;
1157     }
1158 
1159     @Override
1160     public boolean accept(Path rd) {
1161       if (!regionDirPattern.matcher(rd.getName()).matches()) {
1162         return false;
1163       }
1164 
1165       try {
1166         return fs.getFileStatus(rd).isDir();
1167       } catch (IOException ioe) {
1168         // Maybe the file was moved or the fs was disconnected.
1169         LOG.warn("Skipping file " + rd +" due to IOException", ioe);
1170         return false;
1171       }
1172     }
1173   }
1174 
1175   /**
1176    * Given a particular table dir, return all the regiondirs inside it, excluding files such as
1177    * .tableinfo
1178    * @param fs A file system for the Path
1179    * @param tableDir Path to a specific table directory <hbase.rootdir>/<tabledir>
1180    * @return List of paths to valid region directories in table dir.
1181    * @throws IOException
1182    */
1183   public static List<Path> getRegionDirs(final FileSystem fs, final Path tableDir) throws IOException {
1184     // assumes we are in a table dir.
1185     FileStatus[] rds = fs.listStatus(tableDir, new RegionDirFilter(fs));
1186     List<Path> regionDirs = new ArrayList<Path>(rds.length);
1187     for (FileStatus rdfs: rds) {
1188       Path rdPath = rdfs.getPath();
1189       regionDirs.add(rdPath);
1190     }
1191     return regionDirs;
1192   }
1193 
1194   /**
1195    * Filter for all dirs that are legal column family names.  This is generally used for colfam
1196    * dirs <hbase.rootdir>/<tabledir>/<regiondir>/<colfamdir>.
1197    */
1198   public static class FamilyDirFilter implements PathFilter {
1199     final FileSystem fs;
1200 
1201     public FamilyDirFilter(FileSystem fs) {
1202       this.fs = fs;
1203     }
1204 
1205     @Override
1206     public boolean accept(Path rd) {
1207       try {
1208         // throws IAE if invalid
1209         HColumnDescriptor.isLegalFamilyName(Bytes.toBytes(rd.getName()));
1210       } catch (IllegalArgumentException iae) {
1211         // path name is an invalid family name and thus is excluded.
1212         return false;
1213       }
1214 
1215       try {
1216         return fs.getFileStatus(rd).isDir();
1217       } catch (IOException ioe) {
1218         // Maybe the file was moved or the fs was disconnected.
1219         LOG.warn("Skipping file " + rd +" due to IOException", ioe);
1220         return false;
1221       }
1222     }
1223   }
1224 
1225   /**
1226    * Given a particular region dir, return all the familydirs inside it
1227    *
1228    * @param fs A file system for the Path
1229    * @param regionDir Path to a specific region directory
1230    * @return List of paths to valid family directories in region dir.
1231    * @throws IOException
1232    */
1233   public static List<Path> getFamilyDirs(final FileSystem fs, final Path regionDir) throws IOException {
1234     // assumes we are in a region dir.
1235     FileStatus[] fds = fs.listStatus(regionDir, new FamilyDirFilter(fs));
1236     List<Path> familyDirs = new ArrayList<Path>(fds.length);
1237     for (FileStatus fdfs: fds) {
1238       Path fdPath = fdfs.getPath();
1239       familyDirs.add(fdPath);
1240     }
1241     return familyDirs;
1242   }
1243 
1244   /**
1245    * Filter for HFiles that excludes reference files.
1246    */
1247   public static class HFileFilter implements PathFilter {
1248     // This pattern will accept 0.90+ style hex hfies files but reject reference files
1249     final public static Pattern hfilePattern = Pattern.compile("^([0-9a-f]+)$");
1250 
1251     final FileSystem fs;
1252 
1253     public HFileFilter(FileSystem fs) {
1254       this.fs = fs;
1255     }
1256 
1257     @Override
1258     public boolean accept(Path rd) {
1259       if (!hfilePattern.matcher(rd.getName()).matches()) {
1260         return false;
1261       }
1262 
1263       try {
1264         // only files
1265         return !fs.getFileStatus(rd).isDir();
1266       } catch (IOException ioe) {
1267         // Maybe the file was moved or the fs was disconnected.
1268         LOG.warn("Skipping file " + rd +" due to IOException", ioe);
1269         return false;
1270       }
1271     }
1272   }
1273 
1274   /**
1275    * @param conf
1276    * @return Returns the filesystem of the hbase rootdir.
1277    * @throws IOException
1278    */
1279   public static FileSystem getCurrentFileSystem(Configuration conf)
1280   throws IOException {
1281     return getRootDir(conf).getFileSystem(conf);
1282   }
1283 
1284   /**
1285    * Runs through the HBase rootdir and creates a reverse lookup map for
1286    * table StoreFile names to the full Path.
1287    * <br>
1288    * Example...<br>
1289    * Key = 3944417774205889744  <br>
1290    * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
1291    *
1292    * @param fs  The file system to use.
1293    * @param hbaseRootDir  The root directory to scan.
1294    * @return Map keyed by StoreFile name with a value of the full Path.
1295    * @throws IOException When scanning the directory fails.
1296    */
1297   public static Map<String, Path> getTableStoreFilePathMap(
1298     final FileSystem fs, final Path hbaseRootDir)
1299   throws IOException {
1300     Map<String, Path> map = new HashMap<String, Path>();
1301 
1302     // if this method looks similar to 'getTableFragmentation' that is because
1303     // it was borrowed from it.
1304     
1305     DirFilter df = new DirFilter(fs);
1306     // presumes any directory under hbase.rootdir is a table
1307     FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, df);
1308     for (FileStatus tableDir : tableDirs) {
1309       // Skip the .log and other non-table directories.  All others should be tables.
1310       // Inside a table, there are compaction.dir directories to skip.  Otherwise, all else
1311       // should be regions. 
1312       Path d = tableDir.getPath();
1313       if (HConstants.HBASE_NON_TABLE_DIRS.contains(d.getName())) {
1314         continue;
1315       }
1316       FileStatus[] regionDirs = fs.listStatus(d, df);
1317       for (FileStatus regionDir : regionDirs) {
1318         Path dd = regionDir.getPath();
1319         if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
1320           continue;
1321         }
1322         // else its a region name, now look in region for families
1323         FileStatus[] familyDirs = fs.listStatus(dd, df);
1324         for (FileStatus familyDir : familyDirs) {
1325           Path family = familyDir.getPath();
1326           // now in family, iterate over the StoreFiles and
1327           // put in map
1328           FileStatus[] familyStatus = fs.listStatus(family);
1329           for (FileStatus sfStatus : familyStatus) {
1330             Path sf = sfStatus.getPath();
1331             map.put( sf.getName(), sf);
1332           }
1333 
1334         }
1335       }
1336     }
1337       return map;
1338   }
1339 
1340   /**
1341    * Calls fs.listStatus() and treats FileNotFoundException as non-fatal
1342    * This accommodates differences between hadoop versions
1343    *
1344    * @param fs file system
1345    * @param dir directory
1346    * @param filter path filter
1347    * @return null if tabledir doesn't exist, otherwise FileStatus array
1348    */
1349   public static FileStatus [] listStatus(final FileSystem fs,
1350       final Path dir, final PathFilter filter) throws IOException {
1351     FileStatus [] status = null;
1352     try {
1353       status = filter == null ? fs.listStatus(dir) : fs.listStatus(dir, filter);
1354     } catch (FileNotFoundException fnfe) {
1355       // if directory doesn't exist, return null
1356       LOG.debug(dir + " doesn't exist");
1357     }
1358     if (status == null || status.length < 1) return null;
1359     return status;
1360   }
1361 
1362   /**
1363    * Calls fs.listStatus() and treats FileNotFoundException as non-fatal
1364    * This would accommodates differences between hadoop versions
1365    *
1366    * @param fs file system
1367    * @param dir directory
1368    * @return null if tabledir doesn't exist, otherwise FileStatus array
1369    */
1370   public static FileStatus[] listStatus(final FileSystem fs, final Path dir) throws IOException {
1371     return listStatus(fs, dir, null);
1372   }
1373 
1374   /**
1375    * Calls fs.delete() and returns the value returned by the fs.delete()
1376    *
1377    * @param fs
1378    * @param path
1379    * @param recursive
1380    * @return the value returned by the fs.delete()
1381    * @throws IOException
1382    */
1383   public static boolean delete(final FileSystem fs, final Path path, final boolean recursive)
1384       throws IOException {
1385     return fs.delete(path, recursive);
1386   }
1387 
1388   /**
1389    * Calls fs.exists(). Checks if the specified path exists
1390    *
1391    * @param fs
1392    * @param path
1393    * @return the value returned by fs.exists()
1394    * @throws IOException
1395    */
1396   public static boolean isExists(final FileSystem fs, final Path path) throws IOException {
1397     return fs.exists(path);
1398   }
1399 
1400   /**
1401    * Throw an exception if an action is not permitted by a user on a file.
1402    *
1403    * @param ugi
1404    *          the user
1405    * @param file
1406    *          the file
1407    * @param action
1408    *          the action
1409    */
1410   public static void checkAccess(UserGroupInformation ugi, FileStatus file,
1411       FsAction action) throws AccessControlException {
1412     if (ugi.getShortUserName().equals(file.getOwner())) {
1413       if (file.getPermission().getUserAction().implies(action)) {
1414         return;
1415       }
1416     } else if (contains(ugi.getGroupNames(), file.getGroup())) {
1417       if (file.getPermission().getGroupAction().implies(action)) {
1418         return;
1419       }
1420     } else if (file.getPermission().getOtherAction().implies(action)) {
1421       return;
1422     }
1423     throw new AccessControlException("Permission denied:" + " action=" + action
1424         + " path=" + file.getPath() + " user=" + ugi.getShortUserName());
1425   }
1426 
1427   private static boolean contains(String[] groups, String user) {
1428     for (String group : groups) {
1429       if (group.equals(user)) {
1430         return true;
1431       }
1432     }
1433     return false;
1434   }
1435 
1436   /**
1437    * Log the current state of the filesystem from a certain root directory
1438    * @param fs filesystem to investigate
1439    * @param root root file/directory to start logging from
1440    * @param LOG log to output information
1441    * @throws IOException if an unexpected exception occurs
1442    */
1443   public static void logFileSystemState(final FileSystem fs, final Path root, Log LOG)
1444       throws IOException {
1445     LOG.debug("Current file system:");
1446     logFSTree(LOG, fs, root, "|-");
1447   }
1448 
1449   /**
1450    * Recursive helper to log the state of the FS
1451    *
1452    * @see #logFileSystemState(FileSystem, Path, Log)
1453    */
1454   private static void logFSTree(Log LOG, final FileSystem fs, final Path root, String prefix)
1455       throws IOException {
1456     FileStatus[] files = FSUtils.listStatus(fs, root, null);
1457     if (files == null) return;
1458 
1459     for (FileStatus file : files) {
1460       if (file.isDir()) {
1461         LOG.debug(prefix + file.getPath().getName() + "/");
1462         logFSTree(LOG, fs, file.getPath(), prefix + "---");
1463       } else {
1464         LOG.debug(prefix + file.getPath().getName());
1465       }
1466     }
1467   }
1468 }