public abstract class FSUtils extends Object
Modifier and Type | Class and Description |
---|---|
static class |
FSUtils.DirFilter
A
PathFilter that returns directories. |
static class |
FSUtils.FamilyDirFilter
Filter for all dirs that are legal column family names.
|
static class |
FSUtils.HFileFilter
Filter for HFiles that excludes reference files.
|
static class |
FSUtils.RegionDirFilter
Filter for all dirs that don't start with '.'
|
Modifier | Constructor and Description |
---|---|
protected |
FSUtils() |
Modifier and Type | Method and Description |
---|---|
static void |
checkAccess(User user,
org.apache.hadoop.fs.FileStatus file,
org.apache.hadoop.fs.permission.FsAction action)
Throw an exception if an action is not permitted by a user on a file.
|
static boolean |
checkClusterIdExists(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path rootdir,
int wait)
Checks that a cluster ID file exists in the HBase root directory
|
static void |
checkDfsSafeMode(org.apache.hadoop.conf.Configuration conf)
Check whether dfs is in safemode.
|
org.apache.hadoop.fs.Path |
checkdir(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path dir)
Check if directory exists.
|
static void |
checkFileSystemAvailable(org.apache.hadoop.fs.FileSystem fs)
Checks to see if the specified file system is available
|
static void |
checkVersion(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path rootdir,
boolean message)
Verifies current version of file system
|
static void |
checkVersion(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path rootdir,
boolean message,
int wait,
int retries)
Verifies current version of file system
|
static HDFSBlocksDistribution |
computeHDFSBlocksDistribution(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.FileStatus status,
long start,
long length)
Compute HDFS blocks distribution of a given file, or a portion of the file
|
static org.apache.hadoop.fs.FSDataOutputStream |
create(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path,
org.apache.hadoop.fs.permission.FsPermission perm)
Create the specified file on the filesystem.
|
static org.apache.hadoop.fs.FSDataOutputStream |
create(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path,
org.apache.hadoop.fs.permission.FsPermission perm,
boolean overwrite)
Create the specified file on the filesystem.
|
static boolean |
delete(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path,
boolean recursive)
Calls fs.delete() and returns the value returned by the fs.delete()
|
static boolean |
deleteDirectory(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path dir)
Delete if exists.
|
static String |
getClusterId(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path rootdir)
Returns the value of the unique cluster ID stored for this HBase instance.
|
static org.apache.hadoop.fs.FileSystem |
getCurrentFileSystem(org.apache.hadoop.conf.Configuration conf) |
static long |
getDefaultBlockSize(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path)
Return the number of bytes that large input files should be optimally
be split into to minimize i/o time.
|
static int |
getDefaultBufferSize(org.apache.hadoop.fs.FileSystem fs)
Returns the default buffer size to use during writes.
|
static short |
getDefaultReplication(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path) |
static List<org.apache.hadoop.fs.Path> |
getFamilyDirs(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path regionDir)
Given a particular region dir, return all the familydirs inside it
|
static org.apache.hadoop.fs.permission.FsPermission |
getFilePermissions(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.conf.Configuration conf,
String permssionConfKey)
Get the file permissions specified in the configuration, if they are
enabled.
|
static FSUtils |
getInstance(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.conf.Configuration conf) |
static String |
getPath(org.apache.hadoop.fs.Path p)
Return the 'path' component of a Path.
|
static List<org.apache.hadoop.fs.Path> |
getRegionDirs(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path tableDir)
Given a particular table dir, return all the regiondirs inside it, excluding files such as
.tableinfo
|
static org.apache.hadoop.fs.Path |
getRootDir(org.apache.hadoop.conf.Configuration c) |
static List<org.apache.hadoop.fs.Path> |
getTableDirs(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path rootdir) |
static Map<String,Integer> |
getTableFragmentation(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path hbaseRootDir)
Runs through the HBase rootdir and checks how many stores for each table
have more than one file in them.
|
static Map<String,Integer> |
getTableFragmentation(HMaster master)
Runs through the HBase rootdir and checks how many stores for each table
have more than one file in them.
|
static org.apache.hadoop.fs.Path |
getTablePath(org.apache.hadoop.fs.Path rootdir,
byte[] tableName) |
static org.apache.hadoop.fs.Path |
getTablePath(org.apache.hadoop.fs.Path rootdir,
String tableName) |
static Map<String,org.apache.hadoop.fs.Path> |
getTableStoreFilePathMap(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path hbaseRootDir)
Runs through the HBase rootdir and creates a reverse lookup map for
table StoreFile names to the full Path.
|
static int |
getTotalTableFragmentation(HMaster master)
Returns the total overall fragmentation percentage.
|
static String |
getVersion(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path rootdir)
Verifies current version of file system
|
static boolean |
isAppendSupported(org.apache.hadoop.conf.Configuration conf)
Heuristic to determine whether is safe or not to open a file for append
Looks both for dfs.support.append and use reflection to search
for SequenceFile.Writer.syncFs() or FSDataOutputStream.hflush()
|
static boolean |
isExists(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path)
Calls fs.exists().
|
static boolean |
isHDFS(org.apache.hadoop.conf.Configuration conf) |
static boolean |
isMajorCompacted(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path hbaseRootDir)
Runs through the hbase rootdir and checks all stores have only
one file in them -- that is, they've been major compacted.
|
static boolean |
isMajorCompactedPre020(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path hbaseRootDir)
Runs through the hbase rootdir and checks all stores have only
one file in them -- that is, they've been major compacted.
|
static boolean |
isPre020FileLayout(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path hbaseRootDir)
Expects to find -ROOT- directory.
|
static org.apache.hadoop.fs.FileStatus[] |
listStatus(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path dir)
Calls fs.listStatus() and treats FileNotFoundException as non-fatal
This would accommodates differences between hadoop versions
|
static org.apache.hadoop.fs.FileStatus[] |
listStatus(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path dir,
org.apache.hadoop.fs.PathFilter filter)
Calls fs.listStatus() and treats FileNotFoundException as non-fatal
This accommodates differences between hadoop versions
|
static void |
logFileSystemState(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path root,
org.apache.commons.logging.Log LOG)
Log the current state of the filesystem from a certain root directory
|
abstract void |
recoverFileLease(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path p,
org.apache.hadoop.conf.Configuration conf)
Recover file lease.
|
static boolean |
rootRegionExists(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path rootdir)
Checks if root region exists
|
static void |
setClusterId(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path rootdir,
String clusterId,
int wait)
Writes a new unique identifier for this cluster to the "hbase.id" file
in the HBase root directory
|
static void |
setFsDefault(org.apache.hadoop.conf.Configuration c,
org.apache.hadoop.fs.Path root) |
static void |
setRootDir(org.apache.hadoop.conf.Configuration c,
org.apache.hadoop.fs.Path root) |
static void |
setVersion(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path rootdir)
Sets version of file system
|
static void |
setVersion(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path rootdir,
int wait,
int retries)
Sets version of file system
|
static void |
setVersion(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path rootdir,
String version,
int wait,
int retries)
Sets version of file system
|
static org.apache.hadoop.fs.Path |
validateRootPath(org.apache.hadoop.fs.Path root)
Verifies root directory path is a valid URI with a scheme
|
static void |
waitOnSafeMode(org.apache.hadoop.conf.Configuration conf,
long wait)
If DFS, check safe mode and if so, wait until we clear it.
|
public static FSUtils getInstance(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.conf.Configuration conf)
public static boolean deleteDirectory(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path dir) throws IOException
fs
- filesystem objectdir
- directory to deletedir
IOException
- epublic org.apache.hadoop.fs.Path checkdir(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path dir) throws IOException
fs
- filesystem objectdir
- path to checkIOException
- epublic static org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.permission.FsPermission perm) throws IOException
fs
- FileSystem
on which to write the filepath
- Path
to the file to writeIOException
- if the file cannot be createdpublic static org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.permission.FsPermission perm, boolean overwrite) throws IOException
fs
- FileSystem
on which to write the filepath
- Path
to the file to writeperm
- overwrite
- Whether or not the created file should be overwritten.IOException
- if the file cannot be createdpublic static org.apache.hadoop.fs.permission.FsPermission getFilePermissions(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.conf.Configuration conf, String permssionConfKey)
fs
- filesystem that the file will be created on.conf
- configuration to read for determining if permissions are
enabled and which to usepermssionConfKey
- property key in the configuration to use when
finding the permissionpublic static void checkFileSystemAvailable(org.apache.hadoop.fs.FileSystem fs) throws IOException
fs
- filesystemIOException
- epublic static void checkDfsSafeMode(org.apache.hadoop.conf.Configuration conf) throws IOException
conf
- IOException
public static String getVersion(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir) throws IOException
fs
- filesystem objectrootdir
- root hbase directoryIOException
- epublic static void checkVersion(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir, boolean message) throws IOException
fs
- file systemrootdir
- root directory of HBase installationmessage
- if true, issues a message on System.outIOException
- epublic static void checkVersion(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir, boolean message, int wait, int retries) throws IOException
fs
- file systemrootdir
- root directory of HBase installationmessage
- if true, issues a message on System.outwait
- wait intervalretries
- number of times to retryIOException
- epublic static void setVersion(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir) throws IOException
fs
- filesystem objectrootdir
- hbase rootIOException
- epublic static void setVersion(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir, int wait, int retries) throws IOException
fs
- filesystem objectrootdir
- hbase rootwait
- time to wait for retryretries
- number of times to retry before failingIOException
- epublic static long getDefaultBlockSize(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path) throws IOException
fs
- filesystem objectIOException
- epublic static short getDefaultReplication(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path) throws IOException
IOException
public static int getDefaultBufferSize(org.apache.hadoop.fs.FileSystem fs)
fs
- filesystem objectpublic static void setVersion(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir, String version, int wait, int retries) throws IOException
fs
- filesystem objectrootdir
- hbase root directoryversion
- version to setwait
- time to wait for retryretries
- number of times to retry before throwing an IOExceptionIOException
- epublic static boolean checkClusterIdExists(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir, int wait) throws IOException
fs
- the root directory FileSystemrootdir
- the HBase root directory in HDFSwait
- how long to wait between retriestrue
if the file exists, otherwise false
IOException
- if checking the FileSystem failspublic static String getClusterId(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir) throws IOException
fs
- the root directory FileSystemrootdir
- the path to the HBase root directoryIOException
- if reading the cluster ID file failspublic static void setClusterId(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir, String clusterId, int wait) throws IOException
fs
- the root directory FileSystemrootdir
- the path to the HBase root directoryclusterId
- the unique identifier to storewait
- how long (in milliseconds) to wait between retriesIOException
- if writing to the FileSystem fails and no wait valuepublic static org.apache.hadoop.fs.Path validateRootPath(org.apache.hadoop.fs.Path root) throws IOException
root
- root directory pathroot
argument.IOException
- if not a valid URI with a schemepublic static void waitOnSafeMode(org.apache.hadoop.conf.Configuration conf, long wait) throws IOException
conf
- configurationwait
- Sleep between retriesIOException
- epublic static String getPath(org.apache.hadoop.fs.Path p)
hdfs://example.org:9000/hbase_trunk/TestTable/compaction.dir
,
this method returns /hbase_trunk/TestTable/compaction.dir
.
This method is useful if you want to print out a Path without qualifying
Filesystem instance.p
- Filesystem Path whose 'path' component we are to return.public static org.apache.hadoop.fs.Path getRootDir(org.apache.hadoop.conf.Configuration c) throws IOException
c
- configurationhbase.rootdir
from
configuration as a qualified Path.IOException
- epublic static void setRootDir(org.apache.hadoop.conf.Configuration c, org.apache.hadoop.fs.Path root) throws IOException
IOException
public static void setFsDefault(org.apache.hadoop.conf.Configuration c, org.apache.hadoop.fs.Path root) throws IOException
IOException
public static boolean rootRegionExists(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir) throws IOException
fs
- file systemrootdir
- root directory of HBase installationIOException
- epublic static HDFSBlocksDistribution computeHDFSBlocksDistribution(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.FileStatus status, long start, long length) throws IOException
fs
- file systemstatus
- file status of the filestart
- start position of the portionlength
- length of the portionIOException
public static boolean isMajorCompacted(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path hbaseRootDir) throws IOException
fs
- filesystemhbaseRootDir
- hbase root directoryIOException
- epublic static int getTotalTableFragmentation(HMaster master) throws IOException
master
- The master defining the HBase root and file system.IOException
- When scanning the directory fails.public static Map<String,Integer> getTableFragmentation(HMaster master) throws IOException
master
- The master defining the HBase root and file system.IOException
- When scanning the directory fails.public static Map<String,Integer> getTableFragmentation(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path hbaseRootDir) throws IOException
fs
- The file system to use.hbaseRootDir
- The root directory to scan.IOException
- When scanning the directory fails.public static boolean isPre020FileLayout(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path hbaseRootDir) throws IOException
fs
- filesystemhbaseRootDir
- hbase root directoryIOException
- epublic static boolean isMajorCompactedPre020(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path hbaseRootDir) throws IOException
isMajorCompacted(FileSystem, Path)
in that it expects a
pre-0.20.0 hbase layout on the filesystem. Used migrating.fs
- filesystemhbaseRootDir
- hbase root directoryIOException
- epublic static boolean isAppendSupported(org.apache.hadoop.conf.Configuration conf)
conf
- public static boolean isHDFS(org.apache.hadoop.conf.Configuration conf) throws IOException
conf
- IOException
public abstract void recoverFileLease(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path p, org.apache.hadoop.conf.Configuration conf) throws IOException
fs
- FileSystem handlep
- Path of file to recover leaseconf
- Configuration handleIOException
public static List<org.apache.hadoop.fs.Path> getTableDirs(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir) throws IOException
fs
- rootdir
- rootdir
. Ignore non table hbase folders such as
.logs, .oldlogs, .corrupt, .META., and -ROOT- folders.IOException
public static org.apache.hadoop.fs.Path getTablePath(org.apache.hadoop.fs.Path rootdir, byte[] tableName)
public static org.apache.hadoop.fs.Path getTablePath(org.apache.hadoop.fs.Path rootdir, String tableName)
public static List<org.apache.hadoop.fs.Path> getRegionDirs(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path tableDir) throws IOException
fs
- A file system for the PathtableDir
- Path to a specific table directory IOException
public static List<org.apache.hadoop.fs.Path> getFamilyDirs(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path regionDir) throws IOException
fs
- A file system for the PathregionDir
- Path to a specific region directoryIOException
public static org.apache.hadoop.fs.FileSystem getCurrentFileSystem(org.apache.hadoop.conf.Configuration conf) throws IOException
conf
- IOException
public static Map<String,org.apache.hadoop.fs.Path> getTableStoreFilePathMap(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path hbaseRootDir) throws IOException
fs
- The file system to use.hbaseRootDir
- The root directory to scan.IOException
- When scanning the directory fails.public static org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path dir, org.apache.hadoop.fs.PathFilter filter) throws IOException
fs
- file systemdir
- directoryfilter
- path filterIOException
public static org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path dir) throws IOException
fs
- file systemdir
- directoryIOException
public static boolean delete(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, boolean recursive) throws IOException
fs
- path
- recursive
- IOException
public static void checkAccess(User user, org.apache.hadoop.fs.FileStatus file, org.apache.hadoop.fs.permission.FsAction action) throws org.apache.hadoop.security.AccessControlException
user
- the userfile
- the fileaction
- the actionorg.apache.hadoop.security.AccessControlException
public static boolean isExists(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path) throws IOException
fs
- path
- IOException
public static void logFileSystemState(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path root, org.apache.commons.logging.Log LOG) throws IOException
fs
- filesystem to investigateroot
- root file/directory to start logging fromLOG
- log to output informationIOException
- if an unexpected exception occursCopyright © 2014 The Apache Software Foundation. All Rights Reserved.