|
||||||||||
PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD |
java.lang.Objectorg.apache.hadoop.conf.Configured
org.apache.hadoop.fs.FileSystem
public abstract class FileSystem
An abstract base class for a fairly generic filesystem. It may be implemented as a distributed filesystem, or as a "local" one that reflects the locally-connected disk. The local version exists for small Hadopp instances and for testing.
All user code that may potentially use the Hadoop Distributed File System should be written to use a FileSystem object. The Hadoop DFS is a multi-machine system that appears as a single disk. It's useful because of its fault tolerance and potentially very large capacity.
The local implementation is LocalFileSystem
and distributed
implementation is DistributedFileSystem
.
Field Summary | |
---|---|
static org.apache.commons.logging.Log |
LOG
|
Constructor Summary | |
---|---|
protected |
FileSystem(Configuration conf)
|
Method Summary | |
---|---|
void |
close()
No more filesystem operations are needed. |
void |
completeLocalOutput(File src,
File dst)
Deprecated. Call completeLocalOutput(Path, Path) instead. |
abstract void |
completeLocalOutput(Path fsOutputFile,
Path tmpLocalFile)
Called when we're all done writing to the target. |
abstract void |
copyFromLocalFile(Path src,
Path dst)
The src file is on the local disk. |
abstract void |
copyToLocalFile(Path src,
Path dst)
The src file is under FS, and the dst is on the local disk. |
FSDataOutputStream |
create(File f)
Deprecated. Call create(Path) instead. |
FSDataOutputStream |
create(Path f)
Opens an FSDataOutputStream at the indicated Path. |
FSDataOutputStream |
create(Path f,
boolean overwrite,
int bufferSize)
Opens an FSDataOutputStream at the indicated Path. |
FSDataOutputStream |
create(Path f,
boolean overwrite,
int bufferSize,
Progressable progress)
Opens an FSDataOutputStream at the indicated Path with write-progress reporting. |
FSDataOutputStream |
create(Path f,
boolean overwrite,
int bufferSize,
short replication,
long blockSize)
Opens an FSDataOutputStream at the indicated Path. |
FSDataOutputStream |
create(Path f,
boolean overwrite,
int bufferSize,
short replication,
long blockSize,
Progressable progress)
Opens an FSDataOutputStream at the indicated Path with write-progress reporting. |
FSDataOutputStream |
create(Path f,
Progressable progress)
Create an FSDataOutputStream at the indicated Path with write-progress reporting. |
FSDataOutputStream |
create(Path f,
short replication)
Opens an FSDataOutputStream at the indicated Path. |
FSDataOutputStream |
create(Path f,
short replication,
Progressable progress)
Opens an FSDataOutputStream at the indicated Path with write-progress reporting. |
boolean |
createNewFile(File f)
Deprecated. Call createNewFile(Path) instead. |
boolean |
createNewFile(Path f)
Creates the given Path as a brand-new zero-length file. |
abstract FSOutputStream |
createRaw(Path f,
boolean overwrite,
short replication,
long blockSize)
Opens an OutputStream at the indicated Path. |
abstract FSOutputStream |
createRaw(Path f,
boolean overwrite,
short replication,
long blockSize,
Progressable progress)
Opens an OutputStream at the indicated Path with write-progress reporting. |
boolean |
delete(File f)
Deprecated. Call delete(Path) instead. |
boolean |
delete(Path f)
Delete a file. |
abstract boolean |
deleteRaw(Path f)
Deletes Path |
boolean |
exists(File f)
Deprecated. call exists(Path) instead |
abstract boolean |
exists(Path f)
Check if exists. |
static FileSystem |
get(Configuration conf)
Returns the configured filesystem implementation. |
abstract long |
getBlockSize(Path f)
Get the size for a particular file. |
static Path |
getChecksumFile(Path file)
Return the name of the checksum file associated with a file. |
abstract long |
getDefaultBlockSize()
Return the number of bytes that large input files should be optimally be split into to minimize i/o time. |
abstract short |
getDefaultReplication()
Get the default replication. |
abstract String[][] |
getFileCacheHints(Path f,
long start,
long len)
Return a 2D array of size 1x1 or greater, containing hostnames where portions of the given file can be found. |
long |
getLength(File f)
Deprecated. Call getLength(Path) instead. |
abstract long |
getLength(Path f)
The number of bytes in a file. |
abstract String |
getName()
Returns a name for this filesystem, suitable to pass to getNamed(String,Configuration) . |
static FileSystem |
getNamed(String name,
Configuration conf)
Returns a named filesystem. |
abstract short |
getReplication(Path src)
Get replication. |
abstract Path |
getWorkingDirectory()
Get the current working directory for the given file system |
Path[] |
globPaths(Path filePattern)
Return all the files that match filePattern and are not checksum files. |
Path[] |
globPaths(Path filePattern,
PathFilter filter)
glob all the file names that matches filePattern and is accepted by filter |
static boolean |
isChecksumFile(Path file)
Return true iff file is a checksum file name. |
boolean |
isDirectory(File f)
Deprecated. Call isDirectory(Path) instead. |
abstract boolean |
isDirectory(Path f)
True iff the named path is a directory. |
boolean |
isFile(File f)
Deprecated. Call isFile(Path) instead. |
boolean |
isFile(Path f)
True iff the named path is a regular file. |
File[] |
listFiles(File f)
Deprecated. Call listPaths(Path) instead. |
File[] |
listFiles(File f,
FileFilter filt)
Deprecated. Call listPaths(Path) instead. |
Path[] |
listPaths(Path f)
List files in a directory. |
Path[] |
listPaths(Path[] files)
Filter raw files in a list directories using the default checksum filter. |
Path[] |
listPaths(Path[] files,
PathFilter filter)
Filter raw files in a list directories using user-supplied path filter. |
Path[] |
listPaths(Path f,
PathFilter filter)
Filter raw files in a directory. |
abstract Path[] |
listPathsRaw(Path f)
List files in a directory. |
void |
lock(File f,
boolean shared)
Deprecated. Call lock(Path,boolean) instead. |
abstract void |
lock(Path f,
boolean shared)
Obtain a lock on the given Path |
boolean |
mkdirs(File f)
Deprecated. Call mkdirs(Path) instead. |
abstract boolean |
mkdirs(Path f)
Make the given file and all non-existent parents into directories. |
abstract void |
moveFromLocalFile(Path src,
Path dst)
The src file is on the local disk. |
FSDataInputStream |
open(File f)
Deprecated. Call open(Path) instead. |
FSDataInputStream |
open(Path f)
Opens an FSDataInputStream at the indicated Path. |
FSDataInputStream |
open(Path f,
int bufferSize)
Opens an FSDataInputStream at the indicated Path. |
abstract FSInputStream |
openRaw(Path f)
Opens an InputStream for the indicated Path, whether local or via DFS. |
static FileSystem |
parseArgs(String[] argv,
int i,
Configuration conf)
Parse the cmd-line args, starting at i. |
void |
release(File f)
Deprecated. Call release(Path) instead. |
abstract void |
release(Path f)
Release the lock |
boolean |
rename(File src,
File dst)
Deprecated. Call rename(Path, Path) instead. |
boolean |
rename(Path src,
Path dst)
Renames Path src to Path dst. |
abstract boolean |
renameRaw(Path src,
Path dst)
Renames Path src to Path dst. |
abstract void |
reportChecksumFailure(Path f,
FSInputStream in,
long start,
long length,
int crc)
Report a checksum error to the file system. |
boolean |
setReplication(Path src,
short replication)
Set replication for an existing file. |
abstract boolean |
setReplicationRaw(Path src,
short replication)
Set replication for an existing file. |
abstract void |
setWorkingDirectory(Path new_dir)
Set the current working directory for the given file system. |
File |
startLocalOutput(File src,
File dst)
Deprecated. Call startLocalOutput(Path, Path) instead. |
abstract Path |
startLocalOutput(Path fsOutputFile,
Path tmpLocalFile)
Returns a local File that the user can write output to. |
Methods inherited from class org.apache.hadoop.conf.Configured |
---|
getConf, setConf |
Methods inherited from class java.lang.Object |
---|
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
Field Detail |
---|
public static final org.apache.commons.logging.Log LOG
Constructor Detail |
---|
protected FileSystem(Configuration conf)
Method Detail |
---|
public static FileSystem parseArgs(String[] argv, int i, Configuration conf) throws IOException
IOException
public static FileSystem get(Configuration conf) throws IOException
IOException
public abstract String getName()
getNamed(String,Configuration)
.
public static FileSystem getNamed(String name, Configuration conf) throws IOException
IOException
public static Path getChecksumFile(Path file)
public static boolean isChecksumFile(Path file)
public abstract String[][] getFileCacheHints(Path f, long start, long len) throws IOException
IOException
public FSDataInputStream open(File f) throws IOException
open(Path)
instead.
IOException
public FSDataInputStream open(Path f, int bufferSize) throws IOException
f
- the file name to openbufferSize
- the size of the buffer to be used.
IOException
public FSDataInputStream open(Path f) throws IOException
f
- the file to open
IOException
public abstract FSInputStream openRaw(Path f) throws IOException
IOException
public FSDataOutputStream create(File f) throws IOException
create(Path)
instead.
IOException
public FSDataOutputStream create(Path f) throws IOException
IOException
public FSDataOutputStream create(Path f, Progressable progress) throws IOException
IOException
public FSDataOutputStream create(Path f, short replication) throws IOException
IOException
public FSDataOutputStream create(Path f, short replication, Progressable progress) throws IOException
IOException
public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize) throws IOException
f
- the file name to openoverwrite
- if a file with this name already exists, then if true,
the file will be overwritten, and if false an error will be thrown.bufferSize
- the size of the buffer to be used.
IOException
public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, Progressable progress) throws IOException
f
- the file name to openoverwrite
- if a file with this name already exists, then if true,
the file will be overwritten, and if false an error will be thrown.bufferSize
- the size of the buffer to be used.
IOException
public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, short replication, long blockSize) throws IOException
f
- the file name to openoverwrite
- if a file with this name already exists, then if true,
the file will be overwritten, and if false an error will be thrown.bufferSize
- the size of the buffer to be used.replication
- required block replication for the file.
IOException
public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException
f
- the file name to openoverwrite
- if a file with this name already exists, then if true,
the file will be overwritten, and if false an error will be thrown.bufferSize
- the size of the buffer to be used.replication
- required block replication for the file.
IOException
public abstract FSOutputStream createRaw(Path f, boolean overwrite, short replication, long blockSize) throws IOException
f
- the file name to openoverwrite
- if a file with this name already exists, then if true,
the file will be overwritten, and if false an error will be thrown.replication
- required block replication for the file.
IOException
public abstract FSOutputStream createRaw(Path f, boolean overwrite, short replication, long blockSize, Progressable progress) throws IOException
f
- the file name to openoverwrite
- if a file with this name already exists, then if true,
the file will be overwritten, and if false an error will be thrown.replication
- required block replication for the file.
IOException
public boolean createNewFile(File f) throws IOException
createNewFile(Path)
instead.
IOException
public boolean createNewFile(Path f) throws IOException
IOException
public boolean setReplication(Path src, short replication) throws IOException
src
- file namereplication
- new replication
IOException
public abstract short getReplication(Path src) throws IOException
src
- file name
IOException
public abstract boolean setReplicationRaw(Path src, short replication) throws IOException
src
- file namereplication
- new replication
IOException
public boolean rename(File src, File dst) throws IOException
rename(Path, Path)
instead.
IOException
public boolean rename(Path src, Path dst) throws IOException
IOException
public abstract boolean renameRaw(Path src, Path dst) throws IOException
IOException
public boolean delete(File f) throws IOException
delete(Path)
instead.
IOException
public boolean delete(Path f) throws IOException
IOException
public abstract boolean deleteRaw(Path f) throws IOException
IOException
public boolean exists(File f) throws IOException
exists(Path)
instead
IOException
public abstract boolean exists(Path f) throws IOException
IOException
public boolean isDirectory(File f) throws IOException
isDirectory(Path)
instead.
IOException
public abstract boolean isDirectory(Path f) throws IOException
IOException
public boolean isFile(File f) throws IOException
isFile(Path)
instead.
IOException
public boolean isFile(Path f) throws IOException
IOException
public long getLength(File f) throws IOException
getLength(Path)
instead.
IOException
public abstract long getLength(Path f) throws IOException
IOException
public File[] listFiles(File f) throws IOException
listPaths(Path)
instead.
IOException
public Path[] listPaths(Path f) throws IOException
IOException
public abstract Path[] listPathsRaw(Path f) throws IOException
IOException
public File[] listFiles(File f, FileFilter filt) throws IOException
listPaths(Path)
instead.
IOException
public Path[] listPaths(Path f, PathFilter filter) throws IOException
IOException
public Path[] listPaths(Path[] files) throws IOException
files:
- a list of paths
IOException
public Path[] listPaths(Path[] files, PathFilter filter) throws IOException
files:
- a list of paths
IOException
public Path[] globPaths(Path filePattern) throws IOException
Return all the files that match filePattern and are not checksum files. Results are sorted by their names.
A filename pattern is composed of regular characters and special pattern matching characters, which are:
filePattern:
- a regular expression specifying file pattern
IOException
public Path[] globPaths(Path filePattern, PathFilter filter) throws IOException
-
IOException
public abstract void setWorkingDirectory(Path new_dir)
new_dir
- public abstract Path getWorkingDirectory()
public boolean mkdirs(File f) throws IOException
mkdirs(Path)
instead.
IOException
public abstract boolean mkdirs(Path f) throws IOException
IOException
public void lock(File f, boolean shared) throws IOException
lock(Path,boolean)
instead.
IOException
public abstract void lock(Path f, boolean shared) throws IOException
IOException
public void release(File f) throws IOException
release(Path)
instead.
IOException
public abstract void release(Path f) throws IOException
IOException
public abstract void copyFromLocalFile(Path src, Path dst) throws IOException
IOException
public abstract void moveFromLocalFile(Path src, Path dst) throws IOException
IOException
public abstract void copyToLocalFile(Path src, Path dst) throws IOException
IOException
public File startLocalOutput(File src, File dst) throws IOException
startLocalOutput(Path, Path)
instead.
IOException
public abstract Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile) throws IOException
IOException
public void completeLocalOutput(File src, File dst) throws IOException
completeLocalOutput(Path, Path)
instead.
IOException
public abstract void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile) throws IOException
IOException
public void close() throws IOException
IOException
public abstract void reportChecksumFailure(Path f, FSInputStream in, long start, long length, int crc)
f
- the file name containing the errorin
- the stream open on the filestart
- the position of the beginning of the bad data in the filelength
- the length of the bad data in the filecrc
- the expected CRC32 of the datapublic abstract long getBlockSize(Path f) throws IOException
f
- the filename
IOException
public abstract long getDefaultBlockSize()
public abstract short getDefaultReplication()
|
||||||||||
PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD |