|
||||||||||
PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD |
java.lang.Objectorg.apache.hadoop.conf.Configured
org.apache.hadoop.fs.FileSystem
org.apache.hadoop.fs.s3.S3FileSystem
public class S3FileSystem
A FileSystem
backed by Amazon S3.
Field Summary |
---|
Fields inherited from class org.apache.hadoop.fs.FileSystem |
---|
LOG |
Constructor Summary | |
---|---|
S3FileSystem()
|
|
S3FileSystem(org.apache.hadoop.fs.s3.FileSystemStore store)
|
Method Summary | |
---|---|
void |
completeLocalOutput(Path fsOutputFile,
Path tmpLocalFile)
Called when we're all done writing to the target. |
void |
copyFromLocalFile(Path src,
Path dst)
The src file is on the local disk. |
void |
copyToLocalFile(Path src,
Path dst,
boolean copyCrc)
The src file is under FS, and the dst is on the local disk. |
FSOutputStream |
createRaw(Path file,
boolean overwrite,
short replication,
long blockSize)
Opens an OutputStream at the indicated Path. |
FSOutputStream |
createRaw(Path file,
boolean overwrite,
short replication,
long blockSize,
Progressable progress)
Opens an OutputStream at the indicated Path with write-progress reporting. |
boolean |
deleteRaw(Path path)
Deletes Path |
boolean |
exists(Path path)
Check if exists. |
long |
getBlockSize(Path path)
Get the size for a particular file. |
long |
getDefaultBlockSize()
Return the number of bytes that large input files should be optimally be split into to minimize i/o time. |
short |
getDefaultReplication()
Get the default replication. |
String[][] |
getFileCacheHints(Path f,
long start,
long len)
Return 1x1 'localhost' cell if the file exists. |
long |
getLength(Path path)
The number of bytes in a file. |
String |
getName()
|
short |
getReplication(Path path)
Replication is not supported for S3 file systems since S3 handles it for us. |
URI |
getUri()
Returns a URI whose scheme and authority identify this FileSystem. |
Path |
getWorkingDirectory()
Get the current working directory for the given file system |
void |
initialize(URI uri,
Configuration conf)
Called after a new FileSystem instance is constructed. |
boolean |
isDirectory(Path path)
True iff the named path is a directory. |
boolean |
isFile(Path path)
True iff the named path is a regular file. |
Path[] |
listPathsRaw(Path path)
List files in a directory. |
void |
lock(Path path,
boolean shared)
Deprecated. |
boolean |
mkdirs(Path path)
Make the given file and all non-existent parents into directories. |
void |
moveFromLocalFile(Path src,
Path dst)
The src file is on the local disk. |
FSInputStream |
openRaw(Path path)
Opens an InputStream for the indicated Path, whether local or via DFS. |
void |
release(Path path)
Deprecated. |
boolean |
renameRaw(Path src,
Path dst)
Renames Path src to Path dst. |
void |
reportChecksumFailure(Path f,
FSInputStream in,
long inPos,
FSInputStream sums,
long sumsPos)
Report a checksum error to the file system. |
boolean |
setReplicationRaw(Path path,
short replication)
Replication is not supported for S3 file systems since S3 handles it for us. |
void |
setWorkingDirectory(Path dir)
Set the current working directory for the given file system. |
Path |
startLocalOutput(Path fsOutputFile,
Path tmpLocalFile)
Returns a local File that the user can write output to. |
Methods inherited from class org.apache.hadoop.fs.FileSystem |
---|
checkPath, close, copyToLocalFile, create, create, create, create, create, create, create, create, createNewFile, delete, get, get, getChecksumFile, getChecksumFileLength, getContentLength, getLocal, getNamed, globPaths, globPaths, isChecksumFile, listPaths, listPaths, listPaths, listPaths, makeQualified, open, open, parseArgs, rename, setReplication |
Methods inherited from class org.apache.hadoop.conf.Configured |
---|
getConf, setConf |
Methods inherited from class java.lang.Object |
---|
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
Constructor Detail |
---|
public S3FileSystem()
public S3FileSystem(org.apache.hadoop.fs.s3.FileSystemStore store)
Method Detail |
---|
public URI getUri()
FileSystem
getUri
in class FileSystem
public void initialize(URI uri, Configuration conf) throws IOException
FileSystem
initialize
in class FileSystem
uri
- a uri whose authority section names the host, port, etc.
for this FileSystemconf
- the configuration
IOException
public String getName()
getName
in class FileSystem
public Path getWorkingDirectory()
FileSystem
getWorkingDirectory
in class FileSystem
public void setWorkingDirectory(Path dir)
FileSystem
setWorkingDirectory
in class FileSystem
public boolean exists(Path path) throws IOException
FileSystem
exists
in class FileSystem
IOException
public boolean mkdirs(Path path) throws IOException
FileSystem
mkdirs
in class FileSystem
IOException
public boolean isDirectory(Path path) throws IOException
FileSystem
isDirectory
in class FileSystem
IOException
public boolean isFile(Path path) throws IOException
FileSystem
isFile
in class FileSystem
IOException
public Path[] listPathsRaw(Path path) throws IOException
FileSystem
listPathsRaw
in class FileSystem
IOException
public FSOutputStream createRaw(Path file, boolean overwrite, short replication, long blockSize) throws IOException
FileSystem
createRaw
in class FileSystem
file
- the file name to openoverwrite
- if a file with this name already exists, then if true,
the file will be overwritten, and if false an error will be thrown.replication
- required block replication for the file.
IOException
public FSOutputStream createRaw(Path file, boolean overwrite, short replication, long blockSize, Progressable progress) throws IOException
FileSystem
createRaw
in class FileSystem
file
- the file name to openoverwrite
- if a file with this name already exists, then if true,
the file will be overwritten, and if false an error will be thrown.replication
- required block replication for the file.
IOException
public FSInputStream openRaw(Path path) throws IOException
FileSystem
openRaw
in class FileSystem
IOException
public boolean renameRaw(Path src, Path dst) throws IOException
FileSystem
renameRaw
in class FileSystem
IOException
public boolean deleteRaw(Path path) throws IOException
FileSystem
deleteRaw
in class FileSystem
IOException
public long getLength(Path path) throws IOException
FileSystem
getLength
in class FileSystem
IOException
public short getReplication(Path path) throws IOException
getReplication
in class FileSystem
path
- file name
IOException
public short getDefaultReplication()
FileSystem
getDefaultReplication
in class FileSystem
public boolean setReplicationRaw(Path path, short replication) throws IOException
setReplicationRaw
in class FileSystem
path
- file namereplication
- new replication
IOException
public long getBlockSize(Path path) throws IOException
FileSystem
getBlockSize
in class FileSystem
path
- the filename
IOException
public long getDefaultBlockSize()
FileSystem
getDefaultBlockSize
in class FileSystem
public String[][] getFileCacheHints(Path f, long start, long len) throws IOException
getFileCacheHints
in class FileSystem
IOException
@Deprecated public void lock(Path path, boolean shared) throws IOException
FileSystem
lock
in class FileSystem
IOException
@Deprecated public void release(Path path) throws IOException
FileSystem
release
in class FileSystem
IOException
public void reportChecksumFailure(Path f, FSInputStream in, long inPos, FSInputStream sums, long sumsPos)
FileSystem
reportChecksumFailure
in class FileSystem
f
- the file name containing the errorin
- the stream open on the fileinPos
- the position of the beginning of the bad data in the filesums
- the stream open on the checksum filesumsPos
- the position of the beginning of the bad data in the checksum filepublic void moveFromLocalFile(Path src, Path dst) throws IOException
FileSystem
moveFromLocalFile
in class FileSystem
IOException
public void copyFromLocalFile(Path src, Path dst) throws IOException
FileSystem
copyFromLocalFile
in class FileSystem
IOException
public void copyToLocalFile(Path src, Path dst, boolean copyCrc) throws IOException
FileSystem
copyToLocalFile
in class FileSystem
IOException
public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile) throws IOException
FileSystem
startLocalOutput
in class FileSystem
IOException
public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile) throws IOException
FileSystem
completeLocalOutput
in class FileSystem
IOException
|
||||||||||
PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD |