public class IgniteHadoopFileSystem extends FileSystem
IGFS
Hadoop 1.x file system driver over file system API. To use
IGFS
as Hadoop file system, you should configure this class
in Hadoop's core-site.xml
as follows:
<property> <name>fs.default.name</name> <value>igfs:///</value> </property> <property> <name>fs.igfs.impl</name> <value>org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem</value> </property>You should also add Ignite JAR and all libraries to Hadoop classpath. To do this, add following lines to
conf/hadoop-env.sh
script in Hadoop
distribution:
export IGNITE_HOME=/path/to/Ignite/distribution export HADOOP_CLASSPATH=$IGNITE_HOME/ignite*.jar for f in $IGNITE_HOME/libs/*.jar; do export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f; done
job-submitter
or job-scheduler
processes, while data nodes are usually
started together with Hadoop task-tracker
processes.
For sample client and data node configuration refer to config/hadoop/default-config-client.xml
and config/hadoop/default-config.xml
configuration files in Ignite installation.
FileSystem.Statistics
Modifier and Type | Field and Description |
---|---|
static FileStatus[] |
EMPTY_FILE_STATUS
Empty array of file statuses.
|
static String |
IGFS_MANAGEMENT
Internal property to indicate management connection.
|
DEFAULT_FS, FS_DEFAULT_NAME_KEY, LOG, SHUTDOWN_HOOK_PRIORITY, statistics
Constructor and Description |
---|
IgniteHadoopFileSystem() |
Modifier and Type | Method and Description |
---|---|
FSDataOutputStream |
append(Path f,
int bufSize,
Progressable progress) |
protected void |
checkPath(Path path) |
void |
close() |
void |
colocateFileWrites(boolean colocateFileWrites)
Public setter that can be used by direct users of FS or Visor.
|
FSDataOutputStream |
create(Path f,
FsPermission perm,
boolean overwrite,
int bufSize,
short replication,
long blockSize,
Progressable progress) |
boolean |
delete(Path f) |
boolean |
delete(Path f,
boolean recursive) |
protected void |
finalize() |
ContentSummary |
getContentSummary(Path f) |
long |
getDefaultBlockSize() |
short |
getDefaultReplication() |
BlockLocation[] |
getFileBlockLocations(FileStatus status,
long start,
long len) |
FileStatus |
getFileStatus(Path f) |
Path |
getHomeDirectory() |
URI |
getUri() |
Path |
getWorkingDirectory() |
void |
initialize(URI name,
Configuration cfg) |
FileStatus[] |
listStatus(Path f) |
boolean |
mkdirs(Path f,
FsPermission perm) |
IgfsMode |
mode(IgfsPath path)
Resolve path mode.
|
IgfsMode |
mode(Path path)
Resolve path mode.
|
FSDataInputStream |
open(Path f,
int bufSize) |
boolean |
rename(Path src,
Path dst) |
void |
setOwner(Path p,
String username,
String grpName) |
void |
setPermission(Path p,
FsPermission perm) |
void |
setTimes(Path p,
long mtime,
long atime) |
void |
setUser(String userName)
Set user name and default working directory for current thread.
|
void |
setWorkingDirectory(Path newPath) |
String |
toString() |
addDelegationTokens, append, append, areSymlinksEnabled, cancelDeleteOnExit, canonicalizeUri, clearStatistics, closeAll, closeAllForUGI, completeLocalOutput, concat, copyFromLocalFile, copyFromLocalFile, copyFromLocalFile, copyFromLocalFile, copyToLocalFile, copyToLocalFile, copyToLocalFile, create, create, create, create, create, create, create, create, create, create, create, create, createNewFile, createNonRecursive, createNonRecursive, createNonRecursive, createSnapshot, createSnapshot, createSymlink, deleteOnExit, deleteSnapshot, enableSymlinks, exists, fixRelativePart, get, get, get, getAclStatus, getAllStatistics, getBlockSize, getCanonicalServiceName, getCanonicalUri, getChildFileSystems, getDefaultBlockSize, getDefaultPort, getDefaultReplication, getDefaultUri, getDelegationToken, getFileBlockLocations, getFileChecksum, getFileLinkStatus, getFileSystemClass, getFSofPath, getInitialWorkingDirectory, getLength, getLinkTarget, getLocal, getName, getNamed, getReplication, getScheme, getServerDefaults, getServerDefaults, getStatistics, getStatistics, getStatus, getStatus, getUsed, globStatus, globStatus, isDirectory, isFile, listCorruptFileBlocks, listFiles, listLocatedStatus, listLocatedStatus, listStatus, listStatus, listStatus, makeQualified, mkdirs, mkdirs, modifyAclEntries, moveFromLocalFile, moveFromLocalFile, moveToLocalFile, newInstance, newInstance, newInstance, newInstanceLocal, open, primitiveCreate, primitiveMkdir, primitiveMkdir, printStatistics, processDeleteOnExit, removeAcl, removeAclEntries, removeDefaultAcl, rename, renameSnapshot, resolveLink, resolvePath, setAcl, setDefaultUri, setDefaultUri, setReplication, setVerifyChecksum, setWriteChecksum, startLocalOutput, supportsSymlinks
getConf, setConf
public static final String IGFS_MANAGEMENT
public static final FileStatus[] EMPTY_FILE_STATUS
public URI getUri()
getUri
in class FileSystem
public void colocateFileWrites(boolean colocateFileWrites)
colocateFileWrites
- Whether all ongoing file writes should be colocated.public void initialize(URI name, Configuration cfg) throws IOException
initialize
in class FileSystem
IOException
protected void checkPath(Path path)
checkPath
in class FileSystem
public short getDefaultReplication()
getDefaultReplication
in class FileSystem
protected void finalize() throws Throwable
public void close() throws IOException
close
in interface Closeable
close
in interface AutoCloseable
close
in class FileSystem
IOException
public void setTimes(Path p, long mtime, long atime) throws IOException
setTimes
in class FileSystem
IOException
public void setPermission(Path p, FsPermission perm) throws IOException
setPermission
in class FileSystem
IOException
public void setOwner(Path p, String username, String grpName) throws IOException
setOwner
in class FileSystem
IOException
public FSDataInputStream open(Path f, int bufSize) throws IOException
open
in class FileSystem
IOException
public FSDataOutputStream create(Path f, FsPermission perm, boolean overwrite, int bufSize, short replication, long blockSize, Progressable progress) throws IOException
create
in class FileSystem
IOException
public FSDataOutputStream append(Path f, int bufSize, Progressable progress) throws IOException
append
in class FileSystem
IOException
public boolean rename(Path src, Path dst) throws IOException
rename
in class FileSystem
IOException
public boolean delete(Path f) throws IOException
delete
in class FileSystem
IOException
public boolean delete(Path f, boolean recursive) throws IOException
delete
in class FileSystem
IOException
public FileStatus[] listStatus(Path f) throws IOException
listStatus
in class FileSystem
IOException
public Path getHomeDirectory()
getHomeDirectory
in class FileSystem
public void setUser(String userName)
userName
- User name.public void setWorkingDirectory(Path newPath)
setWorkingDirectory
in class FileSystem
public Path getWorkingDirectory()
getWorkingDirectory
in class FileSystem
public boolean mkdirs(Path f, FsPermission perm) throws IOException
mkdirs
in class FileSystem
IOException
public FileStatus getFileStatus(Path f) throws IOException
getFileStatus
in class FileSystem
IOException
public ContentSummary getContentSummary(Path f) throws IOException
getContentSummary
in class FileSystem
IOException
public BlockLocation[] getFileBlockLocations(FileStatus status, long start, long len) throws IOException
getFileBlockLocations
in class FileSystem
IOException
public long getDefaultBlockSize()
getDefaultBlockSize
in class FileSystem
public IgfsMode mode(Path path)
path
- HDFS path.public IgfsMode mode(IgfsPath path)
path
- IGFS path.
Follow @ApacheIgnite
Apache Ignite Fabric : ver. 1.0.0 Release Date : March 31 2015