|
||||||||||
PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD |
java.lang.Objectorg.apache.mahout.common.HadoopUtil
public final class HadoopUtil
Method Summary | |
---|---|
static org.apache.hadoop.fs.Path |
cachedFile(org.apache.hadoop.conf.Configuration conf)
|
static void |
cacheFiles(org.apache.hadoop.fs.Path fileToCache,
org.apache.hadoop.conf.Configuration conf)
|
static long |
countRecords(org.apache.hadoop.fs.Path path,
org.apache.hadoop.conf.Configuration conf)
|
static long |
countRecords(org.apache.hadoop.fs.Path path,
PathType pt,
org.apache.hadoop.fs.PathFilter filter,
org.apache.hadoop.conf.Configuration conf)
Count all the records in a directory using a SequenceFileDirValueIterator |
static void |
delete(org.apache.hadoop.conf.Configuration conf,
Iterable<org.apache.hadoop.fs.Path> paths)
|
static void |
delete(org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.fs.Path... paths)
|
static String |
getCustomJobName(String className,
org.apache.hadoop.mapreduce.JobContext job,
Class<? extends org.apache.hadoop.mapreduce.Mapper> mapper,
Class<? extends org.apache.hadoop.mapreduce.Reducer> reducer)
|
static org.apache.hadoop.fs.FileStatus[] |
getFileStatus(org.apache.hadoop.fs.Path path,
PathType pathType,
org.apache.hadoop.fs.PathFilter filter,
Comparator<org.apache.hadoop.fs.FileStatus> ordering,
org.apache.hadoop.conf.Configuration conf)
|
static InputStream |
openStream(org.apache.hadoop.fs.Path path,
org.apache.hadoop.conf.Configuration conf)
|
static org.apache.hadoop.mapreduce.Job |
prepareJob(org.apache.hadoop.fs.Path inputPath,
org.apache.hadoop.fs.Path outputPath,
Class<? extends org.apache.hadoop.mapreduce.InputFormat> inputFormat,
Class<? extends org.apache.hadoop.mapreduce.Mapper> mapper,
Class<? extends org.apache.hadoop.io.Writable> mapperKey,
Class<? extends org.apache.hadoop.io.Writable> mapperValue,
Class<? extends org.apache.hadoop.mapreduce.OutputFormat> outputFormat,
org.apache.hadoop.conf.Configuration conf)
Create a map-only Hadoop Job out of the passed in parameters. |
static org.apache.hadoop.mapreduce.Job |
prepareJob(org.apache.hadoop.fs.Path inputPath,
org.apache.hadoop.fs.Path outputPath,
Class<? extends org.apache.hadoop.mapreduce.InputFormat> inputFormat,
Class<? extends org.apache.hadoop.mapreduce.Mapper> mapper,
Class<? extends org.apache.hadoop.io.Writable> mapperKey,
Class<? extends org.apache.hadoop.io.Writable> mapperValue,
Class<? extends org.apache.hadoop.mapreduce.Reducer> reducer,
Class<? extends org.apache.hadoop.io.Writable> reducerKey,
Class<? extends org.apache.hadoop.io.Writable> reducerValue,
Class<? extends org.apache.hadoop.mapreduce.OutputFormat> outputFormat,
org.apache.hadoop.conf.Configuration conf)
Create a map and reduce Hadoop job. |
static int |
readInt(org.apache.hadoop.fs.Path path,
org.apache.hadoop.conf.Configuration conf)
|
static void |
setSerializations(org.apache.hadoop.conf.Configuration conf)
|
static void |
writeInt(int value,
org.apache.hadoop.fs.Path path,
org.apache.hadoop.conf.Configuration conf)
|
Methods inherited from class java.lang.Object |
---|
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
Method Detail |
---|
public static org.apache.hadoop.mapreduce.Job prepareJob(org.apache.hadoop.fs.Path inputPath, org.apache.hadoop.fs.Path outputPath, Class<? extends org.apache.hadoop.mapreduce.InputFormat> inputFormat, Class<? extends org.apache.hadoop.mapreduce.Mapper> mapper, Class<? extends org.apache.hadoop.io.Writable> mapperKey, Class<? extends org.apache.hadoop.io.Writable> mapperValue, Class<? extends org.apache.hadoop.mapreduce.OutputFormat> outputFormat, org.apache.hadoop.conf.Configuration conf) throws IOException
IOException
getCustomJobName(String, org.apache.hadoop.mapreduce.JobContext, Class, Class)
public static org.apache.hadoop.mapreduce.Job prepareJob(org.apache.hadoop.fs.Path inputPath, org.apache.hadoop.fs.Path outputPath, Class<? extends org.apache.hadoop.mapreduce.InputFormat> inputFormat, Class<? extends org.apache.hadoop.mapreduce.Mapper> mapper, Class<? extends org.apache.hadoop.io.Writable> mapperKey, Class<? extends org.apache.hadoop.io.Writable> mapperValue, Class<? extends org.apache.hadoop.mapreduce.Reducer> reducer, Class<? extends org.apache.hadoop.io.Writable> reducerKey, Class<? extends org.apache.hadoop.io.Writable> reducerValue, Class<? extends org.apache.hadoop.mapreduce.OutputFormat> outputFormat, org.apache.hadoop.conf.Configuration conf) throws IOException
inputPath
- The input Path
outputPath
- The output Path
inputFormat
- The InputFormat
mapper
- The Mapper
class to usemapperKey
- The Writable
key class. If the Mapper is a no-op, this value may be nullmapperValue
- The Writable
value class. If the Mapper is a no-op, this value may be nullreducer
- The Reducer
to usereducerKey
- The reducer key class.reducerValue
- The reducer value class.outputFormat
- The OutputFormat
.conf
- The Configuration
to use.
Job
.
IOException
- if there is a problem with the IO.getCustomJobName(String, org.apache.hadoop.mapreduce.JobContext, Class, Class)
,
prepareJob(org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path, Class, Class, Class, Class, Class, org.apache.hadoop.conf.Configuration)
public static String getCustomJobName(String className, org.apache.hadoop.mapreduce.JobContext job, Class<? extends org.apache.hadoop.mapreduce.Mapper> mapper, Class<? extends org.apache.hadoop.mapreduce.Reducer> reducer)
public static void delete(org.apache.hadoop.conf.Configuration conf, Iterable<org.apache.hadoop.fs.Path> paths) throws IOException
IOException
public static void delete(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.Path... paths) throws IOException
IOException
public static long countRecords(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration conf) throws IOException
IOException
public static long countRecords(org.apache.hadoop.fs.Path path, PathType pt, org.apache.hadoop.fs.PathFilter filter, org.apache.hadoop.conf.Configuration conf) throws IOException
SequenceFileDirValueIterator
path
- The Path
to countpt
- The PathType
filter
- Apply the PathFilter
. May be nullconf
- The Hadoop Configuration
IOException
- if there was an IO errorpublic static InputStream openStream(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration conf) throws IOException
IOException
public static org.apache.hadoop.fs.FileStatus[] getFileStatus(org.apache.hadoop.fs.Path path, PathType pathType, org.apache.hadoop.fs.PathFilter filter, Comparator<org.apache.hadoop.fs.FileStatus> ordering, org.apache.hadoop.conf.Configuration conf) throws IOException
IOException
public static void cacheFiles(org.apache.hadoop.fs.Path fileToCache, org.apache.hadoop.conf.Configuration conf)
public static org.apache.hadoop.fs.Path cachedFile(org.apache.hadoop.conf.Configuration conf) throws IOException
IOException
public static void setSerializations(org.apache.hadoop.conf.Configuration conf)
public static void writeInt(int value, org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration conf) throws IOException
IOException
public static int readInt(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration conf) throws IOException
IOException
|
||||||||||
PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD |