|
||||||||||
PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD |
java.lang.Objectorg.apache.hadoop.chukwa.extraction.demux.DemuxManager
public class DemuxManager
Field Summary | |
---|---|
protected ChukwaConfiguration |
conf
|
protected SimpleDateFormat |
dayTextFormat
|
protected int |
DEFAULT_MAX_FILES_PER_DEMUX
|
protected int |
DEFAULT_REDUCER_COUNT
|
protected int |
demuxReducerCount
|
protected int |
ERROR_SLEEP_TIME
|
protected org.apache.hadoop.fs.FileSystem |
fs
|
protected boolean |
isRunning
|
protected int |
NO_DATASINK_SLEEP_TIME
|
protected int |
reprocess
|
protected boolean |
sendAlert
|
Constructor Summary | |
---|---|
DemuxManager()
|
|
DemuxManager(ChukwaConfiguration conf)
|
Method Summary | |
---|---|
protected boolean |
checkDemuxInputDir(String demuxInputDir)
Test if demuxInputDir exists |
protected boolean |
checkDemuxOutputDir(String demuxOutputDir)
Test if demuxOutputDir exists |
protected boolean |
deleteDemuxOutputDir(String demuxOutputDir)
Delete DemuxOutput directory |
protected boolean |
dirExists(String directory)
Check if source exists and if source is a directory |
int |
getReprocess()
|
protected void |
init()
|
static void |
main(String[] args)
|
protected boolean |
moveDataSinkFilesToArchiveDirectory(String demuxInputDir,
String archiveDirectory)
Move sourceFolder inside destFolder |
protected boolean |
moveDataSinkFilesToDemuxErrorDirectory(String dataSinkDir,
String demuxErrorDir)
Move sourceFolder inside destFolder |
protected boolean |
moveDataSinkFilesToDemuxInputDirectory(String dataSinkDir,
String demuxInputDir)
Move dataSink files to Demux input directory |
protected boolean |
moveDemuxOutputDirToPostProcessDirectory(String demuxOutputDir,
String postProcessDirectory)
Move sourceFolder inside destFolder |
protected boolean |
moveFolder(String srcDir,
String destDir,
String prefix)
Move sourceFolder inside destFolder |
protected boolean |
processData(String dataSinkDir,
String demuxInputDir,
String demuxOutputDir,
String postProcessDir,
String archiveDir)
Process Data, i.e. |
protected boolean |
runDemux(String demuxInputDir,
String demuxOutputDir)
Submit and Run demux Job |
protected void |
sendDemuxStatusToNagios(String nagiosHost,
int nagiosPort,
String reportingHost,
String demuxInErrorDir,
boolean demuxStatus,
String demuxException)
Send NSCA status to Nagios |
protected void |
setup(org.apache.hadoop.fs.Path directory)
Create directory if !exists |
void |
shutdown()
|
void |
start()
Start the Demux Manager daemon |
Methods inherited from class java.lang.Object |
---|
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
Field Detail |
---|
protected int ERROR_SLEEP_TIME
protected int NO_DATASINK_SLEEP_TIME
protected int DEFAULT_MAX_FILES_PER_DEMUX
protected int DEFAULT_REDUCER_COUNT
protected int demuxReducerCount
protected ChukwaConfiguration conf
protected org.apache.hadoop.fs.FileSystem fs
protected int reprocess
protected boolean sendAlert
protected SimpleDateFormat dayTextFormat
protected volatile boolean isRunning
Constructor Detail |
---|
public DemuxManager() throws Exception
Exception
public DemuxManager(ChukwaConfiguration conf) throws Exception
Exception
Method Detail |
---|
public static void main(String[] args) throws Exception
Exception
protected void init() throws IOException, URISyntaxException
IOException
URISyntaxException
public void shutdown()
public int getReprocess()
public void start() throws Exception
Exception
protected void sendDemuxStatusToNagios(String nagiosHost, int nagiosPort, String reportingHost, String demuxInErrorDir, boolean demuxStatus, String demuxException)
nagiosHost
- nagiosPort
- reportingHost
- demuxInErrorDir
- demuxStatus
- exception
- protected boolean processData(String dataSinkDir, String demuxInputDir, String demuxOutputDir, String postProcessDir, String archiveDir) throws IOException
dataSinkDir
- demuxInputDir
- demuxOutputDir
- postProcessDir
- archiveDir
-
IOException
protected boolean runDemux(String demuxInputDir, String demuxOutputDir)
demuxInputDir
- demuxOutputDir
-
protected boolean moveDataSinkFilesToDemuxInputDirectory(String dataSinkDir, String demuxInputDir) throws IOException
dataSinkDir
- demuxInputDir
-
IOException
protected boolean moveDataSinkFilesToDemuxErrorDirectory(String dataSinkDir, String demuxErrorDir) throws IOException
dataSinkDir
- : ex chukwa/demux/inputDirdemuxErrorDir
- : ex /chukwa/demux/inError
IOException
protected boolean moveDataSinkFilesToArchiveDirectory(String demuxInputDir, String archiveDirectory) throws IOException
demuxInputDir:
- ex chukwa/demux/inputDirarchiveDirectory:
- ex /chukwa/archives
IOException
protected boolean moveDemuxOutputDirToPostProcessDirectory(String demuxOutputDir, String postProcessDirectory) throws IOException
demuxOutputDir:
- ex chukwa/demux/outputDirpostProcessDirectory:
- ex /chukwa/postProcess
IOException
protected boolean checkDemuxInputDir(String demuxInputDir) throws IOException
demuxInputDir
-
IOException
protected boolean checkDemuxOutputDir(String demuxOutputDir) throws IOException
demuxOutputDir
-
IOException
protected boolean deleteDemuxOutputDir(String demuxOutputDir) throws IOException
demuxOutputDir
-
IOException
protected void setup(org.apache.hadoop.fs.Path directory) throws IOException
directory
-
IOException
protected boolean dirExists(String directory) throws IOException
f
- source file
IOException
protected boolean moveFolder(String srcDir, String destDir, String prefix) throws IOException
srcDir
- destDir
-
IOException
|
||||||||||
PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD |