public class UpdateHostDbReducer extends Object implements Reducer<Text,NutchWritable,Text,HostDatum>
Modifier and Type | Field and Description |
---|---|
protected static boolean |
checkFailed |
protected static boolean |
checkKnown |
protected static boolean |
checkNew |
protected ThreadPoolExecutor |
executor |
protected static boolean |
force |
protected static long |
now |
protected static String[] |
numericFields |
protected static Text[] |
numericFieldWritables |
protected Integer |
numResolverThreads |
protected static int[] |
percentiles |
protected static Integer |
purgeFailedHostsThreshold |
protected BlockingQueue<Runnable> |
queue |
protected static Integer |
recheckInterval |
protected ResolverThread |
resolverThread |
protected static String[] |
stringFields |
protected static Text[] |
stringFieldWritables |
Constructor and Description |
---|
UpdateHostDbReducer() |
Modifier and Type | Method and Description |
---|---|
void |
close()
Shut down all running threads and wait for completion.
|
void |
configure(JobConf job)
Configures the thread pool and prestarts all resolver threads.
|
protected boolean |
isEligibleForCheck(HostDatum datum)
Determines whether a record is eligible for recheck.
|
void |
reduce(Text key,
Iterator<NutchWritable> values,
OutputCollector<Text,HostDatum> output,
Reporter reporter) |
protected boolean |
shouldCheck(HostDatum datum)
Determines whether a record should be checked.
|
protected ResolverThread resolverThread
protected Integer numResolverThreads
protected static Integer purgeFailedHostsThreshold
protected static Integer recheckInterval
protected static boolean checkFailed
protected static boolean checkNew
protected static boolean checkKnown
protected static boolean force
protected static long now
protected static String[] numericFields
protected static String[] stringFields
protected static int[] percentiles
protected static Text[] numericFieldWritables
protected static Text[] stringFieldWritables
protected BlockingQueue<Runnable> queue
protected ThreadPoolExecutor executor
public void configure(JobConf job)
configure
in interface JobConfigurable
JobConf
- public void reduce(Text key, Iterator<NutchWritable> values, OutputCollector<Text,HostDatum> output, Reporter reporter) throws IOException
reduce
in interface Reducer<Text,NutchWritable,Text,HostDatum>
IOException
protected boolean shouldCheck(HostDatum datum)
HostDatum
- protected boolean isEligibleForCheck(HostDatum datum)
HostDatum
- public void close()
close
in interface Closeable
close
in interface AutoCloseable
Copyright © 2017 The Apache Software Foundation