public class UpdateHostDbReducer extends Reducer<Text,NutchWritable,Text,HostDatum>
Reducer.Context
Modifier and Type | Field and Description |
---|---|
protected static boolean |
checkFailed |
protected static boolean |
checkKnown |
protected static boolean |
checkNew |
protected java.util.concurrent.ThreadPoolExecutor |
executor |
protected static boolean |
force |
protected static long |
now |
protected static java.lang.String[] |
numericFields |
protected static Text[] |
numericFieldWritables |
protected java.lang.Integer |
numResolverThreads |
protected static int[] |
percentiles |
protected static java.lang.Integer |
purgeFailedHostsThreshold |
protected java.util.concurrent.BlockingQueue<java.lang.Runnable> |
queue |
protected static java.lang.Integer |
recheckInterval |
protected ResolverThread |
resolverThread |
protected static java.lang.String[] |
stringFields |
protected static Text[] |
stringFieldWritables |
Constructor and Description |
---|
UpdateHostDbReducer() |
Modifier and Type | Method and Description |
---|---|
void |
cleanup(Reducer.Context context)
Shut down all running threads and wait for completion.
|
protected boolean |
isEligibleForCheck(HostDatum datum)
Determines whether a record is eligible for recheck.
|
void |
reduce(Text key,
java.lang.Iterable<NutchWritable> values,
Reducer.Context context) |
void |
setup(Reducer.Context context)
Configures the thread pool and prestarts all resolver threads.
|
protected boolean |
shouldCheck(HostDatum datum)
Determines whether a record should be checked.
|
protected ResolverThread resolverThread
protected java.lang.Integer numResolverThreads
protected static java.lang.Integer purgeFailedHostsThreshold
protected static java.lang.Integer recheckInterval
protected static boolean checkFailed
protected static boolean checkNew
protected static boolean checkKnown
protected static boolean force
protected static long now
protected static java.lang.String[] numericFields
protected static java.lang.String[] stringFields
protected static int[] percentiles
protected static Text[] numericFieldWritables
protected static Text[] stringFieldWritables
protected java.util.concurrent.BlockingQueue<java.lang.Runnable> queue
protected java.util.concurrent.ThreadPoolExecutor executor
public void setup(Reducer.Context context)
public void reduce(Text key, java.lang.Iterable<NutchWritable> values, Reducer.Context context) throws java.io.IOException, java.lang.InterruptedException
protected boolean shouldCheck(HostDatum datum)
datum
- protected boolean isEligibleForCheck(HostDatum datum)
datum
- public void cleanup(Reducer.Context context)
Copyright © 2018 The Apache Software Foundation