public class FetchItemQueues
extends java.lang.Object
Modifier and Type | Field and Description |
---|---|
static java.lang.String |
DEFAULT_ID |
static java.lang.String |
QUEUE_MODE_DOMAIN |
static java.lang.String |
QUEUE_MODE_HOST |
static java.lang.String |
QUEUE_MODE_IP |
Constructor and Description |
---|
FetchItemQueues(Configuration conf) |
Modifier and Type | Method and Description |
---|---|
void |
addFetchItem(FetchItem it) |
void |
addFetchItem(Text url,
CrawlDatum datum) |
int |
checkExceptionThreshold(java.lang.String queueid)
Increment the exception counter of a queue in case of an exception e.g.
|
protected static java.lang.String |
checkQueueMode(java.lang.String queueMode)
Check whether queue mode is valid, fall-back to default mode if not.
|
int |
checkTimelimit() |
void |
dump() |
int |
emptyQueues() |
void |
finishFetchItem(FetchItem it) |
void |
finishFetchItem(FetchItem it,
boolean asap) |
FetchItem |
getFetchItem() |
FetchItemQueue |
getFetchItemQueue(java.lang.String id) |
int |
getQueueCount() |
int |
getTotalSize() |
public static final java.lang.String DEFAULT_ID
public static final java.lang.String QUEUE_MODE_HOST
public static final java.lang.String QUEUE_MODE_DOMAIN
public static final java.lang.String QUEUE_MODE_IP
public FetchItemQueues(Configuration conf)
protected static java.lang.String checkQueueMode(java.lang.String queueMode)
queueMode
- queue mode to checkpublic int getTotalSize()
public int getQueueCount()
public void addFetchItem(Text url, CrawlDatum datum)
public void addFetchItem(FetchItem it)
public void finishFetchItem(FetchItem it)
public void finishFetchItem(FetchItem it, boolean asap)
public FetchItemQueue getFetchItemQueue(java.lang.String id)
public FetchItem getFetchItem()
public int checkTimelimit()
public int emptyQueues()
public int checkExceptionThreshold(java.lang.String queueid)
queueid
- public void dump()
Copyright © 2019 The Apache Software Foundation