@InterfaceAudience.LimitedPrivate(value="yarn") @InterfaceStability.Unstable public class FairScheduler extends AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
Modifier and Type | Field and Description |
---|---|
protected boolean |
assignMultiple |
static org.apache.hadoop.yarn.api.records.Resource |
CONTAINER_RESERVED |
protected boolean |
continuousSchedulingEnabled
Deprecated.
|
protected int |
continuousSchedulingSleepMs
Deprecated.
|
protected int |
maxAssign |
protected long |
nodeLocalityDelayMs
Deprecated.
|
protected double |
nodeLocalityThreshold |
protected long |
rackLocalityDelayMs
Deprecated.
|
protected double |
rackLocalityThreshold |
protected boolean |
sizeBasedWeight |
activitiesManager, applications, EMPTY_ALLOCATION, lastNodeUpdateTime, minimumAllocation, nmExpireInterval, nmHeartbeatInterval, nodeTracker, readLock, rmContext, schedulerHealth, schedulingMonitorManager, THREAD_JOIN_TIMEOUT_MS, updateInterval, writeLock
Constructor and Description |
---|
FairScheduler() |
Modifier and Type | Method and Description |
---|---|
protected void |
addApplication(org.apache.hadoop.yarn.api.records.ApplicationId applicationId,
String queueName,
String user,
boolean isAppRecovering)
Add a new application to the scheduler, with a given id, queue name, and
user.
|
protected void |
addApplicationAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId,
boolean transferStateFromPreviousAttempt,
boolean isAttemptRecovering)
Add a new application attempt to the scheduler.
|
Allocation |
allocate(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId,
List<org.apache.hadoop.yarn.api.records.ResourceRequest> ask,
List<org.apache.hadoop.yarn.api.records.SchedulingRequest> schedulingRequests,
List<org.apache.hadoop.yarn.api.records.ContainerId> release,
List<String> blacklistAdditions,
List<String> blacklistRemovals,
ContainerUpdates updateRequests)
The main api between the ApplicationMaster and the Scheduler.
|
boolean |
checkAccess(org.apache.hadoop.security.UserGroupInformation callerUGI,
org.apache.hadoop.yarn.api.records.QueueACL acl,
String queueName)
Check if the user has permission to perform the operation.
|
long |
checkAndGetApplicationLifetime(String queueName,
long lifetime)
Verify whether a submitted application lifetime is valid as per configured
Queue lifetime.
|
protected void |
completedContainerInternal(RMContainer rmContainer,
org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus,
RMContainerEventType event)
Clean up a completed container.
|
protected void |
createPreemptionThread() |
AllocationConfiguration |
getAllocationConfiguration() |
List<org.apache.hadoop.yarn.api.records.ApplicationAttemptId> |
getAppsInQueue(String queueName)
Gets the apps under a given queue
|
FairSchedulerConfiguration |
getConf() |
RMContainerTokenSecretManager |
getContainerTokenSecretManager() |
FSContext |
getContext() |
int |
getContinuousSchedulingSleepMs()
Deprecated.
linked to
isContinuousSchedulingEnabled() deprecation |
org.apache.hadoop.yarn.api.records.Resource |
getIncrementResourceCapability() |
long |
getNodeLocalityDelayMs()
Deprecated.
linked to
isContinuousSchedulingEnabled() deprecation |
double |
getNodeLocalityThreshold() |
org.apache.hadoop.yarn.api.records.Resource |
getNormalizedResource(org.apache.hadoop.yarn.api.records.Resource requestedResource)
Normalize a resource request.
|
int |
getNumClusterNodes()
Get the number of nodes available in the cluster.
|
int |
getNumNodesInRack(String rackName) |
Set<String> |
getPlanQueues()
Gets the list of names for queues managed by the Reservation System
|
org.apache.hadoop.yarn.api.records.QueueInfo |
getQueueInfo(String queueName,
boolean includeChildQueues,
boolean recursive)
Get queue information
|
QueueManager |
getQueueManager() |
List<org.apache.hadoop.yarn.api.records.QueueUserACLInfo> |
getQueueUserAclInfo()
Get acls for queues for current user.
|
long |
getRackLocalityDelayMs()
Deprecated.
linked to
isContinuousSchedulingEnabled() deprecation |
double |
getRackLocalityThreshold() |
float |
getReservableNodesRatio() |
org.apache.hadoop.yarn.util.resource.ResourceCalculator |
getResourceCalculator() |
QueueMetrics |
getRootQueueMetrics()
Get the root queue for the scheduler.
|
FSAppAttempt |
getSchedulerApp(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId) |
EnumSet<org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes> |
getSchedulingResourceTypes()
Return a collection of the resource types that are considered when
scheduling
|
void |
handle(SchedulerEvent event) |
boolean |
isAtLeastReservationThreshold(org.apache.hadoop.yarn.util.resource.ResourceCalculator resourceCalculator,
org.apache.hadoop.yarn.api.records.Resource resource) |
boolean |
isContinuousSchedulingEnabled()
Deprecated.
Continuous scheduling should not be turned ON. It is
deprecated because it can cause scheduler slowness due to locking issues.
Schedulers should use assignmultiple as a replacement.
|
boolean |
isSizeBasedWeight() |
void |
killContainer(RMContainer container)
Kill a RMContainer.
|
String |
moveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId,
String queueName)
Moves the given application to the given queue
|
protected void |
nodeUpdate(RMNode nm)
Process a heartbeat update from a node.
|
void |
preValidateMoveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId,
String newQueue) |
void |
recover(RMStateStore.RMState state) |
void |
reinitialize(org.apache.hadoop.conf.Configuration conf,
RMContext rmContext)
Re-initialize the
ResourceScheduler . |
void |
removeQueue(String queueName)
Only supports removing empty leaf queues
|
void |
serviceInit(org.apache.hadoop.conf.Configuration conf) |
void |
serviceStart() |
void |
serviceStop() |
void |
setEntitlement(String queueName,
QueueEntitlement entitlement)
This method increase the entitlement for current queue (must respect
invariants, e.g., no overcommit of parents, non negative, etc.).
|
void |
setRMContext(RMContext rmContext)
Set RMContext for
ResourceScheduler . |
void |
update()
Recompute the internal variables used by the scheduler - per-job weights,
fair shares, deficits, minimum slot allocations, and amount of used and
required resources per job.
|
void |
updateNodeResource(RMNode nm,
org.apache.hadoop.yarn.api.records.ResourceOption resourceOption)
Process resource update on a node and update Queue.
|
addQueue, asyncContainerRelease, attemptAllocationOnNode, checkAndGetApplicationPriority, clearPendingContainerCache, completedContainer, containerIncreasedOnNode, containerLaunchedOnNode, createReleaseCache, createSchedContainerChangeRequests, getActivitiesManager, getApplicationAttempt, getAppResourceUsageReport, getBlacklistedNodes, getClock, getClusterResource, getCurrentAttemptForContainer, getLastNodeUpdateTime, getMaxClusterLevelAppPriority, getMaximumAllocation, getMaximumApplicationLifetime, getMaximumResourceCapability, getMaximumResourceCapability, getMinimumAllocation, getMinimumResourceCapability, getNode, getNodeIds, getNodeReport, getNodes, getNodeTracker, getPendingResourceRequestsForAttempt, getRMContainer, getSchedulerAppInfo, getSchedulerApplications, getSchedulerHealth, getSchedulerNode, getSchedulingMonitorManager, getTransferredContainers, handleContainerUpdates, initMaximumResourceCapability, killAllAppsInQueue, moveAllApps, normalizeResourceRequests, recoverContainersOnNode, refreshMaximumAllocation, releaseContainers, resetSchedulerMetrics, rollbackContainerUpdate, setClock, setClusterMaxPriority, setLastNodeUpdateTime, shouldContainersBeAutoUpdated, triggerUpdate, updateApplicationPriority, updateNodeResourceUtilization, updateSchedulerHealthInformation
close, getBlockers, getConfig, getFailureCause, getFailureState, getLifecycleHistory, getName, getServiceState, getStartTime, init, isInState, noteFailure, putBlocker, registerGlobalListener, registerServiceListener, removeBlocker, setConfig, start, stop, toString, unregisterGlobalListener, unregisterServiceListener, waitForServiceToStop
public static final org.apache.hadoop.yarn.api.records.Resource CONTAINER_RESERVED
protected boolean sizeBasedWeight
@Deprecated protected boolean continuousSchedulingEnabled
@Deprecated protected volatile int continuousSchedulingSleepMs
protected double nodeLocalityThreshold
protected double rackLocalityThreshold
@Deprecated protected long nodeLocalityDelayMs
@Deprecated protected long rackLocalityDelayMs
protected boolean assignMultiple
protected int maxAssign
public FSContext getContext()
public boolean isAtLeastReservationThreshold(org.apache.hadoop.yarn.util.resource.ResourceCalculator resourceCalculator, org.apache.hadoop.yarn.api.records.Resource resource)
public FairSchedulerConfiguration getConf()
public int getNumNodesInRack(String rackName)
public QueueManager getQueueManager()
public void update()
update
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
public RMContainerTokenSecretManager getContainerTokenSecretManager()
public boolean isSizeBasedWeight()
public org.apache.hadoop.yarn.api.records.Resource getIncrementResourceCapability()
public double getNodeLocalityThreshold()
public double getRackLocalityThreshold()
@Deprecated public long getNodeLocalityDelayMs()
isContinuousSchedulingEnabled()
deprecation@Deprecated public long getRackLocalityDelayMs()
isContinuousSchedulingEnabled()
deprecation@Deprecated public boolean isContinuousSchedulingEnabled()
@Deprecated public int getContinuousSchedulingSleepMs()
isContinuousSchedulingEnabled()
deprecationprotected void addApplication(org.apache.hadoop.yarn.api.records.ApplicationId applicationId, String queueName, String user, boolean isAppRecovering)
protected void addApplicationAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId, boolean transferStateFromPreviousAttempt, boolean isAttemptRecovering)
protected void completedContainerInternal(RMContainer rmContainer, org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus, RMContainerEventType event)
completedContainerInternal
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
public org.apache.hadoop.yarn.api.records.Resource getNormalizedResource(org.apache.hadoop.yarn.api.records.Resource requestedResource)
YarnScheduler
getNormalizedResource
in interface YarnScheduler
getNormalizedResource
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
requestedResource
- the resource to be normalizedpublic void killContainer(RMContainer container)
AbstractYarnScheduler
killContainer
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
container
- the container to killpublic Allocation allocate(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId, List<org.apache.hadoop.yarn.api.records.ResourceRequest> ask, List<org.apache.hadoop.yarn.api.records.SchedulingRequest> schedulingRequests, List<org.apache.hadoop.yarn.api.records.ContainerId> release, List<String> blacklistAdditions, List<String> blacklistRemovals, ContainerUpdates updateRequests)
YarnScheduler
updateRequests
- @return the Allocation
for the applicationprotected void nodeUpdate(RMNode nm)
AbstractYarnScheduler
nodeUpdate
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
nm
- The RMNode corresponding to the NodeManagerpublic FSAppAttempt getSchedulerApp(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId)
public org.apache.hadoop.yarn.util.resource.ResourceCalculator getResourceCalculator()
public QueueMetrics getRootQueueMetrics()
YarnScheduler
public void handle(SchedulerEvent event)
public void recover(RMStateStore.RMState state) throws Exception
Exception
public void setRMContext(RMContext rmContext)
ResourceScheduler
ResourceScheduler
.
This method should be called immediately after instantiating
a scheduler once.rmContext
- created by ResourceManagerprotected void createPreemptionThread()
public void serviceInit(org.apache.hadoop.conf.Configuration conf) throws Exception
serviceInit
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
Exception
public void serviceStart() throws Exception
serviceStart
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
Exception
public void serviceStop() throws Exception
serviceStop
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
Exception
public void reinitialize(org.apache.hadoop.conf.Configuration conf, RMContext rmContext) throws IOException
ResourceScheduler
ResourceScheduler
.reinitialize
in interface ResourceScheduler
reinitialize
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
conf
- configurationIOException
public org.apache.hadoop.yarn.api.records.QueueInfo getQueueInfo(String queueName, boolean includeChildQueues, boolean recursive) throws IOException
YarnScheduler
queueName
- queue nameincludeChildQueues
- include child queues?recursive
- get children queues?IOException
public List<org.apache.hadoop.yarn.api.records.QueueUserACLInfo> getQueueUserAclInfo()
YarnScheduler
public int getNumClusterNodes()
YarnScheduler
public boolean checkAccess(org.apache.hadoop.security.UserGroupInformation callerUGI, org.apache.hadoop.yarn.api.records.QueueACL acl, String queueName)
YarnScheduler
QueueACL.ADMINISTER_QUEUE
permission,
this user can view/modify the applications in this queuetrue
if the user has the permission,
false
otherwisepublic AllocationConfiguration getAllocationConfiguration()
public List<org.apache.hadoop.yarn.api.records.ApplicationAttemptId> getAppsInQueue(String queueName)
YarnScheduler
queueName
- the name of the queue.public String moveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId, String queueName) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
moveApplication
in interface YarnScheduler
moveApplication
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
org.apache.hadoop.yarn.exceptions.YarnException
- if the move cannot be carried outpublic void preValidateMoveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId, String newQueue) throws org.apache.hadoop.yarn.exceptions.YarnException
preValidateMoveApplication
in interface YarnScheduler
preValidateMoveApplication
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
appId
- Application IDnewQueue
- Target QueueNameorg.apache.hadoop.yarn.exceptions.YarnException
- if the pre-validation for move cannot be carried outpublic void updateNodeResource(RMNode nm, org.apache.hadoop.yarn.api.records.ResourceOption resourceOption)
updateNodeResource
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
public EnumSet<org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes> getSchedulingResourceTypes()
getSchedulingResourceTypes
in interface YarnScheduler
getSchedulingResourceTypes
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
public Set<String> getPlanQueues() throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
getPlanQueues
in interface YarnScheduler
getPlanQueues
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
org.apache.hadoop.yarn.exceptions.YarnException
public void setEntitlement(String queueName, QueueEntitlement entitlement) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
setEntitlement
in interface YarnScheduler
setEntitlement
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
queueName
- the queue for which we change entitlemententitlement
- the new entitlement for the queue (capacity,
maxCapacity, etc..)org.apache.hadoop.yarn.exceptions.YarnException
public void removeQueue(String queueName) throws org.apache.hadoop.yarn.exceptions.YarnException
removeQueue
in interface YarnScheduler
removeQueue
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
queueName
- name of queue to removeorg.apache.hadoop.yarn.exceptions.YarnException
- if queue to remove is either not a leaf or if its
not emptypublic float getReservableNodesRatio()
public long checkAndGetApplicationLifetime(String queueName, long lifetime)
YarnScheduler
checkAndGetApplicationLifetime
in interface YarnScheduler
checkAndGetApplicationLifetime
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
queueName
- Name of the Queuelifetime
- configured application lifetimeCopyright © 2008–2020 Apache Software Foundation. All rights reserved.