org.apache.tez.dag.app.rm

Class DagAwareYarnTaskScheduler

java.lang.Object

org.apache.tez.serviceplugins.api.TaskScheduler

org.apache.tez.dag.app.rm.DagAwareYarnTaskScheduler

All Implemented Interfaces:

org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler, org.apache.tez.common.ServicePluginLifecycle

public class DagAwareYarnTaskScheduler
extends org.apache.tez.serviceplugins.api.TaskScheduler
implements org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

A YARN task scheduler that is aware of the dependencies between vertices in the DAG and takes them into account when deciding how to schedule and preempt tasks. This scheduler makes the assumption that vertex IDs start at 0 and are densely allocated (i.e.: there are no "gaps" in the vertex ID space).

Nested Class Summary

Nested classes/interfaces inherited from class org.apache.tez.serviceplugins.api.TaskScheduler

org.apache.tez.serviceplugins.api.TaskScheduler.SchedulerTaskState

Constructor Summary

Constructors

Constructor and Description
DagAwareYarnTaskScheduler(org.apache.tez.serviceplugins.api.TaskSchedulerContext taskSchedulerContext)

Method Summary

Modifier and Type	Method and Description
void	allocateTask(Object task, org.apache.hadoop.yarn.api.records.Resource capability, org.apache.hadoop.yarn.api.records.ContainerId containerId, org.apache.hadoop.yarn.api.records.Priority priority, Object containerSignature, Object clientCookie)
void	allocateTask(Object task, org.apache.hadoop.yarn.api.records.Resource capability, String[] hosts, String[] racks, org.apache.hadoop.yarn.api.records.Priority priority, Object containerSignature, Object clientCookie)
void	blacklistNode(org.apache.hadoop.yarn.api.records.NodeId nodeId)
protected ScheduledExecutorService	createExecutor()
void	dagComplete()
Object	deallocateContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId)
boolean	deallocateTask(Object task, boolean taskSucceeded, org.apache.tez.serviceplugins.api.TaskAttemptEndReason endReason, String diagnostics)
org.apache.hadoop.yarn.api.records.Resource	getAvailableResources()
int	getClusterNodeCount()
float	getProgress()
org.apache.hadoop.yarn.api.records.Resource	getTotalResources()
boolean	hasUnregistered()
void	initialize()
void	initiateStop()
protected long	now()
void	onContainersAllocated(List<org.apache.hadoop.yarn.api.records.Container> containers)
void	onContainersCompleted(List<org.apache.hadoop.yarn.api.records.ContainerStatus> statuses)
void	onError(Throwable e)
void	onNodesUpdated(List<org.apache.hadoop.yarn.api.records.NodeReport> updatedNodes)
void	onShutdownRequest()
void	setShouldUnregister()
void	shutdown()
void	start()
void	unblacklistNode(org.apache.hadoop.yarn.api.records.NodeId nodeId)

Methods inherited from class org.apache.tez.serviceplugins.api.TaskScheduler

getContext, taskStateUpdated

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

DagAwareYarnTaskScheduler

public DagAwareYarnTaskScheduler(org.apache.tez.serviceplugins.api.TaskSchedulerContext taskSchedulerContext)

Method Detail

initialize

public void initialize()
                throws Exception

Specified by:

initialize in interface org.apache.tez.common.ServicePluginLifecycle

Overrides:

initialize in class org.apache.tez.serviceplugins.api.TaskScheduler

Throws:

Exception

start

public void start()
           throws Exception

Specified by:

start in interface org.apache.tez.common.ServicePluginLifecycle

Overrides:

start in class org.apache.tez.serviceplugins.api.TaskScheduler

Throws:

Exception

createExecutor

protected ScheduledExecutorService createExecutor()

now

protected long now()

initiateStop

public void initiateStop()

Overrides:

initiateStop in class org.apache.tez.serviceplugins.api.TaskScheduler

shutdown

public void shutdown()
              throws Exception

Specified by:

shutdown in interface org.apache.tez.common.ServicePluginLifecycle

Overrides:

shutdown in class org.apache.tez.serviceplugins.api.TaskScheduler

Throws:

Exception

onContainersAllocated

public void onContainersAllocated(List<org.apache.hadoop.yarn.api.records.Container> containers)

Specified by:

onContainersAllocated in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onContainersCompleted

public void onContainersCompleted(List<org.apache.hadoop.yarn.api.records.ContainerStatus> statuses)

Specified by:

onContainersCompleted in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onNodesUpdated

public void onNodesUpdated(List<org.apache.hadoop.yarn.api.records.NodeReport> updatedNodes)

Specified by:

onNodesUpdated in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

getProgress

public float getProgress()

Specified by:

getProgress in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onShutdownRequest

public void onShutdownRequest()

Specified by:

onShutdownRequest in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onError

public void onError(Throwable e)

Specified by:

onError in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

getAvailableResources

public org.apache.hadoop.yarn.api.records.Resource getAvailableResources()

Specified by:

getAvailableResources in class org.apache.tez.serviceplugins.api.TaskScheduler

getTotalResources

public org.apache.hadoop.yarn.api.records.Resource getTotalResources()

Specified by:

getTotalResources in class org.apache.tez.serviceplugins.api.TaskScheduler

getClusterNodeCount

public int getClusterNodeCount()

Specified by:

getClusterNodeCount in class org.apache.tez.serviceplugins.api.TaskScheduler

blacklistNode

public void blacklistNode(org.apache.hadoop.yarn.api.records.NodeId nodeId)

Specified by:

blacklistNode in class org.apache.tez.serviceplugins.api.TaskScheduler

unblacklistNode

public void unblacklistNode(org.apache.hadoop.yarn.api.records.NodeId nodeId)

Specified by:

unblacklistNode in class org.apache.tez.serviceplugins.api.TaskScheduler

allocateTask

public void allocateTask(Object task,
                         org.apache.hadoop.yarn.api.records.Resource capability,
                         String[] hosts,
                         String[] racks,
                         org.apache.hadoop.yarn.api.records.Priority priority,
                         Object containerSignature,
                         Object clientCookie)

Specified by:

allocateTask in class org.apache.tez.serviceplugins.api.TaskScheduler

allocateTask

public void allocateTask(Object task,
                         org.apache.hadoop.yarn.api.records.Resource capability,
                         org.apache.hadoop.yarn.api.records.ContainerId containerId,
                         org.apache.hadoop.yarn.api.records.Priority priority,
                         Object containerSignature,
                         Object clientCookie)

Specified by:

allocateTask in class org.apache.tez.serviceplugins.api.TaskScheduler

deallocateTask

public boolean deallocateTask(Object task,
                              boolean taskSucceeded,
                              org.apache.tez.serviceplugins.api.TaskAttemptEndReason endReason,
                              String diagnostics)

Specified by:

deallocateTask in class org.apache.tez.serviceplugins.api.TaskScheduler

deallocateContainer

public Object deallocateContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId)

Specified by:

deallocateContainer in class org.apache.tez.serviceplugins.api.TaskScheduler

setShouldUnregister

public void setShouldUnregister()

Specified by:

setShouldUnregister in class org.apache.tez.serviceplugins.api.TaskScheduler

hasUnregistered

public boolean hasUnregistered()

Specified by:

hasUnregistered in class org.apache.tez.serviceplugins.api.TaskScheduler

dagComplete

public void dagComplete()

Specified by:

dagComplete in class org.apache.tez.serviceplugins.api.TaskScheduler