|
||||||||||
| PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
| SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD | |||||||||
java.lang.Objectorg.apache.hadoop.mapred.TaskTracker
public class TaskTracker
TaskTracker is a process that starts and tracks MR Tasks in a networked environment. It contacts the JobTracker for Task assignments and reporting results.
| Nested Class Summary | |
|---|---|
static class |
TaskTracker.MapOutputServlet
This class is used in TaskTracker's Jetty to serve the map outputs to other nodes. |
| Field Summary | |
|---|---|
static org.apache.commons.logging.Log |
ClientTraceLog
|
static long |
COUNTER_UPDATE_INTERVAL
|
static long |
DEFAULT_DISK_HEALTH_CHECK_INTERVAL
How often TaskTracker needs to check the health of its disks, if not configured using mapred.disk.healthChecker.interval |
static int |
FILE_NOT_FOUND
|
static String |
FOR_REDUCE_TASK
The reduce task number for which this map output is being transferred |
static String |
FROM_MAP_TASK
The map task from which the map output data is being transferred |
static int |
HEARTBEAT_INTERVAL_MIN
|
static org.apache.commons.logging.Log |
LOG
|
static String |
MAP_OUTPUT_LENGTH
The custom http header used for the map output length. |
static String |
MR_CLIENTTRACE_FORMAT
|
static String |
RAW_MAP_OUTPUT_LENGTH
The custom http header used for the "raw" map output length. |
static String |
SUBDIR
|
static int |
SUCCESS
|
static String |
TT_KEYTAB_FILE
|
static String |
TT_LOG_TMP_DIR
|
static String |
TT_RESOURCE_CALCULATOR_PLUGIN
|
static String |
TT_USER_NAME
|
static String |
WORKDIR
|
| Fields inherited from interface org.apache.hadoop.mapred.TaskUmbilicalProtocol |
|---|
versionID |
| Constructor Summary | |
|---|---|
TaskTracker(JobConf conf)
Start with the local machine name, and the default JobTracker |
|
| Method Summary | |
|---|---|
boolean |
canCommit(TaskAttemptID taskid,
org.apache.hadoop.mapred.JvmContext jvmContext)
Child checking whether it can commit |
void |
cleanupStorage()
Removes all contents of temporary storage. |
void |
close()
Close down the TaskTracker and all its components. |
void |
commitPending(TaskAttemptID taskid,
TaskStatus taskStatus,
org.apache.hadoop.mapred.JvmContext jvmContext)
Task is reporting that it is in commit_pending and it is waiting for the commit Response |
void |
done(TaskAttemptID taskid,
org.apache.hadoop.mapred.JvmContext jvmContext)
The task is done. |
void |
fatalError(TaskAttemptID taskId,
String msg,
org.apache.hadoop.mapred.JvmContext jvmContext)
A child task had a fatal error. |
void |
fsError(TaskAttemptID taskId,
String message,
org.apache.hadoop.mapred.JvmContext jvmContext)
A child task had a local filesystem error. |
String |
getConfigVersion()
|
String |
getHostname()
|
int |
getHttpPort()
|
static String |
getJobCacheSubdir(String user)
|
org.apache.hadoop.mapred.InterTrackerProtocol |
getJobClient()
The connection to the JobTracker, used by the TaskRunner for locating remote files. |
static String |
getJobJarFile(String user,
String jobid)
|
String |
getJobTrackerUrl()
|
org.apache.hadoop.mapred.JvmManager |
getJvmManagerInstance()
|
static String |
getLocalJobDir(String user,
String jobid)
|
static String |
getLocalTaskDir(String user,
String jobid,
String taskid)
|
static String |
getLocalTaskDir(String user,
String jobid,
String taskid,
boolean isCleanupAttempt)
|
MapTaskCompletionEventsUpdate |
getMapCompletionEvents(JobID jobId,
int fromEventId,
int maxLocs,
TaskAttemptID id,
org.apache.hadoop.mapred.JvmContext jvmContext)
Called by a reduce task to get the map output locations for finished maps. |
static String |
getPrivateDistributedCacheDir(String user)
|
long |
getProtocolVersion(String protocol,
long clientVersion)
Return protocol version corresponding to protocol interface. |
static String |
getPublicDistributedCacheDir()
|
int |
getRpcPort()
|
JvmTask |
getTask(org.apache.hadoop.mapred.JvmContext context)
Called upon startup by the child process, to fetch Task data. |
TaskController |
getTaskController()
|
org.apache.hadoop.mapred.TaskMemoryManagerThread |
getTaskMemoryManager()
|
String |
getTasksInfoJson()
|
org.apache.hadoop.mapred.TaskTrackerInstrumentation |
getTaskTrackerInstrumentation()
|
InetSocketAddress |
getTaskTrackerReportAddress()
Return the port at which the tasktracker bound to |
static String |
getUserDir(String user)
|
String |
getVersion()
|
boolean |
isHealthy()
|
boolean |
isIdle()
Is this task tracker idle? |
boolean |
isIdleAndClean()
Is this task tracker idle and clean? |
boolean |
isTaskMemoryManagerEnabled()
Is the TaskMemoryManager Enabled on this system? |
protected void |
launchTaskForJob(org.apache.hadoop.mapred.TaskTracker.TaskInProgress tip,
JobConf jobConf,
org.apache.hadoop.mapred.TaskTracker.RunningJob rjob)
|
static void |
main(String[] argv)
Start the TaskTracker, point toward the indicated JobTracker |
void |
mapOutputLost(TaskAttemptID taskid,
String errorMsg)
A completed map task's output has been lost. |
boolean |
ping(TaskAttemptID taskid,
org.apache.hadoop.mapred.JvmContext jvmContext)
Child checking to see if we're alive. |
void |
reportDiagnosticInfo(TaskAttemptID taskid,
String info,
org.apache.hadoop.mapred.JvmContext jvmContext)
Called when the task dies before completion, and we want to report back diagnostic info |
void |
reportNextRecordRange(TaskAttemptID taskid,
org.apache.hadoop.mapred.SortedRanges.Range range,
org.apache.hadoop.mapred.JvmContext jvmContext)
Report the record range which is going to process next by the Task. |
void |
run()
The server retry loop. |
void |
shuffleError(TaskAttemptID taskId,
String message,
org.apache.hadoop.mapred.JvmContext jvmContext)
A reduce-task failed to shuffle the map-outputs. |
void |
shutdown()
|
boolean |
statusUpdate(TaskAttemptID taskid,
TaskStatus taskStatus,
org.apache.hadoop.mapred.JvmContext jvmContext)
Called periodically to report Task progress, from 0.0 to 1.0. |
void |
updatePrivateDistributedCacheSizes(JobID jobId,
long[] sizes)
The job initializer needs to report the sizes of the archive objects and directories in the private distributed cache. |
| Methods inherited from class java.lang.Object |
|---|
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
| Field Detail |
|---|
public static final org.apache.commons.logging.Log LOG
public static final String MR_CLIENTTRACE_FORMAT
public static final org.apache.commons.logging.Log ClientTraceLog
public static final String SUBDIR
public static final String TT_LOG_TMP_DIR
public static final String TT_RESOURCE_CALCULATOR_PLUGIN
public static final String TT_USER_NAME
public static final String TT_KEYTAB_FILE
public static final int HEARTBEAT_INTERVAL_MIN
public static final long COUNTER_UPDATE_INTERVAL
public static final long DEFAULT_DISK_HEALTH_CHECK_INTERVAL
public static final int SUCCESS
public static final int FILE_NOT_FOUND
public static final String MAP_OUTPUT_LENGTH
public static final String RAW_MAP_OUTPUT_LENGTH
public static final String FROM_MAP_TASK
public static final String FOR_REDUCE_TASK
public static final String WORKDIR
| Constructor Detail |
|---|
public TaskTracker(JobConf conf)
throws IOException,
InterruptedException
IOException
InterruptedException| Method Detail |
|---|
public org.apache.hadoop.mapred.TaskTrackerInstrumentation getTaskTrackerInstrumentation()
public TaskController getTaskController()
public static String getUserDir(String user)
public static String getPrivateDistributedCacheDir(String user)
public static String getPublicDistributedCacheDir()
public static String getJobCacheSubdir(String user)
public static String getLocalJobDir(String user,
String jobid)
public static String getJobJarFile(String user,
String jobid)
public static String getLocalTaskDir(String user,
String jobid,
String taskid)
public static String getLocalTaskDir(String user,
String jobid,
String taskid,
boolean isCleanupAttempt)
public long getProtocolVersion(String protocol,
long clientVersion)
throws IOException
VersionedProtocol
getProtocolVersion in interface VersionedProtocolprotocol - The classname of the protocol interfaceclientVersion - The version of the protocol that the client speaks
IOException
public void cleanupStorage()
throws IOException
IOException
protected void launchTaskForJob(org.apache.hadoop.mapred.TaskTracker.TaskInProgress tip,
JobConf jobConf,
org.apache.hadoop.mapred.TaskTracker.RunningJob rjob)
throws IOException
IOException
public void shutdown()
throws IOException,
InterruptedException
IOException
InterruptedException
public void close()
throws IOException,
InterruptedException
InterruptedException
IOExceptionpublic org.apache.hadoop.mapred.InterTrackerProtocol getJobClient()
public InetSocketAddress getTaskTrackerReportAddress()
public org.apache.hadoop.mapred.JvmManager getJvmManagerInstance()
public void run()
run in interface Runnable
public JvmTask getTask(org.apache.hadoop.mapred.JvmContext context)
throws IOException
getTask in interface TaskUmbilicalProtocolcontext - the JvmContext of the JVM w.r.t the TaskTracker that
launched it
IOException
public boolean statusUpdate(TaskAttemptID taskid,
TaskStatus taskStatus,
org.apache.hadoop.mapred.JvmContext jvmContext)
throws IOException
statusUpdate in interface TaskUmbilicalProtocoltaskid - task-id of the childtaskStatus - status of the childjvmContext - context the jvmContext running the task.
IOException
public void reportDiagnosticInfo(TaskAttemptID taskid,
String info,
org.apache.hadoop.mapred.JvmContext jvmContext)
throws IOException
reportDiagnosticInfo in interface TaskUmbilicalProtocoltaskid - the id of the task involvedinfo - the text to reportjvmContext - context the jvmContext running the task.
IOException
public void reportNextRecordRange(TaskAttemptID taskid,
org.apache.hadoop.mapred.SortedRanges.Range range,
org.apache.hadoop.mapred.JvmContext jvmContext)
throws IOException
TaskUmbilicalProtocol
reportNextRecordRange in interface TaskUmbilicalProtocoltaskid - the id of the task involvedrange - the range of record sequence nosjvmContext - context the jvmContext running the task.
IOException
public boolean ping(TaskAttemptID taskid,
org.apache.hadoop.mapred.JvmContext jvmContext)
throws IOException
ping in interface TaskUmbilicalProtocoltaskid - the id of the task involvedjvmContext - context the jvmContext running the task.
IOException
public void commitPending(TaskAttemptID taskid,
TaskStatus taskStatus,
org.apache.hadoop.mapred.JvmContext jvmContext)
throws IOException
commitPending in interface TaskUmbilicalProtocoltaskid - task's idtaskStatus - status of the childjvmContext - context the jvmContext running the task.
IOException
public boolean canCommit(TaskAttemptID taskid,
org.apache.hadoop.mapred.JvmContext jvmContext)
throws IOException
canCommit in interface TaskUmbilicalProtocoljvmContext - context the jvmContext running the task.
IOException
public void done(TaskAttemptID taskid,
org.apache.hadoop.mapred.JvmContext jvmContext)
throws IOException
done in interface TaskUmbilicalProtocoltaskid - task's idjvmContext - context the jvmContext running the task.
IOException
public void shuffleError(TaskAttemptID taskId,
String message,
org.apache.hadoop.mapred.JvmContext jvmContext)
throws IOException
shuffleError in interface TaskUmbilicalProtocolIOException
public void fsError(TaskAttemptID taskId,
String message,
org.apache.hadoop.mapred.JvmContext jvmContext)
throws IOException
fsError in interface TaskUmbilicalProtocolIOException
public void fatalError(TaskAttemptID taskId,
String msg,
org.apache.hadoop.mapred.JvmContext jvmContext)
throws IOException
fatalError in interface TaskUmbilicalProtocolIOException
public MapTaskCompletionEventsUpdate getMapCompletionEvents(JobID jobId,
int fromEventId,
int maxLocs,
TaskAttemptID id,
org.apache.hadoop.mapred.JvmContext jvmContext)
throws IOException
TaskUmbilicalProtocol
getMapCompletionEvents in interface TaskUmbilicalProtocoljobId - the reducer job idfromEventId - the index starting from which the locations should be
fetchedmaxLocs - the max number of locations to fetchid - The attempt id of the task that is trying to communicate
MapTaskCompletionEventsUpdate
IOException
public void mapOutputLost(TaskAttemptID taskid,
String errorMsg)
throws IOException
IOExceptionpublic boolean isIdle()
public boolean isIdleAndClean()
public static void main(String[] argv)
throws Exception
Exceptionpublic boolean isTaskMemoryManagerEnabled()
public org.apache.hadoop.mapred.TaskMemoryManagerThread getTaskMemoryManager()
public String getHostname()
getHostname in interface TaskTrackerMXBeanpublic String getVersion()
getVersion in interface TaskTrackerMXBeanpublic String getConfigVersion()
getConfigVersion in interface TaskTrackerMXBeanpublic String getJobTrackerUrl()
getJobTrackerUrl in interface TaskTrackerMXBeanpublic int getRpcPort()
getRpcPort in interface TaskTrackerMXBeanpublic int getHttpPort()
getHttpPort in interface TaskTrackerMXBeanpublic boolean isHealthy()
isHealthy in interface TaskTrackerMXBeanpublic String getTasksInfoJson()
getTasksInfoJson in interface TaskTrackerMXBean
public void updatePrivateDistributedCacheSizes(JobID jobId,
long[] sizes)
throws IOException
TaskUmbilicalProtocol
updatePrivateDistributedCacheSizes in interface TaskUmbilicalProtocoljobId - the job to updatesizes - the array of sizes that were computed
IOException
|
||||||||||
| PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
| SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD | |||||||||