/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapred;
import java.io.IOException;
import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.corona.ResourceGrant;
import org.apache.hadoop.corona.ResourceRequest;
import org.apache.hadoop.corona.SessionDriver;
import org.apache.hadoop.corona.Utilities;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.mapred.CoronaCommitPermission.CommitPermissionServer;
import org.apache.hadoop.mapred.CoronaSessionInfo.InetSocketAddressWritable;
/**
* The Proxy used by the CoronaJobTracker in the client to communicate
* with the CoronaJobTracker running on the TaskTracker in case of a
* remote CoronaJobTracker
*/
@SuppressWarnings("deprecation")
public class RemoteJTProxy implements InterCoronaJobTrackerProtocol,
JobSubmissionProtocol {
/** Logger */
public static final Log LOG = LogFactory.getLog(CoronaJobTracker.class);
/** Amount of time to wait for remote JT to launch. */
public static final String REMOTE_JT_TIMEOUT_SEC_CONF =
"mapred.coronajobtracker.remotejobtracker.wait";
/** Flag used for test, if to exclude the failed remote job tracker. */
public static final String REMOTE_JT_EXCLUDE_FAILED =
"mpared.coronajobtracker.remotejobtracker.exclude";
/** Default amount of time to wait for remote JT to launch. */
public static final int REMOTE_JT_TIMEOUT_SEC_DEFAULT = 60;
/** Amount of time for a RPC call timeout to remote JT. */
public static final String REMOTE_JT_RPC_TIMEOUT_SEC_CONF =
"mapred.coronajobtracker.remotejobtracker.rpc.timeout";
/** Default amount of a RPC call timeout to remote JT. */
public static final int REMOTE_JT_RPC_TIMEOUT_SEC_DEFAULT = 3600;
/** Boolean, determines whether remote JT restart should restore state */
public static final String REMOTE_JT_STATE_RESTORING_CONF =
"mapred.coronajobtracker.remote.state.restoring";
/** Default use state restoring mechanism */
public static final boolean REMOTE_JT_STATE_RESTORING_DEFAULT = true;
/** The proxy object to the CoronaJobTracker running in the cluster */
private volatile JobSubmissionProtocol client;
/** JobSubmissionProtocol client lock */
// We do need fair lock to enable early recovery after remote JT crash.
private ReadWriteLock clientLock = new ReentrantReadWriteLock(true);
// TODO Giving priority to writers will increase performance.
/** The host where the remote Job Tracker is running. */
private String remoteJTHost;
/** The port where the remote Job Tracker is running. */
private int remoteJTPort;
/** The task id for the current attempt of running CJT */
private TaskAttemptID currentAttemptId;
/** The number of the current attempt */
private int attempt;
/** Job configuration */
private final JobConf conf;
/** Parent JobTracker */
private final CoronaJobTracker jt;
/** The remote JT resource grant. */
private ResourceGrant remoteJTGrant;
/** The id of the job */
private final JobID jobId;
/** Is true iff the job has been submitted */
private boolean isJobSubmitted = false;
/** The session id for the job tracker running in the cluster */
private String remoteSessionId;
/** The number of remote JT restart attempts. */
private volatile int numRemoteJTFailures;
/** The limit for remote JT restart attempts number. */
private final int maxRemoteJTFailures;
/** Current job attempt id */
private JobID attemptJobId;
/** Address of remote JT */
private InetSocketAddress remoteJTAddr;
/** Holds exceptions from restarting */
public volatile IOException restartingException = null;
/** Saved state updates from remote JT */
private final CoronaJTState remoteJTState;
/** Authority that gives permission to commit */
private final CommitPermissionServer commmitPermissionServer;
private enum RemoteJTStatus {
UNINITIALIZED,
SUCCESS,
FAILURE
};
private RemoteJTStatus remoteJTStatus;
//This variable is our internal logic control flag.
//It means when the RJT failover is enabled, which API call failure will cause failover.
//For API call like killJob, killTasks should not fire the RJT failover.
protected volatile boolean isRestartable = true;
/**
* Construct a proxy for the remote job tracker
* @param jt parent job tracker
* @param jobId id of the job the proxy is created for
* @param conf job configuration
* @throws IOException
*/
RemoteJTProxy(CoronaJobTracker jt, JobID jobId, JobConf conf) throws IOException {
this.maxRemoteJTFailures = conf.getInt(CoronaJobTracker.MAX_JT_FAILURES_CONF,
CoronaJobTracker.MAX_JT_FAILURES_DEFAULT);
this.conf = conf;
this.jt = jt;
this.jobId = jobId;
// Prepare first attempt.
this.attemptJobId = jobId;
attempt = 0;
int partitionId = conf.getNumMapTasks() + 100000;
currentAttemptId = new TaskAttemptID(new TaskID(attemptJobId, true,
partitionId), attempt);
remoteJTStatus = RemoteJTStatus.UNINITIALIZED;
// Prepare stuff for restoring state if necessary
if (isStateRestoringEnabled(conf)) {
remoteJTState = new CoronaJTState();
commmitPermissionServer = new CommitPermissionServer();
} else {
remoteJTState = null;
commmitPermissionServer = null;
}
}
public String getRemoteSessionId() {
return remoteSessionId;
}
// ///////////////////////////////////////////////////////////////////////////
// InterCoronaJobTrackerProtocol
// ///////////////////////////////////////////////////////////////////////////
@Override
public void reportRemoteCoronaJobTracker(
String attempt,
String host,
int port,
String sessionId) throws IOException {
TaskAttemptID attemptId = TaskAttemptID.forName(attempt);
synchronized (this) {
checkAttempt(attemptId);
initializeClientUnprotected(host, port, sessionId);
this.notifyAll();
}
}
@Override
public InetSocketAddressWritable getNewJobTrackerAddress(
InetSocketAddressWritable failedTracker) throws IOException {
// Die immediately if restarting is disabled
if (maxRemoteJTFailures == 0) {
throw new IOException("Restarting remote JT is disabled.");
}
assert remoteJTAddr != null : "Not started, but got request to restart.";
if (clientLock.readLock().tryLock()) {
// We're not restarting, check address
InetSocketAddress seenAddr = remoteJTAddr;
clientLock.readLock().unlock();
// seenAddr is safe to use, because even if restarting takes place, this
// is the address of JT that either is fully running or dead
// (not currently restarting)
if (seenAddr.equals(failedTracker.getAddress())) {
// Not restarted yet
return null;
} else {
LOG.info("Serving new job tracker address request with " + seenAddr
+ " old " + failedTracker.getAddress());
return new InetSocketAddressWritable(seenAddr);
}
} else {
// Currently restarting
return null;
}
}
@Override
public void pushCoronaJobTrackerStateUpdate(TaskAttemptID attempt,
CoronaStateUpdate[] updates) throws IOException {
checkAttempt(attempt);
if (remoteJTState == null) {
throw new IOException(
"Logic error: got state update but state restoring is disabled");
} else {
synchronized (remoteJTState) {
for (CoronaStateUpdate update : updates) {
remoteJTState.add(update);
}
}
}
}
@Override
public CoronaJTState getCoronaJobTrackerState(TaskAttemptID attemptId)
throws IOException {
checkAttempt(attemptId);
if (remoteJTState == null) {
throw new IOException("Logic error: asked for remote JT state but state "
+ "restoring is disabled");
} else {
synchronized (remoteJTState) {
return remoteJTState.prepare();
}
}
}
@Override
public TaskAttemptID[] getAndSetCommitting(TaskAttemptID attemptId,
TaskAttemptID[] toCommit) throws IOException {
checkAttempt(attemptId);
if (commmitPermissionServer == null) {
throw new IOException(
"Logic error: got getAndSet for committing attempt "
+ "but commit permission server is down");
} else {
return commmitPermissionServer.getAndSetCommitting(toCommit);
}
}
@Override
public ProtocolSignature getProtocolSignature(String protocol,
long clientVersion, int clientMethodsHash) throws IOException {
return ProtocolSignature.getProtocolSignature(
this, protocol, clientVersion, clientMethodsHash);
}
@Override
public long getProtocolVersion(String protocol, long clientVersion)
throws IOException {
if (protocol.equals(InterCoronaJobTrackerProtocol.class.getName())) {
return InterCoronaJobTrackerProtocol.versionID;
} else {
throw new IOException("Unknown protocol " + protocol);
}
}
/**
* Increment the attempt number for launching a remote corona job tracker.
* Must be called only when holding the object lock.
*/
private void incrementAttemptUnprotected() {
attempt++;
currentAttemptId = new TaskAttemptID(new TaskID(attemptJobId,
currentAttemptId.isMap(), currentAttemptId.getTaskID().getId()),
attempt);
}
/**
* Checks whether provided attempt id of remote JT matches currently set,
* throws if not
* @param attempt attempt id to check
* @throws IOException
*/
private void checkAttempt(TaskAttemptID attemptId) throws IOException {
if (!attemptId.equals(currentAttemptId)) {
throw new IOException("Attempt " + attemptId
+ " does not match current attempt " + currentAttemptId);
}
}
/**
* Create the RPC client to the remote corona job tracker.
* @param host The host running the remote corona job tracker.
* @param port The port of the remote corona job tracker.
* @param sessionId The session for the remote corona job tracker.
* @throws IOException
*/
void initializeClientUnprotected(String host, int port, String sessionId)
throws IOException {
if (client != null) {
return;
}
LOG.info("Creating JT client to " + host + ":" + port);
long connectTimeout = RemoteJTProxy.getRemotJTTimeout(conf);
int rpcTimeout = RemoteJTProxy.getRemoteJTRPCTimeout(conf);
remoteJTAddr = new InetSocketAddress(host, port);
client = RPC.waitForProtocolProxy(
JobSubmissionProtocol.class,
JobSubmissionProtocol.versionID,
remoteJTAddr,
conf,
connectTimeout,
rpcTimeout
).getProxy();
remoteJTStatus = RemoteJTStatus.SUCCESS;
remoteJTHost = host;
remoteJTPort = port;
remoteSessionId = sessionId;
if (remoteJTState != null) {
remoteJTState.setSessionId(sessionId);
}
}
private void reinitClientUnprotected() throws IOException {
if (client != null) {
RPC.stopProxy(client);
client = null;
remoteJTStatus = RemoteJTStatus.UNINITIALIZED;
}
try {
initializeClientUnprotected(remoteJTHost, remoteJTPort, remoteSessionId);
} finally {
if (client == null) {
remoteJTStatus = RemoteJTStatus.FAILURE;
}
}
}
/**
* Waits for the remote Corona JT to be ready.
* This involves
* - getting a JOBTRACKER resource from the cluster manager.
* - starting the remote job tracker by connecting to the corona task
* tracker on the machine.
* - waiting for the remote job tracker to report its port back to this
* process.
* @param jobConf The job configuration to use.
* @throws IOException
*/
public void waitForJTStart(JobConf jobConf) throws IOException {
int maxJTAttempts = jobConf.getInt(
"mapred.coronajobtracker.remotejobtracker.attempts", 4);
ResourceTracker resourceTracker = jt.getResourceTracker();
SessionDriver sessionDriver = jt.getSessionDriver();
List<ResourceGrant> excludeGrants = new ArrayList<ResourceGrant>();
boolean toExcludeFailed = jobConf.getBoolean(REMOTE_JT_EXCLUDE_FAILED, true);
// Release and blacklist failed JT grant.
if (remoteJTGrant != null) {
if (toExcludeFailed) {
excludeGrants.add(remoteJTGrant);
}
resourceTracker.releaseResource(remoteJTGrant.getId());
sessionDriver.releaseResources(resourceTracker.getResourcesToRelease());
}
for (int i = 0; i < maxJTAttempts; i++) {
try {
remoteJTGrant = waitForJTGrant(resourceTracker, sessionDriver,
excludeGrants);
boolean success = startRemoteJT(jobConf, remoteJTGrant);
if (success) {
return;
} else {
excludeGrants.add(remoteJTGrant);
resourceTracker.releaseResource(remoteJTGrant.getId());
List<ResourceRequest> released =
resourceTracker.getResourcesToRelease();
sessionDriver.releaseResources(released);
}
} catch (InterruptedException e) {
throw new IOException(e);
}
}
throw new IOException("Could not start remote JT after " + maxJTAttempts +
" attempts");
}
/**
* Wait for a JOBTRACKER grant.
* @param resourceTracker The resource tracker object for getting the grant
* @param sessionDriver The session driver for getting the grant
* @param previousGrants Previous grants that could not be used successfully.
* @return A new JOBTRACKER grant.
* @throws IOException
* @throws InterruptedException
*/
private ResourceGrant waitForJTGrant(
ResourceTracker resourceTracker,
SessionDriver sessionDriver,
List<ResourceGrant> previousGrants)
throws IOException, InterruptedException {
LOG.info("Waiting for JT grant for " + attemptJobId);
ResourceRequest req = resourceTracker.newJobTrackerRequest();
for (ResourceGrant prev: previousGrants) {
LOG.info("Adding " + prev.getNodeName() + " to excluded hosts");
req.addToExcludeHosts(prev.getAddress().getHost());
}
resourceTracker.recordRequest(req);
List<ResourceRequest> newRequests = resourceTracker.getWantedResources();
sessionDriver.requestResources(newRequests);
final List<ResourceGrant> grants = new ArrayList<ResourceGrant>();
ResourceTracker.ResourceProcessor proc =
new ResourceTracker.ResourceProcessor() {
@Override
public boolean processAvailableResource(ResourceGrant resource) {
grants.add(resource);
final boolean consumed = true;
return consumed;
}
};
while (true) {
// Try to get JT grant while periodically checking for session driver
// exceptions.
long timeout = 60 * 1000; // 1 min.
resourceTracker.processAvailableGrants(proc, 1, timeout);
IOException e = sessionDriver.getFailed();
if (e != null) {
throw e;
}
if (!grants.isEmpty()) {
return grants.get(0);
}
}
}
/**
* Start corona job tracker on the machine provided by using the corona
* task tracker API.
* @param jobConf The job configuration.
* @param grant The grant that specifies the remote machine.
* @return A boolean indicating success.
* @throws InterruptedException
*/
private boolean startRemoteJT(
JobConf jobConf,
ResourceGrant grant) throws InterruptedException {
org.apache.hadoop.corona.InetAddress ttAddr =
Utilities.appInfoToAddress(grant.appInfo);
CoronaTaskTrackerProtocol coronaTT = null;
try {
coronaTT = jt.getTaskTrackerClient(ttAddr.getHost(), ttAddr.getPort());
} catch (IOException e) {
LOG.error("Error while trying to connect to TT at " + ttAddr.getHost() +
":" + ttAddr.getPort(), e);
return false;
}
LOG.warn("Starting remote JT for " + attemptJobId
+ " on " + ttAddr.getHost());
// Get a special map id for the JT task.
Path systemDir = new Path(jt.getSystemDir());
LOG.info("startRemoteJT:systemDir "+systemDir.toString());
String jobFile = CoronaJobInProgress.getJobFile(systemDir, attemptJobId)
.toString();
LOG.info("startRemoteJT:jobFile " + jobFile);
String splitClass = JobClient.RawSplit.class.getName();
BytesWritable split = new BytesWritable();
Task jobTask = new MapTask(
jobFile, currentAttemptId, currentAttemptId.getTaskID().getId(),
splitClass, split, 1, jobConf.getUser());
CoronaSessionInfo info = new CoronaSessionInfo(jt.getSessionId(),
jt.getJobTrackerAddress(), jt.getJobTrackerAddress());
synchronized (this) {
try {
coronaTT.startCoronaJobTracker(jobTask, info);
} catch (IOException e) {
// Increment the attempt so that the older attempt will get an error
// in reportRemoteCoronaJobTracker().
incrementAttemptUnprotected();
LOG.error("Error while performing RPC to TT at " + ttAddr.getHost() +
":" + ttAddr.getPort(), e);
return false;
}
}
// Now wait for the remote CJT to report its address.
final long waitStart = System.currentTimeMillis();
final long timeout = RemoteJTProxy.getRemotJTTimeout(jobConf);
synchronized (this) {
while (client == null) {
LOG.warn("Waiting for remote JT to start on " + ttAddr.getHost());
this.wait(1000);
if (client == null &&
System.currentTimeMillis() - waitStart > timeout) {
// Increment the attempt so that the older attempt will get an error
// in reportRemoteCoronaJobTracker().
incrementAttemptUnprotected();
LOG.warn("Could not start remote JT on " + ttAddr.getHost());
return false;
}
}
}
return true;
}
/**
* Returns the timeout in milliseconds after which we timeout the remote job
* tracker.
*
* @param conf
* The configuration
* @return The timeout in milliseconds.
*/
public static long getRemotJTTimeout(Configuration conf) {
return conf.getInt(RemoteJTProxy.REMOTE_JT_TIMEOUT_SEC_CONF,
RemoteJTProxy.REMOTE_JT_TIMEOUT_SEC_DEFAULT) * 1000;
}
public static int getRemoteJTRPCTimeout(Configuration conf) {
return conf.getInt(RemoteJTProxy.REMOTE_JT_RPC_TIMEOUT_SEC_CONF,
RemoteJTProxy.REMOTE_JT_RPC_TIMEOUT_SEC_DEFAULT) * 1000;
}
// ///////////////////////////////////////////////////////////////////////////
// JobSubmissionProtocol
// ///////////////////////////////////////////////////////////////////////////
@Override
public JobID getNewJobId() throws IOException {
throw new UnsupportedOperationException(
"getNewJobId not supported by proxy");
}
@Override
public JobStatus submitJob(final JobID jobId) throws IOException {
return (new Caller<JobStatus>() {
@Override
JobStatus call(JobSubmissionProtocol myClient) throws IOException {
// This is first time job submission. Called only once
isJobSubmitted = true;
return myClient.submitJob(attemptJobId);
}
}).makeCall();
}
@Override
public ClusterStatus getClusterStatus(boolean detailed) throws IOException {
throw new UnsupportedOperationException(
"getClusterStatus is not supported by proxy");
}
@Override
public void killJob(final JobID jobId) throws IOException {
(new Caller<JobID>() {
// If the job tracker who hosting the job died,
// will not do an automatic failover
@Override
protected boolean isRestartableCall() {
return false;
}
@Override
JobID call(JobSubmissionProtocol myClient) throws IOException {
myClient.killJob(attemptJobId);
return jobId;
}
}).makeCall();
}
@Override
public void setJobPriority(JobID jobId, String priority) throws IOException {
throw new UnsupportedOperationException(
"setJobPriority is not supported by proxy");
}
@Override
public boolean killTask(final TaskAttemptID taskId, final boolean shouldFail)
throws IOException {
return (new Caller<Boolean>() {
// If the job tracker who hosting the task died,
// will not do an automatic failover
@Override
protected boolean isRestartableCall() {
return false;
}
@Override
Boolean call(JobSubmissionProtocol myClient) throws IOException {
return myClient.killTask(taskId, shouldFail);
}
}).makeCall();
}
@Override
public JobProfile getJobProfile(final JobID jobId) throws IOException {
return (new Caller<JobProfile>() {
@Override
JobProfile call(JobSubmissionProtocol myClient) throws IOException {
return myClient.getJobProfile(attemptJobId);
}
}).makeCall();
}
@Override
public JobStatus getJobStatus(final JobID jobId) throws IOException {
return (new Caller<JobStatus>() {
@Override
JobStatus call(JobSubmissionProtocol myClient) throws IOException {
return myClient.getJobStatus(attemptJobId);
}
}).makeCall();
}
@Override
public Counters getJobCounters(final JobID jobId) throws IOException {
return (new Caller<Counters>() {
@Override
Counters call(JobSubmissionProtocol myClient) throws IOException {
return myClient.getJobCounters(attemptJobId);
}
}).makeCall();
}
@Override
public TaskReport[] getMapTaskReports(final JobID jobId) throws IOException {
return (new Caller<TaskReport[]>() {
@Override
TaskReport[] call(JobSubmissionProtocol myClient) throws IOException {
return myClient.getMapTaskReports(attemptJobId);
}
}).makeCall();
}
@Override
public TaskReport[] getReduceTaskReports(final JobID jobId)
throws IOException {
return (new Caller<TaskReport[]>() {
@Override
TaskReport[] call(JobSubmissionProtocol myClient) throws IOException {
return myClient.getReduceTaskReports(attemptJobId);
}
}).makeCall();
}
@Override
public TaskReport[] getCleanupTaskReports(final JobID jobId)
throws IOException {
return (new Caller<TaskReport[]>() {
@Override
TaskReport[] call(JobSubmissionProtocol myClient) throws IOException {
return myClient.getCleanupTaskReports(attemptJobId);
}
}).makeCall();
}
@Override
public TaskReport[] getSetupTaskReports(final JobID jobId)
throws IOException {
return (new Caller<TaskReport[]>() {
@Override
TaskReport[] call(JobSubmissionProtocol myClient) throws IOException {
return myClient.getSetupTaskReports(attemptJobId);
}
}).makeCall();
}
@Override
public String getFilesystemName() throws IOException {
throw new UnsupportedOperationException(
"getFilesystemName is not supported by proxy");
}
@Override
public JobStatus[] jobsToComplete() {
throw new UnsupportedOperationException(
"jobsToComplete is not supported by proxy");
}
@Override
public JobStatus[] getAllJobs() {
throw new UnsupportedOperationException(
"getAllJobs is not supported by proxy");
}
@Override
public TaskCompletionEvent[] getTaskCompletionEvents(final JobID jobid,
final int fromEventId, final int maxEvents) throws IOException {
return (new Caller<TaskCompletionEvent[]>() {
@Override
TaskCompletionEvent[] call(JobSubmissionProtocol myClient) throws IOException {
return myClient.getTaskCompletionEvents(attemptJobId, fromEventId, maxEvents);
}
}).makeCall();
}
@Override
public String[] getTaskDiagnostics(final TaskAttemptID taskId)
throws IOException {
return (new Caller<String[]>() {
@Override
String[] call(JobSubmissionProtocol myClient) throws IOException {
return myClient.getTaskDiagnostics(taskId);
}
}).makeCall();
}
@Override
public String getSystemDir() {
throw new UnsupportedOperationException(
"getSystemDir not supported by proxy.");
}
@Override
public JobQueueInfo[] getQueues() {
throw new UnsupportedOperationException("getQueues method is " +
"not supported by proxy.");
}
@Override
public JobQueueInfo getQueueInfo(String queue) {
throw new UnsupportedOperationException(
"getQueueInfo not supported by proxy.");
}
@Override
public JobStatus[] getJobsFromQueue(String queue) {
throw new UnsupportedOperationException(
"getJobsFromQueue not supported by proxy.");
}
@Override
public QueueAclsInfo[] getQueueAclsForCurrentUser() throws IOException {
throw new UnsupportedOperationException(
"getQueueAclsForCurrentUser not supported by proxy.");
}
/**
* Stop RPC client.
*/
public void close() {
clientLock.writeLock().lock();
try {
if (client != null) {
RPC.stopProxy(client);
client = null;
}
} finally {
clientLock.writeLock().unlock();
}
}
// ///////////////////////////////////////////////////////////////////////////
// Remote CJT reincarnation.
// ///////////////////////////////////////////////////////////////////////////
/**
* Generic caller interface.
*/
private abstract class Caller<T> {
/**
* Perform the call. Must be overridden by a sub-class.
* @param myClient the client to make the call with.
* @return The generic return value.
* @throws IOException
*/
abstract T call(JobSubmissionProtocol myClient) throws IOException;
/**
* Overriding it to let the caller know if the current call is a
* restartable one. It means if failed to call RJT, if we need to
* do an automatic failover
*
*/
protected boolean isRestartableCall() {
return isRestartable;
}
/**
* Template function to make the call.
* @return The generic return value.
* @throws IOException
*/
public T makeCall() throws IOException {
int curRestartNo;
// If restart fails, exception will break this loop
while(true) {
clientLock.readLock().lock();
curRestartNo = numRemoteJTFailures;
try {
try {
return makeCallWithRetries();
} finally {
clientLock.readLock().unlock();
}
} catch (IOException e) {
LOG.error("Error on remote call with retries", e);
if (isRestartableCall()) {
handleRemoteJTFailure(curRestartNo);
} else {
throw e;
}
}
}
}
/**
* Handles remote JT failure.
* @param failureRestartNo numRemoteJTFailures when failure that issued this
* call has occurred
* @return true iff remote call can be repeated
* @throws IOException InterruptedException
*/
private void handleRemoteJTFailure(int failureRestartNo)
throws IOException {
clientLock.writeLock().lock();
try {
if (failureRestartNo == numRemoteJTFailures) {
try {
LOG.warn("failureRestartNo " + failureRestartNo +
" maxRemoteFailures " + maxRemoteJTFailures +
" numFailure " + numRemoteJTFailures);
++numRemoteJTFailures;
if (numRemoteJTFailures <= maxRemoteJTFailures) {
LOG.warn("JobTracker died or is unreachable."
+ " Restarting remote JT.");
synchronized(RemoteJTProxy.this) {
restartRemoteJTUnprotected();
}
} else {
LOG.warn("JobTracker died or is unreachable."
+ " Reached restart number limit."
+ " Reporting to ClusterManager.");
if (remoteSessionId != null) {
// Kill remote session - it will release resources immediately
jt.getSessionDriver().stopRemoteSession(remoteSessionId);
}
jt.close(false, true);
throw new IOException("Reached remote JT restart limit.");
}
} catch (IOException e) {
restartingException = e;
throw restartingException;
} catch (InterruptedException e) {
restartingException = new IOException(e);
throw restartingException;
}
} else {
// Other thread restarted remote JT, check if successfully
if (restartingException != null) {
throw new IOException(restartingException);
}
}
} finally {
clientLock.writeLock().unlock();
}
}
/**
* Restarts remote JT if there was running job and resubmits this job.
* @throws IOException
*/
private void restartRemoteJTUnprotected() throws IOException {
SessionDriver sessionDriver = jt.getSessionDriver();
if (remoteSessionId != null) {
// Kill remote session - new JT will acquire new session
sessionDriver.stopRemoteSession(remoteSessionId);
}
if (!isStateRestoringEnabled(conf)) {
// Change attempt id only if we're not restoring state
attemptJobId = prepareNextAttempt(attemptJobId);
} else {
// notify the remote job tracker the number of
// remote job tracker get restarted
remoteJTState.restartNum = numRemoteJTFailures;
}
// Stop RPC client.
RPC.stopProxy(client);
client = null;
// Increment attempt to kill old JT on next connection attempt.
incrementAttemptUnprotected();
if (sessionDriver != null) {
sessionDriver.setName("Launch pending for " + conf.getJobName());
}
// Restart remote JT, don't release client lock yet.
waitForJTStart(conf);
// Resubmit job directly.
try {
if (isJobSubmitted) {
LOG.warn("Resubmitting job " + jobId.toString());
client.submitJob(attemptJobId);
}
// Set our info server url in parent JT and CM.
String url = getJobProfile(attemptJobId).getURL().toString();
jt.setRemoteJTUrl(url);
if (sessionDriver != null) {
sessionDriver.setName("Launched session " + getRemoteSessionId());
}
// If reached this point assume success.
LOG.warn("Successfully restarted remote JT.");
if (remoteJTState != null) {
if (LOG.isInfoEnabled()) {
synchronized (remoteJTState) {
LOG.warn(remoteJTState.getPrettyReport(attemptJobId));
}
}
}
} catch (IOException e) {
// in case the new job tracker get failed when doing submitJob
// or getJobProfile
LOG.error("Exception happened when doing RJT restart, try it another time",
e);
handleRemoteJTFailure(numRemoteJTFailures);
}
}
/**
* Prepares next attempt of job.
* @param oldId a job id of last submitted attempt or id known by client
* @return job id of next attempt
* @throws IOException
*/
private JobID prepareNextAttempt(final JobID oldId) throws IOException {
JobID newId = CoronaJobTracker.nextJobID(oldId);
// TODO copy only necessary files
Path oldJobDir = new Path(jt.getSystemDir(), oldId.toString()),
newJobDir = new Path(jt.getSystemDir(), newId.toString());
FileSystem fs = FileSystem.get(conf);
LOG.info("oldJobDir " + oldJobDir.toString() + " newJobDir " + newJobDir.toString() );
// Copy job files.
Path localTemp = new Path("file:///tmp/" + newId.toString());
if (fs.exists(newJobDir)) {
LOG.info("newJobDir "+ localTemp.toString() + " exists, delete it");
fs.delete(newJobDir, true);
}
if (!oldJobDir.equals(newJobDir) && fs.exists(oldJobDir)) {
fs.copyToLocalFile(oldJobDir, localTemp);
fs.moveFromLocalFile(localTemp, newJobDir);
}
LOG.info("Job files copied to " + newJobDir.toString());
return newId;
}
private T makeCallWithRetries() throws IOException {
int errorCount = 0;
final int maxErrorCount = 10; // can make configurable later
IOException lastException = null;
while (errorCount < maxErrorCount) {
try {
JobSubmissionProtocol myClient = checkClient();
return call(myClient);
} catch (ConnectException e) {
throw e;
} catch (IOException e) {
lastException = e;
errorCount++;
if (errorCount == maxErrorCount) {
break;
} else {
long backoff = errorCount * 1000;
LOG.warn(
"Retrying after error connecting to remote JT " +
remoteJTHost + ":" + remoteJTPort +
" will wait " + backoff + " msec ", e);
try {
Thread.sleep(backoff);
} catch (InterruptedException ie) {
throw new IOException(ie);
}
synchronized (RemoteJTProxy.this) {
reinitClientUnprotected();
}
}
}
}
LOG.error("Too many errors " + errorCount +
" in connecting to remote JT " +
remoteJTHost + ":" + remoteJTPort, lastException);
throw lastException;
}
}
/**
* Check if the RPC client to the remote job tracker is ready, and wait if
* not.
* @throws IOException
*/
private JobSubmissionProtocol checkClient() throws IOException {
synchronized (this) {
while (client == null) {
try {
if (remoteJTStatus == RemoteJTStatus.FAILURE) {
throw new IOException("Remote Job Tracker is not available");
}
this.wait(1000);
} catch (InterruptedException e) {
throw new IOException(e);
}
}
return client;
}
}
/**
* Check job configuration if state restoring is enabled
* @param conf configuration of job
* @return true iff enabled
*/
public static boolean isStateRestoringEnabled(JobConf conf) {
return isJTRestartingEnabled(conf)
&& conf.getBoolean(REMOTE_JT_STATE_RESTORING_CONF,
REMOTE_JT_STATE_RESTORING_DEFAULT);
}
/**
* Check job configuration if remote JT restarting is enabled
* @param conf configuration of job
* @return true iff enabled
*/
public static boolean isJTRestartingEnabled(JobConf conf) {
return (0 < conf.getInt(CoronaJobTracker.MAX_JT_FAILURES_CONF,
CoronaJobTracker.MAX_JT_FAILURES_DEFAULT));
}
}