/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.addthis.hydra.job.spawn;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import com.addthis.basis.net.HttpUtil;
import com.addthis.basis.net.http.HttpResponse;
import com.addthis.basis.util.Parameter;
import com.addthis.codec.json.CodecJSON;
import com.addthis.hydra.job.Job;
import com.addthis.hydra.job.alias.AliasManager;
import com.addthis.hydra.util.EmailUtil;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import com.google.common.base.Throwables;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.yammer.metrics.Metrics;
import com.yammer.metrics.core.Gauge;
import org.joda.time.DateTimeConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** Supports {@code http://} and {@code kick://} callbacks. */
public class JobOnFinishStateHandlerImpl implements JobOnFinishStateHandler {
private static final Logger log = LoggerFactory.getLogger(JobOnFinishStateHandlerImpl.class);
private static final int backgroundThreads = Parameter.intValue("spawn.background.threads", 4);
private static final int backgroundQueueSize = Parameter.intValue("spawn.background.queuesize", 1_000);
private static final int backgroundEmailMinute = Parameter.intValue("spawn.background.notification.interval.minutes", 60);
private static final int backgroundHttpTimeoutMs = Parameter.intValue("spawn.background.timeout", 300_000);
private static final String backgroundEmailAddress = Parameter.value("spawn.background.notification.address");
private static final String clusterName = Parameter.value("cluster.name", "localhost");
private final Spawn spawn;
private final BlockingQueue<Runnable> backgroundTaskQueue;
private final ExecutorService backgroundService;
private final AtomicLong emailLastFired;
@SuppressWarnings({"FieldCanBeLocal", "unused"}) private final Gauge<Integer> backgroundQueueGauge;
public JobOnFinishStateHandlerImpl(Spawn spawn) {
this.spawn = spawn;
backgroundTaskQueue = new LinkedBlockingQueue<>(backgroundQueueSize);
backgroundService = new ThreadPoolExecutor(
backgroundThreads, backgroundThreads, 0L, TimeUnit.MILLISECONDS, backgroundTaskQueue,
new ThreadFactoryBuilder().setDaemon(true).build());
emailLastFired = new AtomicLong();
backgroundQueueGauge = Metrics.newGauge(
Spawn.class, "backgroundExecutorQueue", new Gauge<Integer>() {
@Override public Integer value() {
return backgroundTaskQueue.size();
}
});
}
@Override
public void handle(Job job, JobOnFinishState state) {
switch (state) {
case OnComplete:
doOnState(job, job.getOnCompleteURL(), job.getOnCompleteTimeout(), state);
break;
case OnError:
doOnState(job, job.getOnErrorURL(), job.getOnErrorTimeout(), state);
break;
}
}
private void doOnState(Job job, String url, int timeoutSecs, JobOnFinishState state) {
if (Strings.isNullOrEmpty(url)) {
return;
}
if (url.startsWith("http://")) {
int timeoutMs = timeoutSecs > 0 ? (timeoutSecs * 1000) : backgroundHttpTimeoutMs;
try {
Runnable task = createBackgroundTask(job, state, url, timeoutMs);
backgroundService.submit(task);
} catch (Exception e) {
log.error("Error making background call: {}", e.getMessage(), e);
emailNotification(job.getId(), state, Throwables.getStackTraceAsString(e));
}
} else if (url.startsWith("kick://")) {
AliasManager aliasManager = spawn.getAliasManager();
for (String kick : Splitter.on(",").omitEmptyStrings().trimResults().split(url.substring(7))) {
List<String> jobIds = aliasManager.aliasToJobs(kick);
if (jobIds != null) {
for (String jobId : jobIds) {
safeStartJob(jobId.trim());
}
} else {
safeStartJob(kick);
}
}
} else {
log.warn("invalid {} url: {} for job {}", state, url, job.getId());
}
}
private Runnable createBackgroundTask(
Job job,
final JobOnFinishState state,
final String url,
final int timeoutMs) throws Exception {
final String jobId = job.getId();
final byte[] jobJson = CodecJSON.encodeString(job).getBytes();
return () -> {
try {
HttpResponse response = HttpUtil.httpPost(url, "javascript/text", jobJson, timeoutMs);
if (response.getStatus() >= 400) {
String err = "HTTP POST to " + url + " in background task \"" + jobId + " "
+ state + "\" returned " + response.getStatus() + " " + response.getReason();
log.error(err);
emailNotification(jobId, state, err);
}
} catch (IOException ex) {
log.error("IOException when attempting to contact \"{}\" in background task \"{} {}\"",
url, jobId, state, ex);
emailNotification(jobId, state, Throwables.getStackTraceAsString(ex));
}
};
}
private void emailNotification(String jobId, JobOnFinishState state, String body) {
if (backgroundEmailAddress != null) {
long currentTime = System.currentTimeMillis();
long lastTime = emailLastFired.get();
long elapse = currentTime - lastTime;
if (elapse > (long) backgroundEmailMinute * DateTimeConstants.MILLIS_PER_MINUTE) {
EmailUtil.email(backgroundEmailAddress, emailSubject(jobId, state), body);
emailLastFired.set(currentTime);
}
}
}
private String emailSubject(String jobId, JobOnFinishState state) {
return "Background operation failed -" + clusterName + "- {" + jobId + " " + state.name() + "}";
}
private void safeStartJob(String jobId) {
try {
spawn.startJob(jobId, 0);
} catch (Exception ex) {
log.warn("[safe.start] {} failed due to {}", jobId, ex.getMessage(), ex);
}
}
@Override public void close() {
MoreExecutors.shutdownAndAwaitTermination(backgroundService, 120, TimeUnit.SECONDS);
}
}