/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.addthis.hydra.task.run; import java.io.File; import java.io.IOException; import java.util.HashSet; import java.util.Set; import java.util.concurrent.CompletableFuture; import com.addthis.basis.util.LessBytes; import com.addthis.basis.util.LessFiles; import com.addthis.codec.jackson.CodecJackson; import com.addthis.codec.jackson.Jackson; import com.addthis.muxy.MuxFileDirectoryCache; import com.fasterxml.jackson.core.JsonProcessingException; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import com.typesafe.config.ConfigParseOptions; import com.typesafe.config.ConfigResolveOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class TaskRunner { private static final Logger log = LoggerFactory.getLogger(TaskRunner.class); public static void main(String[] args) throws Exception { if (args.length < 1) { System.out.println("usage: task <config> <nodes> <node> [jobid] [threads]"); return; } String fileName = args[0]; String configString = loadStringFromFile(fileName); runTask(configString); } static void runTask(String configString) throws Exception { final TaskRunnable task = makeTask(configString); // before starting, we need to make sure that the task will be closed CompletableFuture<AutoCloseable> startedTask = new CompletableFuture<>(); boolean hookAdded; try { Runtime.getRuntime().addShutdownHook(new Thread(new TaskShutdown(startedTask), "Task Shutdown Hook")); hookAdded = true; } catch (IllegalStateException ignored) { log.info("Canceling task because JVM is shutting down."); hookAdded = false; } if (hookAdded) { try { task.start(); startedTask.complete(task); } catch (Throwable t) { startedTask.complete(() -> log.debug("skipping task.close because it failed to start normally")); throw t; } } } private static class TaskShutdown implements Runnable { private final CompletableFuture<AutoCloseable> task; TaskShutdown(CompletableFuture<AutoCloseable> task) { this.task = task; } @Override public void run() { try { task.join().close(); // critical to get any file meta data written before process exits CompletableFuture.runAsync(MuxFileDirectoryCache::waitForWriteClosure).join(); } catch (Exception ex) { log.error("unrecoverable error shutting down task. immediately halting jvm", ex); Runtime.getRuntime().halt(1); } } } public static TaskRunnable makeTask(String configString) throws JsonProcessingException, IOException { return makeTask(configString, Jackson.defaultCodec()); } /** * Creates a TaskRunnable using CodecConfig and a little custom handling. At the root * level object, if there is a field named "global", then that sub tree is hoisted * up to override system properties (removing it from the root of the job config). * Either way, the job config will also then be resolved against config defaults/ system * properties for the purposes of variable substitution. This will not merge them entirely * though and so the job config will be otherwise unaffected. */ public static TaskRunnable makeTask(String configString, CodecJackson defaultCodec) throws JsonProcessingException, IOException { String subbedConfigString = subAt(configString); Config config = ConfigFactory.parseString(subbedConfigString, ConfigParseOptions.defaults().setOriginDescription("job.conf")); Config defaultGlobalDefaults = defaultCodec.getGlobalDefaults(); Config jobConfig = config; CodecJackson codec; if (config.root().containsKey("global")) { jobConfig = config.root().withoutKey("global").toConfig() .resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true)); Config globalDefaults = config.getConfig("global") .withFallback(defaultGlobalDefaults) .resolve(); jobConfig = jobConfig.resolveWith(globalDefaults); codec = defaultCodec.withConfig(globalDefaults); } else { jobConfig = jobConfig.resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true)) .resolveWith(defaultGlobalDefaults); codec = defaultCodec; } return codec.decodeObject(TaskRunnable.class, jobConfig); } static String loadStringFromFile(String fileName) throws IOException { return LessBytes.toString(LessFiles.read(new File(fileName))); } private static final Set<TaskStringReplacement> replaceOperators = new HashSet<>(); static { replaceOperators.add(new TaskReplacementFile()); replaceOperators.add(new TaskReplacementZoo()); } /** replace references with file contents */ static String subAt(String json) throws IOException { boolean transformed; String output = json; do { String begin = output; for(TaskStringReplacement replacement : replaceOperators) { output = replacement.replace(output); } transformed = (begin != output); } while(transformed); return output; } }