package automately.core.services.container;
import automately.core.data.User;
import automately.core.file.nio.UserFilePath;
import automately.core.file.nio.UserFileSystem;
import automately.core.services.core.AutomatelyService;
import automately.core.util.file.FileUtil;
import com.hazelcast.core.IMap;
import com.spotify.docker.client.DefaultDockerClient;
import com.spotify.docker.client.DockerClient;
import com.spotify.docker.client.LogStream;
import com.spotify.docker.client.exceptions.DockerCertificateException;
import com.spotify.docker.client.exceptions.DockerException;
import com.spotify.docker.client.messages.*;
import io.jsync.app.core.Cluster;
import io.jsync.app.core.Logger;
import io.jsync.impl.Windows;
import io.jsync.json.JsonObject;
import org.apache.commons.lang.NullArgumentException;
import org.apache.commons.lang3.ArrayUtils;
import java.io.IOException;
import java.net.InterfaceAddress;
import java.net.NetworkInterface;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import static automately.core.file.VirtualFileSystem.getUserFileSystem;
import static automately.core.util.file.FileUtil.purgeDirectory;
/**
* The ContainerService is an Automately Service and an API that can be used to
* utilize docker containers within the host machine. It is designed to interact with
* docker on the local machine. This will change in the future.
*/
public class ContainerService extends AutomatelyService {
public static boolean BIND_RANDOM = true;
private static String BROADCAST_ADDRESS = "0.0.0.0";
public static String DEFAULT_IMAGE = "automately/automately-docker";
private static DockerClient docker;
private static Logger logger;
private static IMap<String, JsonObject> userContainers = null;
private static Cluster cluster = null;
private static Set<SecureContainer> localContainers = new LinkedHashSet<>();
private static void checkInitialized() {
if (!initialized()) {
throw new RuntimeException("The default DockerClient has not been initialized.");
}
}
@Override
public void start(Cluster owner) {
cluster = owner;
logger = owner.logger();
try {
String broadcastAddress;
JsonObject containerConfig = coreConfig().getObject("container", new JsonObject());
if (!containerConfig.containsField("broadcast_address")) {
logger.info("Creating default configuration...");
containerConfig.putString("broadcast_address", "0.0.0.0");
containerConfig.putBoolean("broadcast_all", false);
coreConfig().putObject("container", containerConfig);
cluster.config().save();
}
if(!containerConfig.getBoolean("enabled", true)){
logger.info("Container support has been disabled...");
return;
}
if (Windows.isWindows()) {
// This is the default docker location
docker = DefaultDockerClient.fromEnv().build();
} else {
// This is the default docker location
docker = DefaultDockerClient.builder()
.uri(URI.create("unix:///var/run/docker.sock")).build();
}
if (!docker.ping().equals("OK")) {
logger.fatal("Could not create the default DockerClient. (Ping Failed)");
}
broadcastAddress = containerConfig.getString("broadcast_address", "0.0.0.0");
// Attempt to retrieve an actual address
if (broadcastAddress.equals("0.0.0.0") &&
!containerConfig.getBoolean("broadcast_all", false)) {
// There should be a default interface defined
logger.info("Searching for the default \"broadcast\" interface and address...");
i:
for (NetworkInterface iface : Collections.list(NetworkInterface.getNetworkInterfaces())) {
if (!iface.isLoopback() && !iface.getDisplayName().matches("^docker\\d+$")) {
for (InterfaceAddress ifaceaddr : iface.getInterfaceAddresses()) {
if (ifaceaddr.getBroadcast() != null) {
broadcastAddress = ifaceaddr.getAddress().getHostAddress();
logger.info("Found possible default \"broadcast\" address: " + broadcastAddress + " (can be changed by updated \"broadcast_address\")");
containerConfig.putString("broadcast_address", broadcastAddress);
coreConfig().putObject("container", containerConfig);
cluster.config().save();
break i;
}
}
}
}
}
// The broadcast address is used to bind ports
// exposed on the container. This address
// Should be accessible via other containers
BROADCAST_ADDRESS = broadcastAddress;
userContainers = cluster().data().persistentMap(ContainerService.class.getCanonicalName() + ".user.containers");
// We can only run this when the main node starts up
if(!cluster().manager().clientMode()){
logger.info("Cleaning up old containers...");
for (String containerId : userContainers.keySet()) {
JsonObject containerData = userContainers.get(containerId);
if (containerData.getString("broadcast_address", "").equals(BROADCAST_ADDRESS)) {
// We know this container belongs to
logger.info("Killing \"" + containerId + "\"...");
try {
docker.killContainer(containerId);
docker.removeContainer(containerId);
} catch (Exception ignored) {
// We want to catch this just in case the container doesn't exist
}
userContainers.remove(containerId);
}
}
}
} catch (IOException | DockerCertificateException | DockerException | InterruptedException e) {
e.printStackTrace();
docker = null;
logger.fatal("Could not create the default DockerClient: " + e.getMessage());
}
}
@Override
public void stop() {
try {
if (initialized()) {
for (SecureContainer container : localContainers) {
try {
if (!container.killed() || container.running()) {
container.kill();
}
} catch (Exception ignored) {
}
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
docker = null;
}
}
@Override
public String name() {
return getClass().getCanonicalName();
}
public static DockerClient getDockerClient(){
checkInitialized();
return docker;
}
/**
* This method returns true if the ContainerService has been initialized on this cluster.
*
* @return
*/
public static boolean initialized() {
return docker != null && userContainers != null && cluster != null;
}
/**
* buildImage() is used to build and store docker images from raw data.
*
* @param imageName the name if the image you wish to build
* @param dockerFileData the data of the Dockerfile you are building
*/
public static void buildImage(String imageName, io.jsync.buffer.Buffer dockerFileData) {
checkInitialized();
Path tmpDocker = Paths.get("/tmp/" + imageName + "_tmp");
if (Files.exists(tmpDocker)) {
purgeDirectory(tmpDocker);
}
try {
Files.createDirectory(tmpDocker);
Path dockerFile = tmpDocker.resolve("Dockerfile");
Files.write(dockerFile, dockerFileData.getBytes());
// This should definitely not be fatal
if (!Files.exists(tmpDocker)) {
logger.fatal("Could not create the file " + dockerFile);
return;
}
logger.info("Building the docker image \"" + imageName + "\"...");
logger.info("The docker image \"" + docker.build(tmpDocker, imageName, msg -> {
if (msg.stream() != null && !msg.stream().isEmpty()) {
logger.info(msg.stream().trim());
}
}) + "\" has been built.");
} catch (Exception e) {
throw new RuntimeException("Build Error", e);
} finally {
purgeDirectory(tmpDocker);
}
}
public static SecureContainer create(String containerName, User user, String imageName, boolean killOnRestart, String[] exposedPorts) {
checkInitialized();
try {
containerName = containerName.trim();
String[] ports = {};
ports = ArrayUtils.addAll(ports, exposedPorts);
Map<String, List<PortBinding>> portBindings = new HashMap<>();
for (String port : ports) {
List<PortBinding> hostPorts = new ArrayList<>();
String newPort = "";
if (!BIND_RANDOM) {
newPort = port;
}
hostPorts.add(PortBinding.of(BROADCAST_ADDRESS, newPort));
portBindings.put(port, hostPorts);
}
// Allow larger memory limits
long memoryLimit = ((long) (512)) * 1000000;
HostConfig hostConfig = HostConfig.builder()
.memory(memoryLimit)
.portBindings(portBindings)
.privileged(false) // This must be always false for security reasons (may change in the future)
.networkMode("bridge")
.build();
if (imageName == null || imageName.isEmpty()) {
imageName = DEFAULT_IMAGE + ":latest"; // Using the Default Container
}
Iterator<Image> iterator = docker.listImages(DockerClient.ListImagesParam.allImages()).iterator();
boolean imageFound = false;
while (iterator.hasNext()) {
Image image = iterator.next();
if (image.repoTags() != null && image.repoTags().contains(imageName)) {
imageFound = true;
break;
}
}
if (!imageFound) {
logger.info("Attempting to pull the image \"" + imageName + "\" from the default Docker repository.");
docker.pull(imageName, p -> {
logger.info(p.toString());
});
}
ContainerConfig containerConfig = ContainerConfig.builder().image(imageName)
.hostConfig(hostConfig).exposedPorts(ports).cmd("sh", "-c", "while :; do sleep 1; done").build();
ContainerCreation creation;
creation = docker.createContainer(containerConfig);
ContainerState state = docker.inspectContainer(creation.id()).state();
while (!state.status().equals("created") &&!state.oomKilled() && !state.running() && !state.paused() && (state.error() == null || state.error().isEmpty())){
// Let's put this thread to sleep for 50ms
ContainerInfo inf = docker.inspectContainer(creation.id());
state = inf.state();
Thread.sleep(50);
if(Thread.interrupted()){
break;
}
}
if(state.oomKilled()){
throw new IOException("It looks like the state of the container is oomKilled.");
}
if(state.restarting()){
throw new IOException("It looks like the state of the container is restarting.");
}
if(state.error() != null && !state.error().isEmpty()){
throw new IOException(state.error());
}
JsonObject containerData = new JsonObject();
containerData.putString("userToken", user.token());
containerData.putString("id", creation.id());
// This is stored so we can access ports
containerData.putString("broadcast_address", BROADCAST_ADDRESS);
// TODO we need to ensure that any cleanup
// containers don't kill
containerData.putBoolean("kill_on_restart", killOnRestart);
SecureContainer container = new SecureContainer(containerName, creation.id(), user);
userContainers.put(creation.id(), containerData);
localContainers.add(container);
return container;
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("Failed to create a new SecureContainer.", e);
}
}
public static SecureContainer connect(User user, String containerName, String containerId) {
checkInitialized();
try {
for (Container container : docker.listContainers(DockerClient.ListContainersParam.allContainers())) {
if (container.id().equals(containerId)) {
containerName = containerName.trim();
SecureContainer nContainer = new SecureContainer(containerName, containerId, user);
JsonObject containerData = userContainers.get(containerId);
if (containerData == null) {
containerData = new JsonObject();
}
containerData.putString("userToken", user.token());
containerData.putString("id", containerId);
containerData.putString("host", cluster.manager().nodeId());
userContainers.put(containerId, containerData);
localContainers.add(nContainer);
return nContainer;
}
}
} catch (DockerException | InterruptedException e) {
throw new RuntimeException(e);
}
return null;
}
public static class SecureContainer {
private String containerId;
private User user;
private boolean initialized = false;
private PortBinding webPort = null;
private String defaultWebPort = "8080";
private String name;
private boolean killed = false;
private UserFileSystem fs = null;
protected SecureContainer(String name, String containerId, User user) {
if (containerId == null) {
throw new NullPointerException();
}
this.containerId = containerId;
this.user = user;
this.name = name;
this.fs = getUserFileSystem(user);
}
public String id() {
return containerId;
}
public String name() {
return this.name;
}
public PortBinding webPort() {
return webPort;
}
public void setDefaultWebPort(Number webPort) {
if (webPort == null) {
throw new NullArgumentException("webPort cannot be null");
}
if (webPort.intValue() <= 0) {
throw new IllegalArgumentException("webPort must be greater than 0");
}
defaultWebPort = String.valueOf(webPort.intValue());
}
public void uploadPath(String srcPath, String dest) throws IOException {
if (killed) {
throw new IOException("This container has been killed.");
}
if (!initialized()) {
throw new IOException("This container does not seem to be initialized.");
}
UserFilePath src = fs.getPath(srcPath);
if (!Files.exists(src) || !Files.isDirectory(src)) {
throw new IOException(srcPath + " does not exist or is not a directory!");
}
Path tmpDir = Files.createDirectories(Paths.get("fs/tmp/" + io.jsync.utils.UUID.generate()));
Files.walk(src).filter(Files::isRegularFile).forEach(path -> {
try {
String fileName = path.toString().replaceFirst(src.toString(), "");
Path localFile = tmpDir.resolve(fileName);
if(!Files.exists(localFile.getParent())){
Files.createDirectories(localFile.getParent());
}
Files.copy(Files.newInputStream(path), localFile);
} catch (IOException ignored) {
}
});
try {
try {
// Let's attempt to create a directory
// in case it's not there
exec(true, "mkdir", "-p", dest);
} catch (Exception ignored){
}
docker.copyToContainer(tmpDir, containerId, dest);
} catch (DockerException | InterruptedException e) {
throw new RuntimeException(e);
} finally {
try {
FileUtil.purgeDirectory(tmpDir);
} catch (Exception ignored){
}
}
}
public boolean running() {
try {
return docker.inspectContainer(containerId).state().running();
} catch (DockerException | InterruptedException ignored) {
}
return false;
}
public boolean paused() {
try {
return docker.inspectContainer(containerId).state().paused();
} catch (DockerException | InterruptedException ignored) {
}
return false;
}
public boolean restarting() {
try {
return docker.inspectContainer(containerId).state().restarting();
} catch (DockerException | InterruptedException ignored) {
}
return false;
}
public boolean killed() {
return killed;
}
public boolean oomKilled() {
try {
return docker.inspectContainer(containerId).state().oomKilled();
} catch (DockerException | InterruptedException ignored) {
}
return false;
}
public int exitCode() {
try {
return docker.inspectContainer(containerId).state().exitCode();
} catch (DockerException | InterruptedException ignored) {
}
return -1;
}
public String error() {
try {
return docker.inspectContainer(containerId).state().error();
} catch (DockerException | InterruptedException ignored) {
}
return "";
}
public ExecState exec(String... command) {
return exec(true, command);
}
public ExecState exec(boolean waitForFinish, String... command) {
try {
if (killed) {
throw new RuntimeException("This container has been killed.");
}
if (!initialized()) {
throw new RuntimeException("This container does not seem to be initialized.");
}
ExecCreation execCreation = docker.execCreate(containerId, command,
DockerClient.ExecCreateParam.attachStdout(),
DockerClient.ExecCreateParam.attachStderr());
String execId = execCreation.id();
if (waitForFinish) {
try {
docker.execStart(execId).readFully();
} catch (Exception ignored){
}
return docker.execInspect(execId);
} else {
docker.execStart(execId);
return docker.execInspect(execId);
}
} catch (DockerException | InterruptedException e) {
throw new RuntimeException(e);
}
}
public LogStream execWithStream(String... command) {
try {
if (killed) {
throw new RuntimeException("This container has been killed.");
}
if (!initialized()) {
throw new RuntimeException("This container does not seem to be initialized.");
}
ExecCreation execCreation = docker.execCreate(containerId, command,
DockerClient.ExecCreateParam.attachStdout(),
DockerClient.ExecCreateParam.attachStderr());
String execId = execCreation.id();
return docker.execStart(execId);
} catch (DockerException | InterruptedException e) {
throw new RuntimeException(e);
}
}
public boolean initialized() {
return initialized;
}
public void initialize() {
try {
if (killed) {
throw new RuntimeException("This container has been killed.");
}
if (initialized()) {
throw new RuntimeException("Cannot call initialize() when the container is already initialized.");
}
if (!running()) {
docker.startContainer(containerId);
while (!running() && error().isEmpty() && !paused()) {
Thread.sleep(15);
}
}
initialized = running();
for (Map.Entry<String, List<PortBinding>> portBindings : docker.inspectContainer(containerId).networkSettings().ports().entrySet()) {
String s = portBindings.getKey();
if(s.matches("^" + defaultWebPort + "?(/tcp)$")){
Iterator<PortBinding> it = portBindings.getValue().iterator();
if(it.hasNext()){
webPort = it.next();
break;
}
}
}
if(webPort == null){
throw new RuntimeException("Could not retrieve binding for the default web port.");
}
JsonObject containerData = userContainers.get(containerId);
if (containerData == null) {
containerData = new JsonObject();
}
containerData.putString("userToken", user.token());
containerData.putString("id", containerId);
containerData.putString("host", cluster.manager().nodeId());
if (webPort != null) {
containerData.putString("web_port", webPort.hostPort());
containerData.putString("web_host", webPort.hostIp());
}
logger.info("Storing container data: " + containerData.encode());
userContainers.put(containerId, containerData);
} catch (DockerException | InterruptedException e) {
e.printStackTrace();
throw new RuntimeException("There was an issue while attempting to initialize this container.", e);
}
}
public void kill() {
try {
docker.killContainer(containerId);
docker.removeContainer(containerId);
} catch (Exception ignored) {
} finally {
initialized = false;
killed = true;
}
}
}
}