package automately.core.services.sdk;
import automately.core.data.Job;
import automately.core.data.User;
import automately.core.data.UserData;
import automately.core.file.VirtualFileSystem;
import automately.core.file.nio.UserFilePath;
import automately.core.file.nio.UserFileSystem;
import automately.core.services.core.AutomatelyService;
import automately.core.services.job.JobServer;
import automately.core.services.ssh.SSHCommandFactory;
import io.jsync.AsyncResult;
import io.jsync.Handler;
import io.jsync.app.core.Cluster;
import io.jsync.app.core.Logger;
import io.jsync.eventbus.Message;
import io.jsync.json.JsonArray;
import io.jsync.json.JsonObject;
import org.apache.sshd.common.channel.ChannelOutputStream;
import org.apache.sshd.server.Command;
import org.apache.sshd.server.Environment;
import org.apache.sshd.server.ExitCallback;
import org.apache.sshd.server.SessionAware;
import org.apache.sshd.server.session.ServerSession;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.InitCommand;
import org.eclipse.jgit.api.ResetCommand;
import org.eclipse.jgit.errors.RepositoryNotFoundException;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.RepositoryCache;
import org.eclipse.jgit.transport.ReceivePack;
import org.eclipse.jgit.transport.UploadPack;
import org.eclipse.jgit.util.FS;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileTime;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Consumer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* The SdkDeploymentService as service that handles the deployment of
* user SDK applications. It will attempt to intelligently scan for any changes
* via the git deployment repositories and on the changes it will attempt to read
* package.json and if package.json is readable then it will continue with the deployment.
*
* This adds a very powerful and unique feature to Automately.
*/
public class SdkDeploymentService extends AutomatelyService {
public static String DEFAULT_DEPLOYMENT_DIR = "fs/git/deploy/";
private static Cluster cluster;
private static JobServer jobServer;
private static Logger logger;
private static ExecutorService deploymentService = Executors.newCachedThreadPool();
private static Set<Consumer<Map.Entry<User, String>>> errorHandlers = new HashSet<>();
public static void addErrorHandler(Consumer<Map.Entry<User, String>> handler){
if(handler != null){
errorHandlers.add(handler);
}
}
@Override
public void start(Cluster cluster) {
SdkDeploymentService.cluster = cluster;
logger = cluster.logger();
logger.info("Starting the SDK Deployment Service...");
JsonObject coreConfig = coreConfig();
JsonObject deploymentConfig = coreConfig.getObject("deployment_service", new JsonObject());
String deploymentDir = deploymentConfig.getString("dir", DEFAULT_DEPLOYMENT_DIR);
boolean enabled = deploymentConfig.getBoolean("enabled", true);
deploymentConfig.putString("dir", deploymentDir);
deploymentConfig.putBoolean("enabled", enabled);
coreConfig().putObject("deployment_service", deploymentConfig);
cluster.config().save();
if(enabled){
try {
jobServer = (JobServer) cluster().getService(JobServer.class.getCanonicalName());
} catch (Exception e) {
throw new RuntimeException(e);
}
Path deploymentData = Paths.get(deploymentDir);
if(!Files.exists(deploymentData)){
try {
Files.createDirectories(deploymentData);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
SSHCommandFactory.addFactory("git", s -> new GitCommand(DEFAULT_DEPLOYMENT_DIR, s));
}
}
@Override
public void stop() {
logger.info("Stopping the SDK Deployment Service...");
}
@Override
public String name() {
return getClass().getCanonicalName();
}
/**
* @author <a href="mailto:dev@mina.apache.org">Apache MINA SSHD Project</a>
*/
public static class GitCommand implements Command, Runnable, SessionAware {
private static final int CHAR = 1;
private static final int DELIMITER = 2;
private static final int STARTQUOTE = 4;
private static final int ENDQUOTE = 8;
private String rootDir;
private String command;
private InputStream in;
private OutputStream out;
private OutputStream err;
private ExitCallback callback;
private User user;
public GitCommand(String rootDir, String command) {
this.rootDir = rootDir;
this.command = command;
}
@Override
public void setInputStream(InputStream in) {
this.in = in;
}
@Override
public void setOutputStream(OutputStream out) {
this.out = out;
if (out instanceof ChannelOutputStream) {
((ChannelOutputStream) out).setNoDelay(true);
}
}
@Override
public void setErrorStream(OutputStream err) {
this.err = err;
if (err instanceof ChannelOutputStream) {
((ChannelOutputStream) err).setNoDelay(true);
}
}
@Override
public void setExitCallback(ExitCallback callback) {
this.callback = callback;
}
@Override
public void start(Environment env) throws IOException {
Thread thread = new Thread(this);
thread.setDaemon(true);
thread.start();
}
private void handlerError(User user, String error){
for (Consumer<Map.Entry<User, String>> errorHandler : errorHandlers) {
errorHandler.accept(new AbstractMap.SimpleEntry<>(user, error));
}
}
@Override
public void run() {
try {
List<String> strs = parseDelimitedString(command, " ", true);
String[] args = strs.toArray(new String[strs.size()]);
for (int i = 0; i < args.length; i++) {
if (args[i].startsWith("'") && args[i].endsWith("'")) {
args[i] = args[i].substring(1, args[i].length() - 1);
}
if (args[i].startsWith("\"") && args[i].endsWith("\"")) {
args[i] = args[i].substring(1, args[i].length() - 1);
}
}
if (args.length != 2) {
throw new IllegalArgumentException("Invalid git command line: " + command);
}
String gitPath = args[1];
if(!gitPath.endsWith(".git")){
throw new RepositoryNotFoundException(gitPath);
}
File srcGitDir = new File(rootDir, gitPath);
RepositoryCache.FileKey key = RepositoryCache.FileKey.lenient(srcGitDir, FS.DETECTED);
Repository db;
try {
db = key.open(true /* must exist */);
} catch (RepositoryNotFoundException e){
// Let's attempt to automatically create this repository
InitCommand command = new InitCommand();
command.setDirectory(srcGitDir);
db = command.call().getRepository();
}
if(db == null){
throw new RepositoryNotFoundException(gitPath);
}
if ("git-upload-pack".equals(args[0])) {
new UploadPack(db).upload(in, out, err);
} else if ("git-receive-pack".equals(args[0])) {
ReceivePack receivePack = new ReceivePack(db);
Repository finalDb = db;
receivePack.setPostReceiveHook((receivePack1, commands) -> {
String branch = commands.iterator().next().getRefName();
branch = branch.substring(branch.lastIndexOf("/") + 1);
// Is the user pushing to the deploy branch??
boolean shouldDeploy = branch.equals("deploy") || branch.equals("debug");
// Let's go ahead and attempt to reset the repo to the HEAD
try {
Git git = new Git(finalDb);
if(shouldDeploy){
git.checkout().setName(branch).call();
}
ObjectId prevHead = finalDb.resolve(Constants.HEAD);
if(prevHead != null){
git.reset().setMode(ResetCommand.ResetType.HARD).call();
}
if(shouldDeploy){
String finalBranch = branch;
deploymentService.submit(() -> {
try {
// We need to check for package.json and check for some parameters
UserFileSystem userFs = VirtualFileSystem.getUserFileSystem(user);
UserFilePath userPath = userFs.getPath(gitPath.substring(0, gitPath.lastIndexOf(".git")));
Path localPath = Paths.get(srcGitDir.toURI()).toAbsolutePath();
JsonObject projectPackage = null;
boolean skipDeployment = false;
Set<Object> shouldIgnore = new HashSet<>();
if(Files.exists(localPath.resolve("package.json"))){
// If package.json exists then we need to handle some things a certain way.
logger.info(localPath.resolve("package.json") + " was found. Reading project configuration...");
try {
projectPackage = new JsonObject(new String(Files.readAllBytes(localPath.resolve("package.json"))));
// Since the project is in debug mode let's go ahead and deploy to _debug
if(projectPackage.getBoolean("debug", false) || finalBranch.equals("debug")){
projectPackage.putBoolean("debug", true); // Let's make sure we set the debug flag to true
userPath = userFs.getPath(gitPath.substring(0, gitPath.lastIndexOf(".git")) + "_debug");
}
// Let's check and see if we want to skip the deployment process
skipDeployment = projectPackage.getBoolean("skip_deployment", false);
Files.write(localPath.resolve("package.json"), projectPackage.encode().getBytes(), StandardOpenOption.CREATE);
Collections.addAll(shouldIgnore, projectPackage.getArray("exclude", new JsonArray()).toArray());
} catch (Exception e){
handlerError(user, e.getLocalizedMessage());
return;
}
}
logger.info("Deploying " + localPath + " to " + userPath + " for the user " + user.username + ".");
if(!skipDeployment){
logger.info("Copying " + localPath + " to " + userPath + " for the user " + user.username + "...");
if(!Files.exists(userPath)){
Files.createDirectories(userPath);
}
UserFilePath finalUserPath = userPath;
Files.walkFileTree(localPath, new FileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
String newPath = finalUserPath.resolve(localPath.relativize(dir).normalize().toString()).normalize().toString().replaceAll(Pattern.quote(File.separator), "/");
Path userPath = userFs.getPath(newPath);
if(IgnoredFiles.check(newPath, shouldIgnore)){
return FileVisitResult.SKIP_SUBTREE;
}
if(!Files.exists(userPath)){
Files.createDirectories(userPath);
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
String newPath = finalUserPath.resolve(localPath.relativize(file).toString()).normalize().toString().replaceAll(Pattern.quote(File.separator), "/");
Path userPath = userFs.getPath(newPath);
if(IgnoredFiles.check(newPath, shouldIgnore)){
return FileVisitResult.CONTINUE;
}
if(Files.exists(userPath)){
FileTime lastModifiedTime = Files.getLastModifiedTime(file);
FileTime lastModifiedTime2 = Files.getLastModifiedTime(userPath);
if(lastModifiedTime.compareTo(lastModifiedTime2) > 0){
Files.copy(file, userPath, StandardCopyOption.REPLACE_EXISTING);
}
} else {
Files.copy(file, userPath, StandardCopyOption.REPLACE_EXISTING);
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
return FileVisitResult.CONTINUE;
}
});
if(projectPackage != null){
JsonObject finalProjectPackage = projectPackage;
Runnable projectJobDeploy = () -> {
logger.info("Attempting to start " + finalProjectPackage.getString("name") + " for the user " + user.username + "...");
if (!finalProjectPackage.containsField("server")) {
if (!finalProjectPackage.containsField("job_config")) {
handlerError(user, "Could not handle deployment for " + finalUserPath.toPathAlias() + " because no server or job_config was found.");
return;
}
Job job = new Job();
job.config = finalProjectPackage.getObject("job_config", new JsonObject());
job.userToken = user.token();
job = jobServer.submit(job);
logger.info("Started the job " + job.token() + " for the user " + user.username + "...");
return;
}
Job job = new Job();
job.config = finalProjectPackage.getObject("job_config", new JsonObject());
JsonObject scriptConfig = new JsonObject();
scriptConfig.putString("scriptData", "require(\"" + finalUserPath.toPathAlias() +
finalProjectPackage.getString("server", "server.js") + "\").start();");
scriptConfig.putString("scriptPath", finalUserPath.toPathAlias());
job.config.putObject("script", scriptConfig);
job.userToken = user.token();
job = jobServer.submit(job);
logger.info("Started the job " + job.token() + " for the user " + user.username + "...");
};
String serviceName = "_sdk_service_" + projectPackage.getString("name");
if(projectPackage.getBoolean("debug", false)){
serviceName += "_debug";
}
Job currentService = JobServer.getService(user, serviceName);
if(currentService != null){
if(currentService.status.equals("running")){
// Let's go ahead and attempt to send a message to it
// This is going to allow us to intercept the deployment process
cluster.eventBus().sendWithTimeout("job.server." + currentService.token() + ".deployment", "restart", 5000, (Handler<AsyncResult<Message<Boolean>>>) event -> {
if(event.succeeded()){
if(event.result().body()){
// Let's continue with the normal deployment
projectJobDeploy.run();
} else {
// We don't need to continue
logger.info("Skipping job deployment for " + finalProjectPackage.getString("name") + " for the user " + user.username + "...");
}
} else {
logger.info("deployment timeout");
// Let's continue with the normal deployment
// Since the job did not respond
projectJobDeploy.run();
}
});
return;
}
}
projectJobDeploy.run();
}
}
} catch (Exception e){
handlerError(user, "Could not handle deployment for " + gitPath + ". " + e.getLocalizedMessage());
}
});
}
} catch (Exception e) {
handlerError(user, "Could not handle deployment for " + gitPath + ". " + e.getLocalizedMessage());
throw new RuntimeException(e);
}
});
(receivePack).receive(this.in, this.out, this.err);
} else {
throw new IllegalArgumentException("Unknown git command: " + command);
}
} catch (Throwable t) {
t.printStackTrace();
}
if (callback != null) {
callback.onExit(0);
}
}
@Override
public void destroy() {
//To change body of implemented methods use File | Settings | File Templates.
}
/**
* Parses delimited string and returns an array containing the tokens. This
* parser obeys quotes, so the delimiter character will be ignored if it is
* inside of a quote. This method assumes that the quote character is not
* included in the set of delimiter characters.
*
* @param value the delimited string to parse.
* @param delim the characters delimiting the tokens.
* @param trim {@code true} if the strings are trimmed before being added to the list
* @return a list of string or an empty list if there are none.
*/
private static List<String> parseDelimitedString(String value, String delim, boolean trim) {
if (value == null) {
value = "";
}
List<String> list = new ArrayList<>();
StringBuilder sb = new StringBuilder();
int expecting = CHAR | DELIMITER | STARTQUOTE;
boolean isEscaped = false;
for (int i = 0; i < value.length(); i++) {
char c = value.charAt(i);
boolean isDelimiter = delim.indexOf(c) >= 0;
if (!isEscaped && c == '\\') {
isEscaped = true;
continue;
}
if (isEscaped) {
sb.append(c);
} else if (isDelimiter && (expecting & DELIMITER) > 0) {
if (trim) {
list.add(sb.toString().trim());
} else {
list.add(sb.toString());
}
sb.delete(0, sb.length());
expecting = CHAR | DELIMITER | STARTQUOTE;
} else if ((c == '"') && (expecting & STARTQUOTE) > 0) {
sb.append(c);
expecting = CHAR | ENDQUOTE;
} else if ((c == '"') && (expecting & ENDQUOTE) > 0) {
sb.append(c);
expecting = CHAR | STARTQUOTE | DELIMITER;
} else if ((expecting & CHAR) > 0) {
sb.append(c);
} else {
throw new IllegalArgumentException("Invalid delimited string: " + value);
}
isEscaped = false;
}
if (sb.length() > 0) {
if (trim) {
list.add(sb.toString().trim());
} else {
list.add(sb.toString());
}
}
return list;
}
@Override
public void setSession(ServerSession serverSession) {
user = UserData.getUserByUsername(serverSession.getUsername());
rootDir = rootDir + serverSession.getUsername();
}
}
public static class IgnoredFiles {
private static Set<String> DEFAULT_IGNORED_FILES;
static {
/*
By default we ignore certain files such as git and IntelliJ projects
*/
Set<String> ignoredFiles = new HashSet<>();
ignoredFiles.add("^.*(/|\\\\)\\.git((/|\\\\)?.*)?$");
ignoredFiles.add("^.*(/|\\\\)\\.idea((/|\\\\)?.*)?$");
ignoredFiles.add("^.*\\.gitignore$");
ignoredFiles.add("^.*\\.iml$");
ignoredFiles.add("^.*\\.DS_Store$");
DEFAULT_IGNORED_FILES = ignoredFiles;
}
public static Set<String> getDefaultRules(){
return DEFAULT_IGNORED_FILES;
}
/**
* Checks a file location if it should be ignored or not.
*
* @param file the file location to check
* @return returns true if the file should be ignored
*/
public static boolean check(String file){
return check(file, null, null);
}
public static boolean check(String file, Set<Object> ignoredFiles){
return check(file, ignoredFiles, null);
}
/**
* Checks a file location if it should be ignored or not.
*
* @param file the file location to check
* @param ignoredFiles files to ignore
* @param allowedFiles files to allow
* @return returns true if the file should be ignored
*/
public static boolean check(String file, Set<Object> ignoredFiles, Set<Object> allowedFiles){
Set<Object> ignored = new HashSet<>();
if(ignoredFiles != null){
ignored.addAll(ignoredFiles);
}
ignored.addAll(DEFAULT_IGNORED_FILES);
for(Object rule : ignored){
if(rule instanceof String){
if(validateRule((String) rule, file)){
return true;
}
}
}
if(allowedFiles != null){
for(Object rule : allowedFiles){
if(rule instanceof String){
if(validateRule((String) rule, file)){
// Return false if it is allowed
return false;
}
}
}
}
return false;
}
private static boolean validateRule(String rule, String data){
try {
Pattern p = Pattern.compile(rule);
Matcher m = p.matcher(data);
return m.find() || m.matches();
} catch (Exception ignored){
ignored.printStackTrace();
}
return data.contains(rule) || data.equals(rule);
}
}
}