SaltRunner.java

package org.zstack.core.salt;

import org.springframework.beans.factory.annotation.Autowire;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Configurable;
import org.zstack.core.Platform;
import org.zstack.core.config.GlobalConfigFacade;
import org.zstack.core.errorcode.ErrorFacade;
import org.zstack.core.job.JobQueueFacade;
import org.zstack.core.thread.ChainTask;
import org.zstack.core.thread.SyncTaskChain;
import org.zstack.core.thread.ThreadFacade;
import org.zstack.header.core.Completion;
import org.zstack.header.core.ReturnValueCompletion;
import org.zstack.header.errorcode.ErrorCode;
import org.zstack.header.exception.CloudRuntimeException;
import org.zstack.utils.ShellResult;
import org.zstack.utils.ShellUtils;
import org.zstack.utils.Utils;
import org.zstack.utils.gson.JSONObjectUtil;
import org.zstack.utils.logging.CLogger;
import org.zstack.utils.network.NetworkUtils;
import org.zstack.utils.ssh.Ssh;
import org.zstack.utils.ssh.SshResult;

import static org.zstack.core.Platform.operr;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;

import static org.zstack.utils.DebugUtils.Assert;

/**
 */
@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE)
public class SaltRunner {
    private static final CLogger logger = Utils.getLogger(SaltRunner.class);
    private static Map<String, String> minionIds = new HashMap<String, String>();

    @Autowired
    private ThreadFacade thdf;
    @Autowired
    private GlobalConfigFacade gcf;
    @Autowired
    private JobQueueFacade jobf;
    @Autowired
    private ErrorFacade errf;
    @Autowired
    private SaltFacade saltf;

    private String saltBootstrapScriptPath;
    private String saltMinionConfPath;

    private String minionId;
    private String targetIp;
    private String privateKey;
    private String password;
    private String username;
    private int sshPort = 22;
    private boolean cleanupMasterPubKey = true;
    private String stateName;
    private boolean fullDeploy;
    private int agentPort;
    private String moduleName;


    public SaltRunner(String saltBootstrapScriptPath, String saltMinionConfPath) {
        this.saltBootstrapScriptPath = saltBootstrapScriptPath;
        this.saltMinionConfPath = saltMinionConfPath;
    }

    public String getModuleName() {
        return moduleName;
    }

    public SaltRunner setModuleName(String moduleName) {
        this.moduleName = moduleName;
        return this;
    }

    public String getMinionId() {
        return minionId;
    }

    public SaltRunner setMinionId(String minionId) {
        this.minionId = minionId;
        return this;
    }

    public String getTargetIp() {
        return targetIp;
    }

    public SaltRunner setTargetIp(String targetIp) {
        this.targetIp = targetIp;
        return this;
    }

    public String getPrivateKey() {
        return privateKey;
    }

    public SaltRunner setPrivateKey(String privateKey) {
        this.privateKey = privateKey;
        return this;
    }

    public String getPassword() {
        return password;
    }

    public SaltRunner setPassword(String password) {
        this.password = password;
        return this;
    }

    public String getUsername() {
        return username;
    }

    public SaltRunner setUsername(String username) {
        this.username = username;
        return this;
    }

    public boolean isCleanupMasterPubKey() {
        return cleanupMasterPubKey;
    }

    public SaltRunner setCleanupMasterPubKey(boolean cleanupMasterPubKey) {
        this.cleanupMasterPubKey = cleanupMasterPubKey;
        return this;
    }

    private void generateMinionId() {
        minionId = minionIds.get(targetIp);
        if (minionId != null) {
            return;
        }

        SshResult machineIdRes = new Ssh().setHostname(targetIp).setUsername(username).setPrivateKey(privateKey)
                .setPassword(password).setPort(sshPort).command("cat /sys/class/dmi/id/product_uuid").runAndClose();
        machineIdRes.raiseExceptionIfFailed();
        minionId = machineIdRes.getStdout().trim();
        minionIds.put(targetIp, minionId);
    }

    private boolean needSetupMinion() {
        if (agentPort != 0) {
            boolean opened = NetworkUtils.isRemotePortOpen(targetIp, agentPort, (int)TimeUnit.SECONDS.toMillis(5));
            if (!opened) {
                logger.debug(String.format("agent port[%s] on target ip[%s] is not opened, run salt[%s]", agentPort, targetIp, moduleName));
                return true;
            }

            boolean changed = saltf.isModuleChanged(moduleName);
            if (changed) {
                logger.debug(String.format("salt module[%s] changed, run salt", moduleName));
                return true;
            }

            logger.debug(String.format("agent port[%s] on target ip[%s] is opened, salt module[%s] is not changed, skip to run salt", agentPort, targetIp, moduleName));
            return false;
        }

        return true;
    }

    private void setupMinion(final Completion completion) {
        final SaltSetupMinionJob job = new SaltSetupMinionJob();
        if (minionId == null)  {
            generateMinionId();
        }

        Assert(minionId != null, "minionId can not be null");

        job.setPrivateKey(privateKey);
        job.setPassword(password);
        job.setPort(sshPort);
        job.setTargetIp(targetIp);
        job.setUsername(username);
        job.setSaltBootstrapScriptPath(saltBootstrapScriptPath);
        job.setSaltMinionConfPath(saltMinionConfPath);
        job.setMinionId(minionId);
        job.setCleanMasterKey(cleanupMasterPubKey);

        boolean useJob = SaltGlobalConfig.SETUP_MINION_IN_JOB.value(Boolean.class);
        if (useJob) {
            jobf.execute(String.format("setup-minion-%s", minionId), Platform.getManagementServerId(), job, completion);
        } else {
            thdf.chainSubmit(new ChainTask(completion) {
                @Override
                public String getSyncSignature() {
                    return minionId;
                }

                @Override
                public void run(final SyncTaskChain chain) {
                    job.run(new ReturnValueCompletion<Object>(chain) {
                        @Override
                        public void success(Object returnValue) {
                            completion.success();
                            chain.next();
                        }

                        @Override
                        public void fail(ErrorCode errorCode) {
                            completion.fail(errorCode);
                            chain.next();
                        }
                    });
                }

                @Override
                public String getName() {
                    return String.format("setup-minion-for-%s", targetIp);
                }
            });
        }
    }

    private boolean doRunState() {
        StringBuilder sb = new StringBuilder(String.format("/usr/bin/salt --out=json '%s' state.sls %s queue=True", minionId, stateName));
        boolean alwaysFullDeploy = Boolean.valueOf(System.getProperty("SaltFacade.alwaysFullDeploy"));
        if (alwaysFullDeploy ? alwaysFullDeploy : fullDeploy) {
            sb.append(String.format(" pillar=\"{'pkg':True}\""));
        }
        String cmd = sb.toString();
        ShellResult res = ShellUtils.runAndReturn(cmd);

        if ("".equals(res.getStdout()) || res.isReturnCode(2)) {
            sb = new StringBuilder(String.format("\nfailed to apply salt state[minion id:%s, state name:%s]", minionId, stateName));
            sb.append(String.format("\ncommand: %s", cmd));
            sb.append(String.format("\nno json output from the command, it's probably caused by the minion hasn't connected to master, will retry", res.getStdout()));
            logger.debug(sb.toString());
            return false;
        }

        res.raiseExceptionIfFail();

        HashMap output = JSONObjectUtil.toObject(res.getStdout(), HashMap.class);
        for (Object val : output.values()) {
            if (val instanceof List) {
                sb = new StringBuilder(String.format("\nfailed to apply salt state[minion id:%s, state name:%s]", minionId, stateName));
                sb.append(String.format("\ncommand: %s", cmd));
                sb.append(String.format("\n%s", res.getStdout()));
                throw new SaltException(sb.toString());
            }

            Map m = (Map)val;
            for (Object v2 : m.values()) {
                Map m2 = (Map)v2;
                Boolean r = (Boolean) m2.get("result");
                if (!r) {
                    sb = new StringBuilder(String.format("\nfailed to apply salt state[minion id:%s, state name:%s]", minionId, stateName));
                    sb.append(String.format("\ncommand: %s", cmd));
                    sb.append(String.format("\n%s", res.getStdout()));
                    throw new SaltException(sb.toString());
                }
            }
        }

        return true;
    }

    public void run(final Completion completion) {
        if (!needSetupMinion()) {
            completion.success();
            return;
        }

        setupMinion(new Completion(completion) {
            @Override
            public void success() {
                try {
                    // a close runState after setting up minion likely fails because the minion hasn't connected to master
                    // we retry to work around this problem
                    int retry = SaltGlobalConfig.SETUP_MINION_RETRY.value(Integer.class);
                    int interval = SaltGlobalConfig.SETUP_MINION_RETRY_INTERVAL.value(Integer.class);
                    boolean ret = false;
                    for (int i=0; i<retry; i++) {
                        ret = doRunState();
                        if (ret) {
                            break;
                        }

                        try {
                            TimeUnit.SECONDS.sleep(interval);
                        } catch (InterruptedException e) {
                            throw new CloudRuntimeException(e);
                        }
                    }

                    if (!ret) {
                        completion.fail(operr("failed to run salt state[%s] on system[%s], failed after %s retries", stateName, targetIp, retry));
                        return;
                    }

                    logger.debug(String.format("successfully run salt state[%s] on system[%s]", stateName, targetIp));
                    completion.success();
                } catch (Exception e) {
                    logger.warn(String.format("failed to run salt state[%s] on system[%s], %s", stateName, targetIp, e.getMessage()));
                    completion.fail(errf.throwableToOperationError(e));
                }
            }

            @Override
            public void fail(ErrorCode errorCode) {
                logger.warn(String.format("failed to setup salt minion for state[%s] on system[%s], %s", stateName, targetIp, errorCode));
                completion.fail(errorCode);
            }
        });
    }

    public int getAgentPort() {
        return agentPort;
    }

    public SaltRunner setAgentPort(int agentPort) {
        this.agentPort = agentPort;
        return this;
    }

    public int getSshPort() {
        return sshPort;
    }

    public SaltRunner setSshPort(int sshPort) {
        this.sshPort = sshPort;
        return this;
    }

    public String getStateName() {
        return stateName;
    }

    public SaltRunner setStateName(String stateName) {
        this.stateName = stateName;
        return this;
    }

    public boolean isFullDeploy() {
        return fullDeploy;
    }

    public SaltRunner setFullDeploy(boolean fullDeploy) {
        this.fullDeploy = fullDeploy;
        return this;
    }
}