/* * Copyright (C) 2006-2016 DLR, Germany * * All rights reserved * * http://www.rcenvironment.de/ */ package de.rcenvironment.core.instancemanagement.internal; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; import java.io.InputStream; import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.io.UnsupportedEncodingException; import java.io.Writer; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.mindrot.jbcrypt.BCrypt; import com.jcraft.jsch.JSchException; import com.jcraft.jsch.Logger; import com.jcraft.jsch.Session; import de.rcenvironment.core.configuration.ConfigurationSegment; import de.rcenvironment.core.configuration.ConfigurationService; import de.rcenvironment.core.configuration.PersistentSettingsService; import de.rcenvironment.core.configuration.bootstrap.BootstrapConfiguration; import de.rcenvironment.core.instancemanagement.InstanceManagementConstants; import de.rcenvironment.core.instancemanagement.InstanceManagementService; import de.rcenvironment.core.toolkitbridge.transitional.TextStreamWatcherFactory; import de.rcenvironment.core.utils.common.TempFileService; import de.rcenvironment.core.utils.common.TempFileServiceAccess; import de.rcenvironment.core.utils.common.textstream.TextOutputReceiver; import de.rcenvironment.core.utils.common.textstream.receivers.AbstractTextOutputReceiver; import de.rcenvironment.core.utils.incubator.FileSystemOperations; import de.rcenvironment.core.utils.ssh.jsch.JschSessionFactory; import de.rcenvironment.core.utils.ssh.jsch.SshParameterException; import de.rcenvironment.core.utils.ssh.jsch.executor.JSchRCECommandLineExecutor; /** * Default {@link InstanceManagementService} implementation. * * @author Robert Mischke * @author David Scholz */ public class InstanceManagementServiceImpl implements InstanceManagementService { private static final String ZIP = ".zip"; private static final String SLASH = "/"; private static final String INDENT = "- "; private static final String TEMPLATES = "templates"; private static final String VERSION = ".version"; private static final Pattern VALID_IDS_REGEXP_PATTERN = Pattern.compile("[a-zA-Z0-9-_]+"); private static final String GENERIC_PLACEHOLDER_STRING = "*"; private static final String CONFIGURATION_SUBTREE_PATH = "instanceManagement"; private static final String CONFIGURATION_FILENAME = "configuration.json"; // TODO find actual value through testing private static final int MAX_INSTALLATION_ROOT_PATH_LENGTH = 30; private static final String DATA_ROOT_DIRECTORY_PROPERTY = "dataRootDirectory"; private static final String INSTALLATIONS_ROOT_DIR_PROPERTY = "installationsRootDirectory"; private static final Map<String, InstanceConfigurationImpl> CONFIG_FILE_NAME_TO_CONFIG_STORE_MAP = new HashMap<>(); private static final String SUCCESS = " was successful."; private static final String TO_INSTANCE = " to instance "; private static final String OF_INSTANCE = " of instance "; private File dataRootDir; private File installationsRootDir; private File profilesRootDir; private File templatesRootDir; private File downloadsCacheDir; private volatile boolean hasValidLocalConfiguration = false; private volatile boolean hasValidDownloadConfiguration = false; private ConcurrentHashMap<String, String> profileIdToInstallationIdMap = new ConcurrentHashMap<>(); private ConfigurationService configurationService; private PersistentSettingsService persistentSettingsService; // split into distinct classes to allow separate testing private final InstanceOperations instanceOperations = new InstanceOperationsImplSynchronizeDecorator( new InstanceOperationsImplReleaseLockDecorator(new InstanceOperationsImpl())); private final DeploymentOperationsImpl deploymentOperations = new DeploymentOperationsImpl(); private TempFileService tfs; private String downloadSourceFolderUrlPattern; private String downloadFilenamePattern; private final TextOutputReceiver fallbackUserOutputReceiver; private final Log log = LogFactory.getLog(getClass()); public InstanceManagementServiceImpl() { fallbackUserOutputReceiver = new AbstractTextOutputReceiver() { @Override public void addOutput(String line) { log.info("Operation progress: "); } }; } /** * OSGi-DS lifecycle method; made public for unit testing. */ public void activate() { tfs = TempFileServiceAccess.getInstance(); hasValidLocalConfiguration = false; hasValidDownloadConfiguration = false; final ConfigurationSegment configuration = this.configurationService.getConfigurationSegment(CONFIGURATION_SUBTREE_PATH); if (!configuration.isPresentInCurrentConfiguration()) { log.debug("No '" + CONFIGURATION_SUBTREE_PATH + "' configuration segment found, disabling instance management"); return; } try { applyConfiguration(configuration); initProfileIdToInstallationMap(); } catch (IOException e) { // do not fail the activate() method; the failure is marked by the "hasValidConfiguration" flag being "false" log.error("Error while configuring " + getClass().getSimpleName(), e); } } /** * OSGi-DS lifecycle method; made public for unit testing. */ public void deactivate() { hasValidLocalConfiguration = false; hasValidDownloadConfiguration = false; } @Override public void setupInstallationFromUrlQualifier(String installationId, String urlQualifier, InstallationPolicy installationPolicy, TextOutputReceiver userOutputReceiver, final long timeout) throws IOException { validateInstallationId(installationId); validateConfiguration(true, true); userOutputReceiver = ensureUserOutputReceiverDefined(userOutputReceiver); deploymentOperations.setUserOutputReceiver(userOutputReceiver); // Do not try to reinstall running installations if (isInstallationRunning(installationId)) { throw new IOException("Cannot replace installation " + installationId + " because instances are currently running using this installation. Stop the " + "instannces or try the \"im reinstall\" command instead."); } switch (installationPolicy) { case IF_PRESENT_CHECK_VERSION_AND_REINSTALL_IF_DIFFERENT: installIfVersionIsDifferent(installationId, urlQualifier, userOutputReceiver); break; case FORCE_NEW_DOWNLOAD_AND_REINSTALL: forceDownloadAndReinstall(installationId, urlQualifier, userOutputReceiver); break; case FORCE_REINSTALL: forceReinstall(installationId, urlQualifier, userOutputReceiver); break; case ONLY_INSTALL_IF_NOT_PRESENT: installIfNotPresent(installationId, urlQualifier, userOutputReceiver); break; default: throw new IOException("Not supported yet: " + installationPolicy); } } @Override public void reinstallFromUrlQualifier(String installationId, String urlQualifier, InstallationPolicy installationPolicy, TextOutputReceiver userOutputReceiver, final long timeout) throws IOException { validateInstallationId(installationId); validateConfiguration(true, true); userOutputReceiver = ensureUserOutputReceiverDefined(userOutputReceiver); deploymentOperations.setUserOutputReceiver(userOutputReceiver); // Get instances running with this installation List<String> instancesForInstallation = getInstancesRunningInstallation(installationId); //Stop running instances if (!instancesForInstallation.isEmpty()) { stopInstance(instancesForInstallation, userOutputReceiver, timeout); } //Reinstall switch (installationPolicy) { case IF_PRESENT_CHECK_VERSION_AND_REINSTALL_IF_DIFFERENT: installIfVersionIsDifferent(installationId, urlQualifier, userOutputReceiver); break; case FORCE_NEW_DOWNLOAD_AND_REINSTALL: forceDownloadAndReinstall(installationId, urlQualifier, userOutputReceiver); break; case FORCE_REINSTALL: forceReinstall(installationId, urlQualifier, userOutputReceiver); break; default: throw new IOException("Not supported yet: " + installationPolicy); } //Start instances with new installation if (!instancesForInstallation.isEmpty()) { startInstance(installationId, instancesForInstallation, userOutputReceiver, timeout, false); } } @Override public void configureInstance(String instanceId, ConfigurationChangeSequence changeSequence, TextOutputReceiver userOutputReceiver) throws IOException { validateConfiguration(true, false); final File destinationConfigFile = new File(profilesRootDir + SLASH + instanceId, CONFIGURATION_FILENAME); createProfileWithEmptyConfigFileIfNotPresent(destinationConfigFile); List<ConfigurationChangeEntry> changeEntries = changeSequence.getAll(); if (changeEntries.isEmpty()) { throw new IllegalArgumentException("There must be at least one configuration step to perform"); } // perform commands "reset" or "apply template" that can only reasonably happen as the first step, // and before creating the in-memory configuration modification class ConfigurationChangeEntry firstEntry = changeEntries.get(0); switch (firstEntry.getFlag()) { case RESET_CONFIGURATION: // FIXME backup?! writeEmptyConfigFile(destinationConfigFile); userOutputReceiver.addOutput("Clearing/resetting the configuration" + OF_INSTANCE + instanceId); break; case APPLY_TEMPLATE: // FIXME backup?! userOutputReceiver.addOutput("Replacing configuration" + OF_INSTANCE + instanceId + " with template " + firstEntry.getValue()); File template = resolveAndCheckTemplateDir((String) firstEntry.getValue() + SLASH + CONFIGURATION_FILENAME); Path src = template.toPath(); Files.copy(src, destinationConfigFile.toPath(), StandardCopyOption.REPLACE_EXISTING); break; default: // ignore standard commands here } applyChangeEntries(changeEntries, destinationConfigFile, instanceId, userOutputReceiver); } private void applyChangeEntries(List<ConfigurationChangeEntry> changeEntries, final File destinationConfigFile, String instanceId, TextOutputReceiver userOutputReceiver) throws IOException { final InstanceConfigurationImpl configOperations; if (CONFIG_FILE_NAME_TO_CONFIG_STORE_MAP.get(destinationConfigFile) == null) { configOperations = new InstanceConfigurationImpl(destinationConfigFile); CONFIG_FILE_NAME_TO_CONFIG_STORE_MAP.put(destinationConfigFile.getName(), configOperations); } else { configOperations = CONFIG_FILE_NAME_TO_CONFIG_STORE_MAP.get(destinationConfigFile.getName()); } boolean isFirstCommand = true; for (ConfigurationChangeEntry entry : changeEntries) { switch (entry.getFlag()) { case SET_NAME: configOperations.setInstanceName((String) entry.getValue()); userOutputReceiver.addOutput("Setting instance name of instance " + instanceId + " to " + (String) entry.getValue()); break; case DISABLE_RELAY: configOperations.setRelayFlag((Boolean) entry.getValue()); userOutputReceiver.addOutput("Disabling relay flag" + OF_INSTANCE + instanceId); break; case ENABLE_RELAY: configOperations.setRelayFlag((Boolean) entry.getValue()); userOutputReceiver.addOutput("Enabling relay flag" + OF_INSTANCE + instanceId); break; case SET_COMMENT: configOperations.setInstanceComment((String) entry.getValue()); userOutputReceiver.addOutput("Setting comment field " + OF_INSTANCE + instanceId + " to " + entry.getValue()); break; case ADD_ALLOWED_IP: configOperations.addAllowedIp((String) entry.getValue()); userOutputReceiver.addOutput("Adding " + (String) entry.getValue() + " to allowed ips was successfull"); break; case ADD_CONNECTION: final ConfigurationConnection connectionData = (ConfigurationConnection) entry.getValue(); configOperations.addConnection(connectionData); userOutputReceiver.addOutput("Adding connection " + connectionData.getConnectionName() + " to instance " + instanceId); break; case ADD_SERVER_PORT: addServerPortToConfig(instanceId, userOutputReceiver, SUCCESS, TO_INSTANCE, configOperations, entry); break; case SET_BACKGROUND_MONITORING: @SuppressWarnings("unchecked") Map<String, Integer> map = (HashMap<String, Integer>) entry.getValue(); configOperations.setBackgroundMonitoring((String) map.keySet().toArray()[0], map.get(map.keySet().toArray()[0])); userOutputReceiver.addOutput("Setting background monitoring " + OF_INSTANCE + instanceId); break; case ADD_SSH_CONNECTION: configOperations.addSshConnection((ConfigurationSshConnection) entry.getValue()); userOutputReceiver.addOutput("Adding ssh connection " + TO_INSTANCE + instanceId); break; case DISABLE_DEP_INPUT_TAB: configOperations.disableDeprecatedInputTab(); userOutputReceiver.addOutput("Disabling deprecated input tab" + OF_INSTANCE + instanceId); break; case DISABLE_SSH_SERVER: configOperations.disableSshServer(); userOutputReceiver.addOutput("Disabling ssh server" + OF_INSTANCE + instanceId); break; case DISABLE_WORKFLOWHOST: configOperations.disableWorkflowHost(); userOutputReceiver.addOutput("Disabling workflow host flag" + OF_INSTANCE + instanceId); break; case ENABLE_DEP_INPUT_TAB: configOperations.enableDeprecatedInputTab(); userOutputReceiver.addOutput("Enabling deprecated input tab" + OF_INSTANCE + instanceId); break; case ENABLE_IP_FILTER: configOperations.enableIpFilter(); userOutputReceiver.addOutput("Enabling ip filter" + OF_INSTANCE + instanceId); break; case DISABLE_IP_FILTER: configOperations.disableIpFilter(); userOutputReceiver.addOutput("Disabling ip filter" + OF_INSTANCE + instanceId); break; case ENABLE_SSH_SERVER: configOperations.enableSshServer(); userOutputReceiver.addOutput("Enabling ssh server" + OF_INSTANCE + instanceId); break; case ENABLE_WORKFLOWHOST: configOperations.enableWorkflowHost(); userOutputReceiver.addOutput("Enabling workflow host flag" + OF_INSTANCE + instanceId); break; case FORWARDING_TIMEOUT: configOperations.setForwardingTimeout((Long) entry.getValue()); userOutputReceiver.addOutput("Setting forwarding timeout" + OF_INSTANCE + instanceId); break; case PUBLISH_COMPONENT: configOperations.publishComponent((String) entry.getValue()); userOutputReceiver.addOutput("Publishing component" + OF_INSTANCE + instanceId); break; case REMOVE_ALLOWED_IP: configOperations.removeAllowedIp((String) entry.getValue()); userOutputReceiver.addOutput("Removing allowed ip" + OF_INSTANCE + instanceId); break; case REMOVE_CONNECTION: configOperations.removeConnection((String) entry.getValue()); userOutputReceiver.addOutput("Removing connection " + (String) entry.getValue() + OF_INSTANCE + instanceId); break; case REMOVE_SERVER_PORT: configOperations.removeServerPort((String) entry.getValue()); userOutputReceiver.addOutput("Removing server port " + (String) entry.getValue() + OF_INSTANCE + instanceId); break; case REMOVE_SSH_CONNECTION: configOperations.removeSshConnection((String) entry.getValue()); userOutputReceiver .addOutput("Removing ssh connection " + (String) entry.getValue() + OF_INSTANCE + instanceId); break; case REQUEST_TIMEOUT: configOperations.setRequestTimeout((Long) entry.getValue()); userOutputReceiver.addOutput("Setting request timeout" + OF_INSTANCE + instanceId); break; case SET_SSH_SERVER_IP: configOperations.setSshServerIP((String) entry.getValue()); userOutputReceiver.addOutput("Setting ssh server ip" + OF_INSTANCE + instanceId); break; case SET_SSH_SERVER_PORT: configOperations.setSshServerPort((Integer) entry.getValue()); userOutputReceiver.addOutput("Setting ssh server port" + OF_INSTANCE + instanceId); break; case TEMP_DIR: configOperations.setTempDirectory((String) entry.getValue()); userOutputReceiver.addOutput("Setting temp directory" + OF_INSTANCE + instanceId); break; case UNPUBLISH_COMPONENT: configOperations.unPublishComponent((String) entry.getValue()); userOutputReceiver.addOutput("Unpublishing component" + OF_INSTANCE + instanceId); break; case ENABLE_IM_SSH_ACCESS: configOperations.enableImSshAccess(((Integer) entry.getValue()), getHashedPassphrase()); userOutputReceiver.addOutput("Configuring ssh access for IM on instance" + instanceId); break; case RESET_CONFIGURATION: case APPLY_TEMPLATE: // these operations were already performed; only run a consistency check here if (!isFirstCommand) { throw new IOException("Resetting the configuration or applying a template " + "must take place *before* applying any other configuration commands"); // TODO better exception type } break; default: throw new IOException("Unhandled configuration change request: " + entry.getFlag()); // TODO better exception type } isFirstCommand = false; } configOperations.update(); userOutputReceiver.addOutput("Updated the configuration file" + OF_INSTANCE + instanceId); } private void createProfileWithEmptyConfigFileIfNotPresent(File config) throws IOException { if (!config.exists()) { config.getParentFile().mkdir(); writeEmptyConfigFile(config); } } private void writeEmptyConfigFile(File config) throws FileNotFoundException { try (PrintWriter writer = new PrintWriter(config)) { writer.println("{"); writer.println("}"); } } /** * The IM master uses the same passphrase for all instances. This method retreives the passphrase from the persistent settings. If no * passphrase is stored yet, it is created randomly. * * @return the password hash */ private String getHashedPassphrase() { String passphrase = persistentSettingsService.readStringValue(InstanceManagementConstants.IM_MASTER_PASSPHRASE_KEY); if (passphrase == null) { passphrase = RandomStringUtils.randomAlphanumeric(10); persistentSettingsService.saveStringValue(InstanceManagementConstants.IM_MASTER_PASSPHRASE_KEY, passphrase); } return BCrypt.hashpw(passphrase, BCrypt.gensalt(10)); } private void addServerPortToConfig(String instanceId, TextOutputReceiver userOutputReceiver, final String success, final String toInstance, InstanceConfigurationImpl configOperations, ConfigurationChangeEntry key) throws IOException { // explicit type cast as we can't guarantee the correct type of the list objects itself with generics. Object o = key.getValue(); if (o instanceof List) { @SuppressWarnings("unchecked") List<String> list = (List<String>) o; if (list.size() > 3) { throw new IOException("Wrong number of parameters."); } String name = list.get(0); String ip = list.get(1); int port = Integer.parseInt(list.get(2)); configOperations.addServerPort((String) name, (String) ip, (int) port); userOutputReceiver.addOutput("Adding server port " + ((String) name) + toInstance + instanceId + success); } else { throw new IOException("Failure."); } } @Override public void startInstance(String installationId, List<String> instanceIdList, TextOutputReceiver userOutputReceiver, final long timeout, boolean startWithGui) throws IOException { // TODO add some user output instead of simply return. if (installationId == null || instanceIdList == null || installationId.isEmpty() || instanceIdList.isEmpty()) { throw new IOException("Malformed command: either no installation id or instance id defined."); } File possibleInstallation = new File(installationsRootDir + SLASH + installationId); if (!possibleInstallation.exists()) { throw new IOException("Installation with id: " + installationId + " does not exist."); } validateInstallationId(installationId); validateInstanceId(instanceIdList, false); validateConfiguration(true, false); userOutputReceiver = ensureUserOutputReceiverDefined(userOutputReceiver); final File installationDir = resolveAndCheckInstallationDir(installationId); for (String s : new ArrayList<String>(instanceIdList)) { if (isInstanceRunning(s)) { instanceIdList.remove(s); userOutputReceiver.addOutput("Profile with id: " + s + " is already in use."); } } List<File> profileDirList = resolveAndCheckProfileDirList(instanceIdList); writeInstallationIdToFile(installationId, profileDirList, userOutputReceiver); try { instanceOperations.startInstanceUsingInstallation(profileDirList, installationDir, timeout, userOutputReceiver, startWithGui); } catch (InstanceOperationException e) { if (e.getMessage().contains("Timeout reached")) { List<String> names = new ArrayList<String>(); for (File profile : e.getFailedInstances()) { names.add(profile.getName()); } userOutputReceiver.addOutput("Timeout reached... Aborting startup process of failed instances: " + names); stopInstance(names, userOutputReceiver, 0); } else { throw new IOException(e); } } addInstanceToMap(instanceIdList, installationId); } @Override public void stopInstance(List<String> instanceIdList, TextOutputReceiver userOutputReceiver, final long timeout) throws IOException { // TODO add some user output instead of simply return. if (instanceIdList == null || instanceIdList.isEmpty()) { userOutputReceiver.addOutput("No instance to stop defined.. aborting."); return; } validateInstanceId(instanceIdList, true); validateConfiguration(true, false); userOutputReceiver = ensureUserOutputReceiverDefined(userOutputReceiver); final List<File> profileDirList = resolveAndCheckProfileDirList(instanceIdList); try { instanceOperations.shutdownInstance(profileDirList, timeout, userOutputReceiver); } catch (IOException e) { checkAndRemoveInstanceLock(profileDirList, instanceIdList, e); } finally { removeInstanceTopMap(instanceIdList); } } private void checkAndRemoveInstanceLock(List<File> profileDirList, List<String> instanceIdList, IOException e) throws IOException { for (File profile : new ArrayList<>(profileDirList)) { if (!isInstanceRunning(profile.getName())) { profileDirList.remove(profile); if (profile.isDirectory()) { for (File file : profile.listFiles()) { if (file.getName().equals(BootstrapConfiguration.PROFILE_DIR_LOCK_FILE_NAME)) { boolean success = file.delete(); if (e != null && !success) { throw new IOException("Failed to delete instance.lock file.", e); } else if (!success) { throw new IOException("Failed to delete instance.lock file."); } } } } } } } @Override public boolean isInstanceRunning(String instanceId) throws IOException { validateConfiguration(true, false); final File profileDir = resolveAndCheckProfileDir(instanceId); return instanceOperations.isProfileLocked(profileDir); } /** * returns Checks if an instance is currently running with the given installation. * * @param installationId * @return true iff an instance is currently running with the given installation * @throws IOException */ private boolean isInstallationRunning(String installationId) throws IOException { for (Map.Entry<String, String> entry : profileIdToInstallationIdMap.entrySet()) { if (entry.getValue().equals(installationId)) { if (isInstanceRunning(entry.getKey())) { return true; } } } return false; } /** * Returns all instances currently running with the given installation ID. * * @param installationId * @return * @throws IOException */ private List<String> getInstancesRunningInstallation(String installationId) throws IOException { List<String> instances = new ArrayList<String>(); for (Map.Entry<String, String> entry : profileIdToInstallationIdMap.entrySet()) { if (entry.getValue().equals(installationId)) { if (isInstanceRunning(entry.getKey())) { instances.add(entry.getKey()); } } } return instances; } protected void bindConfigurationService(ConfigurationService newInstance) { this.configurationService = newInstance; } protected void bindPersistentSettingsService(PersistentSettingsService newService) { this.persistentSettingsService = newService; } @Override public void listInstanceManagementInformation(String scope, TextOutputReceiver userOutputReceiver) throws IOException { if ("all".equals(scope)) { listInstances(userOutputReceiver); listInstallations(userOutputReceiver); listTemplates(userOutputReceiver); } else if ("instances".equals(scope)) { listInstances(userOutputReceiver); } else if ("installations".equals(scope)) { listInstallations(userOutputReceiver); } else if ("templates".equals(scope)) { listTemplates(userOutputReceiver); } } @Override public void disposeInstance(String instanceId, TextOutputReceiver outputReceiver) throws IOException { if (profilesRootDir == null) { throw new IOException("Failed to dispose instance. Instances' root directory not defined."); } for (File instanceFile : profilesRootDir.listFiles()) { if (instanceFile.isDirectory() && instanceId.equals(instanceFile.getName())) { for (File fileInInstanceFolder : instanceFile.listFiles()) { if (fileInInstanceFolder.getName().equals(BootstrapConfiguration.PROFILE_DIR_LOCK_FILE_NAME)) { throw new IOException("Instance with ID " + instanceId + " currently in use. " + "To stop it use 'im stop " + instanceId + "'."); } } FileSystemOperations.deleteSandboxDirectory(instanceFile); return; } } throw new IOException("Instance with ID " + instanceId + " not found."); } @Override public void showInstanceManagementInformation(TextOutputReceiver outputReceiver) { if (profilesRootDir != null) { outputReceiver.addOutput("Instances' root directory: " + profilesRootDir.getAbsolutePath()); } else { outputReceiver.addOutput("Instances' root directory not defined."); } if (installationsRootDir != null) { outputReceiver.addOutput("Installations' root directory: " + installationsRootDir.getAbsolutePath()); } else { outputReceiver.addOutput("Installations' root directory not defined."); } if (templatesRootDir != null) { outputReceiver.addOutput("Templates' root directory: " + templatesRootDir.getAbsolutePath()); } else { outputReceiver.addOutput("Templates' root directory not defined."); } if (dataRootDir != null) { outputReceiver.addOutput("Data root directory: " + dataRootDir.getAbsolutePath()); } else { outputReceiver.addOutput("Data root directory not defined."); } if (downloadsCacheDir != null) { outputReceiver.addOutput("Download cache directory: " + downloadsCacheDir.getAbsolutePath()); if (downloadsCacheDir.listFiles().length > 0) { outputReceiver.addOutput("Downloads cached: "); for (File cachedDownloadFile : downloadsCacheDir.listFiles()) { outputReceiver.addOutput(INDENT + cachedDownloadFile.getName().replace(ZIP, "")); } } else { outputReceiver.addOutput("No download cached."); } } else { outputReceiver.addOutput("Download cache directory not defined."); } } @Override public void startAllInstances(String installationId, TextOutputReceiver userOutputReceiver, final long timeout) throws IOException { List<String> instanceIdList = new ArrayList<>(); addProfiles(profilesRootDir.toPath(), instanceIdList); startInstance(installationId, instanceIdList, userOutputReceiver, timeout, false); } @Override public void stopAllInstances(String installationId, TextOutputReceiver userOutputReceiver, final long timeout) throws IOException { List<String> instanceIdList = new ArrayList<>(); if (!installationId.isEmpty()) { synchronized (profileIdToInstallationIdMap) { for (Map.Entry<String, String> entry : profileIdToInstallationIdMap.entrySet()) { if (entry.getValue().equals(installationId)) { instanceIdList.add(entry.getKey()); } } } } else { addProfiles(profilesRootDir.toPath(), instanceIdList); } stopInstance(instanceIdList, userOutputReceiver, timeout); } /** * * Intended for unit tests. * * @return the profile root dir. */ protected File getProfilesRootDir() { return profilesRootDir; } /** * * Intended for unit tests. * */ protected void setProfilesRootDir(File file) { profilesRootDir = file; } /** * * Intended for unit tests. * */ protected void validateLocalConfig() { hasValidLocalConfiguration = true; } protected String fetchVersionInformationFromDownloadSourceFolder(String urlQualifier) throws IOException { validateConfiguration(false, true); File tempVersionFile = tfs.createTempFileFromPattern("versionfile-*.tmp"); String versionFileUrl = downloadSourceFolderUrlPattern.replace(GENERIC_PLACEHOLDER_STRING, urlQualifier) + "VERSION"; log.debug("Fetching remote version information from " + versionFileUrl); deploymentOperations.downloadFile(versionFileUrl, tempVersionFile, true, false); // allow overwrite as temp file already exists return FileUtils.readFileToString(tempVersionFile).trim(); } protected void downloadInstallationPackage(String urlQualifier, String version, File localFile) throws IOException { validateConfiguration(false, true); String remoteFileUrl = downloadSourceFolderUrlPattern.replace(GENERIC_PLACEHOLDER_STRING, urlQualifier) + downloadFilenamePattern.replace(GENERIC_PLACEHOLDER_STRING, version); log.debug("Downloading installation package from '" + remoteFileUrl + "' to local file '" + localFile.getAbsolutePath() + "'"); deploymentOperations.downloadFile(remoteFileUrl, localFile, true, true); // allow overwrite } private void applyConfiguration(final ConfigurationSegment configuration) throws IOException { try { hasValidLocalConfiguration = false; this.dataRootDir = getConfiguredDirectory(configuration, DATA_ROOT_DIRECTORY_PROPERTY); this.installationsRootDir = getConfiguredDirectory(configuration, INSTALLATIONS_ROOT_DIR_PROPERTY); if (dataRootDir == null || installationsRootDir == null) { throw new IOException("Data or installation root directory (or both) unspecified"); } this.templatesRootDir = new File(dataRootDir, TEMPLATES); this.profilesRootDir = new File(dataRootDir, "profiles"); this.downloadsCacheDir = new File(dataRootDir, "downloads"); // TODO add new combinations if (installationsRootDir.equals(templatesRootDir) || installationsRootDir.equals(profilesRootDir) || templatesRootDir.equals(profilesRootDir)) { throw new IOException("Two or more configured directory are equal, but they must be unique"); } // TODO run this check on Windows only? if (installationsRootDir.getPath().length() > MAX_INSTALLATION_ROOT_PATH_LENGTH) { throw new IOException("Installation root path is too long: " + installationsRootDir.getPath()); } prepareAndValidateDirectory(DATA_ROOT_DIRECTORY_PROPERTY, dataRootDir); prepareAndValidateDirectory(INSTALLATIONS_ROOT_DIR_PROPERTY, installationsRootDir); prepareAndValidateDirectory(TEMPLATES, templatesRootDir); prepareAndValidateDirectory("profiles", profilesRootDir); prepareAndValidateDirectory("downloads", downloadsCacheDir); hasValidLocalConfiguration = true; } catch (IOException e) { log.info("Disabling local instance management due to missing or invalid configuration: " + e.getMessage()); } // note: these settings use an empty string as "undefined" markers this.downloadSourceFolderUrlPattern = configuration.getString("downloadSourceFolderUrlPattern", ""); // normalize URL pattern to end with "/" if (downloadSourceFolderUrlPattern.length() > 0 && !downloadSourceFolderUrlPattern.endsWith(SLASH)) { downloadSourceFolderUrlPattern = downloadSourceFolderUrlPattern + SLASH; } this.downloadFilenamePattern = configuration.getString("downloadFilenamePattern", ""); try { hasValidDownloadConfiguration = false; if (downloadSourceFolderUrlPattern.isEmpty()) { throw new IOException("Parameter 'downloadSourceFolderUrlPattern' has not been defined, but is required"); } if (downloadFilenamePattern.isEmpty()) { throw new IOException("Parameter 'downloadFilenamePattern' has not been defined, but is required"); } hasValidDownloadConfiguration = true; } catch (IOException e) { log.error("Error in instance management download configuration: " + e.getMessage()); } } private File forceFetchingProductZip(String urlQualifier, String remoteVersion) throws IOException { File downloadFile = new File(downloadsCacheDir, remoteVersion + ZIP); int i = 1; while (downloadFile.exists()) { String newFilePath = downloadFile.getAbsolutePath().replace(downloadFile.getName(), "") + remoteVersion + "(" + (i++) + ")" + ZIP; downloadFile = new File(newFilePath); } downloadInstallationPackage(urlQualifier, remoteVersion, downloadFile); return downloadFile; } private String fetchVersionInformation(TextOutputReceiver userOutputReceiver, String urlQualifier) throws IOException { // get remote version userOutputReceiver.addOutput("Fetching remote version information"); return fetchVersionInformationFromDownloadSourceFolder(urlQualifier); } private File fetchProductZipIfNecessary(String urlQualifier, String remoteVersion) throws IOException { File downloadFile = new File(downloadsCacheDir, remoteVersion + ZIP); if (downloadFile.exists()) { log.info("Version " + remoteVersion + " is already present in downloads cache, not downloading"); } else { downloadInstallationPackage(urlQualifier, remoteVersion, downloadFile); } return downloadFile; } private void forceDownloadAndReinstall(String installationId, String urlQualifier, TextOutputReceiver userOutputReceiver) throws IOException { String version = fetchVersionInformation(userOutputReceiver, urlQualifier); reinstall(installationId, version, urlQualifier, userOutputReceiver, true); } private void forceReinstall(String installationId, String urlQualifier, TextOutputReceiver userOutputReceiver) throws IOException { String version = fetchVersionInformation(userOutputReceiver, urlQualifier); // TODO validate remote version: not empty, plausible major version log.info("Identified remote version: " + version); reinstall(installationId, version, urlQualifier, userOutputReceiver, false); } private void installIfNotPresent(String installationId, String urlQualifier, TextOutputReceiver userOutputReceiver) throws IOException { File installationDir = new File(installationsRootDir, installationId); if (installationDir.exists()) { userOutputReceiver.addOutput("Installation with ID " + installationId + " already exists."); } else { installIfVersionIsDifferent(installationId, urlQualifier, userOutputReceiver); } } private void installIfVersionIsDifferent(String installationId, String urlQualifier, TextOutputReceiver userOutputReceiver) throws IOException { String newVersion = fetchVersionInformation(userOutputReceiver, urlQualifier); // TODO validate remote version: not empty, plausible major version log.info("Identified remote version: " + newVersion); // load local installed version information String oldVersion = getVersionOfInstallation(installationId); if (newVersion.isEmpty()) { throw new IOException("Unable to find new version"); } if (newVersion.equals(oldVersion)) { userOutputReceiver.addOutput("Remote and installed version are the same; no change required"); return; } reinstall(installationId, newVersion, urlQualifier, userOutputReceiver, false); } private void reinstall(String installationId, String version, String urlQualifier, TextOutputReceiver userOutputReceiver, boolean force) throws IOException { File zipFile; if (force) { log.info("Forces new download"); zipFile = forceFetchingProductZip(urlQualifier, version); } else { // download installation package if not already present zipFile = fetchProductZipIfNecessary(urlQualifier, version); } File installationDir = new File(installationsRootDir, installationId); if (installationDir.exists()) { userOutputReceiver.addOutput("Deleting old installation " + installationId); deploymentOperations.deleteInstallation(installationDir); } userOutputReceiver.addOutput("Setting up new installation " + installationId); deploymentOperations.installFromProductZip(zipFile, installationDir); log.debug("Writing version information for installation " + installationId); storeVersionOfInstallation(installationId, version); } private void listInstances(TextOutputReceiver userOutputReceiver) { if (profilesRootDir == null) { userOutputReceiver.addOutput("Instances' root directory not defined."); return; } if (profilesRootDir.listFiles().length == 0) { userOutputReceiver.addOutput("No instances found."); return; } userOutputReceiver.addOutput("Instances: "); for (File instanceFile : profilesRootDir.listFiles()) { if (instanceFile.isDirectory()) { String runningState = "Not running"; for (File fileInProfile : instanceFile.listFiles()) { synchronized (profileIdToInstallationIdMap) { if (fileInProfile.isFile() && fileInProfile.getName().equals(BootstrapConfiguration.PROFILE_DIR_LOCK_FILE_NAME)) { runningState = "Running (" + profileIdToInstallationIdMap.get(instanceFile.getName()) + ")"; } } } userOutputReceiver.addOutput(INDENT + instanceFile.getName() + " (" + runningState + ")"); } } } private void listInstallations(TextOutputReceiver userOutputReceiver) throws IOException { if (installationsRootDir == null) { userOutputReceiver.addOutput("Installations' root directory not defined."); return; } if (installationsRootDir.listFiles().length == 0) { userOutputReceiver.addOutput("No installations found."); return; } userOutputReceiver.addOutput("Installations: "); for (File installationFile : installationsRootDir.listFiles()) { if (installationFile.isFile() && installationFile.getName().endsWith(VERSION)) { String installationsId = installationFile.getName().replace(VERSION, ""); String version = FileUtils.readFileToString(installationFile); userOutputReceiver.addOutput(INDENT + installationsId + " (" + version + ")"); } } } private void listTemplates(TextOutputReceiver userOutputReceiver) { if (templatesRootDir == null) { userOutputReceiver.addOutput("Templates' root directory not defined."); return; } if (templatesRootDir.listFiles().length == 0) { userOutputReceiver.addOutput("No templates found."); return; } userOutputReceiver.addOutput("Templates: "); for (File templateFile : templatesRootDir.listFiles()) { if (templateFile.isDirectory()) { userOutputReceiver.addOutput(INDENT + templateFile.getName()); } } } private String getVersionOfInstallation(String installationId) throws IOException { String oldVersion; File installationVersionFile = new File(installationsRootDir, installationId + VERSION); if (installationVersionFile.exists()) { oldVersion = FileUtils.readFileToString(installationVersionFile); } else { oldVersion = ""; // simpler to handle than null } return oldVersion; } private void storeVersionOfInstallation(String installationId, String versionString) throws IOException { File installationVersionFile = new File(installationsRootDir, installationId + VERSION); FileUtils.writeStringToFile(installationVersionFile, versionString); } private List<File> resolveAndCheckProfileDirList(List<String> instanceIdList) throws IOException { final List<File> dirList = new ArrayList<>(); for (String instanceId : instanceIdList) { dirList.add(resolveAndCheckProfileDir(instanceId)); } return dirList; } private File resolveAndCheckInstallationDir(String installationId) throws IOException { final File installationDir = new File(installationsRootDir, installationId); if (installationDir.exists()) { prepareAndValidateDirectory("installation " + installationId, installationDir); } else { throw new IOException("The installation directory " + installationId + " does not exist."); } return installationDir; } private File resolveAndCheckProfileDir(String instanceId) throws IOException { final File profileDir = new File(profilesRootDir, instanceId); prepareAndValidateDirectory("instance " + instanceId, profileDir); return profileDir; } private File resolveAndCheckTemplateDir(String templateId) throws IOException { final File templateDir = new File(templatesRootDir, templateId); // prepareAndValidateDirectory("template " + templateId, templateDir); return templateDir; } private void prepareAndValidateDirectory(String id, File dir) throws IOException { dir.mkdirs(); String absolutePath = dir.getAbsolutePath(); if (!dir.isDirectory()) { throw new IOException("The configured path '" + id + "' ('" + absolutePath + "') could not be created"); } // TODO improve and document validation if (absolutePath.contains("\"")) { throw new IOException("The directory path '" + absolutePath + "' contains illegal characters"); } log.debug("Final path for id '" + id + "' " + absolutePath); } private TextOutputReceiver ensureUserOutputReceiverDefined(TextOutputReceiver userOutputReceiver) { if (userOutputReceiver != null) { return userOutputReceiver; } else { return fallbackUserOutputReceiver; } } private void validateConfiguration(boolean validateLocalConfig, boolean validateDownloadConfig) throws IOException { if (validateLocalConfig && !hasValidLocalConfiguration) { throw new IOException( "The instance management service is disabled or has no valid local configuration - " + "cannot execute the requested operation"); } if (validateDownloadConfig && !hasValidDownloadConfiguration) { throw new IOException( "The instance management service is disabled or has no valid download configuration - " + "cannot execute the requested operation"); } } private <T> boolean isUnique(Collection<T> idList) { Set<T> duplicateIdList = new LinkedHashSet<>(); Set<T> uniqueIdList = new HashSet<T>(); for (T t : idList) { if (!uniqueIdList.add(t)) { duplicateIdList.add(t); } } return duplicateIdList.isEmpty(); } private boolean isIdValid(String id) { // TODO add reasonable maximum length check? if (!VALID_IDS_REGEXP_PATTERN.matcher(id).matches()) { return false; } return true; } private boolean isProfilePresent(String id) { File possibleProfile = new File(profilesRootDir + SLASH + id); return possibleProfile.exists(); } private void validateInstallationId(String id) throws IOException { if (!isIdValid(id)) { // note: this assumes that even malformed ids are safe to print, as it should only affect the user // that issued the command anyway throw new IOException("Malformed id: " + id); } } private void validateInstanceId(List<String> idList, boolean forShutdown) throws IOException { if (!isUnique(idList)) { throw new IOException("Malformed command: multiple instances with identical id"); } for (String id : idList) { if (!isIdValid(id)) { // note: this assumes that even malformed ids are safe to print, as it should only affect the user // that issued the command anyway throw new IOException("Malformed id: " + id); } if (forShutdown && !isProfilePresent(id)) { throw new IOException("Malformed command: tried to shutdown instance, which doesn't exist"); } } } private void initProfileIdToInstallationMap() throws IOException { for (File instanceFile : profilesRootDir.listFiles()) { if (instanceFile.isDirectory()) { for (File fileInProfile : instanceFile.listFiles()) { if (fileInProfile.getName().equals("installation")) { try (BufferedReader br = new BufferedReader(new FileReader(fileInProfile))) { synchronized (profileIdToInstallationIdMap) { profileIdToInstallationIdMap.put(instanceFile.getName(), br.readLine()); } } } } } } } private void addInstanceToMap(List<String> instanceIds, String installationId) { for (String instanceId : instanceIds) { synchronized (profileIdToInstallationIdMap) { profileIdToInstallationIdMap.put(instanceId, installationId); } } } private void removeInstanceTopMap(List<String> instanceIds) throws IOException { for (Iterator<String> iterator = instanceIds.iterator(); iterator.hasNext();) { synchronized (profileIdToInstallationIdMap) { if (profileIdToInstallationIdMap.remove(iterator.next()) == null) { iterator.remove(); } } } } /** * * Adds all existing profiles to a given {@link Collection}. Note: this isn't a recursive search. There won't be a tree structure * containing all children as it is not needed in this particular case. * * @param directory * @param running * @return * @throws IOException */ private void addProfiles(Path directory, Collection<String> all) throws IOException { try (DirectoryStream<Path> ds = Files.newDirectoryStream(directory)) { for (Path child : ds) { all.add(child.getFileName().toString()); } } } private File getConfiguredDirectory(final ConfigurationSegment configuration, String id) throws IOException { String configuredPath = configuration.getString(id); log.debug("Configuration value for property '" + id + "': " + configuredPath); if (configuredPath != null) { return new File(configuredPath).getAbsoluteFile(); } else { return null; } } private void writeInstallationIdToFile(String installationId, final List<File> profileDirList, TextOutputReceiver userOutputReceiver) throws IOException, UnsupportedEncodingException, FileNotFoundException { for (File profile : profileDirList) { try (Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(profile.getAbsolutePath() + "/" + "installation"), "utf-8"))) { writer.write(installationId); } } } @Override public void executeCommandOnInstance(String instanceId, String command, TextOutputReceiver userOutputReceiver) throws JSchException, SshParameterException, IOException, InterruptedException { if (isInstanceRunning(instanceId)) { Logger logger = JschSessionFactory.createDelegateLogger(log); Integer port = getSshPortForInstance(instanceId); String ip = getSshIpForInstance(instanceId); String passphrase = persistentSettingsService.readStringValue(InstanceManagementConstants.IM_MASTER_PASSPHRASE_KEY); if (passphrase != null && port != null && ip != null) { Session session = JschSessionFactory.setupSession(ip, port, InstanceManagementConstants.IM_MASTER_USER_NAME, null, passphrase, logger); JSchRCECommandLineExecutor rceExecutor = new JSchRCECommandLineExecutor(session); rceExecutor.start(command); try (InputStream stdoutStream = rceExecutor.getStdout(); InputStream stderrStream = rceExecutor.getStderr();) { TextStreamWatcherFactory.create(stdoutStream, userOutputReceiver).start(); TextStreamWatcherFactory.create(stderrStream, userOutputReceiver).start(); rceExecutor.waitForTermination(); } session.disconnect(); userOutputReceiver.addOutput("Finished executing command " + command + " on instance " + instanceId); } else { userOutputReceiver.addOutput("Could not retrieve password and/or port for instance " + instanceId + "."); } } else { userOutputReceiver.addOutput("Cannot execute command on instance " + instanceId + " because it is not running."); } } /** * Retrieve SSH port from configuration. * * @return the port * @throws IOException */ private Integer getSshPortForInstance(String instanceId) throws IOException { File config = new File(new File(profilesRootDir, instanceId), CONFIGURATION_FILENAME); if (!config.exists()) { log.warn("No config file for instance " + instanceId + " exists."); return null; } InstanceConfigurationImpl configOperations; if (CONFIG_FILE_NAME_TO_CONFIG_STORE_MAP.get(config) == null) { configOperations = new InstanceConfigurationImpl(config); } else { configOperations = CONFIG_FILE_NAME_TO_CONFIG_STORE_MAP.get(config.getName()); } return configOperations.getSshServerPort(); } /** * Retrieve SSH ip from configuration. * * @return the port * @throws IOException */ private String getSshIpForInstance(String instanceId) throws IOException { File config = new File(new File(profilesRootDir, instanceId), CONFIGURATION_FILENAME); if (!config.exists()) { log.warn("No config file for instance " + instanceId + " exists."); return null; } InstanceConfigurationImpl configOperations; if (CONFIG_FILE_NAME_TO_CONFIG_STORE_MAP.get(config) == null) { configOperations = new InstanceConfigurationImpl(config); } else { configOperations = CONFIG_FILE_NAME_TO_CONFIG_STORE_MAP.get(config.getName()); } return configOperations.getSshServerIp(); } }