// Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. package com.cloud.agent.resource.consoleproxy; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URL; import java.net.URLConnection; import java.util.HashMap; import java.util.Map; import java.util.Properties; import javax.naming.ConfigurationException; import org.apache.log4j.Logger; import com.google.gson.Gson; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import com.cloud.agent.Agent.ExitStatus; import com.cloud.agent.api.AgentControlAnswer; import com.cloud.agent.api.Answer; import com.cloud.agent.api.CheckHealthAnswer; import com.cloud.agent.api.CheckHealthCommand; import com.cloud.agent.api.Command; import com.cloud.agent.api.ConsoleAccessAuthenticationAnswer; import com.cloud.agent.api.ConsoleAccessAuthenticationCommand; import com.cloud.agent.api.ConsoleProxyLoadReportCommand; import com.cloud.agent.api.PingCommand; import com.cloud.agent.api.ReadyAnswer; import com.cloud.agent.api.ReadyCommand; import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.StartupProxyCommand; import com.cloud.agent.api.proxy.CheckConsoleProxyLoadCommand; import com.cloud.agent.api.proxy.ConsoleProxyLoadAnswer; import com.cloud.agent.api.proxy.StartConsoleProxyAgentHttpHandlerCommand; import com.cloud.agent.api.proxy.WatchConsoleProxyLoadCommand; import com.cloud.exception.AgentControlChannelException; import com.cloud.host.Host; import com.cloud.host.Host.Type; import com.cloud.resource.ServerResource; import com.cloud.resource.ServerResourceBase; import com.cloud.utils.NumbersUtil; import com.cloud.utils.net.NetUtils; import com.cloud.utils.script.Script; /** * * I don't want to introduce extra cross-cutting concerns into console proxy * process, as it involves configurations like zone/pod, agent auto self-upgrade * etc. I also don't want to introduce more module dependency issues into our * build system, cross-communication between this resource and console proxy * will be done through reflection. As a result, come out with following * solution to solve the problem of building a communication channel between * consoole proxy and management server. * * We will deploy an agent shell inside console proxy VM, and this agent shell * will launch current console proxy from within this special server resource, * through it console proxy can build a communication channel with management * server. * */ public class ConsoleProxyResource extends ServerResourceBase implements ServerResource { static final Logger s_logger = Logger.getLogger(ConsoleProxyResource.class); private final Properties _properties = new Properties(); private Thread _consoleProxyMain = null; long _proxyVmId; int _proxyPort; String _localgw; String _eth1ip; String _eth1mask; String _pubIp; @Override public Answer executeRequest(final Command cmd) { if (cmd instanceof CheckConsoleProxyLoadCommand) { return execute((CheckConsoleProxyLoadCommand)cmd); } else if (cmd instanceof WatchConsoleProxyLoadCommand) { return execute((WatchConsoleProxyLoadCommand)cmd); } else if (cmd instanceof ReadyCommand) { s_logger.info("Receive ReadyCommand, response with ReadyAnswer"); return new ReadyAnswer((ReadyCommand)cmd); } else if (cmd instanceof CheckHealthCommand) { return new CheckHealthAnswer((CheckHealthCommand)cmd, true); } else if (cmd instanceof StartConsoleProxyAgentHttpHandlerCommand) { return execute((StartConsoleProxyAgentHttpHandlerCommand)cmd); } else { return Answer.createUnsupportedCommandAnswer(cmd); } } private Answer execute(StartConsoleProxyAgentHttpHandlerCommand cmd) { s_logger.info("Invoke launchConsoleProxy() in responding to StartConsoleProxyAgentHttpHandlerCommand"); launchConsoleProxy(cmd.getKeystoreBits(), cmd.getKeystorePassword(), cmd.getEncryptorPassword()); return new Answer(cmd); } private void disableRpFilter() { try (FileWriter fstream = new FileWriter("/proc/sys/net/ipv4/conf/eth2/rp_filter"); BufferedWriter out = new BufferedWriter(fstream);) { out.write("0"); } catch (IOException e) { s_logger.warn("Unable to disable rp_filter"); } } protected Answer execute(final CheckConsoleProxyLoadCommand cmd) { return executeProxyLoadScan(cmd, cmd.getProxyVmId(), cmd.getProxyVmName(), cmd.getProxyManagementIp(), cmd.getProxyCmdPort()); } protected Answer execute(final WatchConsoleProxyLoadCommand cmd) { return executeProxyLoadScan(cmd, cmd.getProxyVmId(), cmd.getProxyVmName(), cmd.getProxyManagementIp(), cmd.getProxyCmdPort()); } private Answer executeProxyLoadScan(final Command cmd, final long proxyVmId, final String proxyVmName, final String proxyManagementIp, final int cmdPort) { String result = null; final StringBuffer sb = new StringBuffer(); sb.append("http://").append(proxyManagementIp).append(":" + cmdPort).append("/cmd/getstatus"); boolean success = true; try { final URL url = new URL(sb.toString()); final URLConnection conn = url.openConnection(); final InputStream is = conn.getInputStream(); final BufferedReader reader = new BufferedReader(new InputStreamReader(is,"UTF-8")); final StringBuilder sb2 = new StringBuilder(); String line = null; try { while ((line = reader.readLine()) != null) sb2.append(line + "\n"); result = sb2.toString(); } catch (final IOException e) { success = false; } finally { try { is.close(); } catch (final IOException e) { s_logger.warn("Exception when closing , console proxy address : " + proxyManagementIp); success = false; } } } catch (final IOException e) { s_logger.warn("Unable to open console proxy command port url, console proxy address : " + proxyManagementIp); success = false; } return new ConsoleProxyLoadAnswer(cmd, proxyVmId, proxyVmName, success, result); } @Override protected String getDefaultScriptsDir() { return null; } @Override public Type getType() { return Host.Type.ConsoleProxy; } @Override public synchronized StartupCommand[] initialize() { final StartupProxyCommand cmd = new StartupProxyCommand(); fillNetworkInformation(cmd); cmd.setProxyPort(_proxyPort); cmd.setProxyVmId(_proxyVmId); if (_pubIp != null) cmd.setPublicIpAddress(_pubIp); return new StartupCommand[] {cmd}; } @Override public void disconnected() { } @Override public PingCommand getCurrentStatus(long id) { return new PingCommand(Type.ConsoleProxy, id); } @Override public boolean configure(String name, Map<String, Object> params) throws ConfigurationException { _localgw = (String)params.get("localgw"); _eth1mask = (String)params.get("eth1mask"); _eth1ip = (String)params.get("eth1ip"); if (_eth1ip != null) { params.put("private.network.device", "eth1"); } else { s_logger.info("eth1ip parameter has not been configured, assuming that we are not inside a system vm"); } String eth2ip = (String)params.get("eth2ip"); if (eth2ip != null) { params.put("public.network.device", "eth2"); } else { s_logger.info("eth2ip parameter is not found, assuming that we are not inside a system vm"); } super.configure(name, params); for (Map.Entry<String, Object> entry : params.entrySet()) { _properties.put(entry.getKey(), entry.getValue()); } String value = (String)params.get("premium"); if (value != null && value.equals("premium")) _proxyPort = 443; else { value = (String)params.get("consoleproxy.httpListenPort"); _proxyPort = NumbersUtil.parseInt(value, 80); } value = (String)params.get("proxy_vm"); _proxyVmId = NumbersUtil.parseLong(value, 0); if (_localgw != null) { String mgmtHost = (String)params.get("host"); if (_eth1ip != null) { addRouteToInternalIpOrCidr(_localgw, _eth1ip, _eth1mask, mgmtHost); String internalDns1 = (String) params.get("internaldns1"); if (internalDns1 == null) { s_logger.warn("No DNS entry found during configuration of NfsSecondaryStorage"); } else { addRouteToInternalIpOrCidr(_localgw, _eth1ip, _eth1mask, internalDns1); } String internalDns2 = (String) params.get("internaldns2"); if (internalDns2 != null) { addRouteToInternalIpOrCidr(_localgw, _eth1ip, _eth1mask, internalDns2); } } } _pubIp = (String)params.get("public.ip"); value = (String)params.get("disable_rp_filter"); if (value != null && value.equalsIgnoreCase("true")) { disableRpFilter(); } if (s_logger.isInfoEnabled()) s_logger.info("Receive proxyVmId in ConsoleProxyResource configuration as " + _proxyVmId); return true; } private void addRouteToInternalIpOrCidr(String localgw, String eth1ip, String eth1mask, String destIpOrCidr) { s_logger.debug("addRouteToInternalIp: localgw=" + localgw + ", eth1ip=" + eth1ip + ", eth1mask=" + eth1mask + ",destIp=" + destIpOrCidr); if (destIpOrCidr == null) { s_logger.debug("addRouteToInternalIp: destIp is null"); return; } if (!NetUtils.isValidIp(destIpOrCidr) && !NetUtils.isValidCIDR(destIpOrCidr)) { s_logger.warn(" destIp is not a valid ip address or cidr destIp=" + destIpOrCidr); return; } boolean inSameSubnet = false; if (NetUtils.isValidIp(destIpOrCidr)) { if (eth1ip != null && eth1mask != null) { inSameSubnet = NetUtils.sameSubnet(eth1ip, destIpOrCidr, eth1mask); } else { s_logger.warn("addRouteToInternalIp: unable to determine same subnet: _eth1ip=" + eth1ip + ", dest ip=" + destIpOrCidr + ", _eth1mask=" + eth1mask); } } else { inSameSubnet = NetUtils.isNetworkAWithinNetworkB(destIpOrCidr, NetUtils.ipAndNetMaskToCidr(eth1ip, eth1mask)); } if (inSameSubnet) { s_logger.debug("addRouteToInternalIp: dest ip " + destIpOrCidr + " is in the same subnet as eth1 ip " + eth1ip); return; } Script command = new Script("/bin/bash", s_logger); command.add("-c"); command.add("ip route delete " + destIpOrCidr); command.execute(); command = new Script("/bin/bash", s_logger); command.add("-c"); command.add("ip route add " + destIpOrCidr + " via " + localgw); String result = command.execute(); if (result != null) { s_logger.warn("Error in configuring route to internal ip err=" + result); } else { s_logger.debug("addRouteToInternalIp: added route to internal ip=" + destIpOrCidr + " via " + localgw); } } @Override public String getName() { return _name; } private void launchConsoleProxy(final byte[] ksBits, final String ksPassword, final String encryptorPassword) { final Object resource = this; if (_consoleProxyMain == null) { _consoleProxyMain = new Thread(new ManagedContextRunnable() { @Override protected void runInContext() { try { Class<?> consoleProxyClazz = Class.forName("com.cloud.consoleproxy.ConsoleProxy"); try { s_logger.info("Invoke setEncryptorPassword(), ecnryptorPassword: " + encryptorPassword); Method methodSetup = consoleProxyClazz.getMethod("setEncryptorPassword", String.class); methodSetup.invoke(null, encryptorPassword); s_logger.info("Invoke startWithContext()"); Method method = consoleProxyClazz.getMethod("startWithContext", Properties.class, Object.class, byte[].class, String.class); method.invoke(null, _properties, resource, ksBits, ksPassword); } catch (SecurityException e) { s_logger.error("Unable to launch console proxy due to SecurityException", e); System.exit(ExitStatus.Error.value()); } catch (NoSuchMethodException e) { s_logger.error("Unable to launch console proxy due to NoSuchMethodException", e); System.exit(ExitStatus.Error.value()); } catch (IllegalArgumentException e) { s_logger.error("Unable to launch console proxy due to IllegalArgumentException", e); System.exit(ExitStatus.Error.value()); } catch (IllegalAccessException e) { s_logger.error("Unable to launch console proxy due to IllegalAccessException", e); System.exit(ExitStatus.Error.value()); } catch (InvocationTargetException e) { s_logger.error("Unable to launch console proxy due to InvocationTargetException " + e.getTargetException().toString(), e); System.exit(ExitStatus.Error.value()); } } catch (final ClassNotFoundException e) { s_logger.error("Unable to launch console proxy due to ClassNotFoundException"); System.exit(ExitStatus.Error.value()); } } }, "Console-Proxy-Main"); _consoleProxyMain.setDaemon(true); _consoleProxyMain.start(); } else { s_logger.info("com.cloud.consoleproxy.ConsoleProxy is already running"); try { Class<?> consoleProxyClazz = Class.forName("com.cloud.consoleproxy.ConsoleProxy"); Method methodSetup = consoleProxyClazz.getMethod("setEncryptorPassword", String.class); methodSetup.invoke(null, encryptorPassword); } catch (SecurityException e) { s_logger.error("Unable to launch console proxy due to SecurityException", e); System.exit(ExitStatus.Error.value()); } catch (NoSuchMethodException e) { s_logger.error("Unable to launch console proxy due to NoSuchMethodException", e); System.exit(ExitStatus.Error.value()); } catch (IllegalArgumentException e) { s_logger.error("Unable to launch console proxy due to IllegalArgumentException", e); System.exit(ExitStatus.Error.value()); } catch (IllegalAccessException e) { s_logger.error("Unable to launch console proxy due to IllegalAccessException", e); System.exit(ExitStatus.Error.value()); } catch (InvocationTargetException e) { s_logger.error("Unable to launch console proxy due to InvocationTargetException " + e.getTargetException().toString(), e); System.exit(ExitStatus.Error.value()); } catch (final ClassNotFoundException e) { s_logger.error("Unable to launch console proxy due to ClassNotFoundException", e); System.exit(ExitStatus.Error.value()); } } } public String authenticateConsoleAccess(String host, String port, String vmId, String sid, String ticket, Boolean isReauthentication) { ConsoleAccessAuthenticationCommand cmd = new ConsoleAccessAuthenticationCommand(host, port, vmId, sid, ticket); cmd.setReauthenticating(isReauthentication); ConsoleProxyAuthenticationResult result = new ConsoleProxyAuthenticationResult(); result.setSuccess(false); result.setReauthentication(isReauthentication); try { AgentControlAnswer answer = getAgentControl().sendRequest(cmd, 10000); if (answer != null) { ConsoleAccessAuthenticationAnswer authAnswer = (ConsoleAccessAuthenticationAnswer)answer; result.setSuccess(authAnswer.succeeded()); result.setHost(authAnswer.getHost()); result.setPort(authAnswer.getPort()); result.setTunnelUrl(authAnswer.getTunnelUrl()); result.setTunnelSession(authAnswer.getTunnelSession()); } else { s_logger.error("Authentication failed for vm: " + vmId + " with sid: " + sid); } } catch (AgentControlChannelException e) { s_logger.error("Unable to send out console access authentication request due to " + e.getMessage(), e); } return new Gson().toJson(result); } public void reportLoadInfo(String gsonLoadInfo) { ConsoleProxyLoadReportCommand cmd = new ConsoleProxyLoadReportCommand(_proxyVmId, gsonLoadInfo); try { getAgentControl().postRequest(cmd); if (s_logger.isDebugEnabled()) s_logger.debug("Report proxy load info, proxy : " + _proxyVmId + ", load: " + gsonLoadInfo); } catch (AgentControlChannelException e) { s_logger.error("Unable to send out load info due to " + e.getMessage(), e); } } public void ensureRoute(String address) { if (_localgw != null) { if (s_logger.isDebugEnabled()) s_logger.debug("Ensure route for " + address + " via " + _localgw); // this method won't be called in high frequency, serialize access // to script execution synchronized (this) { try { addRouteToInternalIpOrCidr(_localgw, _eth1ip, _eth1mask, address); } catch (Throwable e) { s_logger.warn("Unexpected exception while adding internal route to " + address, e); } } } } @Override public void setName(String name) { } @Override public void setConfigParams(Map<String, Object> params) { } @Override public Map<String, Object> getConfigParams() { return new HashMap<String, Object>(); } @Override public int getRunLevel() { return 0; } @Override public void setRunLevel(int level) { } }