/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.ambari.server.api.services.stackadvisor.commands;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.lang.reflect.ParameterizedType;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import org.apache.ambari.server.api.resources.ResourceInstance;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.api.services.BaseService;
import org.apache.ambari.server.api.services.LocalUriInfo;
import org.apache.ambari.server.api.services.Request;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorResponse;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner;
import org.apache.ambari.server.controller.spi.Resource;
import org.apache.ambari.server.state.ServiceInfo;
import org.apache.ambari.server.utils.DateUtils;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.map.SerializationConfig;
import org.codehaus.jackson.node.ArrayNode;
import org.codehaus.jackson.node.ObjectNode;
import org.codehaus.jackson.node.TextNode;
/**
* Parent for all commands.
*/
public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extends BaseService {
/**
* Type of response object provided by extending classes when
* {@link #invoke(StackAdvisorRequest)} is called.
*/
private Class<T> type;
protected static Log LOG = LogFactory.getLog(StackAdvisorCommand.class);
private static final String GET_HOSTS_INFO_URI = "/api/v1/hosts"
+ "?fields=Hosts/*&Hosts/host_name.in(%s)";
private static final String GET_SERVICES_INFO_URI = "/api/v1/stacks/%s/versions/%s/"
+ "?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version"
+ ",services/StackServices/service_name,services/StackServices/service_version"
+ ",services/components/StackServiceComponents,services/components/dependencies/Dependencies/scope"
+ ",services/components/dependencies/Dependencies/conditions,services/components/auto_deploy"
+ ",services/configurations/StackConfigurations/property_depends_on"
+ ",services/configurations/dependencies/StackConfigurationDependency/dependency_name"
+ ",services/configurations/dependencies/StackConfigurationDependency/dependency_type,services/configurations/StackConfigurations/type"
+ "&services/StackServices/service_name.in(%s)";
private static final String SERVICES_PROPERTY = "services";
private static final String SERVICES_COMPONENTS_PROPERTY = "components";
private static final String CONFIG_GROUPS_PROPERTY = "config-groups";
private static final String STACK_SERVICES_PROPERTY = "StackServices";
private static final String COMPONENT_INFO_PROPERTY = "StackServiceComponents";
private static final String COMPONENT_NAME_PROPERTY = "component_name";
private static final String COMPONENT_HOSTNAMES_PROPERTY = "hostnames";
private static final String CONFIGURATIONS_PROPERTY = "configurations";
private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed-configurations";
private static final String USER_CONTEXT_PROPERTY = "user-context";
private static final String AMBARI_SERVER_CONFIGURATIONS_PROPERTY = "ambari-server-properties";
private File recommendationsDir;
private String recommendationsArtifactsLifetime;
private String stackAdvisorScript;
private int requestId;
private File requestDirectory;
private StackAdvisorRunner saRunner;
protected ObjectMapper mapper;
private final AmbariMetaInfo metaInfo;
@SuppressWarnings("unchecked")
public StackAdvisorCommand(File recommendationsDir, String recommendationsArtifactsLifetime, String stackAdvisorScript, int requestId,
StackAdvisorRunner saRunner, AmbariMetaInfo metaInfo) {
this.type = (Class<T>) ((ParameterizedType) getClass().getGenericSuperclass())
.getActualTypeArguments()[0];
this.mapper = new ObjectMapper();
this.mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
this.recommendationsDir = recommendationsDir;
this.recommendationsArtifactsLifetime = recommendationsArtifactsLifetime;
this.stackAdvisorScript = stackAdvisorScript;
this.requestId = requestId;
this.saRunner = saRunner;
this.metaInfo = metaInfo;
}
protected abstract StackAdvisorCommandType getCommandType();
/**
* Simple holder for 'hosts.json' and 'services.json' data.
*/
public static class StackAdvisorData {
protected String hostsJSON;
protected String servicesJSON;
public StackAdvisorData(String hostsJSON, String servicesJSON) {
this.hostsJSON = hostsJSON;
this.servicesJSON = servicesJSON;
}
}
/**
* Name with the result JSON, e.g. "component-layout.json" or
* "validations.json" .
*
* @return the file name
*/
protected abstract String getResultFileName();
protected abstract void validate(StackAdvisorRequest request) throws StackAdvisorException;
protected StackAdvisorData adjust(StackAdvisorData data, StackAdvisorRequest request) {
try {
ObjectNode root = (ObjectNode) this.mapper.readTree(data.servicesJSON);
populateStackHierarchy(root);
populateComponentHostsMap(root, request.getComponentHostsMap());
populateServiceAdvisors(root);
populateConfigurations(root, request);
populateConfigGroups(root, request);
populateAmbariServerInfo(root);
data.servicesJSON = mapper.writeValueAsString(root);
} catch (Exception e) {
// should not happen
String message = "Error parsing services.json file content: " + e.getMessage();
LOG.warn(message, e);
throw new WebApplicationException(Response.status(Status.BAD_REQUEST).entity(message).build());
}
return data;
}
protected void populateAmbariServerInfo(ObjectNode root) throws StackAdvisorException {
Map<String, String> serverProperties = metaInfo.getAmbariServerProperties();
if (serverProperties != null && !serverProperties.isEmpty()) {
JsonNode serverPropertiesNode = mapper.convertValue(serverProperties, JsonNode.class);
root.put(AMBARI_SERVER_CONFIGURATIONS_PROPERTY, serverPropertiesNode);
}
}
private void populateConfigurations(ObjectNode root,
StackAdvisorRequest request) {
Map<String, Map<String, Map<String, String>>> configurations =
request.getConfigurations();
ObjectNode configurationsNode = root.putObject(CONFIGURATIONS_PROPERTY);
for (String siteName : configurations.keySet()) {
ObjectNode siteNode = configurationsNode.putObject(siteName);
Map<String, Map<String, String>> siteMap = configurations.get(siteName);
for (String properties : siteMap.keySet()) {
ObjectNode propertiesNode = siteNode.putObject(properties);
Map<String, String> propertiesMap = siteMap.get(properties);
for (String propertyName : propertiesMap.keySet()) {
String propertyValue = propertiesMap.get(propertyName);
propertiesNode.put(propertyName, propertyValue);
}
}
}
JsonNode changedConfigs = mapper.valueToTree(request.getChangedConfigurations());
root.put(CHANGED_CONFIGURATIONS_PROPERTY, changedConfigs);
JsonNode userContext = mapper.valueToTree(request.getUserContext());
root.put(USER_CONTEXT_PROPERTY, userContext);
}
private void populateConfigGroups(ObjectNode root,
StackAdvisorRequest request) {
if (request.getConfigGroups() != null &&
!request.getConfigGroups().isEmpty()) {
JsonNode configGroups = mapper.valueToTree(request.getConfigGroups());
root.put(CONFIG_GROUPS_PROPERTY, configGroups);
}
}
protected void populateStackHierarchy(ObjectNode root) {
ObjectNode version = (ObjectNode) root.get("Versions");
TextNode stackName = (TextNode) version.get("stack_name");
TextNode stackVersion = (TextNode) version.get("stack_version");
ObjectNode stackHierarchy = version.putObject("stack_hierarchy");
stackHierarchy.put("stack_name", stackName);
ArrayNode parents = stackHierarchy.putArray("stack_versions");
for (String parentVersion : metaInfo.getStackParentVersions(stackName.asText(), stackVersion.asText())) {
parents.add(parentVersion);
}
}
private void populateComponentHostsMap(ObjectNode root, Map<String, Set<String>> componentHostsMap) {
ArrayNode services = (ArrayNode) root.get(SERVICES_PROPERTY);
Iterator<JsonNode> servicesIter = services.getElements();
while (servicesIter.hasNext()) {
JsonNode service = servicesIter.next();
ArrayNode components = (ArrayNode) service.get(SERVICES_COMPONENTS_PROPERTY);
Iterator<JsonNode> componentsIter = components.getElements();
while (componentsIter.hasNext()) {
JsonNode component = componentsIter.next();
ObjectNode componentInfo = (ObjectNode) component.get(COMPONENT_INFO_PROPERTY);
String componentName = componentInfo.get(COMPONENT_NAME_PROPERTY).getTextValue();
Set<String> componentHosts = componentHostsMap.get(componentName);
ArrayNode hostnames = componentInfo.putArray(COMPONENT_HOSTNAMES_PROPERTY);
if (null != componentHosts) {
for (String hostName : componentHosts) {
hostnames.add(hostName);
}
}
}
}
}
private void populateServiceAdvisors(ObjectNode root) {
ArrayNode services = (ArrayNode) root.get(SERVICES_PROPERTY);
Iterator<JsonNode> servicesIter = services.getElements();
ObjectNode version = (ObjectNode) root.get("Versions");
String stackName = version.get("stack_name").asText();
String stackVersion = version.get("stack_version").asText();
while (servicesIter.hasNext()) {
JsonNode service = servicesIter.next();
ObjectNode serviceVersion = (ObjectNode) service.get(STACK_SERVICES_PROPERTY);
String serviceName = serviceVersion.get("service_name").getTextValue();
try {
ServiceInfo serviceInfo = metaInfo.getService(stackName, stackVersion, serviceName);
if (serviceInfo.getAdvisorFile() != null) {
serviceVersion.put("advisor_name", serviceInfo.getAdvisorName());
serviceVersion.put("advisor_path", serviceInfo.getAdvisorFile().getAbsolutePath());
}
}
catch (Exception e) {
LOG.error("Error adding service advisor information to services.json", e);
}
}
}
public synchronized T invoke(StackAdvisorRequest request) throws StackAdvisorException {
validate(request);
String hostsJSON = getHostsInformation(request);
String servicesJSON = getServicesInformation(request);
StackAdvisorData adjusted = adjust(new StackAdvisorData(hostsJSON, servicesJSON), request);
try {
createRequestDirectory();
FileUtils.writeStringToFile(new File(requestDirectory, "hosts.json"), adjusted.hostsJSON);
FileUtils.writeStringToFile(new File(requestDirectory, "services.json"),
adjusted.servicesJSON);
saRunner.runScript(stackAdvisorScript, getCommandType(), requestDirectory);
String result = FileUtils.readFileToString(new File(requestDirectory, getResultFileName()));
T response = this.mapper.readValue(result, this.type);
return updateResponse(request, setRequestId(response));
} catch (StackAdvisorException ex) {
throw ex;
} catch (Exception e) {
String message = "Error occured during stack advisor command invocation: ";
LOG.warn(message, e);
throw new StackAdvisorException(message + e.getMessage());
}
}
protected abstract T updateResponse(StackAdvisorRequest request, T response);
private T setRequestId(T response) {
response.setId(requestId);
return response;
}
/**
* Create request id directory for each call
*/
private void createRequestDirectory() throws IOException {
if (!recommendationsDir.exists()) {
if (!recommendationsDir.mkdirs()) {
throw new IOException("Cannot create " + recommendationsDir);
}
}
cleanupRequestDirectory();
requestDirectory = new File(recommendationsDir, Integer.toString(requestId));
if (requestDirectory.exists()) {
FileUtils.deleteDirectory(requestDirectory);
}
if (!requestDirectory.mkdirs()) {
throw new IOException("Cannot create " + requestDirectory);
}
}
/**
* Deletes folders older than (now - recommendationsArtifactsLifetime)
*/
private void cleanupRequestDirectory() throws IOException {
final Date cutoffDate = DateUtils.getDateSpecifiedTimeAgo(recommendationsArtifactsLifetime); // subdirectories older than this date will be deleted
String[] oldDirectories = recommendationsDir.list(new FilenameFilter() {
@Override
public boolean accept(File current, String name) {
File file = new File(current, name);
return file.isDirectory() && !FileUtils.isFileNewer(file, cutoffDate);
}
});
if(oldDirectories.length > 0) {
LOG.info(String.format("Deleting old directories %s from %s", StringUtils.join(oldDirectories, ", "), recommendationsDir));
}
for(String oldDirectory:oldDirectories) {
FileUtils.deleteDirectory(new File(recommendationsDir, oldDirectory));
}
}
String getHostsInformation(StackAdvisorRequest request) throws StackAdvisorException {
String hostsURI = String.format(GET_HOSTS_INFO_URI, request.getHostsCommaSeparated());
Response response = handleRequest(null, null, new LocalUriInfo(hostsURI), Request.Type.GET,
createHostResource());
if (response.getStatus() != Status.OK.getStatusCode()) {
String message = String.format(
"Error occured during hosts information retrieving, status=%s, response=%s",
response.getStatus(), (String) response.getEntity());
LOG.warn(message);
throw new StackAdvisorException(message);
}
String hostsJSON = (String) response.getEntity();
if (LOG.isDebugEnabled()) {
LOG.debug("Hosts information: " + hostsJSON);
}
Collection<String> unregistered = getUnregisteredHosts(hostsJSON, request.getHosts());
if (unregistered.size() > 0) {
String message = String.format("There are unregistered hosts in the request, %s",
Arrays.toString(unregistered.toArray()));
LOG.warn(message);
throw new StackAdvisorException(message);
}
return hostsJSON;
}
@SuppressWarnings("unchecked")
private Collection<String> getUnregisteredHosts(String hostsJSON, List<String> hosts)
throws StackAdvisorException {
ObjectMapper mapper = new ObjectMapper();
List<String> registeredHosts = new ArrayList<>();
try {
JsonNode root = mapper.readTree(hostsJSON);
Iterator<JsonNode> iterator = root.get("items").getElements();
while (iterator.hasNext()) {
JsonNode next = iterator.next();
String hostName = next.get("Hosts").get("host_name").getTextValue();
registeredHosts.add(hostName);
}
return CollectionUtils.subtract(hosts, registeredHosts);
} catch (Exception e) {
throw new StackAdvisorException("Error occured during calculating unregistered hosts", e);
}
}
String getServicesInformation(StackAdvisorRequest request) throws StackAdvisorException {
String stackName = request.getStackName();
String stackVersion = request.getStackVersion();
String servicesURI = String.format(GET_SERVICES_INFO_URI, stackName, stackVersion,
request.getServicesCommaSeparated());
Response response = handleRequest(null, null, new LocalUriInfo(servicesURI),
Request.Type.GET, createStackVersionResource(stackName, stackVersion));
if (response.getStatus() != Status.OK.getStatusCode()) {
String message = String.format(
"Error occured during services information retrieving, status=%s, response=%s",
response.getStatus(), (String) response.getEntity());
LOG.warn(message);
throw new StackAdvisorException(message);
}
String servicesJSON = (String) response.getEntity();
if (LOG.isDebugEnabled()) {
LOG.debug("Services information: " + servicesJSON);
}
return servicesJSON;
}
private ResourceInstance createHostResource() {
Map<Resource.Type, String> mapIds = new HashMap<>();
return createResource(Resource.Type.Host, mapIds);
}
private ResourceInstance createStackVersionResource(String stackName, String stackVersion) {
Map<Resource.Type, String> mapIds = new HashMap<>();
mapIds.put(Resource.Type.Stack, stackName);
mapIds.put(Resource.Type.StackVersion, stackVersion);
return createResource(Resource.Type.StackVersion, mapIds);
}
}