/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.ambari.server.controller;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_COUNT;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_ON_UNAVAILABILITY;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMPONENT_CATEGORY;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.ObjectNotFoundException;
import org.apache.ambari.server.Role;
import org.apache.ambari.server.RoleCommand;
import org.apache.ambari.server.StackAccessException;
import org.apache.ambari.server.actionmanager.Stage;
import org.apache.ambari.server.actionmanager.TargetHostType;
import org.apache.ambari.server.agent.ExecutionCommand;
import org.apache.ambari.server.agent.ExecutionCommand.KeyNames;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.internal.RequestResourceFilter;
import org.apache.ambari.server.customactions.ActionDefinition;
import org.apache.ambari.server.orm.dao.ClusterVersionDAO;
import org.apache.ambari.server.orm.entities.ClusterVersionEntity;
import org.apache.ambari.server.orm.entities.OperatingSystemEntity;
import org.apache.ambari.server.orm.entities.RepositoryEntity;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.ComponentInfo;
import org.apache.ambari.server.state.ServiceComponentHost;
import org.apache.ambari.server.state.ServiceInfo;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.svccomphost.ServiceComponentHostOpInProgressEvent;
import org.apache.ambari.server.utils.SecretReference;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.inject.Inject;
import com.google.inject.Singleton;
/**
* Helper class containing logic to process custom action execution requests
*/
@Singleton
public class AmbariActionExecutionHelper {
private final static Logger LOG =
LoggerFactory.getLogger(AmbariActionExecutionHelper.class);
private static final String TYPE_PYTHON = "PYTHON";
private static final String ACTION_UPDATE_REPO = "update_repo";
private static final String SUCCESS_FACTOR_PARAMETER = "success_factor";
private static final float UPDATE_REPO_SUCCESS_FACTOR_DEFAULT = 0f;
@Inject
private Clusters clusters;
@Inject
private AmbariManagementController managementController;
@Inject
private AmbariMetaInfo ambariMetaInfo;
@Inject
private MaintenanceStateHelper maintenanceStateHelper;
@Inject
private Configuration configs;
@Inject
private ClusterVersionDAO clusterVersionDAO;
/**
* Validates the request to execute an action.
* @param actionRequest
* @throws AmbariException
*/
public void validateAction(ExecuteActionRequest actionRequest) throws AmbariException {
if (actionRequest.getActionName() == null || actionRequest.getActionName().isEmpty()) {
throw new AmbariException("Action name must be specified");
}
ActionDefinition actionDef = ambariMetaInfo.getActionDefinition(actionRequest.getActionName());
if (actionDef == null) {
throw new AmbariException("Action " + actionRequest.getActionName() + " does not exist");
}
if (actionDef.getInputs() != null) {
String[] inputs = actionDef.getInputs().split(",");
for (String input : inputs) {
String inputName = input.trim();
if (!inputName.isEmpty()) {
boolean mandatory = true;
if (inputName.startsWith("[") && inputName.endsWith("]")) {
mandatory = false;
}
if (mandatory && !actionRequest.getParameters().containsKey(inputName)) {
throw new AmbariException("Action " + actionRequest.getActionName() + " requires input '" +
input.trim() + "' that is not provided.");
}
}
}
}
List<RequestResourceFilter> resourceFilters = actionRequest.getResourceFilters();
RequestResourceFilter resourceFilter = null;
if (resourceFilters != null && !resourceFilters.isEmpty()) {
if (resourceFilters.size() > 1) {
throw new AmbariException("Custom action definition only allows one " +
"resource filter to be specified.");
} else {
resourceFilter = resourceFilters.get(0);
}
}
String targetService = "";
String targetComponent = "";
if (null != actionRequest.getClusterName()) {
Cluster cluster = clusters.getCluster(actionRequest.getClusterName());
if (cluster == null) {
throw new AmbariException("Unable to find cluster. clusterName = " +
actionRequest.getClusterName());
}
StackId stackId = cluster.getCurrentStackVersion();
String expectedService = actionDef.getTargetService() == null ? "" : actionDef.getTargetService();
String actualService = resourceFilter == null || resourceFilter.getServiceName() == null ? "" : resourceFilter.getServiceName();
if (!expectedService.isEmpty() && !actualService.isEmpty() && !expectedService.equals(actualService)) {
throw new AmbariException("Action " + actionRequest.getActionName() + " targets service " + actualService +
" that does not match with expected " + expectedService);
}
targetService = expectedService;
if (targetService == null || targetService.isEmpty()) {
targetService = actualService;
}
if (targetService != null && !targetService.isEmpty()) {
ServiceInfo serviceInfo;
try {
serviceInfo = ambariMetaInfo.getService(stackId.getStackName(), stackId.getStackVersion(),
targetService);
} catch (StackAccessException se) {
serviceInfo = null;
}
if (serviceInfo == null) {
throw new AmbariException("Action " + actionRequest.getActionName() +
" targets service " + targetService + " that does not exist.");
}
}
String expectedComponent = actionDef.getTargetComponent() == null ? "" : actionDef.getTargetComponent();
String actualComponent = resourceFilter == null || resourceFilter.getComponentName() == null ? "" : resourceFilter.getComponentName();
if (!expectedComponent.isEmpty() && !actualComponent.isEmpty() && !expectedComponent.equals(actualComponent)) {
throw new AmbariException("Action " + actionRequest.getActionName() + " targets component " + actualComponent +
" that does not match with expected " + expectedComponent);
}
targetComponent = expectedComponent;
if (targetComponent == null || targetComponent.isEmpty()) {
targetComponent = actualComponent;
}
if (!targetComponent.isEmpty() && targetService.isEmpty()) {
throw new AmbariException("Action " + actionRequest.getActionName() + " targets component " + targetComponent +
" without specifying the target service.");
}
if (targetComponent != null && !targetComponent.isEmpty()) {
ComponentInfo compInfo;
try {
compInfo = ambariMetaInfo.getComponent(stackId.getStackName(), stackId.getStackVersion(),
targetService, targetComponent);
} catch (StackAccessException se) {
compInfo = null;
}
if (compInfo == null) {
throw new AmbariException("Action " + actionRequest.getActionName() + " targets component " + targetComponent +
" that does not exist.");
}
}
}
TargetHostType targetHostType = actionDef.getTargetType();
if (TargetHostType.SPECIFIC.equals(targetHostType)
|| (targetService.isEmpty() && targetComponent.isEmpty())) {
if ((resourceFilter == null || resourceFilter.getHostNames().size() == 0) && !isTargetHostTypeAllowsEmptyHosts(targetHostType)) {
throw new AmbariException("Action " + actionRequest.getActionName() + " requires explicit target host(s)" +
" that is not provided.");
}
}
}
private boolean isTargetHostTypeAllowsEmptyHosts(TargetHostType targetHostType) {
return targetHostType.equals(TargetHostType.ALL) || targetHostType.equals(TargetHostType.ANY)
|| targetHostType.equals(TargetHostType.MAJORITY);
}
/**
* Add tasks to the stage based on the requested action execution
* @param actionContext the context associated with the action
* @param stage stage into which tasks must be inserted
* @param requestParams all request parameters (may be null)
* @throws AmbariException if the task can not be added
*/
public void addExecutionCommandsToStage(final ActionExecutionContext actionContext, Stage stage,
Map<String, String> requestParams) throws AmbariException {
addExecutionCommandsToStage(actionContext, stage, requestParams, true);
}
/**
* Add tasks to the stage based on the requested action execution
* @param actionContext
* @param stage
* @param requestParams
* @param checkHostIsMemberOfCluster if true AmbariException will be thrown in case host is not member of cluster.
* @throws AmbariException
*/
public void addExecutionCommandsToStage(final ActionExecutionContext actionContext, Stage stage,
Map<String, String> requestParams, boolean checkHostIsMemberOfCluster)
throws AmbariException {
String actionName = actionContext.getActionName();
String clusterName = actionContext.getClusterName();
final Cluster cluster;
if (null != clusterName) {
cluster = clusters.getCluster(clusterName);
} else {
cluster = null;
}
ComponentInfo componentInfo = null;
List<RequestResourceFilter> resourceFilters = actionContext.getResourceFilters();
final RequestResourceFilter resourceFilter;
if (resourceFilters != null && !resourceFilters.isEmpty()) {
resourceFilter = resourceFilters.get(0);
} else {
resourceFilter = new RequestResourceFilter();
}
// List of host to select from
Set<String> candidateHosts = new HashSet<>();
final String serviceName = actionContext.getExpectedServiceName();
final String componentName = actionContext.getExpectedComponentName();
LOG.debug("Called addExecutionCommandsToStage() for serviceName: {}, componentName: {}.", serviceName, componentName);
if (resourceFilter.getHostNames().isEmpty()) {
LOG.debug("Resource filter has no hostnames.");
} else {
LOG.debug("Resource filter has hosts: {}", StringUtils.join(resourceFilter.getHostNames(), ", "));
}
if (null != cluster) {
StackId stackId = cluster.getCurrentStackVersion();
if (serviceName != null && !serviceName.isEmpty()) {
if (componentName != null && !componentName.isEmpty()) {
Map<String, ServiceComponentHost> componentHosts =
cluster.getService(serviceName)
.getServiceComponent(componentName).getServiceComponentHosts();
candidateHosts.addAll(componentHosts.keySet());
try {
componentInfo = ambariMetaInfo.getComponent(stackId.getStackName(),
stackId.getStackVersion(), serviceName, componentName);
} catch (ObjectNotFoundException e) {
// do nothing, componentId is checked for null later
LOG.error("Did not find service {} and component {} in stack {}.", serviceName, componentName, stackId.getStackName());
}
} else {
for (String component : cluster.getService(serviceName).getServiceComponents().keySet()) {
Map<String, ServiceComponentHost> componentHosts =
cluster.getService(serviceName)
.getServiceComponent(component).getServiceComponentHosts();
candidateHosts.addAll(componentHosts.keySet());
}
}
} else {
// All hosts are valid target host
candidateHosts.addAll(clusters.getHostsForCluster(cluster.getClusterName()).keySet());
}
LOG.debug("Request for service {} and component {} is set to run on candidate hosts: {}.", serviceName, componentName, StringUtils.join(candidateHosts, ", "));
// Filter hosts that are in MS
Set<String> ignoredHosts = maintenanceStateHelper.filterHostsInMaintenanceState(
candidateHosts, new MaintenanceStateHelper.HostPredicate() {
@Override
public boolean shouldHostBeRemoved(final String hostname)
throws AmbariException {
return ! maintenanceStateHelper.isOperationAllowed(
cluster, actionContext.getOperationLevel(),
resourceFilter, serviceName, componentName, hostname);
}
}
);
if (! ignoredHosts.isEmpty()) {
LOG.debug("Hosts to ignore: {}.", StringUtils.join(ignoredHosts, ", "));
LOG.debug("Ignoring action for hosts due to maintenance state." +
"Ignored hosts =" + ignoredHosts + ", component="
+ componentName + ", service=" + serviceName
+ ", cluster=" + cluster.getClusterName() + ", " +
"actionName=" + actionContext.getActionName());
}
}
// If request did not specify hosts and there exists no host
if (resourceFilter.getHostNames().isEmpty() && candidateHosts.isEmpty()) {
throw new AmbariException("Suitable hosts not found, component="
+ componentName + ", service=" + serviceName
+ ((null == cluster) ? "" : ", cluster=" + cluster.getClusterName() + ", ")
+ "actionName=" + actionContext.getActionName());
}
if (checkHostIsMemberOfCluster) {
// Compare specified hosts to available hosts
if (!resourceFilter.getHostNames().isEmpty() && !candidateHosts.isEmpty()) {
for (String hostname : resourceFilter.getHostNames()) {
if (!candidateHosts.contains(hostname)) {
throw new AmbariException("Request specifies host " + hostname +
" but it is not a valid host based on the " +
"target service=" + serviceName + " and component=" + componentName);
}
}
}
}
List<String> targetHosts = resourceFilter.getHostNames();
//Find target hosts to execute
if (targetHosts.isEmpty()) {
TargetHostType hostType = actionContext.getTargetType();
switch (hostType) {
case ALL:
targetHosts.addAll(candidateHosts);
break;
case ANY:
targetHosts.add(managementController.getHealthyHost(candidateHosts));
break;
case MAJORITY:
for (int i = 0; i < (candidateHosts.size() / 2) + 1; i++) {
String hostname = managementController.getHealthyHost(candidateHosts);
targetHosts.add(hostname);
candidateHosts.remove(hostname);
}
break;
default:
throw new AmbariException("Unsupported target type = " + hostType);
}
}
setAdditionalParametersForStageAccordingToAction(stage, actionContext);
// create tasks for each host
for (String hostName : targetHosts) {
// ensure that any tags that need to be refreshed are extracted from the
// context and put onto the execution command
Map<String, String> actionParameters = actionContext.getParameters();
stage.addHostRoleExecutionCommand(hostName, Role.valueOf(actionContext.getActionName()),
RoleCommand.ACTIONEXECUTE,
new ServiceComponentHostOpInProgressEvent(actionContext.getActionName(), hostName,
System.currentTimeMillis()),
clusterName, serviceName, actionContext.isRetryAllowed(),
actionContext.isFailureAutoSkipped());
Map<String, String> commandParams = new TreeMap<>();
int taskTimeout = Integer.parseInt(configs.getDefaultAgentTaskTimeout(false));
// use the biggest of all these:
// if the action context timeout is bigger than the default, use the context
// if the action context timeout is smaller than the default, use the default
// if the action context timeout is undefined, use the default
if (null != actionContext.getTimeout() && actionContext.getTimeout() > taskTimeout) {
commandParams.put(COMMAND_TIMEOUT, actionContext.getTimeout().toString());
} else {
commandParams.put(COMMAND_TIMEOUT, Integer.toString(taskTimeout));
}
if (requestParams != null && requestParams.containsKey(KeyNames.LOG_OUTPUT)) {
LOG.info("Should command log output?: " + requestParams.get(KeyNames.LOG_OUTPUT));
commandParams.put(KeyNames.LOG_OUTPUT, requestParams.get(KeyNames.LOG_OUTPUT));
}
commandParams.put(SCRIPT, actionName + ".py");
commandParams.put(SCRIPT_TYPE, TYPE_PYTHON);
ExecutionCommand execCmd = stage.getExecutionCommandWrapper(hostName,
actionContext.getActionName()).getExecutionCommand();
// !!! ensure that these are empty so that commands have the correct tags
// applied when the execution is about to be scheduled to run
execCmd.setConfigurations(new TreeMap<String, Map<String, String>>());
execCmd.setConfigurationAttributes(new TreeMap<String, Map<String, Map<String, String>>>());
// if the command should fetch brand new configuration tags before
// execution, then we don't need to fetch them now
if (null != actionParameters && !actionParameters.isEmpty()) {
if (actionParameters.containsKey(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)) {
execCmd.setForceRefreshConfigTagsBeforeExecution(true);
}
}
// when building complex orchestration ahead of time (such as when
// performing ugprades), fetching configuration tags can take a very long
// time - if it's not needed, then don't do it
Map<String, Map<String, String>> configTags = new TreeMap<>();
if (!execCmd.getForceRefreshConfigTagsBeforeExecution()) {
configTags = managementController.findConfigurationTagsWithOverrides(cluster, hostName);
}
execCmd.setConfigurationTags(configTags);
execCmd.setServiceName(serviceName == null || serviceName.isEmpty() ?
resourceFilter.getServiceName() : serviceName);
execCmd.setComponentName(componentName == null || componentName.isEmpty() ?
resourceFilter.getComponentName() : componentName);
Map<String, String> hostLevelParams = execCmd.getHostLevelParams();
hostLevelParams.put(AGENT_STACK_RETRY_ON_UNAVAILABILITY, configs.isAgentStackRetryOnInstallEnabled());
hostLevelParams.put(AGENT_STACK_RETRY_COUNT, configs.getAgentStackRetryOnInstallCount());
for (Map.Entry<String, String> dbConnectorName : configs.getDatabaseConnectorNames().entrySet()) {
hostLevelParams.put(dbConnectorName.getKey(), dbConnectorName.getValue());
}
for (Map.Entry<String, String> previousDBConnectorName : configs.getPreviousDatabaseConnectorNames().entrySet()) {
hostLevelParams.put(previousDBConnectorName.getKey(), previousDBConnectorName.getValue());
}
addRepoInfoToHostLevelParams(cluster, hostLevelParams, hostName);
Map<String, String> roleParams = execCmd.getRoleParams();
if (roleParams == null) {
roleParams = new TreeMap<>();
}
roleParams.putAll(actionParameters);
SecretReference.replaceReferencesWithPasswords(roleParams, cluster);
if (componentInfo != null) {
roleParams.put(COMPONENT_CATEGORY, componentInfo.getCategory());
}
// if there is a stack upgrade which is currently suspended then pass that
// information down with the command as some components may need to know
if (null != cluster && cluster.isUpgradeSuspended()) {
cluster.addSuspendedUpgradeParameters(commandParams, roleParams);
}
execCmd.setCommandParams(commandParams);
execCmd.setRoleParams(roleParams);
if (null != cluster) {
// Generate localComponents
for (ServiceComponentHost sch : cluster.getServiceComponentHosts(hostName)) {
execCmd.getLocalComponents().add(sch.getServiceComponentName());
}
}
}
}
/*
* This method adds additional properties
* to action params. For example: success factor.
*
* */
private void setAdditionalParametersForStageAccordingToAction(Stage stage, ActionExecutionContext actionExecutionContext) throws AmbariException {
if (actionExecutionContext.getActionName().equals(ACTION_UPDATE_REPO)) {
Map<String, String> params = actionExecutionContext.getParameters();
float successFactor = UPDATE_REPO_SUCCESS_FACTOR_DEFAULT;
if (params != null && params.containsKey(SUCCESS_FACTOR_PARAMETER)) {
try{
successFactor = Float.valueOf(params.get(SUCCESS_FACTOR_PARAMETER));
} catch (Exception ex) {
throw new AmbariException("Failed to cast success_factor value to float!", ex.getCause());
}
}
stage.getSuccessFactors().put(Role.UPDATE_REPO, successFactor);
}
}
/*
* This method builds and adds repo info
* to hostLevelParams of action
*
* */
private void addRepoInfoToHostLevelParams(Cluster cluster, Map<String, String> hostLevelParams, String hostName) throws AmbariException {
if (null == cluster) {
return;
}
JsonObject rootJsonObject = new JsonObject();
JsonArray repositories = new JsonArray();
ClusterVersionEntity clusterVersionEntity = clusterVersionDAO.findByClusterAndStateCurrent(
cluster.getClusterName());
if (clusterVersionEntity != null && clusterVersionEntity.getRepositoryVersion() != null) {
String hostOsFamily = clusters.getHost(hostName).getOsFamily();
for (OperatingSystemEntity operatingSystemEntity : clusterVersionEntity.getRepositoryVersion().getOperatingSystems()) {
// ostype in OperatingSystemEntity it's os family. That should be fixed
// in OperatingSystemEntity.
if (operatingSystemEntity.getOsType().equals(hostOsFamily)) {
for (RepositoryEntity repositoryEntity : operatingSystemEntity.getRepositories()) {
JsonObject repositoryInfo = new JsonObject();
repositoryInfo.addProperty("base_url", repositoryEntity.getBaseUrl());
repositoryInfo.addProperty("repo_name", repositoryEntity.getName());
repositoryInfo.addProperty("repo_id", repositoryEntity.getRepositoryId());
repositories.add(repositoryInfo);
}
rootJsonObject.add("repositories", repositories);
}
}
}
hostLevelParams.put(REPO_INFO, rootJsonObject.toString());
StackId stackId = cluster.getCurrentStackVersion();
hostLevelParams.put(STACK_NAME, stackId.getStackName());
hostLevelParams.put(STACK_VERSION, stackId.getStackVersion());
}
}