/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.ambari.server.topology.validators;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import org.apache.ambari.server.controller.internal.Stack;
import org.apache.ambari.server.state.PropertyInfo;
import org.apache.ambari.server.topology.Blueprint;
import org.apache.ambari.server.topology.ClusterTopology;
import org.apache.ambari.server.topology.HostGroup;
import org.apache.ambari.server.topology.InvalidTopologyException;
import org.apache.ambari.server.topology.TopologyValidator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Validates the configuration by checking the existence of required properties for the services listed in the blueprint.
* Required properties are specified in the stack and are tied to config types and services.
*
* The validator ignores password properties that should never be specified in the artifacts (blueprint / cluster creation template)
*/
public class RequiredConfigPropertiesValidator implements TopologyValidator {
private static final Logger LOGGER = LoggerFactory.getLogger(RequiredConfigPropertiesValidator.class);
/**
* Validates the configuration coming from the blueprint and cluster creation template and ensures that all the required properties are provided.
* It's expected, that a in hostrgroup containing components for a given service all required configuration for the given service is available.
*
* @param topology the topology instance holding the configuration for cluster provisioning
* @throws InvalidTopologyException when there are missing configuration types or properties related to services in the blueprint
*/
@Override
public void validate(ClusterTopology topology) throws InvalidTopologyException {
// collect required properties
Map<String, Map<String, Collection<String>>> requiredPropertiesByService = getRequiredPropertiesByService(topology.getBlueprint());
// find missing properties in the cluster configuration
Map<String, Collection<String>> missingProperties = new HashMap<>();
Map<String, Map<String, String>> topologyConfiguration = new HashMap<>(topology.getConfiguration().getFullProperties(1));
for (HostGroup hostGroup : topology.getBlueprint().getHostGroups().values()) {
LOGGER.debug("Processing hostgroup configurations for hostgroup: {}", hostGroup.getName());
// copy of all configurations available in the topology hgConfig -> topologyConfig -> bpConfig
Map<String, Map<String, String>> operationalConfigurations = new HashMap<>(topologyConfiguration);
for (Map.Entry<String, Map<String, String>> hostgroupConfigEntry : hostGroup.getConfiguration().getProperties().entrySet()) {
if (operationalConfigurations.containsKey(hostgroupConfigEntry.getKey())) {
operationalConfigurations.get(hostgroupConfigEntry.getKey()).putAll(hostgroupConfigEntry.getValue());
} else {
operationalConfigurations.put(hostgroupConfigEntry.getKey(), hostgroupConfigEntry.getValue());
}
}
for (String hostGroupService : hostGroup.getServices()) {
if (!requiredPropertiesByService.containsKey(hostGroupService)) {
// there are no required properties for the service
LOGGER.debug("There are no required properties found for hostgroup/service: [{}/{}]", hostGroup.getName(), hostGroupService);
continue;
}
Map<String, Collection<String>> requiredPropertiesByType = requiredPropertiesByService.get(hostGroupService);
for (String configType : requiredPropertiesByType.keySet()) {
// We need a copy not to modify the original
Collection<String> requiredPropertiesForType = new HashSet(requiredPropertiesByType.get(configType));
if (!operationalConfigurations.containsKey(configType)) {
// all required configuration is missing for the config type
missingProperties = addTomissingProperties(missingProperties, hostGroup.getName(), requiredPropertiesForType);
continue;
}
Collection<String> operationalConfigsForType = operationalConfigurations.get(configType).keySet();
requiredPropertiesForType.removeAll(operationalConfigsForType);
if (!requiredPropertiesForType.isEmpty()) {
LOGGER.info("Found missing properties in hostgroup: {}, config type: {}, mising properties: {}", hostGroup.getName(),
configType, requiredPropertiesForType);
missingProperties = addTomissingProperties(missingProperties, hostGroup.getName(), requiredPropertiesForType);
}
}
}
}
if (!missingProperties.isEmpty()) {
throw new InvalidTopologyException("Missing required properties. Specify a value for these " +
"properties in the blueprint or cluster creation template configuration. " + missingProperties);
}
}
/**
* Collects required properties for services in the blueprint. Configuration properties are returned by configuration type.
* service -> configType -> properties
*
* @param blueprint the blueprint from the cluster topology
* @return a map with configuration types mapped to collections of required property names
*/
private Map<String, Map<String, Collection<String>>> getRequiredPropertiesByService(Blueprint blueprint) {
Map<String, Map<String, Collection<String>>> requiredPropertiesForServiceByType = new HashMap<>();
for (String bpService : blueprint.getServices()) {
LOGGER.debug("Collecting required properties for the service: {}", bpService);
Collection<Stack.ConfigProperty> requiredConfigsForService = blueprint.getStack().getRequiredConfigurationProperties(bpService);
Map<String, Collection<String>> requiredPropertiesByConfigType = new HashMap<>();
for (Stack.ConfigProperty configProperty : requiredConfigsForService) {
if (configProperty.getPropertyTypes() != null && configProperty.getPropertyTypes().contains(PropertyInfo.PropertyType.PASSWORD)) {
LOGGER.debug("Skipping required property validation for password type: {}", configProperty.getName());
// skip password types
continue;
}
// add / get service related required propeByType map
if (requiredPropertiesForServiceByType.containsKey(bpService)) {
requiredPropertiesByConfigType = requiredPropertiesForServiceByType.get(bpService);
} else {
LOGGER.debug("Adding required properties entry for service: {}", bpService);
requiredPropertiesForServiceByType.put(bpService, requiredPropertiesByConfigType);
}
// add collection of required properties
Collection<String> requiredPropsForType = new HashSet<>();
if (requiredPropertiesByConfigType.containsKey(configProperty.getType())) {
requiredPropsForType = requiredPropertiesByConfigType.get(configProperty.getType());
} else {
LOGGER.debug("Adding required properties entry for configuration type: {}", configProperty.getType());
requiredPropertiesByConfigType.put(configProperty.getType(), requiredPropsForType);
}
requiredPropsForType.add(configProperty.getName());
LOGGER.debug("Added required property for service; {}, configuration type: {}, property: {}", bpService,
configProperty.getType(), configProperty.getName());
}
}
LOGGER.info("Identified required properties for blueprint services: {}", requiredPropertiesForServiceByType);
return requiredPropertiesForServiceByType;
}
private Map<String, Collection<String>> addTomissingProperties(Map<String, Collection<String>> missingProperties, String hostGroup, Collection<String> values) {
Map<String, Collection<String>> missing;
if (missingProperties == null) {
missing = new HashMap<>();
} else {
missing = new HashMap<>(missingProperties);
}
if (!missing.containsKey(hostGroup)) {
missing.put(hostGroup, new HashSet<String>());
}
missing.get(hostGroup).addAll(values);
return missing;
}
}