PropertyExpressionResolver.java

package com.thinkbiganalytics.feedmgr.nifi;

/*-
 * #%L
 * thinkbig-feed-manager-controller
 * %%
 * Copyright (C) 2017 ThinkBig Analytics
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */


import com.thinkbiganalytics.annotations.AnnotatedFieldProperty;
import com.thinkbiganalytics.feedmgr.MetadataFieldAnnotationFieldNameResolver;
import com.thinkbiganalytics.feedmgr.MetadataFields;
import com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata;
import com.thinkbiganalytics.metadata.MetadataField;
import com.thinkbiganalytics.nifi.feedmgr.ConfigurationPropertyReplacer;
import com.thinkbiganalytics.nifi.rest.model.NifiProperty;

import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.lang.text.StrLookup;
import org.apache.commons.lang.text.StrSubstitutor;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;

import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;

/**
 * Resolves the values for NiFi processor properties using the following logic:
 * <ol>
 * <li>Resolves {@code ${config.<NAME>}} to a property of the same name in the {@code application.properties} file.</li>
 * <li>Looks for a property in {@code application.properties} that matches {@code nifi.<PROCESSOR TYPE>.<PROPERTY KEY>} or {@code nifi.all_processors.<PROPERTY_KEY>}.</li>
 * <li>Resolves {@code ${metadata.<NAME>}} to a {@link MetadataField} property of the {@link FeedMetadata} class.</li>
 * </ol>
 *
 * <p>The {@code <PROCESSOR TYPE>} is the class name of the NiFi processor converted to lowercase. The {@code <PROPERTY KEY>} is the NiFi processor property key converted to lowercase and spaces
 * substituted with underscores.
 * See {@link ConfigurationPropertyReplacer} in the {@code nifi-rest-client} project for more information.</p>
 */
public class PropertyExpressionResolver {

    /**
     * Matches a variable in a property value
     */
    private static final Logger log = LoggerFactory.getLogger(PropertyExpressionResolver.class);

    private static final String VAR_PREFIX = "${";

    /**
     * Prefix for variable-type property replacement
     */
    public static String configPropertyPrefix = "config.";

    /**
     * Prefix for {@link FeedMetadata} property replacement
     */
    private static String metadataPropertyPrefix = MetadataFieldAnnotationFieldNameResolver.metadataPropertyPrefix;

    /**
     * Properties from the {@code application.properties} file
     */
    @Autowired
    private SpringEnvironmentProperties environmentProperties;

    /**
     * Resolves the values for all properties of the specified feed.
     *
     * @param metadata the feed
     * @return the modified properties
     */
    @Nonnull
    public List<NifiProperty> resolvePropertyExpressions(@Nullable final FeedMetadata metadata) {
        if (metadata != null) {
            return resolvePropertyExpressions(metadata.getProperties(), metadata);
        } else {
            return Collections.emptyList();
        }
    }

    public List<NifiProperty> resolvePropertyExpressions(List<NifiProperty> properties, @Nullable final FeedMetadata metadata) {
        if (metadata != null && properties != null) {
            return properties.stream()
                .filter(property -> resolveExpression(metadata, property))
                .collect(Collectors.toList());
        } else {
            return Collections.emptyList();
        }
    }

    public Map<String, Object> getStaticConfigProperties() {
        Map<String, Object> props = environmentProperties.getPropertiesStartingWith(configPropertyPrefix);
        if(props == null){
            props = new HashMap<>();
        }
        Map<String, Object> nifiProps = environmentProperties.getPropertiesStartingWith("nifi.");
        if (nifiProps != null && !nifiProps.isEmpty()) {
            //copy it to a new map
            props = new HashMap<>(props);
            props.putAll(nifiProps);
        }
        return props;
    }

    public List<AnnotatedFieldProperty> getMetadataProperties() {
        return MetadataFields.getInstance().getProperties(FeedMetadata.class);
    }

    /**
     * @return only those properties that were updated
     */
    public List<NifiProperty> resolveStaticProperties(@Nullable final List<NifiProperty> allProperties) {

        if (allProperties != null) {
            return allProperties.stream().filter(property -> resolveStaticConfigProperty(property).isModified).collect(Collectors.toList());
        } else {
            return Collections.emptyList();
        }
    }


    /**
     * Resolves the value of the specified property of the specified feed.
     *
     * @param metadata the feed
     * @param property the property
     * @return {@code true} if the property was modified, or {@code false} otherwise
     */
    boolean resolveExpression(@Nonnull final FeedMetadata metadata, @Nonnull final NifiProperty property) {
        final ResolveResult variableResult = resolveVariables(property, metadata);
        final ResolveResult staticConfigResult = (!variableResult.isFinal) ? resolveStaticConfigProperty(property) : new ResolveResult(false, false);
        return variableResult.isModified || staticConfigResult.isModified;
    }

    public boolean containsVariablesPatterns(String str) {
        return str.contains(VAR_PREFIX);
    }

    public static class ResolvedVariables {

        private Map<String, String> resolvedVariables;

        private String resolvedString;

        ResolvedVariables(String str) {
            this.resolvedString = str;
            this.resolvedVariables = new HashMap<>();
        }

        public Map<String, String> getResolvedVariables() {
            return resolvedVariables;
        }

        public String getResolvedString() {
            return resolvedString;
        }

        void setResolvedString(String resolvedString) {
            this.resolvedString = resolvedString;
        }
    }


    /**
     * Replace any property in the str  ${var} with the respective value in the map of vars
     */
    public String resolveVariables(String str, Map<String, String> vars) {
        StrSubstitutor ss = new StrSubstitutor(vars);
        return ss.replace(str);
    }

    public String replaceAll(String str, String replacement) {
        if (str != null) {
            StrLookup lookup = new StrLookup() {
                @Override
                public String lookup(String key) {
                    return replacement;
                }
            };
            StrSubstitutor ss = new StrSubstitutor(lookup);
            return ss.replace(str);
        }
        return null;
    }

    /**
     * Resolve the str with values from the supplied {@code properties} This will recursively fill out the str looking back at the properties to get the correct value. NOTE the property values will be
     * overwritten if replacement is found!
     */
    public ResolvedVariables resolveVariables(String str, List<NifiProperty> properties) {
        ResolvedVariables variables = new ResolvedVariables(str);

        StrLookup resolver = new StrLookup() {
            @Override
            public String lookup(String key) {
                Optional<NifiProperty> optional = properties.stream().filter(prop -> key.equals(prop.getKey())).findFirst();
                if (optional.isPresent()) {
                    NifiProperty property = optional.get();
                    String value = property.getValue().trim();
                    variables.getResolvedVariables().put(property.getKey(), value);
                    return value;
                } else {
                    return null;
                }
            }
        };

        StrSubstitutor ss = new StrSubstitutor(resolver);
        variables.setResolvedString(ss.replace(str));

        Map<String, String> map = variables.getResolvedVariables();
        Map<String, String> vars = map.entrySet().stream()
            .collect(Collectors.toMap(Map.Entry::getKey, entry -> ss.replace(entry.getValue())));
        variables.getResolvedVariables().putAll(vars);

        return variables;
    }

    private String getMetadataPropertyValue(FeedMetadata metadata, String variableName) throws Exception {
        String fieldPathName = StringUtils.substringAfter(variableName, metadataPropertyPrefix);
        Object obj = null;
        try {
            obj = BeanUtils.getProperty(metadata, fieldPathName);
        } catch (Exception e) {
            //    throw new RuntimeException(e);
        }
        //check to see if the path has a Metadata annotation with a matching field
        String matchingProperty = MetadataFields.getInstance().getMatchingPropertyDescriptor(metadata, variableName);
        if (obj == null && matchingProperty != null) {
            matchingProperty = StringUtils.substringAfter(matchingProperty, metadataPropertyPrefix);
            obj = BeanUtils.getProperty(metadata, matchingProperty);
        }
        if (obj != null) {
            return obj.toString();
        } else {
            return null;
        }
    }

    /**
     * Find a property in the env properties matching one of the following:
     * 1) nifi.<processorType>[<processorName>].<property key>
     * 2)    nifi.<processorType>.<property key>
     * 3) nifi.all_processors.<property key>
     * @param property the property
     * @param propertyKey a config. environment property to match on
     * @return
     */
    private String getConfigurationPropertyValue(NifiProperty property, String propertyKey) {
        if (StringUtils.isNotBlank(propertyKey) && propertyKey.startsWith(configPropertyPrefix)) {
            return ConfigurationPropertyReplacer.fixNiFiExpressionPropertyValue(environmentProperties.getPropertyValueAsString(propertyKey));
        } else {
            //see if the processorType is configured
            String processorTypeWithProcessorNameProperty = ConfigurationPropertyReplacer.getProcessorNamePropertyConfigName(property);
            String value = environmentProperties.getPropertyValueAsString(processorTypeWithProcessorNameProperty);
            if(StringUtils.isBlank(value)) {
                String processorTypeProperty = ConfigurationPropertyReplacer.getProcessorPropertyConfigName(property);
                value = environmentProperties.getPropertyValueAsString(processorTypeProperty);
                if (StringUtils.isBlank(value)) {
                    String globalPropertyType = ConfigurationPropertyReplacer.getGlobalAllProcessorsPropertyConfigName(property);
                    value = environmentProperties.getPropertyValueAsString(globalPropertyType);
                }
            }
            return ConfigurationPropertyReplacer.fixNiFiExpressionPropertyValue(value);
        }
    }



    /**
     * Resolves the value of the specified property using static configuration properties.
     *
     * @param property the property
     * @return the result of the transformation
     */
    private ResolveResult resolveStaticConfigProperty(@Nonnull final NifiProperty property) {
        final String processTypeAndProcessNameProperty = ConfigurationPropertyReplacer.getProcessorNamePropertyConfigName(property);
        final String processTypeAndProcessNamePropertyValue = environmentProperties.getPropertyValueAsString(processTypeAndProcessNameProperty);


        final String processorTypeProperty = ConfigurationPropertyReplacer.getProcessorPropertyConfigName(property);
        final String processorTypePropertyValue = environmentProperties.getPropertyValueAsString(processorTypeProperty);

        final String globalName = ConfigurationPropertyReplacer.getGlobalAllProcessorsPropertyConfigName(property);
        final String globalPropertyValue = environmentProperties.getPropertyValueAsString(globalName);

        //value first == processorTypeAndProcessNamePropertyValue, thne processorTypePropertyValue,  finally globalPropertyValue
        String value = StringUtils.isBlank(processTypeAndProcessNamePropertyValue) ?  ( StringUtils.isBlank(processorTypePropertyValue) ? globalPropertyValue : processorTypePropertyValue) : processTypeAndProcessNamePropertyValue;

        if (value != null) {
            value = ConfigurationPropertyReplacer.fixNiFiExpressionPropertyValue(value);
            property.setValue(value);
            return new ResolveResult(true, true);
        }
        return new ResolveResult(false, false);
    }


    /**
     * Resolves the variables in the value of the specified property.
     *
     * @param property the property
     * @param metadata the feed
     * @return the result of the transformation
     */
    private ResolveResult resolveVariables(@Nonnull final NifiProperty property, @Nonnull final FeedMetadata metadata) {
        // Filter blank values
        final String value = property.getValue();
        if (StringUtils.isBlank(value)) {
            return new ResolveResult(false, false);
        }

        final boolean[] hasConfig = {false};
        final boolean[] isModified = {false};

        StrLookup resolver = new StrLookup() {
            @Override
            public String lookup(String variable) {
                // Resolve configuration variables
                final String configValue = getConfigurationPropertyValue(property, variable);
                if (configValue != null) {
                    hasConfig[0] = true;
                    isModified[0] = true;
                    //if this is the first time we found the config var, set the template value correctly
                    if (!property.isContainsConfigurationVariables()) {
                        property.setTemplateValue(property.getValue());
                        property.setContainsConfigurationVariables(true);
                    }
                    return configValue;
                }

                // Resolve metadata variables
                try {
                    final String metadataValue = getMetadataPropertyValue(metadata, variable);
                    if (metadataValue != null) {
                        isModified[0] = true;
                        return metadataValue;
                    }
                } catch (Exception e) {
                    log.error("Unable to resolve variable: " + variable, e);
                }

                return null;
            }
        };
        StrSubstitutor ss = new StrSubstitutor(resolver);
        ss.setEnableSubstitutionInVariables(true);
        property.setValue(StringUtils.trim(ss.replace(value)));

        return new ResolveResult(hasConfig[0], isModified[0]);
    }

    /**
     * The result of resolving a NiFi property value.
     */
    private static class ResolveResult {

        /**
         * Indicates that the value should not be resolved further
         */
        final boolean isFinal;

        /**
         * Indicates that the value was modified
         */
        final boolean isModified;

        /**
         * Constructs a {@code ResultResult} with the specified attributes.
         *
         * @param isFinal    {@code true} if the value should not be resolved further, or {@code false} otherwise
         * @param isModified {@code true} if the value was modified, or {@code false} otherwise
         */
        ResolveResult(final boolean isFinal, final boolean isModified) {
            this.isFinal = isFinal;
            this.isModified = isModified;
        }
    }
}