PropsUtils.java

/*
 * Copyright 2012 LinkedIn Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package azkaban.utils;

import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.jexl2.Expression;
import org.apache.commons.jexl2.JexlEngine;
import org.apache.commons.jexl2.JexlException;
import org.apache.commons.jexl2.MapContext;
import org.apache.commons.lang.StringUtils;

import org.apache.log4j.Logger;
import org.joda.time.DateTime;

import com.google.common.collect.Maps;
import com.google.common.collect.MapDifference;

import azkaban.executor.ExecutableFlowBase;
import azkaban.flow.CommonJobProperties;

public class PropsUtils {

  private static final Logger logger = Logger.getLogger(PropsUtils.class);
  /**
   * Load job schedules from the given directories ] * @param dir The directory
   * to look in
   *
   * @param suffixes File suffixes to load
   * @return The loaded set of schedules
   */
  public static Props loadPropsInDir(File dir, String... suffixes) {
    return loadPropsInDir(null, dir, suffixes);
  }

  /**
   * Load job schedules from the given directories
   *
   * @param parent The parent properties for these properties
   * @param dir The directory to look in
   * @param suffixes File suffixes to load
   * @return The loaded set of schedules
   */
  public static Props loadPropsInDir(Props parent, File dir, String... suffixes) {
    try {
      Props props = new Props(parent);
      File[] files = dir.listFiles();
      Arrays.sort(files);
      if (files != null) {
        for (File f : files) {
          if (f.isFile() && endsWith(f, suffixes)) {
            props.putAll(new Props(null, f.getAbsolutePath()));
          }
        }
      }
      return props;
    } catch (IOException e) {
      throw new RuntimeException("Error loading properties.", e);
    }
  }

  public static Props loadProps(Props parent, File... propFiles) {
    try {
      Props props = new Props(parent);
      for (File f : propFiles) {
        if (f.isFile()) {
          props = new Props(props, f);
        }
      }

      return props;
    } catch (IOException e) {
      throw new RuntimeException("Error loading properties.", e);
    }
  }

  /**
   * Load job schedules from the given directories
   *
   * @param dirs The directories to check for properties
   * @param suffixes The suffixes to load
   * @return The properties
   */
  public static Props loadPropsInDirs(List<File> dirs, String... suffixes) {
    Props props = new Props();
    for (File dir : dirs) {
      props.putLocal(loadPropsInDir(dir, suffixes));
    }
    return props;
  }

  /**
   * Load properties from the given path
   *
   * @param jobPath The path to load from
   * @param props The parent properties for loaded properties
   * @param suffixes The suffixes of files to load
   */
  public static void loadPropsBySuffix(File jobPath, Props props,
      String... suffixes) {
    try {
      if (jobPath.isDirectory()) {
        File[] files = jobPath.listFiles();
        if (files != null) {
          for (File file : files)
            loadPropsBySuffix(file, props, suffixes);
        }
      } else if (endsWith(jobPath, suffixes)) {
        props.putAll(new Props(null, jobPath.getAbsolutePath()));
      }
    } catch (IOException e) {
      throw new RuntimeException("Error loading schedule properties.", e);
    }
  }

  public static boolean endsWith(File file, String... suffixes) {
    for (String suffix : suffixes)
      if (file.getName().endsWith(suffix))
        return true;
    return false;
  }

  private static final Pattern VARIABLE_REPLACEMENT_PATTERN = Pattern
      .compile("\\$\\{([a-zA-Z_.0-9]+)\\}");

  public static boolean isVarialbeReplacementPattern(String str) {
    Matcher matcher = VARIABLE_REPLACEMENT_PATTERN.matcher(str);
    return matcher.matches();
  }

  public static Props resolveProps(Props props) {
    if (props == null)
      return null;

    Props resolvedProps = new Props();

    LinkedHashSet<String> visitedVariables = new LinkedHashSet<String>();
    for (String key : props.getKeySet()) {
      String value = props.get(key);

      visitedVariables.add(key);
      String replacedValue =
          resolveVariableReplacement(value, props, visitedVariables);
      visitedVariables.clear();

      resolvedProps.put(key, replacedValue);
    }

    for (String key : resolvedProps.getKeySet()) {
      String value = resolvedProps.get(key);
      String expressedValue = resolveVariableExpression(value);
      resolvedProps.put(key, expressedValue);
    }

    return resolvedProps;
  };

  private static String resolveVariableReplacement(String value, Props props,
      LinkedHashSet<String> visitedVariables) {
    StringBuffer buffer = new StringBuffer();
    int startIndex = 0;

    Matcher matcher = VARIABLE_REPLACEMENT_PATTERN.matcher(value);
    while (matcher.find(startIndex)) {
      if (startIndex < matcher.start()) {
        // Copy everything up front to the buffer
        buffer.append(value.substring(startIndex, matcher.start()));
      }

      String subVariable = matcher.group(1);
      // Detected a cycle
      if (visitedVariables.contains(subVariable)) {
        throw new IllegalArgumentException(String.format(
            "Circular variable substitution found: [%s] -> [%s]",
            StringUtils.join(visitedVariables, "->"), subVariable));
      } else {
        // Add substitute variable and recurse.
        String replacement = props.get(subVariable);
        visitedVariables.add(subVariable);

        if (replacement == null) {
          throw new UndefinedPropertyException(String.format(
              "Could not find variable substitution for variable(s) [%s]",
              StringUtils.join(visitedVariables, "->")));
        }

        buffer.append(resolveVariableReplacement(replacement, props,
            visitedVariables));
        visitedVariables.remove(subVariable);
      }

      startIndex = matcher.end();
    }

    if (startIndex < value.length()) {
      buffer.append(value.substring(startIndex));
    }

    return buffer.toString();
  }

  private static String resolveVariableExpression(String value) {
    JexlEngine jexl = new JexlEngine();
    return resolveVariableExpression(value, value.length(), jexl);
  }

  /**
   * Function that looks for expressions to parse. It parses backwards to
   * capture embedded expressions
   *
   * @param value
   * @param last
   * @param jexl
   * @return
   */
  private static String resolveVariableExpression(String value, int last,
      JexlEngine jexl) {
    int lastIndex = value.lastIndexOf("$(", last);
    if (lastIndex == -1) {
      return value;
    }

    // Want to check that everything is well formed, and that
    // we properly capture $( ...(...)...).
    int bracketCount = 0;
    int nextClosed = lastIndex + 2;
    for (; nextClosed < value.length(); ++nextClosed) {
      if (value.charAt(nextClosed) == '(') {
        bracketCount++;
      } else if (value.charAt(nextClosed) == ')') {
        bracketCount--;
        if (bracketCount == -1) {
          break;
        }
      }
    }

    if (nextClosed == value.length()) {
      throw new IllegalArgumentException("Expression " + value
          + " not well formed.");
    }

    String innerExpression = value.substring(lastIndex + 2, nextClosed);
    Object result = null;
    try {
      Expression e = jexl.createExpression(innerExpression);
      result = e.evaluate(new MapContext());
    } catch (JexlException e) {
      throw new IllegalArgumentException("Expression " + value
          + " not well formed. " + e.getMessage(), e);
    }

    if (result == null) {
      // for backward compatibility it is best to return value
      return value;
    }

    String newValue =
        value.substring(0, lastIndex) + result.toString()
            + value.substring(nextClosed + 1);
    return resolveVariableExpression(newValue, lastIndex, jexl);
  }

  public static Props addCommonFlowProperties(Props parentProps,
      final ExecutableFlowBase flow) {
    Props props = new Props(parentProps);

    props.put(CommonJobProperties.FLOW_ID, flow.getFlowId());
    props.put(CommonJobProperties.EXEC_ID, flow.getExecutionId());
    props.put(CommonJobProperties.PROJECT_ID, flow.getProjectId());
    props.put(CommonJobProperties.PROJECT_NAME, flow.getProjectName());
    props.put(CommonJobProperties.PROJECT_VERSION, flow.getVersion());
    props.put(CommonJobProperties.FLOW_UUID, UUID.randomUUID().toString());
    props.put(CommonJobProperties.PROJECT_LAST_CHANGED_BY, flow.getLastModifiedByUser());
    props.put(CommonJobProperties.PROJECT_LAST_CHANGED_DATE, flow.getLastModifiedTimestamp());
    props.put(CommonJobProperties.SUBMIT_USER, flow.getExecutableFlow().getSubmitUser());

    DateTime loadTime = new DateTime();

    props.put(CommonJobProperties.FLOW_START_TIMESTAMP, loadTime.toString());
    props.put(CommonJobProperties.FLOW_START_YEAR, loadTime.toString("yyyy"));
    props.put(CommonJobProperties.FLOW_START_MONTH, loadTime.toString("MM"));
    props.put(CommonJobProperties.FLOW_START_DAY, loadTime.toString("dd"));
    props.put(CommonJobProperties.FLOW_START_HOUR, loadTime.toString("HH"));
    props.put(CommonJobProperties.FLOW_START_MINUTE, loadTime.toString("mm"));
    props.put(CommonJobProperties.FLOW_START_SECOND, loadTime.toString("ss"));
    props.put(CommonJobProperties.FLOW_START_MILLISSECOND,
        loadTime.toString("SSS"));
    props.put(CommonJobProperties.FLOW_START_TIMEZONE,
        loadTime.toString("ZZZZ"));

    return props;
  }

  public static String toJSONString(Props props, boolean localOnly) {
    Map<String, String> map = toStringMap(props, localOnly);
    return JSONUtils.toJSON(map);
  }

  public static Map<String, String> toStringMap(Props props, boolean localOnly) {
    HashMap<String, String> map = new HashMap<String, String>();
    Set<String> keyset = localOnly ? props.localKeySet() : props.getKeySet();

    for (String key : keyset) {
      String value = props.get(key);
      map.put(key, value);
    }

    return map;
  }

  public static Props fromJSONString(String json) throws IOException {
    Map<String, String> obj = (Map<String, String>) JSONUtils.parseJSONFromString(json);
    Props props = new Props(null, obj);
    return props;
  }

  @SuppressWarnings("unchecked")
  public static Props fromHierarchicalMap(Map<String, Object> propsMap) {
    if (propsMap == null) {
      return null;
    }

    String source = (String) propsMap.get("source");
    Map<String, String> propsParams =
        (Map<String, String>) propsMap.get("props");

    Map<String, Object> parent = (Map<String, Object>) propsMap.get("parent");
    Props parentProps = fromHierarchicalMap(parent);

    Props props = new Props(parentProps, propsParams);
    props.setSource(source);
    return props;
  }

  public static Map<String, Object> toHierarchicalMap(Props props) {
    Map<String, Object> propsMap = new HashMap<String, Object>();
    propsMap.put("source", props.getSource());
    propsMap.put("props", toStringMap(props, true));

    if (props.getParent() != null) {
      propsMap.put("parent", toHierarchicalMap(props.getParent()));
    }

    return propsMap;
  }

  /**
   * @param oldProps
   * @param newProps
   * @return the difference between oldProps and newProps.
   */
  public static String getPropertyDiff(Props oldProps, Props newProps) {

    StringBuilder builder = new StringBuilder("");

    // oldProps can not be null during the below comparison process.
    if (oldProps == null) {
      oldProps = new Props();
    }

    if (newProps == null) {
      newProps = new Props();
    }

    MapDifference<String, String> md =
        Maps.difference(toStringMap(oldProps, false), toStringMap(newProps, false));

    Map<String, String> newlyCreatedProperty = md.entriesOnlyOnRight();
    if (newlyCreatedProperty != null && newlyCreatedProperty.size() > 0) {
      builder.append("Newly created Properties: ");
      newlyCreatedProperty.forEach((k, v) -> {
        builder.append("[ " + k + ", " + v + "], ");
      });
      builder.append("\n");
    }

    Map<String, String> deletedProperty = md.entriesOnlyOnLeft();
    if (deletedProperty != null && deletedProperty.size() > 0) {
      builder.append("Deleted Properties: ");
      deletedProperty.forEach((k, v) -> {
        builder.append("[ " + k + ", " + v + "], ");
      });
      builder.append("\n");
    }

    Map<String, MapDifference.ValueDifference<String>> diffProperties = md.entriesDiffering();
    if (diffProperties != null && diffProperties.size() > 0) {
      builder.append("Modified Properties: ");
      diffProperties.forEach((k, v) -> {
        builder.append("[ " + k + ", " + v.leftValue() + "-->" + v.rightValue() + "], ");
      });
    }
    return builder.toString();
  }
}