/*
* Copyright (C) 2006-2016 DLR, Germany
*
* All rights reserved
*
* http://www.rcenvironment.de/
*/
package de.rcenvironment.components.optimizer.common.execution;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.codehaus.jackson.map.ObjectMapper;
import de.rcenvironment.components.optimizer.common.MethodDescription;
import de.rcenvironment.components.optimizer.common.OptimizerComponentConstants;
import de.rcenvironment.core.component.api.ComponentException;
import de.rcenvironment.core.component.execution.api.ComponentContext;
import de.rcenvironment.core.datamodel.api.DataType;
import de.rcenvironment.core.datamodel.api.TypedDatum;
import de.rcenvironment.core.datamodel.api.TypedDatumService;
import de.rcenvironment.core.datamodel.types.api.FloatTD;
import de.rcenvironment.core.datamodel.types.api.VectorTD;
import de.rcenvironment.core.utils.common.JsonUtils;
/**
* A part implementation for the algorithm executor which write config files in python.
*
* @author Sascha Zur
*/
public abstract class CommonPythonAlgorithmExecutor extends OptimizerAlgorithmExecutor {
protected static final Log LOGGER = LogFactory.getLog(OptimizerAlgorithmExecutor.class);
protected static final String CLOSE_BRACKET_AND_NL = ")\n";
protected static final String REGEX_DOT = "\\.";
private static final String DOT = ".";
private static final int NEGATE_VALUE = -1;
private static final String META_BASE = "base";
private static final String VALUE = "value";
protected final String apostroph = "'";
protected final String comma = ",";
protected Map<String, TypedDatum> outputValues;
protected Collection<String> input;
protected String algorithm;
protected Map<String, MethodDescription> methodConfiguration;
protected Map<String, Double> lowerMap;
protected Map<String, Double> upperMap;
private boolean gradRequest;
private List<String> orderedOutputValueKeys;
private Map<String, Double> stepValues;
public CommonPythonAlgorithmExecutor(Map<String, MethodDescription> methodConfiguration,
Map<String, TypedDatum> outputValues,
Collection<String> input, ComponentContext compContext,
Map<String, Double> upperMap, Map<String, Double> lowerMap, Map<String, Double> stepValues, String inputFilename)
throws ComponentException {
super(compContext, compContext.getInstanceName(), inputFilename);
this.algorithm = compContext.getConfigurationValue(OptimizerComponentConstants.ALGORITHMS);
this.methodConfiguration = methodConfiguration;
this.outputValues = outputValues;
this.input = input;
this.lowerMap = lowerMap;
this.upperMap = upperMap;
this.stepValues = stepValues;
typedDatumFactory = compContext.getService(TypedDatumService.class).getFactory();
}
public CommonPythonAlgorithmExecutor() {
}
protected void writeConfigurationFile(File configFile) throws ComponentException {
Map<String, Object> configuration = new HashMap<>();
addOutputsToConfig(configuration);
addInputsToConfig(configuration);
String[] algos = algorithm.split(comma);
for (String algo : algos) {
Map<String, Map<String, String>> allSettings = methodConfiguration.get(algo).getSpecificSettings();
Map<String, Object> settings = new HashMap<>();
for (String key : allSettings.keySet()) {
Map<String, Object> set = new HashMap<>();
String dataType = allSettings.get(key).get("dataType");
String value = allSettings.get(key).get("Value");
if (value == null || value.isEmpty()) {
value = allSettings.get(key).get("DefaultValue");
}
switch (dataType.toLowerCase()) {
case "real":
set.put(VALUE, Double.parseDouble(value));
break;
case "int":
set.put(VALUE, Integer.parseInt(value));
break;
case "bool":
set.put(VALUE, Boolean.parseBoolean(value));
break;
case "None":
default:
set.put(VALUE, value);
}
settings.put(key, set);
}
configuration.put("algorithmSettings", settings);
configuration.put("algorithm", algo.substring(0, algo.lastIndexOf("[") - 1));
}
ObjectMapper mapper = JsonUtils.getDefaultObjectMapper();
try {
mapper.writerWithDefaultPrettyPrinter().writeValue(configFile, configuration);
} catch (IOException e) {
throw new ComponentException("Failed to write configuration file", e);
}
}
private void addInputsToConfig(Map<String, Object> configuration) {
List<String> orderedInputValueKeys = new ArrayList<String>(input.size());
for (String e : input) {
orderedInputValueKeys.add(e);
}
Collections.sort(orderedInputValueKeys);
List<String> orderedObjectives = new LinkedList<>();
List<String> orderedConstraints = new LinkedList<>();
List<String> orderedGradients = new LinkedList<>();
Map<String, Double> minValues = new HashMap<>();
Map<String, Double> maxValues = new HashMap<>();
Map<String, Double> objectiveWeights = new HashMap<>();
for (String key : orderedInputValueKeys) {
if (compContext.getDynamicInputIdentifier(key).equals(OptimizerComponentConstants.ID_OBJECTIVE)
&& !key.contains(OptimizerComponentConstants.GRADIENT_DELTA)) {
orderedObjectives.add(key);
String weight = compContext.getInputMetaDataValue(key, OptimizerComponentConstants.META_WEIGHT);
objectiveWeights.put(key, Double.parseDouble(weight));
}
if (compContext.getDynamicInputIdentifier(key).equals(OptimizerComponentConstants.ID_CONSTRAINT)
&& !key.contains(OptimizerComponentConstants.GRADIENT_DELTA)) {
orderedConstraints.add(key);
minValues.put(key, lowerMap.get(key));
maxValues.put(key, upperMap.get(key));
}
if (key.contains(OptimizerComponentConstants.GRADIENT_DELTA)) {
orderedGradients.add(key);
}
}
configuration.put("objectives", orderedObjectives);
configuration.put("constraints", orderedConstraints);
configuration.put("gradients", orderedGradients);
configuration.put("objectivesWeights", objectiveWeights);
configuration.put("minValuesConstraints", minValues);
configuration.put("maxValuesConstraints", maxValues);
}
private void addOutputsToConfig(Map<String, Object> configuration) {
orderedOutputValueKeys = new ArrayList<String>();
for (String output : outputValues.keySet()) {
if (compContext.getOutputDataType(output) == DataType.Vector) {
for (int i = 0; i < Integer.valueOf(compContext.getOutputMetaDataValue(output,
OptimizerComponentConstants.METADATA_VECTOR_SIZE)); i++) {
orderedOutputValueKeys.add(output + OptimizerComponentConstants.OPTIMIZER_VECTOR_INDEX_SYMBOL + i);
}
} else {
orderedOutputValueKeys.add(output);
}
}
Collections.sort(orderedOutputValueKeys);
Map<String, Double> initValues = new HashMap<>();
Map<String, Double> baseValues = new HashMap<>();
// Map<String, Double> stepValues = new HashMap<>();
Map<String, Boolean> discreteValues = new HashMap<>();
Map<String, Double> minValues = new HashMap<>();
Map<String, Double> maxValues = new HashMap<>();
for (String key : orderedOutputValueKeys) {
if (key.contains(OptimizerComponentConstants.OPTIMIZER_VECTOR_INDEX_SYMBOL)
&& compContext.getOutputDataType(getVectorName(key)) == DataType.Vector) {
if (outputValues.containsKey(key)) {
initValues.put(key, ((FloatTD) outputValues.get(key)).getFloatValue());
}
discreteValues.put(key, Boolean.parseBoolean(
compContext.getOutputMetaDataValue(getVectorName(key), OptimizerComponentConstants.META_IS_DISCRETE)));
} else {
initValues.put(key, ((FloatTD) outputValues.get(key)).getFloatValue());
}
if (!key.contains(OptimizerComponentConstants.GRADIENT_DELTA)) {
if (key.contains(OptimizerComponentConstants.OPTIMIZER_VECTOR_INDEX_SYMBOL)) {
if (compContext.getOutputMetaDataValue(getVectorName(key), META_BASE) != null) {
baseValues.put(key, Double.valueOf(compContext.getOutputMetaDataValue(
getVectorName(key), META_BASE)));
}
} else {
if (compContext.getOutputMetaDataValue(key, META_BASE) != null) {
baseValues.put(key, Double.valueOf(compContext.getOutputMetaDataValue(key, META_BASE)));
}
}
}
minValues.put(key, lowerMap.get(key));
maxValues.put(key, upperMap.get(key));
}
configuration.put("designVariableCount", orderedOutputValueKeys.size());
configuration.put("designVariables", orderedOutputValueKeys);
configuration.put("initValues", initValues);
configuration.put("minValuesVariables", minValues);
configuration.put("maxValuesVariables", maxValues);
configuration.put("baseValues", baseValues);
configuration.put("stepValues", stepValues);
configuration.put("discreteValues", discreteValues);
}
private String getVectorName(String key) {
return key.substring(0, key.indexOf(OptimizerComponentConstants.OPTIMIZER_VECTOR_INDEX_SYMBOL));
}
@SuppressWarnings("unchecked")
@Override
public void readOutputFileFromExternalProgram(Map<String, TypedDatum> outputValueMap) throws IOException {
if (messageFromClient != null) {
String currentWorkingDir = messageFromClient.getCurrentWorkingDir();
if (currentWorkingDir != null) {
File[] cwdFiles = new File(currentWorkingDir).listFiles();
if (cwdFiles != null) {
File outputFile = cwdFiles[0];
ObjectMapper mapper = JsonUtils.getDefaultObjectMapper();
Map<String, Object> result = mapper.readValue(outputFile, new HashMap<String, Object>().getClass());
List<Double> outputs = (List<Double>) result.get("designVar");
int offset = 0;
for (int i = 0; i < orderedOutputValueKeys.size(); i += offset) {
offset = readOutputs(outputValueMap, outputs, i);
}
gradRequest = (Boolean) result.get("gradRequest");
}
}
}
}
private int readOutputs(Map<String, TypedDatum> outputValueMap, List<Double> outputs, int i) {
int offset;
String key = orderedOutputValueKeys.get(i);
offset = 1;
String realKey = "";
if (key.contains(OptimizerComponentConstants.OPTIMIZER_VECTOR_INDEX_SYMBOL)) {
realKey = key.substring(0, key.lastIndexOf(OptimizerComponentConstants.OPTIMIZER_VECTOR_INDEX_SYMBOL));
}
if (!realKey.isEmpty()) {
if (compContext.getOutputDataType(realKey) == DataType.Vector && !orderedOutputValueKeys.contains(realKey)) {
VectorTD resultVec = typedDatumFactory.createVector(Integer.parseInt(compContext.getOutputMetaDataValue(
realKey, OptimizerComponentConstants.METADATA_VECTOR_SIZE)));
for (int j = 0; j < resultVec.getRowDimension(); j++) {
resultVec.setFloatTDForElement(typedDatumFactory.createFloat(outputs.get(i + j)), j);
}
outputValueMap.put(realKey, resultVec);
offset++;
}
} else {
outputValueMap.put(key, typedDatumFactory.createFloat(outputs.get(i)));
}
return offset;
}
@Override
public boolean getDerivativedNeeded() {
return gradRequest;
}
@Override
protected void writeInputFileforExternalProgram(Map<String, Double> functionVariables,
Map<String, Double> functionVariablesGradients, Map<String, Double> constraintVariables, String outputFileName) throws IOException {
File rceInputFile = new File(messageFromClient.getCurrentWorkingDir(), outputFileName);
for (String key : functionVariables.keySet()) {
String name = key;
if (name.contains(OptimizerComponentConstants.OPTIMIZER_VECTOR_INDEX_SYMBOL)
&& !name.contains(OptimizerComponentConstants.GRADIENT_DELTA)) {
name = name.substring(0, name.lastIndexOf(OptimizerComponentConstants.OPTIMIZER_VECTOR_INDEX_SYMBOL));
}
if (!name.contains(OptimizerComponentConstants.GRADIENT_DELTA)
&& compContext.getInputMetaDataValue(name, OptimizerComponentConstants.META_GOAL).equals("Maximize")) { // Maximize
// Optimizer only minimizes functions so for maximizing you need to minimize -f(x)
functionVariables.put(key, functionVariables.get(key) * NEGATE_VALUE);
}
}
// Arrange gradients
Map<String, List<Double>> gradientsPerObjectiveOrConstraint = new HashMap<>();
getGradients(functionVariables, functionVariablesGradients, gradientsPerObjectiveOrConstraint, orderedOutputValueKeys);
getGradients(constraintVariables, functionVariablesGradients, gradientsPerObjectiveOrConstraint, orderedOutputValueKeys);
ObjectMapper mapper = JsonUtils.getDefaultObjectMapper();
Map<String, Object> all = new HashMap<>();
all.put("objective", functionVariables);
all.put("const", constraintVariables);
all.put("grad", gradientsPerObjectiveOrConstraint);
mapper.writerWithDefaultPrettyPrinter().writeValue(rceInputFile, all);
}
private void getGradients(Map<String, Double> functionVariables, Map<String, Double> functionVariablesGradients,
Map<String, List<Double>> gradientsPerObjectiveOrConstraint, List<String> outputNames) {
for (String obj : functionVariables.keySet()) {
List<Double> values = new LinkedList<>();
for (String output : outputNames) {
if (functionVariablesGradients.containsKey(getGradientName(obj, output))) {
values.add(functionVariablesGradients.get(getGradientName(obj, output)));
}
}
if (!values.isEmpty()) {
gradientsPerObjectiveOrConstraint.put(obj, values);
}
}
}
private String getGradientName(String obj, String output) {
return OptimizerComponentConstants.GRADIENT_DELTA + obj + DOT + OptimizerComponentConstants.GRADIENT_DELTA + output;
}
}