/*
* Copyright (C) 2006-2016 DLR, Germany
*
* All rights reserved
*
* http://www.rcenvironment.de/
*/
package de.rcenvironment.core.datamanagement.testutils;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
import org.apache.commons.io.FileUtils;
import de.rcenvironment.core.authorization.AuthorizationException;
import de.rcenvironment.core.command.common.CommandException;
import de.rcenvironment.core.command.spi.CommandContext;
import de.rcenvironment.core.command.spi.CommandDescription;
import de.rcenvironment.core.command.spi.CommandPlugin;
import de.rcenvironment.core.communication.api.PlatformService;
import de.rcenvironment.core.communication.common.CommunicationException;
import de.rcenvironment.core.datamanagement.DataManagementService;
import de.rcenvironment.core.datamanagement.backend.MetaDataBackendService;
import de.rcenvironment.core.datamanagement.commons.ComponentInstance;
import de.rcenvironment.core.datamanagement.commons.EndpointInstance;
import de.rcenvironment.core.datamanagement.commons.MetaData;
import de.rcenvironment.core.datamanagement.commons.MetaDataKeys;
import de.rcenvironment.core.datamanagement.commons.MetaDataSet;
import de.rcenvironment.core.datamanagement.commons.PropertiesKeys;
import de.rcenvironment.core.datamodel.api.DataType;
import de.rcenvironment.core.datamodel.api.EndpointType;
import de.rcenvironment.core.datamodel.api.FinalComponentRunState;
import de.rcenvironment.core.datamodel.api.FinalComponentState;
import de.rcenvironment.core.datamodel.api.FinalWorkflowState;
import de.rcenvironment.core.datamodel.api.TypedDatum;
import de.rcenvironment.core.datamodel.api.TypedDatumFactory;
import de.rcenvironment.core.datamodel.api.TypedDatumSerializer;
import de.rcenvironment.core.datamodel.api.TypedDatumService;
import de.rcenvironment.core.toolkitbridge.transitional.ConcurrencyUtils;
import de.rcenvironment.core.utils.common.StringUtils;
import de.rcenvironment.core.utils.common.TempFileService;
import de.rcenvironment.core.utils.common.TempFileServiceAccess;
import de.rcenvironment.core.utils.common.rpc.RemoteOperationException;
import de.rcenvironment.toolkit.modules.concurrency.api.CallablesGroup;
import de.rcenvironment.toolkit.modules.concurrency.api.TaskDescription;
/**
* A {@link CommandPlugin} providing commands for testing the data management.
*
* @author Brigitte Boden
* @author Jascha Riedel (#13978)
*/
public class DMCommandPlugin implements CommandPlugin {
private static final int NUMBER_100 = 100;
private static final int WORKFLOWS_TO_CREATE_IN_PARALLEL = 20;
private static final int DEFAULT_NUMBER_OF_INPUTS = 5;
private static final int DEFAULT_NUMBER_OF_ITERATIONS = 10;
private static final int DEFAULT_NUMBER_OF_COMPONENTS = 10;
private static final int DEFAULT_NUMBER_OF_WORKFLOWS = 10;
private static final String DEFAULT_WORKFLOWNAME_PREFIX = "dummy_workflow";
/**
* The values for number of iterations, number of components and number of inputs are varied for each workflow. This constant defines
* how much deviation from the default value is allowed (i.e. 20%)
*/
private static final double DEFAULT_ALLOWED_DEVIATION_IN_PERCENT = 20;
private MetaDataBackendService metaDataService;
private PlatformService platformService;
private TypedDatumService typedDatumService;
private DataManagementService dataManagementService;
private TypedDatumFactory typedDatumFactory;
private TypedDatumSerializer typedDatumSerializer;
/**
* Option for file creation.
*
*/
private enum FileCreationOption {
NONE, SMALL, BIG
}
/**
* {@inheritDoc}
*
* @see de.rcenvironment.core.command.spi.SingleCommandHandler#execute(de.rcenvironment.core.command.spi.CommandContext)
*/
@Override
public void execute(CommandContext context) throws CommandException {
context.consumeExpectedToken("dm");
String subCmd = context.consumeNextToken();
if (subCmd == null) {
throw CommandException.unknownCommand(context);
} else {
if ("create-test-data".equals(subCmd)) {
// "dm create-test-data"
performDmCreateTestData(context);
} else {
throw CommandException.unknownCommand(context);
}
}
}
/**
* Fill datamanagement with workflow runs.
*
* @param context
*/
private void performDmCreateTestData(final CommandContext context) throws CommandException {
final int userDefinednumberOfWorkflows;
final int userDefinednumberOfIterations;
final double userDefinedAllowedDeviation;
final String userDefinedWorkflowNamePrefix;
// Read optional parameters
String workflowPrefix = readOptionalStringParameter(context, "--prefix");
if (workflowPrefix != null) {
userDefinedWorkflowNamePrefix = workflowPrefix;
} else {
userDefinedWorkflowNamePrefix = DEFAULT_WORKFLOWNAME_PREFIX;
}
Integer w = readOptionalIntParameter(context, "--workflows");
if (w != null) {
userDefinednumberOfWorkflows = w;
} else {
userDefinednumberOfWorkflows = DEFAULT_NUMBER_OF_WORKFLOWS;
}
Integer iter = readOptionalIntParameter(context, "--iterations");
if (iter != null) {
userDefinednumberOfIterations = iter;
} else {
userDefinednumberOfIterations = DEFAULT_NUMBER_OF_ITERATIONS;
}
Double deviationInPercent = readOptionalDoubleParameter(context, "--allowedDeviation");
if (deviationInPercent != null) {
userDefinedAllowedDeviation = deviationInPercent / NUMBER_100;
} else {
userDefinedAllowedDeviation = DEFAULT_ALLOWED_DEVIATION_IN_PERCENT / NUMBER_100;
}
if (userDefinednumberOfWorkflows < 0 || userDefinednumberOfIterations < 0 || userDefinedAllowedDeviation < 0) {
throw CommandException.executionError("Negative values are not supported.", context);
}
if (userDefinedAllowedDeviation > 1) {
throw CommandException.executionError("Deviation of more than 100% is not supported.", context);
}
List<String> fileTokens = context.consumeRemainingTokens();
FileCreationOption fileCreationOption = FileCreationOption.NONE;
if (fileTokens.contains("--smallfiles")) {
fileCreationOption = FileCreationOption.SMALL;
if (fileTokens.contains("--bigfiles")) {
context.println("Both --smallfiles and --bigfiles were set, --smallfiles will be used.");
}
} else if (fileTokens.contains("--bigfiles")) {
fileCreationOption = FileCreationOption.BIG;
}
// Read files from resources folder
TempFileService tempFileService = TempFileServiceAccess.getInstance();
File wfFile;
File logFile;
File dummyFile;
try (InputStream wfFileStream = getClass().getResourceAsStream("/DummyWorkflowFile.txt");
InputStream logFileStream = getClass().getResourceAsStream("/DummyLogFile.txt");) {
wfFile = tempFileService.createTempFileWithFixedFilename("DummyWorkflowFile.txt");
FileUtils.copyInputStreamToFile(wfFileStream, wfFile);
logFile = tempFileService.createTempFileWithFixedFilename("DummyLogFile.txt");
FileUtils.copyInputStreamToFile(logFileStream, logFile);
} catch (IOException e1) {
throw CommandException.executionError("Could not read dummy files from resources folder.", context);
}
if (fileCreationOption.equals(FileCreationOption.SMALL)) {
try (InputStream dummyFileStream = getClass().getResourceAsStream("/SmallDummyFile.txt");) {
dummyFile = tempFileService.createTempFileWithFixedFilename("SmallDummyFile.txt");
FileUtils.copyInputStreamToFile(dummyFileStream, dummyFile);
} catch (IOException e) {
throw CommandException.executionError("Could not read dummy file from resources folder.", context);
}
} else if (fileCreationOption.equals(FileCreationOption.BIG)) {
try (InputStream dummyFileStream = getClass().getResourceAsStream("/BigDummyFile.xml");) {
dummyFile = tempFileService.createTempFileWithFixedFilename("BigDummyFile.xml");
FileUtils.copyInputStreamToFile(dummyFileStream, dummyFile);
} catch (IOException e) {
throw CommandException.executionError("Could not read dummy file from resources folder.", context);
}
} else {
dummyFile = null;
}
final String localNodeId = platformService.getLocalInstanceNodeSessionId().getInstanceNodeSessionIdString();
typedDatumFactory = typedDatumService.getFactory();
typedDatumSerializer = typedDatumService.getSerializer();
context.println("Creating workflows...");
int workflowIndex = 0;
while (workflowIndex < userDefinednumberOfWorkflows) {
// Create a CallablesGroup with WORKFLOWS_TO_CREATE_IN_PARALLEL threads
CallablesGroup<CommandException> callablesGroup =
ConcurrencyUtils.getFactory().createCallablesGroup(CommandException.class);
for (int i = 0; i < WORKFLOWS_TO_CREATE_IN_PARALLEL; i++) {
workflowIndex++;
// Create random parameters for this workflow
int numberOfComponents =
randomizeValue(DEFAULT_NUMBER_OF_COMPONENTS, userDefinedAllowedDeviation);
int numberOfInputs =
randomizeValue(DEFAULT_NUMBER_OF_INPUTS, userDefinedAllowedDeviation);
int numberOfIterations = randomizeValue(userDefinednumberOfIterations, userDefinedAllowedDeviation);
CreateWorkflowCallableContext workflowCreationContext =
new CreateWorkflowCallableContext(wfFile, userDefinedWorkflowNamePrefix, localNodeId, context, dummyFile,
logFile, numberOfComponents,
numberOfInputs, workflowIndex, numberOfIterations, fileCreationOption);
callablesGroup.add(new CreateWorkflowCallable(workflowCreationContext));
// Stop if required number of workflows is reached.
if (workflowIndex >= userDefinednumberOfWorkflows) {
break;
}
}
List<CommandException> exceptions = callablesGroup.executeParallel(null);
for (CommandException e : exceptions) {
if (e != null) {
throw e;
}
}
// context.println("Created " + workflowIndex + " out of " + userDefinednumberOfWorkflows + " workflows.");
}
context.println("Finished creating test data: created " + userDefinednumberOfWorkflows + " workflows.");
}
private int randomizeValue(int defaultValue, double allowedDeviation) {
return (int) (Math.round((Math.random() * (2 * allowedDeviation) + (1 - allowedDeviation)) * defaultValue));
}
private Integer readOptionalIntParameter(CommandContext context, String parameter) throws CommandException {
if (context.consumeNextTokenIfEquals(parameter)) {
String number = context.consumeNextToken();
if (number == null) {
throw CommandException.syntaxError("Missing number", context);
}
try {
return Integer.parseInt(number);
} catch (NumberFormatException e) {
throw CommandException.executionError(e.getMessage(), context);
}
}
return null;
}
private String readOptionalStringParameter(CommandContext context, String parameter) throws CommandException {
if (context.consumeNextTokenIfEquals(parameter)) {
String prefix = context.consumeNextToken();
if (prefix == null) {
throw CommandException.syntaxError("Missing Workflow Name Prefix", context);
}
return prefix;
}
return null;
}
private Double readOptionalDoubleParameter(CommandContext context, String parameter) throws CommandException {
if (context.consumeNextTokenIfEquals(parameter)) {
String number = context.consumeNextToken();
if (number == null) {
throw CommandException.syntaxError("Missing number", context);
}
try {
return Double.parseDouble(number);
} catch (NumberFormatException e) {
throw CommandException.executionError(e.getMessage(), context);
}
}
return null;
}
/**
* {@inheritDoc}
*
* @see de.rcenvironment.core.command.spi.CommandPlugin#getCommandDescriptions()
*/
@Override
public Collection<CommandDescription> getCommandDescriptions() {
final Collection<CommandDescription> contributions = new ArrayList<CommandDescription>();
contributions
.add(new CommandDescription(
"dm create-test-data [--workflows <number of workflows>] [--iterations <number of iterations per workflow>]\n\t"
+ " [--allowedDeviation <allowed deviation of values in %>]"
+ " [--smallfiles] [--bigfiles]", "", true, "creates test data in the database. ", "If --smallfiles or --bigfiles is "
+ "set, files are generated as inputs/outputs of the components.", " Values for number of iterations, number of "
+ "components, number of inputs/outputs are randomized, deviating at most <allowedDeviation> % from default values "
+ "(default 20%)"));
return contributions;
}
/**
* OSGi-DS bind method.
*
* @param newInstance the new service instance
*/
public void bindMetaDataService(MetaDataBackendService newInstance) {
this.metaDataService = newInstance;
}
/**
* OSGi-DS bind method.
*
* @param newInstance the new service instance
*/
public void bindTypedDatumService(TypedDatumService newInstance) {
this.typedDatumService = newInstance;
}
/**
* OSGi-DS bind method.
*
* @param newInstance the new service instance
*/
public void bindPlatformService(PlatformService newInstance) {
this.platformService = newInstance;
}
/**
* OSGi-DS bind method.
*
* @param newInstance the new service instance
*/
public void bindDataManagementService(DataManagementService newInstance) {
this.dataManagementService = newInstance;
}
/**
* OSGi-DS lifecycle method.
*/
public void activate() {}
/**
* A container class for the parameters and files for the creation of a workflow.
*
* @author Brigitte Boden
*/
private final class CreateWorkflowCallableContext {
private File wfFile;
private String prefixName;
private String localNodeId;
private CommandContext context;
private File dummyFile;
private File logFile;
private int numberOfComponents;
private int numberOfInputs;
private int workflowNumber;
private int numberOfIterations;
private FileCreationOption fileCreationOption;
private CreateWorkflowCallableContext(File wfFile, String prefixName, String localNodeId, CommandContext context, File dummyFile,
File logFile, int numberOfComponents, int numberOfInputs,
int workflowNumber, int numberOfIterations, FileCreationOption fileCreationOption) {
this.wfFile = wfFile;
this.prefixName = prefixName;
this.localNodeId = localNodeId;
this.context = context;
this.dummyFile = dummyFile;
this.logFile = logFile;
this.numberOfComponents = numberOfComponents;
this.numberOfInputs = numberOfInputs;
this.workflowNumber = workflowNumber;
this.numberOfIterations = numberOfIterations;
this.fileCreationOption = fileCreationOption;
}
}
/**
* A "callable" class for creatinng a single dummy workflow in the data management.
*
* @author Brigitte Boden
*/
private final class CreateWorkflowCallable implements Callable<CommandException> {
private CreateWorkflowCallableContext workflowCreationContext;
private CreateWorkflowCallable(CreateWorkflowCallableContext workflowCreationContext) {
this.workflowCreationContext = workflowCreationContext;
}
@Override
@TaskDescription(value = "Create a single workflow entry.")
public CommandException call() {
Map<String, Long> compInstDmIds;
try {
final String workflowTitle = workflowCreationContext.prefixName + "_" + workflowCreationContext.workflowNumber;
Long id =
metaDataService.addWorkflowRun(workflowTitle,
workflowCreationContext.localNodeId, workflowCreationContext.localNodeId,
System.currentTimeMillis());
MetaDataSet mds = new MetaDataSet();
MetaData mdWorkflowRunId = new MetaData(MetaDataKeys.WORKFLOW_RUN_ID, true, true);
mds.setValue(mdWorkflowRunId, id.toString());
try {
final String wfFileReference =
dataManagementService.createReferenceFromLocalFile(workflowCreationContext.wfFile, mds,
platformService.getLocalInstanceNodeSessionId());
TypedDatum fileRefTD = typedDatumFactory.createFileReference(wfFileReference,
workflowCreationContext.wfFile.getName());
metaDataService.addWorkflowFileToWorkflowRun(id, typedDatumSerializer.serialize(fileRefTD));
} catch (AuthorizationException | IOException | InterruptedException | CommunicationException e) {
return CommandException.executionError("Failed to upload wf file." + e, workflowCreationContext.context);
}
Set<ComponentInstance> componentInstances = new HashSet<>();
for (int j = 1; j <= workflowCreationContext.numberOfComponents; j++) {
componentInstances.add(new ComponentInstance(UUID.randomUUID().toString(),
"de.rcenvironment.script/3.4", "Script " + j, null));
}
compInstDmIds = metaDataService.addComponentInstances(id, componentInstances);
Set<EndpointInstance> endpointInstancesIn = new HashSet<>();
Set<EndpointInstance> endpointInstancesOut = new HashSet<>();
for (int z = 1; z <= workflowCreationContext.numberOfInputs; z++) {
Map<String, String> metaData = new HashMap<String, String>();
metaData.put(MetaDataKeys.DATA_TYPE, DataType.Float.getShortName());
endpointInstancesIn.add(new EndpointInstance("input_" + z, EndpointType.INPUT, metaData));
endpointInstancesOut.add(new EndpointInstance("output_" + z, EndpointType.OUTPUT, metaData));
}
for (Long componentInstanceId : compInstDmIds.values()) {
Map<String, Long> endpointIdsIn =
metaDataService.addEndpointInstances(componentInstanceId, endpointInstancesIn);
Map<String, Long> endpointIdsOut =
metaDataService.addEndpointInstances(componentInstanceId, endpointInstancesOut);
long typedDatumId = 0;
for (int j = 1; j <= workflowCreationContext.numberOfIterations; j++) {
Long componentRunId =
metaDataService.addComponentRun(componentInstanceId, workflowCreationContext.localNodeId, j,
System.currentTimeMillis());
for (Long endpointInstanceId : endpointIdsOut.values()) {
TypedDatum datum;
String dummyFileReference;
if ((workflowCreationContext.fileCreationOption.equals(FileCreationOption.SMALL)
|| workflowCreationContext.fileCreationOption.equals(FileCreationOption.BIG))
&& workflowCreationContext.dummyFile != null) {
try {
dummyFileReference =
dataManagementService.createReferenceFromLocalFile(workflowCreationContext.dummyFile, mds,
platformService.getLocalInstanceNodeSessionId());
} catch (AuthorizationException | IOException | InterruptedException | CommunicationException e) {
return CommandException.executionError("Could not create reference for dummy file.",
workflowCreationContext.context);
}
datum =
typedDatumFactory.createFileReference(dummyFileReference, workflowCreationContext.dummyFile.getName());
} else {
datum = typedDatumFactory.createFloat(7.0);
}
String datumSerialized = typedDatumSerializer.serialize(datum);
typedDatumId = metaDataService.addOutputDatum(componentRunId, endpointInstanceId, datumSerialized, j);
}
for (Long endpointInstanceId : endpointIdsIn.values()) {
metaDataService.addInputDatum(componentRunId, typedDatumId, endpointInstanceId, j);
}
metaDataService
.setComponentRunFinished(componentRunId, System.currentTimeMillis(), FinalComponentRunState.FINISHED);
Map<String, String> properties = new HashMap<String, String>();
String logFileReference;
try {
logFileReference =
dataManagementService.createReferenceFromLocalFile(workflowCreationContext.logFile, mds,
platformService.getLocalInstanceNodeSessionId());
} catch (AuthorizationException | IOException | InterruptedException | CommunicationException e) {
return CommandException.executionError("Could not store dummy component log file.",
workflowCreationContext.context);
}
TypedDatum logRefTD = typedDatumFactory.createFileReference(logFileReference,
workflowCreationContext.logFile.getName());
properties.put(PropertiesKeys.COMPONENT_LOG_FILE, typedDatumService.getSerializer().serialize(logRefTD));
metaDataService.addComponentRunProperties(componentRunId, properties);
}
metaDataService.setComponentInstanceFinalState(componentInstanceId, FinalComponentState.FINISHED);
}
metaDataService.setWorkflowRunFinished(id, System.currentTimeMillis(), FinalWorkflowState.FINISHED);
String type = "float";
if (workflowCreationContext.fileCreationOption.equals(FileCreationOption.BIG)) {
type = "big files";
} else if (workflowCreationContext.fileCreationOption.equals(FileCreationOption.SMALL)) {
type = "small files";
}
workflowCreationContext.context.println(StringUtils.format(
"Created workflow %s: %d components, %d iterations, %d inputs/outputs (%s) per component.", workflowTitle,
workflowCreationContext.numberOfComponents, workflowCreationContext.numberOfIterations,
workflowCreationContext.numberOfInputs, type));
} catch (RemoteOperationException e) {
return CommandException.executionError("Could not store workflow runs.", workflowCreationContext.context);
}
return null;
}
}
}