DerbyMetaDataBackendTest.java

/*
 * Copyright (C) 2006-2016 DLR, Germany
 * 
 * All rights reserved
 * 
 * http://www.rcenvironment.de/
 */

package de.rcenvironment.core.datamanagement.backend.metadata.derby.internal;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;

import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.easymock.EasyMock;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.osgi.framework.Bundle;
import org.osgi.framework.BundleContext;

import de.rcenvironment.core.communication.common.NodeIdentifierTestUtils;
import de.rcenvironment.core.configuration.testutils.MockConfigurationService;
import de.rcenvironment.core.datamanagement.DataManagementIdMapping;
import de.rcenvironment.core.datamanagement.FileDataService;
import de.rcenvironment.core.datamanagement.commons.BinaryReference;
import de.rcenvironment.core.datamanagement.commons.ComponentInstance;
import de.rcenvironment.core.datamanagement.commons.DataReference;
import de.rcenvironment.core.datamanagement.commons.EndpointInstance;
import de.rcenvironment.core.datamanagement.commons.MetaData;
import de.rcenvironment.core.datamanagement.commons.MetaDataKeys;
import de.rcenvironment.core.datamanagement.commons.MetaDataSet;
import de.rcenvironment.core.datamanagement.commons.WorkflowRun;
import de.rcenvironment.core.datamodel.api.CompressionFormat;
import de.rcenvironment.core.datamodel.api.DataType;
import de.rcenvironment.core.datamodel.api.EndpointType;
import de.rcenvironment.core.datamodel.api.FinalWorkflowState;
import de.rcenvironment.core.datamodel.api.TypedDatumFactory;
import de.rcenvironment.core.datamodel.api.TypedDatumSerializer;
import de.rcenvironment.core.datamodel.testutils.TypedDatumFactoryDefaultStub;
import de.rcenvironment.core.datamodel.testutils.TypedDatumSerializerDefaultStub;
import de.rcenvironment.core.datamodel.testutils.TypedDatumServiceDefaultStub;
import de.rcenvironment.core.toolkitbridge.transitional.ConcurrencyUtils;
import de.rcenvironment.core.utils.common.StringUtils;
import de.rcenvironment.core.utils.common.TempFileServiceAccess;
import de.rcenvironment.core.utils.common.rpc.RemoteOperationException;
import de.rcenvironment.core.utils.testing.CommonTestOptions;
import de.rcenvironment.toolkit.modules.concurrency.api.CallablesGroup;
import de.rcenvironment.toolkit.modules.concurrency.api.TaskDescription;

/**
 * Test cases for {@link DerbyMetaDataBackendServiceImpl}.
 * 
 * @author Juergen Klein // TODO review: still correct?
 * @author Jan Flink
 * @author Robert Mischke
 */
public class DerbyMetaDataBackendTest {

    // "safety net" test timeout to avoid blocking continuous integration test runs
    private static final int COMPLEX_SCENARIO_TEST_TIMEOUT = 300000;

    private static final String STRING_TEST_RUN = "TestRun";

    private static File tempDirectory;

    private DerbyMetaDataBackendServiceImpl derbyMetaDataBackend;

    private final String bundleName = "superderby";

    private MetaDataSet metaDataSet;

    private TypedDatumFactory typedDatumFactory = new TypedDatumFactoryDefaultStub();

    private TypedDatumSerializer typedDatumSerializer = new TypedDatumSerializerDefaultStub();

    private final Log log = LogFactory.getLog(getClass());

    /**
     * Sets the whole test case up.
     * 
     * @throws IOException on initialization failure
     * @throws RemoteOperationException standard remote operation exception
     */
    @BeforeClass
    public static void setUpTestcase() throws IOException, RemoteOperationException {
        TempFileServiceAccess.setupUnitTestEnvironment();
        tempDirectory = TempFileServiceAccess.getInstance().createManagedTempDir("derby-metadata");
        // perform db startup so the time is not added to the run time of the first test case
        DerbyMetaDataBackendTest dummyTest = new DerbyMetaDataBackendTest();
        dummyTest.setUp();
        dummyTest.tearDown();
    }

    /**
     * Set up.
     * 
     * @throws IOException on initialization failure
     * @throws RemoteOperationException standard remote operation exception
     */
    @Before
    public void setUp() throws IOException, RemoteOperationException {

        metaDataSet = new MetaDataSet();
        metaDataSet.setValue(new MetaData("testkey", true), "testvalue");

        derbyMetaDataBackend = new DerbyMetaDataBackendServiceImpl();
        derbyMetaDataBackend.bindConfigurationService(new DummyConfigurationService());
        derbyMetaDataBackend.bindTypedDatumService(new TypedDatumServiceDefaultStub());

        Bundle bundle = EasyMock.createNiceMock(Bundle.class);
        EasyMock.expect(bundle.getSymbolicName()).andReturn(bundleName).anyTimes();
        EasyMock.replay(bundle);
        BundleContext bundleContext = EasyMock.createNiceMock(BundleContext.class);
        EasyMock.expect(bundleContext.getBundle()).andReturn(bundle).anyTimes();
        EasyMock.replay(bundleContext);

        // create and set a mock that takes blob deletion requests, and ignores them without error
        FileDataService mockDataService = EasyMock.createNiceMock(FileDataService.class);
        mockDataService.deleteReference(EasyMock.anyObject(String.class));
        EasyMock.expectLastCall().anyTimes();
        EasyMock.replay(mockDataService);
        derbyMetaDataBackend.bindDataService(mockDataService);

        derbyMetaDataBackend.activate(bundleContext);

        System.setProperty("derby.locks.monitor", "true");
        System.setProperty("derby.locks.deadlockTrace", "true");
    }

    /** Tear down. */
    @After
    public void tearDown() {
        derbyMetaDataBackend.deactivate();
    }

    /**
     * Tears down the whole test case.
     * 
     * @throws IOException IOException
     * 
     */
    @AfterClass
    public static void tearDownTestcase() throws IOException {
        // FileUtils.deleteDirectory(tempDirectory);
    }

    /** Test. */
    @Test
    public void testAddWorkflowRun() {
        Long wfRunId =
            derbyMetaDataBackend.addWorkflowRun(STRING_TEST_RUN, DataManagementIdMapping.createDummyNodeIdStringForTesting(),
                DataManagementIdMapping.createDummyNodeIdStringForTesting(),
                System.currentTimeMillis());
        assertNotNull(wfRunId);
    }

    /** Test. */
    @Test
    public void testAddComponentInstances() {
        Long wfRunId =
            derbyMetaDataBackend.addWorkflowRun(STRING_TEST_RUN, DataManagementIdMapping.createDummyNodeIdStringForTesting(),
                DataManagementIdMapping.createDummyNodeIdStringForTesting(),
                System.currentTimeMillis());
        assertNotNull(wfRunId);
        Collection<ComponentInstance> componentInstances = createComponentInstances(4);
        Map<String, Long> result = derbyMetaDataBackend.addComponentInstances(wfRunId, componentInstances);
        assertNotNull(result);
        assertEquals(componentInstances.size(), result.size());
        for (ComponentInstance ci : componentInstances) {
            assertTrue(result.containsKey(ci.getComponentExecutionID()));
        }
    }

    private Collection<ComponentInstance> createComponentInstances(int numComponents) {
        Collection<ComponentInstance> componentInstances = new HashSet<ComponentInstance>();
        String componentID = "de.rcenvironment.testcomponent";
        for (int i = 1; i <= numComponents; i++) {
            // TODO review UUID generation; encapsulate in properly named method
            componentInstances.add(new ComponentInstance(UUID.randomUUID().toString(), componentID, "Testcomponent" + i, null));
        }
        return componentInstances;
    }

    /** Test. */
    @Test
    public void testAddEndpointInstances() {
        Long wfRunId =
            derbyMetaDataBackend.addWorkflowRun(STRING_TEST_RUN, DataManagementIdMapping.createDummyNodeIdStringForTesting(),
                DataManagementIdMapping.createDummyNodeIdStringForTesting(),
                System.currentTimeMillis());
        Collection<ComponentInstance> componentInstances = createComponentInstances(4);
        Map<String, Long> componentInstanceIdMap = derbyMetaDataBackend.addComponentInstances(wfRunId, componentInstances);
        Collection<EndpointInstance> endpointInstances = getEndpointInstances();
        Map<String, Long> result = new HashMap<String, Long>();
        for (Long id : componentInstanceIdMap.values()) {
            result.putAll(derbyMetaDataBackend.addEndpointInstances(id, endpointInstances));
        }
        assertFalse(result.isEmpty());
    }

    private Collection<EndpointInstance> getEndpointInstances() {
        Collection<EndpointInstance> endpointInstances = new HashSet<EndpointInstance>();
        Map<String, String> metaData = new HashMap<String, String>();
        metaData.put(MetaDataKeys.DATA_TYPE, DataType.Float.getShortName());
        endpointInstances.add(new EndpointInstance("X_in", EndpointType.INPUT, metaData));
        endpointInstances.add(new EndpointInstance("Y_in", EndpointType.INPUT, metaData));
        endpointInstances.add(new EndpointInstance("Z_in", EndpointType.INPUT, metaData));
        endpointInstances.add(new EndpointInstance("X_out", EndpointType.OUTPUT, metaData));
        endpointInstances.add(new EndpointInstance("Y_out", EndpointType.OUTPUT, metaData));
        endpointInstances.add(new EndpointInstance("Y_out", EndpointType.OUTPUT, metaData));
        return endpointInstances;
    }

    /** Test. */
    @Test
    public void testAddComponentRun() {
        Long wfRunId =
            derbyMetaDataBackend.addWorkflowRun(STRING_TEST_RUN, DataManagementIdMapping.createDummyNodeIdStringForTesting(),
                DataManagementIdMapping.createDummyNodeIdStringForTesting(),
                System.currentTimeMillis());
        Collection<ComponentInstance> componentInstances = createComponentInstances(4);
        Map<String, Long> componentInstanceIdMap = derbyMetaDataBackend.addComponentInstances(wfRunId, componentInstances);
        for (Long ciid : componentInstanceIdMap.values()) {
            Long id =
                derbyMetaDataBackend.addComponentRun(ciid, DataManagementIdMapping.createDummyNodeIdStringForTesting(), 1,
                    System.currentTimeMillis());
            assertNotNull(id);
        }

    }

    /** Test. */
    @Test(timeout = COMPLEX_SCENARIO_TEST_TIMEOUT)
    public void testAddDeleteData() {
        // test single-workflow deletion with the defined components and number of runs per component each
        final int numComponents = 10;
        final int numRunsPerComponent = 50;
        performAddDeleteWorkflowWithDataCycle(numComponents, numRunsPerComponent, 0);
    }

    /** Test. */
    @Test(timeout = COMPLEX_SCENARIO_TEST_TIMEOUT)
    public void testParallelAddDeleteData() {
        // TODO review standard/extended values; commented code are the previously hardcoded ones
        // final int parallelTasks = 50; // low value for CI test runs; increase for manual tests of high-contention problems
        // final int numComponents = 20;
        // final int numRunsPerComponent = 20; // increase in manual testing
        final int parallelTasks = CommonTestOptions.selectStandardOrExtendedValue(10, 50);
        final int numComponents = CommonTestOptions.selectStandardOrExtendedValue(10, 20);
        final int numRunsPerComponent = CommonTestOptions.selectStandardOrExtendedValue(10, 20);

        // the following wait interval can be set to leave the workflow data in the database for some time
        final long minWaitBeforeDeletion = 0;
        final long maxWaitBeforeDeletion = 0;
        CallablesGroup<Boolean> callablesGroup = ConcurrencyUtils.getFactory().createCallablesGroup(Boolean.class);
        for (int i = 0; i < parallelTasks; i++) {
            callablesGroup.add(new Callable<Boolean>() {

                @Override
                @TaskDescription("Parallel add/delete cycle task")
                public Boolean call() throws Exception {
                    try {
                        long wait = minWaitBeforeDeletion + ((long) Math.random() * (maxWaitBeforeDeletion - minWaitBeforeDeletion));
                        performAddDeleteWorkflowWithDataCycle(numComponents, numRunsPerComponent, wait);
                        return true;
                    } catch (RuntimeException e) {
                        log.error("Asynchronous add/delete cycle failed", e);
                        return false;
                    }
                }
            });
        }
        List<Boolean> results = callablesGroup.executeParallel(null);
        int succcessCount = 0;
        for (Boolean result : results) {
            if (result == Boolean.TRUE) {
                succcessCount++;
            }
        }
        assertEquals("One or more parallel add/delete tasks failed; check the log output for asynchronous exceptions", parallelTasks,
            succcessCount);
    }

    private void performAddDeleteWorkflowWithDataCycle(int numComponents, int numRunsPerComponent, long waitBeforeDeleteMsec) {
        Long wfRunId =
            derbyMetaDataBackend.addWorkflowRun(STRING_TEST_RUN, DataManagementIdMapping.createDummyNodeIdStringForTesting(),
                DataManagementIdMapping.createDummyNodeIdStringForTesting(),
                System.currentTimeMillis());
        log.debug(StringUtils.format("Added workflow run (id %d)", wfRunId));
        Collection<ComponentInstance> componentInstances = createComponentInstances(numComponents);
        Map<String, Long> componentInstanceIdMap = derbyMetaDataBackend.addComponentInstances(wfRunId, componentInstances);
        log.debug(StringUtils.format("Added component instances to workflow run id %d", wfRunId));
        Collection<EndpointInstance> endpointInstances = getEndpointInstances();
        Map<String, Long> endpointInstanceIdMap = new HashMap<String, Long>();
        for (Long id : componentInstanceIdMap.values()) {
            endpointInstanceIdMap.putAll(derbyMetaDataBackend.addEndpointInstances(id, endpointInstances));
        }
        log.debug(StringUtils.format("Added endpoint instances to workflow run id %d", wfRunId));
        for (int crunCounter = 1; crunCounter <= numRunsPerComponent; crunCounter++) {
            Set<Long> crunIds = new HashSet<Long>();
            for (Long ciid : componentInstanceIdMap.values()) {
                crunIds.add(
                    derbyMetaDataBackend.addComponentRun(ciid, DataManagementIdMapping.createDummyNodeIdStringForTesting(), crunCounter,
                        System.currentTimeMillis()));
            }
            Iterator<Long> crunIdIterator = crunIds.iterator();
            Iterator<Long> epiIdIterator = endpointInstanceIdMap.values().iterator();
            // TODO review UUID generation; encapsulate in properly named method
            Long id =
                derbyMetaDataBackend.addOutputDatum(crunIdIterator.next(), epiIdIterator.next(),
                    typedDatumSerializer.serialize(typedDatumFactory.createFileReference(UUID.randomUUID().toString(), "test.xml")), 1);
            assertNotNull(id);
            id =
                derbyMetaDataBackend.addOutputDatum(crunIdIterator.next(), epiIdIterator.next(),
                    typedDatumSerializer.serialize(typedDatumFactory.createBoolean(false)), 1);
            assertNotNull(id);
            id =
                derbyMetaDataBackend.addOutputDatum(crunIdIterator.next(), epiIdIterator.next(),
                    typedDatumSerializer.serialize(typedDatumFactory.createFloat(3f)), 1);
            assertNotNull(id);

            Long crunId = crunIdIterator.next();
            derbyMetaDataBackend.addInputDatum(crunId, id, endpointInstanceIdMap.values().iterator().next(), 1);

            // Generate data reference and add it to a component run
            Set<BinaryReference> brefs = new HashSet<BinaryReference>();
            // TODO review UUID generation; encapsulate in properly named method
            String key = UUID.randomUUID().toString();
            brefs.add(new BinaryReference(key, CompressionFormat.GZIP, "1.1"));
            // note: the following line is creating a new instance id on each call; kept this way to maintain pre-8.0 test behavior
            derbyMetaDataBackend.addDataReferenceToComponentRun(crunId,
                new DataReference(key, NodeIdentifierTestUtils.createTestInstanceNodeId(), brefs));
        }
        log.debug(StringUtils.format("Added %d component runs to workflow run id %d", (numComponents * numRunsPerComponent), wfRunId));
        // mark as finished, otherwise file deletion will (correctly) fail
        derbyMetaDataBackend.setWorkflowRunFinished(wfRunId, System.currentTimeMillis(), FinalWorkflowState.FINISHED);
        log.debug(StringUtils.format("Set workflow run id %d finished", wfRunId));

        assertEquals(STRING_TEST_RUN, derbyMetaDataBackend.getWorkflowRun(wfRunId).getWorkflowTitle());
        log.debug(StringUtils.format("Retrieved worklfow run data of id %d", wfRunId));

        try {
            Thread.sleep(waitBeforeDeleteMsec);
        } catch (InterruptedException e) {
            // wrap for simplicity
            throw new RuntimeException("Interrupted while waiting", e);
        }

        if ((wfRunId % 2) == 0) {
            // Delete data references
            derbyMetaDataBackend.deleteWorkflowRunFiles(wfRunId);
            log.debug(StringUtils.format("Deleted files of workflow run id %d", wfRunId));
        } else {
            // Delete workflow run
            derbyMetaDataBackend.deleteWorkflowRun(wfRunId);
            log.debug(StringUtils.format("Deleted workflow run id %d", wfRunId));
        }
    }

    /** Test. */
    @Test
    public void testGetWorkflowRun() {
        Long wfRunId =
            derbyMetaDataBackend.addWorkflowRun(STRING_TEST_RUN, DataManagementIdMapping.createDummyNodeIdStringForTesting(),
                DataManagementIdMapping.createDummyNodeIdStringForTesting(),
                System.currentTimeMillis());
        Collection<ComponentInstance> componentInstances = createComponentInstances(4);
        Map<String, Long> componentInstanceIdMap = derbyMetaDataBackend.addComponentInstances(wfRunId, componentInstances);
        for (Long ciid : componentInstanceIdMap.values()) {
            derbyMetaDataBackend.addComponentRun(ciid, DataManagementIdMapping.createDummyNodeIdStringForTesting(), 1,
                System.currentTimeMillis());
        }
        WorkflowRun wfrun = derbyMetaDataBackend.getWorkflowRun(wfRunId);
        assertNotNull(wfrun);
    }

    /**
     * Test implementation of <code>ConfigurationService</code>.
     * 
     * @author Doreen Seider
     */
    private class DummyConfigurationService extends MockConfigurationService.ThrowExceptionByDefault {

        @Override
        @SuppressWarnings("unchecked")
        public <T> T getConfiguration(String identifier, Class<T> clazz) {
            if (identifier.equals(bundleName) && clazz == DerbyMetaDataBackendConfiguration.class) {
                return (T) new DerbyMetaDataBackendConfiguration();
            }
            return null;
        }

        @Override
        public File getProfileDirectory() {
            return tempDirectory;
        }

        @Override
        public File getConfigurablePath(ConfigurablePathId pathId) {
            assertEquals(ConfigurablePathId.PROFILE_DATA_MANAGEMENT, pathId);
            return tempDirectory;
        }

    }

}