/*******************************************************************************
* Copyright (c) 2011 The Board of Trustees of the Leland Stanford Junior University
* as Operator of the SLAC National Accelerator Laboratory.
* Copyright (c) 2011 Brookhaven National Laboratory.
* EPICS archiver appliance is distributed subject to a Software License Agreement found
* in file LICENSE that is included with this distribution.
*******************************************************************************/
package org.epics.archiverappliance.etl;
import java.io.File;
import java.io.IOException;
import java.sql.Timestamp;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.Callable;
import junit.framework.TestCase;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import org.epics.archiverappliance.Event;
import org.epics.archiverappliance.EventStream;
import org.epics.archiverappliance.common.BasicContext;
import org.epics.archiverappliance.common.POJOEvent;
import org.epics.archiverappliance.common.TimeUtils;
import org.epics.archiverappliance.common.YearSecondTimestamp;
import org.epics.archiverappliance.config.ArchDBRTypes;
import org.epics.archiverappliance.config.ConfigServiceForTests;
import org.epics.archiverappliance.config.PVTypeInfo;
import org.epics.archiverappliance.config.StoragePluginURLParser;
import org.epics.archiverappliance.data.ScalarValue;
import org.epics.archiverappliance.engine.membuf.ArrayListEventStream;
import org.epics.archiverappliance.retrieval.RemotableEventStreamDesc;
import org.epics.archiverappliance.retrieval.postprocessors.PostProcessor;
import org.epics.archiverappliance.retrieval.postprocessors.PostProcessorWithConsolidatedEventStream;
import org.epics.archiverappliance.retrieval.postprocessors.PostProcessors;
import org.epics.archiverappliance.retrieval.workers.CurrentThreadWorkerEventStream;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import edu.stanford.slac.archiverappliance.PlainPB.PlainPBStoragePlugin;
/**
* Variation of DataReductionDailyETLTest; except we test multiple post processors
* @author mshankar
*
*/
public class DataReductionPostProcessorsTest extends TestCase {
private static final Logger logger = Logger.getLogger(DataReductionPostProcessorsTest.class);
String shortTermFolderName=ConfigServiceForTests.getDefaultShortTermFolder()+"/shortTerm";
String mediumTermFolderName=ConfigServiceForTests.getDefaultPBTestFolder()+"/mediumTerm";
String longTermFolderName=ConfigServiceForTests.getDefaultPBTestFolder()+"/longTerm";
private String rawPVName = ConfigServiceForTests.ARCH_UNIT_TEST_PVNAME_PREFIX + DataReductionPostProcessorsTest.class.getSimpleName();
private String reducedPVName = ConfigServiceForTests.ARCH_UNIT_TEST_PVNAME_PREFIX + DataReductionPostProcessorsTest.class.getSimpleName() + "reduced";
@Before
public void setUp() throws Exception {
cleanDataFolders();
}
private void cleanDataFolders() throws IOException {
if(new File(shortTermFolderName).exists()) {
FileUtils.deleteDirectory(new File(shortTermFolderName));
}
if(new File(mediumTermFolderName).exists()) {
FileUtils.deleteDirectory(new File(mediumTermFolderName));
}
if(new File(longTermFolderName).exists()) {
FileUtils.deleteDirectory(new File(longTermFolderName));
}
}
@After
public void tearDown() throws Exception {
cleanDataFolders();
}
@Test
public void testReducedETL() throws Exception {
String[] postProcessors = new String[] {
// No fill versions
"lastSample_3600",
"firstSample_3600",
"firstSample_600",
"lastSample_600",
"meanSample_3600",
"meanSample_600",
"meanSample_1800",
"minSample_3600",
"maxSample_3600",
"medianSample_3600",
// Fill versions
"mean_3600",
"mean_600",
"mean_1800",
"min_3600",
"max_3600",
"median_3600",
"firstFill_3600",
"lastFill_3600"
};
for(String reduceDataUsing : postProcessors) {
testPostProcessor(reduceDataUsing);
}
}
/**
* 1) Set up the raw and reduced PV's
* 2) Generate data in STS
* 3) Run ETL
* 4) Compare
*/
private void testPostProcessor(String reduceDataUsing) throws Exception {
cleanDataFolders();
ConfigServiceForTests configService = new ConfigServiceForTests(new File("./bin"), 1);
// Set up the raw and reduced PV's
PlainPBStoragePlugin etlSTS = (PlainPBStoragePlugin) StoragePluginURLParser.parseStoragePlugin("pb://localhost?name=STS&rootFolder=" + shortTermFolderName + "/&partitionGranularity=PARTITION_HOUR", configService);;
PlainPBStoragePlugin etlMTS = (PlainPBStoragePlugin) StoragePluginURLParser.parseStoragePlugin("pb://localhost?name=MTS&rootFolder=" + mediumTermFolderName + "/&partitionGranularity=PARTITION_DAY", configService);
PlainPBStoragePlugin etlLTSRaw = (PlainPBStoragePlugin) StoragePluginURLParser.parseStoragePlugin("pb://localhost?name=LTS&rootFolder=" + longTermFolderName + "/&partitionGranularity=PARTITION_YEAR", configService);
PlainPBStoragePlugin etlLTSReduced = (PlainPBStoragePlugin) StoragePluginURLParser.parseStoragePlugin("pb://localhost?name=LTS&rootFolder=" + longTermFolderName + "/&partitionGranularity=PARTITION_YEAR&reducedata=" + reduceDataUsing, configService);
{
PVTypeInfo typeInfo = new PVTypeInfo(rawPVName, ArchDBRTypes.DBR_SCALAR_DOUBLE, true, 1);
String[] dataStores = new String[] { etlSTS.getURLRepresentation(), etlMTS.getURLRepresentation(), etlLTSRaw.getURLRepresentation() };
typeInfo.setDataStores(dataStores);
typeInfo.setPaused(true);
configService.updateTypeInfoForPV(rawPVName, typeInfo);
configService.registerPVToAppliance(rawPVName, configService.getMyApplianceInfo());
}
{
PVTypeInfo typeInfo = new PVTypeInfo(reducedPVName, ArchDBRTypes.DBR_SCALAR_DOUBLE, true, 1);
String[] dataStores = new String[] { etlSTS.getURLRepresentation(), etlMTS.getURLRepresentation(), etlLTSReduced.getURLRepresentation() };
typeInfo.setDataStores(dataStores);
typeInfo.setPaused(true);
configService.updateTypeInfoForPV(reducedPVName, typeInfo);
configService.registerPVToAppliance(reducedPVName, configService.getMyApplianceInfo());
}
// Control ETL manually
configService.getETLLookup().manualControlForUnitTests();
short currentYear = TimeUtils.getCurrentYear();
logger.info("Testing data reduction for postprocessor " + reduceDataUsing);
for(int day = 0; day < 40; day++) {
// Generate data into the STS on a daily basis
ArrayListEventStream genDataRaw = new ArrayListEventStream(86400, new RemotableEventStreamDesc(ArchDBRTypes.DBR_SCALAR_DOUBLE, rawPVName, currentYear));
ArrayListEventStream genDataReduced = new ArrayListEventStream(86400, new RemotableEventStreamDesc(ArchDBRTypes.DBR_SCALAR_DOUBLE, reducedPVName, currentYear));
for(int second = 0; second < 86400; second++) {
YearSecondTimestamp ysts = new YearSecondTimestamp(currentYear, day*86400 + second, 0);
Timestamp ts = TimeUtils.convertFromYearSecondTimestamp(ysts);
genDataRaw.add(new POJOEvent(ArchDBRTypes.DBR_SCALAR_DOUBLE, ts, new ScalarValue<Double>(second*1.0),0, 0));
genDataReduced.add(new POJOEvent(ArchDBRTypes.DBR_SCALAR_DOUBLE, ts, new ScalarValue<Double>(second*1.0),0, 0));
}
try(BasicContext context = new BasicContext()) {
etlSTS.appendData(context, rawPVName, genDataRaw);
etlSTS.appendData(context, reducedPVName, genDataReduced);
}
logger.debug("For postprocessor " + reduceDataUsing + " done generating data into the STS for day " + day);
// Run ETL at the end of the day
Timestamp timeETLruns = TimeUtils.convertFromYearSecondTimestamp(new YearSecondTimestamp(currentYear, day*86400 + 86399, 0));
ETLExecutor.runETLs(configService, timeETLruns);
logger.debug("For postprocessor " + reduceDataUsing + " done performing ETL as though today is " + TimeUtils.convertToHumanReadableString(timeETLruns));
// Compare data for raw+postprocessor and reduced PV's.
PostProcessor postProcessor = PostProcessors.findPostProcessor(reduceDataUsing);
postProcessor.initialize(reduceDataUsing, rawPVName);
int rawWithPPCount = 0;
int reducedCount = 0;
try (BasicContext context = new BasicContext()) {
Timestamp startTime = TimeUtils.minusDays(TimeUtils.now(), 10*366);
Timestamp endTime = TimeUtils.plusDays(TimeUtils.now(), 10*366);
LinkedList<Timestamp> rawTimestamps = new LinkedList<Timestamp>();
LinkedList<Timestamp> reducedTimestamps = new LinkedList<Timestamp>();
if(postProcessor instanceof PostProcessorWithConsolidatedEventStream) {
List<Callable<EventStream>> callables = etlLTSRaw.getDataForPV(context, rawPVName, startTime, endTime, postProcessor);
for(Callable<EventStream> callable : callables) {
callable.call();
}
for(Event e : ((PostProcessorWithConsolidatedEventStream) postProcessor).getConsolidatedEventStream()) {
rawTimestamps.add(e.getEventTimeStamp());
rawWithPPCount++;
}
} else {
try(EventStream rawWithPP = new CurrentThreadWorkerEventStream(rawPVName, etlLTSRaw.getDataForPV(context, rawPVName, startTime, endTime, postProcessor))) {
for(Event e : rawWithPP) {
rawTimestamps.add(e.getEventTimeStamp());
rawWithPPCount++;
}
}
}
try(EventStream reduced = new CurrentThreadWorkerEventStream(reducedPVName, etlLTSReduced.getDataForPV(context, reducedPVName, startTime, endTime))) {
for(Event e : reduced) {
reducedTimestamps.add(e.getEventTimeStamp());
reducedCount++;
}
}
logger.debug("For postprocessor " + reduceDataUsing + " for day " + day + " we have " + rawWithPPCount + " raw with postprocessor events and " + reducedCount + " reduced events");
if(rawTimestamps.size() != reducedTimestamps.size()) {
while(!rawTimestamps.isEmpty() || !reducedTimestamps.isEmpty()) {
if(!rawTimestamps.isEmpty()) logger.info("Raw/PP " + TimeUtils.convertToHumanReadableString(rawTimestamps.pop()));
if(!reducedTimestamps.isEmpty()) logger.info("Reduced" + TimeUtils.convertToHumanReadableString(reducedTimestamps.pop()));
}
}
assertTrue("For postprocessor " + reduceDataUsing + " for day " + day + " we have " + rawWithPPCount + " rawWithPP events and " + reducedCount + " reduced events", rawWithPPCount == reducedCount);
}
if(day > 2) {
assertTrue("For postprocessor " + reduceDataUsing + " for day " + day + ", seems like no events were moved by ETL into LTS for " + rawPVName + " Count = " + rawWithPPCount, (rawWithPPCount != 0));
assertTrue("For postprocessor " + reduceDataUsing + " for day " + day + ", seems like no events were moved by ETL into LTS for " + reducedPVName + " Count = " + reducedCount, (reducedCount != 0));
}
}
configService.shutdownNow();
}
}