GradualAccumulationETLTest.java

/*******************************************************************************
 * Copyright (c) 2011 The Board of Trustees of the Leland Stanford Junior University
 * as Operator of the SLAC National Accelerator Laboratory.
 * Copyright (c) 2011 Brookhaven National Laboratory.
 * EPICS archiver appliance is distributed subject to a Software License Agreement found
 * in file LICENSE that is included with this distribution.
 *******************************************************************************/
package org.epics.archiverappliance.etl;

import static org.junit.Assert.assertTrue;

import java.io.File;
import java.io.IOException;
import java.sql.Timestamp;

import org.apache.log4j.Logger;
import org.epics.archiverappliance.Event;
import org.epics.archiverappliance.EventStream;
import org.epics.archiverappliance.common.BasicContext;
import org.epics.archiverappliance.common.PartitionGranularity;
import org.epics.archiverappliance.common.TimeUtils;
import org.epics.archiverappliance.config.ArchDBRTypes;
import org.epics.archiverappliance.config.ConfigServiceForTests;
import org.epics.archiverappliance.config.DefaultConfigService;
import org.epics.archiverappliance.config.PVTypeInfo;
import org.epics.archiverappliance.data.ScalarValue;
import org.epics.archiverappliance.engine.membuf.ArrayListEventStream;
import org.epics.archiverappliance.retrieval.RemotableEventStreamDesc;
import org.epics.archiverappliance.retrieval.workers.CurrentThreadWorkerEventStream;
import org.epics.archiverappliance.utils.simulation.SimulationEvent;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import edu.stanford.slac.archiverappliance.PB.data.PBCommonSetup;
import edu.stanford.slac.archiverappliance.PlainPB.PlainPBStoragePlugin;

/**
 * Complex test for testing ETL as it is supposed to happen in production.
 * @author mshankar
 *
 */
public class GradualAccumulationETLTest {
	private static Logger logger = Logger.getLogger(GradualAccumulationETLTest.class.getName());

	@Before
	public void setUp() throws Exception {
	}

	@After
	public void tearDown() throws Exception {
	}

	
	/**
	 * In a running appserver, the sequence is expected to be as follows
	 * <ol>
	 * <li>The engine generates data for a PV at a certain rate.</li>
	 * <li>ETL runs every so often and copies a subset (the older portion) of the data over to the destination store.</li>
	 * <li>The engine continues to generate data.</li>
	 * </ol>
	 * This aims to test that. We generate data and then fake the time for ETL
	 * @throws Exception
	 */
	@Test
	public void testGradualAccumulation() throws Exception {
		for(PartitionGranularity granularity : PartitionGranularity.values()) {
			if(granularity.getNextLargerGranularity() == null) continue;
			testGradualAccumulation(granularity, true);
			testGradualAccumulation(granularity, false);
		}
	}
	
	private void testGradualAccumulation(PartitionGranularity granularity, boolean backUpfiles) throws Exception {
		PlainPBStoragePlugin etlSrc = new PlainPBStoragePlugin();
		PBCommonSetup srcSetup = new PBCommonSetup();
		PlainPBStoragePlugin etlDest = new PlainPBStoragePlugin();
		PBCommonSetup destSetup = new PBCommonSetup();
		DefaultConfigService configService = new ConfigServiceForTests(new File("./bin"), 1);
		etlDest.setBackupFilesBeforeETL(backUpfiles);

		srcSetup.setUpRootFolder(etlSrc, "GradualAccumulationETLTestSrc_"+granularity, granularity);
		destSetup.setUpRootFolder(etlDest, "GradualAccumulationETLTestDest"+granularity, granularity.getNextLargerGranularity());

		logger.info("Testing gradual accumulation for " + etlSrc.getPartitionGranularity() + " to " + etlDest.getPartitionGranularity() + " with backup = " + Boolean.toString(backUpfiles));

		short year = TimeUtils.getCurrentYear();
		long startOfYearInEpochSeconds = TimeUtils.getStartOfCurrentYearInSeconds();
		long curEpochSeconds = startOfYearInEpochSeconds; 
		int secondsintoyear = 0;
		int incrementSeconds = 450;
		int eventsgenerated = 0;

		String pvName = ConfigServiceForTests.ARCH_UNIT_TEST_PVNAME_PREFIX + "ETL_testGradual" + etlSrc.getPartitionGranularity();

		PVTypeInfo typeInfo = new PVTypeInfo(pvName, ArchDBRTypes.DBR_SCALAR_DOUBLE, true, 1);
		String[] dataStores = new String[] { etlSrc.getURLRepresentation(), etlDest.getURLRepresentation() }; 
		typeInfo.setDataStores(dataStores);
		configService.updateTypeInfoForPV(pvName, typeInfo);
		configService.registerPVToAppliance(pvName, configService.getMyApplianceInfo());
		configService.getETLLookup().manualControlForUnitTests();

		while(secondsintoyear < 60*60*24*365) {
			// The 60x60X20 is to generate 20 hours worth of data in each shot.
			int eventsPerShot = Math.max(60*60*20, granularity.getApproxSecondsPerChunk()*2)/incrementSeconds;
			ArrayListEventStream instream = new ArrayListEventStream(eventsPerShot, new RemotableEventStreamDesc(ArchDBRTypes.DBR_SCALAR_DOUBLE, pvName, year));
			for(int i = 0; i < eventsPerShot; i++) {
				instream.add(new SimulationEvent(secondsintoyear, year, ArchDBRTypes.DBR_SCALAR_DOUBLE, new ScalarValue<Double>((double)secondsintoyear)));
				secondsintoyear += incrementSeconds;
				curEpochSeconds += incrementSeconds;
				eventsgenerated++;
			}
			try(BasicContext context = new BasicContext()) {
				etlSrc.appendData(context, pvName, instream);
			}
			ETLExecutor.runETLs(configService, TimeUtils.convertFromEpochSeconds(curEpochSeconds, 0));
			logger.debug("Done performing ETL");
			checkDataValidity(pvName, etlSrc, etlDest, startOfYearInEpochSeconds, incrementSeconds, eventsgenerated, granularity.toString() + "/" + Boolean.toString(backUpfiles));
		}
		

		srcSetup.deleteTestFolder();
		destSetup.deleteTestFolder();
		configService.shutdownNow();
	}
	
	private void checkDataValidity(String pvName, PlainPBStoragePlugin etlSrc, PlainPBStoragePlugin etlDest, long startOfYearInEpochSeconds, int incrementSeconds, int eventsgenerated, String testDesc) throws IOException {
		Timestamp startOfRequest = TimeUtils.minusDays(TimeUtils.now(), 2*366);
		Timestamp endOfRequest = TimeUtils.plusDays(TimeUtils.now(), 2*366);

		logger.debug(testDesc + "Asking for data between" 
				+ TimeUtils.convertToHumanReadableString(startOfRequest) 
				+ " and " 
				+ TimeUtils.convertToHumanReadableString(endOfRequest)
				);

		long expectedEpochSeconds = startOfYearInEpochSeconds;
		int afterCount = 0;
		
		try (BasicContext context = new BasicContext(); EventStream afterDest = new CurrentThreadWorkerEventStream(pvName, etlDest.getDataForPV(context, pvName, startOfRequest, endOfRequest))) {
			if(afterDest != null) { 
				for(Event e : afterDest) {
					assertTrue(testDesc + "Expected seconds " + TimeUtils.convertToHumanReadableString(expectedEpochSeconds) + " is not the same as actual seconds " + TimeUtils.convertToHumanReadableString(e.getEpochSeconds()), (expectedEpochSeconds == e.getEpochSeconds()));
					expectedEpochSeconds += incrementSeconds;
					afterCount++;
				}
				assertTrue(testDesc + "Seems like no events were moved by ETL " + afterCount, (afterCount != 0));
			}
		}
		
		
		try (BasicContext context = new BasicContext(); EventStream afterSrc = new CurrentThreadWorkerEventStream(pvName, etlSrc.getDataForPV(context, pvName, startOfRequest, endOfRequest))) {
			if(afterSrc != null) {
				for(Event e : afterSrc) { 
					assertTrue(testDesc + "Expected seconds " + TimeUtils.convertToHumanReadableString(expectedEpochSeconds) + " is not the same as actual seconds " + TimeUtils.convertToHumanReadableString(e.getEpochSeconds()), (expectedEpochSeconds == e.getEpochSeconds()));
					expectedEpochSeconds += incrementSeconds;
					afterCount++;
				}
			}
		}
		
		assertTrue(testDesc + "Expected total events " + eventsgenerated + " is not the same as actual events " + afterCount, (eventsgenerated == afterCount));
	}
}