ETLWithRecurringFilesTest.java

/*******************************************************************************
 * Copyright (c) 2011 The Board of Trustees of the Leland Stanford Junior University
 * as Operator of the SLAC National Accelerator Laboratory.
 * Copyright (c) 2011 Brookhaven National Laboratory.
 * EPICS archiver appliance is distributed subject to a Software License Agreement found
 * in file LICENSE that is included with this distribution.
 *******************************************************************************/
package org.epics.archiverappliance.etl;

import static org.junit.Assert.assertTrue;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.sql.Timestamp;

import org.apache.log4j.Logger;
import org.epics.archiverappliance.Event;
import org.epics.archiverappliance.EventStream;
import org.epics.archiverappliance.common.BasicContext;
import org.epics.archiverappliance.common.PartitionGranularity;
import org.epics.archiverappliance.common.TimeUtils;
import org.epics.archiverappliance.config.ArchDBRTypes;
import org.epics.archiverappliance.config.ConfigServiceForTests;
import org.epics.archiverappliance.config.PVTypeInfo;
import org.epics.archiverappliance.data.ScalarValue;
import org.epics.archiverappliance.engine.membuf.ArrayListEventStream;
import org.epics.archiverappliance.retrieval.RemotableEventStreamDesc;
import org.epics.archiverappliance.retrieval.workers.CurrentThreadWorkerEventStream;
import org.epics.archiverappliance.utils.nio.ArchPaths;
import org.epics.archiverappliance.utils.simulation.SimulationEvent;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import edu.stanford.slac.archiverappliance.PB.data.PBCommonSetup;
import edu.stanford.slac.archiverappliance.PlainPB.PlainPBPathNameUtility;
import edu.stanford.slac.archiverappliance.PlainPB.PlainPBStoragePlugin;
import edu.stanford.slac.archiverappliance.PlainPB.PlainPBStoragePlugin.CompressionMode;

/**
 * We want to test what happens when ETL encounters the same stream again; for example, when a delete fails. 
 * @author mshankar
 *
 */
public class ETLWithRecurringFilesTest {
	private static Logger logger = Logger.getLogger(ETLWithRecurringFilesTest.class.getName());

	@Before
	public void setUp() throws Exception {
	}

	@After
	public void tearDown() throws Exception {
	}

	@Test
	public void testRecurringFiles() throws Exception {
		for(PartitionGranularity granularity : PartitionGranularity.values()) {
			if(granularity.getNextLargerGranularity() == null) continue;
			testRecurringFiles(granularity, true, false);
			testRecurringFiles(granularity, false, false);
			testRecurringFiles(granularity, true, true);
			testRecurringFiles(granularity, false, true);
		}
	}

	/**
	 * @param granularity
	 * @param backUpfiles
	 * @param useNewDest - This creates a new ETLDest for the rerun. New dest should hopefully not have any state and should still work.
	 * @throws Exception
	 */
	private void testRecurringFiles(PartitionGranularity granularity, boolean backUpfiles, boolean useNewDest) throws Exception {
		PlainPBStoragePlugin etlSrc = new PlainPBStoragePlugin();
		PBCommonSetup srcSetup = new PBCommonSetup();
		PlainPBStoragePlugin etlDest = new PlainPBStoragePlugin();
		PBCommonSetup destSetup = new PBCommonSetup();
		PlainPBStoragePlugin etlNewDest = new PlainPBStoragePlugin();
		PBCommonSetup newDestSetup = new PBCommonSetup();
		ConfigServiceForTests configService = new ConfigServiceForTests(new File("./bin"), 1);
		etlDest.setBackupFilesBeforeETL(backUpfiles);

		srcSetup.setUpRootFolder(etlSrc, "RecurringFilesTestSrc"+granularity, granularity);
		destSetup.setUpRootFolder(etlDest, "RecurringFilesTestDest"+granularity, granularity.getNextLargerGranularity());
		newDestSetup.setUpRootFolder(etlNewDest, "RecurringFilesTestDest"+granularity, granularity.getNextLargerGranularity());

		logger.info("Testing recurring files for " + etlSrc.getPartitionGranularity() + " to " + etlDest.getPartitionGranularity() + " with backup = " + Boolean.toString(backUpfiles) + " and newDest = " + Boolean.toString(useNewDest));

		short year = TimeUtils.getCurrentYear();
		long startOfYearInEpochSeconds = TimeUtils.getStartOfCurrentYearInSeconds();
		long curEpochSeconds = startOfYearInEpochSeconds; 
		int secondsintoyear = 0;
		int incrementSeconds = 450;
		int eventsgenerated = 0;

		String pvName = ConfigServiceForTests.ARCH_UNIT_TEST_PVNAME_PREFIX + "ETL_testRecurringFiles" + etlSrc.getPartitionGranularity();
		{
			PVTypeInfo typeInfo = new PVTypeInfo(pvName, ArchDBRTypes.DBR_SCALAR_DOUBLE, true, 1);
			String[] dataStores = new String[] { etlSrc.getURLRepresentation(), etlDest.getURLRepresentation() }; 
			typeInfo.setDataStores(dataStores);
			configService.updateTypeInfoForPV(pvName, typeInfo);
			configService.registerPVToAppliance(pvName, configService.getMyApplianceInfo());
			configService.getETLLookup().manualControlForUnitTests();
		}
		
		// Generate 90 days worth of data
		int eventsPerShot = (60*60*24*90)/incrementSeconds;
		ArrayListEventStream instream = new ArrayListEventStream(eventsPerShot, new RemotableEventStreamDesc(ArchDBRTypes.DBR_SCALAR_DOUBLE, pvName, year));
		for(int i = 0; i < eventsPerShot; i++) {
			instream.add(new SimulationEvent(secondsintoyear, year, ArchDBRTypes.DBR_SCALAR_DOUBLE, new ScalarValue<Double>((double)secondsintoyear)));
			secondsintoyear += incrementSeconds;
			curEpochSeconds += incrementSeconds;
			eventsgenerated++;
		}
		
		try(BasicContext context = new BasicContext()) {
			etlSrc.appendData(context, pvName, instream);
		}
		
		// We should now have some data in the src root folder...
		// Make a copy of these files so that we can restore them back later after ETL.
		Path[] allSrcPaths = PlainPBPathNameUtility.getAllPathsForPV(new ArchPaths(), etlSrc.getRootFolder(), pvName, PlainPBStoragePlugin.PB_EXTENSION, etlSrc.getPartitionGranularity(), CompressionMode.NONE, configService.getPVNameToKeyConverter());
		for(Path srcPath : allSrcPaths) {
			Path destPath = srcPath.resolveSibling(srcPath.getFileName().toString().replace(".pb", ".etltest"));
			logger.debug("Path for backup is " + destPath.toString());
			Files.copy(srcPath, destPath, StandardCopyOption.REPLACE_EXISTING);
		}

		// Run ETL as if it was a week later.
		long ETLIsRunningAtEpochSeconds = curEpochSeconds + 24*60*60*7;
		// Now do ETL and the srcFiles should have disappeared.
		ETLExecutor.runETLs(configService, TimeUtils.convertFromEpochSeconds(ETLIsRunningAtEpochSeconds, 0));
		
		// Restore the files from the backup and run ETL again.
		for(Path srcPath : allSrcPaths) {
			assertTrue(srcPath.toAbsolutePath().toString() + " was not deleted in the last run", !srcPath.toFile().exists());
			Path destPath = srcPath.resolveSibling(srcPath.getFileName().toString().replace(".pb", ".etltest"));
			Files.copy(destPath, srcPath, StandardCopyOption.COPY_ATTRIBUTES);
		}
		
		if(useNewDest) {
			ConfigServiceForTests newConfigService = new ConfigServiceForTests(new File("./bin"), 1);
			PVTypeInfo typeInfo2 = new PVTypeInfo(pvName, ArchDBRTypes.DBR_SCALAR_DOUBLE, true, 1);
			String[] dataStores2 = new String[] { etlSrc.getURLRepresentation(), etlNewDest.getURLRepresentation() }; 
			typeInfo2.setDataStores(dataStores2);
			newConfigService.registerPVToAppliance(pvName, newConfigService.getMyApplianceInfo());
			newConfigService.updateTypeInfoForPV(pvName, typeInfo2);
			newConfigService.getETLLookup().manualControlForUnitTests();
			
			logger.debug("Running ETL again against a new plugin; the debug logs should see a lot of skipping events messages from here on.");
			ETLExecutor.runETLs(configService, TimeUtils.convertFromEpochSeconds(ETLIsRunningAtEpochSeconds, 0));
			checkDataValidity(pvName, etlSrc, etlNewDest, startOfYearInEpochSeconds, incrementSeconds, eventsgenerated, granularity.toString() + "/" + Boolean.toString(backUpfiles));
			
		} else {
			logger.debug("Running ETL again; the debug logs should see a lot of skipping events messages from here on.");
			ETLExecutor.runETLs(configService, TimeUtils.convertFromEpochSeconds(ETLIsRunningAtEpochSeconds, 0));
			checkDataValidity(pvName, etlSrc, etlDest, startOfYearInEpochSeconds, incrementSeconds, eventsgenerated, granularity.toString() + "/" + Boolean.toString(backUpfiles));
		}
		

		srcSetup.deleteTestFolder();
		destSetup.deleteTestFolder();
		newDestSetup.deleteTestFolder();
		
		configService.shutdownNow();
	}
	
	private void checkDataValidity(String pvName, PlainPBStoragePlugin etlSrc, PlainPBStoragePlugin etlDest, long startOfYearInEpochSeconds, int incrementSeconds, int eventsgenerated, String testDesc) throws IOException {
		Timestamp startOfRequest = TimeUtils.minusDays(TimeUtils.now(), 366);
		Timestamp endOfRequest = TimeUtils.plusDays(TimeUtils.now(), 366);

		logger.debug(testDesc + "Asking for data between" 
				+ TimeUtils.convertToHumanReadableString(startOfRequest) 
				+ " and " 
				+ TimeUtils.convertToHumanReadableString(endOfRequest)
				);

		long expectedEpochSeconds = startOfYearInEpochSeconds;
		int afterCount = 0;
		
		try (BasicContext context = new BasicContext(); EventStream afterDest = new CurrentThreadWorkerEventStream(pvName, etlDest.getDataForPV(context, pvName, startOfRequest, endOfRequest))) {
			if(afterDest != null) { 
				for(Event e : afterDest) {
					assertTrue(testDesc + "Dest Expected seconds " + TimeUtils.convertToHumanReadableString(expectedEpochSeconds) + " is not the same as actual seconds " + TimeUtils.convertToHumanReadableString(e.getEpochSeconds()) + " for afterCount " + afterCount, (expectedEpochSeconds == e.getEpochSeconds()));
					expectedEpochSeconds += incrementSeconds;
					afterCount++;
				}
				assertTrue(testDesc + "Seems like no events were moved by ETL " + afterCount, (afterCount != 0));
			}
		}
		
		
		try (BasicContext context = new BasicContext(); EventStream afterSrc = new CurrentThreadWorkerEventStream(pvName, etlSrc.getDataForPV(context, pvName, startOfRequest, endOfRequest))) {
			if(afterSrc != null) {
				for(Event e : afterSrc) { 
					assertTrue(testDesc + "Src Expected seconds " + TimeUtils.convertToHumanReadableString(expectedEpochSeconds) + " is not the same as actual seconds " + TimeUtils.convertToHumanReadableString(e.getEpochSeconds()) + " for afterCount " + afterCount, (expectedEpochSeconds == e.getEpochSeconds()));
					expectedEpochSeconds += incrementSeconds;
					afterCount++;
				}
			}
		}
		
		assertTrue(testDesc + "Expected total events " + eventsgenerated + " is not the same as actual events " + afterCount, (eventsgenerated == afterCount));
	}
}