BlackHoleETLTest.java

/*******************************************************************************
 * Copyright (c) 2011 The Board of Trustees of the Leland Stanford Junior University
 * as Operator of the SLAC National Accelerator Laboratory.
 * Copyright (c) 2011 Brookhaven National Laboratory.
 * EPICS archiver appliance is distributed subject to a Software License Agreement found
 * in file LICENSE that is included with this distribution.
 *******************************************************************************/
package org.epics.archiverappliance.etl;

import static org.junit.Assert.assertTrue;

import java.io.File;
import java.nio.file.Path;

import org.apache.log4j.Logger;
import org.epics.archiverappliance.common.BasicContext;
import org.epics.archiverappliance.common.PartitionGranularity;
import org.epics.archiverappliance.common.TimeUtils;
import org.epics.archiverappliance.config.ArchDBRTypes;
import org.epics.archiverappliance.config.ConfigService;
import org.epics.archiverappliance.config.ConfigServiceForTests;
import org.epics.archiverappliance.config.PVTypeInfo;
import org.epics.archiverappliance.data.ScalarValue;
import org.epics.archiverappliance.engine.membuf.ArrayListEventStream;
import org.epics.archiverappliance.retrieval.RemotableEventStreamDesc;
import org.epics.archiverappliance.utils.blackhole.BlackholeStoragePlugin;
import org.epics.archiverappliance.utils.nio.ArchPaths;
import org.epics.archiverappliance.utils.simulation.SimulationEvent;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import edu.stanford.slac.archiverappliance.PB.data.PBCommonSetup;
import edu.stanford.slac.archiverappliance.PlainPB.PlainPBPathNameUtility;
import edu.stanford.slac.archiverappliance.PlainPB.PlainPBStoragePlugin;
import edu.stanford.slac.archiverappliance.PlainPB.PlainPBStoragePlugin.CompressionMode;

/**
 * Test pouring data into the black hole plugin.
 * @author mshankar
 *
 */
public class BlackHoleETLTest {
	private static Logger logger = Logger.getLogger(GradualAccumulationETLTest.class.getName());
	private ConfigService configService;

	@Before
	public void setUp() throws Exception {
		configService = new ConfigServiceForTests(new File("./bin"));
	}

	@After
	public void tearDown() throws Exception {
	}

	
	/**
	 * Variant of the gradual accumulation test where the destination is a black hole plugin
	 * @throws Exception
	 */
	@Test
	public void testBlackHoleETL() throws Exception {
		for(PartitionGranularity granularity : PartitionGranularity.values()) {
			if(granularity.getNextLargerGranularity() == null) continue;
			testBlackHoleETL(granularity);
		}
	}
	
	private void testBlackHoleETL(PartitionGranularity granularity) throws Exception {
		PlainPBStoragePlugin etlSrc = new PlainPBStoragePlugin();
		PBCommonSetup srcSetup = new PBCommonSetup();
		BlackholeStoragePlugin etlDest = new BlackholeStoragePlugin();
		ConfigServiceForTests configService = new ConfigServiceForTests(new File("./bin"));

		srcSetup.setUpRootFolder(etlSrc, "BlackholeETLTestSrc_"+granularity, granularity);

		logger.info("Testing black hole for " + etlSrc.getPartitionGranularity() + " to " + etlDest.getPartitionGranularity());

		short year = TimeUtils.getCurrentYear();
		long startOfYearInEpochSeconds = TimeUtils.getStartOfCurrentYearInSeconds();
		long curEpochSeconds = startOfYearInEpochSeconds; 
		int secondsintoyear = 0;
		int incrementSeconds = 450;

		String pvName = ConfigServiceForTests.ARCH_UNIT_TEST_PVNAME_PREFIX + "ETL_blackhole" + etlSrc.getPartitionGranularity();
		
		PVTypeInfo typeInfo = new PVTypeInfo(pvName, ArchDBRTypes.DBR_SCALAR_DOUBLE, true, 1);
		String[] dataStores = new String[] { etlSrc.getURLRepresentation(), etlDest.getURLRepresentation() }; 
		typeInfo.setDataStores(dataStores);
		configService.updateTypeInfoForPV(pvName, typeInfo);
		configService.registerPVToAppliance(pvName, configService.getMyApplianceInfo());
		configService.getETLLookup().manualControlForUnitTests();

		try(BasicContext context = new BasicContext()) {
			while(secondsintoyear < 60*60*24*365) {
				int eventsPerShot = (60*60*24*31)/incrementSeconds;
				ArrayListEventStream instream = new ArrayListEventStream(eventsPerShot, new RemotableEventStreamDesc(ArchDBRTypes.DBR_SCALAR_DOUBLE, pvName, year));
				for(int i = 0; i < eventsPerShot; i++) {
					instream.add(new SimulationEvent(secondsintoyear, year, ArchDBRTypes.DBR_SCALAR_DOUBLE, new ScalarValue<Double>((double)secondsintoyear)));
					secondsintoyear += incrementSeconds;
					curEpochSeconds += incrementSeconds;
				}
				etlSrc.appendData(context, pvName, instream);
				int filesWithDataBefore = getFilesWithData(pvName, etlSrc);
				ETLExecutor.runETLs(configService, 	TimeUtils.convertFromEpochSeconds(curEpochSeconds, 0));
				logger.debug("Done performing ETL");
				int filesWithDataAfter = getFilesWithData(pvName, etlSrc);
				assertTrue("Black hole did not remove source files before = " + filesWithDataBefore + " and after = " + filesWithDataAfter + " for granularity " + granularity.toString(), filesWithDataAfter < filesWithDataBefore);
			}
		}

		srcSetup.deleteTestFolder();
	}
	
	private int getFilesWithData(String pvName, PlainPBStoragePlugin etlSrc) throws Exception {
		// Check that all the files in the destination store are valid files.
		Path[] allPaths = PlainPBPathNameUtility.getAllPathsForPV(new ArchPaths(), etlSrc.getRootFolder(), pvName, ".pb", etlSrc.getPartitionGranularity(), CompressionMode.NONE, configService.getPVNameToKeyConverter());
		return allPaths.length;
	}
	
}