/*******************************************************************************
* Copyright (c) 2011 The Board of Trustees of the Leland Stanford Junior University
* as Operator of the SLAC National Accelerator Laboratory.
* Copyright (c) 2011 Brookhaven National Laboratory.
* EPICS archiver appliance is distributed subject to a Software License Agreement found
* in file LICENSE that is included with this distribution.
*******************************************************************************/
package org.epics.archiverappliance.etl;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.sql.Timestamp;
import java.util.LinkedList;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import org.epics.archiverappliance.Event;
import org.epics.archiverappliance.EventStream;
import org.epics.archiverappliance.common.BasicContext;
import org.epics.archiverappliance.common.PartitionGranularity;
import org.epics.archiverappliance.common.TimeUtils;
import org.epics.archiverappliance.common.YearSecondTimestamp;
import org.epics.archiverappliance.config.ArchDBRTypes;
import org.epics.archiverappliance.config.ConfigServiceForTests;
import org.epics.archiverappliance.config.PVNameToKeyMapping;
import org.epics.archiverappliance.config.PVTypeInfo;
import org.epics.archiverappliance.config.StoragePluginURLParser;
import org.epics.archiverappliance.data.ScalarValue;
import org.epics.archiverappliance.engine.membuf.ArrayListEventStream;
import org.epics.archiverappliance.retrieval.RemotableEventStreamDesc;
import org.epics.archiverappliance.retrieval.workers.CurrentThreadWorkerEventStream;
import org.epics.archiverappliance.utils.nio.ArchPaths;
import org.epics.archiverappliance.utils.simulation.SimulationEvent;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import edu.stanford.slac.archiverappliance.PlainPB.PlainPBPathNameUtility;
import edu.stanford.slac.archiverappliance.PlainPB.PlainPBStoragePlugin;
import edu.stanford.slac.archiverappliance.PlainPB.PlainPBStoragePlugin.CompressionMode;
import edu.stanford.slac.archiverappliance.PlainPB.utils.ValidatePBFile;
import junit.framework.TestCase;
/**
* Occasionally, we seem to get files that are 0 bytes long; this usually happens in unusual circumstances.
* For example, Terry reported this happening around the time IT changed the network config incorrectly.
* Once we get a zero byte file in the ETL dest, we seem to be struck as we try to determine the last known timestamp in the file and fail.
* This tests zero byte files in both the source and the dest.
* @author mshankar
*
*/
public class ZeroByteFilesTest extends TestCase {
private static final Logger logger = Logger.getLogger(ZeroByteFilesTest.class);
private String shortTermFolderName=ConfigServiceForTests.getDefaultShortTermFolder()+"/shortTerm";
private String mediumTermFolderName=ConfigServiceForTests.getDefaultPBTestFolder()+"/mediumTerm";
private ConfigServiceForTests configService;
private PVNameToKeyMapping pvNameToKeyConverter;
private PlainPBStoragePlugin etlSrc;
private PlainPBStoragePlugin etlDest;
private short currentYear = TimeUtils.getCurrentYear();
@Before
public void setUp() throws Exception {
configService = new ConfigServiceForTests(new File("./bin"));
configService.getETLLookup().manualControlForUnitTests();
cleanUpDataFolders();
pvNameToKeyConverter = configService.getPVNameToKeyConverter();
etlSrc = (PlainPBStoragePlugin) StoragePluginURLParser.parseStoragePlugin("pb://localhost?name=STS&rootFolder=" + shortTermFolderName + "/&partitionGranularity=PARTITION_DAY", configService);
etlDest = (PlainPBStoragePlugin) StoragePluginURLParser.parseStoragePlugin("pb://localhost?name=MTS&rootFolder=" + mediumTermFolderName + "/&partitionGranularity=PARTITION_YEAR", configService);
}
@After
public void tearDown() throws Exception {
cleanUpDataFolders();
}
public void cleanUpDataFolders() throws Exception {
if(new File(shortTermFolderName).exists()) {
FileUtils.deleteDirectory(new File(shortTermFolderName));
}
if(new File(mediumTermFolderName).exists()) {
FileUtils.deleteDirectory(new File(mediumTermFolderName));
}
}
@Test
public void testZeroByteETL() throws Exception {
testZeroByteFileInDest();
testZeroByteFilesInSource();
}
@FunctionalInterface
public interface VoidFunction {
void apply() throws IOException;
}
public void testZeroByteFileInDest() throws Exception {
String pvName = ConfigServiceForTests.ARCH_UNIT_TEST_PVNAME_PREFIX + "ETL_testZeroDest";
// Create an zero byte file in the ETL dest
VoidFunction zeroByteGenerator = () -> {
Path zeroDestPath = Paths.get(etlDest.getRootFolder(), pvNameToKeyConverter.convertPVNameToKey(pvName) + currentYear + PlainPBStoragePlugin.PB_EXTENSION);
logger.info("Creating zero byte file " + zeroDestPath);
Files.write(zeroDestPath, new byte[0], StandardOpenOption.CREATE);
};
runETLAndValidate(pvName, zeroByteGenerator);
}
public void testZeroByteFilesInSource() throws Exception {
// Create zero byte files in the ETL source; since this is a daily partition, we need something like so sine:2016_03_31.pb
String pvName = ConfigServiceForTests.ARCH_UNIT_TEST_PVNAME_PREFIX + "ETL_testZeroSrc";
VoidFunction zeroByteGenerator = () -> {
for(int day = 2; day < 10; day++) {
Path zeroSrcPath = Paths.get(etlSrc.getRootFolder(), pvNameToKeyConverter.convertPVNameToKey(pvName) + TimeUtils.getPartitionName(TimeUtils.getCurrentEpochSeconds() - day*86400, PartitionGranularity.PARTITION_DAY) + PlainPBStoragePlugin.PB_EXTENSION);
logger.info("Creating zero byte file " + zeroSrcPath);
Files.write(zeroSrcPath, new byte[0], StandardOpenOption.CREATE);
}
};
runETLAndValidate(pvName, zeroByteGenerator);
}
/**
* Generates some data in STS; then calls the ETL to move it to MTS which has a zero byte file.
*/
public void runETLAndValidate(String pvName, VoidFunction zeroByteGenerationFunction) throws Exception {
// Generate some data in the src
int totalSamples = 1024;
long currentSeconds = TimeUtils.getCurrentEpochSeconds();
ArrayListEventStream srcData = new ArrayListEventStream(totalSamples, new RemotableEventStreamDesc(ArchDBRTypes.DBR_SCALAR_DOUBLE, pvName, currentYear));
for(int i = 0; i < totalSamples; i++) {
YearSecondTimestamp yts = TimeUtils.convertToYearSecondTimestamp(currentSeconds);
srcData.add(new SimulationEvent(yts.getSecondsintoyear(), yts.getYear(), ArchDBRTypes.DBR_SCALAR_DOUBLE, new ScalarValue<Double>(Math.sin((double)yts.getSecondsintoyear()))));
currentSeconds++;
}
try(BasicContext context = new BasicContext()) {
etlSrc.appendData(context, pvName, srcData);
}
logger.info("Done creating src data for PV " + pvName);
long beforeCount = 0;
List<Event> beforeEvents = new LinkedList<Event>();
try (BasicContext context = new BasicContext(); EventStream before = new CurrentThreadWorkerEventStream(pvName, etlSrc.getDataForPV(context, pvName, TimeUtils.minusDays(TimeUtils.now(), 366), TimeUtils.plusDays(TimeUtils.now(), 366)))) {
for(Event e : before) {
beforeEvents.add(e.makeClone());
beforeCount++;
}
}
logger.debug("Calling lambda to generate zero byte files");
zeroByteGenerationFunction.apply();
// Register the PV
PVTypeInfo typeInfo = new PVTypeInfo(pvName, ArchDBRTypes.DBR_SCALAR_DOUBLE, true, 1);
String[] dataStores = new String[] { etlSrc.getURLRepresentation(), etlDest.getURLRepresentation() };
typeInfo.setDataStores(dataStores);
configService.updateTypeInfoForPV(pvName, typeInfo);
configService.registerPVToAppliance(pvName, configService.getMyApplianceInfo());
// Now do ETL...
Timestamp timeETLruns = TimeUtils.plusDays(TimeUtils.now(), 365*10);
ETLExecutor.runETLs(configService, timeETLruns);
logger.info("Done performing ETL as though today is " + TimeUtils.convertToHumanReadableString(timeETLruns));
// Validation starts here
Timestamp startOfRequest = TimeUtils.minusDays(TimeUtils.now(), 366);
Timestamp endOfRequest = TimeUtils.plusDays(TimeUtils.now(), 366);
// Check that all the files in the destination store are valid files.
Path[] allPaths = PlainPBPathNameUtility.getAllPathsForPV(new ArchPaths(), etlDest.getRootFolder(), pvName, ".pb", etlDest.getPartitionGranularity(), CompressionMode.NONE, pvNameToKeyConverter);
assertTrue("PlainPBFileNameUtility returns null for getAllFilesForPV for " + pvName, allPaths != null);
assertTrue("PlainPBFileNameUtility returns empty array for getAllFilesForPV for " + pvName + " when looking in " + etlDest.getRootFolder() , allPaths.length > 0);
for(Path destPath : allPaths) {
assertTrue("File validation failed for " + destPath.toAbsolutePath().toString(), ValidatePBFile.validatePBFile(destPath, false));
}
logger.info("Asking for data between"
+ TimeUtils.convertToHumanReadableString(startOfRequest)
+ " and "
+ TimeUtils.convertToHumanReadableString(endOfRequest)
);
long afterCount = 0;
try (BasicContext context = new BasicContext(); EventStream afterDest = new CurrentThreadWorkerEventStream(pvName, etlDest.getDataForPV(context, pvName, startOfRequest, endOfRequest))) {
assertNotNull(afterDest);
for(@SuppressWarnings("unused") Event e : afterDest) { afterCount++; }
}
logger.info("Of the " + beforeCount + " events, " + afterCount + " events were moved into the dest store.");
assertTrue("Seems like no events were moved by ETL " + afterCount, (afterCount != 0));
long afterSourceCount = 0;
try (BasicContext context = new BasicContext(); EventStream afterSrc = new CurrentThreadWorkerEventStream(pvName, etlSrc.getDataForPV(context, pvName, startOfRequest, endOfRequest))) {
for(@SuppressWarnings("unused") Event e : afterSrc) { afterSourceCount++; }
}
assertTrue("Seems like we still have " + afterSourceCount + " events in the source ", (afterSourceCount == 0));
// Now compare the events itself
try (BasicContext context = new BasicContext(); EventStream afterDest = new CurrentThreadWorkerEventStream(pvName, etlDest.getDataForPV(context, pvName, startOfRequest, endOfRequest))) {
int index = 0;
for(Event afterEvent : afterDest) {
Event beforeEvent = beforeEvents.get(index);
assertTrue("Before timestamp " + TimeUtils.convertToHumanReadableString(beforeEvent.getEventTimeStamp())
+ " After timestamp " + TimeUtils.convertToHumanReadableString(afterEvent.getEventTimeStamp()), beforeEvent.getEventTimeStamp().equals(afterEvent.getEventTimeStamp()));
assertTrue("Before value " + beforeEvent.getSampleValue().getValue()
+ " After value " + afterEvent.getSampleValue().getValue(), beforeEvent.getSampleValue().getValue().equals(afterEvent.getSampleValue().getValue()));
index++;
}
}
assertTrue("Of the total " + beforeCount + " event, we should have moved " + beforeCount + ". Instead we seem to have moved " + afterCount, beforeCount == afterCount);
}
}