package edu.stanford.slac.archiverappliance.PlainPB; import static org.junit.Assert.assertTrue; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.sql.Timestamp; import java.util.GregorianCalendar; import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; import org.epics.archiverappliance.Event; import org.epics.archiverappliance.common.BasicContext; import org.epics.archiverappliance.common.TimeUtils; import org.epics.archiverappliance.config.ArchDBRTypes; import org.epics.archiverappliance.config.ConfigServiceForTests; import org.epics.archiverappliance.config.StoragePluginURLParser; import org.epics.archiverappliance.data.ScalarValue; import org.epics.archiverappliance.engine.membuf.ArrayListEventStream; import org.epics.archiverappliance.retrieval.RemotableEventStreamDesc; import org.epics.archiverappliance.utils.simulation.SimulationEvent; import org.epics.archiverappliance.utils.simulation.SimulationEventStream; import org.epics.archiverappliance.utils.simulation.SimulationEventStreamIterator; import org.epics.archiverappliance.utils.simulation.SineGenerator; import org.junit.After; import org.junit.Before; import org.junit.Test; /** * Some simple tests for the FileBackedPBEventStream * We generate a years worth of data and then create FileBackedPBEventStream's using various constructors and make sure we get the expected amount of data. * @author mshankar * */ public class FileBackedPBEventStreamTest { private static Logger logger = Logger.getLogger(FileBackedPBEventStreamTest.class.getName()); File testFolder = new File(ConfigServiceForTests.getDefaultPBTestFolder() + File.separator + "FileBackedPBEventStreamTest"); String pvName = ConfigServiceForTests.ARCH_UNIT_TEST_PVNAME_PREFIX + ":FileBackedPBEventStreamTest"; ArchDBRTypes dbrType = ArchDBRTypes.DBR_SCALAR_DOUBLE; PlainPBStoragePlugin storagePlugin; private boolean leapYear = new GregorianCalendar().isLeapYear(TimeUtils.getCurrentYear()); private ConfigServiceForTests configService; @Before public void setUp() throws Exception { configService = new ConfigServiceForTests(new File("./bin")); storagePlugin = (PlainPBStoragePlugin) StoragePluginURLParser.parseStoragePlugin("pb://localhost?name=FileBackedPBEventStreamTest&rootFolder=" + testFolder.getAbsolutePath() + "&partitionGranularity=PARTITION_YEAR", configService); int phasediffindegrees = 10; SimulationEventStream simstream = new SimulationEventStream(dbrType, new SineGenerator(phasediffindegrees)); try(BasicContext context = new BasicContext()) { storagePlugin.appendData(context, pvName, simstream); } } @After public void tearDown() throws Exception { FileUtils.deleteDirectory(testFolder); } @Test public void test() throws Exception { testLocationBasedEventBeforeTime(); testCompleteStream(); testLocationBasedIterator(); testTimeBasedIterator(); makeSureWeGetTheLastEventInTheFile(); testHighRateEndLocation(); } private void testCompleteStream() throws Exception { try(BasicContext context = new BasicContext()) { long startMs = System.currentTimeMillis(); Path path = PlainPBPathNameUtility.getPathNameForTime(storagePlugin, pvName, TimeUtils.getStartOfCurrentYearInSeconds() + 24*60*60*7, context.getPaths(), configService.getPVNameToKeyConverter()); assertTrue("Did we not write any data?", path != null); int eventCount = 0; try(FileBackedPBEventStream stream = new FileBackedPBEventStream(pvName, path, dbrType)) { for(Event e : stream) { e.getEventTimeStamp(); eventCount++; } } int expectedSamples = new GregorianCalendar().isLeapYear(TimeUtils.getCurrentYear()) ? SimulationEventStreamIterator.LEAPYEAR_NUMBER_OF_SAMPLES : SimulationEventStreamIterator.DEFAULT_NUMBER_OF_SAMPLES; assertTrue("Expected " + expectedSamples + " got " + eventCount, eventCount == expectedSamples); long endMs = System.currentTimeMillis(); logger.info("Time for " + eventCount + " samples = " + (endMs - startMs) + "(ms)"); } } private void testLocationBasedIterator() throws Exception { try(BasicContext context = new BasicContext()) { Path path = PlainPBPathNameUtility.getPathNameForTime(storagePlugin, pvName, TimeUtils.getStartOfCurrentYearInSeconds() + 24*60*60*7, context.getPaths(), configService.getPVNameToKeyConverter()); int eventCount = 0; try(FileBackedPBEventStream stream = new FileBackedPBEventStream(pvName, path, dbrType, 0, Files.size(path))) { for(@SuppressWarnings("unused") Event e : stream) { eventCount++; } } int expectedSamples = leapYear ? SimulationEventStreamIterator.LEAPYEAR_NUMBER_OF_SAMPLES : SimulationEventStreamIterator.DEFAULT_NUMBER_OF_SAMPLES; assertTrue("Expected " + expectedSamples + " got " + eventCount, eventCount == expectedSamples); } try(BasicContext context = new BasicContext()) { Path path = PlainPBPathNameUtility.getPathNameForTime(storagePlugin, pvName, TimeUtils.getStartOfCurrentYearInSeconds() + 24*60*60*7, context.getPaths(), configService.getPVNameToKeyConverter()); int eventCount = 0; try(FileBackedPBEventStream stream = new FileBackedPBEventStream(pvName, path, dbrType, Files.size(path), Files.size(path)+1)) { for(@SuppressWarnings("unused") Event e : stream) { eventCount++; } } int expectedSamples = 0; assertTrue("Expected " + expectedSamples + " got " + eventCount, eventCount == expectedSamples); } } private void testTimeBasedIterator() throws Exception { for(int i = 0; i < 2; i++) { boolean skipSearch = (i==0); try(BasicContext context = new BasicContext()) { Path path = PlainPBPathNameUtility.getPathNameForTime(storagePlugin, pvName, TimeUtils.getStartOfCurrentYearInSeconds() + 24*60*60*7, context.getPaths(), configService.getPVNameToKeyConverter()); int eventCount = 0; // Start 11 days into the year and get two days worth of data. long startEpochSeconds = TimeUtils.getStartOfCurrentYearInSeconds() + 24*60*60*11; Timestamp start = TimeUtils.convertFromEpochSeconds(startEpochSeconds, 0); int secondsToExtract = 24*60*60*2; Timestamp end = TimeUtils.convertFromEpochSeconds(startEpochSeconds + secondsToExtract, 0); try(FileBackedPBEventStream stream = new FileBackedPBEventStream(pvName, path, dbrType, start, end, skipSearch)) { long eventEpochSeconds = 0; for(Event e : stream) { eventEpochSeconds = e.getEpochSeconds(); if(eventCount < 2) { logger.info("Starting event timestamp " + TimeUtils.convertToISO8601String(eventEpochSeconds)); } else if (eventCount > (secondsToExtract-10)) { logger.info("Ending event timestamp " + TimeUtils.convertToISO8601String(eventEpochSeconds)); } eventCount++; } logger.info("Final event timestamp " + TimeUtils.convertToHumanReadableString(eventEpochSeconds)); } int expectedSamples = secondsToExtract + 1; assertTrue("Expected " + expectedSamples + " got " + eventCount + " with skipSearch " + skipSearch, eventCount == expectedSamples); } // Same as before expect the start time is before the year try(BasicContext context = new BasicContext()) { Path path = PlainPBPathNameUtility.getPathNameForTime(storagePlugin, pvName, TimeUtils.getStartOfCurrentYearInSeconds() + 24*60*60*7, context.getPaths(), configService.getPVNameToKeyConverter()); int eventCount = 0; long startEpochSeconds = TimeUtils.getStartOfCurrentYearInSeconds() - 24*60*60; Timestamp start = TimeUtils.convertFromEpochSeconds(startEpochSeconds, 0); int secondsToExtract = 24*60*60*2; Timestamp end = TimeUtils.convertFromEpochSeconds(startEpochSeconds + secondsToExtract, 0); logger.debug("Looking for data between " + TimeUtils.convertToISO8601String(start) + " and " + TimeUtils.convertToISO8601String(end) + " with skipSearch " + skipSearch); try(FileBackedPBEventStream stream = new FileBackedPBEventStream(pvName, path, dbrType, start, end, skipSearch)) { long eventEpochSeconds = 0; for(Event e : stream) { eventEpochSeconds = e.getEpochSeconds(); if(eventCount < 2) { logger.info("Starting event timestamp " + TimeUtils.convertToISO8601String(eventEpochSeconds)); } eventCount++; } logger.info("Final event timestamp " + TimeUtils.convertToISO8601String(eventEpochSeconds)); } // We should only get one days worth of data. int expectedSamples = 24*60*60; assertTrue("Expected " + expectedSamples + " got " + eventCount + " with skipSearch " + skipSearch, eventCount == expectedSamples); } // This time, change the end time try(BasicContext context = new BasicContext()) { Path path = PlainPBPathNameUtility.getPathNameForTime(storagePlugin, pvName, TimeUtils.getStartOfCurrentYearInSeconds() + 24*60*60*7, context.getPaths(), configService.getPVNameToKeyConverter()); int eventCount = 0; long startEpochSeconds = TimeUtils.getStartOfCurrentYearInSeconds() + 360*24*60*60; Timestamp start = TimeUtils.convertFromEpochSeconds(startEpochSeconds, 0); int secondsToExtract = 24*60*60*10; Timestamp end = TimeUtils.convertFromEpochSeconds(startEpochSeconds + secondsToExtract, 0); long eventEpochSeconds = 0; try(FileBackedPBEventStream stream = new FileBackedPBEventStream(pvName, path, dbrType, start, end, skipSearch)) { for(Event e : stream) { eventEpochSeconds = e.getEpochSeconds(); if(eventCount < 2) { logger.info("Starting event timestamp " + TimeUtils.convertToHumanReadableString(eventEpochSeconds)); } eventCount++; } } logger.info("Final event timestamp " + TimeUtils.convertToHumanReadableString(eventEpochSeconds)); // Based on whether this is a leap year, we should get 5-6 days worth of data int expectedSamples = leapYear ? 24*60*60*6 + 1 : 24*60*60*5 + 1; assertTrue("Expected " + expectedSamples + " got " + eventCount + " with skipSearch " + skipSearch, eventCount == expectedSamples); } } } private void testLocationBasedEventBeforeTime() throws IOException { try(BasicContext context = new BasicContext()) { Path path = PlainPBPathNameUtility.getPathNameForTime(storagePlugin, pvName, TimeUtils.getStartOfCurrentYearInSeconds() + 24*60*60*7, context.getPaths(), configService.getPVNameToKeyConverter()); // Start 11 days into the year and get two days worth of data. long epochSeconds = TimeUtils.getStartOfCurrentYearInSeconds() + 7*24*60*60; Timestamp time = TimeUtils.convertFromEpochSeconds(epochSeconds, 0); try(FileBackedPBEventStream stream = new FileBackedPBEventStream(pvName, path, dbrType, time, TimeUtils.getEndOfYear(TimeUtils.getCurrentYear()), false)) { boolean firstEvent = true; for(Event e : stream) { if(firstEvent) { assertTrue( "The first event should be before timestamp " + TimeUtils.convertToHumanReadableString(time) + " got " + TimeUtils.convertToHumanReadableString(e.getEventTimeStamp()), e.getEventTimeStamp().before(time)); firstEvent = false; } else { // All other events should be after timestamp assertTrue( "All other events should be on or after timestamp " + TimeUtils.convertToHumanReadableString(time) + " got " + TimeUtils.convertToHumanReadableString(e.getEventTimeStamp()), e.getEventTimeStamp().after(time) || e.getEventTimeStamp().equals(time)); } } } } } private void makeSureWeGetTheLastEventInTheFile() throws IOException { try(BasicContext context = new BasicContext()) { Path path = PlainPBPathNameUtility.getPathNameForTime(storagePlugin, pvName, TimeUtils.getStartOfCurrentYearInSeconds() + 24*60*60*7, context.getPaths(), configService.getPVNameToKeyConverter()); // Start near the end of the year long startEpochSeconds = TimeUtils.getStartOfCurrentYearInSeconds() + 360*24*60*60; Timestamp startTime = TimeUtils.convertFromEpochSeconds(startEpochSeconds, 0); Timestamp endTime = TimeUtils.convertFromEpochSeconds(startEpochSeconds + 20*24*60*60, 0); Event finalEvent = null; try(FileBackedPBEventStream stream = new FileBackedPBEventStream(pvName, path, dbrType, startTime, endTime, false)) { boolean firstEvent = true; for(Event e : stream) { if(firstEvent) { assertTrue( "The first event should be before timestamp " + TimeUtils.convertToHumanReadableString(startTime) + " got " + TimeUtils.convertToHumanReadableString(e.getEventTimeStamp()), e.getEventTimeStamp().before(startTime)); firstEvent = false; } else { finalEvent = e.makeClone(); } } } assertTrue("Final event is null", finalEvent != null); Timestamp finalSecondOfYear = TimeUtils.getEndOfYear(TimeUtils.getCurrentYear()); finalSecondOfYear.setNanos(0); assertTrue("Final event should be the last event in the stream " + TimeUtils.convertToISO8601String(finalSecondOfYear) + " Instead it is " + TimeUtils.convertToISO8601String(finalEvent.getEventTimeStamp()), finalEvent.getEventTimeStamp().equals(finalSecondOfYear)); } } /** * This is Jud Gauden'z use case. We have a high rate PV (more than one event per second). * We then ask for data from the same second (start time and end time is the same second). * For this we generate data into a new PB file. * @throws IOException */ private void testHighRateEndLocation() throws IOException { PlainPBStoragePlugin highRatePlugin = (PlainPBStoragePlugin) StoragePluginURLParser.parseStoragePlugin("pb://localhost?name=FileBackedPBEventStreamTest&rootFolder=" + testFolder.getAbsolutePath() + "&partitionGranularity=PARTITION_YEAR", configService); String highRatePVName = ConfigServiceForTests.ARCH_UNIT_TEST_PVNAME_PREFIX + ":FileBackedPBEventStreamTestHighRate"; int day = 60; int startofdayinseconds = day*24*60*60; try(BasicContext context = new BasicContext()) { short currentYear = TimeUtils.getCurrentYear(); ArrayListEventStream testData = new ArrayListEventStream(24*60*60, new RemotableEventStreamDesc(ArchDBRTypes.DBR_SCALAR_DOUBLE, highRatePVName, currentYear)); for(int secondintoday = 0; secondintoday < 24*60*60; secondintoday++) { // The value should be the secondsIntoYear integer divided by 600. // Add 10 events per second for(int i = 0; i < 10; i++) { SimulationEvent sample = new SimulationEvent(startofdayinseconds + secondintoday, currentYear, ArchDBRTypes.DBR_SCALAR_DOUBLE, new ScalarValue<Double>((double) (((int)(startofdayinseconds + secondintoday)/600)))); sample.setNanos(i*100); testData.add(sample); } } highRatePlugin.appendData(context, highRatePVName, testData); } long requestEpochSeconds = TimeUtils.getStartOfCurrentYearInSeconds() + startofdayinseconds + 12*60*60; // Yes; start and end time are the same epochseconds. We can vary the nanos. Timestamp startTime = TimeUtils.convertFromEpochSeconds(requestEpochSeconds, 0); Timestamp endTime = TimeUtils.convertFromEpochSeconds(requestEpochSeconds, 999999999); try(BasicContext context = new BasicContext()) { Path path = PlainPBPathNameUtility.getPathNameForTime(highRatePlugin, highRatePVName, TimeUtils.getStartOfCurrentYearInSeconds() + 24*60*60*7, context.getPaths(), configService.getPVNameToKeyConverter()); try(FileBackedPBEventStream stream = new FileBackedPBEventStream(highRatePVName, path, dbrType, startTime, endTime, false)) { boolean firstEvent = true; int eventCount = 0; int expectedEventCount = 11; for(Event e : stream) { eventCount++; if(firstEvent) { assertTrue( "The first event should be before timestamp " + TimeUtils.convertToHumanReadableString(startTime) + " got " + TimeUtils.convertToHumanReadableString(e.getEventTimeStamp()), e.getEventTimeStamp().before(startTime)); firstEvent = false; } } assertTrue("We should have " + expectedEventCount + " events. Instead we got " + eventCount, eventCount == expectedEventCount); } } } }