package org.epics.archiverappliance.etl.bpl.reports;
import java.io.IOException;
import java.text.DecimalFormat;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.Map;
import org.apache.log4j.Logger;
import org.epics.archiverappliance.config.ConfigService;
import org.epics.archiverappliance.etl.ETLDest;
import org.epics.archiverappliance.etl.ETLSource;
import org.epics.archiverappliance.etl.StorageMetrics;
import org.epics.archiverappliance.etl.common.ETLMetricsForLifetime;
import org.epics.archiverappliance.etl.common.ETLPVLookupItems;
import org.json.simple.JSONValue;
public class StorageWithLifetime {
private static Logger logger = Logger.getLogger(StorageWithLifetime.class.getName());
StorageMetrics storageMetricsAPI;
int lifetimeid;
double totalETLTimeIntoThisDestInMillis;
int maxTotalETLRunsIntoThisDest;
int minPartitionSourceGranularityInSecs = 366*24*60*60; // Init to a large value; below we use Math.min to pick the correct value.
public StorageWithLifetime(StorageMetrics storageMetricsAPI, int lifetimeid) {
this.storageMetricsAPI = storageMetricsAPI;
this.lifetimeid = lifetimeid;
}
public static String getStorageMetrics(ConfigService configService) {
LinkedList<Map<String, String>> allStorageMetrics = new LinkedList<Map<String, String>>();
LinkedList<StorageWithLifetime> finalStorages = getStorageWithLifetimes(configService);
for(StorageWithLifetime storage : finalStorages) {
try {
ETLMetricsForLifetime metricsForLifetime = configService.getETLLookup().getApplianceMetrics().get(storage.lifetimeid);
HashMap<String, String> storageMetrics = new HashMap<String, String>();
allStorageMetrics.add(storageMetrics);
storageMetrics.put("identity", storage.storageMetricsAPI.getName());
storageMetrics.put("totalSpace", Long.toString(storage.storageMetricsAPI.getTotalSpace(metricsForLifetime)));
storageMetrics.put("availableSpace", Long.toString(storage.storageMetricsAPI.getUsableSpace(metricsForLifetime)));
double avgTimeIntoThisDestInMillis = storage.totalETLTimeIntoThisDestInMillis/storage.maxTotalETLRunsIntoThisDest;
storageMetrics.put("avgTimeConsumedMs", Double.toString(avgTimeIntoThisDestInMillis));
storageMetrics.put("avgTimeConsumedPercent", Double.toString((avgTimeIntoThisDestInMillis/1000)/storage.minPartitionSourceGranularityInSecs));
storageMetrics.put("minPartitionSourceGranularityInSecs", Integer.toString(storage.minPartitionSourceGranularityInSecs));
} catch(IOException ex) {
logger.warn("Exception retrieving details from " + storage.storageMetricsAPI.getName(), ex);
}
}
return JSONValue.toJSONString(allStorageMetrics);
}
public static String getStorageDetails(ConfigService configService) {
DecimalFormat twoSignificantDigits = new DecimalFormat("###,###,###,###,###,###.##");
LinkedList<HashMap<String, String>> details = new LinkedList<HashMap<String, String>>();
LinkedList<StorageWithLifetime> finalStorages = getStorageWithLifetimes(configService);
for(StorageWithLifetime storage : finalStorages) {
ETLMetricsForLifetime metricsForLifetime = configService.getETLLookup().getApplianceMetrics().get(storage.lifetimeid);
HashMap<String, String> detail = new HashMap<String, String>();
details.add(detail);
try {
detail.put("name", storage.storageMetricsAPI.getName());
double totalSpaceGB = storage.storageMetricsAPI.getTotalSpace(metricsForLifetime)*1.0/(1024*1024*1024);
detail.put("total_space", twoSignificantDigits.format(totalSpaceGB));
double availbleSpaceGB = storage.storageMetricsAPI.getUsableSpace(metricsForLifetime)*1.0/(1024*1024*1024);
detail.put("available_space", twoSignificantDigits.format(availbleSpaceGB));
detail.put("available_space_percent", twoSignificantDigits.format(availbleSpaceGB*100/totalSpaceGB));
double avgTimeIntoThisDestInMillis = storage.totalETLTimeIntoThisDestInMillis/storage.maxTotalETLRunsIntoThisDest;
detail.put("time_copy_data_into_store", twoSignificantDigits.format(avgTimeIntoThisDestInMillis/1000));
// Copy time as percent of source granularities
detail.put("time_copy_data_into_store_percent", twoSignificantDigits.format((avgTimeIntoThisDestInMillis/1000)/storage.minPartitionSourceGranularityInSecs));
} catch(IOException ex) {
logger.warn("Exception retrieving details from " + storage.storageMetricsAPI.getName(), ex);
}
}
return JSONValue.toJSONString(details);
}
/**
* Utility method to get all the ETL lookup items as storagemetrics instances if they support it.
* @return
*/
private static LinkedList<StorageWithLifetime> getStorageWithLifetimes(ConfigService configService) {
LinkedHashMap<String, StorageWithLifetime> storages = new LinkedHashMap<String, StorageWithLifetime>();
for(String pvName : configService.getPVsForThisAppliance()) {
for(ETLPVLookupItems lookupItem : configService.getETLLookup().getLookupItemsForPV(pvName)) {
ETLSource etlSrc = lookupItem.getETLSource();
if(etlSrc instanceof StorageMetrics) {
StorageMetrics storageMetricsAPI = (StorageMetrics) etlSrc;
if(!storages.containsKey(storageMetricsAPI.getName())) {
storages.put(storageMetricsAPI.getName(), new StorageWithLifetime(storageMetricsAPI, lookupItem.getLifetimeorder()));
}
}
ETLDest etlDest = lookupItem.getETLDest();
if(etlDest instanceof StorageMetrics) {
StorageMetrics storageMetricsAPI = (StorageMetrics) etlDest;
if(!storages.containsKey(storageMetricsAPI.getName())) {
storages.put(storageMetricsAPI.getName(), new StorageWithLifetime(storageMetricsAPI, lookupItem.getLifetimeorder()));
}
storages.get(storageMetricsAPI.getName()).addETLDestTimes(lookupItem);
}
}
}
LinkedList<StorageWithLifetime> finalStorages = new LinkedList<StorageWithLifetime>(storages.values());
Collections.sort(finalStorages, new Comparator<StorageWithLifetime>() {
@Override
public int compare(StorageWithLifetime o1, StorageWithLifetime o2) {
return o1.lifetimeid - o2.lifetimeid;
}
});
return finalStorages;
}
private void addETLDestTimes(ETLPVLookupItems lookupItem) {
if(lookupItem.getNumberofTimesWeETLed() > 0) {
this.totalETLTimeIntoThisDestInMillis += lookupItem.getTotalTimeWeSpentInETLInMilliSeconds();
this.maxTotalETLRunsIntoThisDest = Math.max(this.maxTotalETLRunsIntoThisDest, lookupItem.getNumberofTimesWeETLed());
// We compute the percent as the percent of the source granularity.
// For example, if the source granularity is an hour, then we have an hour to get the data into the dest.
// What fraction of this did we consume?
int typicalSecsInSrcPG = lookupItem.getETLSource().getPartitionGranularity().getApproxSecondsPerChunk();
this.minPartitionSourceGranularityInSecs = Math.min(this.minPartitionSourceGranularityInSecs, typicalSecsInSrcPG);
} else {
if(logger.isDebugEnabled()) logger.debug("We do not seem to have ETLed for pv " + lookupItem.getPvName());
}
}
public static class StorageConsumedByPV {
public String pvName;
public long storageConsumed;
public StorageConsumedByPV(String pvName, long storageConsumed) {
this.pvName = pvName;
this.storageConsumed = storageConsumed;
}
}
/**
* Get a list of PVs and the storage they consume on all the devices sorted by desc storage consumed...
* @param configService ConfigService
* @return LinkedList StorageConsumedByPV
* @throws IOException
*/
public static LinkedList<StorageConsumedByPV> getPVSByStorageConsumed(ConfigService configService) throws IOException {
//TODO there may be some problems . When visiting the web page of reports and look up the "PVs by storage consumed(100)" , it takes a long time
HashMap<String, HashMap<String, StorageMetrics>> storesForAllPVs = getStoresForAllPVs(configService);
LinkedList<StorageConsumedByPV> storageConsumedList = new LinkedList<StorageConsumedByPV>();
for(String pvName: storesForAllPVs.keySet()) {
long spaceConsumedByPV = 0;
HashMap<String, StorageMetrics> pvStores = storesForAllPVs.get(pvName);
for(StorageMetrics storageMetrics : pvStores.values()) {
long spaceConsumedByPVInThisStore = storageMetrics.spaceConsumedByPV(pvName);
spaceConsumedByPV = spaceConsumedByPV + spaceConsumedByPVInThisStore;
}
storageConsumedList.add(new StorageConsumedByPV(pvName, spaceConsumedByPV));
}
Collections.sort(storageConsumedList, new Comparator<StorageConsumedByPV>() {
@Override
public int compare(StorageConsumedByPV o1, StorageConsumedByPV o2) {
if(o1.storageConsumed == o2.storageConsumed) return 0;
return (o1.storageConsumed < o2.storageConsumed) ? 1 : -1;
}
});
return storageConsumedList;
}
/**
* Get the stores for all PV's indexed by PV name..
* @param configService ConfigService
* @return HashMap
*/
private static HashMap<String, HashMap<String, StorageMetrics>> getStoresForAllPVs(ConfigService configService) {
HashMap<String, HashMap<String, StorageMetrics>> storesForAllPVs = new HashMap<String, HashMap<String, StorageMetrics>>();
for(String pvName : configService.getPVsForThisAppliance()) {
for(ETLPVLookupItems lookupItem : configService.getETLLookup().getLookupItemsForPV(pvName)) {
HashMap<String, StorageMetrics> pvStores = storesForAllPVs.get(pvName);
if(pvStores == null) {
pvStores = new HashMap<String, StorageMetrics>();
storesForAllPVs.put(pvName, pvStores);
}
ETLSource etlSrc = lookupItem.getETLSource();
if(etlSrc instanceof StorageMetrics) {
StorageMetrics storageMetricsAPI = (StorageMetrics) etlSrc;
if(!pvStores.containsKey(storageMetricsAPI.getName())) {
pvStores.put(storageMetricsAPI.getName(), storageMetricsAPI);
}
}
ETLDest etlDest = lookupItem.getETLDest();
if(etlDest instanceof StorageMetrics) {
StorageMetrics storageMetricsAPI = (StorageMetrics) etlDest;
if(!pvStores.containsKey(storageMetricsAPI.getName())) {
pvStores.put(storageMetricsAPI.getName(), storageMetricsAPI);
}
}
}
}
return storesForAllPVs;
}
}