/*
ALMA - Atacama Large Millimiter Array
* Copyright (c) European Southern Observatory, 2014
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
package alma.alarmsystem.statistics;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Vector;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.logging.Logger;
import org.exolab.castor.xml.MarshalException;
import org.exolab.castor.xml.Marshaller;
import org.exolab.castor.xml.ValidationException;
import alma.acs.logging.AcsLogLevel;
import alma.acs.util.IsoDateFormat;
import alma.alarmsystem.statistics.generated.IDType;
import alma.alarmsystem.statistics.generated.MostActivatedAlarms;
import alma.alarmsystem.statistics.generated.MostActivatedTerminatedAlarms;
import alma.alarmsystem.statistics.generated.MostTerminatedAlarms;
import alma.alarmsystem.statistics.generated.Record;
import alma.alarmsystem.statistics.generated.Statistics;
/**
* <code>StatHashMap</code> encapsulate a HashMap where all the alarms
* received during a time interval are stored.
* The purpose of this class is to provide more detailed statistics then those
* published in the log.
* <P>
* <EM>Implementation note</EM><BR>
* To generate and write statistics on file could be quite consuming.
* To reduce the impact, when the time interval elapses, {@link #alarmsMap} point to a newly
* created HashMap and the old one is used by a thread to calculate the statistics.
* <BR>
* All the maps are stored in the FIFO {@link StatHashMap#mapsQueue} queue.
* The thread waits for a new map in the queue the generate the statistics.
* The size of the queue is limited to control the memory usage in the unlikely case
* that generation of statistics lasts longer then a time interval.
* <P>
* Writing on file is only partially done through castor to avoid keeping in
* memory all the statistics.
* The xml is however always generated by castor to ensure it is well formed
* and readable through castor itself.
*
* @author acaproni
* @since 2015.2
*/
public class StatHashMap implements Runnable {
/**
* A struct to hold the statistics at each iterations
* @author almadev
*
*/
private static class StatStruct {
public StatStruct(int maxTotActiavations, int maxTotTerminations,
Map<String, AlarmInfo> alarmsInfo) {
super();
this.numActiavations = maxTotActiavations;
this.numTerminations = maxTotTerminations;
this.alarmsInfo = alarmsInfo;
}
public final int numActiavations;
public final int numTerminations;
public final Map<String,AlarmInfo> alarmsInfo;
}
/**
* To get more useful statistics, the number of activations
* and terminations of each alarm in the last time interval
* is stored in this data structure.
*
* @author acaproni
*
*/
private static class AlarmInfo {
/**
*
* @author The type of a valu to get
*
*/
private enum VALUE_TYPE {
ACTIVATIONS,
TERMINATIONS,
OPERATIONS
};
/**
* The ID of the alarm whose statistics this object holds
*/
public final String alarmID;
/**
* Number of activations in the last time interval
*/
private int activations=0;
/**
* Number of terminations in the last time interval
*/
private int terminations=0;
/**
* Constructor
*
* @param alarmID the ID of the alarm
*/
public AlarmInfo(String alarmID) {
if (alarmID==null || alarmID.isEmpty()) {
throw new IllegalArgumentException("Invalid alarm ID");
}
this.alarmID=alarmID;
}
public void update(boolean active) {
if (active) {
activations++;
} else {
terminations++;
}
}
/**
* Getter
*
* @return The number of activations
*/
public int getActivations() {
return activations;
}
/**
* Getter
*
* @return The number of terminations
*/
public int getTerminations() {
return terminations;
}
/**
*
* @return The total number of activations and terminations executed on this alarm
* during the time interval
*/
public int getTotalOperations() {
return activations+terminations;
}
@Override
public String toString() {
return alarmID+": activations="+getActivations()+", terminations="+getTerminations();
}
public int getValue(VALUE_TYPE type) {
if (type==null) {
throw new IllegalArgumentException("The type can't be null");
}
if (type==VALUE_TYPE.ACTIVATIONS) {
return getActivations();
}
if (type==VALUE_TYPE.TERMINATIONS) {
return getTerminations();
}
return getTotalOperations();
}
}
/**
* A {@link AlarmInfo} comparator based on the passed type
*
* @author acaproni
*/
private static class ComparatorByType implements Comparator<AlarmInfo> {
/**
* The type on which the comparison is based
*/
private final AlarmInfo.VALUE_TYPE type;
/**
* Constructor
*
* @param type The type on which the comparison is based
*/
public ComparatorByType(AlarmInfo.VALUE_TYPE type) {
if (type==null) {
throw new IllegalArgumentException("The type can't be null");
}
this.type=type;
}
@Override
public int compare(AlarmInfo o1, AlarmInfo o2) {
return Integer.valueOf(o1.getValue(type)).compareTo(o2.getValue(type));
}
}
/**
* The logger
*/
private final Logger logger;
/**
* {@link #alarmsMap} records the activations/terminations of
* each alarm during the current time interval.
* It is used later on to calculate the statistics.
* <P>
* A new map is created for each time interval to minimize the impact in performances
* in {@link StatHashMap#processedFS(String, boolean)}.
* <P>
* There is no need to synchronize this map because it is modified only
* by the synchronized {@link StatHashMap#processedFS(String, boolean)} method.
*/
private Map<String, AlarmInfo> alarmsMap = new HashMap<String, AlarmInfo>();
/**
* The max number of maps to store in the {@link #mapsQueue}.
*/
private static final int MAXQUEUECAPACITY=10;
/**
* When the time interval elapses the {@link StatHashMap#alarmsMap} is pushed
* into this queue and a new {@link StatHashMap#alarmsMap}.
* <P>
* The thread gets maps from this queue and generate the statistics.
*/
private final BlockingQueue<StatStruct> mapsQueue = new ArrayBlockingQueue<StatStruct>(MAXQUEUECAPACITY);
/**
* The thread to calculate statistics
* @see #run()
*/
private final Thread thread = new Thread(this,this.getClass().getName());
/**
* Statistics are calculated every time interval (in minutes)
*/
private final int timeInterval;
/**
* Signal if the processing of statistics has been terminated
*/
private volatile boolean closed=false;
/**
* The folder to write the files of statistics into
*/
private final File folder;
/**
* The name of each file is composed of a prefix plus the index
* and the ".xml" extension.
*/
public static final String fileNamePrefix="AlarmSystemStats-";
/**
* The name of each file of statistics is indexed with the following integer
*/
private int fileNumber=0;
/**
* A new file is created when the size of actual one is greater then
* the value of this property
*/
private static final long maxFileSize= 2000000000;
private static final String closeXMLTag="</Statistics>\n";
/**
* Constructor
*
* @param lgger The logger
* @param intervalLength The length of each time interval (in minutes)
* @param folder The folder to write the files of statistics into
*/
public StatHashMap(Logger logger, int intervalLength, File folder) {
if (logger==null) {
throw new IllegalArgumentException("The logger can't be null");
}
this.logger=logger;
if (intervalLength<0) {
throw new IllegalArgumentException("The time interval must be greater or equal to 0");
}
this.timeInterval=intervalLength;
if (folder==null) {
throw new IllegalArgumentException("The logger can't be null");
}
if (!folder.isDirectory()) {
throw new IllegalArgumentException(folder.getAbsolutePath()+ " not a directory");
}
if (!folder.canWrite()) {
throw new IllegalArgumentException("Can't create files in "+folder.getAbsolutePath());
}
this.folder=folder;
logger.log(AcsLogLevel.DEBUG,"Files with statistics will be saved in "+this.folder.getAbsolutePath());
}
/**
* A new FS has been processed: if not already present,
* it is stored in the {@link #alarmsMap} map otherwise its state is updated.
*
* @param alarmID the ID of the FS
* @param active The activation state of the FS
*/
public void processedFS(String alarmID,boolean active) {
if (closed || timeInterval==0) {
return;
}
synchronized (this) {
AlarmInfo info = alarmsMap.get(alarmID);
if (info==null) {
info = new AlarmInfo(alarmID);
alarmsMap.put(alarmID, info);
}
info.update(active);
}
}
/**
* Create a new structure for the timer task:the eal calculation will be done there
*
* @param activations Activations in the given interval
* @param terminations Terminations in the given interval
*/
public void calcStatistics(int activations, int terminations) {
if (closed || timeInterval==0) {
return;
}
// Create new map and release the lock so that processedFS does not block
// impacting alarm server performances
synchronized (this) {
if (!mapsQueue.offer(new StatStruct(activations, terminations, alarmsMap))) {
// Queue full: log a warning
logger.warning("Alarm statistics lost for this time interval: the queue is full");
}
alarmsMap = new HashMap<String, AlarmInfo>();
}
}
/**
* The thread picks maps from the {@link #mapsQueue} then calculate and write on file
* the statistics.
*/
public void run() {
while (!closed) {
// Get one map out of the queue
StatStruct struct;
try {
struct = mapsQueue.take();
} catch (InterruptedException ie) {
continue;
}
try {
calculate(struct);
} catch (Throwable t) {
logger.log(AcsLogLevel.ERROR, "Error calculating/writing alarm system statistics: record lost!", t);
}
}
}
/**
* Start the thread to evaluate the statistics
*/
public void start() {
if (timeInterval!=0) {
thread.setDaemon(true);
thread.start();
}
}
/**
* Stop the thread to calculate the statistics
* and free the resources.
*/
public void shutdown() {
closed=false;
if (timeInterval!=0) {
thread.interrupt();
}
}
/**
* Calculate the statistics of the passed time interval.
* <P>
* The figures calculated are:
* <UL>
* <LI>Total number of different alarms issued
* <LI>Total number of activations and terminations
* <LI>Total number of terminations
* <LI>Total number of activations
* <LI>Average number of activations and terminations per second
* <LI>The 5 alarms that has been changed (activated and/or terminated) more often
* <LI>The 5 alarms that has been activated more often
* <LI>The 5 alarms that has been terminated more often
* </UL>
*
* @param statStruct The object with numbers for the statistics
* @throws IOException In case of error with the file
* @throws ValidationException In case of error validating data
* @throws MarshalException IN case of error "marshalling" data on file
*/
private void calculate(StatStruct statStruct) throws IOException, MarshalException, ValidationException {
// A collection to iterate over the values
List<AlarmInfo> infos = new ArrayList<AlarmInfo>(statStruct.alarmsInfo.values());
// The number of different alarms published in the interval
int totAlarms = infos.size();
// Total number of operations (i.e. activations and terminations)
int totOperations=statStruct.numActiavations+statStruct.numTerminations;
// Average number of operations per second
float avgOpPerSecond=(float)totOperations/(float)(timeInterval*60);
// Get the file to write the statistics
BufferedWriter outF;
try {
outF=openOutputFile();
} catch (Throwable t) {
logger.log(AcsLogLevel.ERROR, "Can't write on file: statistics lost", t);
return;
}
// Build the string to write on disk (help of castor)
Record statRec = new Record();
statRec.setTimestamp(IsoDateFormat.formatCurrentDate());
statRec.setProcessedAlarmTypes(totAlarms);
statRec.setActivations(statStruct.numActiavations);
statRec.setTerminations(statStruct.numTerminations);
statRec.setTotalAlarms(totOperations);
statRec.setAvgAlarmsPerSecond(Float.valueOf(String.format("%.2f", avgOpPerSecond)));
MostActivatedAlarms maa = new MostActivatedAlarms();
Collections.sort(infos, new ComparatorByType(AlarmInfo.VALUE_TYPE.ACTIVATIONS));
maa.setID(appendListOfAlarms(infos,AlarmInfo.VALUE_TYPE.ACTIVATIONS,4));
statRec.setMostActivatedAlarms(maa);
MostTerminatedAlarms mta = new MostTerminatedAlarms();
Collections.sort(infos, new ComparatorByType(AlarmInfo.VALUE_TYPE.TERMINATIONS));
mta.setID(appendListOfAlarms(infos,AlarmInfo.VALUE_TYPE.TERMINATIONS,4));
statRec.setMostTerminatedAlarms(mta);
MostActivatedTerminatedAlarms mata = new MostActivatedTerminatedAlarms();
Collections.sort(infos, new ComparatorByType(AlarmInfo.VALUE_TYPE.OPERATIONS));
mata.setID(appendListOfAlarms(infos,AlarmInfo.VALUE_TYPE.OPERATIONS,4));
statRec.setMostActivatedTerminatedAlarms(mata);
Marshaller marshaller = getMarshaller(outF);
marshaller.marshal(statRec);
// This string appears at the end of the XML file
outF.write(closeXMLTag);
statStruct.alarmsInfo.clear();
infos.clear();
outF.flush();
outF.close();
}
/**
* Return a properly configured marshaller for
* writing in the file.
*
* @return The marshaller
* @throws IOException If the witer is invalid for the marshaller
*/
private Marshaller getMarshaller(Writer writer) throws IOException {
if (writer==null) {
throw new IllegalArgumentException("Invalid null writer");
}
Marshaller marshaller = new Marshaller(writer);
marshaller.setEncoding("ISO-8859-1");
marshaller.setSuppressNamespaces(true);
marshaller.setSupressXMLDeclaration(true);
marshaller.setSuppressXSIType(true);
return marshaller;
}
/**
* Return the list of the top 5 five alarms of the passed list.
* <P>
* The list reports the number of alarms with the highest numbers (alarms
* with the same values are grouped) so the returned
* list can actually be longer of <code>depth</code>.
* <P>
* Alarms with a number of 0 are discarded.
*
* @param infos The list of alarms received in the time interval
* @param The type of value shown by the list
* @param depth The number of items to append
* @return The vector of alarms to add to the record
*/
private IDType[] appendListOfAlarms(
List<AlarmInfo> infos,
AlarmInfo.VALUE_TYPE type,
int depth) {
Vector<IDType> temp= new Vector<IDType>();
int count=0;
int oldVal=0;
int pos=infos.size()-1;
while (count<=depth && pos>=0) {
AlarmInfo alarm = infos.get(pos--);
int actualValue=alarm.getValue(type);
if (actualValue==0) {
// We do not want to report alarms with a value of 0
break;
}
IDType idt = new IDType();
idt.setValue(alarm.getValue(type));
idt.setContent(alarm.alarmID);
temp.add(idt);
if (oldVal!=actualValue) {
count++;
}
oldVal=actualValue;
}
IDType[] ret = new IDType[temp.size()];
return temp.toArray(ret);
}
/**
* Open and create the output stream for writing statistics on file.
* <P>
* The file is opened for each writing and closed immediately.
* A new file is created, whenever the size of the actual file is
* greater then {@link #maxFileSize}.
*
* @return The stream for writing into the file
* @throws IOException If can't open/create the file for writing
* @throws ValidationException In case of error validating the Statistics element (should never happen)
* @throws MarshalException Error writing Statistics header in the file
*/
private BufferedWriter openOutputFile() throws IOException, MarshalException, ValidationException {
String actualFileName=fileNamePrefix+fileNumber+".xml";
String folderName= folder.getAbsolutePath();
if (!folderName.endsWith(""+File.separator)) {
folderName=folderName+File.separator;
}
// Check the size of the file if it exists
File f = new File(folderName+actualFileName);
if (f.exists() && f.length()>maxFileSize) {
fileNumber++;
return openOutputFile();
}
if (f.length()==0) {
// New file: write the header
Statistics statistics = new Statistics();
BufferedWriter outF = new BufferedWriter(new FileWriter(f, true));
Marshaller marshaller = getMarshaller(outF);
marshaller.setSupressXMLDeclaration(false);
marshaller.marshal(statistics);
outF.flush();
outF.close();
RandomAccessFile raf = new RandomAccessFile(f, "rw");
raf.setLength(raf.length()-3);
raf.seek(raf.length());
raf.writeBytes(">\n");
raf.close();
} else {
// Remove the closing tag (closeXMLTag) before adding a new record
RandomAccessFile raf = new RandomAccessFile(f, "rw");
raf.setLength(raf.length()-closeXMLTag.length()-1);
raf.seek(raf.length());
raf.writeBytes("\n");
raf.close();
}
return new BufferedWriter(new FileWriter(f, true));
}
}