FlowReader.java

/*******************************************************************************
 * gMix open source project - https://svs.informatik.uni-hamburg.de/gmix/
 * Copyright (C) 2014  SVS
 *
 * This program is free software: you can redistribute it and/or modify 
 * it under the terms of the GNU General Public License as published by 
 * the Free Software Foundation, either version 3 of the License, or 
 * (at your option) any later version.
 *  
 * This program is distributed in the hope that it will be useful, 
 * but WITHOUT ANY WARRANTY; without even the implied warranty of 
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License 
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *******************************************************************************/
package staticContent.evaluation.traceParser.engine.fileReader;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.channels.Channels;
import java.util.Vector;

import staticContent.evaluation.traceParser.engine.Protocol;
import staticContent.evaluation.traceParser.engine.TraceInfo;
import staticContent.evaluation.traceParser.engine.converter.EndOfFlow;
import staticContent.evaluation.traceParser.engine.converter.ToGMF;
import staticContent.evaluation.traceParser.engine.converter.TransactionInfo;
import staticContent.evaluation.traceParser.engine.dataStructure.Flow;
import staticContent.evaluation.traceParser.engine.dataStructure.Flow.Restriction;
import staticContent.evaluation.traceParser.interfaces.FlowFilter;
import staticContent.framework.util.Util;


public class FlowReader extends CountingBufferedReader implements FlowIterator {

	protected FlowFilter filter;
	private Flow nextFlow;
	private boolean wasHasNextCalled = false;
	private long offsetOfLastFlow = 0;
	private long offsetOfNextFlow = 0;
	private long tmp_offsetOfLastFlow = 0;
	private long tmp_offsetOfNextFlow = 0;
	private long endOfLastFlowGroup = 0;
	//private int idOfLatestFinishedFlowOfLastFlowGroup = Util.NOT_SET;
	private int ownerOfLastFlowGroup = Util.NOT_SET;
	private final boolean calculateRestrictions;
	private EndOfFlow latestFinishedFlowOfLastFlowGroup = null;
	
	
	/*public FlowReader(String pathToTraceFolder) throws FileNotFoundException, IOException {
		this(new TraceInfo(pathToTraceFolder), null);
	}*/
	
	
	public FlowReader(String pathToTraceFolder, FlowFilter filter) throws FileNotFoundException, IOException {
		this(new TraceInfo(pathToTraceFolder), filter);
	}
	
	
	public FlowReader(String pathToTraceFolder, FlowFilter filter, boolean calculateRestrictions) throws FileNotFoundException, IOException {
		this(new TraceInfo(pathToTraceFolder), filter, calculateRestrictions);
	}
	
	/*public FlowReader(TraceInfo traceInfo) throws FileNotFoundException, IOException {
		this(traceInfo, null);
	}*/
	
	
	public FlowReader(TraceInfo traceInfo, FlowFilter filter) throws FileNotFoundException, IOException {
		super(getGmfFilePointer(traceInfo));
		this.filter = filter;
		this.calculateRestrictions = true;
	}
	
	
	public FlowReader(TraceInfo traceInfo, FlowFilter filter, boolean calculateRestrictions) throws FileNotFoundException, IOException {
		super(getGmfFilePointer(traceInfo));
		this.filter = filter;
		this.calculateRestrictions = calculateRestrictions;
	}
	
	
	/*public FlowReader(BufferedReader bufferedReader) throws FileNotFoundException, IOException {
		super(bufferedReader);
	}*/
	
	
	public FlowReader(BufferedReader bufferedReader, FlowFilter filter) throws FileNotFoundException, IOException {
		super(bufferedReader);
		this.filter = filter;
		this.calculateRestrictions = true;
	}
	
	
	public FlowReader(BufferedReader bufferedReader, FlowFilter filter, boolean calculateRestrictions) throws FileNotFoundException, IOException {
		super(bufferedReader);
		this.filter = filter;
		this.calculateRestrictions = calculateRestrictions;
	}
	
	
	public FlowReader(BufferedReader bufferedReader, FlowFilter filter, boolean calculateRestrictions, RandomAccessFile raf) throws FileNotFoundException, IOException {
		super(bufferedReader, raf);
		this.filter = filter;
		this.calculateRestrictions = calculateRestrictions;
	}


	/**
	 * returns the path to the created (or loaded if already existing) .gmf file
	 */
	protected static String getGmfFilePointer(TraceInfo traceInfo) {
		Util.displayWarningOnLowReservedMemory();
		String outputFile = Util.removeFileExtension(traceInfo.getPathToTraceFile()) + ".gmf";
		if (traceInfo.getTraceFormat() != Protocol.GMF) { // no .gmf selected -> detect/create .gmf
			// detect if .gmf file is already present: (but was not selected in traceInfo.txt, e.g. because it was created automatically last run)
			boolean alreadyPresent = false;
			try {
				FileInputStream trace = new FileInputStream(outputFile);
				trace.close();
				alreadyPresent = true;
				System.out.println("detected existing .gmf file"); 
			} catch (Exception e) {}
			if (!alreadyPresent) { // use PacketSource as source (will converter to .gmp if not present) and then use toGMF converter to create .gmf.
				System.out.println("extracting flows (converting to .gmf)"); 
				PacketSource packetSource = new PacketSource(traceInfo);
				new ToGMF(packetSource, traceInfo).convert();
				System.out.println("finished extracting flows (converting to .gmf)"); 
			}
		}
		return outputFile;
	}
	
	
	public boolean hasNextFlow() throws IOException {
		return peekNextFlow() != null;
	}
	
	
	public Flow peekNextFlow() throws IOException {
		if (!wasHasNextCalled) {
			try {
				this.nextFlow = tryReadFlowFromFile();
			} catch (IOException e) {
				e.printStackTrace();
				super.close();
				nextFlow = null;
				return null;
			}
			wasHasNextCalled = true;
		}
		return this.nextFlow;
	}
	
	
	protected Flow tryReadFlowFromFile() throws IOException {
		String serializedFlow = super.readLine();
		if (filter == null) {
			this.tmp_offsetOfLastFlow = this.tmp_offsetOfNextFlow;
			this.tmp_offsetOfNextFlow = super.getPositionOfNextLine();
			return serializedFlow == null ? null : new Flow(serializedFlow);
		} else {
			if (serializedFlow == null) {
				this.tmp_offsetOfLastFlow = this.tmp_offsetOfNextFlow;
				this.tmp_offsetOfNextFlow = super.getPositionOfNextLine();
				return null;
			}
			Flow flow = new Flow(serializedFlow);
			while (filter.filterFlow(flow)) {
				this.tmp_offsetOfNextFlow = super.getPositionOfNextLine();
				serializedFlow = super.readLine();
				if (serializedFlow == null) {
					this.tmp_offsetOfLastFlow = this.tmp_offsetOfNextFlow;
					this.tmp_offsetOfNextFlow = super.getPositionOfNextLine();
					return null;
				}
				flow = new Flow(serializedFlow);
			} 
			this.tmp_offsetOfLastFlow = this.tmp_offsetOfNextFlow;
			this.tmp_offsetOfNextFlow = super.getPositionOfNextLine();
			return flow;
		}
	}
	
	
	public Flow readFlow() throws IOException {
		if (!wasHasNextCalled)
			hasNextFlow();
		wasHasNextCalled = false;
		this.offsetOfLastFlow = this.tmp_offsetOfLastFlow;
		this.offsetOfNextFlow = this.tmp_offsetOfNextFlow;
		return nextFlow;
	}
	
	
	public FlowGroupFlowIterator getFlowGroupFlowIterator() throws IOException {
		return new FgFlowIterator();
	}
	
	
	@Override
	public final long getPositionOfLastLine() {
		return getOffsetOfLastFlow();
	}
	
	
	public long getOffsetOfLastFlow() {
		return offsetOfLastFlow;
	}
	
	
	public long getOffsetOfNextFlow() {
		return offsetOfNextFlow;
	}


	public void reset() {
		this.nextFlow = null;
		this.wasHasNextCalled = false;
		this.offsetOfLastFlow = 0;
		this.offsetOfNextFlow = 0;
		this.tmp_offsetOfLastFlow = 0;
		this.tmp_offsetOfNextFlow = 0;
	}
	
	
	public class FgFlowIterator implements FlowGroupFlowIterator {
		
		public int hostId;	
		public long startOffset;
		public long offsetOfNextFlowGroup;
		public long endOfLatestFlow = Util.NOT_SET;
		private long endOfCurrentFlowGroup = Long.MIN_VALUE;
		private Vector<EndOfFlow> openFlows = new Vector<EndOfFlow>(100);
		private EndOfFlow latestFinishedFlow = null;
		private boolean hasNext = false;
		private boolean wasHasNextCalled = false;
		private long lastRestriction = Long.MIN_VALUE;

		
		public FgFlowIterator() throws IOException {
			if (peekNextFlow() == null)
				throw new RuntimeException("cannot init FlowGroupFlowIterator as Flow(Group)Reader has no more flows"); 
			this.hostId = peekNextFlow().senderId;
			this.startOffset = getOffsetOfNextFlow();
		}
		
		
		public boolean hasNext() throws IOException {
			if (wasHasNextCalled)
				return hasNext;
			wasHasNextCalled = true;
			if (peekNextFlow() == null || peekNextFlow().senderId != hostId) { // no more flows at all || next flow belongs to another host
				hasNext = false;
				offsetOfNextFlowGroup = getOffsetOfNextFlow();
				//close();
				return false;
			} else {
				// check if next flow belongs to a new flow group (due to user think time):
				if (openFlows.size() == 0) { // first flow
					hasNext = true;
					return true;
				} else { // not first flow -> drop finished flows
					long now = peekNextFlow().startOfFlow; // note that flows is ordered by flow.startOfFlow
					for (int i=0; i<openFlows.size(); i++) {
						EndOfFlow flow = openFlows.get(i);
						if (flow.endOfFlow <= now) {
							openFlows.remove(i);
							if (latestFinishedFlow == null || latestFinishedFlow.endOfFlow < flow.endOfFlow) 
								latestFinishedFlow = flow;
							i--;
						}
					}
					if (openFlows.size() == 0) { // end of flow group
						hasNext = false;
						offsetOfNextFlowGroup = getOffsetOfNextFlow();
						return false;
					} else { // still open flows -> NOT end of flow group
						hasNext = true;
						return true;
					}
				}
			}
		}
		
		
		public Flow next() throws IOException {
			if (!hasNext())
				throw new RuntimeException("no more flows in this flow group. use hasNext() to avoid this exception"); 
			wasHasNextCalled = false;
			Flow result = readFlow();
			if (calculateRestrictions) {
				if (openFlows.size() == 0 && ownerOfLastFlowGroup != result.senderId) { // first flow of current host
					result.restriction = Restriction.NONE;
					//System.out.println("flow " +result.flowId +" is not restricted at all (ownerOfLastFlowGroup: " +ownerOfLastFlowGroup +", current owner: " +result.senderId +")"); 
				} else { // not the first flow of the current host
					TransactionInfo blockingTransaction = EndOfFlow.getTransactionWithLatestReplyBefore(result.startOfFlow, openFlows);
					boolean isBlockedByTransaction = blockingTransaction != null;
					if (blockingTransaction != null && blockingTransaction.endOfRestrictingReply <= endOfLatestFlow) {
						//System.out.println("override: " +blockingTransaction.endOfRestrictingReply + " <= " +endOfLatestFlow); 
						isBlockedByTransaction = false;
					}
					if (isBlockedByTransaction) { // flow IS blocked by a reply of an open flow
						result.restriction = Restriction.NOT_BEFORE_END_OF_TRANSACTION;
						result.idOfRestrictingFlow = blockingTransaction.idOfRestrictingFlow;
						result.idOfRestrictingTransaction = blockingTransaction.arrayOffsetOfRestrictingTransaction;
						result.idOfRestrictingReply = blockingTransaction.arrayOffsetOfRestrictingReply;
						result.offsetFromRestriction = (int) (result.startOfFlow - blockingTransaction.endOfRestrictingReply);
						//System.out.println("flow " +result.flowId +" is restricted by reply " +blockingTransaction.arrayOffsetOfRestrictingReply +" of transaction " +blockingTransaction.arrayOffsetOfRestrictingTransaction +" of flow " +blockingTransaction.idOfRestrictingFlow +" -> offsetFromRestriction: " +result.offsetFromRestriction +" (" +result.startOfFlow +" - " +blockingTransaction.endOfRestrictingReply +")"); 
						assert blockingTransaction.endOfRestrictingReply >= lastRestriction;
						lastRestriction = blockingTransaction.endOfRestrictingReply;
					} else { // flow IS NOT blocked by a reply of an open flow -> use end of latest finished flow as restriction
						if (latestFinishedFlow == null) { // no flows finished yet -> use offset from end of last flow group or start of trace as delay
							if (ownerOfLastFlowGroup != result.senderId) { // no previous flow group -> use offset from start of trace as delay
								result.restriction = Restriction.SIMPLE_DELAY;
								result.offsetFromRestriction = (int)result.startOfFlow; // note that the trace always starts at 0
								//System.out.println("flow " +result.flowId +" is restricted by start of trace. offsetFromRestriction: " +result.offsetFromRestriction +"(" +result.startOfFlow +" - 0)");
								assert 0 >= lastRestriction;
								lastRestriction = 0;
							} else { // previous flow group PRESENT-> use offset from last flow group as delay
								result.restriction = Restriction.NOT_BEFORE_END_OF_OTHER_FLOW;
								result.idOfRestrictingFlow = latestFinishedFlowOfLastFlowGroup.flowId;
								result.offsetFromRestriction = (int) (result.startOfFlow - endOfLastFlowGroup);
								assert latestFinishedFlowOfLastFlowGroup.endOfFlow == endOfLastFlowGroup;
								//System.out.println("flow " +result.flowId +" is restricted by previous flow group only. idOfRestrictingFlow: " +result.idOfRestrictingFlow +", offsetFromRestriction: " +result.offsetFromRestriction +" (" +result.startOfFlow +" - " +endOfLastFlowGroup +")"); 
								assert endOfLastFlowGroup >= lastRestriction;
								lastRestriction = endOfLastFlowGroup;
							}
						} else { // latest finished flow is available -> use offset from latest finished flow
							result.restriction = Restriction.NOT_BEFORE_END_OF_OTHER_FLOW;
							result.idOfRestrictingFlow = latestFinishedFlow.flowId;
							result.offsetFromRestriction = (int) (result.startOfFlow - latestFinishedFlow.endOfFlow);
							//System.out.println("flow " +result.flowId +" is restricted by latest finished flow. idOfRestrictingFlow: " +result.idOfRestrictingFlow +", offsetFromRestriction: " +result.offsetFromRestriction +" (" +result.startOfFlow +" - " +latestFinishedFlow.endOfFlow +")"); 
							assert latestFinishedFlow.endOfFlow >= lastRestriction;
							lastRestriction = latestFinishedFlow.endOfFlow;
						}
					}
				}
			}
			openFlows.add(new EndOfFlow(result));
			if (result.endOfFlow > endOfLatestFlow)
				endOfLatestFlow = result.endOfFlow;
			if (result.endOfFlow > endOfCurrentFlowGroup)
				endOfCurrentFlowGroup = result.endOfFlow;
			if (!hasNext()) {
				endOfLastFlowGroup = endOfCurrentFlowGroup;
				latestFinishedFlowOfLastFlowGroup = latestFinishedFlow;
				ownerOfLastFlowGroup = result.senderId;
				//System.out.println("end of flow group"); 
			}
			return result;
		}
		
	}
	
	
	/**
	 * Comment
	 *
	 * @param args Not used.
	 */
	public static void main(String[] args) {
		try {
			testRestrictionMechanism();
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	} 
	
	
	public static void testRestrictionMechanism() throws IOException {
		FlowReader readerOfClient;
		try {
			RandomAccessFile raf = new RandomAccessFile("./inputOutput/global/traces/pcapTests/localCapture3/TestExtractorTest.gmf", "r");
			raf.seek(0);
			// use BufferedReader to read line instead of raf.readLine() for performance reasons:
			BufferedReader reader = new BufferedReader(Channels.newReader(raf.getChannel(), "ISO-8859-1"));
			readerOfClient = new FlowReader(reader, null);
		} catch (Exception e) {
			e.printStackTrace();
			throw new RuntimeException();
		}
		FlowGroupFlowIterator currentFlowGroup = null;
		while (readerOfClient.hasNextFlow()) {
			if (currentFlowGroup == null || !currentFlowGroup.hasNext()) {
				//System.out.println("next flow group");
				currentFlowGroup = readerOfClient.getFlowGroupFlowIterator();
			}
			currentFlowGroup.next();
		} 
		
	}
	
}