PathState.java

/*******************************************************************************
 * Copyright (c) 2011 Subgraph.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 * 
 * Contributors:
 *     Subgraph - initial API and implementation
 ******************************************************************************/
package com.subgraph.vega.impl.scanner.state;

import java.net.URI;
import java.util.ArrayList;
import java.util.List;

import org.apache.http.NameValuePair;
import org.apache.http.client.methods.HttpUriRequest;

import com.subgraph.vega.api.analysis.IContentAnalyzer;
import com.subgraph.vega.api.analysis.IContentAnalyzerResult;
import com.subgraph.vega.api.analysis.MimeType;
import com.subgraph.vega.api.crawler.ICrawlerResponseProcessor;
import com.subgraph.vega.api.http.requests.IHttpResponse;
import com.subgraph.vega.api.http.requests.IPageFingerprint;
import com.subgraph.vega.api.model.web.IWebPath;
import com.subgraph.vega.api.model.web.IWebPath.PathType;
import com.subgraph.vega.api.scanner.IInjectionModuleContext;
import com.subgraph.vega.api.scanner.IPathState;
import com.subgraph.vega.api.scanner.modules.IBasicModuleScript;
import com.subgraph.vega.impl.scanner.requests.BasicRequestBuilder;
import com.subgraph.vega.impl.scanner.requests.GetParameterRequestBuilder;
import com.subgraph.vega.impl.scanner.requests.IRequestBuilder;
import com.subgraph.vega.impl.scanner.requests.PostParameterRequestBuilder;

public class PathState implements IPathState {

	public static PathState createBasicPathState(ICrawlerResponseProcessor fetchProcessor, PathStateManager stateManager, PathState parentState, IWebPath path) {
		final IRequestBuilder rb = new BasicRequestBuilder(path);
		final PathState st = new PathState(fetchProcessor, stateManager, parentState, path, rb);
		if(parentState != null)
			parentState.addChildState(st, true);
		else {
			st.setLocked();
			st.performInitialFetch();
		}
		return st;
	}

	public static PathState createParameterPathState(ICrawlerResponseProcessor fetchProcessor, PathState parentState, List<NameValuePair> parameters, int fuzzIndex) {
		if(parentState == null)
			throw new IllegalArgumentException("Parent of parameter path cannot be null");
		final IRequestBuilder rb = new GetParameterRequestBuilder(parentState.getPath(), parameters, fuzzIndex);
		final PathState st = new PathState(fetchProcessor, parentState.getPathStateManager(), parentState, parentState.getPath(), rb);
		parentState.addChildState(st, false);
		return st;
	}

	public static PathState createPostParameterPathState(ICrawlerResponseProcessor fetchProcessor, PathState parentState, List<NameValuePair> parameters, int fuzzIndex) {
		if(parentState == null)
			throw new IllegalArgumentException("Parent of parameter path cannot be null");
		final IRequestBuilder rb = new PostParameterRequestBuilder(parentState.getPath(), parameters, fuzzIndex);
		final PathState st = new PathState(fetchProcessor, parentState.getPathStateManager(), parentState, parentState.getPath(), rb);
		parentState.addChildState(st, false);
		return st;
	}

	private final PathStateManager pathStateManager;
	private final IWebPath path;
	private final PathState parentState;
	private final List<PathState> childStates = new ArrayList<PathState>();
	private final IRequestBuilder requestBuilder;
	private final PathState404 state404;
	private final ICrawlerResponseProcessor initialFetchProcessor;

	private IHttpResponse response;
	private IPageFingerprint pathFingerprint;
	private IPageFingerprint unknownFingerprint;

	private boolean isDone;
	private boolean hasFailed404;
	private boolean hasBadParent;
	private boolean ipsDetected;

	private boolean isSureDirectory;
	private boolean isPageMissing;
	private boolean isBogusParameter;
	private boolean responseVaries;
	private boolean lockedFlag;
	private PathStateParameterManager parameterManager;
	
	private final Object childCountLock = new Object();
	private int descendantCount = 0;
	private int childCount = 0;

	private PathState(ICrawlerResponseProcessor fetchProcessor, PathStateManager stateManager, PathState parentState, IWebPath path, IRequestBuilder requestBuilder) {
		this.initialFetchProcessor = fetchProcessor;
		this.pathStateManager = stateManager;
		this.path = path;
		this.parentState = parentState;
		this.requestBuilder = requestBuilder;
		this.state404 = new PathState404(this);
	}

	@Override
	public PathState getParentState() {
		return parentState;
	}

	private void addChildState(PathState state, boolean checkDup) {
		synchronized(childStates) {
			if(checkDup) {
				for(IPathState cs: childStates) {
					if(cs.getPath().equals(state.getPath()))
						return;
				}
			}
			childStates.add(state);
			if(lockedFlag)
				state.setLocked();
			else
				state.performInitialFetch();
			
			synchronized(childCountLock) {
				childCount += 1;
				incrementDescendants();
			}
		}
	}

	private void incrementDescendants() {
		synchronized(childCountLock) {
			descendantCount += 1;
			if(parentState != null) {
				parentState.incrementDescendants();
			}
		}
	}

	public int getDescendantCount() {
		return descendantCount;
	}
	
	public int getChildCount() {
		return childCount;
	}

	public synchronized int getDepth() {
		if(parentState == null) {
			return 1;
		} else {
			return 1 + parentState.getDepth();
		}
	}

	public PathStateManager getPathStateManager() {
		return pathStateManager;
	}
	@Override
	public boolean isParametric() {
		return requestBuilder.isFuzzable();
	}

	@Override
	public boolean isDone() {
		return isDone;
	}

	@Override
	public void setDone() {
		isDone = true;
		response = null;
	}

	private void setLocked() {
		lockedFlag = true;
	}

	private void performInitialFetch() {
		final IInjectionModuleContext ctx = new ModuleContext(pathStateManager, requestBuilder, this, 0);
		final HttpUriRequest req = createRequest();

		if(response != null) {
			initialFetchProcessor.processResponse(pathStateManager.getCrawler(), req, response, ctx);
		} else {
			submitRequest(req, initialFetchProcessor, ctx);
		}
	}

	@Override
	public void setBogusParameter() {
		isBogusParameter = true;
	}

	@Override
	public boolean isBogusParameter() {
		return isBogusParameter;
	}

	@Override
	public IWebPath getPath() {
		return path;
	}

	@Override
	public void setUnknownFingerprint(IPageFingerprint fp) {
		unknownFingerprint = fp;
	}

	@Override
	public IPageFingerprint getUnknownFingerprint() {
		return unknownFingerprint;
	}

	public void submitRequest(ICrawlerResponseProcessor callback) {
		final HttpUriRequest req = requestBuilder.createBasicRequest();
		final IInjectionModuleContext ctx = createModuleContext();
		submitRequest(req, callback, ctx);
	}

	public void submitRequest(HttpUriRequest request, ICrawlerResponseProcessor callback) {
		final IInjectionModuleContext ctx = createModuleContext();
		submitRequest(request, callback, ctx);
	}

	public void submitRequest(HttpUriRequest request, ICrawlerResponseProcessor callback, IInjectionModuleContext ctx) {
		pathStateManager.getCrawler().submitTask(request, getWrappedCallback(callback), ctx);
	}

	private ICrawlerResponseProcessor getWrappedCallback(ICrawlerResponseProcessor callback) {
		if(pathStateManager.requestLoggingEnabled())
			return CrawlerCallbackWrapper.createLogging(pathStateManager.getRequestLog(), callback);
		else
			return CrawlerCallbackWrapper.create(callback);
	}

	@Override
	public HttpUriRequest createRequest() {
		return requestBuilder.createBasicRequest();
	}

	@Override
	public HttpUriRequest createAlteredRequest(String value, boolean append) {
		return requestBuilder.createAlteredRequest(value, append);
	}

	public boolean hasMaximum404Fingerprints() {
		return state404.hasMaximum404Fingerprints();
	}

	@Override
	public boolean add404Fingerprint(IPageFingerprint fp) {
		return state404.add404Fingerprint(fp);
	}

	@Override
	public boolean isRootPath() {
		return path.getParentPath() == null;
	}

	@Override
	public boolean has404Fingerprints() {
		return state404.has404Fingerprints();
	}

	@Override
	public IPathState get404Parent() {
		return state404.get404Parent();
	}

	@Override
	public boolean has404FingerprintMatching(IPageFingerprint fp) {
		return state404.has404FingerprintMatching(fp);
	}
	@Override
	public boolean hasParent404Fingerprint(IPageFingerprint fp) {
		return state404.hasParent404Fingerprint(fp);
	}

	public void dump404Fingerprints() {
		state404.dumpFingerprints();
	}
	@Override
	public boolean hasParent404FingerprintMatchingThis() {
		return state404.hasParent404Fingerprint(pathFingerprint);
	}

	@Override
	public void setSureDirectory() {
		isSureDirectory = true;
	}

	@Override
	public void clear404Fingerprints() {
		state404.clear404Fingerprints();
	}

	public void setSkip404() {
		state404.setSkip404();
	}

	public boolean getSkip404() {
		return state404.getSkip404();
	}

	@Override
	public boolean isSureDirectory() {
		return isSureDirectory;
	}

	@Override
	public void setResponse(IHttpResponse response) {
		this.response = response;
		if(response != null) {
			this.pathFingerprint = response.getPageFingerprint();
		} else {
			this.pathFingerprint = null;
			return;
		}

		if(response.getResponseCode() == 200) {
			addWebResponseToPath(response);
		}
	}

	private void addWebResponseToPath(IHttpResponse response) {
		final IContentAnalyzer contentAnalyzer = pathStateManager.getContentAnalyzer();
		final IContentAnalyzerResult result = contentAnalyzer.processResponse(response, false, false);
		final URI uri = createRequest().getURI();
		path.addGetResponse(uri.getQuery(), contentAnalyzerResultToMimeString(result));
	}

	private String contentAnalyzerResultToMimeString(IContentAnalyzerResult result) {
		if(result.getSniffedMimeType() != MimeType.MIME_NONE)
			return result.getSniffedMimeType().getCanonicalName();
		else if(result.getDeclaredMimeType() != MimeType.MIME_NONE)
			return result.getDeclaredMimeType().getCanonicalName();
		else
			return null;
	}

	@Override
	public IHttpResponse getResponse() {
		return response;
	}

	public boolean isPageMissing() {
		return isPageMissing;
	}

	@Override
	public void setPageMissing() {
		isPageMissing = true;
	}

	@Override
	public void setResponseVaries() {
		responseVaries = true;
	}

	@Override
	public boolean getResponseVaries() {
		return responseVaries;
	}

	public void debug(String msg) {
		pathStateManager.debug("["+path.getUri()+"] "+ msg);
	}

	@Override
	public IPageFingerprint getPathFingerprint() {
		return pathFingerprint;
	}

	@Override
	public boolean matchesPathFingerprint(IPageFingerprint fp) {
		if(pathFingerprint == null) {
			debug("Whoops no path fingerprint for "+ path.getUri() + " : " + this);
			return false;
		}
		return pathFingerprint.isSame(fp);
	}

	@Override
	public int allocateXssId() {
		return pathStateManager.allocateXssId();
	}
	@Override
	public String createXssTag(int xssId) {
		return pathStateManager.createXssTag(xssId);
	}

	@Override
	public String createXssTag(String prefix, int xssId) {
		return pathStateManager.createXssTag(prefix, xssId);
	}

	@Override
	public void registerXssRequest(HttpUriRequest request, int xssId) {
		pathStateManager.registerXssRequest(request, xssId);
	}

	@Override
	public HttpUriRequest getXssRequest(int xssId, int scanId) {
		return pathStateManager.getXssRequest(xssId, scanId);
	}

	@Override
	public NameValuePair getFuzzableParameter() {
		return requestBuilder.getFuzzableParameter();
	}

	@Override
	public void maybeAddParameters(List<NameValuePair> parameters) {
		final PathStateParameterManager pm = getParameterManager();
		synchronized(pm) {
			if(parameters.size() > pathStateManager.getMaxParameterCount()) {
				return;
			} else if(pm.hasParameterList(parameters)) {
				return;
			} else if(pathStateManager.hasExceededLimits(this)) {
				return;
			} else {
				pm.addParameterList(parameters);
			}
		}
	}

	@Override
	public void maybeAddPostParameters(List<NameValuePair> parameters) {
		final PathStateParameterManager pm = getParameterManager();
		synchronized(pm) {
			if(parameters.size() > pathStateManager.getMaxParameterCount()) {
				return;
			}
			if(!pm.hasPostParameterList(parameters))
				pm.addPostParameterList(parameters);
		}
	}

	private synchronized PathStateParameterManager getParameterManager() {
		if(parameterManager == null) {
			parameterManager = new PathStateParameterManager(this);
		}
		return parameterManager;
	}

	@Override
	public void unlockChildren() {
		synchronized(childStates) {
			if(!lockedFlag)
				return;
			lockedFlag = false;
			for(PathState c: childStates) {
				c.performInitialFetch();
			}
		}
	}

	@Override
	public String toString() {
		return "STATE: ["+ requestBuilder + "]";
	}

	@Override
	public IInjectionModuleContext createModuleContext() {
		return new ModuleContext(pathStateManager, requestBuilder, this);
	}

	@Override
	public void setFailed404Detection() {
		hasFailed404 = true;
	}

	@Override
	public boolean hasFailed404Detection() {
		return hasFailed404;
	}

	@Override
	public void setBadParentDirectory() {
		hasBadParent = true;
	}

	@Override
	public boolean hasBadParentDirectory() {
		return hasBadParent;
	}

	@Override
	public boolean isIPSDetected() {
		return ipsDetected;
	}

	@Override
	public void setIPSDetected() {
		ipsDetected = true;
	}

	@Override
	public boolean doInjectionChecks() {
		if(isParametric())
			return true;
		else if(path.getPathType() == PathType.PATH_DIRECTORY)
			return pathStateManager.getDirectoryInjectionChecksFlag();
		else
			return pathStateManager.getNonParameterFileInjectionChecksFlag();
	}

	@Override
	public List<IBasicModuleScript> getInjectionModules() {
		return pathStateManager.getInjectionModules();
	}
}