DereferencerBatch.java

/*
 *   Copyright (c) 2009, MediaEvent Services GmbH & Co. KG
 *   http://mediaeventservices.com
 *   
 *   This file is part of Marbles.
 *
 *   Marbles is free software: you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation, either version 3 of the License, or
 *   (at your option) any later version.
 *
 *   Marbles is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with Marbles.  If not, see <http://www.gnu.org/licenses/>.
 *   
 */
package de.fuberlin.wiwiss.marbles.loading;

import info.aduna.iteration.Iterations;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;

import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.URI;
import org.apache.commons.httpclient.URIException;
import org.openrdf.model.Resource;
import org.openrdf.model.Statement;
import org.openrdf.model.Value;
import org.openrdf.model.impl.URIImpl;
import org.openrdf.repository.RepositoryConnection;
import org.openrdf.repository.RepositoryException;
import org.openrdf.repository.RepositoryResult;

import de.fuberlin.wiwiss.marbles.Constants;
import de.fuberlin.wiwiss.marbles.dataproviders.DataProvider;

/**
 * Starting with one URL, the DereferencerBatch handles the nested retrieval of data
 * by following known predicates in retrieved data, and processing retrieval results 
 * with data providers.
 *  
 * @author Christian Becker
 */
public class DereferencerBatch implements DereferencingListener {
	
	private List<ExtendedDereferencingTask> pendingTasks = new ArrayList<ExtendedDereferencingTask>();
	private List<URI> retrievedURLs = new ArrayList<URI>();
	private CacheController cacheController;
	private Resource mainResource;
	private DereferencingTaskQueue uriQueue;
	private Collection<DataProvider> dataProviders;
	private int maxSteps;
	private int maxRedirects;
	
	/**
	 * Constructs a new <code>DereferencerBatch</code>
	 * @param cacheController
	 * @param uriQueue
	 * @param dataProviders
	 * @param mainResource
	 * @param maxSteps
	 */
	public DereferencerBatch(CacheController cacheController, DereferencingTaskQueue uriQueue, Collection<DataProvider> dataProviders, Resource mainResource, int maxSteps, int maxRedirects) {
		this.cacheController = cacheController;
		this.mainResource = mainResource;
		this.uriQueue = uriQueue;
		this.dataProviders = dataProviders;
		this.maxSteps = maxSteps;
		this.maxRedirects = maxRedirects;
	}

	/**
	 * Loads URL if not yet loaded
	 * 
	 * @param url	The URL to load
	 * @param step	The distance from the focal resource
	 * @param redirectCount	The number of redirects performed in the course of this individual request
	 * @param forceReload	Set this to true if the URL should be loaded even if a valid copy is already in the cache
	 * @throws URIException
	 */
	public void loadURL(URI url, int step, int redirectCount, boolean forceReload) throws URIException {
		if (step > maxSteps || redirectCount > maxRedirects)
			return;
		
		/* Cut off local names from URI */ 
		url.setFragment("");

		if (retrievedURLs.contains(url)) /* force reload doesn't apply on batch level, as they are short-lived and this could cause infinite loops */
			return;
		
		if (!forceReload && cacheController.hasURLData(url.toString())) {
			/* Treat as retrieved when reading from cache */
			retrievedURLs.add(url);
			
			String redirect = cacheController.getCachedRedirect(url.toString());
			
			/* Process a cached redirect */
			if (redirect != null) {
				URI redirectUrl = new URI(url, redirect, true);
				loadURL(redirectUrl, step, redirectCount + 1, forceReload);
			}
			else {
				/* Data is already loaded; try to find new links within it */
				try {
					org.openrdf.model.URI sesameUri = new URIImpl(url.toString()); 
					processLinks(step + 1, sesameUri);
				}
				catch (IllegalArgumentException e) {
					e.printStackTrace();
				}
			}
		}
		else {
			/* No data about this URL; get it */
			ExtendedDereferencingTask task = new ExtendedDereferencingTask(this, url.toString(), step, redirectCount, forceReload);
			if (uriQueue.addTask(task)) {
				pendingTasks.add(task);
				retrievedURLs.add(url);
			}
		}
	}
	
	/**
	 * Determines whether requests are pending below a specified step level
	 * @param maxLevel	Maximum step level to consider
	 * @return	true, if requests are pending
	 */
	public boolean hasPending(int maxLevel) {
		boolean pending = false;
		
		for (ExtendedDereferencingTask task : pendingTasks) {
			if (task.getStep() <= maxLevel && !task.isDone()) {
				pending = true;
				break;
			}
		}
		return pending;
	}
	
	/**
	 * Determines whether any requests are pending
	 * @return	true, if requests are pending
	 */
	public boolean hasPending() {
		return hasPending(Integer.MAX_VALUE);
	}
	
	/*
	 * TODO Determine whether a retrieval batch was executed successfully
	 * Problem: To do this, {@link DereferencingResult} should be a member of {@link DereferencingTask}, not vice versa 
	 */
	/*public boolean wasSuccess() {
		boolean success = true;
		
		for (ExtendedDereferencingTask task : pendingTasks) {
			if (task.isDone() && task.) {
				pending = true;
				break;
			}
		}
		return pending;
	}*/
	
	/**
	 * Called by {@link DereferencerThread} once data has been retrieved.
	 * Handles insertion into cache, processes redirects, and initiates following of known links
	 * for the retrieved URL using {@link #processLinks(int, Resource...)} 
	 */
	public void dereferenced(DereferencingResult result) {
		ExtendedDereferencingTask task = (ExtendedDereferencingTask) result.getTask();
		
		/* Add to cache - including header data for redirects */
		cacheController.addURLData(result.getURI(), result.getResultData(), result.getMethod());

		/* Handle known redirect */
		if (null != result.getMethod() && null != result.getMethod().getStatusLine()) /* against NullPointerException with getStatusCode() */ {
			int resultCode = result.getMethod().getStatusCode();
			if (HttpStatusCodes.isRedirect(resultCode)) {
				Header locationHeader;
		        if (null != (locationHeader = result.getMethod().getResponseHeader("location"))) {
					try {
						loadURL(new URI(new URI(result.getURI(), true), locationHeader.getValue(), true), task.getStep(), task.getRedirectStep() + 1, task.isForceReload());
					} catch (URIException e) {
						e.printStackTrace();
					} 
		        }
			}
		}

		task.setDone(true);
		
		/* Wake up parent */
		synchronized(this) {
			notify();
		}
		
		/* find new links */
		if (result.isSuccess())
			processLinks(task.getStep() + 1, new URIImpl(result.getURI()));
	}
	
	/**
	 * Identifies known links from loaded data and submits them to <code>{@link #loadURL(URI, int, int, boolean)}</code> 
	 * @param step	Current step level
	 * @param contexts	Contexts that are to be considered to find links
	 */
	public void processLinks(int step, Resource ... contexts) {
		if (step > maxSteps)
			return;
		
		RepositoryConnection conn = null;
		try {
			conn = cacheController.getDataRepository().getConnection();
			for (org.openrdf.model.URI predicate : Constants.interestingPredicates) {
				List<Statement> statementsList;
				RepositoryResult<Statement> statements = conn.getStatements(mainResource, predicate, null /* obj */, true /* includeInferred */, contexts);
				statementsList = Iterations.addAll(statements, new ArrayList<Statement>());
				statements.close();
				
				/* Also include inverse properties */
				statements = conn.getStatements(null, predicate, mainResource, true /* includeInferred */, contexts);
				Iterations.addAll(statements, statementsList);
				statements.close();
				
				List<URI> urlsToBeFetched = new ArrayList<URI>();
	
				for (Statement st : statementsList) {
				      Value obj = (st.getSubject().equals(mainResource) ? st.getObject() : st.getSubject());
				      if (obj instanceof org.openrdf.model.URI && !urlsToBeFetched.contains(obj.toString()))
						try {
							urlsToBeFetched.add(new URI(obj.toString(), true));
						} catch (URIException e) {
							e.printStackTrace();
						} catch (NullPointerException e) {
							e.printStackTrace();
						}
				}
				
				/* Ask data providers */
				for (DataProvider p : dataProviders) {
					List<URI> newURLs = p.getURLsFromData(cacheController, conn, mainResource);
					if (newURLs != null)
						urlsToBeFetched.addAll(newURLs);
				}
				
				/* Load URLs */
				for (URI url : urlsToBeFetched) {
						try {
							loadURL(url, step, 0 /* redirectStep */, false);
						} catch (URIException e) {
							e.printStackTrace();
						}
				}
			}
		}
		catch (RepositoryException e) {
			e.printStackTrace();
		}
		finally {
			try {
				if (conn != null)
					conn.close();
			} catch (RepositoryException e) {
				e.printStackTrace();
			}
		}
		
	}

	public List<URI> getRetrievedURLs() {
		return retrievedURLs;
	}
}