RemoteExclusionFilter.java

/*
 *  This file is part of the Wayback archival access software
 *   (http://archive-access.sourceforge.net/projects/wayback/).
 *
 *  Licensed to the Internet Archive (IA) by one or more individual 
 *  contributors. 
 *
 *  The IA licenses this file to You under the Apache License, Version 2.0
 *  (the "License"); you may not use this file except in compliance with
 *  the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */
package org.archive.wayback.accesscontrol.remote;

import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;

import java.util.logging.Logger;
import org.archive.wayback.core.CaptureSearchResult;
import org.archive.wayback.resourceindex.filters.ExclusionFilter;

/**
 * SearchResultFilter which uses remote access control/exclusion service to
 * filter results.
 *
 * @deprecated superseded by ExclusionOracle
 * @author brad
 * @version $Date$, $Revision$
 */
public class RemoteExclusionFilter extends ExclusionFilter {
	private static final Logger LOGGER = Logger.getLogger(RemoteExclusionFilter.class
			.getName());


	private final static int BYTE_BUFFER_SIZE = 4096;
	
	private static String OPERATION_ARGUMENT = "operation";
	private static String CHECK_OPERATION = "check";
	
	private static String URL_ARGUMENT = "url";
	private static String USER_AGENT_ARGUMENT = "useragent";
	private static String TIMESTAMP_ARGUMENT = "timestamp"; 
	private static String OK_CONTENT = "OK";
	
	private String urlPrefix = null;
	private String userAgent = null;

	// allocate byte buffer once
	private byte[] bbuffer = new byte[BYTE_BUFFER_SIZE];
	
	/**
	 * Constructor
	 * 
	 * @param urlPrefix String prefix for remote exclusion service
	 * @param userAgent String user agent to send to remote exclusion service
	 */
	public RemoteExclusionFilter(final String urlPrefix, final String userAgent) {
		this.urlPrefix = urlPrefix;
		this.userAgent = userAgent;
	}
	
	private boolean isBlocked(final String urlString, final String captureDate) {
		boolean blocked = true;
		
		StringBuffer finalUrl = new StringBuffer(urlPrefix);
		finalUrl.append("?");

		finalUrl.append(OPERATION_ARGUMENT);
		finalUrl.append("=");
		finalUrl.append(CHECK_OPERATION);


		finalUrl.append("&");
		finalUrl.append(URL_ARGUMENT);
		finalUrl.append("=");
		try {
			finalUrl.append(URLEncoder.encode(urlString,"UTF-8"));
		} catch (UnsupportedEncodingException e) {
			// TODO what happenned here?
			e.printStackTrace();
		}

		finalUrl.append("&");
		finalUrl.append(USER_AGENT_ARGUMENT);
		finalUrl.append("=");
		try {
			finalUrl.append(URLEncoder.encode(userAgent,"UTF-8"));
		} catch (UnsupportedEncodingException e) {
			// TODO what happenned here?
			e.printStackTrace();
		}

		finalUrl.append("&");
		finalUrl.append(TIMESTAMP_ARGUMENT);
		finalUrl.append("=");
		try {
			finalUrl.append(URLEncoder.encode(captureDate,"UTF-8"));
		} catch (UnsupportedEncodingException e) {
			// TODO what happenned here?
			e.printStackTrace();
		}
		
		try {

			// TODO: this is so ugly -- there must be a better/easy way.
			URL url = new URL(finalUrl.toString());
			InputStream is = url.openStream();
			// slurp the whole thing into RAM:
			// TODO: character encoding detection & handling!!
			StringBuilder sbuffer = new StringBuilder(BYTE_BUFFER_SIZE);
			for (int r = -1; (r = is.read(bbuffer, 0, BYTE_BUFFER_SIZE)) != -1;) {
				sbuffer.append(new String(bbuffer, 0, r));
			}
			String content = sbuffer.toString();
			if(content.equals(OK_CONTENT)) {
				blocked = false;
			} else {
				LOGGER.info("Disallowing " + urlString + " because " + content);
			}
		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return blocked;
	}
	
	/* (non-Javadoc)
	 * @see org.archive.wayback.resourceindex.SearchResultFilter#filterSearchResult(org.archive.wayback.core.SearchResult)
	 */
	public int filterObject(CaptureSearchResult r) {
		String captureDate = r.getCaptureTimestamp();
		String url = r.getOriginalUrl();
		return isBlocked(url,captureDate) ?	FILTER_EXCLUDE : FILTER_INCLUDE;
	}
}