WarcResource.java

/*
 *  This file is part of the Wayback archival access software
 *   (http://archive-access.sourceforge.net/projects/wayback/).
 *
 *  Licensed to the Internet Archive (IA) by one or more individual
 *  contributors.
 *
 *  The IA licenses this file to You under the Apache License, Version 2.0
 *  (the "License"); you may not use this file except in compliance with
 *  the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */
package org.archive.wayback.resourcestore.resourcefile;

import java.io.IOException;
import java.text.ParseException;
import java.util.Date;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.Map;

import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.StatusLine;
import org.apache.commons.httpclient.util.EncodingUtil;
import org.archive.format.arc.ARCConstants;
import org.archive.format.warc.WARCConstants.WARCRecordType;
import org.archive.io.ArchiveReader;
import org.archive.io.ArchiveRecordHeader;
import org.archive.io.RecoverableIOException;
import org.archive.io.warc.WARCRecord;
import org.archive.util.ArchiveUtils;
import org.archive.util.DateUtils;
import org.archive.util.LaxHttpParser;
import org.archive.wayback.core.Resource;
import org.archive.wayback.replay.HttpHeaderOperation;

public class WarcResource extends Resource {
	private WARCRecord rec = null;
	private ArchiveReader reader = null;
	private Map<String, String> headers = null;
	private long length = 0;
	private int status = 0;
	private boolean parsedHeaders = false;
	public WarcResource(WARCRecord rec, ArchiveReader reader) {
		this.rec = rec;
		this.reader = reader;
	}

	/**
	 * @param bytes Array of bytes to examine for an EOL.
	 * @return Count of end-of-line characters or zero if none.
	 */
	private int getEolCharsCount(byte [] bytes) {
		int count = 0;
		if (bytes != null && bytes.length >=1 &&
				bytes[bytes.length - 1] == '\n') {
			count++;
			if (bytes.length >=2 && bytes[bytes.length -2] == '\r') {
				count++;
			}
		}
		return count;
	}

	public void parseHeaders() throws IOException {
		if(parsedHeaders) {
			return;
		}

		// If warc or arc record is 0 length, don't do any more parsing!
		// Hopefully caller code will check this before proceeding as well
		if (getRecordLength() <= 0) {
			parsedHeaders = true;
			return;
		}

		// WARCRecord should have getRecordType() method returning WARCRecordType.
		String rectypeStr = (String)rec.getHeader().getHeaderValue("WARC-Type");
		WARCRecordType rectype;
		try {
		    rectype = WARCRecordType.valueOf(rectypeStr);
		} catch (IllegalArgumentException ex) {
		    throw new RecoverableIOException("unrecognized WARC-Type \"" + rectypeStr + "\"");
		}

		if (rectype == WARCRecordType.response || rectype == WARCRecordType.revisit) {
		    byte [] statusBytes = LaxHttpParser.readRawLine(rec);
		    int eolCharCount = getEolCharsCount(statusBytes);
		    if (eolCharCount <= 0) {
		        throw new RecoverableIOException("Failed to read http status where one " +
		                " was expected: " + new String(statusBytes));
		    }
		    String statusLineStr = EncodingUtil.getString(statusBytes, 0,
		            statusBytes.length - eolCharCount, ARCConstants.DEFAULT_ENCODING);
		    if ((statusLineStr == null) ||
		            !StatusLine.startsWithHTTP(statusLineStr)) {
		        throw new RecoverableIOException("Failed parse of http status line.");
		    }
		    StatusLine statusLine = new StatusLine(statusLineStr);

		    this.status = statusLine.getStatusCode();

		    Header[] tmpHeaders = LaxHttpParser.parseHeaders(rec,
		            ARCConstants.DEFAULT_ENCODING);
		    headers = new Hashtable<String,String>();
		    this.setInputStream(rec);
		    for(Header header: tmpHeaders) {
		        headers.put(header.getName(), header.getValue());
		        if(header.getName().toUpperCase().contains(
		                HttpHeaderOperation.HTTP_TRANSFER_ENC_HEADER)) {
		            if(header.getValue().toUpperCase().contains(
		                    HttpHeaderOperation.HTTP_CHUNKED_ENCODING_HEADER)) {
		                setChunkedEncoding();
		            }
		        }
		    }
		} else if (rectype == WARCRecordType.metadata || rectype == WARCRecordType.resource) {
		    status = 200;
		    headers = new HashMap<String, String>();
		    String ct = (String)rec.getHeader().getHeaderValue("Content-Type");
		    if (ct != null) {
		        headers.put("Content-Type", ct);
		    }
		    // necessary?
		    String date = rec.getHeader().getDate();
		    if (date != null) {
		        try {
		            Date d = org.apache.commons.lang.time.DateUtils.parseDate(date, new String[] { "yyyy-MM-dd'T'HH:mm:ss'Z'"});
		            String httpDate = DateUtils.getRFC1123Date(d);
		            headers.put("Date", httpDate);
		        } catch (ParseException ex) {
		            //
		        }
		    }
		    setInputStream(rec);
		}
		parsedHeaders = true;
	}

	@Override
	public Map<String, String> getHttpHeaders() {
		return headers;
	}

	public ArchiveRecordHeader getWarcHeaders() {
		return rec.getHeader();
	}

	@Override
	public long getRecordLength() {
		if ((length == 0) && (rec.getHeader() != null)) {
			length = rec.getHeader().getContentLength();
		}
		return length;
	}

	@Override
	public int getStatusCode() {
		return status;
	}

	@Override
	public void close() throws IOException {
		rec.close();
		reader.close();
	}

	public String getRefersToTargetURI() {
		return (String)getWarcHeaders().getHeaderFields().get(
			"WARC-Refers-To-Target-URI");
	}

	public String getRefersToDate() {
		String dateString = (String)getWarcHeaders().getHeaderFields().get(
			"WARC-Refers-To-Date");
		if (dateString != null) {
			Date date = ArchiveUtils.parse14DigitISODate(dateString, null);
			if (date != null) {
				return ArchiveUtils.get14DigitDate(date);
			}
		}
		return null;
	}

	public static final String PROFILE_REVISIT_SERVER_NOT_MODIFIED =
			"http://netpreserve.org/warc/1.0/revisit/server-not-modified";

	/**
	 * whether this Resource is {@code server-not-modified} revisit.
	 * (this method used to be {@code AccessPoint#isWarcRevisitNotModified(Resource)}.
	 * Not made a part of {@code Resource} interface because it was unused.)
	 * @return {@code true} if it is
	 */
	public boolean isRevisitNotModified() {
		Map<String, Object> warcHeaders = getWarcHeaders().getHeaderFields();
		String warcProfile = (String)warcHeaders.get("WARC-Profile");
		return PROFILE_REVISIT_SERVER_NOT_MODIFIED.equals(warcProfile);
	}
}