NcsaLogParserTest.java

/*******************************************************************************
 * Copyright 2011 André Rouél
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ******************************************************************************/
package net.sf.jacclog.service.importer.internal.parser;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Locale;

import net.sf.jacclog.api.domain.ReadonlyLogEntry;
import net.sf.jacclog.api.domain.http.HttpRequestHeader;
import net.sf.jacclog.api.domain.http.HttpRequestHeaderField;
import net.sf.jacclog.api.domain.http.HttpRequestMethod;
import net.sf.jacclog.api.domain.http.HttpStatus;
import net.sf.jacclog.logformat.LogFormat;

import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Test for NCSA Log parser.
 */
public class NcsaLogParserTest {

	private static final Logger LOG = LoggerFactory.getLogger(NcsaLogParserTest.class);

	private void printResult(final String line, final List<String> tokens) {
		LOG.info(">>>\n" + line);
		LOG.info("tokens={");
		int i = 0;
		for (final String token : tokens) {
			LOG.info("  [" + i + "] = " + token);
			i++;
		}
		LOG.info("}");
	}

	@Test
	public void testParseCombinedLogFormat() throws Exception {
		final String line = "192.168.123.12 - - [19/Oct/2008:19:45:38 -0700] \""
				+ "GET /search?q1=foo&st=bar HTTP/1.1\" 200 323 "
				+ "\"-\" \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.14) "
				+ "Gecko/20080416 Fedora/2.0.0.14-1.fc7 Firefox/2.0.0.14\"";
		final List<String> tokens = NcsaLogParser.parse(line);
		printResult(line, tokens);

		final ReadonlyLogEntry entry = new NcsaLogParser(LogFormat.Defaults.COMBINED.getFormat()).parseLine(line);
		// TODO %l is not tested yet
		Assert.assertEquals("192.168.123.12", entry.getRemoteHost());
		Assert.assertEquals("-", entry.getRemoteUser());
		Assert.assertSame(HttpRequestMethod.GET, entry.getRequestMethod());
		final long epoch = new SimpleDateFormat("dd/MMM/yyyy:HH:mm:ss Z", Locale.ENGLISH).parse(
				"19/Oct/2008:19:45:38 -0700").getTime();
		Assert.assertEquals(new Date(epoch), entry.getRequestTime());
		Assert.assertEquals("/search", entry.getUrlPath());
		Assert.assertEquals("q1=foo&st=bar", entry.getQueryString());
		Assert.assertSame(HttpStatus.OK, entry.getLastStatusCode());
		Assert.assertEquals(Long.valueOf(323), entry.getResponseInBytes());

		// check size
		Assert.assertEquals(2, entry.getRequestHeaders().size());

		// check referer
		HttpRequestHeaderField referer = new HttpRequestHeaderField(HttpRequestHeader.REFERER, "-");
		Assert.assertTrue(entry.getRequestHeaders().contains(referer));

		// check user agent
		HttpRequestHeaderField userAgent = new HttpRequestHeaderField(HttpRequestHeader.USER_AGENT,
				"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.14) Gecko/20080416 Fedora/2.0.0.14-1.fc7 Firefox/2.0.0.14");
		Assert.assertTrue(entry.getRequestHeaders().contains(userAgent));

		LOG.info(entry.toString());
	}

	@Test
	public void testParseCombinedLogFormatWithCookie() throws Exception {
		final String line = "192.168.123.12 - - [19/Oct/2008:19:45:38 -0700] \""
				+ "GET /search?q1=foo&st=bar HTTP/1.1\" 200 323 "
				+ "\"-\" \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.14) "
				+ "Gecko/20080416 Fedora/2.0.0.14-1.fc7 Firefox/2.0.0.14\" "
				+ "\"USER_ID=12345,jsession_id=3BFY342211\"";
		final List<String> tokens = NcsaLogParser.parse(line);
		printResult(line, tokens);
	}

	@Test
	public void testParseCommonLogFormatNoCookie() throws Exception {
		final String line = "192.168.123.12 - - [19/Oct/2008:19:45:38 -0700] \""
				+ "GET /search?q1=foo&st=bar HTTP/1.1\" 200 323";
		final List<String> tokens = NcsaLogParser.parse(line);
		printResult(line, tokens);

		final ReadonlyLogEntry entry = new NcsaLogParser(LogFormat.Defaults.COMMON.getFormat()).parseLine(line);
		// TODO %l is not tested yet
		Assert.assertEquals("192.168.123.12", entry.getRemoteHost());
		Assert.assertEquals("-", entry.getRemoteUser());
		Assert.assertSame(HttpRequestMethod.GET, entry.getRequestMethod());
		final long epoch = new SimpleDateFormat("dd/MMM/yyyy:HH:mm:ss Z", Locale.ENGLISH).parse(
				"19/Oct/2008:19:45:38 -0700").getTime();
		Assert.assertEquals(new Date(epoch), entry.getRequestTime());
		Assert.assertEquals("/search", entry.getUrlPath());
		Assert.assertEquals("q1=foo&st=bar", entry.getQueryString());
		Assert.assertSame(HttpStatus.OK, entry.getLastStatusCode());
		Assert.assertEquals(Long.valueOf(323), entry.getResponseInBytes());

		LOG.info(entry.toString());
	}

	@Test
	public void testParseCustomLogFormat() throws Exception {
		LOG.info("TEST--------------------------------");
		final String line = "192.168.123.12 - - [19/Oct/2008:19:45:38 -0700] \""
				+ "GET /search?q1=foo&st=bar HTTP/1.1\" 200 323 "
				+ "\"-\" 34567 \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.14) "
				+ "Gecko/20080416 Fedora/2.0.0.14-1.fc7 Firefox/2.0.0.14\"";
		final List<String> tokens = NcsaLogParser.parse(line);
		printResult(line, tokens);

		final ReadonlyLogEntry entry = new NcsaLogParser(
				LogFormat.parse("%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" %D \"%{User-Agent}i\"")).parseLine(line);
		// TODO %l is not tested yet
		// TODO %D is not tested yet
		Assert.assertEquals("192.168.123.12", entry.getRemoteHost());
		Assert.assertEquals("-", entry.getRemoteUser());
		Assert.assertSame(HttpRequestMethod.GET, entry.getRequestMethod());
		final long epoch = new SimpleDateFormat("dd/MMM/yyyy:HH:mm:ss Z", Locale.ENGLISH).parse(
				"19/Oct/2008:19:45:38 -0700").getTime();
		Assert.assertEquals(new Date(epoch), entry.getRequestTime());
		Assert.assertEquals("/search", entry.getUrlPath());
		Assert.assertEquals("q1=foo&st=bar", entry.getQueryString());
		Assert.assertSame(HttpStatus.OK, entry.getLastStatusCode());
		Assert.assertEquals(Long.valueOf(323), entry.getResponseInBytes());

		// check size
		Assert.assertEquals(2, entry.getRequestHeaders().size());

		// check referer
		HttpRequestHeaderField referer = new HttpRequestHeaderField(HttpRequestHeader.REFERER, "-");
		Assert.assertTrue(entry.getRequestHeaders().contains(referer));

		// check user agent
		HttpRequestHeaderField userAgent = new HttpRequestHeaderField(HttpRequestHeader.USER_AGENT,
				"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.14) Gecko/20080416 Fedora/2.0.0.14-1.fc7 Firefox/2.0.0.14");
		Assert.assertTrue(entry.getRequestHeaders().contains(userAgent));

		LOG.info(entry.toString());
	}

	// @Test
	public void testParsing() throws Exception {
		final File[] accesslogs = (new File("src/test/resources/access_logs")).listFiles();
		for (final File accesslog : accesslogs) {
			String line;
			final BufferedReader reader = new BufferedReader(new FileReader(accesslog));
			while ((line = reader.readLine()) != null) {
				final List<String> tokens = NcsaLogParser.parse(line);
				printResult(line, tokens);
				// final String url = tokens.get(4);
				// if (url.contains("/search")) {
				// printResult(line, tokens);
				// }
			}
			reader.close();
		}
	}

}