/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nutch.tools;
import java.io.*;
import java.util.*;
import java.util.regex.*;
import javax.xml.parsers.*;
import org.xml.sax.*;
import org.xml.sax.helpers.*;
import org.apache.xerces.util.XMLChar;
// Commons Logging imports
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.util.LogUtil;
import org.apache.nutch.util.NutchConfiguration;
/** Utility that converts DMOZ RDF into a flat file of URLs to be injected. */
public class DmozParser {
public static final Log LOG = LogFactory.getLog(DmozParser.class);
long pages = 0;
/**
* This filter fixes characters that might offend our parser.
* This lets us be tolerant of errors that might appear in the input XML.
*/
private static class XMLCharFilter extends FilterReader {
private boolean lastBad = false;
public XMLCharFilter(Reader reader) {
super(reader);
}
public int read() throws IOException {
int c = in.read();
int value = c;
if (c != -1 && !(XMLChar.isValid(c))) // fix invalid characters
value = 'X';
else if (lastBad && c == '<') { // fix mis-matched brackets
in.mark(1);
if (in.read() != '/')
value = 'X';
in.reset();
}
lastBad = (c == 65533);
return value;
}
public int read(char[] cbuf, int off, int len)
throws IOException {
int n = in.read(cbuf, off, len);
if (n != -1) {
for (int i = 0; i < n; i++) {
char c = cbuf[off+i];
char value = c;
if (!(XMLChar.isValid(c))) // fix invalid characters
value = 'X';
else if (lastBad && c == '<') { // fix mis-matched brackets
if (i != n-1 && cbuf[off+i+1] != '/')
value = 'X';
}
lastBad = (c == 65533);
cbuf[off+i] = value;
}
}
return n;
}
}
/**
* The RDFProcessor receives tag messages during a parse
* of RDF XML data. We build whatever structures we need
* from these messages.
*/
private class RDFProcessor extends DefaultHandler {
String curURL = null, curSection = null;
boolean titlePending = false, descPending = false, insideAdultSection = false;
Pattern topicPattern = null;
StringBuffer title = new StringBuffer(), desc = new StringBuffer();
XMLReader reader;
int subsetDenom;
int hashSkew;
boolean includeAdult;
Locator location;
/**
* Pass in an XMLReader, plus a flag as to whether we
* should include adult material.
*/
public RDFProcessor(XMLReader reader, int subsetDenom, boolean includeAdult, int skew, Pattern topicPattern) throws IOException {
this.reader = reader;
this.subsetDenom = subsetDenom;
this.includeAdult = includeAdult;
this.topicPattern = topicPattern;
this.hashSkew = skew != 0 ? skew : new Random().nextInt();
}
//
// Interface ContentHandler
//
/**
* Start of an XML elt
*/
public void startElement(String namespaceURI, String localName, String qName, Attributes atts) throws SAXException {
if ("Topic".equals(qName)) {
curSection = atts.getValue("r:id");
} else if ("ExternalPage".equals(qName)) {
// Porn filter
if ((! includeAdult) && curSection.startsWith("Top/Adult")) {
return;
}
if (topicPattern != null && !topicPattern.matcher(curSection).matches()) {
return;
}
// Subset denominator filter.
// Only emit with a chance of 1/denominator.
String url = atts.getValue("about");
int hashValue = MD5Hash.digest(url).hashCode();
hashValue = Math.abs(hashValue ^ hashSkew);
if ((hashValue % subsetDenom) != 0) {
return;
}
// We actually claim the URL!
curURL = url;
} else if (curURL != null && "d:Title".equals(qName)) {
titlePending = true;
} else if (curURL != null && "d:Description".equals(qName)) {
descPending = true;
}
}
/**
* The contents of an XML elt
*/
public void characters(char ch[], int start, int length) {
if (titlePending) {
title.append(ch, start, length);
} else if (descPending) {
desc.append(ch, start, length);
}
}
/**
* Termination of XML elt
*/
public void endElement(String namespaceURI, String localName, String qName)
throws SAXException {
if (curURL != null) {
if ("ExternalPage".equals(qName)) {
//
// Inc the number of pages, insert the page, and
// possibly print status.
//
System.out.println(curURL);
pages++;
//
// Clear out the link text. This is what
// you would use for adding to the linkdb.
//
if (title.length() > 0) {
title.delete(0, title.length());
}
if (desc.length() > 0) {
desc.delete(0, desc.length());
}
// Null out the URL.
curURL = null;
} else if ("d:Title".equals(qName)) {
titlePending = false;
} else if ("d:Description".equals(qName)) {
descPending = false;
}
}
}
/**
* When parsing begins
*/
public void startDocument() {
LOG.info("Begin parse");
}
/**
* When parsing ends
*/
public void endDocument() {
LOG.info("Completed parse. Found " + pages + " pages.");
}
/**
* From time to time the Parser will set the "current location"
* by calling this function. It's useful for emitting locations
* for error messages.
*/
public void setDocumentLocator(Locator locator) {
location = locator;
}
//
// Interface ErrorHandler
//
/**
* Emit the exception message
*/
public void error(SAXParseException spe) {
if (LOG.isFatalEnabled()) {
LOG.fatal("Error: " + spe.toString() + ": " + spe.getMessage());
spe.printStackTrace(LogUtil.getFatalStream(LOG));
}
}
/**
* Emit the exception message, with line numbers
*/
public void fatalError(SAXParseException spe) {
if (LOG.isFatalEnabled()) {
LOG.fatal("Fatal err: " + spe.toString() + ": " + spe.getMessage());
LOG.fatal("Last known line is " + location.getLineNumber() +
", column " + location.getColumnNumber());
spe.printStackTrace(LogUtil.getFatalStream(LOG));
}
}
/**
* Emit exception warning message
*/
public void warning(SAXParseException spe) {
if (LOG.isWarnEnabled()) {
LOG.warn("Warning: " + spe.toString() + ": " + spe.getMessage());
spe.printStackTrace(LogUtil.getWarnStream(LOG));
}
}
}
/**
* Iterate through all the items in this structured DMOZ file.
* Add each URL to the web db.
*/
public void parseDmozFile(File dmozFile, int subsetDenom,
boolean includeAdult,
int skew,
Pattern topicPattern)
throws IOException, SAXException, ParserConfigurationException {
SAXParserFactory parserFactory = SAXParserFactory.newInstance();
SAXParser parser = parserFactory.newSAXParser();
XMLReader reader = parser.getXMLReader();
// Create our own processor to receive SAX events
RDFProcessor rp =
new RDFProcessor(reader, subsetDenom, includeAdult,
skew, topicPattern);
reader.setContentHandler(rp);
reader.setErrorHandler(rp);
LOG.info("skew = " + rp.hashSkew);
//
// Open filtered text stream. The TextFilter makes sure that
// only appropriate XML-approved Text characters are received.
// Any non-conforming characters are silently skipped.
//
XMLCharFilter in = new XMLCharFilter(new BufferedReader(new InputStreamReader(new BufferedInputStream(new FileInputStream(dmozFile)), "UTF-8")));
try {
InputSource is = new InputSource(in);
reader.parse(is);
} catch (Exception e) {
if (LOG.isFatalEnabled()) {
LOG.fatal(e.toString());
e.printStackTrace(LogUtil.getFatalStream(LOG));
}
System.exit(0);
} finally {
in.close();
}
}
private static void addTopicsFromFile(String topicFile,
Vector<String> topics)
throws IOException {
BufferedReader in = null;
try {
in = new BufferedReader(new InputStreamReader(new FileInputStream(topicFile), "UTF-8"));
String line = null;
while ((line = in.readLine()) != null) {
topics.addElement(new String(line));
}
}
catch (Exception e) {
if (LOG.isFatalEnabled()) {
LOG.fatal(e.toString());
e.printStackTrace(LogUtil.getFatalStream(LOG));
}
System.exit(0);
} finally {
in.close();
}
}
/**
* Command-line access. User may add URLs via a flat text file
* or the structured DMOZ file. By default, we ignore Adult
* material (as categorized by DMOZ).
*/
public static void main(String argv[]) throws Exception {
if (argv.length < 1) {
System.err.println("Usage: DmozParser <dmoz_file> [-subset <subsetDenominator>] [-includeAdultMaterial] [-skew skew] [-topicFile <topic list file>] [-topic <topic> [-topic <topic> [...]]]");
return;
}
//
// Parse the command line, figure out what kind of
// URL file we need to load
//
int subsetDenom = 1;
int skew = 0;
String dmozFile = argv[0];
boolean includeAdult = false;
Pattern topicPattern = null;
Vector<String> topics = new Vector<String>();
Configuration conf = NutchConfiguration.create();
FileSystem fs = FileSystem.get(conf);
try {
for (int i = 1; i < argv.length; i++) {
if ("-includeAdultMaterial".equals(argv[i])) {
includeAdult = true;
} else if ("-subset".equals(argv[i])) {
subsetDenom = Integer.parseInt(argv[i+1]);
i++;
} else if ("-topic".equals(argv[i])) {
topics.addElement(argv[i+1]);
i++;
} else if ("-topicFile".equals(argv[i])) {
addTopicsFromFile(argv[i+1], topics);
i++;
} else if ("-skew".equals(argv[i])) {
skew = Integer.parseInt(argv[i+1]);
i++;
}
}
DmozParser parser = new DmozParser();
if (!topics.isEmpty()) {
String regExp = new String("^(");
int j = 0;
for ( ; j < topics.size() - 1; ++j) {
regExp = regExp.concat(topics.get(j));
regExp = regExp.concat("|");
}
regExp = regExp.concat(topics.get(j));
regExp = regExp.concat(").*");
LOG.info("Topic selection pattern = " + regExp);
topicPattern = Pattern.compile(regExp);
}
parser.parseDmozFile(new File(dmozFile), subsetDenom,
includeAdult, skew, topicPattern);
} finally {
fs.close();
}
}
}