/* ======================================================================= * Copyright 2004-2006 The OpenMentor Team. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * ======================================================================== */ package uk.org.openmentor.classifier.impl; import uk.org.openmentor.classifier.Classifier; import java.util.regex.Pattern; import java.util.regex.Matcher; import java.io.InputStreamReader; import java.io.Reader; import java.io.InputStream; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.HashSet; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.InputSource; import org.xml.sax.SAXParseException; /** * @author Stuart Watt, Gordon Eccleston, Hannah Cumming, Sree Nuti * * <p> * This class contains the pattern classifier, which is the first of the main * classification engines in OpenMentor. It is not necessarily the only one, * and future engines will be needed as machine learning and other * classification techniques become more important. * </p> * <p> * The pattern classifier uses a set of regular expressions, each of which is * associated with some categories to include, and possibly others to exclude. * The algorithm essentially tries all the patterns one at a time, and then * builds up a list of matching (and non-matching) categories. When complete, * all the non-matching ones are removed, and the remaining matching categories * are turned into a comma-separated string. * </p> */ public class PatternClassifier implements Classifier { /** * The name of the classpath resource containing the rules. */ private String sourceFile; /** List (in order) of rules to try for classification. */ private List<Rule> vRules; /** Table of excluded categories. */ private Set<String> vExclude = new HashSet<String>(); /** Table of included categories. */ private Set<String> vInclude = new HashSet<String>(); /** Our standard logger. */ private static Log log = LogFactory.getLog(PatternClassifier.class); /** * Constructor for the pattern classifier. Doesn't do much by * itself. */ public PatternClassifier () { } /** * This simply sets the corresponding instance variable. It is * called by Spring each time an applicationContext is set up. * * @param newSourceFile the required file name. */ public final void setSourceFile(final String newSourceFile) { this.sourceFile = newSourceFile; } /** * Loads the set of rules from the classpath resource described in * the instance variable <code>sourceFile</code>. This was * originally done in <code>setSourceFile</code>; the two tasks * were separated to avoid reloading rules frequently during units * tests, when they were never being used. This is now called the * first time an attempt is made to use <code>vRules</code>. * * @exception Exception if the rule set is invalid. */ public final void setRules() throws Exception { if (log.isDebugEnabled()) { log.debug("Using source file: " + sourceFile); } InputStream in = this.getClass().getClassLoader(). getResourceAsStream(sourceFile); if (in == null) { throw new Exception("Can't open rule resource: " + sourceFile); } Reader theReader = new InputStreamReader(in); InputSource theInput = new InputSource(theReader); setRulesFromInput(theInput); } /** * Classifies a string and returns a comma-separated list of * matching categories. This is by far the most important part of * the classifier API. The method uses whatever rules or training * are provided to classify the text that is passed, and returns a * comma-separated list of matching categories. * * @param theText the text to be classified * @return a comma-separated list of categories */ public final Set<String> classifyString(final String theText) { if (vRules == null ) { vRules = new ArrayList<Rule>(); try { setRules(); } catch (Exception e) { log.warn("Could not load rules " + sourceFile); } } int theCount = vRules.size(); vInclude.clear(); vExclude.clear(); for (int theIndex = 0; theIndex < theCount; theIndex++) { Rule theRule = (Rule) vRules.get(theIndex); if (theRule.tryRule(theText)) { theRule.applyRule(); } } for (Iterator i = vExclude.iterator(); i.hasNext();) { vInclude.remove(i.next()); } Set<String> theResult = new HashSet<String>(); for (Iterator i = vInclude.iterator(); i.hasNext();) { theResult.add((String) i.next()); } return theResult; } /** * Clears the current rule data. */ public final void clearRules() { vRules.clear(); } /** * Adds a rule to the current rule set. This is called as the XML is * parsed to process the rules in the file. * * @param theRule the rule pattern text * @param theInclude a comma-separated list of categories to include * @param theExclude a comma-separated list of categories to exclude */ private void addRule(final String theRule, final String theInclude, final String theExclude) { vRules.add(new Rule(theRule, theInclude, theExclude)); } /** * Gets the text out of a DOM node. This is used in parsing the * rule set. * * @param theNode the DOM node * @return the text inside the node */ private static String getText(final Node theNode) { NodeList nl = theNode.getChildNodes(); Node ndCurrent; StringBuffer theResult = new StringBuffer(); for (int i = 0; i < nl.getLength(); i++) { ndCurrent = nl.item(i); if (ndCurrent.getNodeType() == Node.TEXT_NODE) { theResult.append(ndCurrent.getNodeValue()); } else if (ndCurrent.getNodeType() == Node.ELEMENT_NODE) { theResult.append(getText(ndCurrent)); } } return theResult.toString(); } /** * Sets the rule from a DOM element. * @param theElement the element * @throws Exception if there is a problem, such as parsing */ private void setRuleFromElement(final Element theElement) throws Exception { String theID = theElement.getAttribute("id"); if (theID == null) { throw new Exception("Invalid rule: no rule identifier set"); } NodeList matches = theElement.getElementsByTagName("match"); if (matches.getLength() != 1) { throw new Exception("Invalid rule: no pattern set"); } String thePattern = getText(matches.item(0)); matches = theElement.getElementsByTagName("categories"); if (matches.getLength() != 1) { throw new Exception("Invalid rule: no categories set"); } Element theCats = (Element) matches.item(0); String theInclude = theCats.getAttribute("include"); String theExclude = theCats.getAttribute("exclude"); addRule(thePattern, theInclude, theExclude); } /** * Sets the rule from the input in an InputStream. * @param theInput the input stream * @throws Exception if there is a problem, such as parsing */ private void setRulesFromInput(final InputSource theInput) throws Exception { Document document; DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); factory.setValidating(false); factory.setNamespaceAware(true); try { DocumentBuilder builder = factory.newDocumentBuilder(); document = builder.parse(theInput); NodeList all = document.getElementsByTagName("rule"); int theNodes = all.getLength(); for (int i = 0; i < theNodes; i++) { Element theElement = (Element) all.item(i); setRuleFromElement(theElement); } } catch (SAXParseException spe) { throw spe; } } /** * Class to implement a pattern rule. This is used within the * PatternClassifier; each rule consists of a regular expression * and a set of categories to include, and a second set to * exclude. The classifier runs through these in order, updating * current sets of categories. * * @author Stuart Watt */ private class Rule { /** The matcher pattern for the rule. */ private Matcher locPattern = null; /** A list of classes that are included. */ private String[] locInclude = null; /** A list of classes that are excluded. */ private String[] locExclude = null; /** * Constructs a rule, initialising the patterns accordingly. * @param pattern the rule regular expression pattern * @param include comma-separated categories to include * @param exclude comma-separated categories to exclude */ protected Rule (final String pattern, final String include, final String exclude) { Pattern thePattern = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE); locPattern = thePattern.matcher(""); locInclude = include.split(","); locExclude = exclude.split(","); } /** * Checks to see if a rule can be used. * @param theComment the comment to be used with the rule * @return true if the rule fires */ protected boolean tryRule (final String theComment) { locPattern.reset(theComment); return locPattern.find(); } /** * Applies a rule, updating the include and exclude lists. */ protected void applyRule() { if (locInclude != null) { for (int i = 0; i < locInclude.length; i++) { vInclude.add(locInclude[i]); } } if (locExclude != null) { for (int i = 0; i < locExclude.length; i++) { vExclude.add(locExclude[i]); } } } } }