LibSvmEntityFilter.java

/**
 *  Copyright 2014 Marco Cornolti
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

package it.acubelab.smaph.entityfilters;

import it.acubelab.smaph.SmaphAnnotatorDebugger;
import it.acubelab.smaph.learn.LibSvmFilter;

import java.io.IOException;
import java.util.HashMap;
import java.util.Vector;

import org.apache.commons.lang.ArrayUtils;

/**
 * An SVM-based entity filter.
 */
public class LibSvmEntityFilter extends LibSvmFilter implements EntityFilter {

	public static String[] ftrNames = new String[] {
			"is_s1", // 1
			"is_s2",
			"is_s3",
			"is_s4",
			"is_s5",
			"s1_freq",
			"s1_rhoScore", //
			"s1_localCoherence", //
			"s1_lp",
			"s1_editDistance", // 10
			"s1_commonness", //
			"s1_avgRank",
			"s1_ambiguity",
			"s1_pageRank", //
			"s2_editDistanceTitle",
			"s2_rank",
			"s2_wikiWebTotal",
			"s2_webTotal",
			"s3_rank",
			"s3_wikiWebTotal", // 20
			"s3_editDistanceTitle",
			"s3_editDistanceNoPar",
			"s3_editDistanceBolds",
			"s3_capitalizedBolds",
			"s3_avgBoldsWords",
			"s5_rank",
			"s5_wikiWebTotal",
			"s5_editDistanceTitle",
			"s5_editDistanceNoPar",
			"s5_editDistanceBolds", // 30
			"s5_capitalizedBolds",
			"s5_avgBoldsWords",
			"s3_webTotal",
			"s2_editDistanceNoPar",
			"s2_editDistanceBolds",
			"s2_capitalizedBolds",
			"s2_avgBoldsWords", // 37
	};

	public LibSvmEntityFilter(String modelFileBase) throws IOException {
		super(modelFileBase + ".model", modelFileBase + ".range");
	}

	@Override
	public boolean filterEntity(HashMap<String, Double> features) {
		boolean result = predict(features);
		String ftrDesc = "";
		for (String key : features.keySet())
			ftrDesc += String.format("%s:%.3f ", key, features.get(key));
		SmaphAnnotatorDebugger.out.printf("EF: %s has been %s.%n", ftrDesc,
				result ? "accepted" : "discarded");
		return result;
	}

	private static double getOrDefault(HashMap<String, Double> features,
			String key, double defaultVal) {
		Double res = features.get(key);
		if (res == null)
			return defaultVal;
		return res;
	}

	/**
	 * Turns a frature_name-feature_value mapping to an array of features.
	 * 
	 * @param features
	 *            the mapping from feature names to feature values.
	 * @return an array of feature values.
	 */
	public static double[] featuresToFtrVectStatic(
			HashMap<String, Double> features) {

		if (!checkFeatures(features)) {
			for (String ftrName : features.keySet())
				System.err.printf("%s -> %f%n", ftrName, features.get(ftrName));
			throw new RuntimeException(
					"Implementation error -- check the features");
		}

		Vector<Double> ftrValues = new Vector<>();
		for (String ftrName : ftrNames)
			ftrValues.add(getOrDefault(features, ftrName, 0.0));
		
		return ArrayUtils.toPrimitive(ftrValues.toArray(new Double[] {}));
	}

	private static boolean checkFeatures(HashMap<String, Double> features) {
		if (getOrDefault(features, "is_s1", 0.0)
				+ getOrDefault(features, "is_s2", 0.0)
				+ getOrDefault(features, "is_s3", 0.0)
				+ getOrDefault(features, "is_s4", 0.0)
				+ getOrDefault(features, "is_s5", 0.0) != 1)
			return false;
		boolean found = false;
		for (String sourcePrefix : new String[] { "s1_", "s2_", "s3_", "s5_" }) {
			int sourceFtrCount = 0;

			for (String ftrName : features.keySet())
				if (ftrName.startsWith(sourcePrefix))
					sourceFtrCount++;

			if (sourcePrefix.equals("s1_"))
				found = sourceFtrCount == 9
						&& features.size() == sourceFtrCount + 1;
			if (sourcePrefix.equals("s2_"))
				found = sourceFtrCount == 8
						&& features.size() == sourceFtrCount + 1;
			if (sourcePrefix.equals("s3_"))
				found = sourceFtrCount == 8
						&& features.size() == sourceFtrCount + 1;
			if (sourcePrefix.equals("s4_"))
				found = sourceFtrCount == 0
						&& features.size() == sourceFtrCount + 1;
			if (sourcePrefix.equals("s5_"))
				found = sourceFtrCount == 8
						&& features.size() == sourceFtrCount + 1;

			if (found)
				return true;
		}
		return false;
	}

	@Override
	public double[] featuresToFtrVect(HashMap<String, Double> features) {
		return featuresToFtrVectStatic(features);
	}
}