package org.wikibrain.sr.evaluation; import gnu.trove.map.TIntDoubleMap; import gnu.trove.map.hash.TIntDoubleHashMap; import org.apache.commons.lang3.ArrayUtils; import org.wikibrain.sr.SRResult; import org.wikibrain.sr.SRResultList; import org.wikibrain.sr.utils.KnownSim; import java.util.ArrayList; import java.util.Arrays; import java.util.List; /** * A most generated similar list that can be evaluated against a known human-labeled gold-standard. * @author Shilad Sen */ public class MostSimilarGuess { private final KnownMostSim known; private final int length; // length of most similar guess list private final double minScore; private final double maxScore; private final List<Observation> observations = new ArrayList<Observation>(); public MostSimilarGuess(KnownMostSim known, String str) { this.known = known; TIntDoubleMap actual = new TIntDoubleHashMap(); for (KnownSim ks : known.getMostSimilar()) { actual.put(ks.wpId2, ks.similarity); } String tokens[] = str.split("[|]"); length = Integer.valueOf(tokens[0]); minScore = Double.valueOf(tokens[1]); maxScore = Double.valueOf(tokens[2]); for (int i = 3; i < tokens.length; i++) { String tuple[] = tokens[i].split("[@]"); int id = Integer.valueOf(tuple[1]); observations.add(new Observation( Integer.valueOf(tuple[0]), id, Double.valueOf(tuple[2]), actual.get(id))); } } public MostSimilarGuess(KnownMostSim known, SRResultList guess) { this.known = known; length = guess.numDocs(); minScore = guess.minScore(); maxScore = guess.maxScore(); TIntDoubleMap actual = new TIntDoubleHashMap(); for (KnownSim ks : known.getMostSimilar()) { actual.put(ks.wpId2, ks.similarity); } for (int i = 0; i < guess.numDocs(); i++) { SRResult sr = guess.get(i); if (actual.containsKey(sr.getId())) { observations.add(new Observation(i+1, sr.getId(), sr.getScore(), actual.get(sr.getId()))); } } } public String toString() { StringBuilder sb = new StringBuilder(); sb.append(length); sb.append("|"); sb.append(minScore); sb.append("|"); sb.append(maxScore); for (Observation observation : observations) { sb.append("|") .append(observation.rank) .append("@") .append(observation.id) .append("@") .append(observation.estimate); } return sb.toString(); } public List<Observation> getObservations() { return observations; } public int getLength() { return length; } public KnownMostSim getKnown() { return known; } /** * @see <a href="https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Normalized_DCG">Wikipedia</a> * @return The normalized discounted cummulative gain. Results with no known * entry are totally ignored. */ public double getNDGC() { if (observations.isEmpty()) { return 0.0; } TIntDoubleMap actual = new TIntDoubleHashMap(); for (KnownSim ks : known.getMostSimilar()) { actual.put(ks.wpId2, ks.similarity); } int ranks[] = new int[observations.size()]; double scores[] = new double[observations.size()]; double s = 0.0; for (int i = 0; i < observations.size(); i++) { Observation o = observations.get(i); double k = (o.rank == 1) ? 1 : Math.log(o.rank + 1); s += actual.get(o.id) / k; scores[i] = actual.get(o.id); ranks[i] = o.rank; } Arrays.sort(ranks); Arrays.sort(scores); ArrayUtils.reverse(scores); double t = 0; for (int i = 0; i < scores.length; i++) { double k = (ranks[i] == 1) ? 1 : Math.log(ranks[i] + 1); t += scores[i] / k; } return s / t; } /** * @see <a href="https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Normalized_DCG">Wikipedia</a> * @return The normalized discounted cummulative gain, but assumes unobserved * KnownSim entries lie somewhere below the observed list. */ public double getPenalizedNDGC() { if (observations.isEmpty()) { return 0.0; } TIntDoubleMap actual = new TIntDoubleHashMap(); for (KnownSim ks : known.getMostSimilar()) { actual.put(ks.wpId2, ks.similarity); } double s = 0.0; int ranks[] = new int[observations.size()]; for (int i = 0; i < observations.size(); i++) { Observation o = observations.get(i); double k = (o.rank == 1) ? 1 : Math.log(o.rank + 1); s += actual.get(o.id) / k; ranks[i] = o.rank; } Arrays.sort(ranks); // there are known.size() - observations.size() unobserved items. // calculate the expected rank and score of them // unobserved rank is 3 * length of the list (totally random!) // unobserved similarity is the mean between the minimum score and 0 int unobservedCount = (known.getMostSimilar().size() - observations.size()); int unobservedRank = 3 * length; double unobservedScore = minScore / 2; s += unobservedCount * unobservedScore / Math.log(unobservedRank + 1); // Calculate maximum over ALL entries - not just observed ones. double t = 0; for (int i = 0; i < known.getMostSimilar().size(); i++) { double k; if (i < ranks.length) { k = (ranks[i] == 1) ? 1 : Math.log(ranks[i] + 1); } else { k = Math.log(unobservedRank + 1); } t += known.getMostSimilar().get(i).similarity / k; } return s / t; } public PrecisionRecallAccumulator getPrecisionRecall(int n, double threshold) { PrecisionRecallAccumulator pr = new PrecisionRecallAccumulator(n, threshold); TIntDoubleMap actual = new TIntDoubleHashMap(); for (KnownSim ks : known.getMostSimilar()) { pr.observe(ks.similarity); actual.put(ks.wpId2, ks.similarity); } for (Observation o : observations) { if (o.rank > n) { break; } pr.observeRetrieved(actual.get(o.id)); } return pr; } /** * Represents an entry in the most similar result list with known similarity. */ public static class Observation { public int rank; // rank in result list (range is [1...length of list]) public int id; // wp local page id public double estimate; // estimated relatedness public double actual; // actual human score public Observation(int rank, int id, double estimate, double actual) { this.rank = rank; this.id = id; this.estimate = estimate; this.actual = actual; } @Override public String toString() { return "Observation{" + "rank=" + rank + ", id=" + id + ", estimate=" + estimate + ", actual=" + actual + '}'; } } }