Robust04_WSD.java example

Explorer
Ivory-master
- src
  - java
/*
 * Ivory: A Hadoop toolkit for web-scale information retrieval research
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License. You may
 * obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

package ivory.regression.basic;

import static ivory.regression.RegressionUtils.loadScoresIntoMap;
import static org.junit.Assert.assertEquals;
import ivory.core.eval.Qrels;
import ivory.core.eval.RankedListEvaluator;
import ivory.smrf.retrieval.Accumulator;
import ivory.smrf.retrieval.BatchQueryRunner;

import java.util.Map;

import junit.framework.JUnit4TestAdapter;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.log4j.Logger;
import org.junit.Test;

import com.google.common.collect.Maps;

import edu.umd.cloud9.collection.DocnoMapping;

public class Robust04_WSD {
  private static final Logger LOG = Logger.getLogger(Robust04_WSD.class);

  private static String[] sDir_WSD_SD_RawAP = new String[] {
    "601", "0.4951", "602", "0.3076", "603", "0.2561", "604", "0.8518", "605", "0.0680",
    "606", "0.5995", "607", "0.2564", "608", "0.0766", "609", "0.3063", "610", "0.0294",
    "611", "0.2613", "612", "0.4517", "613", "0.2492", "614", "0.2043", "615", "0.0634",
    "616", "0.7377", "617", "0.2553", "618", "0.2091", "619", "0.5563", "620", "0.0736",
    "621", "0.3446", "622", "0.1210", "623", "0.2954", "624", "0.2713", "625", "0.0251",
    "626", "0.2027", "627", "0.0213", "628", "0.2213", "629", "0.2001", "630", "0.7169",
    "631", "0.1831", "632", "0.2441", "633", "0.4995", "634", "0.7412", "635", "0.5952",
    "636", "0.1971", "637", "0.5269", "638", "0.0624", "639", "0.1589", "640", "0.3741",
    "641", "0.3269", "642", "0.1632", "643", "0.5512", "644", "0.2277", "645", "0.5907",
    "646", "0.3490", "647", "0.2457", "648", "0.3688", "649", "0.6840", "650", "0.0845",
    "651", "0.0491", "652", "0.3279", "653", "0.5957", "654", "0.4214", "655", "0.0011",
    "656", "0.5310", "657", "0.4787", "658", "0.1321", "659", "0.3972", "660", "0.6448",
    "661", "0.6152", "662", "0.6510", "663", "0.4894", "664", "0.5061", "665", "0.1623",
    "666", "0.0186", "667", "0.3714", "668", "0.3651", "669", "0.1597", "670", "0.1270",
    "671", "0.4059", "672", "0.0000", "673", "0.2950", "674", "0.1371", "675", "0.3248",
    "676", "0.2861", "677", "0.8840", "678", "0.2190", "679", "0.8556", "680", "0.0926",
    "681", "0.5499", "682", "0.2641", "683", "0.0242", "684", "0.1308", "685", "0.2044",
    "686", "0.2071", "687", "0.2285", "688", "0.1157", "689", "0.0149", "690", "0.0080",
    "691", "0.3478", "692", "0.4843", "693", "0.3228", "694", "0.4819", "695", "0.2510",
    "696", "0.2824", "697", "0.1709", "698", "0.4457", "699", "0.5590", "700", "0.5904" };

  private static String[] sDir_WSD_SD_RawP10 = new String[] {
    "601", "0.3000", "602", "0.3000", "603", "0.2000", "604", "0.6000", "605", "0.2000",
    "606", "0.5000", "607", "0.3000", "608", "0.1000", "609", "0.6000", "610", "0.0000",
    "611", "0.5000", "612", "0.6000", "613", "0.5000", "614", "0.2000", "615", "0.0000",
    "616", "0.9000", "617", "0.5000", "618", "0.3000", "619", "0.6000", "620", "0.1000",
    "621", "0.8000", "622", "0.2000", "623", "0.6000", "624", "0.4000", "625", "0.1000",
    "626", "0.3000", "627", "0.1000", "628", "0.4000", "629", "0.4000", "630", "0.3000",
    "631", "0.1000", "632", "0.7000", "633", "1.0000", "634", "0.8000", "635", "0.8000",
    "636", "0.3000", "637", "0.8000", "638", "0.2000", "639", "0.3000", "640", "0.6000",
    "641", "0.6000", "642", "0.2000", "643", "0.4000", "644", "0.2000", "645", "0.9000",
    "646", "0.4000", "647", "0.7000", "648", "0.7000", "649", "1.0000", "650", "0.2000",
    "651", "0.1000", "652", "0.7000", "653", "0.6000", "654", "1.0000", "655", "0.0000",
    "656", "0.7000", "657", "0.6000", "658", "0.4000", "659", "0.5000", "660", "0.9000",
    "661", "0.9000", "662", "0.8000", "663", "0.6000", "664", "0.6000", "665", "0.3000",
    "666", "0.0000", "667", "0.8000", "668", "0.5000", "669", "0.1000", "670", "0.3000",
    "671", "0.5000", "672", "0.0000", "673", "0.6000", "674", "0.1000", "675", "0.6000",
    "676", "0.2000", "677", "0.7000", "678", "0.4000", "679", "0.6000", "680", "0.3000",
    "681", "0.8000", "682", "0.6000", "683", "0.4000", "684", "0.3000", "685", "0.2000",
    "686", "0.3000", "687", "0.8000", "688", "0.5000", "689", "0.0000", "690", "0.0000",
    "691", "0.5000", "692", "0.7000", "693", "0.6000", "694", "0.5000", "695", "0.8000",
    "696", "0.6000", "697", "0.3000", "698", "0.3000", "699", "0.7000", "700", "0.7000" };

  private static String[] sDir_WSD_FD_RawAP = new String[] {
    "601", "0.6823", "602", "0.3070", "603", "0.2825", "604", "0.8538", "605", "0.0631",
    "606", "0.6033", "607", "0.2293", "608", "0.1029", "609", "0.3282", "610", "0.0202",
    "611", "0.2402", "612", "0.4517", "613", "0.2521", "614", "0.2287", "615", "0.0512",
    "616", "0.7377", "617", "0.2343", "618", "0.1954", "619", "0.5567", "620", "0.0805",
    "621", "0.4508", "622", "0.1210", "623", "0.2548", "624", "0.2855", "625", "0.0254",
    "626", "0.2027", "627", "0.0154", "628", "0.2213", "629", "0.1868", "630", "0.7885",
    "631", "0.2676", "632", "0.2205", "633", "0.4995", "634", "0.7412", "635", "0.5711",
    "636", "0.1890", "637", "0.5654", "638", "0.0624", "639", "0.1397", "640", "0.3584",
    "641", "0.3197", "642", "0.1633", "643", "0.3963", "644", "0.2612", "645", "0.5907",
    "646", "0.3579", "647", "0.2457", "648", "0.2937", "649", "0.6840", "650", "0.0795",
    "651", "0.0491", "652", "0.3278", "653", "0.5802", "654", "0.4214", "655", "0.0012",
    "656", "0.5421", "657", "0.4605", "658", "0.1321", "659", "0.4778", "660", "0.6404",
    "661", "0.6217", "662", "0.6510", "663", "0.4885", "664", "0.6499", "665", "0.1482",
    "666", "0.0168", "667", "0.3751", "668", "0.3651", "669", "0.1597", "670", "0.1270",
    "671", "0.3937", "672", "0.0000", "673", "0.2946", "674", "0.1371", "675", "0.2878",
    "676", "0.2861", "677", "0.8803", "678", "0.1951", "679", "0.8000", "680", "0.1028",
    "681", "0.5690", "682", "0.2597", "683", "0.0242", "684", "0.1119", "685", "0.2136",
    "686", "0.2376", "687", "0.3293", "688", "0.1060", "689", "0.0149", "690", "0.0064",
    "691", "0.3333", "692", "0.4887", "693", "0.3518", "694", "0.4819", "695", "0.2052",
    "696", "0.2983", "697", "0.1811", "698", "0.4651", "699", "0.5590", "700", "0.5904" };

  private static String[] sDir_WSD_FD_RawP10 = new String[] {
    "601", "0.4000", "602", "0.2000", "603", "0.3000", "604", "0.6000", "605", "0.3000",
    "606", "0.5000", "607", "0.4000", "608", "0.1000", "609", "0.7000", "610", "0.0000",
    "611", "0.4000", "612", "0.6000", "613", "0.5000", "614", "0.3000", "615", "0.0000",
    "616", "0.9000", "617", "0.5000", "618", "0.3000", "619", "0.7000", "620", "0.1000",
    "621", "0.8000", "622", "0.2000", "623", "0.5000", "624", "0.5000", "625", "0.1000",
    "626", "0.3000", "627", "0.1000", "628", "0.4000", "629", "0.4000", "630", "0.3000",
    "631", "0.8000", "632", "0.7000", "633", "1.0000", "634", "0.8000", "635", "0.7000",
    "636", "0.3000", "637", "0.7000", "638", "0.2000", "639", "0.2000", "640", "0.6000",
    "641", "0.7000", "642", "0.3000", "643", "0.4000", "644", "0.3000", "645", "0.9000",
    "646", "0.4000", "647", "0.7000", "648", "0.6000", "649", "1.0000", "650", "0.0000",
    "651", "0.1000", "652", "0.7000", "653", "0.6000", "654", "1.0000", "655", "0.0000",
    "656", "0.7000", "657", "0.6000", "658", "0.4000", "659", "0.5000", "660", "0.9000",
    "661", "0.8000", "662", "0.8000", "663", "0.6000", "664", "0.6000", "665", "0.3000",
    "666", "0.0000", "667", "0.8000", "668", "0.5000", "669", "0.1000", "670", "0.3000",
    "671", "0.4000", "672", "0.0000", "673", "0.5000", "674", "0.1000", "675", "0.5000",
    "676", "0.2000", "677", "0.7000", "678", "0.3000", "679", "0.6000", "680", "0.4000",
    "681", "0.8000", "682", "0.5000", "683", "0.4000", "684", "0.3000", "685", "0.3000",
    "686", "0.3000", "687", "0.7000", "688", "0.5000", "689", "0.0000", "690", "0.0000",
    "691", "0.5000", "692", "0.7000", "693", "0.6000", "694", "0.5000", "695", "0.6000",
    "696", "0.7000", "697", "0.3000", "698", "0.4000", "699", "0.7000", "700", "0.7000" };

  private static Qrels sQrels;
  private static DocnoMapping sMapping;

  @Test
  public void runRegression() throws Exception {
    Map<String, Map<String, Float>> AllModelsAPScores = Maps.newHashMap();
    AllModelsAPScores.put("robust04-dir-wsd-sd", loadScoresIntoMap(sDir_WSD_SD_RawAP));
    AllModelsAPScores.put("robust04-dir-wsd-fd", loadScoresIntoMap(sDir_WSD_FD_RawAP));

    Map<String, Map<String, Float>> AllModelsP10Scores = Maps.newHashMap();
    AllModelsP10Scores.put("robust04-dir-wsd-sd", loadScoresIntoMap(sDir_WSD_SD_RawP10));
    AllModelsP10Scores.put("robust04-dir-wsd-fd", loadScoresIntoMap(sDir_WSD_FD_RawP10));

    sQrels = new Qrels("data/trec/qrels.robust04.noCRFR.txt");

    String[] params = new String[] {
            "data/trec/run.robust04.wsd.xml",
            "data/trec/queries.robust04.xml" };

    FileSystem fs = FileSystem.getLocal(new Configuration());

    BatchQueryRunner qr = new BatchQueryRunner(params, fs);

    long start = System.currentTimeMillis();
    qr.runQueries();
    long end = System.currentTimeMillis();

    LOG.info("Total query time: " + (end - start) + "ms");

    sMapping = qr.getDocnoMapping();

    for (String model : qr.getModels()) {
      LOG.info("Verifying results of model \"" + model + "\"");
      Map<String, Accumulator[]> results = qr.getResults(model);
      verifyResults(model, results, AllModelsAPScores.get(model),
          AllModelsP10Scores.get(model));
      LOG.info("Done!");
    }

  }

  private static void verifyResults(String model, Map<String, Accumulator[]> results,
      Map<String, Float> apScores, Map<String, Float> p10Scores) {
    float apSum = 0, p10Sum = 0;
    for (String qid : results.keySet()) {
      float ap = (float) RankedListEvaluator.computeAP(results.get(qid), sMapping,
          sQrels.getReldocsForQid(qid));

      float p10 = (float) RankedListEvaluator.computePN(10, results.get(qid), sMapping,
          sQrels.getReldocsForQid(qid));

      apSum += ap;
      p10Sum += p10;

      LOG.info("verifying qid " + qid + " for model " + model);
      assertEquals(apScores.get(qid), ap, 10e-5);
      assertEquals(p10Scores.get(qid), p10, 10e-5);
    }

    // one topic didn't contain qrels, so trec_eval only picked up 99 topics
    float MAP = (float) RankedListEvaluator.roundTo4SigFigs(apSum / 99f);
    float P10Avg = (float) RankedListEvaluator.roundTo4SigFigs(p10Sum / 99f);

    if (model.equals("robust04-dir-wsd-sd")) {
      assertEquals(0.3246, MAP, 10e-5);
      assertEquals(0.4626, P10Avg, 10e-5);
    } else if (model.equals("robust04-dir-wsd-fd")) {
      assertEquals(0.3283, MAP, 10e-5);
      assertEquals(0.4667, P10Avg, 10e-5);
    }
  }

  public static junit.framework.Test suite() {
    return new JUnit4TestAdapter(Robust04_WSD.class);
  }
}