/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.benchmark.quality;
import java.io.PrintWriter;
import java.text.NumberFormat;
import java.util.ArrayList;
/**
* Results of quality benchmark run for a single query or for a set of queries.
*/
public class QualityStats {
/** Number of points for which precision is computed. */
public static final int MAX_POINTS = 20;
private double maxGoodPoints;
private double recall;
private double pAt[];
private double pReleventSum = 0;
private double numPoints = 0;
private double numGoodPoints = 0;
private double mrr = 0;
private long searchTime;
private long docNamesExtractTime;
/**
* A certain rank in which a relevant doc was found.
*/
public static class RecallPoint {
private int rank;
private double recall;
private RecallPoint(int rank, double recall) {
this.rank = rank;
this.recall = recall;
}
/** Returns the rank: where on the list of returned docs this relevant doc appeared. */
public int getRank() {
return rank;
}
/** Returns the recall: how many relevant docs were returned up to this point, inclusive. */
public double getRecall() {
return recall;
}
}
private ArrayList<RecallPoint> recallPoints;
/**
* Construct a QualityStats object with anticipated maximal number of relevant hits.
* @param maxGoodPoints maximal possible relevant hits.
*/
public QualityStats(double maxGoodPoints, long searchTime) {
this.maxGoodPoints = maxGoodPoints;
this.searchTime = searchTime;
this.recallPoints = new ArrayList<RecallPoint>();
pAt = new double[MAX_POINTS+1]; // pAt[0] unused.
}
/**
* Add a (possibly relevant) doc.
* @param n rank of the added doc (its ordinal position within the query results).
* @param isRelevant true if the added doc is relevant, false otherwise.
*/
public void addResult(int n, boolean isRelevant, long docNameExtractTime) {
if (Math.abs(numPoints+1 - n) > 1E-6) {
throw new IllegalArgumentException("point "+n+" illegal after "+numPoints+" points!");
}
if (isRelevant) {
numGoodPoints+=1;
recallPoints.add(new RecallPoint(n,numGoodPoints));
if (recallPoints.size()==1 && n<=5) { // first point, but only within 5 top scores.
mrr = 1.0 / n;
}
}
numPoints = n;
double p = numGoodPoints / numPoints;
if (isRelevant) {
pReleventSum += p;
}
if (n<pAt.length) {
pAt[n] = p;
}
recall = maxGoodPoints<=0 ? p : numGoodPoints/maxGoodPoints;
docNamesExtractTime += docNameExtractTime;
}
/**
* Return the precision at rank n:
* |{relevant hits within first <code>n</code> hits}| / <code>n</code>.
* @param n requested precision point, must be at least 1 and at most {@link #MAX_POINTS}.
*/
public double getPrecisionAt(int n) {
if (n<1 || n>MAX_POINTS) {
throw new IllegalArgumentException("n="+n+" - but it must be in [1,"+MAX_POINTS+"] range!");
}
if (n>numPoints) {
return (numPoints * pAt[(int)numPoints])/n;
}
return pAt[n];
}
/**
* Return the average precision at recall points.
*/
public double getAvp() {
return maxGoodPoints==0 ? 0 : pReleventSum/maxGoodPoints;
}
/**
* Return the recall: |{relevant hits found}| / |{relevant hits existing}|.
*/
public double getRecall() {
return recall;
}
/**
* Log information on this QualityStats object.
* @param logger Logger.
* @param prefix prefix before each log line.
*/
public void log(String title, int paddLines, PrintWriter logger, String prefix) {
for (int i=0; i<paddLines; i++) {
logger.println();
}
if (title!=null && title.trim().length()>0) {
logger.println(title);
}
prefix = prefix==null ? "" : prefix;
NumberFormat nf = NumberFormat.getInstance();
nf.setMaximumFractionDigits(3);
nf.setMinimumFractionDigits(3);
nf.setGroupingUsed(true);
int M = 19;
logger.println(prefix+format("Search Seconds: ",M)+
fracFormat(nf.format((double)searchTime/1000)));
logger.println(prefix+format("DocName Seconds: ",M)+
fracFormat(nf.format((double)docNamesExtractTime/1000)));
logger.println(prefix+format("Num Points: ",M)+
fracFormat(nf.format(numPoints)));
logger.println(prefix+format("Num Good Points: ",M)+
fracFormat(nf.format(numGoodPoints)));
logger.println(prefix+format("Max Good Points: ",M)+
fracFormat(nf.format(maxGoodPoints)));
logger.println(prefix+format("Average Precision: ",M)+
fracFormat(nf.format(getAvp())));
logger.println(prefix+format("MRR: ",M)+
fracFormat(nf.format(getMRR())));
logger.println(prefix+format("Recall: ",M)+
fracFormat(nf.format(getRecall())));
for (int i=1; i<(int)numPoints && i<pAt.length; i++) {
logger.println(prefix+format("Precision At "+i+": ",M)+
fracFormat(nf.format(getPrecisionAt(i))));
}
for (int i=0; i<paddLines; i++) {
logger.println();
}
}
private static String padd = " ";
private String format(String s, int minLen) {
s = (s==null ? "" : s);
int n = Math.max(minLen,s.length());
return (s+padd).substring(0,n);
}
private String fracFormat(String frac) {
int k = frac.indexOf('.');
String s1 = padd+frac.substring(0,k);
int n = Math.max(k,6);
s1 = s1.substring(s1.length()-n);
return s1 + frac.substring(k);
}
/**
* Create a QualityStats object that is the average of the input QualityStats objects.
* @param stats array of input stats to be averaged.
* @return an average over the input stats.
*/
public static QualityStats average(QualityStats[] stats) {
QualityStats avg = new QualityStats(0,0);
if (stats.length==0) {
// weired, no stats to average!
return avg;
}
int m = 0; // queries with positive judgements
// aggregate
for (int i=0; i<stats.length; i++) {
avg.searchTime += stats[i].searchTime;
avg.docNamesExtractTime += stats[i].docNamesExtractTime;
if (stats[i].maxGoodPoints>0) {
m++;
avg.numGoodPoints += stats[i].numGoodPoints;
avg.numPoints += stats[i].numPoints;
avg.pReleventSum += stats[i].getAvp();
avg.recall += stats[i].recall;
avg.mrr += stats[i].getMRR();
avg.maxGoodPoints += stats[i].maxGoodPoints;
for (int j=1; j<avg.pAt.length; j++) {
avg.pAt[j] += stats[i].getPrecisionAt(j);
}
}
}
assert m>0 : "Fishy: no \"good\" queries!";
// take average: times go by all queries, other measures go by "good" queries only.
avg.searchTime /= stats.length;
avg.docNamesExtractTime /= stats.length;
avg.numGoodPoints /= m;
avg.numPoints /= m;
avg.recall /= m;
avg.mrr /= m;
avg.maxGoodPoints /= m;
for (int j=1; j<avg.pAt.length; j++) {
avg.pAt[j] /= m;
}
avg.pReleventSum /= m; // this is actually avgp now
avg.pReleventSum *= avg.maxGoodPoints; // so that getAvgP() would be correct
return avg;
}
/**
* Returns the time it took to extract doc names for judging the measured query, in milliseconds.
*/
public long getDocNamesExtractTime() {
return docNamesExtractTime;
}
/**
* Returns the maximal number of good points.
* This is the number of relevant docs known by the judge for the measured query.
*/
public double getMaxGoodPoints() {
return maxGoodPoints;
}
/**
* Returns the number of good points (only relevant points).
*/
public double getNumGoodPoints() {
return numGoodPoints;
}
/**
* Returns the number of points (both relevant and irrelevant points).
*/
public double getNumPoints() {
return numPoints;
}
/**
* Returns the recallPoints.
*/
public RecallPoint [] getRecallPoints() {
return recallPoints.toArray(new RecallPoint[0]);
}
/**
* Returns the Mean reciprocal rank over the queries or RR for a single query.
* <p>
* Reciprocal rank is defined as <code>1/r</code> where <code>r</code> is the
* rank of the first correct result, or <code>0</code> if there are no correct
* results within the top 5 results.
* <p>
* This follows the definition in
* <a href="http://www.cnlp.org/publications/02cnlptrec10.pdf">
* Question Answering - CNLP at the TREC-10 Question Answering Track</a>.
*/
public double getMRR() {
return mrr;
}
/**
* Returns the search time in milliseconds for the measured query.
*/
public long getSearchTime() {
return searchTime;
}
}