1
2
3 package org.galagosearch.core.eval;
4
5 import java.util.Collection;
6 import java.util.Map;
7 import java.util.TreeMap;
8
9 /***
10 * Computes summary statistics over a set of queries.
11 *
12 * @author Trevor Strohman
13 */
14 public class SetRetrievalEvaluator {
15 Collection<RetrievalEvaluator> _evaluators;
16
17 /*** Creates a new instance of SetRetrievalEvaluator */
18 public SetRetrievalEvaluator(Collection<RetrievalEvaluator> evaluators) {
19 _evaluators = evaluators;
20 }
21
22 /***
23 * Returns a collection of evaluators.
24 */
25 public Collection<RetrievalEvaluator> getEvaluators() {
26 return _evaluators;
27 }
28
29 /***
30 * Returns the mean average precision; the mean of the average
31 * precision values for all queries.
32 */
33 public double meanAveragePrecision() {
34 double sumAveragePrecision = 0;
35
36 for (RetrievalEvaluator evaluator : _evaluators) {
37 sumAveragePrecision += evaluator.averagePrecision();
38 }
39
40 return sumAveragePrecision / (double) _evaluators.size();
41 }
42
43 /***
44 * Returns the mean of the binary preference values
45 * for all queries.
46 */
47 public double meanBinaryPreference() {
48 double sumBinaryPreference = 0;
49
50 for (RetrievalEvaluator evaluator : _evaluators) {
51 sumBinaryPreference += evaluator.binaryPreference();
52 }
53
54 return sumBinaryPreference / (double) _evaluators.size();
55 }
56
57 /***
58 * Returns the geometric mean of average precision values
59 * for all queries.
60 */
61 public double geometricMeanAveragePrecision() {
62 double productAveragePrecision = 0;
63
64 for (RetrievalEvaluator evaluator : _evaluators) {
65 productAveragePrecision *= evaluator.averagePrecision();
66 }
67
68 return Math.pow(productAveragePrecision, 1.0 / _evaluators.size());
69 }
70
71 /***
72 * Returns the mean of the precision values
73 * for all queries.
74 */
75 public double meanPrecision(int documentsRetrieved) {
76 double sumPrecision = 0;
77
78 for (RetrievalEvaluator evaluator : _evaluators) {
79 sumPrecision += evaluator.precision(documentsRetrieved);
80 }
81
82 return sumPrecision / _evaluators.size();
83 }
84
85 /***
86 * Returns the mean of the reciprocal rank values for all queries.
87 */
88 public double meanReciprocalRank() {
89 double sumRR = 0;
90
91 for (RetrievalEvaluator evaluator : _evaluators) {
92 sumRR += evaluator.reciprocalRank();
93 }
94
95 return sumRR / _evaluators.size();
96 }
97
98 /***
99 * Returns the mean of the R-precision values for all queries.
100 */
101 public double meanRPrecision() {
102 double sumRPrecision = 0;
103
104 for (RetrievalEvaluator evaluator : _evaluators) {
105 sumRPrecision += evaluator.rPrecision();
106 }
107
108 return sumRPrecision / _evaluators.size();
109 }
110
111 /***
112 * Returns the mean of the NDCG values for all queries.
113 */
114 public double meanNormalizedDiscountedCumulativeGain() {
115 double sumNDCG = 0;
116
117 for (RetrievalEvaluator evaluator : _evaluators) {
118 sumNDCG += evaluator.normalizedDiscountedCumulativeGain();
119 }
120
121 return sumNDCG / _evaluators.size();
122 }
123
124 /***
125 * Returns the mean of the NDCG values for all queries.
126 */
127 public double meanNormalizedDiscountedCumulativeGain(int documentsRetrieved) {
128 double sumNDCG = 0;
129
130 for (RetrievalEvaluator evaluator : _evaluators) {
131 sumNDCG += evaluator.normalizedDiscountedCumulativeGain(documentsRetrieved);
132 }
133
134 return sumNDCG / _evaluators.size();
135 }
136
137 /***
138 * Returns a Map containing a particular metric value for each query.
139 * For instance, if metric == "averagePrecision", this returns
140 * a map where the keys are query identifiers and the values are the
141 * average precision metric evaluated for each query.
142 */
143 public Map<String, Double> evaluateAll(String metric) throws IllegalArgumentException {
144 TreeMap<String, Double> result = new TreeMap<String, Double>();
145
146 for (RetrievalEvaluator evaluator : _evaluators) {
147 double value = 0;
148
149 if (metric.equals("averagePrecision") || metric.equals("ap") || metric.equals("map")) {
150 value = evaluator.averagePrecision();
151 } else if (metric.equals("ndcg")) {
152 value = evaluator.normalizedDiscountedCumulativeGain();
153 } else if (metric.startsWith("ndcg")) {
154 value = evaluator.normalizedDiscountedCumulativeGain(Integer.parseInt(metric.
155 substring(4)));
156 } else if (metric.equals("reciprocalRank") || metric.equals("mrr")) {
157 value = evaluator.reciprocalRank();
158 } else if (metric.equals("rPrecision")) {
159 value = evaluator.rPrecision();
160 } else if (metric.equals("bpref")) {
161 value = evaluator.binaryPreference();
162 } else if (metric.startsWith("P")) {
163 value = evaluator.precision(Integer.parseInt(metric.substring(1)));
164 } else if (metric.startsWith("R")) {
165 value = evaluator.recall(Integer.parseInt(metric.substring(1)));
166 } else {
167 throw new IllegalArgumentException("Unknown metric: " + metric);
168 }
169
170 result.put(evaluator.queryName(), value);
171 }
172
173 return result;
174 }
175
176 /***
177 * The number of documents retrieved for all queries.
178 */
179 public int numberRetrieved() {
180 int sumRetrieved = 0;
181
182 for (RetrievalEvaluator evaluator : _evaluators) {
183 sumRetrieved += evaluator.retrievedDocuments().size();
184 }
185
186 return sumRetrieved;
187 }
188
189 /***
190 * The total number of relevant documents to any of the queries.
191 */
192 public int numberRelevant() {
193 int sumRelevant = 0;
194
195 for (RetrievalEvaluator evaluator : _evaluators) {
196 sumRelevant += evaluator.relevantDocuments().size();
197 }
198
199 return sumRelevant;
200 }
201
202 /***
203 * The total number of relevant documents retrieved for any of the queries.
204 */
205 public int numberRelevantRetrieved() {
206 int sumRelevantRetrieved = 0;
207
208 for (RetrievalEvaluator evaluator : _evaluators) {
209 sumRelevantRetrieved += evaluator.relevantRetrievedDocuments().size();
210 }
211
212 return sumRelevantRetrieved;
213 }
214 }