| Classes in this File | Line Coverage | Branch Coverage | Complexity | ||||
| SetRetrievalEvaluator |
|
| 0.0;0 |
| 1 | // BSD License (http://www.galagosearch.org/license) | |
| 2 | ||
| 3 | package org.galagosearch.core.eval; | |
| 4 | ||
| 5 | import java.util.Collection; | |
| 6 | import java.util.Map; | |
| 7 | import java.util.TreeMap; | |
| 8 | ||
| 9 | /** | |
| 10 | * Computes summary statistics over a set of queries. | |
| 11 | * | |
| 12 | * @author Trevor Strohman | |
| 13 | */ | |
| 14 | public class SetRetrievalEvaluator { | |
| 15 | Collection<RetrievalEvaluator> _evaluators; | |
| 16 | ||
| 17 | /** Creates a new instance of SetRetrievalEvaluator */ | |
| 18 | 0 | public SetRetrievalEvaluator(Collection<RetrievalEvaluator> evaluators) { |
| 19 | 0 | _evaluators = evaluators; |
| 20 | 0 | } |
| 21 | ||
| 22 | /** | |
| 23 | * Returns a collection of evaluators. | |
| 24 | */ | |
| 25 | public Collection<RetrievalEvaluator> getEvaluators() { | |
| 26 | 0 | return _evaluators; |
| 27 | } | |
| 28 | ||
| 29 | /** | |
| 30 | * Returns the mean average precision; the mean of the average | |
| 31 | * precision values for all queries. | |
| 32 | */ | |
| 33 | public double meanAveragePrecision() { | |
| 34 | 0 | double sumAveragePrecision = 0; |
| 35 | ||
| 36 | 0 | for (RetrievalEvaluator evaluator : _evaluators) { |
| 37 | 0 | sumAveragePrecision += evaluator.averagePrecision(); |
| 38 | } | |
| 39 | ||
| 40 | 0 | return sumAveragePrecision / (double) _evaluators.size(); |
| 41 | } | |
| 42 | ||
| 43 | /** | |
| 44 | * Returns the mean of the binary preference values | |
| 45 | * for all queries. | |
| 46 | */ | |
| 47 | public double meanBinaryPreference() { | |
| 48 | 0 | double sumBinaryPreference = 0; |
| 49 | ||
| 50 | 0 | for (RetrievalEvaluator evaluator : _evaluators) { |
| 51 | 0 | sumBinaryPreference += evaluator.binaryPreference(); |
| 52 | } | |
| 53 | ||
| 54 | 0 | return sumBinaryPreference / (double) _evaluators.size(); |
| 55 | } | |
| 56 | ||
| 57 | /** | |
| 58 | * Returns the geometric mean of average precision values | |
| 59 | * for all queries. | |
| 60 | */ | |
| 61 | public double geometricMeanAveragePrecision() { | |
| 62 | 0 | double productAveragePrecision = 0; |
| 63 | ||
| 64 | 0 | for (RetrievalEvaluator evaluator : _evaluators) { |
| 65 | 0 | productAveragePrecision *= evaluator.averagePrecision(); |
| 66 | } | |
| 67 | ||
| 68 | 0 | return Math.pow(productAveragePrecision, 1.0 / _evaluators.size()); |
| 69 | } | |
| 70 | ||
| 71 | /** | |
| 72 | * Returns the mean of the precision values | |
| 73 | * for all queries. | |
| 74 | */ | |
| 75 | public double meanPrecision(int documentsRetrieved) { | |
| 76 | 0 | double sumPrecision = 0; |
| 77 | ||
| 78 | 0 | for (RetrievalEvaluator evaluator : _evaluators) { |
| 79 | 0 | sumPrecision += evaluator.precision(documentsRetrieved); |
| 80 | } | |
| 81 | ||
| 82 | 0 | return sumPrecision / _evaluators.size(); |
| 83 | } | |
| 84 | ||
| 85 | /** | |
| 86 | * Returns the mean of the reciprocal rank values for all queries. | |
| 87 | */ | |
| 88 | public double meanReciprocalRank() { | |
| 89 | 0 | double sumRR = 0; |
| 90 | ||
| 91 | 0 | for (RetrievalEvaluator evaluator : _evaluators) { |
| 92 | 0 | sumRR += evaluator.reciprocalRank(); |
| 93 | } | |
| 94 | ||
| 95 | 0 | return sumRR / _evaluators.size(); |
| 96 | } | |
| 97 | ||
| 98 | /** | |
| 99 | * Returns the mean of the R-precision values for all queries. | |
| 100 | */ | |
| 101 | public double meanRPrecision() { | |
| 102 | 0 | double sumRPrecision = 0; |
| 103 | ||
| 104 | 0 | for (RetrievalEvaluator evaluator : _evaluators) { |
| 105 | 0 | sumRPrecision += evaluator.rPrecision(); |
| 106 | } | |
| 107 | ||
| 108 | 0 | return sumRPrecision / _evaluators.size(); |
| 109 | } | |
| 110 | ||
| 111 | /** | |
| 112 | * Returns the mean of the NDCG values for all queries. | |
| 113 | */ | |
| 114 | public double meanNormalizedDiscountedCumulativeGain() { | |
| 115 | 0 | double sumNDCG = 0; |
| 116 | ||
| 117 | 0 | for (RetrievalEvaluator evaluator : _evaluators) { |
| 118 | 0 | sumNDCG += evaluator.normalizedDiscountedCumulativeGain(); |
| 119 | } | |
| 120 | ||
| 121 | 0 | return sumNDCG / _evaluators.size(); |
| 122 | } | |
| 123 | ||
| 124 | /** | |
| 125 | * Returns the mean of the NDCG values for all queries. | |
| 126 | */ | |
| 127 | public double meanNormalizedDiscountedCumulativeGain(int documentsRetrieved) { | |
| 128 | 0 | double sumNDCG = 0; |
| 129 | ||
| 130 | 0 | for (RetrievalEvaluator evaluator : _evaluators) { |
| 131 | 0 | sumNDCG += evaluator.normalizedDiscountedCumulativeGain(documentsRetrieved); |
| 132 | } | |
| 133 | ||
| 134 | 0 | return sumNDCG / _evaluators.size(); |
| 135 | } | |
| 136 | ||
| 137 | /** | |
| 138 | * Returns a Map containing a particular metric value for each query. | |
| 139 | * For instance, if metric == "averagePrecision", this returns | |
| 140 | * a map where the keys are query identifiers and the values are the | |
| 141 | * average precision metric evaluated for each query. | |
| 142 | */ | |
| 143 | public Map<String, Double> evaluateAll(String metric) throws IllegalArgumentException { | |
| 144 | 0 | TreeMap<String, Double> result = new TreeMap<String, Double>(); |
| 145 | ||
| 146 | 0 | for (RetrievalEvaluator evaluator : _evaluators) { |
| 147 | 0 | double value = 0; |
| 148 | ||
| 149 | 0 | if (metric.equals("averagePrecision") || metric.equals("ap") || metric.equals("map")) { |
| 150 | 0 | value = evaluator.averagePrecision(); |
| 151 | 0 | } else if (metric.equals("ndcg")) { |
| 152 | 0 | value = evaluator.normalizedDiscountedCumulativeGain(); |
| 153 | 0 | } else if (metric.startsWith("ndcg")) { |
| 154 | 0 | value = evaluator.normalizedDiscountedCumulativeGain(Integer.parseInt(metric. |
| 155 | substring(4))); | |
| 156 | 0 | } else if (metric.equals("reciprocalRank") || metric.equals("mrr")) { |
| 157 | 0 | value = evaluator.reciprocalRank(); |
| 158 | 0 | } else if (metric.equals("rPrecision")) { |
| 159 | 0 | value = evaluator.rPrecision(); |
| 160 | 0 | } else if (metric.equals("bpref")) { |
| 161 | 0 | value = evaluator.binaryPreference(); |
| 162 | 0 | } else if (metric.startsWith("P")) { |
| 163 | 0 | value = evaluator.precision(Integer.parseInt(metric.substring(1))); |
| 164 | 0 | } else if (metric.startsWith("R")) { |
| 165 | 0 | value = evaluator.recall(Integer.parseInt(metric.substring(1))); |
| 166 | } else { | |
| 167 | 0 | throw new IllegalArgumentException("Unknown metric: " + metric); |
| 168 | } | |
| 169 | ||
| 170 | 0 | result.put(evaluator.queryName(), value); |
| 171 | 0 | } |
| 172 | ||
| 173 | 0 | return result; |
| 174 | } | |
| 175 | ||
| 176 | /** | |
| 177 | * The number of documents retrieved for all queries. | |
| 178 | */ | |
| 179 | public int numberRetrieved() { | |
| 180 | 0 | int sumRetrieved = 0; |
| 181 | ||
| 182 | 0 | for (RetrievalEvaluator evaluator : _evaluators) { |
| 183 | 0 | sumRetrieved += evaluator.retrievedDocuments().size(); |
| 184 | } | |
| 185 | ||
| 186 | 0 | return sumRetrieved; |
| 187 | } | |
| 188 | ||
| 189 | /** | |
| 190 | * The total number of relevant documents to any of the queries. | |
| 191 | */ | |
| 192 | public int numberRelevant() { | |
| 193 | 0 | int sumRelevant = 0; |
| 194 | ||
| 195 | 0 | for (RetrievalEvaluator evaluator : _evaluators) { |
| 196 | 0 | sumRelevant += evaluator.relevantDocuments().size(); |
| 197 | } | |
| 198 | ||
| 199 | 0 | return sumRelevant; |
| 200 | } | |
| 201 | ||
| 202 | /** | |
| 203 | * The total number of relevant documents retrieved for any of the queries. | |
| 204 | */ | |
| 205 | public int numberRelevantRetrieved() { | |
| 206 | 0 | int sumRelevantRetrieved = 0; |
| 207 | ||
| 208 | 0 | for (RetrievalEvaluator evaluator : _evaluators) { |
| 209 | 0 | sumRelevantRetrieved += evaluator.relevantRetrievedDocuments().size(); |
| 210 | } | |
| 211 | ||
| 212 | 0 | return sumRelevantRetrieved; |
| 213 | } | |
| 214 | } |