diff --git a/rre-core/src/main/java/io/sease/rre/core/domain/Query.java b/rre-core/src/main/java/io/sease/rre/core/domain/Query.java index 3021a312..0740b850 100644 --- a/rre-core/src/main/java/io/sease/rre/core/domain/Query.java +++ b/rre-core/src/main/java/io/sease/rre/core/domain/Query.java @@ -22,6 +22,7 @@ import io.sease.rre.Func; import io.sease.rre.core.domain.metrics.HitsCollector; import io.sease.rre.core.domain.metrics.Metric; +import io.sease.rre.core.domain.metrics.MetricClassConfigurationManager; import java.util.*; import java.util.function.Function; @@ -86,7 +87,8 @@ public void collect(final Map hit, final int rank, final String judgment(id(hit)).ifPresent(jNode -> { hit.put("_isRelevant", true); - hit.put("_gain", Func.gainOrRatingNode(jNode).map(JsonNode::asInt).orElse(2)); + hit.put("_gain", Func.gainOrRatingNode(jNode).map(JsonNode::decimalValue) + .orElse(MetricClassConfigurationManager.getInstance().getDefaultMissingGrade())); }); results.computeIfAbsent(version, v -> new MutableQueryOrSearchResponse()).collect(hit, rank, version); diff --git a/rre-core/src/main/java/io/sease/rre/core/domain/metrics/MetricClassConfigurationManager.java b/rre-core/src/main/java/io/sease/rre/core/domain/metrics/MetricClassConfigurationManager.java new file mode 100644 index 00000000..d4a4f270 --- /dev/null +++ b/rre-core/src/main/java/io/sease/rre/core/domain/metrics/MetricClassConfigurationManager.java @@ -0,0 +1,83 @@ +package io.sease.rre.core.domain.metrics; + +import java.math.BigDecimal; +import java.util.Collection; +import java.util.Map; + +/** + * Singleton utility class for instantiating the metric class manager, + * and managing metric configuration details. + * + * @author Matt Pearce (mpearce@opensourceconnections.com) + */ +public class MetricClassConfigurationManager { + + private static MetricClassConfigurationManager INSTANCE = new MetricClassConfigurationManager(); + + private BigDecimal defaultMaximumGrade = BigDecimal.valueOf(3); + private BigDecimal defaultMissingGrade = BigDecimal.valueOf(2); + + public static MetricClassConfigurationManager getInstance() { + return INSTANCE; + } + + /** + * Build the appropriate {@link MetricClassManager} for the metric + * configuration passed. + * + * @param metrics the simple metric configurations - a list of metric classes. + * @param parameterizedMetrics the parameterized metric configuration, consisting + * of class names and additional configuration. + * @return a {@link MetricClassManager} that can instantiate all of the + * configured metrics. + */ + @SuppressWarnings("rawtypes") + public MetricClassManager buildMetricClassManager(final Collection metrics, final Map parameterizedMetrics) { + final MetricClassManager metricClassManager; + if (parameterizedMetrics == null || parameterizedMetrics.isEmpty()) { + metricClassManager = new SimpleMetricClassManager(metrics); + } else { + metricClassManager = new ParameterizedMetricClassManager(metrics, parameterizedMetrics); + } + return metricClassManager; + } + + /** + * @return the default maximum grade to use when evaluating metrics. May be + * overridden in parameterized metric configuration. + */ + public BigDecimal getDefaultMaximumGrade() { + return defaultMaximumGrade; + } + + /** + * Set the default maximum grade to use when evaluating metrics. + * + * @param defaultMaximumGrade the grade to use. + * @return the singleton manager instance. + */ + public MetricClassConfigurationManager setDefaultMaximumGrade(final float defaultMaximumGrade) { + this.defaultMaximumGrade = BigDecimal.valueOf(defaultMaximumGrade); + return this; + } + + /** + * @return the default grade to use when evaluating metrics, and no judgement + * is present for the current document. May be overridden in parameterized + * metric configuration. + */ + public BigDecimal getDefaultMissingGrade() { + return defaultMissingGrade; + } + + /** + * Set the default missing judgement grade to use when evaluating metrics. + * + * @param defaultMissingGrade the grade to use. + * @return the singleton manager instance. + */ + public MetricClassConfigurationManager setDefaultMissingGrade(final float defaultMissingGrade) { + this.defaultMissingGrade = BigDecimal.valueOf(defaultMissingGrade); + return this; + } +} diff --git a/rre-core/src/main/java/io/sease/rre/core/domain/metrics/MetricUtils.java b/rre-core/src/main/java/io/sease/rre/core/domain/metrics/MetricUtils.java index f708b0a4..206bda31 100644 --- a/rre-core/src/main/java/io/sease/rre/core/domain/metrics/MetricUtils.java +++ b/rre-core/src/main/java/io/sease/rre/core/domain/metrics/MetricUtils.java @@ -49,7 +49,8 @@ public abstract class MetricUtils { * used in a database or search engine field name. *

* Names will be camel-cased, for the most part, with '@' and '.' symbols - * converted to words. + * converted to words. Any whitespace characters will be substituted with + * '_'. * * @param m the metric. * @return the sanitised version of the metric name. @@ -63,7 +64,8 @@ public static String sanitiseName(final Metric m) { // Do some basic sanitisation ourselves ret = m.getName().toLowerCase() .replace("@", "At") - .replace(".", "Point"); + .replace(".", "Point") + .replaceAll("\\s+", "_"); } return ret; diff --git a/rre-core/src/main/java/io/sease/rre/core/domain/metrics/ParameterizedMetricClassManager.java b/rre-core/src/main/java/io/sease/rre/core/domain/metrics/ParameterizedMetricClassManager.java index 94ae4b78..c12b73a9 100644 --- a/rre-core/src/main/java/io/sease/rre/core/domain/metrics/ParameterizedMetricClassManager.java +++ b/rre-core/src/main/java/io/sease/rre/core/domain/metrics/ParameterizedMetricClassManager.java @@ -21,12 +21,17 @@ public class ParameterizedMetricClassManager extends SimpleMetricClassManager im private final static Logger LOGGER = LogManager.getLogger(ParameterizedMetricClassManager.class); + public static final String NAME_KEY = "name"; + public static final String MAXIMUM_GRADE_KEY = "maximumGrade"; + public static final String MISSING_GRADE_KEY = "missingGrade"; + private static final String METRIC_CLASS_KEY = "class"; private final Map> metricConfiguration; private final Map metricClasses; - public ParameterizedMetricClassManager(Collection metricNames, Map metricConfiguration) { + @SuppressWarnings("rawtypes") + ParameterizedMetricClassManager(Collection metricNames, Map metricConfiguration) { super(metricNames); this.metricClasses = extractParameterizedClassNames(metricConfiguration); this.metricConfiguration = convertMetricConfiguration(metricConfiguration); @@ -44,17 +49,18 @@ public ParameterizedMetricClassManager(Collection metricNames, Map extractParameterizedClassNames(final Map incoming) throws IllegalArgumentException { final Map classNames; if (incoming == null) { classNames = Collections.emptyMap(); } else { classNames = new HashMap<>(); - incoming.forEach((k, configMap) -> { + incoming.forEach((metricName, configMap) -> { if (!configMap.containsKey(METRIC_CLASS_KEY)) { - throw new IllegalArgumentException("No class set for metric " + k); + throw new IllegalArgumentException("No class set for metric " + metricName); } else { - classNames.put(k, (String) configMap.get(METRIC_CLASS_KEY)); + classNames.put(metricName, (String) configMap.get(METRIC_CLASS_KEY)); } }); } @@ -70,21 +76,21 @@ private Map extractParameterizedClassNames(final Map> convertMetricConfiguration(final Map incoming) { final Map> configurations; if (incoming == null) { configurations = Collections.emptyMap(); } else { configurations = new HashMap<>(); - incoming.forEach((n, m) -> { + incoming.forEach((metricName, configOptions) -> { Map config = new HashMap<>(); - m.forEach((k, v) -> { + configOptions.forEach((k, v) -> { if (!k.equals(METRIC_CLASS_KEY)) { config.put((String) k, v); } }); - configurations.put(n, config); + configurations.put(metricName, config); }); } return configurations; diff --git a/rre-core/src/main/java/io/sease/rre/core/domain/metrics/SimpleMetricClassManager.java b/rre-core/src/main/java/io/sease/rre/core/domain/metrics/SimpleMetricClassManager.java index 607f9299..f245a970 100644 --- a/rre-core/src/main/java/io/sease/rre/core/domain/metrics/SimpleMetricClassManager.java +++ b/rre-core/src/main/java/io/sease/rre/core/domain/metrics/SimpleMetricClassManager.java @@ -17,7 +17,7 @@ public class SimpleMetricClassManager implements MetricClassManager { private final Collection metricNames; private final Map> metricClasses = new HashMap<>(); - public SimpleMetricClassManager(Collection metricClasses) { + SimpleMetricClassManager(Collection metricClasses) { this.metricNames = new ArrayList<>(metricClasses); } diff --git a/rre-core/src/main/java/io/sease/rre/core/domain/metrics/impl/ExpectedReciprocalRank.java b/rre-core/src/main/java/io/sease/rre/core/domain/metrics/impl/ExpectedReciprocalRank.java index 912c9c04..40608b88 100644 --- a/rre-core/src/main/java/io/sease/rre/core/domain/metrics/impl/ExpectedReciprocalRank.java +++ b/rre-core/src/main/java/io/sease/rre/core/domain/metrics/impl/ExpectedReciprocalRank.java @@ -19,18 +19,17 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; import io.sease.rre.core.domain.metrics.Metric; +import io.sease.rre.core.domain.metrics.MetricClassConfigurationManager; +import io.sease.rre.core.domain.metrics.ParameterizedMetricClassManager; import io.sease.rre.core.domain.metrics.ValueFactory; import java.math.BigDecimal; import java.math.RoundingMode; -import java.util.Arrays; -import java.util.List; import java.util.Map; -import java.util.stream.StreamSupport; +import java.util.Optional; import static io.sease.rre.Func.gainOrRatingNode; -import static java.util.Collections.emptyList; -import static java.util.stream.Collectors.groupingBy; +import static java.math.BigDecimal.ONE; /** * ERR metric. @@ -48,11 +47,27 @@ public class ExpectedReciprocalRank extends Metric { /** * Builds a new ExpectedReciprocalRank metric with the default gain unit function and one diversity topic. + * + * @param k the top k reference elements used for building the measure. + * @param maxgrade the maximum grade available when judging documents. If + * {@code null}, will default to 3. + * @param defaultgrade the default grade to use when judging documents. If + * {@code null}, will default to either {@code maxgrade / 2} + * or 2, depending whether or not {@code maxgrade} has been specified. + * @param name the name to use for this metric. If {@code null}, will default to {@code ERR@k}. */ - public ExpectedReciprocalRank(@JsonProperty("maxgrade") final float maxgrade, @JsonProperty("k") final int k) { - super("ERR" + "@" + k); - this.fairgrade = BigDecimal.valueOf(Math.round(maxgrade/2)); - this.maxgrade = BigDecimal.valueOf(maxgrade); + public ExpectedReciprocalRank(@JsonProperty(ParameterizedMetricClassManager.MAXIMUM_GRADE_KEY) final Float maxgrade, + @JsonProperty(ParameterizedMetricClassManager.MISSING_GRADE_KEY) final Float defaultgrade, + @JsonProperty("k") final int k, + @JsonProperty(ParameterizedMetricClassManager.NAME_KEY) final String name) { + super(Optional.ofNullable(name).orElse("ERR@" + k)); + if (maxgrade == null) { + this.maxgrade = MetricClassConfigurationManager.getInstance().getDefaultMaximumGrade(); + this.fairgrade = Optional.ofNullable(defaultgrade).map(BigDecimal::valueOf).orElse(MetricClassConfigurationManager.getInstance().getDefaultMissingGrade()); + } else { + this.maxgrade = BigDecimal.valueOf(maxgrade); + this.fairgrade = Optional.ofNullable(defaultgrade).map(BigDecimal::valueOf).orElseGet(() -> this.maxgrade.divide(TWO, 8, RoundingMode.HALF_UP)); + } this.k = k; } @@ -60,33 +75,31 @@ public ExpectedReciprocalRank(@JsonProperty("maxgrade") final float maxgrade, @J public ValueFactory createValueFactory(final String version) { return new ValueFactory(this, version) { private BigDecimal ERR = BigDecimal.ZERO; - private BigDecimal trust = BigDecimal.ONE; + private BigDecimal trust = ONE; private BigDecimal value = fairgrade; private int totalHits = 0; private int totalDocs = 0; @Override public void collect(final Map hit, final int rank, final String version) { - if (++totalDocs>k) return; + if (++totalDocs > k) return; value = fairgrade; judgment(id(hit)) - .ifPresent(judgment -> { - value = gainOrRatingNode(judgment).map(JsonNode::decimalValue).orElse(fairgrade); - totalHits++; - }); + .ifPresent(judgment -> { + value = gainOrRatingNode(judgment).map(JsonNode::decimalValue).orElse(fairgrade); + totalHits++; + }); BigDecimal r = BigDecimal.valueOf(rank); - BigDecimal usefulness = gain(value,maxgrade); - BigDecimal discounted = usefulness.divide(r,8,RoundingMode.HALF_UP); + BigDecimal usefulness = gain(value, maxgrade); + BigDecimal discounted = usefulness.divide(r, 8, RoundingMode.HALF_UP); ERR = ERR.add(trust.multiply(discounted)); - trust = trust.multiply(BigDecimal.ONE.subtract(usefulness)); - //System.out.println(String.valueOf(rank) + " -> " + value.toPlainString()); - //System.out.println(value.toPlainString()); + trust = trust.multiply(ONE.subtract(usefulness)); } @Override public BigDecimal value() { - if (totalHits==0) { - return (totalDocs == 0) ? BigDecimal.ONE : BigDecimal.ZERO; + if (totalHits == 0) { + return (totalDocs == 0) ? ONE : BigDecimal.ZERO; } return ERR; } @@ -94,12 +107,13 @@ public BigDecimal value() { } private BigDecimal gain(BigDecimal grade, BigDecimal max) { - final BigDecimal numer = TWO.pow(grade.intValue()).subtract(BigDecimal.ONE); - final BigDecimal denom = TWO.pow(max.intValue()); + // Need to use Math.pow() here - BigDecimal.pow() is integer-only + final BigDecimal numer = BigDecimal.valueOf(Math.pow(TWO.doubleValue(), grade.doubleValue())).subtract(ONE); + final BigDecimal denom = BigDecimal.valueOf(Math.pow(TWO.doubleValue(), max.doubleValue())); if (denom.equals(BigDecimal.ZERO)) { return BigDecimal.ZERO; } - return numer.divide(denom,8,RoundingMode.HALF_UP); + return numer.divide(denom, 8, RoundingMode.HALF_UP); } @Override diff --git a/rre-core/src/main/java/io/sease/rre/core/domain/metrics/impl/NDCGAtK.java b/rre-core/src/main/java/io/sease/rre/core/domain/metrics/impl/NDCGAtK.java index d3d5bcf8..cef94a5d 100644 --- a/rre-core/src/main/java/io/sease/rre/core/domain/metrics/impl/NDCGAtK.java +++ b/rre-core/src/main/java/io/sease/rre/core/domain/metrics/impl/NDCGAtK.java @@ -19,6 +19,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; import io.sease.rre.core.domain.metrics.Metric; +import io.sease.rre.core.domain.metrics.MetricClassConfigurationManager; +import io.sease.rre.core.domain.metrics.ParameterizedMetricClassManager; import io.sease.rre.core.domain.metrics.ValueFactory; import java.math.BigDecimal; @@ -28,6 +30,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.StreamSupport; @@ -42,15 +45,44 @@ */ public class NDCGAtK extends Metric { private final static BigDecimal TWO = new BigDecimal(2); - final int k; + + private final BigDecimal fairgrade; + private final BigDecimal maxgrade; + private final int k; + + /** + * Builds a new NDCGAtK metric with default maximum and missing judgement + * grades. + * + * @param k the top k reference elements used for building the measure. + */ + public NDCGAtK(final int k) { + this(k, null, null, null); + } /** * Builds a new NDCGAtK metric. * * @param k the top k reference elements used for building the measure. + * @param maxgrade the maximum grade available when judging documents. If + * {@code null}, will default to 3. + * @param defaultgrade the default grade to use when judging documents. If + * {@code null}, will default to either {@code maxgrade / 2} + * or 2, depending whether or not {@code maxgrade} has been specified. + * @param name the name to use for this metric. If {@code null}, will default to {@code NDCG@k}. */ - public NDCGAtK(@JsonProperty("k") final int k) { - super("NDCG@" + k); + public NDCGAtK(@JsonProperty("k") final int k, + @JsonProperty(ParameterizedMetricClassManager.MAXIMUM_GRADE_KEY) final Float maxgrade, + @JsonProperty(ParameterizedMetricClassManager.MISSING_GRADE_KEY) final Float defaultgrade, + @JsonProperty(ParameterizedMetricClassManager.NAME_KEY) final String name) { + super(Optional.ofNullable(name).orElse("NDCG@" + k)); + if (maxgrade == null) { + this.maxgrade = MetricClassConfigurationManager.getInstance().getDefaultMaximumGrade(); + this.fairgrade = Optional.ofNullable(defaultgrade).map(BigDecimal::valueOf).orElse(MetricClassConfigurationManager.getInstance().getDefaultMissingGrade()); + } else { + this.maxgrade = BigDecimal.valueOf(maxgrade); + this.fairgrade = Optional.ofNullable(defaultgrade).map(BigDecimal::valueOf).orElseGet(() -> this.maxgrade.divide(TWO, 8, RoundingMode.HALF_UP)); + } this.k = k; } @@ -69,8 +101,8 @@ public void collect(final Map hit, final int rank, final String if (rank > k) return; judgment(id(hit)) .ifPresent(judgment -> { - final BigDecimal value = gainOrRatingNode(judgment).map(JsonNode::decimalValue).orElse(TWO); - BigDecimal numerator = TWO.pow(value.intValue()).subtract(BigDecimal.ONE); + final BigDecimal value = gainOrRatingNode(judgment).map(JsonNode::decimalValue).orElse(fairgrade); + BigDecimal numerator = BigDecimal.valueOf(Math.pow(TWO.doubleValue(), value.doubleValue())).subtract(BigDecimal.ONE); if (rank == 1) { dcg = numerator; } else { @@ -98,28 +130,28 @@ public BigDecimal value() { private BigDecimal idealDcg(final JsonNode relevantDocuments) { final int windowSize = Math.min(relevantDocuments.size(), k); - final int[] gains = new int[windowSize]; + final double[] gains = new double[windowSize]; - final Map> groups = + final Map> groups = StreamSupport.stream(relevantDocuments.spliterator(), false) - .collect(groupingBy(doc -> gainOrRatingNode(doc).map(JsonNode::intValue).orElse(2))); + .collect(groupingBy(doc -> gainOrRatingNode(doc).map(JsonNode::decimalValue).orElse(fairgrade))); - Set ratingValues = groups.keySet(); - List ratingsSorted = new ArrayList<>(ratingValues); + Set ratingValues = groups.keySet(); + List ratingsSorted = new ArrayList<>(ratingValues); ratingsSorted.sort(Collections.reverseOrder()); int startIndex = 0; - for (Integer ratingValue : ratingsSorted) { + for (BigDecimal ratingValue : ratingsSorted) { if (startIndex < windowSize) { List docsPerRating = groups.get(ratingValue); int endIndex = startIndex + docsPerRating.size(); - Arrays.fill(gains, startIndex, Math.min(windowSize, endIndex), ratingValue); + Arrays.fill(gains, startIndex, Math.min(windowSize, endIndex), ratingValue.doubleValue()); startIndex = endIndex; } } BigDecimal result = BigDecimal.ZERO; for (int i = 1; i <= gains.length; i++) { - BigDecimal num = TWO.pow(gains[i-1]).subtract(BigDecimal.ONE); + BigDecimal num = BigDecimal.valueOf(Math.pow(TWO.doubleValue(), gains[i-1])).subtract(BigDecimal.ONE); double den = Math.log(i + 1) / Math.log(2); result = result.add((num.divide(new BigDecimal(den), 2, RoundingMode.FLOOR))); } diff --git a/rre-core/src/main/java/io/sease/rre/core/domain/metrics/impl/ReciprocalRank.java b/rre-core/src/main/java/io/sease/rre/core/domain/metrics/impl/ReciprocalRank.java index 951f3f5b..6253bd4d 100644 --- a/rre-core/src/main/java/io/sease/rre/core/domain/metrics/impl/ReciprocalRank.java +++ b/rre-core/src/main/java/io/sease/rre/core/domain/metrics/impl/ReciprocalRank.java @@ -16,14 +16,18 @@ */ package io.sease.rre.core.domain.metrics.impl; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; import io.sease.rre.Func; import io.sease.rre.core.domain.metrics.Metric; +import io.sease.rre.core.domain.metrics.MetricClassConfigurationManager; +import io.sease.rre.core.domain.metrics.ParameterizedMetricClassManager; import io.sease.rre.core.domain.metrics.ValueFactory; import java.math.BigDecimal; import java.math.RoundingMode; import java.util.Map; +import java.util.Optional; /** * The reciprocal rank of a query response is the multiplicative inverse of the rank of the first correct answer. @@ -32,25 +36,58 @@ * @since 1.0 */ public class ReciprocalRank extends Metric { + + private final int k; + private final BigDecimal maxgrade; + private final BigDecimal fairgrade; + /** - * Builds a new ReciprocalRank at X metric. + * Builds a new ReciprocalRank at 10 metric. */ public ReciprocalRank() { - super("RR@10"); + this(10, null, null, null); + } + + /** + * Builds a new Reciprocal Rank at K metric. + * + * @param k the top k reference elements used for building the measure. + * @param maxgrade the maximum grade available when judging documents. If + * {@code null}, will default to 3. + * @param defaultgrade the default grade to use when judging documents. If + * {@code null}, will default to either {@code maxgrade / 2} + * or 2, depending whether or not {@code maxgrade} has been specified. + * @param name the name to use for this metric. If {@code null}, will default to {@code RR@k}. + */ + public ReciprocalRank(@JsonProperty("k") final int k, + @JsonProperty(ParameterizedMetricClassManager.MAXIMUM_GRADE_KEY) final Float maxgrade, + @JsonProperty(ParameterizedMetricClassManager.MISSING_GRADE_KEY) final Float defaultgrade, + @JsonProperty(ParameterizedMetricClassManager.NAME_KEY) final String name) { + super(Optional.ofNullable(name).orElse("RR@" + k)); + this.k = k; + if (maxgrade == null) { + this.maxgrade = MetricClassConfigurationManager.getInstance().getDefaultMaximumGrade(); + this.fairgrade = Optional.ofNullable(defaultgrade).map(BigDecimal::valueOf).orElse(MetricClassConfigurationManager.getInstance().getDefaultMissingGrade()); + } else { + this.maxgrade = BigDecimal.valueOf(maxgrade); + this.fairgrade = Optional.ofNullable(defaultgrade).map(BigDecimal::valueOf).orElseGet(() -> this.maxgrade.divide(BigDecimal.valueOf(2), 8, RoundingMode.HALF_UP)); + } } @Override public ValueFactory createValueFactory(final String version) { return new ValueFactory(this, version) { private int rank; - private int maxGain; + private BigDecimal maxGain = BigDecimal.ZERO; + private int totalDocs = 0; @Override public void collect(final Map hit, final int rank, final String version) { + if (++totalDocs > k) return; judgment(id(hit)) .ifPresent(hitData -> { - final int gain = Func.gainOrRatingNode(hitData).map(JsonNode::asInt).orElse(2); - if (gain > maxGain) { + final BigDecimal gain = Func.gainOrRatingNode(hitData).map(JsonNode::decimalValue).orElse(fairgrade); + if (gain.compareTo(maxGain) > 0) { this.rank = rank; this.maxGain = gain; } diff --git a/rre-core/src/test/java/io.sease.rre.core.domain.metrics.impl/ExpectedReciprocalRankTestCase.java b/rre-core/src/test/java/io.sease.rre.core.domain.metrics.impl/ExpectedReciprocalRankTestCase.java index e4432e07..a80dc663 100644 --- a/rre-core/src/test/java/io.sease.rre.core.domain.metrics.impl/ExpectedReciprocalRankTestCase.java +++ b/rre-core/src/test/java/io.sease.rre.core.domain.metrics.impl/ExpectedReciprocalRankTestCase.java @@ -34,7 +34,7 @@ public class ExpectedReciprocalRankTestCase extends BaseTestCase { @Before public void setUp() { - cut = new ExpectedReciprocalRank(3, K); + cut = new ExpectedReciprocalRank(3.0f, 2.0f, K, null); cut.setVersions(Collections.singletonList(A_VERSION)); counter = new AtomicInteger(0); } diff --git a/rre-core/src/test/java/io.sease.rre.core.domain.metrics.impl/NDCGAtKTestCase.java b/rre-core/src/test/java/io.sease.rre.core.domain.metrics.impl/NDCGAtKTestCase.java index 18ee741d..4068b4aa 100644 --- a/rre-core/src/test/java/io.sease.rre.core.domain.metrics.impl/NDCGAtKTestCase.java +++ b/rre-core/src/test/java/io.sease.rre.core.domain.metrics.impl/NDCGAtKTestCase.java @@ -256,4 +256,44 @@ public void _10_judgments_15_search_only_results_10th_relevant_result() { cut.valueFactory(A_VERSION).value().doubleValue(), 0); } + + /** + * Scenario: 10 judgments, 15 search results, 5 relevant results in top positions. + */ + @Test + public void _10_judgments_15_search_only_results_10th_relevant_result_topscore() { + final ObjectNode judgements = mapper.createObjectNode(); + stream(FIFTEEN_SEARCH_HITS).limit(10).forEach(docid -> judgements.set(docid, createJudgmentNode(4))); + cut.setRelevantDocuments(judgements); + + cut.setTotalHits(FIFTEEN_SEARCH_HITS.length, A_VERSION); + stream(ANOTHER_FIVE_SEARCH_HITS) + .map(this::searchHit) + .forEach(hit -> cut.collect(hit, counter.incrementAndGet(), A_VERSION)); + + stream(ANOTHER_FOUR_SEARCH_HITS) + .map(this::searchHit) + .forEach(hit -> cut.collect(hit, counter.incrementAndGet(), A_VERSION)); + + cut.collect(searchHit(FIFTEEN_SEARCH_HITS[9]), counter.incrementAndGet(), A_VERSION); + + Map expectations = new HashMap() + {{ + put(1,0.0); + put(2,0.0); + put(3,0.0); + put(4,0.0); + put(5,0.0); + put(6,0.0); + put(7,0.0); + put(8, 0.0); + put(9,0.0); + put(10, 0.06); + }}; + + assertEquals( + expectations.get(currentAppliedK), + cut.valueFactory(A_VERSION).value().doubleValue(), + 0); + } } diff --git a/rre-core/src/test/java/io.sease.rre.core.domain.metrics.impl/ReciprocalRankTestCase.java b/rre-core/src/test/java/io.sease.rre.core.domain.metrics.impl/ReciprocalRankTestCase.java index 472a3355..ff8ab0aa 100644 --- a/rre-core/src/test/java/io.sease.rre.core.domain.metrics.impl/ReciprocalRankTestCase.java +++ b/rre-core/src/test/java/io.sease.rre.core.domain.metrics.impl/ReciprocalRankTestCase.java @@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicInteger; import static io.sease.rre.core.TestData.A_VERSION; +import static io.sease.rre.core.TestData.FIFTEEN_SEARCH_HITS; import static java.util.Arrays.stream; import static org.junit.Assert.assertEquals; @@ -174,4 +175,22 @@ public void firstRelevantResultAtSecondPosition() { assertEquals(0.5d, cut.valueFactory(A_VERSION).value().doubleValue(), 0); } + + /** + * If the first relevant result is outside the first K results, the value + * should be 0. + */ + @Test + public void firstRelevantResultOutsideKResults() { + final ObjectNode judgements = mapper.createObjectNode(); + stream(FIFTEEN_SEARCH_HITS).skip(11).limit(1).forEach(docid -> judgements.set(docid, createJudgmentNode(3))); + cut.setRelevantDocuments(judgements); + + cut.setTotalHits(FIFTEEN_SEARCH_HITS.length, A_VERSION); + stream(FIFTEEN_SEARCH_HITS) + .map(this::searchHit) + .forEach(hit -> cut.collect(hit, counter.incrementAndGet(), A_VERSION)); + + assertEquals(0.0d, cut.valueFactory(A_VERSION).value().doubleValue(), 0); + } } \ No newline at end of file diff --git a/rre-core/src/test/java/io/sease/rre/core/domain/metrics/MetricClassConfigurationManagerTest.java b/rre-core/src/test/java/io/sease/rre/core/domain/metrics/MetricClassConfigurationManagerTest.java new file mode 100644 index 00000000..81f99fc9 --- /dev/null +++ b/rre-core/src/test/java/io/sease/rre/core/domain/metrics/MetricClassConfigurationManagerTest.java @@ -0,0 +1,50 @@ +package io.sease.rre.core.domain.metrics; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Unit tests for the MetricClassManagerFactory class. + * + * @author Matt Pearce (mpearce@opensourceconnections.com) + */ +public class MetricClassConfigurationManagerTest { + + private final MetricClassConfigurationManager factory = MetricClassConfigurationManager.getInstance(); + + private static final Collection METRICS = Arrays.asList( + "io.sease.rre.core.domain.metrics.impl.PrecisionAtOne", + "io.sease.rre.core.domain.metrics.impl.PrecisionAtK" + ); + + + @Test + public void returnsSimpleMetricManager_whenParameterizedMetricsAreNull() { + assertThat(factory.buildMetricClassManager(METRICS, null)).isInstanceOf(SimpleMetricClassManager.class); + } + + @Test + public void returnsSimpleMetricManager_whenParameterizedMetricsIsEmpty() { + assertThat(factory.buildMetricClassManager(METRICS, new HashMap<>())).isInstanceOf(SimpleMetricClassManager.class); + } + + @Test + @SuppressWarnings({"unchecked", "rawtypes"}) + public void returnsParameterizedMetricManager_whenParameterizedMetricsSet() { + final Map precisionAtFiveConfig = new HashMap(); + precisionAtFiveConfig.put("class", "io.sease.rre.core.domain.metrics.impl.PrecisionAtK"); + precisionAtFiveConfig.put("v", 5); + final Map parameterizedConfig = new HashMap<>(); + parameterizedConfig.put("pAt5", precisionAtFiveConfig); + + MetricClassManager test = factory.buildMetricClassManager(METRICS, parameterizedConfig); + + assertThat(test).isInstanceOf(ParameterizedMetricClassManager.class); + } +} diff --git a/rre-maven-plugin/rre-maven-elasticsearch-plugin/src/main/java/io/sease/rre/maven/plugin/elasticsearch/RREvaluateMojo.java b/rre-maven-plugin/rre-maven-elasticsearch-plugin/src/main/java/io/sease/rre/maven/plugin/elasticsearch/RREvaluateMojo.java index 20809af8..f4b1d948 100644 --- a/rre-maven-plugin/rre-maven-elasticsearch-plugin/src/main/java/io/sease/rre/maven/plugin/elasticsearch/RREvaluateMojo.java +++ b/rre-maven-plugin/rre-maven-elasticsearch-plugin/src/main/java/io/sease/rre/maven/plugin/elasticsearch/RREvaluateMojo.java @@ -17,9 +17,8 @@ package io.sease.rre.maven.plugin.elasticsearch; import io.sease.rre.core.Engine; +import io.sease.rre.core.domain.metrics.MetricClassConfigurationManager; import io.sease.rre.core.domain.metrics.MetricClassManager; -import io.sease.rre.core.domain.metrics.ParameterizedMetricClassManager; -import io.sease.rre.core.domain.metrics.SimpleMetricClassManager; import io.sease.rre.core.evaluation.EvaluationConfiguration; import io.sease.rre.persistence.PersistenceConfiguration; import io.sease.rre.search.api.SearchPlatform; @@ -92,6 +91,12 @@ public class RREvaluateMojo extends AbstractMojo { @Parameter(name = "port", defaultValue = "9200") private int port; + @Parameter(name="maximumGrade", defaultValue="3") + private float maximumGrade; + + @Parameter(name="missingGrade", defaultValue="2") + private float missingGrade; + @Parameter(name = "persistence") private PersistenceConfiguration persistence = PersistenceConfiguration.DEFAULT_CONFIG; @@ -116,8 +121,10 @@ public void execute() throws MojoExecutionException { Thread.currentThread().getContextClassLoader())); try (final SearchPlatform platform = new Elasticsearch()) { - final MetricClassManager metricClassManager = - parameterizedMetrics == null ? new SimpleMetricClassManager(metrics) : new ParameterizedMetricClassManager(metrics, parameterizedMetrics); + final MetricClassManager metricClassManager = MetricClassConfigurationManager.getInstance() + .setDefaultMaximumGrade(maximumGrade) + .setDefaultMissingGrade(missingGrade) + .buildMetricClassManager(metrics, parameterizedMetrics); final Engine engine = new Engine( platform, configurationsFolder, diff --git a/rre-maven-plugin/rre-maven-external-elasticsearch-plugin/src/main/java/io/sease/rre/maven/plugin/external/elasticsearch/RREvaluateMojo.java b/rre-maven-plugin/rre-maven-external-elasticsearch-plugin/src/main/java/io/sease/rre/maven/plugin/external/elasticsearch/RREvaluateMojo.java index 6f233878..f1ab45a5 100644 --- a/rre-maven-plugin/rre-maven-external-elasticsearch-plugin/src/main/java/io/sease/rre/maven/plugin/external/elasticsearch/RREvaluateMojo.java +++ b/rre-maven-plugin/rre-maven-external-elasticsearch-plugin/src/main/java/io/sease/rre/maven/plugin/external/elasticsearch/RREvaluateMojo.java @@ -19,9 +19,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.sease.rre.core.Engine; import io.sease.rre.core.domain.Evaluation; +import io.sease.rre.core.domain.metrics.MetricClassConfigurationManager; import io.sease.rre.core.domain.metrics.MetricClassManager; -import io.sease.rre.core.domain.metrics.ParameterizedMetricClassManager; -import io.sease.rre.core.domain.metrics.SimpleMetricClassManager; import io.sease.rre.core.evaluation.EvaluationConfiguration; import io.sease.rre.persistence.PersistenceConfiguration; import io.sease.rre.search.api.SearchPlatform; @@ -37,7 +36,6 @@ import java.net.URL; import java.net.URLClassLoader; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -76,6 +74,12 @@ public class RREvaluateMojo extends AbstractMojo { @Parameter(name = "exclude") private List exclude; + @Parameter(name="maximumGrade", defaultValue="3") + private float maximumGrade; + + @Parameter(name="missingGrade", defaultValue="2") + private float missingGrade; + @Parameter(name = "persistence") private PersistenceConfiguration persistence = PersistenceConfiguration.DEFAULT_CONFIG; @@ -100,8 +104,10 @@ public void execute() throws MojoExecutionException { Thread.currentThread().getContextClassLoader())); try (final SearchPlatform platform = new ExternalElasticsearch()) { - final MetricClassManager metricClassManager = - parameterizedMetrics == null ? new SimpleMetricClassManager(metrics) : new ParameterizedMetricClassManager(metrics, parameterizedMetrics); + final MetricClassManager metricClassManager = MetricClassConfigurationManager.getInstance() + .setDefaultMaximumGrade(maximumGrade) + .setDefaultMissingGrade(missingGrade) + .buildMetricClassManager(metrics, parameterizedMetrics); final Engine engine = new Engine( platform, configurationsFolder, diff --git a/rre-maven-plugin/rre-maven-external-solr-plugin/src/main/java/io/sease/rre/maven/plugin/externalsolr/RREvaluateMojo.java b/rre-maven-plugin/rre-maven-external-solr-plugin/src/main/java/io/sease/rre/maven/plugin/externalsolr/RREvaluateMojo.java index 35a06e2f..964c6cfd 100644 --- a/rre-maven-plugin/rre-maven-external-solr-plugin/src/main/java/io/sease/rre/maven/plugin/externalsolr/RREvaluateMojo.java +++ b/rre-maven-plugin/rre-maven-external-solr-plugin/src/main/java/io/sease/rre/maven/plugin/externalsolr/RREvaluateMojo.java @@ -17,9 +17,8 @@ package io.sease.rre.maven.plugin.externalsolr; import io.sease.rre.core.Engine; +import io.sease.rre.core.domain.metrics.MetricClassConfigurationManager; import io.sease.rre.core.domain.metrics.MetricClassManager; -import io.sease.rre.core.domain.metrics.ParameterizedMetricClassManager; -import io.sease.rre.core.domain.metrics.SimpleMetricClassManager; import io.sease.rre.core.evaluation.EvaluationConfiguration; import io.sease.rre.persistence.PersistenceConfiguration; import io.sease.rre.search.api.SearchPlatform; @@ -67,6 +66,12 @@ public class RREvaluateMojo extends AbstractMojo { @Parameter(name = "exclude") private List exclude; + @Parameter(name="maximumGrade", defaultValue="3") + private float maximumGrade; + + @Parameter(name="missingGrade", defaultValue="2") + private float missingGrade; + @Parameter(name = "persistence") private PersistenceConfiguration persistence = PersistenceConfiguration.DEFAULT_CONFIG; @@ -76,8 +81,10 @@ public class RREvaluateMojo extends AbstractMojo { @Override public void execute() throws MojoExecutionException { try (final SearchPlatform platform = new ExternalApacheSolr()) { - final MetricClassManager metricClassManager = - parameterizedMetrics == null ? new SimpleMetricClassManager(metrics) : new ParameterizedMetricClassManager(metrics, parameterizedMetrics); + final MetricClassManager metricClassManager = MetricClassConfigurationManager.getInstance() + .setDefaultMaximumGrade(maximumGrade) + .setDefaultMissingGrade(missingGrade) + .buildMetricClassManager(metrics, parameterizedMetrics); final Engine engine = new Engine( platform, configurationsFolder, diff --git a/rre-maven-plugin/rre-maven-solr-plugin/src/main/java/io/sease/rre/maven/plugin/solr/RREvaluateMojo.java b/rre-maven-plugin/rre-maven-solr-plugin/src/main/java/io/sease/rre/maven/plugin/solr/RREvaluateMojo.java index 81a737cb..d204d087 100644 --- a/rre-maven-plugin/rre-maven-solr-plugin/src/main/java/io/sease/rre/maven/plugin/solr/RREvaluateMojo.java +++ b/rre-maven-plugin/rre-maven-solr-plugin/src/main/java/io/sease/rre/maven/plugin/solr/RREvaluateMojo.java @@ -17,9 +17,8 @@ package io.sease.rre.maven.plugin.solr; import io.sease.rre.core.Engine; +import io.sease.rre.core.domain.metrics.MetricClassConfigurationManager; import io.sease.rre.core.domain.metrics.MetricClassManager; -import io.sease.rre.core.domain.metrics.ParameterizedMetricClassManager; -import io.sease.rre.core.domain.metrics.SimpleMetricClassManager; import io.sease.rre.core.evaluation.EvaluationConfiguration; import io.sease.rre.persistence.PersistenceConfiguration; import io.sease.rre.search.api.SearchPlatform; @@ -80,6 +79,12 @@ public class RREvaluateMojo extends AbstractMojo { @Parameter(name = "exclude") private List exclude; + @Parameter(name="maximumGrade", defaultValue="3") + private float maximumGrade; + + @Parameter(name="missingGrade", defaultValue="2") + private float missingGrade; + @Parameter(name = "persistence") private PersistenceConfiguration persistence = PersistenceConfiguration.DEFAULT_CONFIG; @@ -89,8 +94,10 @@ public class RREvaluateMojo extends AbstractMojo { @Override public void execute() throws MojoExecutionException { try (final SearchPlatform platform = new ApacheSolr()) { - final MetricClassManager metricClassManager = - parameterizedMetrics == null ? new SimpleMetricClassManager(metrics) : new ParameterizedMetricClassManager(metrics, parameterizedMetrics); + final MetricClassManager metricClassManager = MetricClassConfigurationManager.getInstance() + .setDefaultMaximumGrade(maximumGrade) + .setDefaultMissingGrade(missingGrade) + .buildMetricClassManager(metrics, parameterizedMetrics); final Engine engine = new Engine( platform, configurationsFolder,