Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#109: Make maximum and default grades configurable. #111

Merged
merged 11 commits into from
Mar 25, 2020
4 changes: 3 additions & 1 deletion rre-core/src/main/java/io/sease/rre/core/domain/Query.java
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import io.sease.rre.Func;
import io.sease.rre.core.domain.metrics.HitsCollector;
import io.sease.rre.core.domain.metrics.Metric;
import io.sease.rre.core.domain.metrics.MetricClassConfigurationManager;

import java.util.*;
import java.util.function.Function;
Expand Down Expand Up @@ -86,7 +87,8 @@ public void collect(final Map<String, Object> hit, final int rank, final String

judgment(id(hit)).ifPresent(jNode -> {
hit.put("_isRelevant", true);
hit.put("_gain", Func.gainOrRatingNode(jNode).map(JsonNode::asInt).orElse(2));
hit.put("_gain", Func.gainOrRatingNode(jNode).map(JsonNode::decimalValue)
.orElse(MetricClassConfigurationManager.getInstance().getDefaultMissingGrade()));
});

results.computeIfAbsent(version, v -> new MutableQueryOrSearchResponse()).collect(hit, rank, version);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
package io.sease.rre.core.domain.metrics;

import java.math.BigDecimal;
import java.util.Collection;
import java.util.Map;

/**
* Singleton utility class for instantiating the metric class manager,
* and managing metric configuration details.
*
* @author Matt Pearce ([email protected])
*/
public class MetricClassConfigurationManager {

private static MetricClassConfigurationManager instance;

private BigDecimal defaultMaximumGrade = BigDecimal.valueOf(3);
private BigDecimal defaultMissingGrade = BigDecimal.valueOf(2);

private MetricClassConfigurationManager() {
// Private constructor
}

public static MetricClassConfigurationManager getInstance() {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The constructor is empty so we could do

private final static MetricClassConfigurationManager INSTANCE = new MetricClassConfigurationManager();

what do you think?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, I've just made that change. My thinking was to not create before it is required, but it is used at the very beginning, and is not a big class either, so it's a sensible change to make.

if (instance == null) {
instance = new MetricClassConfigurationManager();
}
return instance;
}

/**
* Build the appropriate {@link MetricClassManager} for the metric
* configuration passed.
*
* @param metrics the simple metric configurations - a list of metric classes.
* @param parameterizedMetrics the parameterized metric configuration, consisting
* of class names and additional configuration.
* @return a {@link MetricClassManager} that can instantiate all of the
* configured metrics.
*/
@SuppressWarnings("rawtypes")
public MetricClassManager buildMetricClassManager(final Collection<String> metrics, final Map<String, Map> parameterizedMetrics) {
final MetricClassManager metricClassManager;
if (parameterizedMetrics == null || parameterizedMetrics.isEmpty()) {
metricClassManager = new SimpleMetricClassManager(metrics);
} else {
metricClassManager = new ParameterizedMetricClassManager(metrics, parameterizedMetrics);
}
return metricClassManager;
}

/**
* @return the default maximum grade to use when evaluating metrics. May be
* overridden in parameterized metric configuration.
*/
public BigDecimal getDefaultMaximumGrade() {
return defaultMaximumGrade;
}

/**
* Set the default maximum grade to use when evaluating metrics.
*
* @param defaultMaximumGrade the grade to use.
* @return the singleton manager instance.
*/
public MetricClassConfigurationManager setDefaultMaximumGrade(final float defaultMaximumGrade) {
this.defaultMaximumGrade = BigDecimal.valueOf(defaultMaximumGrade);
return this;
}

/**
* @return the default grade to use when evaluating metrics, and no judgement
* is present for the current document. May be overridden in parameterized
* metric configuration.
*/
public BigDecimal getDefaultMissingGrade() {
return defaultMissingGrade;
}

/**
* Set the default missing judgement grade to use when evaluating metrics.
*
* @param defaultMissingGrade the grade to use.
* @return the singleton manager instance.
*/
public MetricClassConfigurationManager setDefaultMissingGrade(final float defaultMissingGrade) {
this.defaultMissingGrade = BigDecimal.valueOf(defaultMissingGrade);
return this;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,8 @@ public abstract class MetricUtils {
* used in a database or search engine field name.
* <p>
* Names will be camel-cased, for the most part, with '@' and '.' symbols
* converted to words.
* converted to words. Any whitespace characters will be substituted with
* '_'.
*
* @param m the metric.
* @return the sanitised version of the metric name.
Expand All @@ -63,7 +64,8 @@ public static String sanitiseName(final Metric m) {
// Do some basic sanitisation ourselves
ret = m.getName().toLowerCase()
.replace("@", "At")
.replace(".", "Point");
.replace(".", "Point")
.replaceAll("\\s+", "_");
}

return ret;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,17 @@ public class ParameterizedMetricClassManager extends SimpleMetricClassManager im

private final static Logger LOGGER = LogManager.getLogger(ParameterizedMetricClassManager.class);

public static final String NAME_KEY = "name";
public static final String MAXIMUM_GRADE_KEY = "maximumGrade";
public static final String MISSING_GRADE_KEY = "missingGrade";

private static final String METRIC_CLASS_KEY = "class";

private final Map<String, Map<String, Object>> metricConfiguration;
private final Map<String, String> metricClasses;

public ParameterizedMetricClassManager(Collection<String> metricNames, Map<String, Map> metricConfiguration) {
@SuppressWarnings("rawtypes")
ParameterizedMetricClassManager(Collection<String> metricNames, Map<String, Map> metricConfiguration) {
super(metricNames);
this.metricClasses = extractParameterizedClassNames(metricConfiguration);
this.metricConfiguration = convertMetricConfiguration(metricConfiguration);
Expand All @@ -44,17 +49,18 @@ public ParameterizedMetricClassManager(Collection<String> metricNames, Map<Strin
* @throws IllegalArgumentException if any of the configurations do not have a
* class property.
*/
@SuppressWarnings("rawtypes")
private Map<String, String> extractParameterizedClassNames(final Map<String, Map> incoming) throws IllegalArgumentException {
final Map<String, String> classNames;
if (incoming == null) {
classNames = Collections.emptyMap();
} else {
classNames = new HashMap<>();
incoming.forEach((k, configMap) -> {
incoming.forEach((metricName, configMap) -> {
if (!configMap.containsKey(METRIC_CLASS_KEY)) {
throw new IllegalArgumentException("No class set for metric " + k);
throw new IllegalArgumentException("No class set for metric " + metricName);
} else {
classNames.put(k, (String) configMap.get(METRIC_CLASS_KEY));
classNames.put(metricName, (String) configMap.get(METRIC_CLASS_KEY));
}
});
}
Expand All @@ -70,21 +76,21 @@ private Map<String, String> extractParameterizedClassNames(final Map<String, Map
* @return an equivalent map containing configuration that can be used to
* construct a Metric without stripping any content.
*/
@SuppressWarnings("unchecked")
@SuppressWarnings({"unchecked", "rawtypes"})
private Map<String, Map<String, Object>> convertMetricConfiguration(final Map<String, Map> incoming) {
final Map<String, Map<String, Object>> configurations;
if (incoming == null) {
configurations = Collections.emptyMap();
} else {
configurations = new HashMap<>();
incoming.forEach((n, m) -> {
incoming.forEach((metricName, configOptions) -> {
Map<String, Object> config = new HashMap<>();
m.forEach((k, v) -> {
configOptions.forEach((k, v) -> {
if (!k.equals(METRIC_CLASS_KEY)) {
config.put((String) k, v);
}
});
configurations.put(n, config);
configurations.put(metricName, config);
});
}
return configurations;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ public class SimpleMetricClassManager implements MetricClassManager {
private final Collection<String> metricNames;
private final Map<String, Class<? extends Metric>> metricClasses = new HashMap<>();

public SimpleMetricClassManager(Collection<String> metricClasses) {
SimpleMetricClassManager(Collection<String> metricClasses) {
this.metricNames = new ArrayList<>(metricClasses);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,17 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.JsonNode;
import io.sease.rre.core.domain.metrics.Metric;
import io.sease.rre.core.domain.metrics.MetricClassConfigurationManager;
import io.sease.rre.core.domain.metrics.ParameterizedMetricClassManager;
import io.sease.rre.core.domain.metrics.ValueFactory;

import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.StreamSupport;
import java.util.Optional;

import static io.sease.rre.Func.gainOrRatingNode;
import static java.util.Collections.emptyList;
import static java.util.stream.Collectors.groupingBy;
import static java.math.BigDecimal.ONE;

/**
* ERR metric.
Expand All @@ -48,58 +47,73 @@ public class ExpectedReciprocalRank extends Metric {

/**
* Builds a new ExpectedReciprocalRank metric with the default gain unit function and one diversity topic.
*
* @param k the top k reference elements used for building the measure.
* @param maxgrade the maximum grade available when judging documents. If
* {@code null}, will default to 3.
* @param defaultgrade the default grade to use when judging documents. If
* {@code null}, will default to either {@code maxgrade / 2}
* or 2, depending whether or not {@code maxgrade} has been specified.
* @param name the name to use for this metric. If {@code null}, will default to {@code ERR@k}.
*/
public ExpectedReciprocalRank(@JsonProperty("maxgrade") final float maxgrade, @JsonProperty("k") final int k) {
super("ERR" + "@" + k);
this.fairgrade = BigDecimal.valueOf(Math.round(maxgrade/2));
this.maxgrade = BigDecimal.valueOf(maxgrade);
public ExpectedReciprocalRank(@JsonProperty(ParameterizedMetricClassManager.MAXIMUM_GRADE_KEY) final Float maxgrade,
@JsonProperty(ParameterizedMetricClassManager.MISSING_GRADE_KEY) final Float defaultgrade,
@JsonProperty("k") final int k,
@JsonProperty(ParameterizedMetricClassManager.NAME_KEY) final String name) {
super(Optional.ofNullable(name).orElse("ERR@" + k));
if (maxgrade == null) {
this.maxgrade = MetricClassConfigurationManager.getInstance().getDefaultMaximumGrade();
this.fairgrade = Optional.ofNullable(defaultgrade).map(BigDecimal::valueOf).orElse(MetricClassConfigurationManager.getInstance().getDefaultMissingGrade());
} else {
this.maxgrade = BigDecimal.valueOf(maxgrade);
this.fairgrade = Optional.ofNullable(defaultgrade).map(BigDecimal::valueOf).orElseGet(() -> this.maxgrade.divide(TWO, 8, RoundingMode.HALF_UP));
}
this.k = k;
}

@Override
public ValueFactory createValueFactory(final String version) {
return new ValueFactory(this, version) {
private BigDecimal ERR = BigDecimal.ZERO;
private BigDecimal trust = BigDecimal.ONE;
private BigDecimal trust = ONE;
private BigDecimal value = fairgrade;
private int totalHits = 0;
private int totalDocs = 0;

@Override
public void collect(final Map<String, Object> hit, final int rank, final String version) {
if (++totalDocs>k) return;
if (++totalDocs > k) return;
value = fairgrade;
judgment(id(hit))
.ifPresent(judgment -> {
value = gainOrRatingNode(judgment).map(JsonNode::decimalValue).orElse(fairgrade);
totalHits++;
});
.ifPresent(judgment -> {
value = gainOrRatingNode(judgment).map(JsonNode::decimalValue).orElse(fairgrade);
totalHits++;
});
BigDecimal r = BigDecimal.valueOf(rank);
BigDecimal usefulness = gain(value,maxgrade);
BigDecimal discounted = usefulness.divide(r,8,RoundingMode.HALF_UP);
BigDecimal usefulness = gain(value, maxgrade);
BigDecimal discounted = usefulness.divide(r, 8, RoundingMode.HALF_UP);
ERR = ERR.add(trust.multiply(discounted));
trust = trust.multiply(BigDecimal.ONE.subtract(usefulness));
//System.out.println(String.valueOf(rank) + " -> " + value.toPlainString());
//System.out.println(value.toPlainString());
trust = trust.multiply(ONE.subtract(usefulness));
}

@Override
public BigDecimal value() {
if (totalHits==0) {
return (totalDocs == 0) ? BigDecimal.ONE : BigDecimal.ZERO;
if (totalHits == 0) {
return (totalDocs == 0) ? ONE : BigDecimal.ZERO;
}
return ERR;
}
};
}

private BigDecimal gain(BigDecimal grade, BigDecimal max) {
final BigDecimal numer = TWO.pow(grade.intValue()).subtract(BigDecimal.ONE);
final BigDecimal denom = TWO.pow(max.intValue());
// Need to use Math.pow() here - BigDecimal.pow() is integer-only
final BigDecimal numer = BigDecimal.valueOf(Math.pow(TWO.doubleValue(), grade.doubleValue())).subtract(ONE);
final BigDecimal denom = BigDecimal.valueOf(Math.pow(TWO.doubleValue(), max.doubleValue()));
if (denom.equals(BigDecimal.ZERO)) {
return BigDecimal.ZERO;
}
return numer.divide(denom,8,RoundingMode.HALF_UP);
return numer.divide(denom, 8, RoundingMode.HALF_UP);
}

@Override
Expand Down
Loading