Skip to content

Commit

Permalink
SeaseLtd#109: Changes to allow default max grade, default missing gra…
Browse files Browse the repository at this point in the history
…de to be set via Maven pom.xml:

- Add singleton factory for instantiating MetricClassManager instances;
- Allow access to default grade values via factory class.
- Modify Maven plugins to use factory.
  • Loading branch information
Matt Pearce committed Mar 16, 2020
1 parent ef1bc11 commit 0784fdc
Show file tree
Hide file tree
Showing 13 changed files with 178 additions and 40 deletions.
4 changes: 3 additions & 1 deletion rre-core/src/main/java/io/sease/rre/core/domain/Query.java
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import io.sease.rre.Func;
import io.sease.rre.core.domain.metrics.HitsCollector;
import io.sease.rre.core.domain.metrics.Metric;
import io.sease.rre.core.domain.metrics.MetricClassManagerFactory;

import java.util.*;
import java.util.function.Function;
Expand Down Expand Up @@ -86,7 +87,8 @@ public void collect(final Map<String, Object> hit, final int rank, final String

judgment(id(hit)).ifPresent(jNode -> {
hit.put("_isRelevant", true);
hit.put("_gain", Func.gainOrRatingNode(jNode).map(JsonNode::asInt).orElse(2));
hit.put("_gain", Func.gainOrRatingNode(jNode).map(JsonNode::decimalValue)
.orElse(MetricClassManagerFactory.getInstance().getDefaultMissingGrade()));
});

results.computeIfAbsent(version, v -> new MutableQueryOrSearchResponse()).collect(hit, rank, version);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,6 @@ public abstract class Metric implements HitsCollector {

public static final int DEFAULT_REQUIRED_RESULTS = 10;

public static final BigDecimal DEFAULT_MAX_GRADE = BigDecimal.valueOf(3);
public static final BigDecimal DEFAULT_MISSING_GRADE = BigDecimal.valueOf(2);

private final String name;

protected String idFieldName = DEFAULT_ID_FIELD_NAME;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package io.sease.rre.core.domain.metrics;

import java.math.BigDecimal;
import java.util.List;
import java.util.Map;

/**
* Singleton utility class for instantiating the metric class manager,
* and managing metric configuration details.
*
* @author Matt Pearce ([email protected])
*/
public class MetricClassManagerFactory {

private static MetricClassManagerFactory instance;

private BigDecimal defaultMaximumGrade = BigDecimal.valueOf(3);
private BigDecimal defaultMissingGrade = BigDecimal.valueOf(2);

private MetricClassManagerFactory() {
// Private constructor
}

public static MetricClassManagerFactory getInstance() {
if (instance == null) {
instance = new MetricClassManagerFactory();
}
return instance;
}

public MetricClassManager buildMetricClassManager(List<String> metrics, Map<String, Map> parameterizedMetrics) {
final MetricClassManager metricClassManager;
if (parameterizedMetrics == null || parameterizedMetrics.isEmpty()) {
metricClassManager = new SimpleMetricClassManager(metrics);
} else {
metricClassManager = new ParameterizedMetricClassManager(metrics, parameterizedMetrics);
}
return metricClassManager;
}

public BigDecimal getDefaultMaximumGrade() {
return defaultMaximumGrade;
}

public MetricClassManagerFactory setDefaultMaximumGrade(BigDecimal defaultMaximumGrade) {
this.defaultMaximumGrade = defaultMaximumGrade;
return this;
}

public BigDecimal getDefaultMissingGrade() {
return defaultMissingGrade;
}

public MetricClassManagerFactory setDefaultMissingGrade(BigDecimal defaultMissingGrade) {
this.defaultMissingGrade = defaultMissingGrade;
return this;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public class ParameterizedMetricClassManager extends SimpleMetricClassManager im
private final Map<String, Map<String, Object>> metricConfiguration;
private final Map<String, String> metricClasses;

public ParameterizedMetricClassManager(Collection<String> metricNames, Map<String, Map> metricConfiguration) {
ParameterizedMetricClassManager(Collection<String> metricNames, Map<String, Map> metricConfiguration) {
super(metricNames);
this.metricClasses = extractParameterizedClassNames(metricConfiguration);
this.metricConfiguration = convertMetricConfiguration(metricConfiguration);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ public class SimpleMetricClassManager implements MetricClassManager {
private final Collection<String> metricNames;
private final Map<String, Class<? extends Metric>> metricClasses = new HashMap<>();

public SimpleMetricClassManager(Collection<String> metricClasses) {
SimpleMetricClassManager(Collection<String> metricClasses) {
this.metricNames = new ArrayList<>(metricClasses);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.JsonNode;
import io.sease.rre.core.domain.metrics.Metric;
import io.sease.rre.core.domain.metrics.MetricClassManagerFactory;
import io.sease.rre.core.domain.metrics.ValueFactory;

import java.math.BigDecimal;
Expand Down Expand Up @@ -48,19 +49,18 @@ public class ExpectedReciprocalRank extends Metric {
*
* @param k the top k reference elements used for building the measure.
* @param maxgrade the maximum grade available when judging documents. If
* {@code null}, will default to {@link Metric#DEFAULT_MAX_GRADE}.
* {@code null}, will default to 3.
* @param defaultgrade the default grade to use when judging documents. If
* {@code null}, will default to either {@code maxgrade / 2}
* or {@link Metric#DEFAULT_MISSING_GRADE}, depending
* whether or not {@code maxgrade} has been specified.
* or 2, depending whether or not {@code maxgrade} has been specified.
*/
public ExpectedReciprocalRank(@JsonProperty("maxgrade") final Float maxgrade,
@JsonProperty("defaultgrade") final Float defaultgrade,
@JsonProperty("k") final int k) {
super("ERR" + "@" + k);
if (maxgrade == null) {
this.maxgrade = DEFAULT_MAX_GRADE;
this.fairgrade = Optional.ofNullable(defaultgrade).map(BigDecimal::valueOf).orElse(DEFAULT_MISSING_GRADE);
this.maxgrade = MetricClassManagerFactory.getInstance().getDefaultMaximumGrade();
this.fairgrade = Optional.ofNullable(defaultgrade).map(BigDecimal::valueOf).orElse(MetricClassManagerFactory.getInstance().getDefaultMissingGrade());
} else {
this.maxgrade = BigDecimal.valueOf(maxgrade);
this.fairgrade = Optional.ofNullable(defaultgrade).map(BigDecimal::valueOf).orElseGet(() -> this.maxgrade.divide(TWO, 8, RoundingMode.HALF_UP));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.JsonNode;
import io.sease.rre.core.domain.metrics.Metric;
import io.sease.rre.core.domain.metrics.MetricClassManagerFactory;
import io.sease.rre.core.domain.metrics.ValueFactory;

import java.math.BigDecimal;
Expand Down Expand Up @@ -62,20 +63,19 @@ public NDCGAtK(final int k) {
* Builds a new NDCGAtK metric.
*
* @param k the top k reference elements used for building the measure.
* @param maxgrade the maximum grade available when judging documents. If
* {@code null}, will default to {@link Metric#DEFAULT_MAX_GRADE}.
* @param maxgrade the maximum grade available when judging documents. If
* {@code null}, will default to 3.
* @param defaultgrade the default grade to use when judging documents. If
* {@code null}, will default to either {@code maxgrade / 2}
* or {@link Metric#DEFAULT_MISSING_GRADE}, depending
* whether or not {@code maxgrade} has been specified.
* or 2, depending whether or not {@code maxgrade} has been specified.
*/
public NDCGAtK(@JsonProperty("k") final int k,
@JsonProperty("maxgrade") final Float maxgrade,
@JsonProperty("defaultgrade") final Float defaultgrade) {
super("NDCG@" + k);
if (maxgrade == null) {
this.maxgrade = DEFAULT_MAX_GRADE;
this.fairgrade = Optional.ofNullable(defaultgrade).map(BigDecimal::valueOf).orElse(DEFAULT_MISSING_GRADE);
this.maxgrade = MetricClassManagerFactory.getInstance().getDefaultMaximumGrade();
this.fairgrade = Optional.ofNullable(defaultgrade).map(BigDecimal::valueOf).orElse(MetricClassManagerFactory.getInstance().getDefaultMissingGrade());
} else {
this.maxgrade = BigDecimal.valueOf(maxgrade);
this.fairgrade = Optional.ofNullable(defaultgrade).map(BigDecimal::valueOf).orElseGet(() -> this.maxgrade.divide(TWO, 8, RoundingMode.HALF_UP));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.fasterxml.jackson.databind.JsonNode;
import io.sease.rre.Func;
import io.sease.rre.core.domain.metrics.Metric;
import io.sease.rre.core.domain.metrics.MetricClassManagerFactory;
import io.sease.rre.core.domain.metrics.ValueFactory;

import java.math.BigDecimal;
Expand Down Expand Up @@ -50,21 +51,20 @@ public ReciprocalRank() {
* Builds a new Reciprocal Rank at K metric.
*
* @param k the top k reference elements used for building the measure.
* @param maxgrade the maximum grade available when judging documents. If
* {@code null}, will default to {@link Metric#DEFAULT_MAX_GRADE}.
* @param maxgrade the maximum grade available when judging documents. If
* {@code null}, will default to 3.
* @param defaultgrade the default grade to use when judging documents. If
* {@code null}, will default to either {@code maxgrade / 2}
* or {@link Metric#DEFAULT_MISSING_GRADE}, depending
* whether or not {@code maxgrade} has been specified.
* or 2, depending whether or not {@code maxgrade} has been specified.
*/
public ReciprocalRank(@JsonProperty("k") final int k,
@JsonProperty("maxgrade") final Float maxgrade,
@JsonProperty("defaultgrade") final Float defaultgrade) {
super("RR@" + k);
this.k = k;
if (maxgrade == null) {
this.maxgrade = DEFAULT_MAX_GRADE;
this.fairgrade = Optional.ofNullable(defaultgrade).map(BigDecimal::valueOf).orElse(DEFAULT_MISSING_GRADE);
this.maxgrade = MetricClassManagerFactory.getInstance().getDefaultMaximumGrade();
this.fairgrade = Optional.ofNullable(defaultgrade).map(BigDecimal::valueOf).orElse(MetricClassManagerFactory.getInstance().getDefaultMissingGrade());
} else {
this.maxgrade = BigDecimal.valueOf(maxgrade);
this.fairgrade = Optional.ofNullable(defaultgrade).map(BigDecimal::valueOf).orElseGet(() -> this.maxgrade.divide(BigDecimal.valueOf(2), 8, RoundingMode.HALF_UP));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package io.sease.rre.core.domain.metrics;

import org.junit.Test;

import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static org.assertj.core.api.Assertions.assertThat;

/**
* Unit tests for the MetricClassManagerFactory class.
*
* @author Matt Pearce ([email protected])
*/
public class MetricClassManagerFactoryTest {

private final MetricClassManagerFactory factory = MetricClassManagerFactory.getInstance();

private static final List<String> METRICS = Arrays.asList(
"io.sease.rre.core.domain.metrics.impl.PrecisionAtOne",
"io.sease.rre.core.domain.metrics.impl.PrecisionAtK"
);


@Test
public void returnsSimpleMetricManager_whenParameterizedMetricsAreNull() {
assertThat(factory.buildMetricClassManager(METRICS, null)).isInstanceOf(SimpleMetricClassManager.class);
}

@Test
public void returnsSimpleMetricManager_whenParameterizedMetricsIsEmpty() {
assertThat(factory.buildMetricClassManager(METRICS, new HashMap<>())).isInstanceOf(SimpleMetricClassManager.class);
}

@Test
@SuppressWarnings({"unchecked", "rawtypes"})
public void returnsParameterizedMetricManager_whenParameterizedMetricsSet() {
final Map precisionAtFiveConfig = new HashMap();
precisionAtFiveConfig.put("class", "io.sease.rre.core.domain.metrics.impl.PrecisionAtK");
precisionAtFiveConfig.put("v", 5);
final Map<String, Map> parameterizedConfig = new HashMap<>();
parameterizedConfig.put("pAt5", precisionAtFiveConfig);

MetricClassManager test = factory.buildMetricClassManager(METRICS, parameterizedConfig);

assertThat(test).isInstanceOf(ParameterizedMetricClassManager.class);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,7 @@

import io.sease.rre.core.Engine;
import io.sease.rre.core.domain.metrics.MetricClassManager;
import io.sease.rre.core.domain.metrics.ParameterizedMetricClassManager;
import io.sease.rre.core.domain.metrics.SimpleMetricClassManager;
import io.sease.rre.core.domain.metrics.MetricClassManagerFactory;
import io.sease.rre.core.evaluation.EvaluationConfiguration;
import io.sease.rre.persistence.PersistenceConfiguration;
import io.sease.rre.search.api.SearchPlatform;
Expand All @@ -32,6 +31,7 @@

import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.HashMap;
Expand Down Expand Up @@ -92,6 +92,12 @@ public class RREvaluateMojo extends AbstractMojo {
@Parameter(name = "port", defaultValue = "9200")
private int port;

@Parameter(name="maximumGrade", defaultValue="3")
private float maxGrade;

@Parameter(name="missingGrade", defaultValue="2")
private float missingGrade;

@Parameter(name = "persistence")
private PersistenceConfiguration persistence = PersistenceConfiguration.DEFAULT_CONFIG;

Expand All @@ -116,8 +122,10 @@ public void execute() throws MojoExecutionException {
Thread.currentThread().getContextClassLoader()));

try (final SearchPlatform platform = new Elasticsearch()) {
final MetricClassManager metricClassManager =
parameterizedMetrics == null ? new SimpleMetricClassManager(metrics) : new ParameterizedMetricClassManager(metrics, parameterizedMetrics);
final MetricClassManager metricClassManager = MetricClassManagerFactory.getInstance()
.setDefaultMaximumGrade(BigDecimal.valueOf(maxGrade))
.setDefaultMissingGrade(BigDecimal.valueOf(missingGrade))
.buildMetricClassManager(metrics, parameterizedMetrics);
final Engine engine = new Engine(
platform,
configurationsFolder,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,7 @@
import io.sease.rre.core.Engine;
import io.sease.rre.core.domain.Evaluation;
import io.sease.rre.core.domain.metrics.MetricClassManager;
import io.sease.rre.core.domain.metrics.ParameterizedMetricClassManager;
import io.sease.rre.core.domain.metrics.SimpleMetricClassManager;
import io.sease.rre.core.domain.metrics.MetricClassManagerFactory;
import io.sease.rre.core.evaluation.EvaluationConfiguration;
import io.sease.rre.persistence.PersistenceConfiguration;
import io.sease.rre.search.api.SearchPlatform;
Expand All @@ -34,10 +33,10 @@

import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

Expand Down Expand Up @@ -76,6 +75,12 @@ public class RREvaluateMojo extends AbstractMojo {
@Parameter(name = "exclude")
private List<String> exclude;

@Parameter(name="maximumGrade", defaultValue="3")
private float maxGrade;

@Parameter(name="missingGrade", defaultValue="2")
private float missingGrade;

@Parameter(name = "persistence")
private PersistenceConfiguration persistence = PersistenceConfiguration.DEFAULT_CONFIG;

Expand All @@ -100,8 +105,10 @@ public void execute() throws MojoExecutionException {
Thread.currentThread().getContextClassLoader()));

try (final SearchPlatform platform = new ExternalElasticsearch()) {
final MetricClassManager metricClassManager =
parameterizedMetrics == null ? new SimpleMetricClassManager(metrics) : new ParameterizedMetricClassManager(metrics, parameterizedMetrics);
final MetricClassManager metricClassManager = MetricClassManagerFactory.getInstance()
.setDefaultMaximumGrade(BigDecimal.valueOf(maxGrade))
.setDefaultMissingGrade(BigDecimal.valueOf(missingGrade))
.buildMetricClassManager(metrics, parameterizedMetrics);
final Engine engine = new Engine(
platform,
configurationsFolder,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,7 @@

import io.sease.rre.core.Engine;
import io.sease.rre.core.domain.metrics.MetricClassManager;
import io.sease.rre.core.domain.metrics.ParameterizedMetricClassManager;
import io.sease.rre.core.domain.metrics.SimpleMetricClassManager;
import io.sease.rre.core.domain.metrics.MetricClassManagerFactory;
import io.sease.rre.core.evaluation.EvaluationConfiguration;
import io.sease.rre.persistence.PersistenceConfiguration;
import io.sease.rre.search.api.SearchPlatform;
Expand All @@ -31,6 +30,7 @@
import org.apache.maven.plugins.annotations.Parameter;

import java.io.IOException;
import java.math.BigDecimal;
import java.util.Collections;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -67,6 +67,12 @@ public class RREvaluateMojo extends AbstractMojo {
@Parameter(name = "exclude")
private List<String> exclude;

@Parameter(name="maximumGrade", defaultValue="3")
private float maxGrade;

@Parameter(name="missingGrade", defaultValue="2")
private float missingGrade;

@Parameter(name = "persistence")
private PersistenceConfiguration persistence = PersistenceConfiguration.DEFAULT_CONFIG;

Expand All @@ -76,8 +82,10 @@ public class RREvaluateMojo extends AbstractMojo {
@Override
public void execute() throws MojoExecutionException {
try (final SearchPlatform platform = new ExternalApacheSolr()) {
final MetricClassManager metricClassManager =
parameterizedMetrics == null ? new SimpleMetricClassManager(metrics) : new ParameterizedMetricClassManager(metrics, parameterizedMetrics);
final MetricClassManager metricClassManager = MetricClassManagerFactory.getInstance()
.setDefaultMaximumGrade(BigDecimal.valueOf(maxGrade))
.setDefaultMissingGrade(BigDecimal.valueOf(missingGrade))
.buildMetricClassManager(metrics, parameterizedMetrics);
final Engine engine = new Engine(
platform,
configurationsFolder,
Expand Down
Loading

0 comments on commit 0784fdc

Please sign in to comment.