76 irvcomparisonaudit

server/eclipse-project/src/main/java/au/org/democracydevelopers/corla/model/IRVComparisonAudit.java

@@ -459,7 +459,8 @@ public void removeDiscrepancy(final CVRAuditInfo theRecord, final int theType) {
       // exists, in each of the audit's assertions.
       boolean removed = false;
       for (Assertion a : assertions) {
-        removed = removed || a.removeDiscrepancy(theRecord);
+        boolean removed_a = a.removeDiscrepancy(theRecord);
+        removed = removed || removed_a;
       }
 
       // Update the discrepancy tallies in the base ComparisonAudit class, for reporting purposes,

server/eclipse-project/src/test/java/au/org/democracydevelopers/corla/model/IRVComparisonAuditTests.java

@@ -23,7 +23,7 @@
 
 import static au.org.democracydevelopers.corla.util.testUtils.log;
 import static org.mockito.Mockito.when;
-import static org.testng.Assert.assertEquals;
+import static org.testng.AssertJUnit.assertEquals;
 
 import au.org.democracydevelopers.corla.model.assertion.Assertion;
 import au.org.democracydevelopers.corla.model.assertion.AssertionTests;
@@ -38,6 +38,7 @@
 import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
 import org.mockito.Mock;
+import org.testng.Assert;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 import us.freeandfair.corla.model.AuditReason;
@@ -49,18 +50,10 @@
 import us.freeandfair.corla.model.ContestResult;
 
 import javax.transaction.Transactional;
+import us.freeandfair.corla.util.Pair;
 
 /**
  * This class contains tests for the functionality present in IRVComparisonAudit.
- * TODO: tests for removing discrepancies -2, -1, 0, 1, 2
- * TODO: tests for risk measurement.
- * TODO: tests of the reaudit ballot workflow.
- * TODO: tests for remaining error modes.
- * TODO: tests for when there are no assertions, but there are some assertion generation summaries.
- * TODO: tests for when there are neither assertions nor assertion generation summaries (at this
- *       stage, just throw an exception).
- *       (These may both be irrelevant depending on how we resolve the construction of IRVComparisonAudits -
- *       see issue <a href="https://github.com/orgs/DemocracyDevelopers/projects/1?pane=issue&itemId=60493439">...</a>)
  */
 @Transactional
 public class IRVComparisonAuditTests extends AssertionTests {
@@ -139,6 +132,12 @@ public class IRVComparisonAuditTests extends AssertionTests {
   @Mock
   private CVRAuditInfo auditInfo;
 
+  /**
+   * Mock of a CVRAuditInfo object, matched to a null CVR
+   */
+  @Mock
+  private CVRAuditInfo auditInfoNullCVR;
+
   /**
    * Initialise mocked objects prior to the first test. Note that the diluted margin
    * returned by ContestResult's for IRV will not have a sensible value, and it will
@@ -175,6 +174,10 @@ public void initContestResultMocks() {
     when(auditInfo.id()).thenReturn(1L);
     when(auditInfo.cvr()).thenReturn(cvr);
     when(auditInfo.acvr()).thenReturn(auditedCvr);
+
+    when(auditInfoNullCVR.id()).thenReturn(1L);
+    when(auditInfoNullCVR.cvr()).thenReturn(null);
+    when(auditInfoNullCVR.acvr()).thenReturn(auditedCvr);
   }
 
   /**
@@ -216,7 +219,18 @@ public void testCreateIRVAuditOneNEBAssertion(){
         0, 0, 23, 23, 1, Map.of(), 0.32);
   }
 
+  /**
+   * Create an IRVComparisonAudit for a contest with one NEB assertion stored in the database, and
+   * test that riskMeasurement() returns the maximum risk (no samples audited).
+   */
+  @Test
+  public void testCreateIRVAuditOneNEBAssertionRiskMeasurement(){
+    testUtils.log(LOGGER, "testCreateIRVAuditOneNEBAssertionRiskMeasurement");
+    IRVComparisonAudit ca = new IRVComparisonAudit(oneNEBContestResult, AssertionTests.riskLimit3,
+        AuditReason.COUNTY_WIDE_CONTEST);
 
+    assertEquals(0, testUtils.doubleComparator.compare(1.0, ca.riskMeasurement().doubleValue()));
+  }
 
   /**
    * Create an IRVComparisonAudit for a contest with one NEN assertion stored in the database. This
@@ -239,6 +253,19 @@ public void testCreateIRVAuditOneNENAssertion(){
         61, 1, Map.of(), 0.12);
   }
 
+  /**
+   * Create an IRVComparisonAudit for a contest with one NEN assertion stored in the database, and
+   * test that riskMeasurement() returns the maximum risk (no samples audited).
+   */
+  @Test
+  public void testCreateIRVAuditOneNENAssertionRiskMeasurement(){
+    testUtils.log(LOGGER, "testCreateIRVAuditOneNENAssertionRiskMeasurement");
+    IRVComparisonAudit ca = new IRVComparisonAudit(oneNENContestResult, AssertionTests.riskLimit3,
+        AuditReason.GEOGRAPHICAL_SCOPE);
+
+    assertEquals(0, testUtils.doubleComparator.compare(1.0, ca.riskMeasurement().doubleValue()));
+  }
+
   /**
    * Create an IRVComparisonAudit for a contest with one NEN and one NEB assertion stored in the
    * database. The smallest diluted margin across these assertions is 0.1.
@@ -263,6 +290,19 @@ public void testCreateIRVAuditOneNENNEBAssertion(){
         15, 1, Map.of(), 0.5);
   }
 
+  /**
+   * Create an IRVComparisonAudit for a contest with one NEN and one NEN assertion stored in the database, and
+   * test that riskMeasurement() returns the maximum risk (no samples audited).
+   */
+  @Test
+  public void testCreateIRVAuditOneNENNEBAssertionRiskMeasurement(){
+    testUtils.log(LOGGER, "testCreateIRVAuditOneNENNEBAssertionRiskMeasurement");
+    IRVComparisonAudit ca = new IRVComparisonAudit(oneNENNEBContestResult, AssertionTests.riskLimit3,
+        AuditReason.COUNTY_WIDE_CONTEST);
+
+    assertEquals(0, testUtils.doubleComparator.compare(1.0, ca.riskMeasurement().doubleValue()));
+  }
+
   /**
    * Create an IRVComparisonAudit for a multi-county contest with two NEBs and one NEN stored in the
    * database. The smallest diluted margin across these assertions is 0.1.
@@ -467,6 +507,72 @@ public void testNENRecordNoDiscrepancy(int theType){
     ca.recordDiscrepancy(auditInfo, theType);
   }
 
+  /**
+   * If we call removeDiscrepancy() with an invalid discrepancy type (3), an IllegalArgumentException
+   * should be thrown.
+   */
+  @Test(expectedExceptions = IllegalArgumentException.class)
+  public void testRemoveInvalidDiscrepancy1(){
+    log(LOGGER, "testRemoveInvalidDiscrepancy1");
+    IRVComparisonAudit ca = new IRVComparisonAudit(testEstimationNEBOnly,
+        AssertionTests.riskLimit3, AuditReason.OPPORTUNISTIC_BENEFITS);
+
+    ca.removeDiscrepancy(auditInfo, 3);
+  }
+
+  /**
+   * If we call removeDiscrepancy() with an invalid discrepancy type (-3), an IllegalArgumentException
+   * should be thrown.
+   */
+  @Test(expectedExceptions = IllegalArgumentException.class)
+  public void testRemoveInvalidDiscrepancy2(){
+    log(LOGGER, "testRemoveInvalidDiscrepancy2");
+    IRVComparisonAudit ca = new IRVComparisonAudit(testEstimationNEBOnly,
+        AssertionTests.riskLimit3, AuditReason.OPPORTUNISTIC_BENEFITS);
+
+    ca.removeDiscrepancy(auditInfo, -3);
+  }
+
+  /**
+   * If we call removeDiscrepancy() on an audit that does not contain any discrepancies, no
+   * discrepancies will be removed.
+   */
+  @Test
+  public void testRemoveInvalidDiscrepancy3(){
+    log(LOGGER, "testRemoveInvalidDiscrepancy3");
+    IRVComparisonAudit ca = new IRVComparisonAudit(mixedContest2,
+        AssertionTests.riskLimit3, AuditReason.OPPORTUNISTIC_BENEFITS);
+
+    checkDiscrepancies(ca, 0, 0, 0, 0, 0);
+    ca.removeDiscrepancy(auditInfo, 1);
+    checkDiscrepancies(ca, 0, 0, 0, 0, 0);
+  }
+
+  /**
+   * If we call removeDiscrepancy() with a null audit info, an IllegalArgumentException will be
+   * thrown.
+   */
+  @Test(expectedExceptions = IllegalArgumentException.class)
+  public void testRemoveNullAuditInfo(){
+    log(LOGGER, "testRemoveNullAuditInfo");
+    IRVComparisonAudit ca = new IRVComparisonAudit(testEstimationNEBOnly,
+        AssertionTests.riskLimit3, AuditReason.OPPORTUNISTIC_BENEFITS);
+
+    ca.removeDiscrepancy(null, -1);
+  }
+
+  /**
+   * If we call removeDiscrepancy() with a null cvr in the provided audit info, an
+   * IllegalArgumentException will be thrown.
+   */
+  @Test(expectedExceptions = IllegalArgumentException.class)
+  public void testRemoveNullAuditInfoCVR(){
+    log(LOGGER, "testRemoveNullAuditInfoCVR");
+    IRVComparisonAudit ca = createIRVComparisonAuditMixed();
+
+    ca.removeDiscrepancy(auditInfoNullCVR, -1);
+  }
+
   /**
    * Discrepancy computation for 'Mixed Contest' with CVR "A" and audited ballot "A", "B", "C" ,"D".
    * The maximum discrepancy is 0.
@@ -487,7 +593,7 @@ public void testComputeDiscrepancyCVR_A_ACVR_ABCD(RecordType auditedType){
   }
 
   /**
-   * Test that if we follow up discrepancy computation with an invalid call to recordDiscrepancy().
+   * Test that if we follow discrepancy computation with an invalid call to recordDiscrepancy().
    * (Invalid in the sense that the provided discrepancy type is not the maximum across the
    * audit's assertions for the given CVR).
    */
@@ -511,11 +617,12 @@ public void testRecordWrongDiscrepancy(){
 
   /**
    * Discrepancy computation for 'Mixed Contest' with CVR "A","B","C","D" and audited ballot
-   * "B","A","C","D" and recording of the resulting maximum discrepancy of type 2.
+   * "B","A","C","D" and recording of the resulting maximum discrepancy of type 2. Also checks
+   * risk measurement before and after the removal of the recorded discrepancy.
    */
   @Test(dataProvider = "AuditedRecordTypes", dataProviderClass = AssertionTests.class)
-  public void testComputeRecordDiscrepancyCVR_ABCD_ACVR_BACD(RecordType auditedType){
-    log(LOGGER, String.format("testComputeRecordDiscrepancyCVR_ABCD_ACVR_BACD[%s]", auditedType));
+  public void testComputeRecordRemoveDiscrepancyCVR_ABCD_ACVR_BACD(RecordType auditedType){
+    log(LOGGER, String.format("testComputeRecordRemoveDiscrepancyCVR_ABCD_ACVR_BACD[%s]", auditedType));
     resetMocks(ABCD, BACD, RecordType.UPLOADED, ConsensusValue.YES, auditedType, "Mixed Contest");
     IRVComparisonAudit ca = createIRVComparisonAuditMixed();
 
@@ -531,6 +638,34 @@ public void testComputeRecordDiscrepancyCVR_ABCD_ACVR_BACD(RecordType auditedTyp
     ca.recordDiscrepancy(auditInfo, 2);
 
     checkDiscrepancies(ca, 0, 1, 0, 0, 0);
+
+    riskMeasurementCheck(ca, List.of(Pair.make(1000, 0.214),
+        Pair.make(1500, 0.019)));
+
+    ca.removeDiscrepancy(auditInfo, 2);
+    checkDiscrepancies(ca, 0, 0, 0, 0, 0);
+
+    // ca.riskMeasurement() at this point, where the audited sample count is 1500, should yield
+    // a risk of 0.001.
+    Assert.assertEquals(testUtils.doubleComparator.compare(0.001,
+        ca.riskMeasurement().doubleValue()), 0);
+  }
+
+  /**
+   * Given an ordered list of sample size and expected risk value, check that the risk
+   * measurement method in the given IRVComparisonAudit produces expected values.
+   * @param audit IRVComparisonAudit whose risk measurement method is being tested.
+   * @param risks Expected risks for varying audited sample counts (ordered by sample size).
+   */
+  private void riskMeasurementCheck(IRVComparisonAudit audit, final List<Pair<Integer, Double>> risks){
+    int sample = 0;
+    for(Pair<Integer,Double> entry : risks){
+      // Increment sample count according to entry key
+      audit.signalSampleAudited(entry.first()-sample);
+      sample += (entry.first()-sample);
+      final BigDecimal r = audit.riskMeasurement();
+      Assert.assertEquals(testUtils.doubleComparator.compare(entry.second(), r.doubleValue()), 0);
+    }
   }
 
   /**
@@ -578,11 +713,12 @@ public void testComputeDiscrepancyCVR_BACD_ACVR_ABCD(RecordType auditedType){
 
   /**
    * Discrepancy computation and recording for 'Mixed Contest' with CVR "B","A","C","D" and
-   * audited ballot "A","B","C","D". The maximum discrepancy is 1.
+   * audited ballot "A","B","C","D". The maximum discrepancy is 1. Also checks risk measurement
+   * before and after the removal of the recorded discrepancies against the ballot/CVR pair.
    */
   @Test(dataProvider = "AuditedRecordTypes", dataProviderClass = AssertionTests.class)
-  public void testComputeRecordDiscrepancyCVR_BACD_ACVR_ABCD(RecordType auditedType){
-    log(LOGGER, String.format("testComputeRecordDiscrepancyCVR_BACD_ACVR_ABCD[%s]", auditedType));
+  public void testComputeRecordRemoveDiscrepancyCVR_BACD_ACVR_ABCD(RecordType auditedType){
+    log(LOGGER, String.format("testComputeRecordRemoveDiscrepancyCVR_BACD_ACVR_ABCD[%s]", auditedType));
     resetMocks(BACD, ABCD, RecordType.UPLOADED, ConsensusValue.YES, auditedType, "Mixed Contest");
     IRVComparisonAudit ca = createIRVComparisonAuditMixed();
 
@@ -598,15 +734,27 @@ public void testComputeRecordDiscrepancyCVR_BACD_ACVR_ABCD(RecordType auditedTyp
     ca.recordDiscrepancy(auditInfo, 1);
 
     checkDiscrepancies(ca, 1, 0, 0, 0, 0);
+
+    riskMeasurementCheck(ca, List.of(Pair.make(10, 1.0),
+        Pair.make(100, 0.894), Pair.make(200, 0.415)));
+
+    ca.removeDiscrepancy(auditInfo, 1);
+    checkDiscrepancies(ca, 0, 0, 0, 0, 0);
+
+    // ca.riskMeasurement() at this point, where the audited sample count is 200, should yield
+    // a risk of 0.381.
+    Assert.assertEquals(testUtils.doubleComparator.compare(0.381,
+        ca.riskMeasurement().doubleValue()), 0);
   }
 
   /**
    * Discrepancy computation and recording for 'Mixed Contest 2' with CVR "B","A","C","D" and
-   * audited ballot "A","B","C","D". The maximum discrepancy is -1.
+   * audited ballot "A","B","C","D". The maximum discrepancy is -1. Also checks risk measurement
+   * before and after the removal of the recorded discrepancies against the ballot/CVR pair.
    */
   @Test(dataProvider = "AuditedRecordTypes", dataProviderClass = AssertionTests.class)
-  public void testComputeRecordDiscrepancyCVR_BACD_ACVR_ABCD_mixed2(RecordType auditedType){
-    log(LOGGER, String.format("testComputeRecordDiscrepancyCVR_BACD_ACVR_ABCD_mixed2[%s]", auditedType));
+  public void testComputeRecordRemoveDiscrepancyCVR_BACD_ACVR_ABCD_mixed2(RecordType auditedType){
+    log(LOGGER, String.format("testComputeRecordRemoveDiscrepancyCVR_BACD_ACVR_ABCD_mixed2[%s]", auditedType));
     resetMocks(BACD, ABCD, RecordType.UPLOADED, ConsensusValue.YES, auditedType, "Mixed Contest 2");
     IRVComparisonAudit ca = createIRVComparisonAuditMixed2();
 
@@ -622,14 +770,26 @@ public void testComputeRecordDiscrepancyCVR_BACD_ACVR_ABCD_mixed2(RecordType aud
     ca.recordDiscrepancy(auditInfo, -1);
 
     checkDiscrepancies(ca, 0, 0, 1, 0, 0);
+
+    riskMeasurementCheck(ca, List.of(Pair.make(10, 0.412),
+        Pair.make(50, 0.151), Pair.make(100, 0.045)));
+
+    ca.removeDiscrepancy(auditInfo, -1);
+    checkDiscrepancies(ca, 0, 0, 0, 0, 0);
+
+    // ca.riskMeasurement() at this point, where the audited sample count is 100, should yield
+    // a risk of 0.088.
+    Assert.assertEquals(testUtils.doubleComparator.compare(0.088,
+        ca.riskMeasurement().doubleValue()), 0);
   }
 
   /**
    * Discrepancy computation and recording for 'Simple Contest 3' with CVR "B","A","C","D" and
-   * audited ballot "A","B","C","D". The maximum discrepancy is -2.
+   * audited ballot "A","B","C","D". The maximum discrepancy is -2. Also checks risk measurement
+   * before and after the removal of the recorded discrepancies against the ballot/CVR pair.
    */
   @Test(dataProvider = "AuditedRecordTypes", dataProviderClass = AssertionTests.class)
-  public void testComputeRecordDiscrepancyCVR_BACD_ACVR_ABCD_simple3(RecordType auditedType){
+  public void testComputeRecordRemoveDiscrepancyCVR_BACD_ACVR_ABCD_simple3(RecordType auditedType){
     log(LOGGER, String.format("testComputeRecordDiscrepancyCVR_BACD_ACVR_ABCD_simple3[%s]", auditedType));
     resetMocks(BACD, ABCD, RecordType.UPLOADED, ConsensusValue.YES, auditedType, "Simple Contest 3");
     IRVComparisonAudit ca = createIRVComparisonAuditSimple3();
@@ -646,6 +806,17 @@ public void testComputeRecordDiscrepancyCVR_BACD_ACVR_ABCD_simple3(RecordType au
     ca.recordDiscrepancy(auditInfo, -2);
 
     checkDiscrepancies(ca, 0, 0, 0, 1, 0);
+
+    riskMeasurementCheck(ca, List.of(Pair.make(10, 0.463),
+        Pair.make(50, 0.314), Pair.make(100, 0.194)));
+
+    ca.removeDiscrepancy(auditInfo, -2);
+    checkDiscrepancies(ca, 0, 0, 0, 0, 0);
+
+    // ca.riskMeasurement() at this point, where the audited sample count is 100, should yield
+    // a risk of 0.380.
+    Assert.assertEquals(testUtils.doubleComparator.compare(0.380,
+        ca.riskMeasurement().doubleValue()), 0);
   }
 
   /**
@@ -707,11 +878,12 @@ public void testComputeDiscrepancyCVR_blank_ACVR_ABCD(RecordType auditedType){
 
   /**
    * Discrepancy computation and recording for 'Mixed Contest' with CVR blank and audited ballot
-   * "A","B","C","D". The maximum discrepancy is 0.
+   * "A","B","C","D". The maximum discrepancy is 0. Also checks risk measurement before and after
+   * the removal of the recorded discrepancies against the ballot/CVR pair.
    */
   @Test(dataProvider = "AuditedRecordTypes", dataProviderClass = AssertionTests.class)
-  public void testComputeRecordDiscrepancyCVR_blank_ACVR_ABCD(RecordType auditedType){
-    log(LOGGER, String.format("testComputeRecordDiscrepancyCVR_blank_ACVR_ABCD[%s]", auditedType));
+  public void testComputeRecordRemoveDiscrepancyCVR_blank_ACVR_ABCD(RecordType auditedType){
+    log(LOGGER, String.format("testComputeRecordRemoveDiscrepancyCVR_blank_ACVR_ABCD[%s]", auditedType));
     resetMocks(blank, ABCD, RecordType.UPLOADED, ConsensusValue.YES, auditedType, "Mixed Contest");
     IRVComparisonAudit ca = createIRVComparisonAuditMixed();
 
@@ -727,6 +899,17 @@ public void testComputeRecordDiscrepancyCVR_blank_ACVR_ABCD(RecordType auditedTy
     ca.recordDiscrepancy(auditInfo, 0);
 
     checkDiscrepancies(ca, 0, 0, 0, 0, 1);
+
+    riskMeasurementCheck(ca, List.of(Pair.make(10, 0.926),
+        Pair.make(50, 0.681), Pair.make(100, 0.464)));
+
+    ca.removeDiscrepancy(auditInfo, 0);
+    checkDiscrepancies(ca, 0, 0, 0, 0, 0);
+
+    // ca.riskMeasurement() at this point, where the audited sample count is , should yield
+    // a risk of .
+    Assert.assertEquals(testUtils.doubleComparator.compare(0.617,
+        ca.riskMeasurement().doubleValue()), 0);
   }
 
   /**